Merge pull request #4247 from getlantern/issue-4246

Added copying of headers to the fronted request closes #4246

src/github.com/getlantern/flashlight/util/http.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"strings"
 	"sync"
 	"time"
 
@@ -22,7 +23,7 @@ var (
 
 // HTTPFetcher is a simple interface for types that are able to fetch data over HTTP.
 type HTTPFetcher interface {
-	Do(req *http.Request) (*http.Response, error)
+	Do(*http.Request) (*http.Response, error)
 }
 
 func success(resp *http.Response) bool {
@@ -94,19 +95,49 @@ type dualFetcher struct {
 // arrive. Callers MUST use the Lantern-Fronted-URL HTTP header to
 // specify the fronted URL to use.
 func (df *dualFetcher) Do(req *http.Request) (*http.Response, error) {
+	if directClient, err := HTTPClient("", df.cf.proxyAddrFN); err != nil {
+		log.Errorf("Could not create http client? %v", err)
+		return nil, err
+	} else {
+		frontedClient := fronted.NewDirectHttpClient(5 * time.Minute)
+		return df.do(req, directClient.Do, frontedClient.Do)
+	}
+}
+
+// Do will attempt to execute the specified HTTP request using both
+// chained and fronted servers, simply returning the first response to
+// arrive. Callers MUST use the Lantern-Fronted-URL HTTP header to
+// specify the fronted URL to use.
+func (df *dualFetcher) do(req *http.Request, chainedFunc func(*http.Request) (*http.Response, error), ddfFunc func(*http.Request) (*http.Response, error)) (*http.Response, error) {
 	log.Debugf("Using dual fronter")
-	frontedUrl := req.Header.Get("Lantern-Fronted-URL")
+	frontedURL := req.Header.Get("Lantern-Fronted-URL")
 	req.Header.Del("Lantern-Fronted-URL")
 
-	if frontedUrl == "" {
+	if frontedURL == "" {
 		return nil, errors.New("Callers MUST specify the fronted URL in the Lantern-Fronted-URL header")
 	}
+
+	// Make a copy of the original requeest headers to include in the fronted
+	// request. This will ensure that things like the caching headers are
+	// included in both requests.
+	headersCopy := make(http.Header, len(req.Header))
+	for k, vv := range req.Header {
+		// Since we're doing domain fronting don't copy the host just in case
+		// it ever makes any difference under the covers.
+		if strings.EqualFold("Host", k) {
+			continue
+		}
+		vv2 := make([]string, len(vv))
+		copy(vv2, vv)
+		headersCopy[k] = vv2
+	}
+
 	responses := make(chan *http.Response, 2)
 	errs := make(chan error, 2)
 
-	request := func(client HTTPFetcher, req *http.Request) error {
-		if resp, err := client.Do(req); err != nil {
-			log.Errorf("Could not complete request with: %v, %v", frontedUrl, err)
+	request := func(clientFunc func(*http.Request) (*http.Response, error), req *http.Request) error {
+		if resp, err := clientFunc(req); err != nil {
+			log.Errorf("Could not complete request with: %v, %v", frontedURL, err)
 			errs <- err
 			return err
 		} else {
@@ -115,43 +146,42 @@ func (df *dualFetcher) Do(req *http.Request) (*http.Response, error) {
 				responses <- resp
 				return nil
 			} else {
-				// If the local proxy can't connect to any upstread proxies, for example,
+				// If the local proxy can't connect to any upstream proxies, for example,
 				// it will return a 502.
 				err := fmt.Errorf("Bad response code: %v", resp.StatusCode)
-				_ = resp.Body.Close()
+				if resp.Body != nil {
+					_ = resp.Body.Close()
+				}
 				errs <- err
 				return err
 			}
 		}
 	}
 
 	go func() {
-		if req, err := http.NewRequest("GET", frontedUrl, nil); err != nil {
-			log.Errorf("Could not create request for: %v, %v", frontedUrl, err)
+		if frontedReq, err := http.NewRequest("GET", frontedURL, nil); err != nil {
+			log.Errorf("Could not create request for: %v, %v", frontedURL, err)
 			errs <- err
 		} else {
 			log.Debug("Sending request via DDF")
-			direct := fronted.NewDirectHttpClient(5 * time.Minute)
-			if err := request(direct, req); err != nil {
+			frontedReq.Header = headersCopy
+
+			if err := request(ddfFunc, frontedReq); err != nil {
 				log.Errorf("Fronted request failed: %v", err)
 			} else {
 				log.Debug("Fronted request succeeded")
 			}
 		}
 	}()
 	go func() {
-		if client, err := HTTPClient("", df.cf.proxyAddrFN); err != nil {
-			log.Errorf("Could not create HTTP client: %v", err)
-			errs <- err
+		log.Debug("Sending chained request")
+		if err := request(chainedFunc, req); err != nil {
+			log.Errorf("Chained request failed %v", err)
 		} else {
-			log.Debug("Sending chained request")
-			if err := request(client, req); err != nil {
-				log.Errorf("Chained request failed %v", err)
-			} else {
-				log.Debug("Switching to chained fronter for future requests since it succeeded")
-				df.cf.setFetcher(&chainedFetcher{df.cf.proxyAddrFN})
-			}
+			log.Debug("Switching to chained fronter for future requests since it succeeded")
+			df.cf.setFetcher(&chainedFetcher{df.cf.proxyAddrFN})
 		}
+
 	}()
 
 	// Create channels for the final response or error. The response channel will be filled

src/github.com/getlantern/flashlight/util/util_test.go

@@ -1,10 +1,12 @@
 package util
 
 import (
+	"bytes"
 	"crypto/x509"
 	"io/ioutil"
 	"net"
 	"net/http"
+	"net/http/httputil"
 	"os"
 	"strings"
 	"testing"
@@ -30,6 +32,62 @@ func TestGetFileHash(t *testing.T) {
 		"hashes not equal! has hashes.go changed?")
 }
 
+// TestChainedAndFrontedHeaders tests to make sure headers are correctly
+// copied to the fronted request from the original chained request.
+func TestChainedAndFrontedHeaders(t *testing.T) {
+	geo := "http://d3u5fqukq7qrhd.cloudfront.net/lookup/198.199.72.101"
+	req, err := http.NewRequest("GET", geo, nil)
+	if !assert.NoError(t, err) {
+		return
+	}
+	req.Header.Set("Lantern-Fronted-URL", geo)
+	req.Header.Set("Accept", "application/x-gzip")
+	// Prevents intermediate nodes (domain-fronters) from caching the content
+	req.Header.Set("Cache-Control", "no-cache")
+	etag := "473892jdfda"
+	req.Header.Set("X-Lantern-If-None-Match", etag)
+
+	// Make sure the chained response fails.
+	chainedFunc := func(req *http.Request) (*http.Response, error) {
+		headers, _ := httputil.DumpRequest(req, false)
+		log.Debugf("Got chained request headers:\n%v", string(headers))
+		return &http.Response{
+			Status:     "503 OK",
+			StatusCode: 503,
+		}, nil
+	}
+
+	frontedHeaders := eventual.NewValue()
+	frontedFunc := func(req *http.Request) (*http.Response, error) {
+		headers, _ := httputil.DumpRequest(req, false)
+		log.Debugf("Got FRONTED request headers:\n%v", string(headers))
+		frontedHeaders.Set(req.Header)
+		return &http.Response{
+			Status:     "200 OK",
+			StatusCode: 200,
+			Body:       ioutil.NopCloser(bytes.NewBufferString("Fronted")),
+		}, nil
+	}
+
+	df := &dualFetcher{&chainedAndFronted{}}
+
+	df.do(req, chainedFunc, frontedFunc)
+
+	headersVal, ok := frontedHeaders.Get(2 * time.Second)
+	if !assert.True(t, ok) {
+		return
+	}
+	headers := headersVal.(http.Header)
+	assert.Equal(t, etag, headers.Get("X-Lantern-If-None-Match"))
+	assert.Equal(t, "no-cache", headers.Get("Cache-Control"))
+
+	// There should not be a host header here -- the go http client will populate
+	// it automatically based on the URL.
+	assert.Equal(t, "", headers.Get("Host"))
+}
+
+// TestChainedAndFronted tests to make sure chained and fronted requests are
+// both working in parallel.
 func TestChainedAndFronted(t *testing.T) {
 	fwd, _ := forward.New()