Merge pull request #280 from baidu/release/v0.7.0

Release/v0.7.0

CHANGELOG.md

@@ -10,6 +10,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [v0.7.0] - 2020-02-26
+### Added
+- mod_access support NCSA Common Log Format(CLF) and W3C Extended Log Format(ELF)
+- mod_static suuport HTTP HEAD method
+- Add mod_userid for client identification
+- Add mod_tag for tagging and tracking groups of requests
+- http cookie: support Expires attribute in GMT format
+- http cookie: support SameSite attribute 
+- Add static check in Makefile
+- Compiling on Windows OS is supported
+- Documents optimization
+
+### Fixed
+- Fix a bug that causes invalid ips are parsed and treated as domain names
+
 ## [v0.6.0] - 2020-01-21
 ### Added
 - Add mod_prison to limit the amount of requests a user can make in a given period of time.
@@ -23,7 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Optimize number of accept goroutines
 - Optimize lock of bfe_balance.BalTable
 - Optimize io.Copy while forwarding responses
-- Compiling on MacOS is supported
+- Compiling on Mac OS is supported
 - Documents optimization
 
 ### Changed
@@ -100,6 +115,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Flexible plugin framework to extend functionality. Based on the framework, developer can add new features rapidly
 - Detailed built-in metrics available for service status monitor
 
+[v0.7.0]: https://github.com/baidu/bfe/compare/v0.6.0...v0.7.0
 [v0.6.0]: https://github.com/baidu/bfe/compare/v0.5.0...v0.6.0
 [v0.5.0]: https://github.com/baidu/bfe/compare/v0.4.0...v0.5.0
 [v0.4.0]: https://github.com/baidu/bfe/compare/v0.3.0...v0.4.0

Makefile

@@ -21,13 +21,13 @@ export PATH        := $(shell go env GOPATH)/bin:$(PATH)
 export GO111MODULE := on
 
 # init command params
-GO      := go
-GOBUILD := $(GO) build
-GOTEST  := $(GO) test
-GOVET   := $(GO) vet
-GOGET   := $(GO) get
-GOGEN   := $(GO) generate
-GOCLEAN := $(GO) clean
+GO           := go
+GOBUILD      := $(GO) build
+GOTEST       := $(GO) test
+GOVET        := $(GO) vet
+GOGET        := $(GO) get
+GOGEN        := $(GO) generate
+STATICCHECK  := staticcheck
 
 # init bfe version
 BFE_VERSION ?= $(shell cat VERSION)
@@ -68,6 +68,11 @@ package:
 	mv bfe  $(OUTDIR)/bin
 	cp -r conf $(OUTDIR)
 
+# make check
+check:
+	$(GO) get honnef.co/go/tools/cmd/staticcheck
+	$(STATICCHECK) ./...
+
 # make docker
 docker:
 	docker build \

VERSION

@@ -1 +1 @@
-0.6.0
+0.7.0

bfe_config/bfe_route_conf/vip_rule_conf/vip_table_load.go

@@ -71,8 +71,8 @@ func VipTableConfCheck(conf *VipTableConf) error {
 	for product, vipList := range conf.Vips {
 		var formattedVipList VipList
 		for _, vip := range vipList {
-			ip, err := net.ResolveIPAddr("ip", vip)
-			if err != nil {
+			ip := net.ParseIP(vip)
+			if ip == nil {
 				return fmt.Errorf("invalid vip %s for %s", vip, product)
 			}
 

bfe_config/bfe_route_conf/vip_rule_conf/vip_table_load_test.go

@@ -50,7 +50,7 @@ func TestVipTableLoad_3(t *testing.T) {
 		return
 	}
 
-	vip, _ := net.ResolveIPAddr("ip", "2001:0:1111:A:B0::9000:200")
+	vip := net.ParseIP("2001:0:1111:A:B0::9000:200")
 	if config.VipMap[vip.String()] != "pb" {
 		t.Errorf("config.VipMap['2001:0:1111:A:B0::9000:200'] should be 'pb', not %s",
 			config.VipMap[vip.String()])

bfe_http/cookie.go

@@ -51,12 +51,28 @@ type Cookie struct {
 	MaxAge   int
 	Secure   bool
 	HttpOnly bool
+	SameSite SameSite
 	Raw      string
 	Unparsed []string // Raw text of unparsed attribute-value pairs
 }
 
 type CookieMap map[string]*Cookie
 
+// SameSite allows a server to define a cookie attribute making it impossible for
+// the browser to send this cookie along with cross-site requests. The main
+// goal is to mitigate the risk of cross-origin information leakage, and provide
+// some protection against cross-site request forgery attacks.
+//
+// See https://tools.ietf.org/html/draft-ietf-httpbis-cookie-same-site-00 for details.
+type SameSite int
+
+const (
+	SameSiteDefaultMode SameSite = iota + 1
+	SameSiteLaxMode
+	SameSiteStrictMode
+	SameSiteNoneMode
+)
+
 // CookieMapGet parse cookies(slice) to req.Route.CookieMap(map)
 func CookieMapGet(cookies []*Cookie) CookieMap {
 	cookieMap := make(CookieMap, len(cookies))
@@ -82,6 +98,32 @@ func (cm CookieMap) Get(key string) (*Cookie, bool) {
 	return val, ok
 }
 
+// http://tools.ietf.org/html/rfc6265#section-5.2.1 only specify an algorithm to parse a cookie-date
+// According to https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie, Expires format should be HTTP-date
+// Preferred syntax: <day-name>, <day> <month> <year> <hour>:<minute>:<second> GMT
+// However, there are some obsolete formats, specified in https://tools.ietf.org/html/rfc7231#section-7.1.1.2
+func parseExpireTime(val string) (time.Time, bool) {
+	// preferred format
+	exptime, err := time.Parse(TimeFormat, val)
+	if err == nil {
+		return exptime.UTC(), true
+	}
+	exptime, err = time.Parse("Mon, 02-Jan-06 15:04:05 GMT", val)
+	if err == nil {
+		return exptime.UTC(), true
+	}
+	exptime, err = time.Parse(time.RFC1123, val)
+	if err == nil {
+		return exptime.UTC(), true
+	}
+	// set-cookie expire format of php version: Sun, 07-May-2028 10:53:08 GMT
+	exptime, err = time.Parse("Mon, 02-Jan-2006 15:04:05 MST", val)
+	if err == nil {
+		return exptime.UTC(), true
+	}
+	return time.Time{}, false
+}
+
 // readSetCookies parses all "Set-Cookie" values from
 // the header h and returns the successfully parsed Cookies.
 func readSetCookies(h Header) []*Cookie {
@@ -130,6 +172,19 @@ func readSetCookies(h Header) []*Cookie {
 				continue
 			}
 			switch lowerAttr {
+			case "samesite":
+				lowerVal := strings.ToLower(val)
+				switch lowerVal {
+				case "lax":
+					c.SameSite = SameSiteLaxMode
+				case "strict":
+					c.SameSite = SameSiteStrictMode
+				case "none":
+					c.SameSite = SameSiteNoneMode
+				default:
+					c.SameSite = SameSiteDefaultMode
+				}
+				continue
 			case "secure":
 				c.Secure = true
 				continue
@@ -153,16 +208,12 @@ func readSetCookies(h Header) []*Cookie {
 				continue
 			case "expires":
 				c.RawExpires = val
-				exptime, err := time.Parse(time.RFC1123, val)
-				if err != nil {
-					exptime, err = time.Parse("Mon, 02-Jan-2006 15:04:05 MST", val)
-					if err != nil {
-						c.Expires = time.Time{}
-						break
-					}
+				// if parse failed, will add to Unparsed
+				exp, ok := parseExpireTime(val)
+				if ok {
+					c.Expires = exp
+					continue
 				}
-				c.Expires = exptime.UTC()
-				continue
 			case "path":
 				c.Path = val
 				// TODO: Add path parsing
@@ -206,7 +257,7 @@ func (c *Cookie) String() string {
 		}
 	}
 	if c.Expires.Unix() > 0 {
-		fmt.Fprintf(&b, "; Expires=%s", c.Expires.UTC().Format(time.RFC1123))
+		fmt.Fprintf(&b, "; Expires=%s", c.Expires.UTC().Format(TimeFormat))
 	}
 	if c.MaxAge > 0 {
 		fmt.Fprintf(&b, "; Max-Age=%d", c.MaxAge)
@@ -219,6 +270,16 @@ func (c *Cookie) String() string {
 	if c.Secure {
 		fmt.Fprintf(&b, "; Secure")
 	}
+	switch c.SameSite {
+	case SameSiteDefaultMode:
+		b.WriteString("; SameSite")
+	case SameSiteNoneMode:
+		b.WriteString("; SameSite=None")
+	case SameSiteLaxMode:
+		b.WriteString("; SameSite=Lax")
+	case SameSiteStrictMode:
+		b.WriteString("; SameSite=Strict")
+	}
 	return b.String()
 }
 

bfe_http/cookie_test.go

@@ -62,6 +62,22 @@ var writeSetCookiesTests = []struct {
 		&Cookie{Name: "cookie-8", Value: "eight", Domain: "::1"},
 		"cookie-8=eight",
 	},
+	{
+		&Cookie{Name: "cookie-12", Value: "samesite-default", SameSite: SameSiteDefaultMode},
+		"cookie-12=samesite-default; SameSite",
+	},
+	{
+		&Cookie{Name: "cookie-13", Value: "samesite-lax", SameSite: SameSiteLaxMode},
+		"cookie-13=samesite-lax; SameSite=Lax",
+	},
+	{
+		&Cookie{Name: "cookie-14", Value: "samesite-strict", SameSite: SameSiteStrictMode},
+		"cookie-14=samesite-strict; SameSite=Strict",
+	},
+	{
+		&Cookie{Name: "cookie-15", Value: "samesite-none", SameSite: SameSiteNoneMode},
+		"cookie-15=samesite-none; SameSite=None",
+	},
 }
 
 func TestWriteSetCookies(t *testing.T) {
@@ -180,7 +196,70 @@ var readSetCookiesTests = []struct {
 			Raw:      "ASP.NET_SessionId=foo; path=/; HttpOnly",
 		}},
 	},
-
+	{
+		Header{"Set-Cookie": {"samesitedefault=foo; SameSite"}},
+		[]*Cookie{{
+			Name:     "samesitedefault",
+			Value:    "foo",
+			SameSite: SameSiteDefaultMode,
+			Raw:      "samesitedefault=foo; SameSite",
+		}},
+	},
+	{
+		Header{"Set-Cookie": {"samesitelax=foo; SameSite=Lax"}},
+		[]*Cookie{{
+			Name:     "samesitelax",
+			Value:    "foo",
+			SameSite: SameSiteLaxMode,
+			Raw:      "samesitelax=foo; SameSite=Lax",
+		}},
+	},
+	{
+		Header{"Set-Cookie": {"samesitestrict=foo; SameSite=Strict"}},
+		[]*Cookie{{
+			Name:     "samesitestrict",
+			Value:    "foo",
+			SameSite: SameSiteStrictMode,
+			Raw:      "samesitestrict=foo; SameSite=Strict",
+		}},
+	},
+	{
+		Header{"Set-Cookie": {"samesitenone=foo; SameSite=None"}},
+		[]*Cookie{{
+			Name:     "samesitenone",
+			Value:    "foo",
+			SameSite: SameSiteNoneMode,
+			Raw:      "samesitenone=foo; SameSite=None",
+		}},
+	},
+	{
+		Header{"Set-Cookie": {"SID=XXX; expires=Thu, 18-Feb-21 06:59:27 GMT; max-age=31536000; path=/; domain=.test.com; version=1"}},
+		[]*Cookie{{
+			Name:       "SID",
+			Value:      "XXX",
+			Expires:    time.Date(2021, time.February, 18, 6, 59, 27, 0, time.UTC),
+			RawExpires: "Thu, 18-Feb-21 06:59:27 GMT",
+			MaxAge:     31536000,
+			Path:       "/",
+			Domain:     ".test.com",
+			Unparsed:   []string{"version=1"},
+			Raw:        "SID=XXX; expires=Thu, 18-Feb-21 06:59:27 GMT; max-age=31536000; path=/; domain=.test.com; version=1",
+		}},
+	},
+	{
+		Header{"Set-Cookie": {"STOKEN=xxx; expires=Tue, 25-Apr-2028 08:07:15 GMT; path=/; domain=test2.com; secure; httponly"}},
+		[]*Cookie{{
+			Name:       "STOKEN",
+			Value:      "xxx",
+			Expires:    time.Date(2028, time.April, 25, 8, 7, 15, 0, time.UTC),
+			RawExpires: "Tue, 25-Apr-2028 08:07:15 GMT",
+			Path:       "/",
+			Domain:     "test2.com",
+			HttpOnly:   true,
+			Secure:     true,
+			Raw:        "STOKEN=xxx; expires=Tue, 25-Apr-2028 08:07:15 GMT; path=/; domain=test2.com; secure; httponly",
+		}},
+	},
 	// TODO(bradfitz): users have reported seeing this in the
 	// wild, but do browsers handle it? RFC 6265 just says "don't
 	// do that" (section 3) and then never mentions header folding

bfe_modules/bfe_modules.go

@@ -32,7 +32,9 @@ import (
 	"github.com/baidu/bfe/bfe_modules/mod_redirect"
 	"github.com/baidu/bfe/bfe_modules/mod_rewrite"
 	"github.com/baidu/bfe/bfe_modules/mod_static"
+	"github.com/baidu/bfe/bfe_modules/mod_tag"
 	"github.com/baidu/bfe/bfe_modules/mod_trust_clientip"
+	"github.com/baidu/bfe/bfe_modules/mod_userid"
 )
 
 // list of all modules, the order is very important
@@ -44,10 +46,16 @@ var moduleList = []bfe_module.BfeModule{
 	// Requirement: After mod_trust_clientip
 	mod_logid.NewModuleLogId(),
 
+	// mode_userid
+	mod_userid.NewModuleUserID(),
+
 	// mod_geo
 	// Requirement: After mod_logid
 	mod_geo.NewModuleGeo(),
 
+	// mod_tag
+	mod_tag.NewModuleTag(),
+
 	// mod_block
 	// Requirement: After mod_logid
 	mod_block.NewModuleBlock(),

bfe_modules/mod_access/conf_mod_access.go

@@ -53,6 +53,8 @@ func ConfLoad(filePath string) (*ConfModAccess, error) {
 		return &cfg, err
 	}
 
+	cfg.Convert()
+
 	return &cfg, nil
 }
 
@@ -84,6 +86,15 @@ func (cfg *ConfModAccess) Check() error {
 	return nil
 }
 
+func (cfg *ConfModAccess) Convert() {
+	switch cfg.Template.RequestTemplate {
+	case "COMMON":
+		cfg.Template.RequestTemplate = "$host - - $request_time \"$request_line\" $status_code $res_len"
+	case "COMBINED":
+		cfg.Template.RequestTemplate = "$host - - $request_time \"$request_line\" $status_code $res_len \"${Referer}req_header\" \"${User-Agent}req_header\""
+	}
+}
+
 func checkLogFmt(item LogFmtItem, logFmtType string) error {
 	if logFmtType != Request && logFmtType != Session {
 		return fmt.Errorf("logFmtType should be Request or Session")