Skip to content

Commit

Permalink
feat: Functionality to mask sensitive field(s) in request body and re…
Browse files Browse the repository at this point in the history
…sponse body (#46)

Co-authored-by: marselab <[email protected]>
  • Loading branch information
marselsampe and marselab authored Feb 3, 2022
1 parent 9b53e0d commit 7637e35
Show file tree
Hide file tree
Showing 5 changed files with 599 additions and 11 deletions.
47 changes: 37 additions & 10 deletions pkg/logger/common/README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Common Log Format Logger

This package enables logging using Common Log Format in go-restful apps.
This package enables logging in go-restful apps.

## Usage

Expand All @@ -27,14 +27,41 @@ ws.Route(ws.GET("/user/{id}").
}))
```

### Environment variables
#### FULL_ACCESS_LOG_ENABLED
Enable full access log mode. Default: false.
### Full access log mode

#### FULL_ACCESS_LOG_SUPPORTED_CONTENT_TYPES
Supported content types to shown in request_body and response_body log.
Default: application/json,application/xml,application/x-www-form-urlencoded,text/plain,text/html.
Full access log mode is used to view the full body detail of all request and response in the endpoints.

#### FULL_ACCESS_LOG_MAX_BODY_SIZE
Maximum size of request body or response body that will be processed, will be ignored if exceed more than it.
Default: 10240 bytes
#### Environment variables

- **FULL_ACCESS_LOG_ENABLED**

Enable full access log mode. Default: `false`

- **FULL_ACCESS_LOG_SUPPORTED_CONTENT_TYPES**

Supported content types to shown in request_body and response_body log.
Default: `application/json,application/xml,application/x-www-form-urlencoded,text/plain,text/html`

- **FULL_ACCESS_LOG_MAX_BODY_SIZE**

Maximum size of request body or response body that will be processed, will be ignored if exceed more than it. Default: `10240` bytes

#### Filter sensitive field(s) in request body or response body

Some endpoint might have sensitive field value in its query params, request body or response body.
For security reason, those sensitive field value should be masked before it printed as a log.

The `log.Attribute` filter can be used to define the field(s) that need to be masked.

```go
ws := new(restful.WebService)
ws.Route(ws.GET("/user/{id}").
Filter(common.Log).
Filter(log.Attribute(log.Option{
MaskedQueryParams: "param1,param2",
MaskedRequestFields: "field1,field2",
MaskedResponseFields: "field3,field4",
})).
To(func(request *restful.Request, response *restful.Response) {
}))
```
16 changes: 15 additions & 1 deletion pkg/logger/common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (

"github.com/AccelByte/go-restful-plugins/v4/pkg/auth/iam"
"github.com/AccelByte/go-restful-plugins/v4/pkg/constant"
"github.com/AccelByte/go-restful-plugins/v4/pkg/logger/log"
"github.com/AccelByte/go-restful-plugins/v4/pkg/trace"
"github.com/AccelByte/go-restful-plugins/v4/pkg/util"
publicsourceip "github.com/AccelByte/public-source-ip"
Expand Down Expand Up @@ -151,12 +152,25 @@ func fullAccessLogFilter(req *restful.Request, resp *restful.Response, chain *re
tokenClientID = jwtClaims.ClientID
}

requestUri := req.Request.URL.RequestURI()

// mask sensitive field(s) in query params, request body and response body
if maskedQueryParams := req.Attribute(log.MaskedQueryParams); maskedQueryParams != nil {
requestUri = log.MaskQueryParams(requestUri, maskedQueryParams.(string))
}
if maskedRequestFields := req.Attribute(log.MaskedRequestFields); maskedRequestFields != nil && requestBody != "" {
requestBody = log.MaskFields(requestContentType, requestBody, maskedRequestFields.(string))
}
if maskedResponseFields := req.Attribute(log.MaskedResponseFields); maskedResponseFields != nil && responseBody != "" {
responseBody = log.MaskFields(responseContentType, responseBody, maskedResponseFields.(string))
}

duration := time.Since(start)

fullAccessLogLogger.Infof(fullAccessLogFormat,
time.Now().UTC().Format("2006-01-02T15:04:05.000Z"),
req.Request.Method,
req.Request.URL.RequestURI(),
requestUri,
resp.StatusCode(),
duration.Milliseconds(),
resp.ContentLength(),
Expand Down
49 changes: 49 additions & 0 deletions pkg/logger/log/log.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
// Copyright 2022 AccelByte Inc
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package log

import (
"github.com/emicklei/go-restful/v3"
)

const MaskedQueryParams = "MaskedQueryParams"
const MaskedRequestFields = "MaskedRequestFields"
const MaskedResponseFields = "MaskedResponseFields"

// Option contains attribute options for log functionality
type Option struct {
// Query param that need to masked in url, separated with comma
MaskedQueryParams string
// Field that need to masked in request body, separated with comma
MaskedRequestFields string
// Field that need to masked in response body, separated with comma
MaskedResponseFields string
}

// Attribute filter is used to define the log attribute for the endpoint.
func Attribute(option Option) restful.FilterFunction {
return func(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
if option.MaskedQueryParams != "" {
req.SetAttribute(MaskedQueryParams, option.MaskedQueryParams)
}
if option.MaskedRequestFields != "" {
req.SetAttribute(MaskedRequestFields, option.MaskedRequestFields)
}
if option.MaskedResponseFields != "" {
req.SetAttribute(MaskedResponseFields, option.MaskedResponseFields)
}
chain.ProcessFilter(req, resp)
}
}
102 changes: 102 additions & 0 deletions pkg/logger/log/maskutil.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
// Copyright 2022 AccelByte Inc
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package log

import (
"fmt"
"regexp"
"strings"
"sync"
)

var FieldRegexCache = sync.Map{}

const (
MaskedValue = "******"
)

// FieldRegex contains regex patterns for field name in varied content-types.
type FieldRegex struct {
FieldName string
JsonPattern *regexp.Regexp
QueryStringPattern *regexp.Regexp
}

// InitFieldRegex initialize the FieldRegex along with its regex patterns.
func (f *FieldRegex) InitFieldRegex(fieldName string) {
f.FieldName = fieldName
// "fieldName":"(.*?)"
f.JsonPattern = regexp.MustCompile(fmt.Sprintf("\"%s\":\"(.*?)\"", fieldName))
// fieldName=(.*?[^&]*)|fieldName=(.*?)$
f.QueryStringPattern = regexp.MustCompile(fmt.Sprintf("%s=(.*?[^&]*)|%s=(.*?)$", fieldName, fieldName))
}

// MaskFields will mask the field value on the content string based on the
// provided field name(s) in "fields" parameter separated by comma.
func MaskFields(contentType, content, fields string) string {
if content == "" || fields == "" {
return content
}

fieldNames := strings.Split(fields, ",")
for _, fieldName := range fieldNames {
var fieldRegex FieldRegex
if val, ok := FieldRegexCache.Load(fieldName); ok {
fieldRegex = val.(FieldRegex)
} else {
fieldRegex = FieldRegex{}
fieldRegex.InitFieldRegex(fieldName)
FieldRegexCache.Store(fieldName, fieldRegex)
}

if strings.Contains(contentType, "application/json") {
content = fieldRegex.JsonPattern.ReplaceAllString(content, fmt.Sprintf("\"%s\":\"%s\"", fieldName, MaskedValue))
} else if strings.Contains(contentType, "application/x-www-form-urlencoded") {
content = fieldRegex.QueryStringPattern.ReplaceAllString(content, fmt.Sprintf("%s=%s", fieldName, MaskedValue))
} else {
// try json pattern and form-data pattern
if fieldRegex.JsonPattern.MatchString(content) {
content = fieldRegex.JsonPattern.ReplaceAllString(content, fmt.Sprintf("\"%s\":\"%s\"", fieldName, MaskedValue))
} else if fieldRegex.QueryStringPattern.MatchString(content) {
content = fieldRegex.QueryStringPattern.ReplaceAllString(content, fmt.Sprintf("%s=%s", fieldName, MaskedValue))
}
}
}

return content
}

// MaskQueryParams will mask the field value on the uri based on the
// provided field name(s) in "fields" parameter separated by comma.
func MaskQueryParams(uri string, fields string) string {
if uri == "" || fields == "" {
return uri
}

fieldNames := strings.Split(fields, ",")
for _, fieldName := range fieldNames {
var fieldRegex FieldRegex
if val, ok := FieldRegexCache.Load(fieldName); ok {
fieldRegex = val.(FieldRegex)
} else {
fieldRegex = FieldRegex{}
fieldRegex.InitFieldRegex(fieldName)
FieldRegexCache.Store(fieldName, fieldRegex)
}

uri = fieldRegex.QueryStringPattern.ReplaceAllString(uri, fmt.Sprintf("%s=%s", fieldName, MaskedValue))
}
return uri
}
Loading

0 comments on commit 7637e35

Please sign in to comment.