From 32f233e7d8627e869dea9cb4d62cb91d2dab8ec8 Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Fri, 1 Dec 2023 14:16:14 +0400 Subject: [PATCH] auth: Read aws-chunked data properly closes #913. Signed-off-by: Evgenii Baidakov --- api/auth/center.go | 30 +++- api/auth/center_test.go | 245 +++++++++++++++++++++++++++ api/auth/signer/v4/chunk_signer.go | 76 +++++++++ api/auth/signer/v4/chunked_reader.go | 150 ++++++++++++---- 4 files changed, 459 insertions(+), 42 deletions(-) create mode 100644 api/auth/signer/v4/chunk_signer.go diff --git a/api/auth/center.go b/api/auth/center.go index f014dc81..e7e25fd2 100644 --- a/api/auth/center.go +++ b/api/auth/center.go @@ -69,14 +69,17 @@ const ( authHeaderPartsNum = 6 maxFormSizeMemory = 50 * 1048576 // 50 MB - AmzAlgorithm = "X-Amz-Algorithm" - AmzCredential = "X-Amz-Credential" - AmzSignature = "X-Amz-Signature" - AmzSignedHeaders = "X-Amz-SignedHeaders" - AmzExpires = "X-Amz-Expires" - AmzDate = "X-Amz-Date" - AuthorizationHdr = "Authorization" - ContentTypeHdr = "Content-Type" + AmzAlgorithm = "X-Amz-Algorithm" + AmzCredential = "X-Amz-Credential" + AmzSignature = "X-Amz-Signature" + AmzSignedHeaders = "X-Amz-SignedHeaders" + AmzExpires = "X-Amz-Expires" + AmzDate = "X-Amz-Date" + AuthorizationHdr = "Authorization" + ContentTypeHdr = "Content-Type" + ContentEncodingHdr = "Content-Encoding" + ContentEncodingAwsChunked = "aws-chunked" + timeFormatISO8601 = "20060102T150405Z" ) @@ -207,6 +210,17 @@ func (c *center) Authenticate(r *http.Request) (*Box, error) { return nil, err } + if hdr := r.Header.Get(ContentEncodingHdr); hdr == ContentEncodingAwsChunked { + sig, err := hex.DecodeString(authHdr.SignatureV4) + if err != nil { + return nil, fmt.Errorf("DecodeString: %w", err) + } + + awsCreds := credentials.NewStaticCredentials(authHdr.AccessKeyID, box.Gate.AccessKey, "") + streamSigner := v4.NewChunkSigner(authHdr.Region, authHdr.Service, sig, signatureDateTime, awsCreds) + r.Body = v4.NewChunkedReader(r.Body, streamSigner) + } + result := &Box{AccessBox: box} if needClientTime { result.ClientTime = signatureDateTime diff --git a/api/auth/center_test.go b/api/auth/center_test.go index 772dd6b6..25251c70 100644 --- a/api/auth/center_test.go +++ b/api/auth/center_test.go @@ -1,10 +1,15 @@ package auth import ( + "bytes" + "encoding/hex" + "io" "strings" "testing" "time" + "github.com/aws/aws-sdk-go/aws/credentials" + v4 "github.com/nspcc-dev/neofs-s3-gw/api/auth/signer/v4" "github.com/nspcc-dev/neofs-s3-gw/api/s3errors" "github.com/stretchr/testify/require" ) @@ -99,3 +104,243 @@ func TestSignature(t *testing.T) { signature := signStr(secret, "s3", "us-east-1", signTime, strToSign) require.Equal(t, "dfbe886241d9e369cf4b329ca0f15eb27306c97aa1022cc0bb5a914c4ef87634", signature) } + +// TestAwsEncodedChunkReader checks example from https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html +func TestAwsEncodedChunkReader(t *testing.T) { + chunkOnePayload := make([]byte, 65536) + for i := 0; i < 65536; i++ { + chunkOnePayload[i] = 'a' + } + + chunkOneBody := append([]byte("10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\n"), chunkOnePayload...) + awsCreds := credentials.NewStaticCredentials("AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "") + + ts, err := time.Parse(timeFormatISO8601, "20130524T000000Z") + require.NoError(t, err) + + seedSignature, err := hex.DecodeString("4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9") + require.NoError(t, err) + + chunkTwoPayload := make([]byte, 1024) + for i := 0; i < 1024; i++ { + chunkTwoPayload[i] = 'a' + } + + chunkTwoBody := append([]byte("400;chunk-signature=0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497\n"), chunkTwoPayload...) + + t.Run("correct signature", func(t *testing.T) { + streamSigner := v4.NewChunkSigner("us-east-1", "s3", seedSignature, ts, awsCreds) + chunkThreeBody := []byte("0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\n") + + buf := bytes.NewBuffer(nil) + chunks := [][]byte{chunkOneBody, chunkTwoBody, chunkThreeBody} + + for _, chunk := range chunks { + _, err = buf.Write(chunk) + require.NoError(t, err) + _, err = buf.Write([]byte{'\r', '\n'}) + require.NoError(t, err) + } + + chunkedReader := v4.NewChunkedReader(io.NopCloser(buf), streamSigner) + defer func() { + _ = chunkedReader.Close() + }() + + chunk := make([]byte, 4096) + payload := bytes.NewBuffer(nil) + _, err = io.CopyBuffer(payload, chunkedReader, chunk) + + require.NoError(t, err) + + require.Equal(t, append(chunkOnePayload, chunkTwoPayload...), payload.Bytes()) + }) + + t.Run("err invalid chunk signature", func(t *testing.T) { + streamSigner := v4.NewChunkSigner("us-east-1", "s3", seedSignature, ts, awsCreds) + chunkThreeBody := []byte("0;chunk-signature=000\n") + + buf := bytes.NewBuffer(nil) + chunks := [][]byte{chunkOneBody, chunkTwoBody, chunkThreeBody} + + for _, chunk := range chunks { + _, err = buf.Write(chunk) + require.NoError(t, err) + _, err = buf.Write([]byte{'\r', '\n'}) + require.NoError(t, err) + } + + chunkedReader := v4.NewChunkedReader(io.NopCloser(buf), streamSigner) + defer func() { + _ = chunkedReader.Close() + }() + + chunk := make([]byte, 4096) + payload := bytes.NewBuffer(nil) + _, err = io.CopyBuffer(payload, chunkedReader, chunk) + + require.ErrorIs(t, err, v4.ErrInvalidChunkSignature) + }) + + t.Run("err missing separator", func(t *testing.T) { + streamSigner := v4.NewChunkSigner("us-east-1", "s3", seedSignature, ts, awsCreds) + chunkThreeBody := []byte("0chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\n") + + buf := bytes.NewBuffer(nil) + chunks := [][]byte{chunkOneBody, chunkTwoBody, chunkThreeBody} + + for _, chunk := range chunks { + _, err = buf.Write(chunk) + require.NoError(t, err) + _, err = buf.Write([]byte{'\r', '\n'}) + require.NoError(t, err) + } + + chunkedReader := v4.NewChunkedReader(io.NopCloser(buf), streamSigner) + defer func() { + _ = chunkedReader.Close() + }() + + chunk := make([]byte, 4096) + payload := bytes.NewBuffer(nil) + _, err = io.CopyBuffer(payload, chunkedReader, chunk) + + require.ErrorIs(t, err, v4.ErrMissingSeparator) + }) + + t.Run("err missing equality byte", func(t *testing.T) { + streamSigner := v4.NewChunkSigner("us-east-1", "s3", seedSignature, ts, awsCreds) + chunkThreeBody := []byte("0;chunk-signatureb6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\n") + + buf := bytes.NewBuffer(nil) + chunks := [][]byte{chunkOneBody, chunkTwoBody, chunkThreeBody} + + for _, chunk := range chunks { + _, err = buf.Write(chunk) + require.NoError(t, err) + _, err = buf.Write([]byte{'\r', '\n'}) + require.NoError(t, err) + } + + chunkedReader := v4.NewChunkedReader(io.NopCloser(buf), streamSigner) + defer func() { + _ = chunkedReader.Close() + }() + + chunk := make([]byte, 4096) + payload := bytes.NewBuffer(nil) + _, err = io.CopyBuffer(payload, chunkedReader, chunk) + + require.Error(t, err) + }) + + t.Run("invalid hex byte", func(t *testing.T) { + streamSigner := v4.NewChunkSigner("us-east-1", "s3", seedSignature, ts, awsCreds) + chunkThreeBody := []byte("h;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\n") + + buf := bytes.NewBuffer(nil) + chunks := [][]byte{chunkOneBody, chunkTwoBody, chunkThreeBody} + + for _, chunk := range chunks { + _, err = buf.Write(chunk) + require.NoError(t, err) + _, err = buf.Write([]byte{'\r', '\n'}) + require.NoError(t, err) + } + + chunkedReader := v4.NewChunkedReader(io.NopCloser(buf), streamSigner) + defer func() { + _ = chunkedReader.Close() + }() + + chunk := make([]byte, 4096) + payload := bytes.NewBuffer(nil) + _, err = io.CopyBuffer(payload, chunkedReader, chunk) + + require.Error(t, err) + }) + + t.Run("invalid hex length", func(t *testing.T) { + streamSigner := v4.NewChunkSigner("us-east-1", "s3", seedSignature, ts, awsCreds) + chunkThreeBody := []byte("11111111111111111;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\n") + + buf := bytes.NewBuffer(nil) + chunks := [][]byte{chunkOneBody, chunkTwoBody, chunkThreeBody} + + for _, chunk := range chunks { + _, err = buf.Write(chunk) + require.NoError(t, err) + _, err = buf.Write([]byte{'\r', '\n'}) + require.NoError(t, err) + } + + chunkedReader := v4.NewChunkedReader(io.NopCloser(buf), streamSigner) + defer func() { + _ = chunkedReader.Close() + }() + + chunk := make([]byte, 4096) + payload := bytes.NewBuffer(nil) + _, err = io.CopyBuffer(payload, chunkedReader, chunk) + + require.Error(t, err) + }) + + t.Run("err missing between chunks separator", func(t *testing.T) { + streamSigner := v4.NewChunkSigner("us-east-1", "s3", seedSignature, ts, awsCreds) + chunkThreeBody := []byte("0;chunk-signatureb6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\n") + + buf := bytes.NewBuffer(nil) + chunks := [][]byte{chunkOneBody, chunkTwoBody, chunkThreeBody} + + for _, chunk := range chunks { + _, err = buf.Write(chunk) + require.NoError(t, err) + _, err = buf.Write([]byte{'\n'}) + require.NoError(t, err) + } + + chunkedReader := v4.NewChunkedReader(io.NopCloser(buf), streamSigner) + defer func() { + _ = chunkedReader.Close() + }() + + chunk := make([]byte, 4096) + payload := bytes.NewBuffer(nil) + _, err = io.CopyBuffer(payload, chunkedReader, chunk) + + require.ErrorIs(t, err, v4.ErrNoChunksSeparator) + }) + + t.Run("err chunk header too long", func(t *testing.T) { + streamSigner := v4.NewChunkSigner("us-east-1", "s3", seedSignature, ts, awsCreds) + chunkThreeBody := make([]byte, 4097) + for i := 0; i < len(chunkThreeBody); i++ { + chunkThreeBody[i] = 'a' + } + + chunkThreeBody[4] = ';' + chunkThreeBody[len(chunkThreeBody)-1] = '\n' + + buf := bytes.NewBuffer(nil) + chunks := [][]byte{chunkOneBody, chunkTwoBody, chunkThreeBody} + + for _, chunk := range chunks { + _, err = buf.Write(chunk) + require.NoError(t, err) + _, err = buf.Write([]byte{'\r', '\n'}) + require.NoError(t, err) + } + + chunkedReader := v4.NewChunkedReader(io.NopCloser(buf), streamSigner) + defer func() { + _ = chunkedReader.Close() + }() + + chunk := make([]byte, 4096) + payload := bytes.NewBuffer(nil) + _, err = io.CopyBuffer(payload, chunkedReader, chunk) + + require.ErrorIs(t, err, v4.ErrLineTooLong) + }) +} diff --git a/api/auth/signer/v4/chunk_signer.go b/api/auth/signer/v4/chunk_signer.go new file mode 100644 index 00000000..99834794 --- /dev/null +++ b/api/auth/signer/v4/chunk_signer.go @@ -0,0 +1,76 @@ +package v4 + +import ( + "encoding/hex" + "hash" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws/credentials" +) + +const ( + // precalculated hash for the zero chunk length. + emptyChunkSHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +) + +// ChunkSigner implements signing of aws-chunked payloads. +type ChunkSigner struct { + region string + service string + + credentials credentialValueProvider + + prevSig []byte + seedDate time.Time +} + +// NewChunkSigner creates a SigV4 signer used to sign Event Stream encoded messages. +func NewChunkSigner(region, service string, seedSignature []byte, seedDate time.Time, credentials *credentials.Credentials) *ChunkSigner { + return &ChunkSigner{ + region: region, + service: service, + credentials: credentials, + seedDate: seedDate, + prevSig: seedSignature, + } +} + +// GetSignature takes an event stream encoded headers and payload and returns a signature. +func (s *ChunkSigner) GetSignature(payload []byte) ([]byte, error) { + return s.getSignature(hashSHA256(payload)) +} + +// GetSignatureByHash takes an event stream encoded headers and payload and returns a signature. +func (s *ChunkSigner) GetSignatureByHash(payloadHash hash.Hash) ([]byte, error) { + return s.getSignature(payloadHash.Sum(nil)) +} + +func (s *ChunkSigner) getSignature(payloadHash []byte) ([]byte, error) { + credValue, err := s.credentials.Get() + if err != nil { + return nil, err + } + + sigKey := deriveSigningKey(s.region, s.service, credValue.SecretAccessKey, s.seedDate) + + keyPath := buildSigningScope(s.region, s.service, s.seedDate) + + stringToSign := buildStringToSign(payloadHash, s.prevSig, keyPath, s.seedDate) + + signature := hmacSHA256(sigKey, []byte(stringToSign)) + s.prevSig = signature + + return signature, nil +} + +func buildStringToSign(payloadHash, prevSig []byte, scope string, date time.Time) string { + return strings.Join([]string{ + "AWS4-HMAC-SHA256-PAYLOAD", + formatTime(date), + scope, + hex.EncodeToString(prevSig), + emptyChunkSHA256, + hex.EncodeToString(payloadHash), + }, "\n") +} diff --git a/api/auth/signer/v4/chunked_reader.go b/api/auth/signer/v4/chunked_reader.go index a30ebc16..5308d301 100644 --- a/api/auth/signer/v4/chunked_reader.go +++ b/api/auth/signer/v4/chunked_reader.go @@ -7,40 +7,64 @@ package v4 import ( "bufio" "bytes" + "crypto/sha256" + "encoding/hex" "errors" + "fmt" + "hash" "io" ) const maxLineLength = 4096 // assumed <= bufio.defaultBufSize -var ErrLineTooLong = errors.New("header line too long") +var ( + // ErrLineTooLong appears if chunk header exceeds maxLineLength. + ErrLineTooLong = errors.New("header line too long") + + // ErrInvalidChunkSignature appears if passed chunk signature differs from calculated. + ErrInvalidChunkSignature = errors.New("invalid chunk signature") + + // ErrMissingSeparator appears if chunk header doesn't contain ';' separator. + ErrMissingSeparator = errors.New("missing header separator") + + // ErrNoChunksSeparator appears if chunks not properly separated between each other. + // They should be divided with \r\n bytes. + ErrNoChunksSeparator = errors.New("no chunk separator") +) // NewChunkedReader returns a new chunkedReader that translates the data read from r // out of HTTP "chunked" format before returning it. // The chunkedReader returns io.EOF when the final 0-length chunk is read. -// -// NewChunkedReader is not needed by normal applications. The http package -// automatically decodes chunking when reading response bodies. -func NewChunkedReader(r io.Reader) io.Reader { - br, ok := r.(*bufio.Reader) - if !ok { - br = bufio.NewReader(r) - } - return &chunkedReader{r: br} +func NewChunkedReader(r io.ReadCloser, streamSigner *ChunkSigner) io.ReadCloser { + return &chunkedReader{ + r: bufio.NewReader(r), + // bufio.Reader can't be closed, thus left link to the original reader to close it later. + origReader: r, + streamSigner: streamSigner, + } } type chunkedReader struct { - r *bufio.Reader - n uint64 // unread bytes in chunk - err error - buf [2]byte - checkEnd bool // whether need to check for \r\n chunk footer + chunkHash hash.Hash + chunkSignature string + r *bufio.Reader + origReader io.ReadCloser + n uint64 // unread bytes in chunk + err error + buf [2]byte + checkEnd bool // whether need to check for \r\n chunk footer + streamSigner *ChunkSigner +} + +// Close implements [io.ReadCloser]. +func (cr *chunkedReader) Close() (err error) { + return cr.origReader.Close() } func (cr *chunkedReader) beginChunk() { // chunk-size CRLF - var line []byte - line, cr.err = readChunkLine(cr.r) + var line, chunkSignature []byte + line, chunkSignature, cr.err = readChunkLine(cr.r) if cr.err != nil { return } @@ -48,11 +72,46 @@ func (cr *chunkedReader) beginChunk() { if cr.err != nil { return } + + if err := cr.validatePreviousChunkData(); err != nil { + cr.err = err + return + } + + // creating instance here to avoid validating non-existent chunk in the first validatePreviousChunkData call. + if cr.chunkHash == nil { + cr.chunkHash = sha256.New() + } else { + cr.chunkHash.Reset() + } + + cr.chunkSignature = string(chunkSignature) + if cr.n == 0 { + if err := cr.validatePreviousChunkData(); err != nil { + cr.err = err + return + } + cr.err = io.EOF } } +func (cr *chunkedReader) validatePreviousChunkData() error { + if cr.chunkHash != nil { + calculatedSignature, err := cr.streamSigner.GetSignatureByHash(cr.chunkHash) + if err != nil { + return fmt.Errorf("GetSignature: %w", err) + } + + if cr.chunkSignature != hex.EncodeToString(calculatedSignature) { + return ErrInvalidChunkSignature + } + } + + return nil +} + func (cr *chunkedReader) chunkHeaderAvailable() bool { n := cr.r.Buffered() if n > 0 { @@ -62,6 +121,7 @@ func (cr *chunkedReader) chunkHeaderAvailable() bool { return false } +// Read gets data from reader. Implements [io.ReadCloser]. func (cr *chunkedReader) Read(b []uint8) (n int, err error) { for cr.err == nil { if cr.checkEnd { @@ -73,7 +133,7 @@ func (cr *chunkedReader) Read(b []uint8) (n int, err error) { } if _, cr.err = io.ReadFull(cr.r, cr.buf[:2]); cr.err == nil { if string(cr.buf[:]) != "\r\n" { - cr.err = errors.New("malformed chunked encoding") + cr.err = ErrNoChunksSeparator break } } else { @@ -105,6 +165,13 @@ func (cr *chunkedReader) Read(b []uint8) (n int, err error) { n += n0 b = b[n0:] cr.n -= uint64(n0) + // Hashing chunk data to calculate the signature. + // rbuf may contain payload and empty bytes, taking only payload. + if _, err = cr.chunkHash.Write(rbuf[:n0]); err != nil { + cr.err = err + break + } + // If we're at the end of a chunk, read the next two // bytes to verify they are "\r\n". if cr.n == 0 && cr.err == nil { @@ -120,27 +187,37 @@ func (cr *chunkedReader) Read(b []uint8) (n int, err error) { // Give up if the line exceeds maxLineLength. // The returned bytes are owned by the bufio.Reader // so they are only valid until the next bufio read. -func readChunkLine(b *bufio.Reader) ([]byte, error) { +func readChunkLine(b *bufio.Reader) ([]byte, []byte, error) { p, err := b.ReadSlice('\n') if err != nil { // We always know when EOF is coming. // If the caller asked for a line, there should be a line. if err == io.EOF { err = io.ErrUnexpectedEOF - } else if err == bufio.ErrBufferFull { + } else if errors.Is(err, bufio.ErrBufferFull) { err = ErrLineTooLong } - return nil, err + return nil, nil, err } if len(p) >= maxLineLength { - return nil, ErrLineTooLong + return nil, nil, ErrLineTooLong } + + var signaturePart []byte + p = trimTrailingWhitespace(p) - p, err = removeChunkExtension(p) + p, signaturePart, err = removeChunkExtension(p) if err != nil { - return nil, err + return nil, nil, err + } + + pos := bytes.IndexByte(signaturePart, '=') + if pos == -1 { + return nil, nil, errors.New("chunk header is malformed") } - return p, nil + + // even if '=' is the latest symbol, the new slice will be just empty + return p, signaturePart[pos+1:], nil } func trimTrailingWhitespace(b []byte) []byte { @@ -160,15 +237,20 @@ var semi = []byte(";") // For example, // // "0" => "0" -// "0;token" => "0" -// "0;token=val" => "0" -// `0;token="quoted string"` => "0" -func removeChunkExtension(p []byte) ([]byte, error) { - p, _, _ = bytes.Cut(p, semi) - // TODO: care about exact syntax of chunk extensions? We're - // ignoring and stripping them anyway. For now just never - // return an error. - return p, nil +// "0;chunk-signature" => "0" +// "0;chunk-signature=val" => "0" +// `0;chunk-signature="quoted string"` => "0" +func removeChunkExtension(p []byte) ([]byte, []byte, error) { + var ( + chunkSignature []byte + found bool + ) + p, chunkSignature, found = bytes.Cut(p, semi) + if !found { + return nil, nil, ErrMissingSeparator + } + + return p, chunkSignature, nil } func parseHexUint(v []byte) (n uint64, err error) {