From 2949a5fb9b5468590cd939e416344485239031e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Wed, 25 Jan 2023 14:48:26 +0200 Subject: [PATCH] remote: implement compact writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This implements the compact writer for api/v1/read API. It works like the other XOR streamed type except that as much as possible data is put into individual writes. This significantly improves throughput. Signed-off-by: Giedrius Statkevičius --- prompb/remote.pb.go | 71 ++++++++++++----------- prompb/remote.proto | 7 +++ storage/remote/codec.go | 30 ++++++---- storage/remote/compact_writer.go | 89 +++++++++++++++++++++++++++++ storage/remote/read_handler.go | 98 ++++++++++++++++++++++++++++++-- 5 files changed, 249 insertions(+), 46 deletions(-) create mode 100644 storage/remote/compact_writer.go diff --git a/prompb/remote.pb.go b/prompb/remote.pb.go index b3cf44884f9..f42553cc45b 100644 --- a/prompb/remote.pb.go +++ b/prompb/remote.pb.go @@ -41,16 +41,22 @@ const ( // Content-Type: "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse" // Content-Encoding: "" ReadRequest_STREAMED_XOR_CHUNKS ReadRequest_ResponseType = 1 + // Response headers: + // Content-Type: "application/x-compact-protobuf; proto=prometheus.ChunkedReadResponse" + // Content-Encoding: "" + ReadRequest_COMPACT_XOR_CHUNKS ReadRequest_ResponseType = 2 ) var ReadRequest_ResponseType_name = map[int32]string{ 0: "SAMPLES", 1: "STREAMED_XOR_CHUNKS", + 2: "COMPACT_XOR_CHUNKS", } var ReadRequest_ResponseType_value = map[string]int32{ "SAMPLES": 0, "STREAMED_XOR_CHUNKS": 1, + "COMPACT_XOR_CHUNKS": 2, } func (x ReadRequest_ResponseType) String() string { @@ -418,38 +424,39 @@ func init() { func init() { proto.RegisterFile("remote.proto", fileDescriptor_eefc82927d57d89b) } var fileDescriptor_eefc82927d57d89b = []byte{ - // 496 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0xcd, 0x6e, 0xd3, 0x40, - 0x10, 0xee, 0x26, 0x69, 0x13, 0x8d, 0x43, 0x14, 0xb6, 0x2d, 0x09, 0x39, 0xa4, 0x91, 0xc5, 0x21, - 0x52, 0x51, 0x10, 0xa1, 0xe2, 0xd4, 0x03, 0x69, 0x89, 0x54, 0xa0, 0xe6, 0x67, 0x13, 0x04, 0x42, - 0x48, 0xd6, 0xc6, 0x1e, 0x35, 0x16, 0xf5, 0x4f, 0x77, 0xd7, 0x52, 0xf3, 0x16, 0x3c, 0x13, 0xa7, - 0x9e, 0x10, 0x4f, 0x80, 0x50, 0x9e, 0x04, 0x79, 0x6d, 0x87, 0x2d, 0x5c, 0xb8, 0xad, 0xbf, 0x3f, - 0xcf, 0xcc, 0xce, 0x42, 0x53, 0x60, 0x18, 0x2b, 0x1c, 0x25, 0x22, 0x56, 0x31, 0x85, 0x44, 0xc4, - 0x21, 0xaa, 0x25, 0xa6, 0xb2, 0x67, 0xa9, 0x55, 0x82, 0x32, 0x27, 0x7a, 0x7b, 0x17, 0xf1, 0x45, - 0xac, 0x8f, 0x8f, 0xb2, 0x53, 0x8e, 0xda, 0x5f, 0x09, 0x34, 0x3f, 0x88, 0x40, 0x21, 0xc3, 0xab, - 0x14, 0xa5, 0xa2, 0xc7, 0x00, 0x2a, 0x08, 0x51, 0xa2, 0x08, 0x50, 0x76, 0xc9, 0xa0, 0x3a, 0xb4, - 0xc6, 0xf7, 0x46, 0x7f, 0x42, 0x47, 0xf3, 0x20, 0xc4, 0x99, 0x66, 0x4f, 0x6a, 0x37, 0x3f, 0x0f, - 0xb6, 0x98, 0xa1, 0xa7, 0xc7, 0xd0, 0x08, 0x51, 0x71, 0x9f, 0x2b, 0xde, 0xad, 0x6a, 0x6f, 0xcf, - 0xf4, 0x3a, 0xa8, 0x44, 0xe0, 0x39, 0x85, 0xa2, 0xf0, 0x6f, 0x1c, 0x2f, 0x6b, 0x8d, 0x4a, 0xbb, - 0x6a, 0x7f, 0x27, 0x60, 0x31, 0xe4, 0x7e, 0x59, 0xd1, 0x21, 0xd4, 0xaf, 0x52, 0xb3, 0x9c, 0xbb, - 0x66, 0xe4, 0xbb, 0x14, 0xc5, 0x8a, 0x95, 0x0a, 0xfa, 0x19, 0x3a, 0xdc, 0xf3, 0x30, 0x51, 0xe8, - 0xbb, 0x02, 0x65, 0x12, 0x47, 0x12, 0x5d, 0x3d, 0x86, 0x6e, 0x65, 0x50, 0x1d, 0xb6, 0xc6, 0x0f, - 0x4c, 0xb3, 0xf1, 0x9b, 0x11, 0x2b, 0xd4, 0xf3, 0x55, 0x82, 0x6c, 0xbf, 0x0c, 0x31, 0x51, 0x69, - 0x1f, 0x41, 0xd3, 0x04, 0xa8, 0x05, 0xf5, 0xd9, 0xc4, 0x79, 0x7b, 0x3e, 0x9d, 0xb5, 0xb7, 0x68, - 0x07, 0x76, 0x67, 0x73, 0x36, 0x9d, 0x38, 0xd3, 0xe7, 0xee, 0xc7, 0x37, 0xcc, 0x3d, 0x3d, 0x7b, - 0xff, 0xfa, 0xd5, 0xac, 0x4d, 0xec, 0x49, 0xe6, 0xe2, 0x9b, 0x28, 0xfa, 0x18, 0xea, 0x02, 0x65, - 0x7a, 0xa9, 0xca, 0x86, 0x3a, 0xff, 0x36, 0xa4, 0x79, 0x56, 0xea, 0xec, 0x6f, 0x04, 0xb6, 0x35, - 0x41, 0x1f, 0x02, 0x95, 0x8a, 0x0b, 0xe5, 0xea, 0xa9, 0x2b, 0x1e, 0x26, 0x6e, 0x98, 0xe5, 0x90, - 0x61, 0x95, 0xb5, 0x35, 0x33, 0x2f, 0x09, 0x47, 0xd2, 0x21, 0xb4, 0x31, 0xf2, 0x6f, 0x6b, 0x2b, - 0x5a, 0xdb, 0xc2, 0xc8, 0x37, 0x95, 0x47, 0xd0, 0x08, 0xb9, 0xf2, 0x96, 0x28, 0x64, 0x71, 0x73, - 0x5d, 0xb3, 0xaa, 0x73, 0xbe, 0xc0, 0x4b, 0x27, 0x17, 0xb0, 0x8d, 0x92, 0x1e, 0xc2, 0xf6, 0x32, - 0x88, 0x94, 0xec, 0xd6, 0x06, 0x64, 0x68, 0x8d, 0xf7, 0xff, 0x1e, 0xee, 0x59, 0x46, 0xb2, 0x5c, - 0x63, 0x4f, 0xc1, 0x32, 0x9a, 0xa3, 0x4f, 0xff, 0x7f, 0xd3, 0xcc, 0x1d, 0xb3, 0xaf, 0x61, 0xf7, - 0x74, 0x99, 0x46, 0x5f, 0xb2, 0xcb, 0x31, 0xa6, 0xfa, 0x0c, 0x5a, 0x5e, 0x0e, 0xbb, 0xb7, 0x22, - 0xef, 0x9b, 0x91, 0x85, 0xb1, 0x48, 0xbd, 0xe3, 0x99, 0x9f, 0xf4, 0x00, 0xac, 0x6c, 0x8d, 0x56, - 0x6e, 0x10, 0xf9, 0x78, 0x5d, 0xcc, 0x09, 0x34, 0xf4, 0x22, 0x43, 0x4e, 0xf6, 0x6e, 0xd6, 0x7d, - 0xf2, 0x63, 0xdd, 0x27, 0xbf, 0xd6, 0x7d, 0xf2, 0x69, 0x27, 0xcb, 0x4d, 0x16, 0x8b, 0x1d, 0xfd, - 0x92, 0x9e, 0xfc, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x13, 0x18, 0x12, 0x0a, 0x88, 0x03, 0x00, 0x00, + // 508 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0xcd, 0x6e, 0x12, 0x41, + 0x1c, 0xef, 0x40, 0x5b, 0xc8, 0x7f, 0x91, 0xe0, 0xf4, 0x03, 0xe4, 0x40, 0xc9, 0xc6, 0x03, 0x49, + 0x0d, 0x46, 0x34, 0x9e, 0x7a, 0x90, 0x22, 0x49, 0x55, 0xd6, 0xd6, 0x01, 0xa3, 0x31, 0x26, 0x9b, + 0x61, 0xf7, 0x9f, 0xb2, 0xb1, 0xfb, 0xd1, 0x99, 0xd9, 0xa4, 0xbc, 0x85, 0xcf, 0xe4, 0xa9, 0x47, + 0x9f, 0xc0, 0x18, 0x0e, 0x3e, 0x87, 0xd9, 0x61, 0x17, 0xa7, 0x7a, 0xf1, 0x36, 0xfb, 0xfb, 0x9a, + 0xf9, 0x7f, 0x2c, 0xd4, 0x04, 0x86, 0xb1, 0xc2, 0x7e, 0x22, 0x62, 0x15, 0x53, 0x48, 0x44, 0x1c, + 0xa2, 0x5a, 0x60, 0x2a, 0xdb, 0x96, 0x5a, 0x26, 0x28, 0xd7, 0x44, 0x7b, 0xff, 0x32, 0xbe, 0x8c, + 0xf5, 0xf1, 0x71, 0x76, 0x5a, 0xa3, 0xf6, 0x57, 0x02, 0xb5, 0x0f, 0x22, 0x50, 0xc8, 0xf0, 0x3a, + 0x45, 0xa9, 0xe8, 0x09, 0x80, 0x0a, 0x42, 0x94, 0x28, 0x02, 0x94, 0x2d, 0xd2, 0x2d, 0xf7, 0xac, + 0xc1, 0x61, 0xff, 0x4f, 0x68, 0x7f, 0x16, 0x84, 0x38, 0xd5, 0xec, 0xe9, 0xf6, 0xed, 0x8f, 0xa3, + 0x2d, 0x66, 0xe8, 0xe9, 0x09, 0x54, 0x43, 0x54, 0xdc, 0xe7, 0x8a, 0xb7, 0xca, 0xda, 0xdb, 0x36, + 0xbd, 0x0e, 0x2a, 0x11, 0x78, 0x4e, 0xae, 0xc8, 0xfd, 0x1b, 0xc7, 0xeb, 0xed, 0x6a, 0xa9, 0x51, + 0xb6, 0x7f, 0x11, 0xb0, 0x18, 0x72, 0xbf, 0x78, 0xd1, 0x31, 0x54, 0xae, 0x53, 0xf3, 0x39, 0xf7, + 0xcd, 0xc8, 0x77, 0x29, 0x8a, 0x25, 0x2b, 0x14, 0xf4, 0x33, 0x34, 0xb9, 0xe7, 0x61, 0xa2, 0xd0, + 0x77, 0x05, 0xca, 0x24, 0x8e, 0x24, 0xba, 0xba, 0x0d, 0xad, 0x52, 0xb7, 0xdc, 0xab, 0x0f, 0x1e, + 0x9a, 0x66, 0xe3, 0x9a, 0x3e, 0xcb, 0xd5, 0xb3, 0x65, 0x82, 0xec, 0xa0, 0x08, 0x31, 0x51, 0x69, + 0x4f, 0xa0, 0x66, 0x02, 0xd4, 0x82, 0xca, 0x74, 0xe8, 0x5c, 0x4c, 0xc6, 0xd3, 0xc6, 0x16, 0x6d, + 0xc2, 0xde, 0x74, 0xc6, 0xc6, 0x43, 0x67, 0xfc, 0xd2, 0xfd, 0x78, 0xce, 0xdc, 0xd1, 0xd9, 0xfb, + 0xb7, 0x6f, 0xa6, 0x0d, 0x42, 0x0f, 0x81, 0x8e, 0xce, 0x9d, 0x8b, 0xe1, 0x68, 0x66, 0xe2, 0x25, + 0x7b, 0x98, 0xa5, 0xf1, 0xcd, 0x15, 0xf4, 0x09, 0x54, 0x04, 0xca, 0xf4, 0x4a, 0x15, 0x85, 0x36, + 0xff, 0x2d, 0x54, 0xf3, 0xac, 0xd0, 0xd9, 0xdf, 0x08, 0xec, 0x68, 0x82, 0x3e, 0x02, 0x2a, 0x15, + 0x17, 0xca, 0xd5, 0xd3, 0x50, 0x3c, 0x4c, 0xdc, 0x30, 0xcb, 0x21, 0xbd, 0x32, 0x6b, 0x68, 0x66, + 0x56, 0x10, 0x8e, 0xa4, 0x3d, 0x68, 0x60, 0xe4, 0xdf, 0xd5, 0x96, 0xb4, 0xb6, 0x8e, 0x91, 0x6f, + 0x2a, 0x9f, 0x41, 0x35, 0xe4, 0xca, 0x5b, 0xa0, 0x90, 0xf9, 0x44, 0x5b, 0xe6, 0xab, 0x26, 0x7c, + 0x8e, 0x57, 0xce, 0x5a, 0xc0, 0x36, 0x4a, 0x7a, 0x0c, 0x3b, 0x8b, 0x20, 0x52, 0xb2, 0xb5, 0xdd, + 0x25, 0x3d, 0x6b, 0x70, 0xf0, 0x77, 0xd3, 0xcf, 0x32, 0x92, 0xad, 0x35, 0xf6, 0x18, 0x2c, 0xa3, + 0x38, 0xfa, 0xfc, 0xff, 0x37, 0xd0, 0xdc, 0x3d, 0xfb, 0x06, 0xf6, 0x46, 0x8b, 0x34, 0xfa, 0x92, + 0x0d, 0xcd, 0xe8, 0xea, 0x0b, 0xa8, 0x7b, 0x6b, 0xd8, 0xbd, 0x13, 0xf9, 0xc0, 0x8c, 0xcc, 0x8d, + 0x79, 0xea, 0x3d, 0xcf, 0xfc, 0xa4, 0x47, 0x60, 0x65, 0xeb, 0xb5, 0x74, 0x83, 0xc8, 0xc7, 0x9b, + 0xbc, 0x4f, 0xa0, 0xa1, 0x57, 0x19, 0x72, 0xba, 0x7f, 0xbb, 0xea, 0x90, 0xef, 0xab, 0x0e, 0xf9, + 0xb9, 0xea, 0x90, 0x4f, 0xbb, 0x59, 0x6e, 0x32, 0x9f, 0xef, 0xea, 0x3f, 0xec, 0xe9, 0xef, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xdd, 0x5e, 0x94, 0xdb, 0xa0, 0x03, 0x00, 0x00, } func (m *WriteRequest) Marshal() (dAtA []byte, err error) { diff --git a/prompb/remote.proto b/prompb/remote.proto index 70c6dd3fbbc..78af4834ecb 100644 --- a/prompb/remote.proto +++ b/prompb/remote.proto @@ -46,6 +46,13 @@ message ReadRequest { // Content-Type: "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse" // Content-Encoding: "" STREAMED_XOR_CHUNKS = 1; + + // It is like the previous one but series are maximally compacted into frames. + // This significantly improves throughput. + // Response headers: + // Content-Type: "application/x-compact-protobuf; proto=prometheus.ChunkedReadResponse" + // Content-Encoding: "" + COMPACT_XOR_CHUNKS = 2; } // accepted_response_types allows negotiating the content type of the response. diff --git a/storage/remote/codec.go b/storage/remote/codec.go index 2ec6658ddea..c67d54eb6f6 100644 --- a/storage/remote/codec.go +++ b/storage/remote/codec.go @@ -174,6 +174,7 @@ func NegotiateResponseType(accepted []prompb.ReadRequest_ResponseType) (prompb.R supported := map[prompb.ReadRequest_ResponseType]struct{}{ prompb.ReadRequest_SAMPLES: {}, prompb.ReadRequest_STREAMED_XOR_CHUNKS: {}, + prompb.ReadRequest_COMPACT_XOR_CHUNKS: {}, } for _, resType := range accepted { @@ -193,10 +194,12 @@ func StreamChunkedReadResponses( sortedExternalLabels []prompb.Label, maxBytesInFrame int, marshalPool *sync.Pool, -) (storage.Warnings, error) { + freeImmediately bool, +) (storage.Warnings, func(), error) { var ( - chks []prompb.Chunk - lbls []prompb.Label + chks []prompb.Chunk + lbls []prompb.Label + returnSlices []*[]byte ) for ss.Next() { @@ -217,7 +220,7 @@ func StreamChunkedReadResponses( chk := iter.At() if chk.Chunk == nil { - return ss.Warnings(), fmt.Errorf("StreamChunkedReadResponses: found not populated chunk returned by SeriesSet at ref: %v", chk.Ref) + return ss.Warnings(), nil, fmt.Errorf("StreamChunkedReadResponses: found not populated chunk returned by SeriesSet at ref: %v", chk.Ref) } // Cut the chunk. @@ -244,23 +247,30 @@ func StreamChunkedReadResponses( b, err := resp.PooledMarshal(marshalPool) if err != nil { - return ss.Warnings(), fmt.Errorf("marshal ChunkedReadResponse: %w", err) + return ss.Warnings(), nil, fmt.Errorf("marshal ChunkedReadResponse: %w", err) } if _, err := stream.Write(b); err != nil { - return ss.Warnings(), fmt.Errorf("write to stream: %w", err) + return ss.Warnings(), nil, fmt.Errorf("write to stream: %w", err) } - // We immediately flush the Write() so it is safe to return to the pool. - marshalPool.Put(&b) + if freeImmediately { + marshalPool.Put(&b) + } else { + returnSlices = append(returnSlices, &b) + } chks = chks[:0] frameBytesLeft = maxDataLength } if err := iter.Err(); err != nil { - return ss.Warnings(), err + return ss.Warnings(), nil, err } } - return ss.Warnings(), ss.Err() + return ss.Warnings(), func() { + for _, rs := range returnSlices { + marshalPool.Put(rs) + } + }, ss.Err() } // MergeLabels merges two sets of sorted proto labels, preferring those in diff --git a/storage/remote/compact_writer.go b/storage/remote/compact_writer.go new file mode 100644 index 00000000000..ab877ef3103 --- /dev/null +++ b/storage/remote/compact_writer.go @@ -0,0 +1,89 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "encoding/binary" + "fmt" + "hash" + "hash/crc32" + "io" + "net/http" +) + +type ChunkedCompactWriter struct { + writer io.Writer + flusher http.Flusher + + crc32 hash.Hash32 + writeBuf []byte +} + +// NewChunkedCompactWriter constructs a ChunkedCompactWriter. +func NewChunkedCompactWriter(writeBuf []byte, w io.Writer, f http.Flusher) *ChunkedCompactWriter { + return &ChunkedCompactWriter{writeBuf: writeBuf[:0], writer: w, flusher: f, crc32: crc32.New(castagnoliTable)} +} + +func (w *ChunkedCompactWriter) Close() error { + if len(w.writeBuf) == 0 { + return nil + } + + n, err := w.writer.Write(w.writeBuf) + if err != nil { + return err + } + if n != len(w.writeBuf) { + return fmt.Errorf("short write: wrote %v but buffer is %v", n, len(w.writeBuf)) + } + + w.flusher.Flush() + return nil +} + +func (w *ChunkedCompactWriter) Write(b []byte) (int, error) { + if len(b) == 0 { + return 0, nil + } + + // len(b) + crc32 + binary.MaxVarintLen64 + requiredSpaceBytes := len(b) + 32/8 + binary.MaxVarintLen64 + + leftSpaceBytes := cap(w.writeBuf) - len(w.writeBuf) + + if len(w.writeBuf) > 0 && leftSpaceBytes < requiredSpaceBytes { + n, err := w.writer.Write(w.writeBuf) + if err != nil { + return n, err + } + if n != len(w.writeBuf) { + return n, fmt.Errorf("short write: wrote %v but buf is %v", n, len(w.writeBuf)) + } + w.flusher.Flush() + w.writeBuf = w.writeBuf[:0] + } + + var buf [binary.MaxVarintLen64]byte + v := binary.PutUvarint(buf[:], uint64(len(b))) + w.writeBuf = append(w.writeBuf, buf[:v]...) + + w.crc32.Reset() + if _, err := w.crc32.Write(b); err != nil { + return 0, err + } + w.writeBuf = binary.BigEndian.AppendUint32(w.writeBuf, w.crc32.Sum32()) + w.writeBuf = append(w.writeBuf, b...) + + return len(b), nil +} diff --git a/storage/remote/read_handler.go b/storage/remote/read_handler.go index 116eb9596c4..3b71cdaf4d4 100644 --- a/storage/remote/read_handler.go +++ b/storage/remote/read_handler.go @@ -38,7 +38,8 @@ type readHandler struct { remoteReadMaxBytesInFrame int remoteReadGate *gate.Gate queries prometheus.Gauge - marshalPool *sync.Pool + chunkPool *sync.Pool + writeBufPool *sync.Pool } // NewReadHandler creates a http.Handler that accepts remote read requests and @@ -51,7 +52,8 @@ func NewReadHandler(logger log.Logger, r prometheus.Registerer, queryable storag remoteReadSampleLimit: remoteReadSampleLimit, remoteReadGate: gate.New(remoteReadConcurrencyLimit), remoteReadMaxBytesInFrame: remoteReadMaxBytesInFrame, - marshalPool: &sync.Pool{}, + chunkPool: &sync.Pool{}, + writeBufPool: &sync.Pool{}, queries: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "prometheus", @@ -103,6 +105,8 @@ func (h *readHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } switch responseType { + case prompb.ReadRequest_COMPACT_XOR_CHUNKS: + h.remoteReadCompactXORChunks(ctx, w, req, externalLabels, sortedExternalLabels) case prompb.ReadRequest_STREAMED_XOR_CHUNKS: h.remoteReadStreamedXORChunks(ctx, w, req, externalLabels, sortedExternalLabels) default: @@ -182,6 +186,91 @@ func (h *readHandler) remoteReadSamples( } } +func (h *readHandler) remoteReadCompactXORChunks(ctx context.Context, w http.ResponseWriter, req *prompb.ReadRequest, externalLabels map[string]string, sortedExternalLabels []prompb.Label) { + w.Header().Set("Content-Type", "application/x-compact-protobuf; proto=prometheus.ChunkedReadResponse") + + f, ok := w.(http.Flusher) + if !ok { + http.Error(w, "internal http.ResponseWriter does not implement http.Flusher interface", http.StatusInternalServerError) + return + } + + var writeBuf []byte + buf := h.writeBufPool.Get() + if buf == nil { + writeBuf = make([]byte, 0, h.remoteReadMaxBytesInFrame) + } else { + writeBuf = *(buf.(*[]byte)) + } + defer h.writeBufPool.Put(&writeBuf) + + for i, query := range req.Queries { + if err := func() error { + filteredMatchers, err := filterExtLabelsFromMatchers(query.Matchers, externalLabels) + if err != nil { + return err + } + + querier, err := h.queryable.ChunkQuerier(ctx, query.StartTimestampMs, query.EndTimestampMs) + if err != nil { + return err + } + defer func() { + if err := querier.Close(); err != nil { + level.Warn(h.logger).Log("msg", "Error on chunk querier close", "err", err.Error()) + } + }() + + var hints *storage.SelectHints + if query.Hints != nil { + hints = &storage.SelectHints{ + Start: query.Hints.StartMs, + End: query.Hints.EndMs, + Step: query.Hints.StepMs, + Func: query.Hints.Func, + Grouping: query.Hints.Grouping, + Range: query.Hints.RangeMs, + By: query.Hints.By, + } + } + + wr := NewChunkedCompactWriter(writeBuf, w, f) + + ws, release, err := StreamChunkedReadResponses( + wr, + int64(i), + // The streaming API has to provide the series sorted. + querier.Select(true, hints, filteredMatchers...), + sortedExternalLabels, + h.remoteReadMaxBytesInFrame, + h.chunkPool, + false, + ) + if err != nil { + return err + } + defer func() { + if release != nil { + release() + } + }() + defer wr.Close() + + for _, w := range ws { + level.Warn(h.logger).Log("msg", "Warnings on chunked remote read query", "warnings", w.Error()) + } + return nil + }(); err != nil { + if httpErr, ok := err.(HTTPError); ok { + http.Error(w, httpErr.Error(), httpErr.Status()) + return + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + func (h *readHandler) remoteReadStreamedXORChunks(ctx context.Context, w http.ResponseWriter, req *prompb.ReadRequest, externalLabels map[string]string, sortedExternalLabels []prompb.Label) { w.Header().Set("Content-Type", "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse") @@ -221,14 +310,15 @@ func (h *readHandler) remoteReadStreamedXORChunks(ctx context.Context, w http.Re } } - ws, err := StreamChunkedReadResponses( + ws, _, err := StreamChunkedReadResponses( NewChunkedWriter(w, f), int64(i), // The streaming API has to provide the series sorted. querier.Select(true, hints, filteredMatchers...), sortedExternalLabels, h.remoteReadMaxBytesInFrame, - h.marshalPool, + h.chunkPool, + true, ) if err != nil { return err