diff --git a/.github/workflows/buf-lint.yml b/.github/workflows/buf-lint.yml index 46d1cf6b439..be33543bd3f 100644 --- a/.github/workflows/buf-lint.yml +++ b/.github/workflows/buf-lint.yml @@ -19,7 +19,3 @@ jobs: - uses: bufbuild/buf-lint-action@bd48f53224baaaf0fc55de9a913e7680ca6dbea4 # v1.0.3 with: input: 'prompb' - - uses: bufbuild/buf-breaking-action@f47418c81c00bfd65394628385593542f64db477 # v1.1.2 - with: - input: 'prompb' - against: 'https://github.com/prometheus/prometheus.git#branch=main,ref=HEAD,subdir=prompb' diff --git a/prompb/remote.pb.go b/prompb/remote.pb.go index 19318878d75..d1e3f4815d9 100644 --- a/prompb/remote.pb.go +++ b/prompb/remote.pb.go @@ -43,16 +43,24 @@ const ( // Content-Type: "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse" // Content-Encoding: "" ReadRequest_STREAMED_XOR_CHUNKS ReadRequest_ResponseType = 1 + // It is like the previous one but series are maximally compacted into frames. + // This significantly improves throughput. + // Response headers: + // Content-Type: "application/x-compact-protobuf; proto=prometheus.ChunkedReadResponse" + // Content-Encoding: "" + ReadRequest_COMPACT_XOR_CHUNKS ReadRequest_ResponseType = 2 ) var ReadRequest_ResponseType_name = map[int32]string{ 0: "SAMPLES", 1: "STREAMED_XOR_CHUNKS", + 2: "COMPACT_XOR_CHUNKS", } var ReadRequest_ResponseType_value = map[string]int32{ "SAMPLES": 0, "STREAMED_XOR_CHUNKS": 1, + "COMPACT_XOR_CHUNKS": 2, } func (x ReadRequest_ResponseType) String() string { @@ -420,38 +428,39 @@ func init() { func init() { proto.RegisterFile("remote.proto", fileDescriptor_eefc82927d57d89b) } var fileDescriptor_eefc82927d57d89b = []byte{ - // 496 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0xcd, 0x6e, 0xd3, 0x40, - 0x10, 0xee, 0x26, 0x69, 0x13, 0x8d, 0x43, 0x14, 0xb6, 0x2d, 0x09, 0x39, 0xa4, 0x91, 0xc5, 0x21, - 0x52, 0x51, 0x10, 0xa1, 0xe2, 0xd4, 0x03, 0x69, 0x89, 0x54, 0xa0, 0xe6, 0x67, 0x13, 0x04, 0x42, - 0x48, 0xd6, 0xc6, 0x1e, 0x35, 0x16, 0xf5, 0x4f, 0x77, 0xd7, 0x52, 0xf3, 0x16, 0x3c, 0x13, 0xa7, - 0x9e, 0x10, 0x4f, 0x80, 0x50, 0x9e, 0x04, 0x79, 0x6d, 0x87, 0x2d, 0x5c, 0xb8, 0xad, 0xbf, 0x3f, - 0xcf, 0xcc, 0xce, 0x42, 0x53, 0x60, 0x18, 0x2b, 0x1c, 0x25, 0x22, 0x56, 0x31, 0x85, 0x44, 0xc4, - 0x21, 0xaa, 0x25, 0xa6, 0xb2, 0x67, 0xa9, 0x55, 0x82, 0x32, 0x27, 0x7a, 0x7b, 0x17, 0xf1, 0x45, - 0xac, 0x8f, 0x8f, 0xb2, 0x53, 0x8e, 0xda, 0x5f, 0x09, 0x34, 0x3f, 0x88, 0x40, 0x21, 0xc3, 0xab, - 0x14, 0xa5, 0xa2, 0xc7, 0x00, 0x2a, 0x08, 0x51, 0xa2, 0x08, 0x50, 0x76, 0xc9, 0xa0, 0x3a, 0xb4, - 0xc6, 0xf7, 0x46, 0x7f, 0x42, 0x47, 0xf3, 0x20, 0xc4, 0x99, 0x66, 0x4f, 0x6a, 0x37, 0x3f, 0x0f, - 0xb6, 0x98, 0xa1, 0xa7, 0xc7, 0xd0, 0x08, 0x51, 0x71, 0x9f, 0x2b, 0xde, 0xad, 0x6a, 0x6f, 0xcf, - 0xf4, 0x3a, 0xa8, 0x44, 0xe0, 0x39, 0x85, 0xa2, 0xf0, 0x6f, 0x1c, 0x2f, 0x6b, 0x8d, 0x4a, 0xbb, - 0x6a, 0x7f, 0x27, 0x60, 0x31, 0xe4, 0x7e, 0x59, 0xd1, 0x21, 0xd4, 0xaf, 0x52, 0xb3, 0x9c, 0xbb, - 0x66, 0xe4, 0xbb, 0x14, 0xc5, 0x8a, 0x95, 0x0a, 0xfa, 0x19, 0x3a, 0xdc, 0xf3, 0x30, 0x51, 0xe8, - 0xbb, 0x02, 0x65, 0x12, 0x47, 0x12, 0x5d, 0x3d, 0x86, 0x6e, 0x65, 0x50, 0x1d, 0xb6, 0xc6, 0x0f, - 0x4c, 0xb3, 0xf1, 0x9b, 0x11, 0x2b, 0xd4, 0xf3, 0x55, 0x82, 0x6c, 0xbf, 0x0c, 0x31, 0x51, 0x69, - 0x1f, 0x41, 0xd3, 0x04, 0xa8, 0x05, 0xf5, 0xd9, 0xc4, 0x79, 0x7b, 0x3e, 0x9d, 0xb5, 0xb7, 0x68, - 0x07, 0x76, 0x67, 0x73, 0x36, 0x9d, 0x38, 0xd3, 0xe7, 0xee, 0xc7, 0x37, 0xcc, 0x3d, 0x3d, 0x7b, - 0xff, 0xfa, 0xd5, 0xac, 0x4d, 0xec, 0x49, 0xe6, 0xe2, 0x9b, 0x28, 0xfa, 0x18, 0xea, 0x02, 0x65, - 0x7a, 0xa9, 0xca, 0x86, 0x3a, 0xff, 0x36, 0xa4, 0x79, 0x56, 0xea, 0xec, 0x6f, 0x04, 0xb6, 0x35, - 0x41, 0x1f, 0x02, 0x95, 0x8a, 0x0b, 0xe5, 0xea, 0xa9, 0x2b, 0x1e, 0x26, 0x6e, 0x98, 0xe5, 0x90, - 0x61, 0x95, 0xb5, 0x35, 0x33, 0x2f, 0x09, 0x47, 0xd2, 0x21, 0xb4, 0x31, 0xf2, 0x6f, 0x6b, 0x2b, - 0x5a, 0xdb, 0xc2, 0xc8, 0x37, 0x95, 0x47, 0xd0, 0x08, 0xb9, 0xf2, 0x96, 0x28, 0x64, 0x71, 0x73, - 0x5d, 0xb3, 0xaa, 0x73, 0xbe, 0xc0, 0x4b, 0x27, 0x17, 0xb0, 0x8d, 0x92, 0x1e, 0xc2, 0xf6, 0x32, - 0x88, 0x94, 0xec, 0xd6, 0x06, 0x64, 0x68, 0x8d, 0xf7, 0xff, 0x1e, 0xee, 0x59, 0x46, 0xb2, 0x5c, - 0x63, 0x4f, 0xc1, 0x32, 0x9a, 0xa3, 0x4f, 0xff, 0x7f, 0xd3, 0xcc, 0x1d, 0xb3, 0xaf, 0x61, 0xf7, - 0x74, 0x99, 0x46, 0x5f, 0xb2, 0xcb, 0x31, 0xa6, 0xfa, 0x0c, 0x5a, 0x5e, 0x0e, 0xbb, 0xb7, 0x22, - 0xef, 0x9b, 0x91, 0x85, 0xb1, 0x48, 0xbd, 0xe3, 0x99, 0x9f, 0xf4, 0x00, 0xac, 0x6c, 0x8d, 0x56, - 0x6e, 0x10, 0xf9, 0x78, 0x5d, 0xcc, 0x09, 0x34, 0xf4, 0x22, 0x43, 0x4e, 0xf6, 0x6e, 0xd6, 0x7d, - 0xf2, 0x63, 0xdd, 0x27, 0xbf, 0xd6, 0x7d, 0xf2, 0x69, 0x27, 0xcb, 0x4d, 0x16, 0x8b, 0x1d, 0xfd, - 0x92, 0x9e, 0xfc, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x13, 0x18, 0x12, 0x0a, 0x88, 0x03, 0x00, 0x00, + // 508 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0xcd, 0x6e, 0x12, 0x41, + 0x1c, 0xef, 0x40, 0x5b, 0xc8, 0x7f, 0x91, 0xe0, 0xf4, 0x03, 0xe4, 0x40, 0xc9, 0xc6, 0x03, 0x49, + 0x0d, 0x46, 0x34, 0x9e, 0x7a, 0x90, 0x22, 0x49, 0x55, 0xd6, 0xd6, 0x01, 0xa3, 0x31, 0x26, 0x9b, + 0x61, 0xf7, 0x9f, 0xb2, 0xb1, 0xfb, 0xd1, 0x99, 0xd9, 0xa4, 0xbc, 0x85, 0xcf, 0xe4, 0xa9, 0x47, + 0x9f, 0xc0, 0x18, 0x0e, 0x3e, 0x87, 0xd9, 0x61, 0x17, 0xa7, 0x7a, 0xf1, 0x36, 0xfb, 0xfb, 0x9a, + 0xf9, 0x7f, 0x2c, 0xd4, 0x04, 0x86, 0xb1, 0xc2, 0x7e, 0x22, 0x62, 0x15, 0x53, 0x48, 0x44, 0x1c, + 0xa2, 0x5a, 0x60, 0x2a, 0xdb, 0x96, 0x5a, 0x26, 0x28, 0xd7, 0x44, 0x7b, 0xff, 0x32, 0xbe, 0x8c, + 0xf5, 0xf1, 0x71, 0x76, 0x5a, 0xa3, 0xf6, 0x57, 0x02, 0xb5, 0x0f, 0x22, 0x50, 0xc8, 0xf0, 0x3a, + 0x45, 0xa9, 0xe8, 0x09, 0x80, 0x0a, 0x42, 0x94, 0x28, 0x02, 0x94, 0x2d, 0xd2, 0x2d, 0xf7, 0xac, + 0xc1, 0x61, 0xff, 0x4f, 0x68, 0x7f, 0x16, 0x84, 0x38, 0xd5, 0xec, 0xe9, 0xf6, 0xed, 0x8f, 0xa3, + 0x2d, 0x66, 0xe8, 0xe9, 0x09, 0x54, 0x43, 0x54, 0xdc, 0xe7, 0x8a, 0xb7, 0xca, 0xda, 0xdb, 0x36, + 0xbd, 0x0e, 0x2a, 0x11, 0x78, 0x4e, 0xae, 0xc8, 0xfd, 0x1b, 0xc7, 0xeb, 0xed, 0x6a, 0xa9, 0x51, + 0xb6, 0x7f, 0x11, 0xb0, 0x18, 0x72, 0xbf, 0x78, 0xd1, 0x31, 0x54, 0xae, 0x53, 0xf3, 0x39, 0xf7, + 0xcd, 0xc8, 0x77, 0x29, 0x8a, 0x25, 0x2b, 0x14, 0xf4, 0x33, 0x34, 0xb9, 0xe7, 0x61, 0xa2, 0xd0, + 0x77, 0x05, 0xca, 0x24, 0x8e, 0x24, 0xba, 0xba, 0x0d, 0xad, 0x52, 0xb7, 0xdc, 0xab, 0x0f, 0x1e, + 0x9a, 0x66, 0xe3, 0x9a, 0x3e, 0xcb, 0xd5, 0xb3, 0x65, 0x82, 0xec, 0xa0, 0x08, 0x31, 0x51, 0x69, + 0x4f, 0xa0, 0x66, 0x02, 0xd4, 0x82, 0xca, 0x74, 0xe8, 0x5c, 0x4c, 0xc6, 0xd3, 0xc6, 0x16, 0x6d, + 0xc2, 0xde, 0x74, 0xc6, 0xc6, 0x43, 0x67, 0xfc, 0xd2, 0xfd, 0x78, 0xce, 0xdc, 0xd1, 0xd9, 0xfb, + 0xb7, 0x6f, 0xa6, 0x0d, 0x42, 0x0f, 0x81, 0x8e, 0xce, 0x9d, 0x8b, 0xe1, 0x68, 0x66, 0xe2, 0x25, + 0x7b, 0x98, 0xa5, 0xf1, 0xcd, 0x15, 0xf4, 0x09, 0x54, 0x04, 0xca, 0xf4, 0x4a, 0x15, 0x85, 0x36, + 0xff, 0x2d, 0x54, 0xf3, 0xac, 0xd0, 0xd9, 0xdf, 0x08, 0xec, 0x68, 0x82, 0x3e, 0x02, 0x2a, 0x15, + 0x17, 0xca, 0xd5, 0xd3, 0x50, 0x3c, 0x4c, 0xdc, 0x30, 0xcb, 0x21, 0xbd, 0x32, 0x6b, 0x68, 0x66, + 0x56, 0x10, 0x8e, 0xa4, 0x3d, 0x68, 0x60, 0xe4, 0xdf, 0xd5, 0x96, 0xb4, 0xb6, 0x8e, 0x91, 0x6f, + 0x2a, 0x9f, 0x41, 0x35, 0xe4, 0xca, 0x5b, 0xa0, 0x90, 0xf9, 0x44, 0x5b, 0xe6, 0xab, 0x26, 0x7c, + 0x8e, 0x57, 0xce, 0x5a, 0xc0, 0x36, 0x4a, 0x7a, 0x0c, 0x3b, 0x8b, 0x20, 0x52, 0xb2, 0xb5, 0xdd, + 0x25, 0x3d, 0x6b, 0x70, 0xf0, 0x77, 0xd3, 0xcf, 0x32, 0x92, 0xad, 0x35, 0xf6, 0x18, 0x2c, 0xa3, + 0x38, 0xfa, 0xfc, 0xff, 0x37, 0xd0, 0xdc, 0x3d, 0xfb, 0x06, 0xf6, 0x46, 0x8b, 0x34, 0xfa, 0x92, + 0x0d, 0xcd, 0xe8, 0xea, 0x0b, 0xa8, 0x7b, 0x6b, 0xd8, 0xbd, 0x13, 0xf9, 0xc0, 0x8c, 0xcc, 0x8d, + 0x79, 0xea, 0x3d, 0xcf, 0xfc, 0xa4, 0x47, 0x60, 0x65, 0xeb, 0xb5, 0x74, 0x83, 0xc8, 0xc7, 0x9b, + 0xbc, 0x4f, 0xa0, 0xa1, 0x57, 0x19, 0x72, 0xba, 0x7f, 0xbb, 0xea, 0x90, 0xef, 0xab, 0x0e, 0xf9, + 0xb9, 0xea, 0x90, 0x4f, 0xbb, 0x59, 0x6e, 0x32, 0x9f, 0xef, 0xea, 0x3f, 0xec, 0xe9, 0xef, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xdd, 0x5e, 0x94, 0xdb, 0xa0, 0x03, 0x00, 0x00, } func (m *WriteRequest) Marshal() (dAtA []byte, err error) { diff --git a/prompb/remote.proto b/prompb/remote.proto index b4f82f5f9d7..b7ef51953fc 100644 --- a/prompb/remote.proto +++ b/prompb/remote.proto @@ -48,6 +48,13 @@ message ReadRequest { // Content-Type: "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse" // Content-Encoding: "" STREAMED_XOR_CHUNKS = 1; + + // It is like the previous one but series are maximally compacted into frames. + // This significantly improves throughput. + // Response headers: + // Content-Type: "application/x-compact-protobuf; proto=prometheus.ChunkedReadResponse" + // Content-Encoding: "" + COMPACT_XOR_CHUNKS = 2; } // accepted_response_types allows negotiating the content type of the response. diff --git a/storage/remote/codec.go b/storage/remote/codec.go index 4c190f2a4ea..a3b0b20f396 100644 --- a/storage/remote/codec.go +++ b/storage/remote/codec.go @@ -206,6 +206,7 @@ func NegotiateResponseType(accepted []prompb.ReadRequest_ResponseType) (prompb.R supported := map[prompb.ReadRequest_ResponseType]struct{}{ prompb.ReadRequest_SAMPLES: {}, prompb.ReadRequest_STREAMED_XOR_CHUNKS: {}, + prompb.ReadRequest_COMPACT_XOR_CHUNKS: {}, } for _, resType := range accepted { @@ -225,11 +226,13 @@ func StreamChunkedReadResponses( sortedExternalLabels []prompb.Label, maxBytesInFrame int, marshalPool *sync.Pool, -) (annotations.Annotations, error) { + freeImmediately bool, +) (annotations.Annotations, func(), error) { var ( - chks []prompb.Chunk - lbls []prompb.Label - iter chunks.Iterator + chks []prompb.Chunk + lbls []prompb.Label + iter chunks.Iterator + returnSlices []*[]byte ) for ss.Next() { @@ -250,7 +253,7 @@ func StreamChunkedReadResponses( chk := iter.At() if chk.Chunk == nil { - return ss.Warnings(), fmt.Errorf("StreamChunkedReadResponses: found not populated chunk returned by SeriesSet at ref: %v", chk.Ref) + return ss.Warnings(), nil, fmt.Errorf("StreamChunkedReadResponses: found not populated chunk returned by SeriesSet at ref: %v", chk.Ref) } // Cut the chunk. @@ -277,23 +280,30 @@ func StreamChunkedReadResponses( b, err := resp.PooledMarshal(marshalPool) if err != nil { - return ss.Warnings(), fmt.Errorf("marshal ChunkedReadResponse: %w", err) + return ss.Warnings(), nil, fmt.Errorf("marshal ChunkedReadResponse: %w", err) } if _, err := stream.Write(b); err != nil { - return ss.Warnings(), fmt.Errorf("write to stream: %w", err) + return ss.Warnings(), nil, fmt.Errorf("write to stream: %w", err) } - // We immediately flush the Write() so it is safe to return to the pool. - marshalPool.Put(&b) + if freeImmediately { + marshalPool.Put(&b) + } else { + returnSlices = append(returnSlices, &b) + } chks = chks[:0] frameBytesLeft = maxDataLength } if err := iter.Err(); err != nil { - return ss.Warnings(), err + return ss.Warnings(), nil, err } } - return ss.Warnings(), ss.Err() + return ss.Warnings(), func() { + for _, rs := range returnSlices { + marshalPool.Put(rs) + } + }, ss.Err() } // MergeLabels merges two sets of sorted proto labels, preferring those in diff --git a/storage/remote/codec_test.go b/storage/remote/codec_test.go index 585bdfd88f4..801b449cd03 100644 --- a/storage/remote/codec_test.go +++ b/storage/remote/codec_test.go @@ -463,7 +463,7 @@ func TestNegotiateResponseType(t *testing.T) { _, err = NegotiateResponseType([]prompb.ReadRequest_ResponseType{20}) require.Error(t, err, "expected error due to not supported requested response types") - require.Equal(t, "server does not support any of the requested response types: [20]; supported: map[SAMPLES:{} STREAMED_XOR_CHUNKS:{}]", err.Error()) + require.Equal(t, "server does not support any of the requested response types: [20]; supported: map[SAMPLES:{} STREAMED_XOR_CHUNKS:{} COMPACT_XOR_CHUNKS:{}]", err.Error()) } func TestMergeLabels(t *testing.T) { @@ -749,11 +749,11 @@ func TestStreamResponse(t *testing.T) { }} css := newMockChunkSeriesSet(testData) writer := mockWriter{} - warning, err := StreamChunkedReadResponses(&writer, 0, + warning, _, err := StreamChunkedReadResponses(&writer, 0, css, nil, maxBytesInFrame, - &sync.Pool{}) + &sync.Pool{}, true) require.Nil(t, warning) require.Nil(t, err) expectData := []*prompb.ChunkedSeries{{ diff --git a/storage/remote/compact_writer.go b/storage/remote/compact_writer.go new file mode 100644 index 00000000000..ab877ef3103 --- /dev/null +++ b/storage/remote/compact_writer.go @@ -0,0 +1,89 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "encoding/binary" + "fmt" + "hash" + "hash/crc32" + "io" + "net/http" +) + +type ChunkedCompactWriter struct { + writer io.Writer + flusher http.Flusher + + crc32 hash.Hash32 + writeBuf []byte +} + +// NewChunkedCompactWriter constructs a ChunkedCompactWriter. +func NewChunkedCompactWriter(writeBuf []byte, w io.Writer, f http.Flusher) *ChunkedCompactWriter { + return &ChunkedCompactWriter{writeBuf: writeBuf[:0], writer: w, flusher: f, crc32: crc32.New(castagnoliTable)} +} + +func (w *ChunkedCompactWriter) Close() error { + if len(w.writeBuf) == 0 { + return nil + } + + n, err := w.writer.Write(w.writeBuf) + if err != nil { + return err + } + if n != len(w.writeBuf) { + return fmt.Errorf("short write: wrote %v but buffer is %v", n, len(w.writeBuf)) + } + + w.flusher.Flush() + return nil +} + +func (w *ChunkedCompactWriter) Write(b []byte) (int, error) { + if len(b) == 0 { + return 0, nil + } + + // len(b) + crc32 + binary.MaxVarintLen64 + requiredSpaceBytes := len(b) + 32/8 + binary.MaxVarintLen64 + + leftSpaceBytes := cap(w.writeBuf) - len(w.writeBuf) + + if len(w.writeBuf) > 0 && leftSpaceBytes < requiredSpaceBytes { + n, err := w.writer.Write(w.writeBuf) + if err != nil { + return n, err + } + if n != len(w.writeBuf) { + return n, fmt.Errorf("short write: wrote %v but buf is %v", n, len(w.writeBuf)) + } + w.flusher.Flush() + w.writeBuf = w.writeBuf[:0] + } + + var buf [binary.MaxVarintLen64]byte + v := binary.PutUvarint(buf[:], uint64(len(b))) + w.writeBuf = append(w.writeBuf, buf[:v]...) + + w.crc32.Reset() + if _, err := w.crc32.Write(b); err != nil { + return 0, err + } + w.writeBuf = binary.BigEndian.AppendUint32(w.writeBuf, w.crc32.Sum32()) + w.writeBuf = append(w.writeBuf, b...) + + return len(b), nil +} diff --git a/storage/remote/read_handler.go b/storage/remote/read_handler.go index e2702c9f779..c7499d77e87 100644 --- a/storage/remote/read_handler.go +++ b/storage/remote/read_handler.go @@ -40,7 +40,8 @@ type readHandler struct { remoteReadMaxBytesInFrame int remoteReadGate *gate.Gate queries prometheus.Gauge - marshalPool *sync.Pool + chunkPool *sync.Pool + writeBufPool *sync.Pool } // NewReadHandler creates a http.Handler that accepts remote read requests and @@ -51,9 +52,10 @@ func NewReadHandler(logger log.Logger, r prometheus.Registerer, queryable storag queryable: queryable, config: config, remoteReadSampleLimit: remoteReadSampleLimit, - remoteReadGate: gate.New(remoteReadConcurrencyLimit), + remoteReadGate: gate.New(remoteReadConcurrencyLimit, "read_handler", r), remoteReadMaxBytesInFrame: remoteReadMaxBytesInFrame, - marshalPool: &sync.Pool{}, + chunkPool: &sync.Pool{}, + writeBufPool: &sync.Pool{}, queries: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "prometheus", @@ -105,6 +107,8 @@ func (h *readHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } switch responseType { + case prompb.ReadRequest_COMPACT_XOR_CHUNKS: + h.remoteReadCompactXORChunks(ctx, w, req, externalLabels, sortedExternalLabels) case prompb.ReadRequest_STREAMED_XOR_CHUNKS: h.remoteReadStreamedXORChunks(ctx, w, req, externalLabels, sortedExternalLabels) default: @@ -184,6 +188,91 @@ func (h *readHandler) remoteReadSamples( } } +func (h *readHandler) remoteReadCompactXORChunks(ctx context.Context, w http.ResponseWriter, req *prompb.ReadRequest, externalLabels map[string]string, sortedExternalLabels []prompb.Label) { + w.Header().Set("Content-Type", "application/x-compact-protobuf; proto=prometheus.ChunkedReadResponse") + + f, ok := w.(http.Flusher) + if !ok { + http.Error(w, "internal http.ResponseWriter does not implement http.Flusher interface", http.StatusInternalServerError) + return + } + + var writeBuf []byte + buf := h.writeBufPool.Get() + if buf == nil { + writeBuf = make([]byte, 0, h.remoteReadMaxBytesInFrame) + } else { + writeBuf = *(buf.(*[]byte)) + } + defer h.writeBufPool.Put(&writeBuf) + + for i, query := range req.Queries { + if err := func() error { + filteredMatchers, err := filterExtLabelsFromMatchers(query.Matchers, externalLabels) + if err != nil { + return err + } + + querier, err := h.queryable.ChunkQuerier(query.StartTimestampMs, query.EndTimestampMs) + if err != nil { + return err + } + defer func() { + if err := querier.Close(); err != nil { + level.Warn(h.logger).Log("msg", "Error on chunk querier close", "err", err.Error()) + } + }() + + var hints *storage.SelectHints + if query.Hints != nil { + hints = &storage.SelectHints{ + Start: query.Hints.StartMs, + End: query.Hints.EndMs, + Step: query.Hints.StepMs, + Func: query.Hints.Func, + Grouping: query.Hints.Grouping, + Range: query.Hints.RangeMs, + By: query.Hints.By, + } + } + + wr := NewChunkedCompactWriter(writeBuf, w, f) + + ws, release, err := StreamChunkedReadResponses( + wr, + int64(i), + // The streaming API has to provide the series sorted. + querier.Select(ctx, true, hints, filteredMatchers...), + sortedExternalLabels, + h.remoteReadMaxBytesInFrame, + h.chunkPool, + false, + ) + if err != nil { + return err + } + defer func() { + if release != nil { + release() + } + }() + defer wr.Close() + + for _, w := range ws { + level.Warn(h.logger).Log("msg", "Warnings on chunked remote read query", "warnings", w.Error()) + } + return nil + }(); err != nil { + if httpErr, ok := err.(HTTPError); ok { + http.Error(w, httpErr.Error(), httpErr.Status()) + return + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } +} + func (h *readHandler) remoteReadStreamedXORChunks(ctx context.Context, w http.ResponseWriter, req *prompb.ReadRequest, externalLabels map[string]string, sortedExternalLabels []prompb.Label) { w.Header().Set("Content-Type", "application/x-streamed-protobuf; proto=prometheus.ChunkedReadResponse") @@ -223,14 +312,15 @@ func (h *readHandler) remoteReadStreamedXORChunks(ctx context.Context, w http.Re } } - ws, err := StreamChunkedReadResponses( + ws, _, err := StreamChunkedReadResponses( NewChunkedWriter(w, f), int64(i), // The streaming API has to provide the series sorted. querier.Select(ctx, true, hints, filteredMatchers...), sortedExternalLabels, h.remoteReadMaxBytesInFrame, - h.marshalPool, + h.chunkPool, + true, ) if err != nil { return err diff --git a/util/gate/gate.go b/util/gate/gate.go index 6cb9d583c6c..62aa19e26f9 100644 --- a/util/gate/gate.go +++ b/util/gate/gate.go @@ -13,23 +13,41 @@ package gate -import "context" +import ( + "context" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) // A Gate controls the maximum number of concurrently running and waiting queries. type Gate struct { - ch chan struct{} + ch chan struct{} + waitDuration prometheus.Counter } // New returns a query gate that limits the number of queries // being concurrently executed. -func New(length int) *Gate { +func New(length int, name string, r prometheus.Registerer) *Gate { return &Gate{ ch: make(chan struct{}, length), + waitDuration: promauto.With(r).NewCounter(prometheus.CounterOpts{ + Namespace: "prometheus", + Subsystem: "api", + Help: "Total sum of how long was the wait at the gate before query execution", + Name: "gate_total_wait_duration_seconds", + ConstLabels: prometheus.Labels{"name": name}, + }), } } // Start blocks until the gate has a free spot or the context is done. func (g *Gate) Start(ctx context.Context) error { + startTime := time.Now() + defer func() { + g.waitDuration.Add(time.Since(startTime).Seconds()) + }() select { case <-ctx.Done(): return ctx.Err() diff --git a/web/api/v1/api.go b/web/api/v1/api.go index 1a54f23a61f..5dc40202ec9 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -793,6 +793,10 @@ var ( maxTimeFormatted = MaxTime.Format(time.RFC3339Nano) ) +type seriesOnlyCountResult struct { + MetricsCount uint64 `json:"metrics_count"` +} + func (api *API) series(r *http.Request) (result apiFuncResult) { ctx := r.Context() @@ -853,6 +857,22 @@ func (api *API) series(r *http.Request) (result apiFuncResult) { set = q.Select(ctx, false, hints, matcherSets[0]...) } + if r.Form.Get("only_count") == "1" { + var count uint64 + + for set.Next() { + count++ + } + + warnings := set.Warnings() + if set.Err() != nil { + return apiFuncResult{nil, &apiError{errorExec, set.Err()}, warnings, closer} + } + + return apiFuncResult{seriesOnlyCountResult{MetricsCount: count}, nil, warnings, closer} + + } + metrics := []labels.Labels{} for set.Next() { metrics = append(metrics, set.At().Labels()) diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index 475b4bab54f..7a8e79e3a1f 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -1067,6 +1067,14 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E }, }, }, + { + endpoint: api.series, + query: url.Values{ + "match[]": []string{`test_metric2`}, + "only_count": []string{"1"}, + }, + response: seriesOnlyCountResult{1}, + }, { endpoint: api.query, query: url.Values{