Node/CCQ: Proxy log stats on timeout (#4067)

node/cmd/ccq/http.go

@@ -167,7 +167,14 @@ func (s *httpServer) handleQuery(w http.ResponseWriter, r *http.Request) {
 	// Wait for the response or timeout
 	select {
 	case <-time.After(query.RequestTimeout + 5*time.Second):
-		s.logger.Info("publishing time out to client", zap.String("userId", permEntry.userName), zap.String("requestId", requestId))
+		maxMatchingResponses, outstandingResponses, quorum := pendingResponse.getStats()
+		s.logger.Info("publishing time out to client",
+			zap.String("userId", permEntry.userName),
+			zap.String("requestId", requestId),
+			zap.Int("maxMatchingResponses", maxMatchingResponses),
+			zap.Int("outstandingResponses", outstandingResponses),
+			zap.Int("quorum", quorum),
+		)
 		http.Error(w, "Timed out waiting for response", http.StatusGatewayTimeout)
 		queryTimeoutsByUser.WithLabelValues(permEntry.userName).Inc()
 		failedQueriesByUser.WithLabelValues(permEntry.userName).Inc()

node/cmd/ccq/p2p.go

@@ -270,6 +270,7 @@ func runP2P(
 							}
 						}
 						outstandingResponses := len(guardianSet.Keys) - totalSigners
+						pendingResponse.updateStats(maxMatchingResponses, outstandingResponses, quorum)
 						if maxMatchingResponses+outstandingResponses < quorum {
 							quorumNotMetByUser.WithLabelValues(pendingResponse.userName).Inc()
 							failedQueriesByUser.WithLabelValues(pendingResponse.userName).Inc()

node/cmd/ccq/pending_request.go

@@ -16,6 +16,12 @@ type PendingResponse struct {
 	queryRequest *query.QueryRequest
 	ch           chan *SignedResponse
 	errCh        chan *ErrorEntry
+
+	// statsLock protects the data items below.
+	statsLock            sync.RWMutex
+	maxMatchingResponses int
+	outstandingResponses int
+	quorum               int
 }
 
 type ErrorEntry struct {
@@ -111,3 +117,17 @@ func (p *PendingResponses) updateMetricsAlreadyLocked(reqRemoved *PendingRespons
 		}
 	}
 }
+
+func (p *PendingResponse) updateStats(maxMatchingResponses int, outstandingResponses int, quorum int) {
+	p.statsLock.Lock()
+	defer p.statsLock.Unlock()
+	p.maxMatchingResponses = maxMatchingResponses
+	p.outstandingResponses = outstandingResponses
+	p.quorum = quorum
+}
+
+func (p *PendingResponse) getStats() (int, int, int) {
+	p.statsLock.Lock()
+	defer p.statsLock.Unlock()
+	return p.maxMatchingResponses, p.outstandingResponses, p.quorum
+}