expose eligible_at and signed_at in data api

cmd/tool/export-data-api-payloads-bids.go

@@ -60,9 +60,9 @@ var DataAPIExportBids = &cobra.Command{
 		}
 
 		log.Infof("got %d bids", len(bids))
-		entries := make([]common.BidTraceV2WithTimestampJSON, len(bids))
+		entries := make([]common.BidTraceV3JSON, len(bids))
 		for i, bid := range bids {
-			entries[i] = database.BuilderSubmissionEntryToBidTraceV2WithTimestampJSON(bid)
+			entries[i] = database.BuilderSubmissionEntryToBidTraceV3JSON(bid)
 		}
 
 		if len(entries) == 0 {

cmd/tool/export-data-api-payloads-delivered.go

@@ -72,9 +72,9 @@ var DataAPIExportPayloads = &cobra.Command{
 		}
 
 		log.Infof("got %d payloads", len(deliveredPayloads))
-		entries := make([]common.BidTraceV2JSON, len(deliveredPayloads))
+		entries := make([]common.BidTraceV3JSON, len(deliveredPayloads))
 		for i, payload := range deliveredPayloads {
-			entries[i] = database.DeliveredPayloadEntryToBidTraceV2JSON(payload)
+			entries[i] = database.DeliveredPayloadEntryToBidTraceV3JSON(payload)
 		}
 
 		if len(entries) == 0 {

common/types.go

@@ -299,6 +299,64 @@ func (b *BidTraceV2WithTimestampJSON) ToCSVRecord() []string {
 	}
 }
 
+type BidTraceV3JSON struct {
+	Slot                  uint64 `json:"slot,string"`
+	ParentHash            string `json:"parent_hash"`
+	BlockHash             string `json:"block_hash"`
+	BuilderPubkey         string `json:"builder_pubkey"`
+	ProposerPubkey        string `json:"proposer_pubkey"`
+	ProposerFeeRecipient  string `json:"proposer_fee_recipient"`
+	GasLimit              uint64 `json:"gas_limit,string"`
+	GasUsed               uint64 `json:"gas_used,string"`
+	Value                 string `json:"value"`
+	NumTx                 uint64 `json:"num_tx,string"`
+	BlockNumber           uint64 `json:"block_number,string"`
+	Timestamp             int64  `json:"timestamp,string,omitempty"`
+	TimestampMs           int64  `json:"timestamp_ms,string,omitempty"`
+	EligibleAtTimestampMs int64  `json:"eligible_at_timestamp_ms,string,omitempty"`
+	SignedAtTimestampMs   int64  `json:"signed_at_timestamp_ms,string,omitempty"`
+}
+
+func (b *BidTraceV3JSON) CSVHeader() []string {
+	return []string{
+		"slot",
+		"parent_hash",
+		"block_hash",
+		"builder_pubkey",
+		"proposer_pubkey",
+		"proposer_fee_recipient",
+		"gas_limit",
+		"gas_used",
+		"value",
+		"num_tx",
+		"block_number",
+		"timestamp",
+		"timestamp_ms",
+		"eligible_at_timestamp_ms",
+		"signed_at_timestamp_ms",
+	}
+}
+
+func (b *BidTraceV3JSON) ToCSVRecord() []string {
+	return []string{
+		fmt.Sprint(b.Slot),
+		b.ParentHash,
+		b.BlockHash,
+		b.BuilderPubkey,
+		b.ProposerPubkey,
+		b.ProposerFeeRecipient,
+		fmt.Sprint(b.GasLimit),
+		fmt.Sprint(b.GasUsed),
+		b.Value,
+		fmt.Sprint(b.NumTx),
+		fmt.Sprint(b.BlockNumber),
+		fmt.Sprint(b.Timestamp),
+		fmt.Sprint(b.TimestampMs),
+		fmt.Sprint(b.EligibleAtTimestampMs),
+		fmt.Sprint(b.SignedAtTimestampMs),
+	}
+}
+
 type SignedBlindedBeaconBlock struct {
 	Bellatrix *boostTypes.SignedBlindedBeaconBlock
 	Capella   *apiv1capella.SignedBlindedBeaconBlock

database/typesconv.go

@@ -74,3 +74,60 @@ func BuilderSubmissionEntryToBidTraceV2WithTimestampJSON(payload *BuilderBlockSu
 		},
 	}
 }
+
+func DeliveredPayloadEntryToBidTraceV3JSON(payload *DeliveredPayloadEntry) common.BidTraceV3JSON {
+	bidTrace := common.BidTraceV3JSON{
+		Slot:                  payload.Slot,
+		ParentHash:            payload.ParentHash,
+		BlockHash:             payload.BlockHash,
+		BuilderPubkey:         payload.BuilderPubkey,
+		ProposerPubkey:        payload.ProposerPubkey,
+		ProposerFeeRecipient:  payload.ProposerFeeRecipient,
+		GasLimit:              payload.GasLimit,
+		GasUsed:               payload.GasUsed,
+		Value:                 payload.Value,
+		NumTx:                 payload.NumTx,
+		BlockNumber:           payload.BlockNumber,
+		Timestamp:             int64(0),
+		TimestampMs:           int64(0),
+		SignedAtTimestampMs:   int64(0),
+		EligibleAtTimestampMs: int64(0),
+	}
+
+	if payload.SignedAt.Valid {
+		bidTrace.SignedAtTimestampMs = payload.SignedAt.Time.UnixMilli()
+	}
+
+	return bidTrace
+}
+
+func BuilderSubmissionEntryToBidTraceV3JSON(payload *BuilderBlockSubmissionEntry) common.BidTraceV3JSON {
+	timestamp := payload.InsertedAt
+	if payload.ReceivedAt.Valid {
+		timestamp = payload.ReceivedAt.Time
+	}
+
+	bidtrace := common.BidTraceV3JSON{
+		Timestamp:             timestamp.Unix(),
+		TimestampMs:           timestamp.UnixMilli(),
+		Slot:                  payload.Slot,
+		ParentHash:            payload.ParentHash,
+		BlockHash:             payload.BlockHash,
+		BuilderPubkey:         payload.BuilderPubkey,
+		ProposerPubkey:        payload.ProposerPubkey,
+		ProposerFeeRecipient:  payload.ProposerFeeRecipient,
+		GasLimit:              payload.GasLimit,
+		GasUsed:               payload.GasUsed,
+		Value:                 payload.Value,
+		NumTx:                 payload.NumTx,
+		BlockNumber:           payload.BlockNumber,
+		EligibleAtTimestampMs: int64(0),
+		SignedAtTimestampMs:   int64(0),
+	}
+
+	if payload.EligibleAt.Valid {
+		bidtrace.EligibleAtTimestampMs = payload.EligibleAt.Time.UnixMilli()
+	}
+
+	return bidtrace
+}

services/api/service.go

@@ -1625,9 +1625,9 @@ func (api *RelayAPI) handleDataProposerPayloadDelivered(w http.ResponseWriter, r
 		return
 	}
 
-	response := make([]common.BidTraceV2JSON, len(deliveredPayloads))
+	response := make([]common.BidTraceV3JSON, len(deliveredPayloads))
 	for i, payload := range deliveredPayloads {
-		response[i] = database.DeliveredPayloadEntryToBidTraceV2JSON(payload)
+		response[i] = database.DeliveredPayloadEntryToBidTraceV3JSON(payload)
 	}
 
 	api.RespondOK(w, response)
@@ -1710,9 +1710,9 @@ func (api *RelayAPI) handleDataBuilderBidsReceived(w http.ResponseWriter, req *h
 		return
 	}
 
-	response := make([]common.BidTraceV2WithTimestampJSON, len(blockSubmissions))
+	response := make([]common.BidTraceV3JSON, len(blockSubmissions))
 	for i, payload := range blockSubmissions {
-		response[i] = database.BuilderSubmissionEntryToBidTraceV2WithTimestampJSON(payload)
+		response[i] = database.BuilderSubmissionEntryToBidTraceV3JSON(payload)
 	}
 
 	api.RespondOK(w, response)

services/api/service_test.go

@@ -329,3 +329,160 @@ func TestDataApiGetDataProposerPayloadDelivered(t *testing.T) {
 		}
 	})
 }
+
+func TestDataApiGetBuilderBlocksReceived(t *testing.T) {
+	path := "/relay/v1/data/bidtraces/builder_blocks_received"
+
+	t.Run("Reject requests with cursor", func(t *testing.T) {
+		backend := newTestBackend(t, 1)
+		rr := backend.request(http.MethodGet, path+"?cursor=1", nil)
+		require.Equal(t, http.StatusBadRequest, rr.Code)
+		require.Contains(t, rr.Body.String(), "cursor argument not supported")
+	})
+
+	t.Run("Accept valid slot", func(t *testing.T) {
+		backend := newTestBackend(t, 1)
+
+		validSlot := uint64(2)
+		validSlotPath := fmt.Sprintf("%s?slot=%d", path, validSlot)
+		rr := backend.request(http.MethodGet, validSlotPath, nil)
+		require.Equal(t, http.StatusOK, rr.Code)
+	})
+
+	t.Run("Accept valid slot", func(t *testing.T) {
+		backend := newTestBackend(t, 1)
+
+		validSlot := uint64(2)
+		validSlotPath := fmt.Sprintf("%s?slot=%d", path, validSlot)
+		rr := backend.request(http.MethodGet, validSlotPath, nil)
+		require.Equal(t, http.StatusOK, rr.Code)
+	})
+
+	t.Run("Reject invalid slot", func(t *testing.T) {
+		backend := newTestBackend(t, 1)
+
+		invalidSlots := []string{
+			"-1",
+			"1.1",
+		}
+
+		for _, invalidSlot := range invalidSlots {
+			invalidSlotPath := fmt.Sprintf("%s?slot=%s", path, invalidSlot)
+			rr := backend.request(http.MethodGet, invalidSlotPath, nil)
+			require.Equal(t, http.StatusBadRequest, rr.Code)
+			require.Contains(t, rr.Body.String(), "invalid slot argument")
+		}
+	})
+
+	t.Run("Accept valid block_hash", func(t *testing.T) {
+		backend := newTestBackend(t, 1)
+
+		validBlockHash := "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+		rr := backend.request(http.MethodGet, path+"?block_hash="+validBlockHash, nil)
+		require.Equal(t, http.StatusOK, rr.Code)
+	})
+
+	t.Run("Reject invalid block_hash", func(t *testing.T) {
+		backend := newTestBackend(t, 1)
+
+		invalidBlockHashes := []string{
+			// One character too long.
+			"0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab",
+			// One character too short.
+			"0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+			// Missing the 0x prefix.
+			"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+			// Has an invalid hex character ('z' at the end).
+			"0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaz",
+		}
+
+		for _, invalidBlockHash := range invalidBlockHashes {
+			rr := backend.request(http.MethodGet, path+"?block_hash="+invalidBlockHash, nil)
+			t.Log(invalidBlockHash)
+			require.Equal(t, http.StatusBadRequest, rr.Code)
+			require.Contains(t, rr.Body.String(), "invalid block_hash argument")
+		}
+	})
+
+	t.Run("Accept valid block_number", func(t *testing.T) {
+		backend := newTestBackend(t, 1)
+
+		validBlockNumber := uint64(2)
+		validBlockNumberPath := fmt.Sprintf("%s?block_number=%d", path, validBlockNumber)
+		rr := backend.request(http.MethodGet, validBlockNumberPath, nil)
+		require.Equal(t, http.StatusOK, rr.Code)
+	})
+
+	t.Run("Reject invalid block_number", func(t *testing.T) {
+		backend := newTestBackend(t, 1)
+
+		invalidBlockNumbers := []string{
+			"-1",
+			"1.1",
+		}
+
+		for _, invalidBlockNumber := range invalidBlockNumbers {
+			invalidBlockNumberPath := fmt.Sprintf("%s?block_number=%s", path, invalidBlockNumber)
+			rr := backend.request(http.MethodGet, invalidBlockNumberPath, nil)
+			require.Equal(t, http.StatusBadRequest, rr.Code)
+			require.Contains(t, rr.Body.String(), "invalid block_number argument")
+		}
+	})
+
+	t.Run("Accept valid builder_pubkey", func(t *testing.T) {
+		backend := newTestBackend(t, 1)
+
+		validBuilderPubkey := "0x6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b90890792"
+		rr := backend.request(http.MethodGet, path+"?builder_pubkey="+validBuilderPubkey, nil)
+		require.Equal(t, http.StatusOK, rr.Code)
+	})
+
+	t.Run("Reject invalid builder_pubkey", func(t *testing.T) {
+		backend := newTestBackend(t, 1)
+
+		invalidBuilderPubkeys := []string{
+			// One character too long.
+			"0x6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b908907921",
+			// One character too short.
+			"0x6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b9089079",
+			// Missing the 0x prefix.
+			"6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b90890792",
+			// Has an invalid hex character ('z' at the end).
+			"0x6ae5932d1e248d987d51b58665b81848814202d7b23b343d20f2a167d12f07dcb01ca41c42fdd60b7fca9c4b9089079z",
+		}
+
+		for _, invalidBuilderPubkey := range invalidBuilderPubkeys {
+			rr := backend.request(http.MethodGet, path+"?builder_pubkey="+invalidBuilderPubkey, nil)
+			t.Log(invalidBuilderPubkey)
+			require.Equal(t, http.StatusBadRequest, rr.Code)
+			require.Contains(t, rr.Body.String(), "invalid builder_pubkey argument")
+		}
+	})
+
+	t.Run("Reject no slot or block_hash or block_number or builder_pubkey", func(t *testing.T) {
+		backend := newTestBackend(t, 1)
+		rr := backend.request(http.MethodGet, path, nil)
+		require.Equal(t, http.StatusBadRequest, rr.Code)
+		require.Contains(t, rr.Body.String(), "need to query for specific slot or block_hash or block_number or builder_pubkey")
+	})
+
+	t.Run("Accept valid limit", func(t *testing.T) {
+		backend := newTestBackend(t, 1)
+		blockNumber := uint64(1)
+		limit := uint64(1)
+		limitPath := fmt.Sprintf("%s?block_number=%d&limit=%d", path, blockNumber, limit)
+		rr := backend.request(http.MethodGet, limitPath, nil)
+		require.Equal(t, http.StatusOK, rr.Code)
+	})
+
+	t.Run("Reject above max limit", func(t *testing.T) {
+		backend := newTestBackend(t, 1)
+		blockNumber := uint64(1)
+		maximumLimit := uint64(500)
+		oneAboveMaxLimit := maximumLimit + 1
+		limitPath := fmt.Sprintf("%s?block_number=%d&limit=%d", path, blockNumber, oneAboveMaxLimit)
+		rr := backend.request(http.MethodGet, limitPath, nil)
+		require.Equal(t, http.StatusBadRequest, rr.Code)
+		require.Contains(t, rr.Body.String(), fmt.Sprintf("maximum limit is %d", maximumLimit))
+	})
+}