From ec90b94c4d6880aa3305a77aae52e5bbb74623a0 Mon Sep 17 00:00:00 2001 From: miiu Date: Mon, 7 Oct 2024 16:09:48 +0300 Subject: [PATCH 01/10] inner transactions processor --- data/transaction.go | 119 ++++++++++-------- integrationtests/relayedTxV3_test.go | 13 ++ .../testdata/relayedTxV3/innerTx1.json | 12 ++ .../testdata/relayedTxV3/innerTx2.json | 12 ++ .../relayed-tx-v3-with-events.json | 3 + .../testdata/relayedTxV3/relayed-tx-v3.json | 2 + .../relayed-v3-execution-scr-on-dest.json | 2 + .../relayed-v3-execution-source.json | 2 + process/dataindexer/errors.go | 3 + process/elasticproc/check.go | 3 + process/elasticproc/elasticProcessor.go | 11 +- process/elasticproc/elasticProcessor_test.go | 3 + .../factory/elasticProcessorFactory.go | 2 + .../elasticproc/innerTxs/innerTxsProcessor.go | 32 +++++ .../innerTxs/innerTxsProcessor_test.go | 51 ++++++++ process/elasticproc/innerTxs/serialize.go | 42 +++++++ .../elasticproc/innerTxs/serialize_test.go | 1 + process/elasticproc/interface.go | 6 + .../transactions/transactionDBBuilder.go | 25 +++- .../transactions/transactionDBBuilder_test.go | 2 +- .../transactions/transactionsGrouper_test.go | 8 +- .../transactions/transactionsProcessor.go | 2 +- 22 files changed, 295 insertions(+), 61 deletions(-) create mode 100644 integrationtests/testdata/relayedTxV3/innerTx1.json create mode 100644 integrationtests/testdata/relayedTxV3/innerTx2.json create mode 100644 process/elasticproc/innerTxs/innerTxsProcessor.go create mode 100644 process/elasticproc/innerTxs/innerTxsProcessor_test.go create mode 100644 process/elasticproc/innerTxs/serialize.go create mode 100644 process/elasticproc/innerTxs/serialize_test.go diff --git a/data/transaction.go b/data/transaction.go index 2a85ce52..726c7be7 100644 --- a/data/transaction.go +++ b/data/transaction.go @@ -2,61 +2,59 @@ package data import ( "time" - - "github.com/multiversx/mx-chain-core-go/data/transaction" ) // Transaction is a structure containing all the fields that need // to be saved for a transaction. It has all the default fields // plus some extra information for ease of search and filter type Transaction struct { - MBHash string `json:"miniBlockHash"` - Nonce uint64 `json:"nonce"` - Round uint64 `json:"round"` - Value string `json:"value"` - ValueNum float64 `json:"valueNum"` - Receiver string `json:"receiver"` - Sender string `json:"sender"` - ReceiverShard uint32 `json:"receiverShard"` - SenderShard uint32 `json:"senderShard"` - GasPrice uint64 `json:"gasPrice"` - GasLimit uint64 `json:"gasLimit"` - GasUsed uint64 `json:"gasUsed"` - Fee string `json:"fee"` - FeeNum float64 `json:"feeNum"` - InitialPaidFee string `json:"initialPaidFee,omitempty"` - Data []byte `json:"data"` - Signature string `json:"signature"` - Timestamp time.Duration `json:"timestamp"` - Status string `json:"status"` - SearchOrder uint32 `json:"searchOrder"` - SenderUserName []byte `json:"senderUserName,omitempty"` - ReceiverUserName []byte `json:"receiverUserName,omitempty"` - HasSCR bool `json:"hasScResults,omitempty"` - IsScCall bool `json:"isScCall,omitempty"` - HasOperations bool `json:"hasOperations,omitempty"` - HasLogs bool `json:"hasLogs,omitempty"` - Tokens []string `json:"tokens,omitempty"` - ESDTValues []string `json:"esdtValues,omitempty"` - ESDTValuesNum []float64 `json:"esdtValuesNum,omitempty"` - Receivers []string `json:"receivers,omitempty"` - ReceiversShardIDs []uint32 `json:"receiversShardIDs,omitempty"` - Type string `json:"type,omitempty"` - Operation string `json:"operation,omitempty"` - Function string `json:"function,omitempty"` - IsRelayed bool `json:"isRelayed,omitempty"` - Version uint32 `json:"version,omitempty"` - GuardianAddress string `json:"guardian,omitempty"` - GuardianSignature string `json:"guardianSignature,omitempty"` - ErrorEvent bool `json:"errorEvent,omitempty"` - CompletedEvent bool `json:"completedEvent,omitempty"` - RelayedAddr string `json:"relayed,omitempty"` - InnerTransactions []*transaction.FrontendTransaction `json:"innerTransactions,omitempty"` - ExecutionOrder int `json:"-"` - SmartContractResults []*ScResult `json:"-"` - Hash string `json:"-"` - BlockHash string `json:"-"` - HadRefund bool `json:"-"` + MBHash string `json:"miniBlockHash"` + Nonce uint64 `json:"nonce"` + Round uint64 `json:"round"` + Value string `json:"value"` + ValueNum float64 `json:"valueNum"` + Receiver string `json:"receiver"` + Sender string `json:"sender"` + ReceiverShard uint32 `json:"receiverShard"` + SenderShard uint32 `json:"senderShard"` + GasPrice uint64 `json:"gasPrice"` + GasLimit uint64 `json:"gasLimit"` + GasUsed uint64 `json:"gasUsed"` + Fee string `json:"fee"` + FeeNum float64 `json:"feeNum"` + InitialPaidFee string `json:"initialPaidFee,omitempty"` + Data []byte `json:"data"` + Signature string `json:"signature"` + Timestamp time.Duration `json:"timestamp"` + Status string `json:"status"` + SearchOrder uint32 `json:"searchOrder"` + SenderUserName []byte `json:"senderUserName,omitempty"` + ReceiverUserName []byte `json:"receiverUserName,omitempty"` + HasSCR bool `json:"hasScResults,omitempty"` + IsScCall bool `json:"isScCall,omitempty"` + HasOperations bool `json:"hasOperations,omitempty"` + HasLogs bool `json:"hasLogs,omitempty"` + Tokens []string `json:"tokens,omitempty"` + ESDTValues []string `json:"esdtValues,omitempty"` + ESDTValuesNum []float64 `json:"esdtValuesNum,omitempty"` + Receivers []string `json:"receivers,omitempty"` + ReceiversShardIDs []uint32 `json:"receiversShardIDs,omitempty"` + Type string `json:"type,omitempty"` + Operation string `json:"operation,omitempty"` + Function string `json:"function,omitempty"` + IsRelayed bool `json:"isRelayed,omitempty"` + Version uint32 `json:"version,omitempty"` + GuardianAddress string `json:"guardian,omitempty"` + GuardianSignature string `json:"guardianSignature,omitempty"` + ErrorEvent bool `json:"errorEvent,omitempty"` + CompletedEvent bool `json:"completedEvent,omitempty"` + RelayedAddr string `json:"relayed,omitempty"` + InnerTransactions []*InnerTransaction `json:"innerTransactions,omitempty"` + ExecutionOrder int `json:"-"` + SmartContractResults []*ScResult `json:"-"` + Hash string `json:"-"` + BlockHash string `json:"-"` + HadRefund bool `json:"-"` } // Receipt is a structure containing all the fields that need to be safe for a Receipt @@ -96,3 +94,26 @@ type FeeData struct { GasUsed uint64 Receiver string } + +// InnerTransaction is the structure that contains data about an inner transaction +type InnerTransaction struct { + Hash string `json:"hash,omitempty"` + Type string `json:"type,omitempty"` + Nonce uint64 `json:"nonce"` + Value string `json:"value"` + Receiver string `json:"receiver"` + Sender string `json:"sender"` + SenderUsername []byte `json:"senderUsername,omitempty"` + ReceiverUsername []byte `json:"receiverUsername,omitempty"` + GasPrice uint64 `json:"gasPrice"` + GasLimit uint64 `json:"gasLimit"` + Data []byte `json:"data,omitempty"` + Signature string `json:"signature,omitempty"` + ChainID string `json:"chainID"` + Version uint32 `json:"version"` + Options uint32 `json:"options,omitempty"` + GuardianAddr string `json:"guardian,omitempty"` + GuardianSignature string `json:"guardianSignature,omitempty"` + Relayer string `json:"relayer,omitempty"` + RelayedTxHash string `json:"relayedTxHash,omitempty"` +} diff --git a/integrationtests/relayedTxV3_test.go b/integrationtests/relayedTxV3_test.go index e440c215..6cf95a60 100644 --- a/integrationtests/relayedTxV3_test.go +++ b/integrationtests/relayedTxV3_test.go @@ -99,6 +99,19 @@ func TestRelayedTxV3(t *testing.T) { readExpectedResult("./testdata/relayedTxV3/relayed-tx-v3.json"), string(genericResponse.Docs[0].Source), ) + + ids = []string{"13b41efddcb2c01dbaba26ebc387a7f58a4ea0757a73420267818224f939e77a", "f8b8b93e42afb737a59dd622af054e34d970663b09b90138e7dc2712565ca8db"} + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.OperationsIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/relayedTxV3/innerTx1.json"), + string(genericResponse.Docs[0].Source), + ) + require.JSONEq(t, + readExpectedResult("./testdata/relayedTxV3/innerTx2.json"), + string(genericResponse.Docs[1].Source), + ) } func TestRelayedTxV3WithSignalErrorAndCompletedEvent(t *testing.T) { diff --git a/integrationtests/testdata/relayedTxV3/innerTx1.json b/integrationtests/testdata/relayedTxV3/innerTx1.json new file mode 100644 index 00000000..b3e2611f --- /dev/null +++ b/integrationtests/testdata/relayedTxV3/innerTx1.json @@ -0,0 +1,12 @@ +{ + "type": "innerTx", + "nonce": 20, + "value": "1000", + "receiver": "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht", + "sender": "erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u", + "gasPrice": 1000000000, + "gasLimit": 15406000, + "chainID": "", + "version": 0, + "relayedTxHash": "72656c6179656454785633" +} \ No newline at end of file diff --git a/integrationtests/testdata/relayedTxV3/innerTx2.json b/integrationtests/testdata/relayedTxV3/innerTx2.json new file mode 100644 index 00000000..d60838b0 --- /dev/null +++ b/integrationtests/testdata/relayedTxV3/innerTx2.json @@ -0,0 +1,12 @@ +{ + "type": "innerTx", + "nonce": 10, + "value": "0", + "receiver": "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht", + "sender": "erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u", + "gasPrice": 1000000000, + "gasLimit": 15406000, + "chainID": "", + "version": 0, + "relayedTxHash": "72656c6179656454785633" +} \ No newline at end of file diff --git a/integrationtests/testdata/relayedTxV3/relayed-tx-v3-with-events.json b/integrationtests/testdata/relayedTxV3/relayed-tx-v3-with-events.json index f60cafb2..f94f42b5 100644 --- a/integrationtests/testdata/relayedTxV3/relayed-tx-v3-with-events.json +++ b/integrationtests/testdata/relayedTxV3/relayed-tx-v3-with-events.json @@ -37,6 +37,7 @@ "completedEvent": true, "innerTransactions": [ { + "hash": "0521127b7d727e5d5e82ce963870ac16dde71f438b5dc2a64bd476ad3b8e96c2", "nonce": 5, "value": "10000000000000000", "receiver": "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", @@ -47,6 +48,7 @@ "version": 0 }, { + "hash": "c4d7ee5288fb2f02789e36970a2b22d977870e506eb6df07e10aad12aaf67dbd", "nonce": 3, "value": "10000000000000000", "receiver": "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", @@ -57,6 +59,7 @@ "version": 0 }, { + "hash": "5ab08906cd2d05abaca912fa450df5fb6f9cf64481f649d118f8fd14aa81b1be", "nonce": 4, "value": "10000000000000000", "receiver": "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", diff --git a/integrationtests/testdata/relayedTxV3/relayed-tx-v3.json b/integrationtests/testdata/relayedTxV3/relayed-tx-v3.json index 210df682..fae9e097 100644 --- a/integrationtests/testdata/relayedTxV3/relayed-tx-v3.json +++ b/integrationtests/testdata/relayedTxV3/relayed-tx-v3.json @@ -31,6 +31,7 @@ "isRelayed": true, "innerTransactions": [ { + "hash": "f8b8b93e42afb737a59dd622af054e34d970663b09b90138e7dc2712565ca8db", "nonce": 10, "value": "0", "receiver": "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht", @@ -41,6 +42,7 @@ "version": 0 }, { + "hash": "13b41efddcb2c01dbaba26ebc387a7f58a4ea0757a73420267818224f939e77a", "nonce": 20, "value": "1000", "receiver": "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht", diff --git a/integrationtests/testdata/relayedTxV3/relayed-v3-execution-scr-on-dest.json b/integrationtests/testdata/relayedTxV3/relayed-v3-execution-scr-on-dest.json index e525f876..de1ff465 100644 --- a/integrationtests/testdata/relayedTxV3/relayed-v3-execution-scr-on-dest.json +++ b/integrationtests/testdata/relayedTxV3/relayed-v3-execution-scr-on-dest.json @@ -35,6 +35,7 @@ "errorEvent": true, "innerTransactions": [ { + "hash": "0521127b7d727e5d5e82ce963870ac16dde71f438b5dc2a64bd476ad3b8e96c2", "nonce": 5, "value": "10000000000000000", "receiver": "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", @@ -45,6 +46,7 @@ "version": 0 }, { + "hash": "c4d7ee5288fb2f02789e36970a2b22d977870e506eb6df07e10aad12aaf67dbd", "nonce": 3, "value": "10000000000000000", "receiver": "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", diff --git a/integrationtests/testdata/relayedTxV3/relayed-v3-execution-source.json b/integrationtests/testdata/relayedTxV3/relayed-v3-execution-source.json index e9d692b7..32ae918f 100644 --- a/integrationtests/testdata/relayedTxV3/relayed-v3-execution-source.json +++ b/integrationtests/testdata/relayedTxV3/relayed-v3-execution-source.json @@ -34,6 +34,7 @@ "completedEvent": true, "innerTransactions": [ { + "hash": "0521127b7d727e5d5e82ce963870ac16dde71f438b5dc2a64bd476ad3b8e96c2", "nonce": 5, "value": "10000000000000000", "receiver": "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", @@ -44,6 +45,7 @@ "version": 0 }, { + "hash": "c4d7ee5288fb2f02789e36970a2b22d977870e506eb6df07e10aad12aaf67dbd", "nonce": 3, "value": "10000000000000000", "receiver": "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", diff --git a/process/dataindexer/errors.go b/process/dataindexer/errors.go index 9b4fbd0f..c9d500bf 100644 --- a/process/dataindexer/errors.go +++ b/process/dataindexer/errors.go @@ -86,5 +86,8 @@ var ErrNilBalanceConverter = errors.New("nil balance converter") // ErrNilOperationsHandler signals that a nil operations handler has been provided var ErrNilOperationsHandler = errors.New("nil operations handler") +// ErrNilInnerTxsHandler signals that a nil inner transaction handler has been provided +var ErrNilInnerTxsHandler = errors.New("nil inner transactions handler") + // ErrNilBlockContainerHandler signals that a nil block container handler has been provided var ErrNilBlockContainerHandler = errors.New("nil bock container handler") diff --git a/process/elasticproc/check.go b/process/elasticproc/check.go index 9284376e..0596274f 100644 --- a/process/elasticproc/check.go +++ b/process/elasticproc/check.go @@ -39,6 +39,9 @@ func checkArguments(arguments *ArgElasticProcessor) error { if check.IfNilReflect(arguments.OperationsProc) { return elasticIndexer.ErrNilOperationsHandler } + if check.IfNilReflect(arguments.InnerTxsHandler) { + return elasticIndexer.ErrNilInnerTxsHandler + } return nil } diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 89634a95..2b616325 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -56,6 +56,7 @@ type ArgElasticProcessor struct { DBClient DatabaseClientHandler LogsAndEventsProc DBLogsAndEventsHandler OperationsProc OperationsHandler + InnerTxsHandler InnerTxsHandler Version string } @@ -73,6 +74,7 @@ type elasticProcessor struct { validatorsProc DBValidatorsHandler logsAndEventsProc DBLogsAndEventsHandler operationsProc OperationsHandler + innerTxsHandler InnerTxsHandler } // NewElasticProcessor handles Elasticsearch operations such as initialization, adding, modifying or removing data @@ -94,6 +96,7 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err logsAndEventsProc: arguments.LogsAndEventsProc, operationsProc: arguments.OperationsProc, bulkRequestMaxSize: arguments.BulkRequestMaxSize, + innerTxsHandler: arguments.InnerTxsHandler, } err = ei.init(arguments.UseKibana, arguments.IndexTemplates, arguments.IndexPolicies, arguments.ExtraMappings) @@ -584,9 +587,15 @@ func (ei *elasticProcessor) prepareAndIndexOperations( return nil } + innerTxs := ei.innerTxsHandler.ExtractInnerTxs(txs) + err := ei.innerTxsHandler.SerializeInnerTxs(innerTxs, buffSlice, elasticIndexer.OperationsIndex) + if err != nil { + return err + } + processedTxs, processedSCRs := ei.operationsProc.ProcessTransactionsAndSCRs(txs, scrs, isImportDB, header.GetShardID()) - err := ei.transactionsProc.SerializeTransactions(processedTxs, txHashStatusInfo, header.GetShardID(), buffSlice, elasticIndexer.OperationsIndex) + err = ei.transactionsProc.SerializeTransactions(processedTxs, txHashStatusInfo, header.GetShardID(), buffSlice, elasticIndexer.OperationsIndex) if err != nil { return err } diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 9f3311af..b7c5b739 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "errors" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/innerTxs" "strings" "testing" @@ -73,6 +74,7 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { } lp, _ := logsevents.NewLogsAndEventsProcessor(args) op, _ := operations.NewOperationsProcessor() + ip := innerTxs.NewInnerTxsProcessor() return &ArgElasticProcessor{ DBClient: &mock.DatabaseWriterStub{}, @@ -87,6 +89,7 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { BlockProc: bp, LogsAndEventsProc: lp, OperationsProc: op, + InnerTxsHandler: ip, } } diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index ad865953..c8ce0eae 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -9,6 +9,7 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/accounts" blockProc "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/block" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/innerTxs" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/logsevents" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/miniblocks" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/operations" @@ -127,6 +128,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E OperationsProc: operationsProc, ImportDB: arguments.ImportDB, Version: arguments.Version, + InnerTxsHandler: innerTxs.NewInnerTxsProcessor(), } return elasticproc.NewElasticProcessor(args) diff --git a/process/elasticproc/innerTxs/innerTxsProcessor.go b/process/elasticproc/innerTxs/innerTxsProcessor.go new file mode 100644 index 00000000..581c748d --- /dev/null +++ b/process/elasticproc/innerTxs/innerTxsProcessor.go @@ -0,0 +1,32 @@ +package innerTxs + +import "github.com/multiversx/mx-chain-es-indexer-go/data" + +// InnerTxType is the type for an inner transaction +const InnerTxType = "innerTx" + +type innerTxsProcessor struct { +} + +// NewInnerTxsProcessor will create a new instance of inner transactions processor +func NewInnerTxsProcessor() *innerTxsProcessor { + return &innerTxsProcessor{} +} + +// ExtractInnerTxs will extract the inner transactions from the transaction array +func (ip *innerTxsProcessor) ExtractInnerTxs( + txs []*data.Transaction, +) []*data.InnerTransaction { + innerTxs := make([]*data.InnerTransaction, 0) + for _, tx := range txs { + for _, innerTx := range tx.InnerTransactions { + innerTxCopy := *innerTx + + innerTxCopy.Type = InnerTxType + innerTxCopy.RelayedTxHash = tx.Hash + innerTxs = append(innerTxs, &innerTxCopy) + } + } + + return innerTxs +} diff --git a/process/elasticproc/innerTxs/innerTxsProcessor_test.go b/process/elasticproc/innerTxs/innerTxsProcessor_test.go new file mode 100644 index 00000000..02a6b1d0 --- /dev/null +++ b/process/elasticproc/innerTxs/innerTxsProcessor_test.go @@ -0,0 +1,51 @@ +package innerTxs + +import ( + "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/stretchr/testify/require" + "testing" +) + +func TestInnerTxsProcessor_ExtractInnerTxsNoTransactions(t *testing.T) { + t.Parallel() + + innerTxsProc := NewInnerTxsProcessor() + + res := innerTxsProc.ExtractInnerTxs(nil) + require.Equal(t, 0, len(res)) + + res = innerTxsProc.ExtractInnerTxs([]*data.Transaction{{}, {}}) + require.Equal(t, 0, len(res)) +} + +func TestInnerTxsProcessor_ExtractInnerTxs(t *testing.T) { + t.Parallel() + + innerTxsProc := NewInnerTxsProcessor() + + res := innerTxsProc.ExtractInnerTxs([]*data.Transaction{{ + Hash: "txHash", + InnerTransactions: []*data.InnerTransaction{ + { + Hash: "inner1", + }, + { + Hash: "inner2", + }, + }, + }}) + + require.Equal(t, 2, len(res)) + require.Equal(t, []*data.InnerTransaction{ + { + Type: InnerTxType, + Hash: "inner1", + RelayedTxHash: "txHash", + }, + { + Type: InnerTxType, + Hash: "inner2", + RelayedTxHash: "txHash", + }, + }, res) +} diff --git a/process/elasticproc/innerTxs/serialize.go b/process/elasticproc/innerTxs/serialize.go new file mode 100644 index 00000000..3d13f094 --- /dev/null +++ b/process/elasticproc/innerTxs/serialize.go @@ -0,0 +1,42 @@ +package innerTxs + +import ( + "encoding/json" + "fmt" + "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" +) + +// SerializeInnerTxs will serialize the provided array of inner transaction and add them in the BufferSlice +func (ip *innerTxsProcessor) SerializeInnerTxs(innerTxs []*data.InnerTransaction, buffSlice *data.BufferSlice, index string) error { + for _, innerTx := range innerTxs { + meta, serializedData, err := prepareSerializedDataForAInnerTx(innerTx, index) + if err != nil { + return err + } + + err = buffSlice.PutData(meta, serializedData) + if err != nil { + return err + } + } + + return nil +} + +func prepareSerializedDataForAInnerTx( + innerTx *data.InnerTransaction, + index string, +) ([]byte, []byte, error) { + innerTxHash := innerTx.Hash + innerTx.Hash = "" + + marshaledSCR, err := json.Marshal(innerTx) + if err != nil { + return nil, nil, err + } + + meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s","_id" : "%s" } }%s`, index, converters.JsonEscape(innerTxHash), "\n")) + + return meta, marshaledSCR, nil +} diff --git a/process/elasticproc/innerTxs/serialize_test.go b/process/elasticproc/innerTxs/serialize_test.go new file mode 100644 index 00000000..e6757f9c --- /dev/null +++ b/process/elasticproc/innerTxs/serialize_test.go @@ -0,0 +1 @@ +package innerTxs diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index fe4b565f..2d44b376 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -120,3 +120,9 @@ type OperationsHandler interface { ProcessTransactionsAndSCRs(txs []*data.Transaction, scrs []*data.ScResult, isImportDB bool, shardID uint32) ([]*data.Transaction, []*data.ScResult) SerializeSCRs(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string, shardID uint32) error } + +// InnerTxsHandler defines the actions that an inner transactions handler should do +type InnerTxsHandler interface { + ExtractInnerTxs(txs []*data.Transaction) []*data.InnerTransaction + SerializeInnerTxs(innerTxs []*data.InnerTransaction, buffSlice *data.BufferSlice, index string) error +} diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index f22e1a31..a6bf3010 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -3,6 +3,8 @@ package transactions import ( "encoding/hex" "fmt" + "github.com/multiversx/mx-chain-core-go/hashing" + "github.com/multiversx/mx-chain-core-go/marshal" "time" "github.com/multiversx/mx-chain-core-go/core" @@ -21,17 +23,23 @@ type dbTransactionBuilder struct { addressPubkeyConverter core.PubkeyConverter dataFieldParser DataFieldParser balanceConverter dataindexer.BalanceConverter + hasher hashing.Hasher + marshaller marshal.Marshalizer } func newTransactionDBBuilder( addressPubkeyConverter core.PubkeyConverter, dataFieldParser DataFieldParser, balanceConverter dataindexer.BalanceConverter, + hasher hashing.Hasher, + marshaller marshal.Marshalizer, ) *dbTransactionBuilder { return &dbTransactionBuilder{ addressPubkeyConverter: addressPubkeyConverter, dataFieldParser: dataFieldParser, balanceConverter: balanceConverter, + hasher: hasher, + marshaller: marshaller, } } @@ -124,7 +132,7 @@ func (dtb *dbTransactionBuilder) prepareTransaction( isRelayedV3 := len(tx.InnerTransactions) > 0 if isRelayedV3 { - dtb.addRelayedV3InfoInIndexerTx(tx, eTx, numOfShards) + dtb.addRelayedV3InfoInIndexerTx(txHash, tx, eTx, numOfShards) return eTx } @@ -137,16 +145,23 @@ func (dtb *dbTransactionBuilder) prepareTransaction( return eTx } -func (dtb *dbTransactionBuilder) addRelayedV3InfoInIndexerTx(tx *transaction.Transaction, indexerTx *data.Transaction, numOfShards uint32) { +func (dtb *dbTransactionBuilder) addRelayedV3InfoInIndexerTx(txHash []byte, tx *transaction.Transaction, indexerTx *data.Transaction, numOfShards uint32) { if len(tx.InnerTransactions) == 0 { return } - innerTxs := make([]*transaction.FrontendTransaction, 0, len(tx.InnerTransactions)) + innerTxs := make([]*data.InnerTransaction, 0, len(tx.InnerTransactions)) receivers := make([]string, 0, len(tx.InnerTransactions)) receiversShardIDs := make([]uint32, 0, len(tx.InnerTransactions)) - for _, innerTx := range tx.InnerTransactions { - frontEndTx := &transaction.FrontendTransaction{ + for idx, innerTx := range tx.InnerTransactions { + innerTxHash, err := core.CalculateHash(dtb.marshaller, dtb.hasher, innerTx) + if err != nil { + log.Warn("dbTransactionBuilder.addRelayedV3InfoInIndexerTx: cannot compute inner tx hash", + "index inner tx", idx, "hash", txHash, "error", err) + } + + frontEndTx := &data.InnerTransaction{ + Hash: hex.EncodeToString(innerTxHash), Nonce: innerTx.Nonce, Value: innerTx.Value.String(), Receiver: dtb.addressPubkeyConverter.SilentEncode(innerTx.RcvAddr, log), diff --git a/process/elasticproc/transactions/transactionDBBuilder_test.go b/process/elasticproc/transactions/transactionDBBuilder_test.go index b892166e..0ea16b58 100644 --- a/process/elasticproc/transactions/transactionDBBuilder_test.go +++ b/process/elasticproc/transactions/transactionDBBuilder_test.go @@ -299,7 +299,7 @@ func TestTxsDatabaseProcessor_PrepareTransactionRelayedV3(t *testing.T) { Version: 1, Receivers: []string{"72637631", "72637632"}, ReceiversShardIDs: []uint32{1, 2}, - InnerTransactions: []*transaction.FrontendTransaction{ + InnerTransactions: []*data.InnerTransaction{ { Nonce: 2, Value: "1000", diff --git a/process/elasticproc/transactions/transactionsGrouper_test.go b/process/elasticproc/transactions/transactionsGrouper_test.go index 12c6b75e..f4718aae 100644 --- a/process/elasticproc/transactions/transactionsGrouper_test.go +++ b/process/elasticproc/transactions/transactionsGrouper_test.go @@ -19,7 +19,7 @@ func TestGroupNormalTxs(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap, &mock.HasherMock{}, &mock.MarshalizerMock{}) grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") @@ -55,7 +55,7 @@ func TestGroupRewardsTxs(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap, &mock.HasherMock{}, &mock.MarshalizerMock{}) grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") @@ -83,7 +83,7 @@ func TestGroupInvalidTxs(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) - txBuilder := newTransactionDBBuilder(mock.NewPubkeyConverterMock(32), parser, ap) + txBuilder := newTransactionDBBuilder(mock.NewPubkeyConverterMock(32), parser, ap, &mock.HasherMock{}, &mock.MarshalizerMock{}) grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") @@ -115,7 +115,7 @@ func TestGroupReceipts(t *testing.T) { parser := createDataFieldParserMock() ap, _ := converters.NewBalanceConverter(18) - txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap) + txBuilder := newTransactionDBBuilder(&mock.PubkeyConverterMock{}, parser, ap, &mock.HasherMock{}, &mock.MarshalizerMock{}) grouper := newTxsGrouper(txBuilder, &mock.HasherMock{}, &mock.MarshalizerMock{}) txHash1 := []byte("txHash1") diff --git a/process/elasticproc/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go index 86e59b38..a137c681 100644 --- a/process/elasticproc/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -51,7 +51,7 @@ func NewTransactionsProcessor(args *ArgsTransactionProcessor) (*txsDatabaseProce return nil, err } - txBuilder := newTransactionDBBuilder(args.AddressPubkeyConverter, operationsDataParser, args.BalanceConverter) + txBuilder := newTransactionDBBuilder(args.AddressPubkeyConverter, operationsDataParser, args.BalanceConverter, args.Hasher, args.Marshalizer) txsDBGrouper := newTxsGrouper(txBuilder, args.Hasher, args.Marshalizer) scrProc := newSmartContractResultsProcessor(args.AddressPubkeyConverter, args.Marshalizer, args.Hasher, operationsDataParser, args.BalanceConverter) scrsDataToTxs := newScrsDataToTransactions(args.BalanceConverter) From fdeb371fe41c1e41a90c5f4e4795c43ba1152553 Mon Sep 17 00:00:00 2001 From: miiu Date: Mon, 7 Oct 2024 16:13:27 +0300 Subject: [PATCH 02/10] fix imports --- process/elasticproc/innerTxs/innerTxsProcessor_test.go | 3 ++- process/elasticproc/innerTxs/serialize.go | 1 + process/elasticproc/transactions/transactionDBBuilder.go | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/process/elasticproc/innerTxs/innerTxsProcessor_test.go b/process/elasticproc/innerTxs/innerTxsProcessor_test.go index 02a6b1d0..3931e766 100644 --- a/process/elasticproc/innerTxs/innerTxsProcessor_test.go +++ b/process/elasticproc/innerTxs/innerTxsProcessor_test.go @@ -1,9 +1,10 @@ package innerTxs import ( + "testing" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/stretchr/testify/require" - "testing" ) func TestInnerTxsProcessor_ExtractInnerTxsNoTransactions(t *testing.T) { diff --git a/process/elasticproc/innerTxs/serialize.go b/process/elasticproc/innerTxs/serialize.go index 3d13f094..7a7e1f86 100644 --- a/process/elasticproc/innerTxs/serialize.go +++ b/process/elasticproc/innerTxs/serialize.go @@ -3,6 +3,7 @@ package innerTxs import ( "encoding/json" "fmt" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" ) diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index a6bf3010..23f41093 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -3,8 +3,6 @@ package transactions import ( "encoding/hex" "fmt" - "github.com/multiversx/mx-chain-core-go/hashing" - "github.com/multiversx/mx-chain-core-go/marshal" "time" "github.com/multiversx/mx-chain-core-go/core" @@ -14,6 +12,8 @@ import ( "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/receipt" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-core-go/hashing" + "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" From 9e25a5f84d3d6eeaf7c45f04cb07db3da2860762 Mon Sep 17 00:00:00 2001 From: miiu Date: Mon, 7 Oct 2024 16:21:17 +0300 Subject: [PATCH 03/10] unit test --- .../testdata/relayedTxV3/innerTx1.json | 2 +- .../testdata/relayedTxV3/innerTx2.json | 2 +- process/elasticproc/elasticProcessor_test.go | 2 +- .../elasticproc/innerTxs/serialize_test.go | 33 +++++++++++++++++++ 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/integrationtests/testdata/relayedTxV3/innerTx1.json b/integrationtests/testdata/relayedTxV3/innerTx1.json index b3e2611f..8a6a88ae 100644 --- a/integrationtests/testdata/relayedTxV3/innerTx1.json +++ b/integrationtests/testdata/relayedTxV3/innerTx1.json @@ -9,4 +9,4 @@ "chainID": "", "version": 0, "relayedTxHash": "72656c6179656454785633" -} \ No newline at end of file +} diff --git a/integrationtests/testdata/relayedTxV3/innerTx2.json b/integrationtests/testdata/relayedTxV3/innerTx2.json index d60838b0..93a6539a 100644 --- a/integrationtests/testdata/relayedTxV3/innerTx2.json +++ b/integrationtests/testdata/relayedTxV3/innerTx2.json @@ -9,4 +9,4 @@ "chainID": "", "version": 0, "relayedTxHash": "72656c6179656454785633" -} \ No newline at end of file +} diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index b7c5b739..3bd41377 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/hex" "errors" - "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/innerTxs" "strings" "testing" @@ -19,6 +18,7 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/accounts" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/block" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" + "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/innerTxs" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/logsevents" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/miniblocks" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/operations" diff --git a/process/elasticproc/innerTxs/serialize_test.go b/process/elasticproc/innerTxs/serialize_test.go index e6757f9c..0d9aa77d 100644 --- a/process/elasticproc/innerTxs/serialize_test.go +++ b/process/elasticproc/innerTxs/serialize_test.go @@ -1 +1,34 @@ package innerTxs + +import ( + "testing" + + "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/stretchr/testify/require" +) + +func TestInnerTxsProcessor_SerializeInnerTxs(t *testing.T) { + t.Parallel() + + innerTxs := []*data.InnerTransaction{ + { + Hash: "h1", + Type: InnerTxType, + }, + { + Hash: "h2", + Type: InnerTxType, + }, + } + + innerTxsProc := NewInnerTxsProcessor() + buffSlice := data.NewBufferSlice(0) + err := innerTxsProc.SerializeInnerTxs(innerTxs, buffSlice, dataindexer.OperationsIndex) + require.Nil(t, err) + require.Equal(t, `{ "index" : { "_index":"operations","_id" : "h1" } } +{"type":"innerTx","nonce":0,"value":"","receiver":"","sender":"","gasPrice":0,"gasLimit":0,"chainID":"","version":0} +{ "index" : { "_index":"operations","_id" : "h2" } } +{"type":"innerTx","nonce":0,"value":"","receiver":"","sender":"","gasPrice":0,"gasLimit":0,"chainID":"","version":0} +`, buffSlice.Buffers()[0].String()) +} From 327bd04507c51115fef1778612eea9ff771cf183 Mon Sep 17 00:00:00 2001 From: miiu Date: Fri, 11 Oct 2024 18:42:50 +0300 Subject: [PATCH 04/10] id inner txs --- data/transaction.go | 1 + integrationtests/relayedTxV3_test.go | 2 +- .../testdata/relayedTxV3/innerTx1.json | 1 + .../testdata/relayedTxV3/innerTx2.json | 1 + process/elasticproc/elasticProcessor_test.go | 2 +- .../factory/elasticProcessorFactory.go | 7 +++++- .../elasticproc/innerTxs/innerTxsProcessor.go | 22 ++++++++++++++++--- .../innerTxs/innerTxsProcessor_test.go | 7 ++++-- process/elasticproc/innerTxs/serialize.go | 5 +---- .../elasticproc/innerTxs/serialize_test.go | 13 ++++++----- 10 files changed, 44 insertions(+), 17 deletions(-) diff --git a/data/transaction.go b/data/transaction.go index 726c7be7..64f6347a 100644 --- a/data/transaction.go +++ b/data/transaction.go @@ -97,6 +97,7 @@ type FeeData struct { // InnerTransaction is the structure that contains data about an inner transaction type InnerTransaction struct { + ID string `json:"-"` Hash string `json:"hash,omitempty"` Type string `json:"type,omitempty"` Nonce uint64 `json:"nonce"` diff --git a/integrationtests/relayedTxV3_test.go b/integrationtests/relayedTxV3_test.go index 6cf95a60..1828a408 100644 --- a/integrationtests/relayedTxV3_test.go +++ b/integrationtests/relayedTxV3_test.go @@ -100,7 +100,7 @@ func TestRelayedTxV3(t *testing.T) { string(genericResponse.Docs[0].Source), ) - ids = []string{"13b41efddcb2c01dbaba26ebc387a7f58a4ea0757a73420267818224f939e77a", "f8b8b93e42afb737a59dd622af054e34d970663b09b90138e7dc2712565ca8db"} + ids = []string{"59d001d2804b6b4f5d56223577b4469852743622521cd5081b9f49e5c791af44", "a37e88461df25c1989b912b6a451c4e7c703d3d9b02eb1b4fd964c6362e7cbd4"} err = esClient.DoMultiGet(context.Background(), ids, indexerdata.OperationsIndex, true, genericResponse) require.Nil(t, err) diff --git a/integrationtests/testdata/relayedTxV3/innerTx1.json b/integrationtests/testdata/relayedTxV3/innerTx1.json index 8a6a88ae..7ef8a5c9 100644 --- a/integrationtests/testdata/relayedTxV3/innerTx1.json +++ b/integrationtests/testdata/relayedTxV3/innerTx1.json @@ -1,4 +1,5 @@ { + "hash":"13b41efddcb2c01dbaba26ebc387a7f58a4ea0757a73420267818224f939e77a", "type": "innerTx", "nonce": 20, "value": "1000", diff --git a/integrationtests/testdata/relayedTxV3/innerTx2.json b/integrationtests/testdata/relayedTxV3/innerTx2.json index 93a6539a..bd446486 100644 --- a/integrationtests/testdata/relayedTxV3/innerTx2.json +++ b/integrationtests/testdata/relayedTxV3/innerTx2.json @@ -1,4 +1,5 @@ { + "hash":"f8b8b93e42afb737a59dd622af054e34d970663b09b90138e7dc2712565ca8db", "type": "innerTx", "nonce": 10, "value": "0", diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index 3bd41377..9ebe0d1c 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -74,7 +74,7 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { } lp, _ := logsevents.NewLogsAndEventsProcessor(args) op, _ := operations.NewOperationsProcessor() - ip := innerTxs.NewInnerTxsProcessor() + ip, _ := innerTxs.NewInnerTxsProcessor(args.Hasher) return &ArgElasticProcessor{ DBClient: &mock.DatabaseWriterStub{}, diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index c8ce0eae..d07ca16b 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -110,6 +110,11 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E return nil, err } + innerTxsProc, err := innerTxs.NewInnerTxsProcessor(arguments.Hasher) + if err != nil { + return nil, err + } + args := &elasticproc.ArgElasticProcessor{ BulkRequestMaxSize: arguments.BulkRequestMaxSize, TransactionsProc: txsProc, @@ -128,7 +133,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E OperationsProc: operationsProc, ImportDB: arguments.ImportDB, Version: arguments.Version, - InnerTxsHandler: innerTxs.NewInnerTxsProcessor(), + InnerTxsHandler: innerTxsProc, } return elasticproc.NewElasticProcessor(args) diff --git a/process/elasticproc/innerTxs/innerTxsProcessor.go b/process/elasticproc/innerTxs/innerTxsProcessor.go index 581c748d..5df631f9 100644 --- a/process/elasticproc/innerTxs/innerTxsProcessor.go +++ b/process/elasticproc/innerTxs/innerTxsProcessor.go @@ -1,16 +1,30 @@ package innerTxs -import "github.com/multiversx/mx-chain-es-indexer-go/data" +import ( + "encoding/hex" + + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/hashing" + "github.com/multiversx/mx-chain-es-indexer-go/data" +) // InnerTxType is the type for an inner transaction const InnerTxType = "innerTx" type innerTxsProcessor struct { + hasher hashing.Hasher } // NewInnerTxsProcessor will create a new instance of inner transactions processor -func NewInnerTxsProcessor() *innerTxsProcessor { - return &innerTxsProcessor{} +func NewInnerTxsProcessor(hasher hashing.Hasher) (*innerTxsProcessor, error) { + if check.IfNil(hasher) { + return nil, core.ErrNilHasher + } + + return &innerTxsProcessor{ + hasher: hasher, + }, nil } // ExtractInnerTxs will extract the inner transactions from the transaction array @@ -22,6 +36,8 @@ func (ip *innerTxsProcessor) ExtractInnerTxs( for _, innerTx := range tx.InnerTransactions { innerTxCopy := *innerTx + id := ip.hasher.Compute(innerTxCopy.Hash + innerTxCopy.RelayedTxHash) + innerTxCopy.ID = hex.EncodeToString(id) innerTxCopy.Type = InnerTxType innerTxCopy.RelayedTxHash = tx.Hash innerTxs = append(innerTxs, &innerTxCopy) diff --git a/process/elasticproc/innerTxs/innerTxsProcessor_test.go b/process/elasticproc/innerTxs/innerTxsProcessor_test.go index 3931e766..22d1db63 100644 --- a/process/elasticproc/innerTxs/innerTxsProcessor_test.go +++ b/process/elasticproc/innerTxs/innerTxsProcessor_test.go @@ -4,13 +4,14 @@ import ( "testing" "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/stretchr/testify/require" ) func TestInnerTxsProcessor_ExtractInnerTxsNoTransactions(t *testing.T) { t.Parallel() - innerTxsProc := NewInnerTxsProcessor() + innerTxsProc, _ := NewInnerTxsProcessor(&mock.HasherMock{}) res := innerTxsProc.ExtractInnerTxs(nil) require.Equal(t, 0, len(res)) @@ -22,7 +23,7 @@ func TestInnerTxsProcessor_ExtractInnerTxsNoTransactions(t *testing.T) { func TestInnerTxsProcessor_ExtractInnerTxs(t *testing.T) { t.Parallel() - innerTxsProc := NewInnerTxsProcessor() + innerTxsProc, _ := NewInnerTxsProcessor(&mock.HasherMock{}) res := innerTxsProc.ExtractInnerTxs([]*data.Transaction{{ Hash: "txHash", @@ -39,11 +40,13 @@ func TestInnerTxsProcessor_ExtractInnerTxs(t *testing.T) { require.Equal(t, 2, len(res)) require.Equal(t, []*data.InnerTransaction{ { + ID: "054efde0c8bb1ee713c9fe5981340d7efbf23b7aa72abeab9a63b64c21000188", Type: InnerTxType, Hash: "inner1", RelayedTxHash: "txHash", }, { + ID: "e50a3fd30c5c7bba673dd18a0b329760f1bff34342978821c5d341067da70fa1", Type: InnerTxType, Hash: "inner2", RelayedTxHash: "txHash", diff --git a/process/elasticproc/innerTxs/serialize.go b/process/elasticproc/innerTxs/serialize.go index 7a7e1f86..3149b4c8 100644 --- a/process/elasticproc/innerTxs/serialize.go +++ b/process/elasticproc/innerTxs/serialize.go @@ -29,15 +29,12 @@ func prepareSerializedDataForAInnerTx( innerTx *data.InnerTransaction, index string, ) ([]byte, []byte, error) { - innerTxHash := innerTx.Hash - innerTx.Hash = "" - marshaledSCR, err := json.Marshal(innerTx) if err != nil { return nil, nil, err } - meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s","_id" : "%s" } }%s`, index, converters.JsonEscape(innerTxHash), "\n")) + meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s","_id" : "%s" } }%s`, index, converters.JsonEscape(innerTx.ID), "\n")) return meta, marshaledSCR, nil } diff --git a/process/elasticproc/innerTxs/serialize_test.go b/process/elasticproc/innerTxs/serialize_test.go index 0d9aa77d..f49cc3c4 100644 --- a/process/elasticproc/innerTxs/serialize_test.go +++ b/process/elasticproc/innerTxs/serialize_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/multiversx/mx-chain-es-indexer-go/data" + "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/stretchr/testify/require" ) @@ -13,22 +14,24 @@ func TestInnerTxsProcessor_SerializeInnerTxs(t *testing.T) { innerTxs := []*data.InnerTransaction{ { + ID: "id1", Hash: "h1", Type: InnerTxType, }, { + ID: "id2", Hash: "h2", Type: InnerTxType, }, } - innerTxsProc := NewInnerTxsProcessor() + innerTxsProc, _ := NewInnerTxsProcessor(&mock.HasherMock{}) buffSlice := data.NewBufferSlice(0) err := innerTxsProc.SerializeInnerTxs(innerTxs, buffSlice, dataindexer.OperationsIndex) require.Nil(t, err) - require.Equal(t, `{ "index" : { "_index":"operations","_id" : "h1" } } -{"type":"innerTx","nonce":0,"value":"","receiver":"","sender":"","gasPrice":0,"gasLimit":0,"chainID":"","version":0} -{ "index" : { "_index":"operations","_id" : "h2" } } -{"type":"innerTx","nonce":0,"value":"","receiver":"","sender":"","gasPrice":0,"gasLimit":0,"chainID":"","version":0} + require.Equal(t, `{ "index" : { "_index":"operations","_id" : "id1" } } +{"hash":"h1","type":"innerTx","nonce":0,"value":"","receiver":"","sender":"","gasPrice":0,"gasLimit":0,"chainID":"","version":0} +{ "index" : { "_index":"operations","_id" : "id2" } } +{"hash":"h2","type":"innerTx","nonce":0,"value":"","receiver":"","sender":"","gasPrice":0,"gasLimit":0,"chainID":"","version":0} `, buffSlice.Buffers()[0].String()) } From e4356eb4808930ce301cb0dff52e5d5d6f233a18 Mon Sep 17 00:00:00 2001 From: miiu Date: Fri, 11 Oct 2024 18:54:12 +0300 Subject: [PATCH 05/10] fixes --- integrationtests/relayedTxV3_test.go | 2 +- process/elasticproc/innerTxs/innerTxsProcessor.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/integrationtests/relayedTxV3_test.go b/integrationtests/relayedTxV3_test.go index 1828a408..28329ebb 100644 --- a/integrationtests/relayedTxV3_test.go +++ b/integrationtests/relayedTxV3_test.go @@ -100,7 +100,7 @@ func TestRelayedTxV3(t *testing.T) { string(genericResponse.Docs[0].Source), ) - ids = []string{"59d001d2804b6b4f5d56223577b4469852743622521cd5081b9f49e5c791af44", "a37e88461df25c1989b912b6a451c4e7c703d3d9b02eb1b4fd964c6362e7cbd4"} + ids = []string{"90cb33aad72cca66ec5f6caace6bdd7a8eb5b36b6d4e02b2712d264a9c53d7b8", "33e70412cb96952914955ad0497e0a4505dfbc795917c4488632f0a53bd05c2a"} err = esClient.DoMultiGet(context.Background(), ids, indexerdata.OperationsIndex, true, genericResponse) require.Nil(t, err) diff --git a/process/elasticproc/innerTxs/innerTxsProcessor.go b/process/elasticproc/innerTxs/innerTxsProcessor.go index 5df631f9..c42b4d74 100644 --- a/process/elasticproc/innerTxs/innerTxsProcessor.go +++ b/process/elasticproc/innerTxs/innerTxsProcessor.go @@ -36,10 +36,11 @@ func (ip *innerTxsProcessor) ExtractInnerTxs( for _, innerTx := range tx.InnerTransactions { innerTxCopy := *innerTx + innerTxCopy.RelayedTxHash = tx.Hash + id := ip.hasher.Compute(innerTxCopy.Hash + innerTxCopy.RelayedTxHash) innerTxCopy.ID = hex.EncodeToString(id) innerTxCopy.Type = InnerTxType - innerTxCopy.RelayedTxHash = tx.Hash innerTxs = append(innerTxs, &innerTxCopy) } } From d1739bada80e9f807b6b0ff2268e48e963a0d647 Mon Sep 17 00:00:00 2001 From: miiu Date: Fri, 11 Oct 2024 20:28:45 +0300 Subject: [PATCH 06/10] fix test --- process/elasticproc/innerTxs/innerTxsProcessor_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/process/elasticproc/innerTxs/innerTxsProcessor_test.go b/process/elasticproc/innerTxs/innerTxsProcessor_test.go index 22d1db63..4e76f568 100644 --- a/process/elasticproc/innerTxs/innerTxsProcessor_test.go +++ b/process/elasticproc/innerTxs/innerTxsProcessor_test.go @@ -40,13 +40,13 @@ func TestInnerTxsProcessor_ExtractInnerTxs(t *testing.T) { require.Equal(t, 2, len(res)) require.Equal(t, []*data.InnerTransaction{ { - ID: "054efde0c8bb1ee713c9fe5981340d7efbf23b7aa72abeab9a63b64c21000188", + ID: "b0a05b65beb800084f80b9b5c7c5894378213cd58e09663aa99322597f86c23e", Type: InnerTxType, Hash: "inner1", RelayedTxHash: "txHash", }, { - ID: "e50a3fd30c5c7bba673dd18a0b329760f1bff34342978821c5d341067da70fa1", + ID: "a9188f77578a4b629c91a237e66f2e833d3f9b850f1b9341bbeab5c7e38a14d9", Type: InnerTxType, Hash: "inner2", RelayedTxHash: "txHash", From 5709b0b0c463e0baf74114ea2738c7e064eda0b1 Mon Sep 17 00:00:00 2001 From: miiu Date: Thu, 17 Oct 2024 13:32:33 +0300 Subject: [PATCH 07/10] treat multiple scrs --- data/scresult.go | 1 + data/transaction.go | 9 +- go.mod | 2 +- go.sum | 4 +- integrationtests/relayedTxV3_test.go | 140 ++++++++++++++++++ .../relayedTxV3/relayed-v3-no-refund.json | 58 ++++++++ .../relayed-v3-with-one-refund.json | 58 ++++++++ .../relayed-v3-with-two-refunds.json | 58 ++++++++ .../transactions/scrsDataToTransactions.go | 9 +- process/elasticproc/transactions/serialize.go | 24 ++- .../smartContractResultsProcessor.go | 1 + 11 files changed, 352 insertions(+), 12 deletions(-) create mode 100644 integrationtests/testdata/relayedTxV3/relayed-v3-no-refund.json create mode 100644 integrationtests/testdata/relayedTxV3/relayed-v3-with-one-refund.json create mode 100644 integrationtests/testdata/relayedTxV3/relayed-v3-with-two-refunds.json diff --git a/data/scresult.go b/data/scresult.go index b246528d..f3fc93ad 100644 --- a/data/scresult.go +++ b/data/scresult.go @@ -43,4 +43,5 @@ type ScResult struct { SenderAddressBytes []byte `json:"-"` InitialTxGasUsed uint64 `json:"-"` InitialTxFee string `json:"-"` + ForRelayed bool `json:"-"` } diff --git a/data/transaction.go b/data/transaction.go index 64f6347a..cde848c5 100644 --- a/data/transaction.go +++ b/data/transaction.go @@ -89,10 +89,11 @@ type ResponseTransactionDB struct { // FeeData is the structure that contains data about transaction fee and gas used type FeeData struct { - FeeNum float64 - Fee string - GasUsed uint64 - Receiver string + FeeNum float64 + Fee string + GasUsed uint64 + Receiver string + ForRelayed bool } // InnerTransaction is the structure that contains data about an inner transaction diff --git a/go.mod b/go.mod index eb32dd80..51148ec8 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/multiversx/mx-chain-communication-go v1.1.0 - github.com/multiversx/mx-chain-core-go v1.2.21 + github.com/multiversx/mx-chain-core-go v1.2.23-0.20241017083841-5e7b1485b6a1 github.com/multiversx/mx-chain-logger-go v1.0.15 github.com/multiversx/mx-chain-vm-common-go v1.5.13 github.com/prometheus/client_model v0.4.0 diff --git a/go.sum b/go.sum index 82f3a0f7..2347e509 100644 --- a/go.sum +++ b/go.sum @@ -249,8 +249,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.1.0 h1:J7bX6HoN3HiHY7cUeEjG8AJWgQDDPcY+OPDOsSUOkRE= github.com/multiversx/mx-chain-communication-go v1.1.0/go.mod h1:WK6bP4pGEHGDDna/AYRIMtl6G9OA0NByI1Lw8PmOnRM= -github.com/multiversx/mx-chain-core-go v1.2.21 h1:+XVKznPTlUU5EFS1A8chtS8fStW60upRIyF4Pgml19I= -github.com/multiversx/mx-chain-core-go v1.2.21/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20241017083841-5e7b1485b6a1 h1:TyWMSr3wx1DR3rlKnp7Iw/nEMYDp7any7DtMpz2aAlU= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20241017083841-5e7b1485b6a1/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.12 h1:zWip7rpUS4CGthJxfKn5MZfMfYPjVjIiCID6uX5BSOk= github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+wRqOwi3n+m2QIHXc= github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= diff --git a/integrationtests/relayedTxV3_test.go b/integrationtests/relayedTxV3_test.go index 28329ebb..6204b69f 100644 --- a/integrationtests/relayedTxV3_test.go +++ b/integrationtests/relayedTxV3_test.go @@ -380,3 +380,143 @@ func TestRelayedV3WithSCRCross(t *testing.T) { string(genericResponse.Docs[0].Source), ) } + +func TestRelayedV3TransactionWithMultipleRefunds(t *testing.T) { + setLogLevelDebug() + + esClient, err := createESClient(esURL) + require.Nil(t, err) + + esProc, err := CreateElasticProcessor(esClient) + require.Nil(t, err) + + txHash := []byte("relayedTxV3WithMultipleRefunds") + header := &dataBlock.Header{ + Round: 50, + TimeStamp: 5040, + } + + body := &dataBlock.Body{ + MiniBlocks: dataBlock.MiniBlockSlice{ + { + Type: dataBlock.TxBlock, + SenderShardID: 0, + ReceiverShardID: 0, + TxHashes: [][]byte{txHash}, + }, + }, + } + + initialTx := &transaction.Transaction{ + Nonce: 1000, + SndAddr: decodeAddress("erd1ykqd64fxxpp4wsz0v7sjqem038wfpzlljhx4mhwx8w9lcxmdzcfszrp64a"), + RcvAddr: decodeAddress("erd1ykqd64fxxpp4wsz0v7sjqem038wfpzlljhx4mhwx8w9lcxmdzcfszrp64a"), + GasLimit: 500_000_000, + GasPrice: 1000000000, + Value: big.NewInt(0), + InnerTransactions: []*transaction.Transaction{ + { + Nonce: 5, + SndAddr: decodeAddress("erd10ksryjr065ad5475jcg82pnjfg9j9qtszjsrp24anl6ym7cmeddshwnru8"), + RcvAddr: decodeAddress("erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r"), + GasLimit: 60_000_000, + GasPrice: 1000000000, + Value: big.NewInt(10000000000000000), + Data: []byte("doSomething"), + }, + { + Nonce: 3, + SndAddr: decodeAddress("erd10ksryjr065ad5475jcg82pnjfg9j9qtszjsrp24anl6ym7cmeddshwnru8"), + RcvAddr: decodeAddress("erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r"), + GasLimit: 60_000_000, + GasPrice: 1000000000, + Value: big.NewInt(10000000000000000), + Data: []byte("doSomething"), + }, + }, + } + + txInfo := &outport.TxInfo{ + Transaction: initialTx, + FeeInfo: &outport.FeeInfo{ + GasUsed: 180_150_000, + Fee: big.NewInt(2864760000000000), + InitialPaidFee: big.NewInt(2864760000000000), + }, + ExecutionOrder: 0, + } + + pool := &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): txInfo, + }, + } + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + ids := []string{hex.EncodeToString(txHash)} + genericResponse := &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/relayedTxV3/relayed-v3-no-refund.json"), + string(genericResponse.Docs[0].Source), + ) + + // execute first SCR with refund + pool = &outport.TransactionPool{ + SmartContractResults: map[string]*outport.SCRInfo{ + "scrHash": { + SmartContractResult: &smartContractResult.SmartContractResult{ + OriginalTxHash: txHash, + }, + FeeInfo: &outport.FeeInfo{ + ForRelayed: true, + GasUsed: 9_692_000, + Fee: big.NewInt(96920000000000), + }, + }, + }, + } + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + ids = []string{hex.EncodeToString(txHash)} + genericResponse = &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/relayedTxV3/relayed-v3-with-one-refund.json"), + string(genericResponse.Docs[0].Source), + ) + + // execute second SCR with refund + pool = &outport.TransactionPool{ + SmartContractResults: map[string]*outport.SCRInfo{ + "scrHash": { + SmartContractResult: &smartContractResult.SmartContractResult{ + OriginalTxHash: txHash, + }, + FeeInfo: &outport.FeeInfo{ + ForRelayed: true, + GasUsed: 9_692_000, + Fee: big.NewInt(96920000000000), + }, + }, + }, + } + err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards)) + require.Nil(t, err) + + ids = []string{hex.EncodeToString(txHash)} + genericResponse = &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, + readExpectedResult("./testdata/relayedTxV3/relayed-v3-with-two-refunds.json"), + string(genericResponse.Docs[0].Source), + ) +} diff --git a/integrationtests/testdata/relayedTxV3/relayed-v3-no-refund.json b/integrationtests/testdata/relayedTxV3/relayed-v3-no-refund.json new file mode 100644 index 00000000..cfdcd81f --- /dev/null +++ b/integrationtests/testdata/relayedTxV3/relayed-v3-no-refund.json @@ -0,0 +1,58 @@ +{ + "miniBlockHash": "785a251c08b314939528e553ac879dbee0627fbe2b76c0bca601c3f1e7162640", + "nonce": 1000, + "round": 50, + "value": "0", + "valueNum": 0, + "receiver": "erd1ykqd64fxxpp4wsz0v7sjqem038wfpzlljhx4mhwx8w9lcxmdzcfszrp64a", + "sender": "erd1ykqd64fxxpp4wsz0v7sjqem038wfpzlljhx4mhwx8w9lcxmdzcfszrp64a", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 500000000, + "gasUsed": 180150000, + "fee": "2864760000000000", + "feeNum": 0.00286476, + "initialPaidFee": "2864760000000000", + "data": null, + "signature": "", + "timestamp": 5040, + "status": "success", + "searchOrder": 0, + "receivers": [ + "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", + "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r" + ], + "receiversShardIDs": [ + 1, + 1 + ], + "operation": "transfer", + "isRelayed": true, + "innerTransactions": [ + { + "hash": "1f2283b94bcc0ff11237db3f2048c54794f0199565cdddd722d3a18bde214809", + "nonce": 5, + "value": "10000000000000000", + "receiver": "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", + "sender": "erd10ksryjr065ad5475jcg82pnjfg9j9qtszjsrp24anl6ym7cmeddshwnru8", + "gasPrice": 1000000000, + "gasLimit": 60000000, + "data": "ZG9Tb21ldGhpbmc=", + "chainID": "", + "version": 0 + }, + { + "hash": "aeac946abbb792f91d3498bf70d7ce1a1be91f6e5acffc316066474c36c1b181", + "nonce": 3, + "value": "10000000000000000", + "receiver": "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", + "sender": "erd10ksryjr065ad5475jcg82pnjfg9j9qtszjsrp24anl6ym7cmeddshwnru8", + "gasPrice": 1000000000, + "gasLimit": 60000000, + "data": "ZG9Tb21ldGhpbmc=", + "chainID": "", + "version": 0 + } + ] +} \ No newline at end of file diff --git a/integrationtests/testdata/relayedTxV3/relayed-v3-with-one-refund.json b/integrationtests/testdata/relayedTxV3/relayed-v3-with-one-refund.json new file mode 100644 index 00000000..b94735a5 --- /dev/null +++ b/integrationtests/testdata/relayedTxV3/relayed-v3-with-one-refund.json @@ -0,0 +1,58 @@ +{ + "miniBlockHash": "785a251c08b314939528e553ac879dbee0627fbe2b76c0bca601c3f1e7162640", + "nonce": 1000, + "round": 50, + "value": "0", + "valueNum": 0, + "receiver": "erd1ykqd64fxxpp4wsz0v7sjqem038wfpzlljhx4mhwx8w9lcxmdzcfszrp64a", + "sender": "erd1ykqd64fxxpp4wsz0v7sjqem038wfpzlljhx4mhwx8w9lcxmdzcfszrp64a", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 500000000, + "gasUsed": 170458000, + "fee": "2767840000000000", + "feeNum": 0.00276784, + "initialPaidFee": "2864760000000000", + "data": null, + "signature": "", + "timestamp": 5040, + "status": "success", + "searchOrder": 0, + "receivers": [ + "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", + "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r" + ], + "receiversShardIDs": [ + 1, + 1 + ], + "operation": "transfer", + "isRelayed": true, + "innerTransactions": [ + { + "hash": "1f2283b94bcc0ff11237db3f2048c54794f0199565cdddd722d3a18bde214809", + "nonce": 5, + "value": "10000000000000000", + "receiver": "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", + "sender": "erd10ksryjr065ad5475jcg82pnjfg9j9qtszjsrp24anl6ym7cmeddshwnru8", + "gasPrice": 1000000000, + "gasLimit": 60000000, + "data": "ZG9Tb21ldGhpbmc=", + "chainID": "", + "version": 0 + }, + { + "hash": "aeac946abbb792f91d3498bf70d7ce1a1be91f6e5acffc316066474c36c1b181", + "nonce": 3, + "value": "10000000000000000", + "receiver": "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", + "sender": "erd10ksryjr065ad5475jcg82pnjfg9j9qtszjsrp24anl6ym7cmeddshwnru8", + "gasPrice": 1000000000, + "gasLimit": 60000000, + "data": "ZG9Tb21ldGhpbmc=", + "chainID": "", + "version": 0 + } + ] +} diff --git a/integrationtests/testdata/relayedTxV3/relayed-v3-with-two-refunds.json b/integrationtests/testdata/relayedTxV3/relayed-v3-with-two-refunds.json new file mode 100644 index 00000000..7c6c3607 --- /dev/null +++ b/integrationtests/testdata/relayedTxV3/relayed-v3-with-two-refunds.json @@ -0,0 +1,58 @@ +{ + "miniBlockHash": "785a251c08b314939528e553ac879dbee0627fbe2b76c0bca601c3f1e7162640", + "nonce": 1000, + "round": 50, + "value": "0", + "valueNum": 0, + "receiver": "erd1ykqd64fxxpp4wsz0v7sjqem038wfpzlljhx4mhwx8w9lcxmdzcfszrp64a", + "sender": "erd1ykqd64fxxpp4wsz0v7sjqem038wfpzlljhx4mhwx8w9lcxmdzcfszrp64a", + "receiverShard": 0, + "senderShard": 0, + "gasPrice": 1000000000, + "gasLimit": 500000000, + "gasUsed": 160766000, + "fee": "2670920000000000", + "feeNum": 0.0026709200000000002, + "initialPaidFee": "2864760000000000", + "data": null, + "signature": "", + "timestamp": 5040, + "status": "success", + "searchOrder": 0, + "receivers": [ + "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", + "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r" + ], + "receiversShardIDs": [ + 1, + 1 + ], + "operation": "transfer", + "isRelayed": true, + "innerTransactions": [ + { + "hash": "1f2283b94bcc0ff11237db3f2048c54794f0199565cdddd722d3a18bde214809", + "nonce": 5, + "value": "10000000000000000", + "receiver": "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", + "sender": "erd10ksryjr065ad5475jcg82pnjfg9j9qtszjsrp24anl6ym7cmeddshwnru8", + "gasPrice": 1000000000, + "gasLimit": 60000000, + "data": "ZG9Tb21ldGhpbmc=", + "chainID": "", + "version": 0 + }, + { + "hash": "aeac946abbb792f91d3498bf70d7ce1a1be91f6e5acffc316066474c36c1b181", + "nonce": 3, + "value": "10000000000000000", + "receiver": "erd1aduqqezzw0u3j7tywlq3mrl0yn4z6f6vytdju8gg0neq38fauyzsa5yy6r", + "sender": "erd10ksryjr065ad5475jcg82pnjfg9j9qtszjsrp24anl6ym7cmeddshwnru8", + "gasPrice": 1000000000, + "gasLimit": 60000000, + "data": "ZG9Tb21ldGhpbmc=", + "chainID": "", + "version": 0 + } + ] +} diff --git a/process/elasticproc/transactions/scrsDataToTransactions.go b/process/elasticproc/transactions/scrsDataToTransactions.go index f18aa741..13b509ea 100644 --- a/process/elasticproc/transactions/scrsDataToTransactions.go +++ b/process/elasticproc/transactions/scrsDataToTransactions.go @@ -70,10 +70,11 @@ func (st *scrsDataToTransactions) processSCRsWithoutTx(scrs []*data.ScResult) ma } txHashRefund[scr.OriginalTxHash] = &data.FeeData{ - FeeNum: feeNum, - Fee: scr.InitialTxFee, - GasUsed: scr.InitialTxGasUsed, - Receiver: scr.Receiver, + FeeNum: feeNum, + Fee: scr.InitialTxFee, + GasUsed: scr.InitialTxGasUsed, + Receiver: scr.Receiver, + ForRelayed: scr.ForRelayed, } } diff --git a/process/elasticproc/transactions/serialize.go b/process/elasticproc/transactions/serialize.go index 939742a0..0ece7737 100644 --- a/process/elasticproc/transactions/serialize.go +++ b/process/elasticproc/transactions/serialize.go @@ -51,7 +51,28 @@ func (tdp *txsDatabaseProcessor) SerializeReceipts(receipts []*data.Receipt, buf func (tdp *txsDatabaseProcessor) SerializeTransactionsFeeData(txHashRefund map[string]*data.FeeData, buffSlice *data.BufferSlice, index string) error { for txHash, feeData := range txHashRefund { meta := []byte(fmt.Sprintf(`{"update":{ "_index":"%s","_id":"%s"}}%s`, index, converters.JsonEscape(txHash), "\n")) - codeToExecute := ` + + var codeToExecute string + if feeData.ForRelayed { + codeToExecute = ` + if ('create' == ctx.op) { + ctx.op = 'noop' + } else { + BigInteger feeFromSource = new BigInteger(ctx._source.fee); + BigInteger fee = new BigInteger(params.fee); + if (feeFromSource.compareTo(fee) > 0) { + ctx._source.fee = feeFromSource.subtract(fee).toString(); + } + if (ctx._source.feeNum > params.feeNum) { + ctx._source.feeNum -= params.feeNum; + } + if (ctx._source.gasUsed > params.gasUsed) { + ctx._source.gasUsed -= params.gasUsed; + } + } +` + } else { + codeToExecute = ` if ('create' == ctx.op) { ctx.op = 'noop' } else { @@ -60,6 +81,7 @@ func (tdp *txsDatabaseProcessor) SerializeTransactionsFeeData(txHashRefund map[s ctx._source.gasUsed = params.gasUsed; } ` + } serializedDataStr := fmt.Sprintf(`{"scripted_upsert": true, "script": {`+ `"source": "%s",`+ diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index 2fcb7ccf..5a3797b4 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -189,6 +189,7 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( OriginalSender: originalSenderAddr, InitialTxFee: feeInfo.Fee.String(), InitialTxGasUsed: feeInfo.GasUsed, + ForRelayed: feeInfo.ForRelayed, ExecutionOrder: int(scrInfo.ExecutionOrder), } } From c260b96878f77840728d7298d93e2e5f5ce21282 Mon Sep 17 00:00:00 2001 From: miiu Date: Thu, 17 Oct 2024 15:06:03 +0300 Subject: [PATCH 08/10] empty line --- integrationtests/testdata/relayedTxV3/relayed-v3-no-refund.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrationtests/testdata/relayedTxV3/relayed-v3-no-refund.json b/integrationtests/testdata/relayedTxV3/relayed-v3-no-refund.json index cfdcd81f..2a7e3feb 100644 --- a/integrationtests/testdata/relayedTxV3/relayed-v3-no-refund.json +++ b/integrationtests/testdata/relayedTxV3/relayed-v3-no-refund.json @@ -55,4 +55,4 @@ "version": 0 } ] -} \ No newline at end of file +} From b821f9d154197e95f8b4dd738b22cd814d77012f Mon Sep 17 00:00:00 2001 From: miiu Date: Fri, 18 Oct 2024 09:09:25 +0300 Subject: [PATCH 09/10] gas refunded --- data/scresult.go | 2 +- data/transaction.go | 10 +++++----- go.mod | 2 +- go.sum | 4 ++-- integrationtests/relayedTxV3_test.go | 10 ++++------ .../transactions/scrsDataToTransactions.go | 12 ++++++------ process/elasticproc/transactions/serialize.go | 10 +++++----- .../transactions/smartContractResultsProcessor.go | 2 +- 8 files changed, 25 insertions(+), 27 deletions(-) diff --git a/data/scresult.go b/data/scresult.go index f3fc93ad..95a71c5c 100644 --- a/data/scresult.go +++ b/data/scresult.go @@ -43,5 +43,5 @@ type ScResult struct { SenderAddressBytes []byte `json:"-"` InitialTxGasUsed uint64 `json:"-"` InitialTxFee string `json:"-"` - ForRelayed bool `json:"-"` + GasRefunded uint64 `json:"-"` } diff --git a/data/transaction.go b/data/transaction.go index cde848c5..ff031ed4 100644 --- a/data/transaction.go +++ b/data/transaction.go @@ -89,11 +89,11 @@ type ResponseTransactionDB struct { // FeeData is the structure that contains data about transaction fee and gas used type FeeData struct { - FeeNum float64 - Fee string - GasUsed uint64 - Receiver string - ForRelayed bool + FeeNum float64 + Fee string + GasUsed uint64 + Receiver string + GasRefunded uint64 } // InnerTransaction is the structure that contains data about an inner transaction diff --git a/go.mod b/go.mod index 51148ec8..a5f21f1c 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/multiversx/mx-chain-communication-go v1.1.0 - github.com/multiversx/mx-chain-core-go v1.2.23-0.20241017083841-5e7b1485b6a1 + github.com/multiversx/mx-chain-core-go v1.2.23-0.20241018055916-203f32de05ad github.com/multiversx/mx-chain-logger-go v1.0.15 github.com/multiversx/mx-chain-vm-common-go v1.5.13 github.com/prometheus/client_model v0.4.0 diff --git a/go.sum b/go.sum index 2347e509..00b0ea98 100644 --- a/go.sum +++ b/go.sum @@ -249,8 +249,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.1.0 h1:J7bX6HoN3HiHY7cUeEjG8AJWgQDDPcY+OPDOsSUOkRE= github.com/multiversx/mx-chain-communication-go v1.1.0/go.mod h1:WK6bP4pGEHGDDna/AYRIMtl6G9OA0NByI1Lw8PmOnRM= -github.com/multiversx/mx-chain-core-go v1.2.23-0.20241017083841-5e7b1485b6a1 h1:TyWMSr3wx1DR3rlKnp7Iw/nEMYDp7any7DtMpz2aAlU= -github.com/multiversx/mx-chain-core-go v1.2.23-0.20241017083841-5e7b1485b6a1/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20241018055916-203f32de05ad h1:EFWFNts9am6Dp+tJxYiHfsepK4TF0J931mTEZ3Aireo= +github.com/multiversx/mx-chain-core-go v1.2.23-0.20241018055916-203f32de05ad/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.12 h1:zWip7rpUS4CGthJxfKn5MZfMfYPjVjIiCID6uX5BSOk= github.com/multiversx/mx-chain-logger-go v1.0.15 h1:HlNdK8etyJyL9NQ+6mIXyKPEBo+wRqOwi3n+m2QIHXc= github.com/multiversx/mx-chain-logger-go v1.0.15/go.mod h1:t3PRKaWB1M+i6gUfD27KXgzLJJC+mAQiN+FLlL1yoGQ= diff --git a/integrationtests/relayedTxV3_test.go b/integrationtests/relayedTxV3_test.go index 6204b69f..e537e7e9 100644 --- a/integrationtests/relayedTxV3_test.go +++ b/integrationtests/relayedTxV3_test.go @@ -472,9 +472,8 @@ func TestRelayedV3TransactionWithMultipleRefunds(t *testing.T) { OriginalTxHash: txHash, }, FeeInfo: &outport.FeeInfo{ - ForRelayed: true, - GasUsed: 9_692_000, - Fee: big.NewInt(96920000000000), + GasRefunded: 9_692_000, + Fee: big.NewInt(96920000000000), }, }, }, @@ -500,9 +499,8 @@ func TestRelayedV3TransactionWithMultipleRefunds(t *testing.T) { OriginalTxHash: txHash, }, FeeInfo: &outport.FeeInfo{ - ForRelayed: true, - GasUsed: 9_692_000, - Fee: big.NewInt(96920000000000), + GasRefunded: 9_692_000, + Fee: big.NewInt(96920000000000), }, }, }, diff --git a/process/elasticproc/transactions/scrsDataToTransactions.go b/process/elasticproc/transactions/scrsDataToTransactions.go index 13b509ea..af2a5148 100644 --- a/process/elasticproc/transactions/scrsDataToTransactions.go +++ b/process/elasticproc/transactions/scrsDataToTransactions.go @@ -54,7 +54,7 @@ func (st *scrsDataToTransactions) processTransactionsAfterSCRsWereAttached(trans func (st *scrsDataToTransactions) processSCRsWithoutTx(scrs []*data.ScResult) map[string]*data.FeeData { txHashRefund := make(map[string]*data.FeeData) for _, scr := range scrs { - if scr.InitialTxGasUsed == 0 { + if scr.InitialTxGasUsed == 0 && scr.GasRefunded == 0 { continue } @@ -70,11 +70,11 @@ func (st *scrsDataToTransactions) processSCRsWithoutTx(scrs []*data.ScResult) ma } txHashRefund[scr.OriginalTxHash] = &data.FeeData{ - FeeNum: feeNum, - Fee: scr.InitialTxFee, - GasUsed: scr.InitialTxGasUsed, - Receiver: scr.Receiver, - ForRelayed: scr.ForRelayed, + FeeNum: feeNum, + Fee: scr.InitialTxFee, + GasUsed: scr.InitialTxGasUsed, + Receiver: scr.Receiver, + GasRefunded: scr.GasRefunded, } } diff --git a/process/elasticproc/transactions/serialize.go b/process/elasticproc/transactions/serialize.go index 0ece7737..eea07a0d 100644 --- a/process/elasticproc/transactions/serialize.go +++ b/process/elasticproc/transactions/serialize.go @@ -53,7 +53,7 @@ func (tdp *txsDatabaseProcessor) SerializeTransactionsFeeData(txHashRefund map[s meta := []byte(fmt.Sprintf(`{"update":{ "_index":"%s","_id":"%s"}}%s`, index, converters.JsonEscape(txHash), "\n")) var codeToExecute string - if feeData.ForRelayed { + if feeData.GasRefunded > 0 { codeToExecute = ` if ('create' == ctx.op) { ctx.op = 'noop' @@ -66,8 +66,8 @@ func (tdp *txsDatabaseProcessor) SerializeTransactionsFeeData(txHashRefund map[s if (ctx._source.feeNum > params.feeNum) { ctx._source.feeNum -= params.feeNum; } - if (ctx._source.gasUsed > params.gasUsed) { - ctx._source.gasUsed -= params.gasUsed; + if (ctx._source.gasUsed > params.gasRefunded) { + ctx._source.gasUsed -= params.gasRefunded; } } ` @@ -86,9 +86,9 @@ func (tdp *txsDatabaseProcessor) SerializeTransactionsFeeData(txHashRefund map[s serializedDataStr := fmt.Sprintf(`{"scripted_upsert": true, "script": {`+ `"source": "%s",`+ `"lang": "painless",`+ - `"params": {"fee": "%s", "gasUsed": %d, "feeNum": %g}},`+ + `"params": {"fee": "%s", "gasUsed": %d, "feeNum": %g, "gasRefunded": %d}},`+ `"upsert": {}}`, - converters.FormatPainlessSource(codeToExecute), feeData.Fee, feeData.GasUsed, feeData.FeeNum, + converters.FormatPainlessSource(codeToExecute), feeData.Fee, feeData.GasUsed, feeData.FeeNum, feeData.GasRefunded, ) err := buffSlice.PutData(meta, []byte(serializedDataStr)) diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index 5a3797b4..ce52062a 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -189,7 +189,7 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( OriginalSender: originalSenderAddr, InitialTxFee: feeInfo.Fee.String(), InitialTxGasUsed: feeInfo.GasUsed, - ForRelayed: feeInfo.ForRelayed, + GasRefunded: feeInfo.GasRefunded, ExecutionOrder: int(scrInfo.ExecutionOrder), } } From 4c779fec382479d1e5421ae0cafde8aa00b77511 Mon Sep 17 00:00:00 2001 From: miiu Date: Fri, 18 Oct 2024 10:04:19 +0300 Subject: [PATCH 10/10] fix unit test --- process/elasticproc/transactions/serialize_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/elasticproc/transactions/serialize_test.go b/process/elasticproc/transactions/serialize_test.go index 1565ee2d..b4e2bf96 100644 --- a/process/elasticproc/transactions/serialize_test.go +++ b/process/elasticproc/transactions/serialize_test.go @@ -146,7 +146,7 @@ func TestTxsDatabaseProcessor_SerializeTransactionWithRefund(t *testing.T) { require.Nil(t, err) expectedBuff := `{"update":{ "_index":"transactions","_id":"txHash"}} -{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx.op = 'noop'} else {ctx._source.fee = params.fee;ctx._source.feeNum = params.feeNum;ctx._source.gasUsed = params.gasUsed;}","lang": "painless","params": {"fee": "100000", "gasUsed": 5000, "feeNum": 5e-15}},"upsert": {}} +{"scripted_upsert": true, "script": {"source": "if ('create' == ctx.op) {ctx.op = 'noop'} else {ctx._source.fee = params.fee;ctx._source.feeNum = params.feeNum;ctx._source.gasUsed = params.gasUsed;}","lang": "painless","params": {"fee": "100000", "gasUsed": 5000, "feeNum": 5e-15, "gasRefunded": 0}},"upsert": {}} ` require.Equal(t, expectedBuff, buffSlice.Buffers()[0].String()) }