diff --git a/.env.example b/.env.example index b8a5089..91dbfae 100644 --- a/.env.example +++ b/.env.example @@ -1,10 +1,14 @@ INDEXER_BRIDGED_TOKENS_FILE=mainnet.json # full list of files you can find in repo ./build/bridged_tokens/ INDEXER_CLASS_INTERFACES_DIR=./interfaces/ # REQUIRED +INDEXER_DATASOURCE=sequencer # REQUIRED: one of 'node' or 'sequencer' HASURA_HOST=hasura HASURA_POSTGRES_HOST=db LOG_LEVEL=info -CACHE_ENABLED=false POSTGRES_PORT=5432 POSTGRES_HOST=db POSTGRES_DB=starknet -POSTGRES_PASSWORD= # REQUIRED \ No newline at end of file +POSTGRES_PASSWORD= # REQUIRED +STARKNET_NODE_URL= # REQUIRED if INDEXER_DATASOURCE=node +STARKNET_SEQUENCER_FEEDER_GATEWAY= # REQUIRED if INDEXER_DATASOURCE=sequencer +NODE_APIKEY= # REQUIRED if your node provider has api key. It's api key. +NODE_HEADER_APIKEY= # REQUIRED if your node provider has api key. It's header name. \ No newline at end of file diff --git a/build/bridged_tokens/goerli.json b/build/bridged_tokens/goerli.json new file mode 100644 index 0000000..961224a --- /dev/null +++ b/build/bridged_tokens/goerli.json @@ -0,0 +1,125 @@ +[ + { + "name": "Wrapped BTC", + "symbol": "WBTC", + "decimals": 8, + "l1_token_address": "0xC04B0d3107736C32e19F1c62b2aF67BE61d63a05", + "l2_token_address": "0x12d537dc323c439dc65c976fad242d5610d27cfb5f31689a0a319b8be7f3d56", + "l1_bridge_address": "0xf29aE3446Ce4688fCc792b232C21D1B9581E7baC", + "l2_bridge_address": "0x72eeb90833bae233a9585f2fa9afc99c187f0a3a82693becd6a4d700b37fc6b" + }, + { + "name": "USD Coin", + "symbol": "USDC", + "decimals": 6, + "l1_token_address": "0x07865c6e87b9f70255377e024ace6630c1eaa37f", + "l2_bridge_address": "0x001d5b64feabc8ac7c839753994f469704c6fabdd45c8fe6d26ed57b5eb79057", + "l2_token_address": "0x005a643907b9a4bc6a55e9069c4fd5fd1f5c79a22470690f75556c4736e34426", + "l1_bridge_address": "0xBA9cE9F22A3Cfa7Fcb5c31f6B2748b1e72C06204" + }, + { + "name": "Tether USD", + "symbol": "USDT", + "decimals": 6, + "l1_token_address": "0x509Ee0d083DdF8AC028f2a56731412edD63223B9", + "l2_token_address": "0x386e8d061177f19b3b485c20e31137e6f6bc497cc635ccdfcab96fadf5add6a", + "l1_bridge_address": "0xA1f590F18b23EFece02804704E5006E91348C997", + "l2_bridge_address": "0x71d54658ca3c6ccd84ff958adb7498b2e71ba008e29b643983221ed2bd71b69" + }, + { + "name": "SelfService", + "symbol": "SLF", + "decimals": 6, + "token_address": "0xd44BB808bfE43095dBb94c83077766382D63952a", + "l2_bridge_address": "0x00fd2a9843c19436542e0ac7fc7b5cbf1d0b69fc2abea6d68591e46a5ca2d75a", + "l2_token_address": "0x07a39a50bf689e9430fc81fba0f4d46e245e1657e77455548ed7e32c808cfc10", + "l1_bridge_address": "0x160e7631f22035149A01420cADD1012267551181" + }, + { + "name": "Ether", + "symbol": "ETH", + "decimals": 18, + "l1_token_address": "0x0000000000000000000000000000000000000000", + "l2_token_address": "0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7", + "l1_bridge_address": "0xc3511006C04EF1d78af4C8E0e74Ec18A6E64Ff9e", + "l2_bridge_address": "0x073314940630fd6dcda0d772d4c972c4e0a9946bef9dabf4ef84eda8ef542b82" + }, + { + "name": "DAI", + "symbol": "DAI", + "decimals": 18, + "l1_token_address": "0x11fE4B6AE13d2a6055C8D9cF65c55bac32B5d844", + "l2_bridge_address": "0x057b7fe4e59d295de5e7955c373023514ede5b972e872e9aa5dcdf563f5cfacb", + "l2_token_address": "0x03e85bfbb8e2a42b7bead9e88e9a1b19dbccf661471061807292120462396ec9", + "l1_bridge_address": "0xaB00D7EE6cFE37cCCAd006cEC4Db6253D7ED3a22" + }, + { + "name": "Wrapped liquid staked Ether 2.0", + "symbol": "wstETH", + "decimals": 18, + "l1_token_address": "0x6320cD32aA674d2898A68ec82e869385Fc5f7E2f", + "l1_bridge_address": "0x190c98506a5396A30CA759A139F3Fb59EF519A5D", + "l2_bridge_address": "0x0399a6011b666888d647665fd65d6dcc7c2690c72d4c4454cae987f19f6ef609", + "l2_token_address": "0x0335bc6e1cf6d9527da4f8044c505906ad6728aeeddfba8d7000b01b32ffe66b" + }, + { + "name": "Rocket Pool ETH", + "symbol": "rETH", + "decimals": 18, + "l1_token_address": "0x178E141a0E3b34152f73Ff610437A7bf9B83267A", + "l1_bridge_address": "0xD2ef821C56B20a7451dbbEd1ec003De6C44F8dC0", + "l2_bridge_address": "0x00214e168720c6eed858066bea070afa828512e83edcfc28846f0e87221f77f6", + "l2_token_address": "0x002133188109385fedaac0b1bf9de1134e271b88efcd21e2ea0dac460639fbe2" + }, + { + "name": "LUSD Stablecoin", + "symbol": "LUSD", + "decimals": 18, + "l1_token_address": "0x76ea225E132609D387464e11ce5EFA1764A3799B", + "l2_bridge_address": "0x02edc855e8e45fa1826d53a144c080fd41bb33f9468190cf60089f45cbc3c223", + "l2_token_address": "0x025731f5f9629ff74d1c5f787ad1ea0ebb9157210047f6c9e3a974f771550cf4", + "l1_bridge_address": "0xc2AFba3f4f6a88Ad738aa0e9cf746B38370415a4" + }, + { + "name": "R Stablecoin", + "symbol": "R", + "decimals": 18, + "l1_token_address": "0x9b41fE4EE4F23507953CCA339A4eC27eAc9e02b8", + "l2_bridge_address": "0x060fbf0392c84eae8bf3d5a79e29e6f250933fabeab00cef13c8f1a68c1cca6f", + "l2_token_address": "0x02c479575aa7399ca4757927c02a71334ff6f1b12693fa9043cf9f49b83d0000", + "l1_bridge_address": "0xe2969b9d9de178cccc7199234d3e0543da3a7733" + }, + { + "name": "Uniswap", + "symbol": "UNI", + "decimals": 18, + "l1_token_address": "0x1f9840a85d5af5bf1d1762f925bdaddc4201f984", + "l2_bridge_address": "0x05f1299e76372f9b7f3d6b4be58c67af8eb27af040bf288206f38e6d5afd0abd", + "l2_token_address": "0x02a19c5ec71efb0022cae7db810d71280e3fccd2b7b7503c87bcb851089e0f34", + "l1_bridge_address": "0xc0c0eB9eeb90243C3FE4e562F12Ff01e8fE8Ff03" + }, + { + "name": "Starknet Token", + "symbol": "STRK", + "decimals": 18, + "l1_token_address": "0xCa14007Eff0dB1f8135f4C25B34De49AB0d42766", + "l2_token_address": "0x04718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d", + "l1_bridge_address": "0x988bb31B3246004e636bbCA7A40ae723f9511808", + "l2_bridge_address": "0x0594c1582459ea03f77deaf9eb7e3917d6994a03c13405ba42867f83d85f085d", + "l2_locking_contract": "0x01a881a75bb478cedfd4d3ea19d2a4564350d78ea463a5287833526a416d5e31" + }, + { + "name": "Starknet Voting Token", + "symbol": "vSTRK", + "decimals": 18, + "l2_token_address": "0x01a881a75bb478cedfd4d3ea19d2a4564350d78ea463a5287833526a416d5e31", + "l2_locked_token": "0x04718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d" + }, + { + "name": "MultiBridge", + "StarkgateManager": "0x5C04F89A136749b7e4d0E427B606241a0A41B32f", + "StarkgateRegistry": "0xEc05a1EEc2fC769F44A87Be2CD7f0CC2BF3730AB", + "StarknetTokenBridge": "0x25E569e0652EB8A6265bAf6D7C146E901f59554C", + "l2_token_bridge": "0x0627582c893c1506750d28a40a2e781031554c16544ff7b390c117978bc03de7" + } +] \ No newline at end of file diff --git a/build/bridged_tokens/goerli1.json b/build/bridged_tokens/goerli1.json deleted file mode 100644 index 577196f..0000000 --- a/build/bridged_tokens/goerli1.json +++ /dev/null @@ -1,56 +0,0 @@ -[ - { - "name": "Wrapped BTC", - "symbol": "WBTC", - "decimals": 8, - "l1_token_address": "0xC04B0d3107736C32e19F1c62b2aF67BE61d63a05", - "l2_token_address": "0x12d537dc323c439dc65c976fad242d5610d27cfb5f31689a0a319b8be7f3d56", - "l1_bridge_address": "0xf29aE3446Ce4688fCc792b232C21D1B9581E7baC", - "l2_bridge_address": "0x72eeb90833bae233a9585f2fa9afc99c187f0a3a82693becd6a4d700b37fc6b" - }, - { - "name": "Goerli USD Coin", - "symbol": "USDC", - "decimals": 6, - "l1_token_address": "0x07865c6e87b9f70255377e024ace6630c1eaa37f", - "l2_bridge_address": "0x001d5b64feabc8ac7c839753994f469704c6fabdd45c8fe6d26ed57b5eb79057", - "l2_token_address": "0x005a643907b9a4bc6a55e9069c4fd5fd1f5c79a22470690f75556c4736e34426", - "l1_bridge_address": "0xBA9cE9F22A3Cfa7Fcb5c31f6B2748b1e72C06204" - }, - { - "name": "Tether USD", - "symbol": "USDT", - "decimals": 6, - "l1_token_address": "0x509Ee0d083DdF8AC028f2a56731412edD63223B9", - "l2_token_address": "0x386e8d061177f19b3b485c20e31137e6f6bc497cc635ccdfcab96fadf5add6a", - "l1_bridge_address": "0xA1f590F18b23EFece02804704E5006E91348C997", - "l2_bridge_address": "0x71d54658ca3c6ccd84ff958adb7498b2e71ba008e29b643983221ed2bd71b69" - }, - { - "name": "SelfService", - "symbol": "SLF", - "decimals": 6, - "token_address": "0xd44BB808bfE43095dBb94c83077766382D63952a", - "l2_bridge_address": "0x00fd2a9843c19436542e0ac7fc7b5cbf1d0b69fc2abea6d68591e46a5ca2d75a", - "l2_token_address": "0x07a39a50bf689e9430fc81fba0f4d46e245e1657e77455548ed7e32c808cfc10", - "l1_bridge_address": "0x160e7631f22035149A01420cADD1012267551181" - }, - { - "name": "Ether", - "symbol": "ETH", - "decimals": 18, - "l1_token_address": "0x0000000000000000000000000000000000000000", - "l2_token_address": "0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7", - "l1_bridge_address": "0xc3511006C04EF1d78af4C8E0e74Ec18A6E64Ff9e", - "l2_bridge_address": "0x073314940630fd6dcda0d772d4c972c4e0a9946bef9dabf4ef84eda8ef542b82" - }, - { - "name": "DAI", - "symbol": "DAI", - "decimals": 18, - "l1_token_address": "0x11fE4B6AE13d2a6055C8D9cF65c55bac32B5d844", - "l2_bridge_address": "0x0278f24c3e74cbf7a375ec099df306289beb0605a346277d200b791a7f811a19", - "l2_token_address": "0x03e85bfbb8e2a42b7bead9e88e9a1b19dbccf661471061807292120462396ec9", - "l1_bridge_address": "0xd8beAa22894Cd33F24075459cFba287a10a104E4" - } -] \ No newline at end of file diff --git a/build/bridged_tokens/goerli2.json b/build/bridged_tokens/goerli2.json deleted file mode 100644 index 4ea6264..0000000 --- a/build/bridged_tokens/goerli2.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "name": "Ether", - "symbol": "ETH", - "decimals": 18, - "l1_token_address": "0x0000000000000000000000000000000000000000", - "l2_token_address": "0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7", - "l1_bridge_address": "0xaea4513378eb6023cf9ce730a26255d0e3f075b9", - "l2_bridge_address": "0x073314940630fd6dcda0d772d4c972c4e0a9946bef9dabf4ef84eda8ef542b82" - } -] \ No newline at end of file diff --git a/build/bridged_tokens/sepolia.json b/build/bridged_tokens/sepolia.json new file mode 100644 index 0000000..d278eaf --- /dev/null +++ b/build/bridged_tokens/sepolia.json @@ -0,0 +1,28 @@ +[ + { + "name": "Ether", + "symbol": "ETH", + "decimals": 18, + "l1_token_address": "0x0000000000000000000000000000000000000000", + "l2_token_address": "0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7", + "l1_bridge_address": "0x8453FC6Cd1bCfE8D4dFC069C400B433054d47bDc", + "l2_bridge_address": "0x04c5772d1914fe6ce891b64eb35bf3522aeae1315647314aac58b01137607f3f" + }, + { + "name": "Starknet Token", + "symbol": "STRK", + "decimals": 18, + "l1_token_address": "0xCa14007Eff0dB1f8135f4C25B34De49AB0d42766", + "l2_token_address": "0x04718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d", + "l1_bridge_address": "0xcE5485Cfb26914C5dcE00B9BAF0580364daFC7a4", + "l2_bridge_address": "0x0594c1582459ea03f77deaf9eb7e3917d6994a03c13405ba42867f83d85f085d", + "l2_locking_contract": "0x035c332b8de00874e702b4831c84b22281fb3246f714475496d74e644f35d492" + }, + { + "name": "Starknet Voting Token", + "symbol": "vSTRK", + "decimals": 18, + "l2_token_address": "0x035c332b8de00874e702b4831c84b22281fb3246f714475496d74e644f35d492", + "l2_locked_token": "0x04718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d" + } +] diff --git a/build/dipdup.yml b/build/dipdup.yml index dad6f8f..a7b3ee2 100644 --- a/build/dipdup.yml +++ b/build/dipdup.yml @@ -1,32 +1,28 @@ -version: 0.0.1 +version: 0.0.2 log_level: ${LOG_LEVEL:-info} indexer: name: ${INDEXER_NAME:-dipdup_starknet_indexer} - - sequencer: - feeder_gateway: ${STARKNET_SEQUENCER_FEEDER_GATEWAY:-https://alpha-mainnet.starknet.io/feeder_gateway} - gateway: ${STARKNET_SEQUENCER_GATEWAY:-https://alpha-mainnet.starknet.io/gateway} - requests_per_second: ${STARKNET_SEQUENCER_RPS:-3} - - node: - url: ${STARKNET_NODE_URL} - requests_per_second: ${STARKNET_NODE_RPS:-5} - + datasource: ${INDEXER_DATASOURCE:-sequencer} threads_count: ${INDEXER_THREADS_COUNT:-10} start_level: ${INDEXER_START_LEVEL:-0} timeout: ${INDEXER_REQUEST_TIMEOUT:-10} - requests_per_second: ${STARKNET_RPS:-2} class_interfaces_dir: ${INDEXER_CLASS_INTERFACES_DIR:-./interfaces/} bridged_tokens_file: ${INDEXER_BRIDGED_TOKENS_FILE:-mainnet.json} - cache_dir: ${INDEXER_CACHE_DIR} - cache: ${CACHE_ENABLED:-false} grpc: bind: ${GRPC_BIND:-127.0.0.1:7779} log: true +datasources: + node: + url: ${STARKNET_NODE_URL} + rps: ${STARKNET_NODE_RPS:-5} + sequencer: + url: ${STARKNET_SEQUENCER_FEEDER_GATEWAY:-https://alpha-mainnet.starknet.io/feeder_gateway} + rps: ${STARKNET_SEQUENCER_RPS:-3} + database: kind: postgres host: ${POSTGRES_HOST:-db} diff --git a/cmd/indexer/main.go b/cmd/indexer/main.go index 83c051f..791b673 100644 --- a/cmd/indexer/main.go +++ b/cmd/indexer/main.go @@ -103,7 +103,12 @@ func main() { } } - indexerModule := indexer.New(cfg.Indexer, postgres) + indexerModule, err := indexer.New(cfg.Indexer, postgres, cfg.DataSources) + if err != nil { + log.Panic().Err(err).Msg("creating indexer module") + cancel() + return + } grpcModule, err := grpc.NewServer( cfg.GRPC, postgres, diff --git a/go.mod b/go.mod index fafc438..4754106 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/dipdup-io/starknet-indexer go 1.21 require ( - github.com/dipdup-io/starknet-go-api v0.0.0-20240130002112-e07f784ea34e + github.com/dipdup-io/starknet-go-api v0.0.0-20240307134453-42ca4ba88776 github.com/dipdup-io/workerpool v0.0.4 github.com/dipdup-net/go-lib v0.3.3 github.com/dipdup-net/indexer-sdk v0.0.4 @@ -80,7 +80,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc4 // indirect - github.com/opencontainers/runc v1.1.9 // indirect + github.com/opencontainers/runc v1.1.12 // indirect github.com/paulmach/orb v0.10.0 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index a60d279..93b493b 100644 --- a/go.sum +++ b/go.sum @@ -39,10 +39,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw= github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo= -github.com/dipdup-io/starknet-go-api v0.0.0-20240110155038-c97c6d82a0eb h1:IAJCAcl/bS47DVWAedA94j/EwfG4Xt/LeBLwEOonKn4= -github.com/dipdup-io/starknet-go-api v0.0.0-20240110155038-c97c6d82a0eb/go.mod h1:y3KGLFQtwzUBcT0X2LMj6CxocUimr/A9XYg+j0KIRDE= -github.com/dipdup-io/starknet-go-api v0.0.0-20240130002112-e07f784ea34e h1:oFFZ77W368tOQuCwReeB3PEpjkJeEGOciOgJFYp+1Y8= -github.com/dipdup-io/starknet-go-api v0.0.0-20240130002112-e07f784ea34e/go.mod h1:y3KGLFQtwzUBcT0X2LMj6CxocUimr/A9XYg+j0KIRDE= +github.com/dipdup-io/starknet-go-api v0.0.0-20240307134453-42ca4ba88776 h1:j7CcqHyilB750xiPtUO/HueGE7VtTzixRvLKp4ccPsY= +github.com/dipdup-io/starknet-go-api v0.0.0-20240307134453-42ca4ba88776/go.mod h1:y3KGLFQtwzUBcT0X2LMj6CxocUimr/A9XYg+j0KIRDE= github.com/dipdup-io/workerpool v0.0.4 h1:m58fuFY3VIPRc+trWpjw2Lsm4FvIgtjP/4VRe79r+/s= github.com/dipdup-io/workerpool v0.0.4/go.mod h1:m6YMqx7M+fORTyabHD/auKymBRpbDax0t1aPZ1i7GZA= github.com/dipdup-net/go-lib v0.3.3 h1:vTUI+sT4L+x+eiMf712Cg8EtlqUCMiN6M3vcNaPlCw8= @@ -222,8 +220,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= -github.com/opencontainers/runc v1.1.9 h1:XR0VIHTGce5eWPkaPesqTBrhW2yAcaraWfsEalNwQLM= -github.com/opencontainers/runc v1.1.9/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= +github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss= +github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= github.com/paulmach/orb v0.10.0 h1:guVYVqzxHE/CQ1KpfGO077TR0ATHSNjp4s6XGLn3W9s= github.com/paulmach/orb v0.10.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= diff --git a/internal/storage/deploy.go b/internal/storage/deploy.go index 5440753..08edf7c 100644 --- a/internal/storage/deploy.go +++ b/internal/storage/deploy.go @@ -39,7 +39,7 @@ type Deploy struct { Hash []byte `comment:"Transaction hash"` ContractAddressSalt []byte `comment:"A random salt that determines the account address"` ConstructorCalldata []string `bun:",array" comment:"Raw constructor calldata"` - ParsedCalldata map[string]any `comment:"Calldata parsed according to contract ABI"` + ParsedCalldata map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"` Error *string `bun:"error" comment:"Reverted error"` Class Class `bun:"rel:belongs-to" hasura:"table:class,field:class_id,remote_field:id,type:oto,name:class"` @@ -89,12 +89,15 @@ func (d Deploy) Flat() []any { d.Hash, d.ContractAddressSalt, pq.StringArray(d.ConstructorCalldata), + nil, + d.Error, } - parsed, err := json.MarshalWithOption(d.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil, d.Error) - } else { - data = append(data, string(parsed), d.Error) + + if d.ParsedCalldata != nil { + parsed, err := json.MarshalWithOption(d.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[10] = string(parsed) + } } return data } diff --git a/internal/storage/deploy_account.go b/internal/storage/deploy_account.go index 2b1b48d..403e91b 100644 --- a/internal/storage/deploy_account.go +++ b/internal/storage/deploy_account.go @@ -42,7 +42,7 @@ type DeployAccount struct { MaxFee decimal.Decimal `bun:",type:numeric" comment:"The maximum fee that the sender is willing to pay for the transaction"` Nonce decimal.Decimal `bun:",type:numeric" comment:"The transaction nonce"` ConstructorCalldata []string `bun:",array" comment:"Raw constructor calldata"` - ParsedCalldata map[string]any `comment:"Calldata parsed according to contract ABI"` + ParsedCalldata map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"` Error *string `bun:"error" comment:"Reverted error"` Class Class `bun:"rel:belongs-to" hasura:"table:class,field:class_id,remote_field:id,type:oto,name:class"` @@ -94,12 +94,14 @@ func (d DeployAccount) Flat() []any { d.MaxFee, d.Nonce, pq.StringArray(d.ConstructorCalldata), + nil, + d.Error, } - parsed, err := json.MarshalWithOption(d.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil, d.Error) - } else { - data = append(data, string(parsed), d.Error) + if d.ParsedCalldata != nil { + parsed, err := json.MarshalWithOption(d.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[12] = string(parsed) + } } return data } diff --git a/internal/storage/event.go b/internal/storage/event.go index 55e9701..ceb7f33 100644 --- a/internal/storage/event.go +++ b/internal/storage/event.go @@ -48,7 +48,7 @@ type Event struct { Keys []string `bun:",array" comment:"Raw event keys"` Data []string `bun:",array" comment:"Raw event data"` Name string `comment:"Event name"` - ParsedData map[string]any `comment:"Event data parsed according to contract ABI"` + ParsedData map[string]any `bun:",nullzero" comment:"Event data parsed according to contract ABI"` From Address `bun:"rel:belongs-to" hasura:"table:address,field:from_id,remote_field:id,type:oto,name:from"` Contract Address `bun:"rel:belongs-to" hasura:"table:address,field:contract_id,remote_field:id,type:oto,name:contract"` @@ -98,13 +98,14 @@ func (e Event) Flat() []any { pq.StringArray(e.Keys), pq.StringArray(e.Data), e.Name, + nil, } - parsed, err := json.MarshalWithOption(e.ParsedData, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil) - } else { - data = append(data, string(parsed)) + if e.ParsedData != nil { + parsed, err := json.MarshalWithOption(e.ParsedData, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[16] = string(parsed) + } } return data } diff --git a/internal/storage/fee.go b/internal/storage/fee.go index 1424dc1..a551f2f 100644 --- a/internal/storage/fee.go +++ b/internal/storage/fee.go @@ -57,7 +57,7 @@ type Fee struct { Entrypoint string `comment:"Entrypoint name"` Calldata []string `bun:",array" comment:"Raw calldata"` Result []string `bun:",array" comment:"Raw result"` - ParsedCalldata map[string]any `comment:"Calldata parsed according to contract ABI"` + ParsedCalldata map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"` Class Class `bun:"rel:belongs-to" hasura:"table:class,field:class_id,remote_field:id,type:oto,name:class"` Caller Address `bun:"rel:belongs-to" hasura:"table:address,field:caller_id,remote_field:id,type:oto,name:caller"` @@ -115,12 +115,14 @@ func (f Fee) Flat() []any { f.Entrypoint, pq.StringArray(f.Calldata), pq.StringArray(f.Result), + nil, } - parsed, err := json.MarshalWithOption(f.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil) - } else { - data = append(data, string(parsed)) + + if f.ParsedCalldata != nil { + parsed, err := json.MarshalWithOption(f.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[18] = string(parsed) + } } return data } diff --git a/internal/storage/internal.go b/internal/storage/internal.go index 3d322aa..42efd42 100644 --- a/internal/storage/internal.go +++ b/internal/storage/internal.go @@ -58,8 +58,8 @@ type Internal struct { Entrypoint string `comment:"Entrypoint name"` Result []string `bun:",array" comment:"Raw result"` Calldata []string `bun:",array" comment:"Raw calldata"` - ParsedCalldata map[string]any `comment:"Calldata parsed according to contract ABI"` - ParsedResult map[string]any `comment:"Result parsed according to contract ABI"` + ParsedCalldata map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"` + ParsedResult map[string]any `bun:",nullzero" comment:"Result parsed according to contract ABI"` Class Class `bun:"rel:belongs-to" hasura:"table:class,field:class_id,remote_field:id,type:oto,name:class"` Caller Address `bun:"rel:belongs-to" hasura:"table:address,field:caller_id,remote_field:id,type:oto,name:caller"` @@ -120,20 +120,22 @@ func (i Internal) Flat() []any { i.Entrypoint, pq.StringArray(i.Calldata), pq.StringArray(i.Result), + nil, + nil, } - parsed, err := json.MarshalWithOption(i.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil) - } else { - data = append(data, string(parsed)) + if i.ParsedCalldata != nil { + parsed, err := json.MarshalWithOption(i.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[20] = string(parsed) + } } - result, err := json.MarshalWithOption(i.ParsedResult, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil) - } else { - data = append(data, string(result)) + if i.ParsedResult != nil { + result, err := json.MarshalWithOption(i.ParsedResult, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[21] = string(result) + } } return data } diff --git a/internal/storage/invoke.go b/internal/storage/invoke.go index 3a663ab..455df55 100644 --- a/internal/storage/invoke.go +++ b/internal/storage/invoke.go @@ -46,7 +46,7 @@ type Invoke struct { MaxFee decimal.Decimal `bun:",type:numeric" comment:"The maximum fee that the sender is willing to pay for the transaction"` Nonce decimal.Decimal `bun:",type:numeric" comment:"The transaction nonce"` CallData []string `bun:",array" comment:"Raw calldata"` - ParsedCalldata map[string]any `comment:"Calldata parsed according to contract ABI"` + ParsedCalldata map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"` Error *string `bun:"error" comment:"Reverted error"` Contract Address `bun:"rel:belongs-to" hasura:"table:address,field:contract_id,remote_field:id,type:oto,name:contract"` @@ -98,12 +98,14 @@ func (i Invoke) Flat() []any { i.MaxFee, i.Nonce, pq.StringArray(i.CallData), + nil, + i.Error, } - parsed, err := json.MarshalWithOption(i.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil, i.Error) - } else { - data = append(data, string(parsed), i.Error) + if i.ParsedCalldata != nil { + parsed, err := json.MarshalWithOption(i.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[13] = string(parsed) + } } return data diff --git a/internal/storage/l1_handler.go b/internal/storage/l1_handler.go index 5e0e504..d2121ad 100644 --- a/internal/storage/l1_handler.go +++ b/internal/storage/l1_handler.go @@ -44,7 +44,7 @@ type L1Handler struct { MaxFee decimal.Decimal `bun:",type:numeric" comment:"The maximum fee that the sender is willing to pay for the transaction"` Nonce decimal.Decimal `bun:",type:numeric" comment:"The transaction nonce"` CallData []string `bun:",array" comment:"Raw calldata"` - ParsedCalldata map[string]any `comment:"Calldata parsed according to contract ABI"` + ParsedCalldata map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"` Error *string `bun:"error" comment:"Reverted error"` Contract Address `bun:"rel:belongs-to" hasura:"table:address,field:contract_id,remote_field:id,type:oto,name:contract"` @@ -94,13 +94,15 @@ func (l1 L1Handler) Flat() []any { l1.MaxFee, l1.Nonce, pq.StringArray(l1.CallData), + nil, + l1.Error, } - parsed, err := json.MarshalWithOption(l1.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil, l1.Error) - } else { - data = append(data, string(parsed), l1.Error) + if l1.ParsedCalldata != nil { + parsed, err := json.MarshalWithOption(l1.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[12] = string(parsed) + } } return data } diff --git a/pkg/indexer/config/config.go b/pkg/indexer/config/config.go index 7c4ca9d..1364cc5 100644 --- a/pkg/indexer/config/config.go +++ b/pkg/indexer/config/config.go @@ -2,27 +2,11 @@ package config // Config - configuration structure for indexer type Config struct { - Name string `yaml:"name" validate:"omitempty"` - StartLevel uint64 `yaml:"start_level" validate:"omitempty"` - ThreadsCount int `yaml:"threads_count" validate:"omitempty,min=1"` - Timeout uint64 `yaml:"timeout" validate:"omitempty"` - Node *Node `yaml:"node" validate:"omitempty"` - Sequencer Sequencer `yaml:"sequencer" validate:"required"` - ClassInterfacesDir string `yaml:"class_interfaces_dir" validate:"required,dir"` - BridgedTokensFile string `yaml:"bridged_tokens_file" validate:"required,file"` - CacheDir string `yaml:"cache_dir" validate:"omitempty,dir"` - Cache bool `yaml:"cache" validate:"omitempty"` -} - -// Node - -type Node struct { - Url string `yaml:"url" validate:"omitempty,url"` - Rps int `yaml:"requests_per_second" validate:"omitempty,min=1"` -} - -// Sequencer - -type Sequencer struct { - FeederGateway string `yaml:"feeder_gateway" validate:"required,url"` - Gateway string `yaml:"gateway" validate:"required,url"` - Rps int `yaml:"requests_per_second" validate:"omitempty,min=1"` + Name string `yaml:"name" validate:"omitempty"` + StartLevel uint64 `yaml:"start_level" validate:"omitempty"` + ThreadsCount int `yaml:"threads_count" validate:"omitempty,min=1"` + Timeout uint64 `yaml:"timeout" validate:"omitempty"` + ClassInterfacesDir string `yaml:"class_interfaces_dir" validate:"required,dir"` + BridgedTokensFile string `yaml:"bridged_tokens_file" validate:"required,file"` + Datasource string `yaml:"datasource" validate:"required,oneof=sequencer node"` } diff --git a/pkg/indexer/indexer.go b/pkg/indexer/indexer.go index 956c3df..0b3f9e5 100644 --- a/pkg/indexer/indexer.go +++ b/pkg/indexer/indexer.go @@ -8,7 +8,6 @@ import ( "time" "github.com/dipdup-io/starknet-go-api/pkg/data" - "github.com/dipdup-io/starknet-go-api/pkg/sequencer" "github.com/dipdup-io/starknet-indexer/internal/starknet" models "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/internal/storage/postgres" @@ -18,6 +17,7 @@ import ( "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/generator" "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" "github.com/dipdup-io/starknet-indexer/pkg/indexer/store" + ddConfig "github.com/dipdup-net/go-lib/config" "github.com/dipdup-net/indexer-sdk/pkg/modules" sdk "github.com/dipdup-net/indexer-sdk/pkg/storage" "github.com/pkg/errors" @@ -68,7 +68,8 @@ type Indexer struct { func New( cfg config.Config, storage postgres.Storage, -) *Indexer { + datasource map[string]ddConfig.DataSource, +) (*Indexer, error) { indexer := &Indexer{ BaseModule: modules.New("indexer"), cfg: cfg, @@ -87,13 +88,18 @@ func New( proxy: storage.Proxy, state: newState(nil), cache: cache.New(storage.Address, storage.Class, storage.Proxy), - receiver: receiver.NewReceiver(cfg), rollbackManager: storage.RollbackManager, rollback: make(chan struct{}, 1), rollbackRerun: make(chan struct{}, 1), txWriteMutex: new(sync.Mutex), rollbackWait: new(sync.WaitGroup), } + rcvr, err := receiver.NewReceiver(cfg, datasource) + if err != nil { + return nil, err + } + indexer.receiver = rcvr + indexer.CreateOutput(OutputBlocks) indexer.idGenerator = generator.NewIdGenerator(storage.Address, storage.Class, indexer.cache, indexer.state.Current()) @@ -118,7 +124,7 @@ func New( storage.Transactable, ) - return indexer + return indexer, nil } // Start - @@ -301,7 +307,7 @@ func (indexer *Indexer) saveBlocks(ctx context.Context) { return case result := <-indexer.receiver.Results(): - indexer.queue[result.Block.BlockNumber] = result + indexer.queue[result.Block.Height] = result if indexer.state.Height() == 0 && !zeroBlock { if data, ok := indexer.queue[0]; ok { @@ -354,7 +360,7 @@ func (indexer *Indexer) saveBlocks(ctx context.Context) { } } -func (indexer *Indexer) handleReorg(ctx context.Context, newBlock sequencer.Block) (bool, error) { +func (indexer *Indexer) handleReorg(ctx context.Context, newBlock receiver.Block) (bool, error) { lastBlock, err := indexer.blocks.Last(ctx) if err != nil { if indexer.blocks.IsNoRows(err) { @@ -363,13 +369,12 @@ func (indexer *Indexer) handleReorg(ctx context.Context, newBlock sequencer.Bloc return false, err } - parentHash := data.Felt(newBlock.ParentHash).Bytes() - if bytes.Equal(lastBlock.Hash, parentHash) { + if bytes.Equal(lastBlock.Hash, newBlock.ParentHash) { return false, nil } log.Warn(). - Str("parent_hash_of_new_block", newBlock.ParentHash). + Hex("parent_hash_of_new_block", newBlock.ParentHash). Hex("indexer_head_block_hash", lastBlock.Hash). Msg("rollback detected by parent hash") @@ -420,16 +425,16 @@ func (indexer *Indexer) handleBlock(ctx context.Context, result receiver.Result) indexer.statusChecker.addBlock(parseResult.Block) } - delete(indexer.queue, result.Block.BlockNumber) + delete(indexer.queue, result.Block.Height) l := indexer.Log.Info(). - Uint64("height", result.Block.BlockNumber). + Uint64("height", result.Block.Height). Int("tx_count", parseResult.Block.TxCount). Time("block_time", parseResult.Block.Time). Int64("process_time_in_ms", time.Since(start).Milliseconds()). Int64("save_time_in_ms", saveTime) - if result.Block.StarknetVersion != nil { - l.Str("version", *result.Block.StarknetVersion) + if result.Block.Version != nil && *result.Block.Version != "" { + l.Str("version", *result.Block.Version) } l.Msg("indexed") diff --git a/pkg/indexer/parser/interfaces/interfaces.go b/pkg/indexer/parser/interfaces/interfaces.go index 3f56e3a..13d8124 100644 --- a/pkg/indexer/parser/interfaces/interfaces.go +++ b/pkg/indexer/parser/interfaces/interfaces.go @@ -8,17 +8,18 @@ import ( "github.com/dipdup-io/starknet-go-api/pkg/sequencer" "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" ) // Parser - type Parser interface { - ParseDeclare(ctx context.Context, version starknetData.Felt, raw *starknetData.Declare, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Declare, *storage.Fee, error) - ParseDeployAccount(ctx context.Context, raw *starknetData.DeployAccount, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.DeployAccount, *storage.Fee, error) - ParseDeploy(ctx context.Context, raw *starknetData.Deploy, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Deploy, *storage.Fee, error) - ParseInvokeV0(ctx context.Context, raw *starknetData.Invoke, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Invoke, *storage.Fee, error) - ParseInvokeV1(ctx context.Context, raw *starknetData.Invoke, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Invoke, *storage.Fee, error) - ParseInvokeV3(ctx context.Context, raw *starknetData.Invoke, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Invoke, *storage.Fee, error) - ParseL1Handler(ctx context.Context, raw *starknetData.L1Handler, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.L1Handler, *storage.Fee, error) + ParseDeclare(ctx context.Context, version starknetData.Felt, raw *starknetData.Declare, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Declare, *storage.Fee, error) + ParseDeployAccount(ctx context.Context, raw *starknetData.DeployAccount, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.DeployAccount, *storage.Fee, error) + ParseDeploy(ctx context.Context, raw *starknetData.Deploy, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Deploy, *storage.Fee, error) + ParseInvokeV0(ctx context.Context, raw *starknetData.Invoke, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Invoke, *storage.Fee, error) + ParseInvokeV1(ctx context.Context, raw *starknetData.Invoke, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Invoke, *storage.Fee, error) + ParseInvokeV3(ctx context.Context, raw *starknetData.Invoke, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Invoke, *storage.Fee, error) + ParseL1Handler(ctx context.Context, raw *starknetData.L1Handler, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.L1Handler, *storage.Fee, error) } // InternalTxParser - diff --git a/pkg/indexer/parser/parser.go b/pkg/indexer/parser/parser.go index 6a5e741..d8b0639 100644 --- a/pkg/indexer/parser/parser.go +++ b/pkg/indexer/parser/parser.go @@ -2,11 +2,8 @@ package parser import ( "context" - "time" - "github.com/dipdup-io/starknet-go-api/pkg/data" starknetData "github.com/dipdup-io/starknet-go-api/pkg/data" - "github.com/dipdup-io/starknet-go-api/pkg/encoding" "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/pkg/indexer/cache" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" @@ -24,7 +21,7 @@ func createParser( cache *cache.Cache, blocks storage.IBlock, ) (interfaces.Parser, error) { - if version == nil { + if version == nil || *version == "" { return v0.New(resolver, cache, blocks), nil } @@ -52,15 +49,15 @@ func Parse( result receiver.Result, ) (parserData.Result, error) { block := storage.Block{ - ID: result.Block.BlockNumber + 1, - Height: result.Block.BlockNumber, - Time: time.Unix(result.Block.Timestamp, 0).UTC(), - Hash: data.Felt(result.Block.BlockHash).Bytes(), - ParentHash: data.Felt(result.Block.ParentHash).Bytes(), - NewRoot: encoding.MustDecodeHex(result.Block.NewRoot), - SequencerAddress: encoding.MustDecodeHex(result.Block.SequencerAddress), - Version: result.Block.StarknetVersion, - Status: storage.NewStatus(result.Block.Status), + ID: result.Block.Height + 1, + Height: result.Block.Height, + Time: result.Block.Time, + Hash: result.Block.Hash, + ParentHash: result.Block.ParentHash, + NewRoot: result.Block.NewRoot, + SequencerAddress: result.Block.SequencerAddress, + Version: result.Block.Version, + Status: result.Block.Status, TxCount: len(result.Block.Transactions), Invoke: make([]storage.Invoke, 0), @@ -70,10 +67,7 @@ func Parse( L1Handler: make([]storage.L1Handler, 0), } - if len(result.Block.Transactions) != len(result.Trace.Traces) { - return parserData.Result{}, errors.Errorf("invalid data length") - } - if len(result.Block.Transactions) != len(result.Block.Receipts) { + if len(result.Block.Transactions) != len(result.Traces) { return parserData.Result{}, errors.Errorf("invalid data length") } @@ -100,16 +94,16 @@ func Parse( ) switch result.Block.Transactions[i].Version { case starknetData.Version0: - invoke, fee, err = p.ParseInvokeV0(ctx, typed, block, result.Trace.Traces[i], result.Block.Receipts[i]) + invoke, fee, err = p.ParseInvokeV0(ctx, typed, block, result.Block.Transactions[i], result.Traces[i]) case starknetData.Version1: - invoke, fee, err = p.ParseInvokeV1(ctx, typed, block, result.Trace.Traces[i], result.Block.Receipts[i]) + invoke, fee, err = p.ParseInvokeV1(ctx, typed, block, result.Block.Transactions[i], result.Traces[i]) case starknetData.Version3: - invoke, fee, err = p.ParseInvokeV3(ctx, typed, block, result.Trace.Traces[i], result.Block.Receipts[i]) + invoke, fee, err = p.ParseInvokeV3(ctx, typed, block, result.Block.Transactions[i], result.Traces[i]) default: return parserData.Result{}, errors.Errorf("unknown invoke version: %s", result.Block.Transactions[i].Version) } if err != nil { - return parserData.Result{}, errors.Wrapf(err, "%s invoke version=%s", result.Block.Transactions[i].TransactionHash, result.Block.Transactions[i].Version) + return parserData.Result{}, errors.Wrapf(err, "%s invoke version=%s", result.Block.Transactions[i].Hash, result.Block.Transactions[i].Version) } invoke.Position = i block.Invoke = append(block.Invoke, invoke) @@ -117,9 +111,9 @@ func Parse( block.Fee = append(block.Fee, *fee) } case *starknetData.Declare: - tx, fee, err := p.ParseDeclare(ctx, result.Block.Transactions[i].Version, typed, block, result.Trace.Traces[i], result.Block.Receipts[i]) + tx, fee, err := p.ParseDeclare(ctx, result.Block.Transactions[i].Version, typed, block, result.Block.Transactions[i], result.Traces[i]) if err != nil { - return parserData.Result{}, errors.Wrapf(err, "%s declare", result.Block.Transactions[i].TransactionHash) + return parserData.Result{}, errors.Wrapf(err, "%s declare", result.Block.Transactions[i].Hash) } tx.Position = i block.Declare = append(block.Declare, tx) @@ -127,9 +121,9 @@ func Parse( block.Fee = append(block.Fee, *fee) } case *starknetData.Deploy: - tx, fee, err := p.ParseDeploy(ctx, typed, block, result.Trace.Traces[i], result.Block.Receipts[i]) + tx, fee, err := p.ParseDeploy(ctx, typed, block, result.Block.Transactions[i], result.Traces[i]) if err != nil { - return parserData.Result{}, errors.Wrapf(err, "%s deploy", result.Block.Transactions[i].TransactionHash) + return parserData.Result{}, errors.Wrapf(err, "%s deploy", result.Block.Transactions[i].Hash) } tx.Position = i block.Deploy = append(block.Deploy, tx) @@ -137,9 +131,9 @@ func Parse( block.Fee = append(block.Fee, *fee) } case *starknetData.DeployAccount: - tx, fee, err := p.ParseDeployAccount(ctx, typed, block, result.Trace.Traces[i], result.Block.Receipts[i]) + tx, fee, err := p.ParseDeployAccount(ctx, typed, block, result.Block.Transactions[i], result.Traces[i]) if err != nil { - return parserData.Result{}, errors.Wrapf(err, "%s deploy account", result.Block.Transactions[i].TransactionHash) + return parserData.Result{}, errors.Wrapf(err, "%s deploy account", result.Block.Transactions[i].Hash) } tx.Position = i block.DeployAccount = append(block.DeployAccount, tx) @@ -147,9 +141,9 @@ func Parse( block.Fee = append(block.Fee, *fee) } case *starknetData.L1Handler: - tx, fee, err := p.ParseL1Handler(ctx, typed, block, result.Trace.Traces[i], result.Block.Receipts[i]) + tx, fee, err := p.ParseL1Handler(ctx, typed, block, result.Block.Transactions[i], result.Traces[i]) if err != nil { - return parserData.Result{}, errors.Wrapf(err, "%s l1 handler", result.Block.Transactions[i].TransactionHash) + return parserData.Result{}, errors.Wrapf(err, "%s l1 handler", result.Block.Transactions[i].Hash) } tx.Position = i block.L1Handler = append(block.L1Handler, tx) diff --git a/pkg/indexer/parser/version/v0/declare.go b/pkg/indexer/parser/version/v0/declare.go index aac233a..85d8c7e 100644 --- a/pkg/indexer/parser/version/v0/declare.go +++ b/pkg/indexer/parser/version/v0/declare.go @@ -7,10 +7,11 @@ import ( "github.com/dipdup-io/starknet-go-api/pkg/sequencer" "github.com/dipdup-io/starknet-indexer/internal/storage" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" ) // ParseDeclare - -func (parser Parser) ParseDeclare(ctx context.Context, version data.Felt, raw *data.Declare, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Declare, *storage.Fee, error) { +func (parser Parser) ParseDeclare(ctx context.Context, version data.Felt, raw *data.Declare, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Declare, *storage.Fee, error) { tx := storage.Declare{ ID: parser.Resolver.NextTxId(), Height: block.Height, @@ -102,7 +103,7 @@ func (parser Parser) ParseDeclare(ctx context.Context, version data.Felt, raw *d return tx, fee, nil } } else { - transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receipts.ActualFee) + transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receiverTx.ActualFee) if err != nil { return tx, nil, nil } diff --git a/pkg/indexer/parser/version/v0/deploy.go b/pkg/indexer/parser/version/v0/deploy.go index 1b08312..bb95713 100644 --- a/pkg/indexer/parser/version/v0/deploy.go +++ b/pkg/indexer/parser/version/v0/deploy.go @@ -11,10 +11,11 @@ import ( "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/pkg/indexer/decode" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" ) // ParseDeploy - -func (parser Parser) ParseDeploy(ctx context.Context, raw *data.Deploy, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Deploy, *storage.Fee, error) { +func (parser Parser) ParseDeploy(ctx context.Context, raw *data.Deploy, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Deploy, *storage.Fee, error) { tx := storage.Deploy{ ID: parser.Resolver.NextTxId(), Height: block.Height, @@ -102,7 +103,7 @@ func (parser Parser) ParseDeploy(ctx context.Context, raw *data.Deploy, block st return tx, fee, nil } } else { - transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receipts.ActualFee) + transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receiverTx.ActualFee) if err != nil { return tx, nil, nil } diff --git a/pkg/indexer/parser/version/v0/deploy_account.go b/pkg/indexer/parser/version/v0/deploy_account.go index ab9f93e..b5ededc 100644 --- a/pkg/indexer/parser/version/v0/deploy_account.go +++ b/pkg/indexer/parser/version/v0/deploy_account.go @@ -11,11 +11,12 @@ import ( "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/pkg/indexer/decode" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" "github.com/goccy/go-json" ) // ParseDeployAccount - -func (parser Parser) ParseDeployAccount(ctx context.Context, raw *data.DeployAccount, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.DeployAccount, *storage.Fee, error) { +func (parser Parser) ParseDeployAccount(ctx context.Context, raw *data.DeployAccount, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.DeployAccount, *storage.Fee, error) { tx := storage.DeployAccount{ ID: parser.Resolver.NextTxId(), Height: block.Height, @@ -107,7 +108,7 @@ func (parser Parser) ParseDeployAccount(ctx context.Context, raw *data.DeployAcc return tx, fee, nil } } else { - transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receipts.ActualFee) + transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receiverTx.ActualFee) if err != nil { return tx, nil, nil } diff --git a/pkg/indexer/parser/version/v0/events.go b/pkg/indexer/parser/version/v0/events.go index d92d807..1219f65 100644 --- a/pkg/indexer/parser/version/v0/events.go +++ b/pkg/indexer/parser/version/v0/events.go @@ -65,6 +65,14 @@ func (parser EventParser) Parse(ctx context.Context, txCtx data.TxContext, contr model.Keys[i] = event.Keys[i].String() } + for i := 0; i < len(event.Data); i++ { + model.Data[i] = event.Data[i].String() + } + + for i := 0; i < len(event.Keys); i++ { + model.Keys[i] = event.Keys[i].String() + } + if address, err := parser.resolver.FindAddressByHash(ctx, starknetData.Felt(event.FromAddress)); err != nil { return model, err } else if address != nil { diff --git a/pkg/indexer/parser/version/v0/fee.go b/pkg/indexer/parser/version/v0/fee.go index cb488b9..885417e 100644 --- a/pkg/indexer/parser/version/v0/fee.go +++ b/pkg/indexer/parser/version/v0/fee.go @@ -132,12 +132,12 @@ func (parser FeeParser) ParseInvocation(ctx context.Context, txCtx data.TxContex Messages: make([]storage.Message, 0), Internals: make([]storage.Internal, 0), } - for i := range feeInvocation.Result { - tx.Result[i] = feeInvocation.Result[i].String() - } - for i := range feeInvocation.Calldata { + for i := 0; i < len(feeInvocation.Calldata); i++ { tx.Calldata[i] = feeInvocation.Calldata[i].String() } + for i := 0; i < len(feeInvocation.Result); i++ { + tx.Result[i] = feeInvocation.Result[i].String() + } if class, err := parser.resolver.FindClassByHash(ctx, feeInvocation.ClassHash, tx.Height); err != nil { return nil, err @@ -213,7 +213,7 @@ func (parser FeeParser) ParseInvocation(ctx context.Context, txCtx data.TxContex } } - if len(feeInvocation.Calldata) > 0 && len(tx.Selector) > 0 { + if len(tx.Calldata) > 0 && len(tx.Selector) > 0 { if isExecute && !hasExecute { tx.Entrypoint = encoding.ExecuteEntrypoint tx.ParsedCalldata, err = abi.DecodeExecuteCallData(tx.Calldata) diff --git a/pkg/indexer/parser/version/v0/internal_tx.go b/pkg/indexer/parser/version/v0/internal_tx.go index eedfee7..ec9133d 100644 --- a/pkg/indexer/parser/version/v0/internal_tx.go +++ b/pkg/indexer/parser/version/v0/internal_tx.go @@ -75,13 +75,14 @@ func (parser InternalTxParser) Parse(ctx context.Context, txCtx parserData.TxCon Messages: make([]storage.Message, 0), Internals: make([]storage.Internal, 0), } - for i := range internal.Result { - tx.Result[i] = internal.Result[i].String() - } - for i := range internal.Calldata { + for i := 0; i < len(internal.Calldata); i++ { tx.Calldata[i] = internal.Calldata[i].String() } + for i := 0; i < len(internal.Result); i++ { + tx.Result[i] = internal.Result[i].String() + } + if class, err := parser.Resolver.FindClassByHash(ctx, internal.ClassHash, tx.Height); err != nil { return tx, err } else if class != nil { @@ -169,7 +170,7 @@ func (parser InternalTxParser) Parse(ctx context.Context, txCtx parserData.TxCon } } - if len(internal.Calldata) > 0 && !isUnknownProxy { + if len(tx.Calldata) > 0 && !isUnknownProxy { switch { case isExecute && !has: tx.Entrypoint = encoding.ExecuteEntrypoint diff --git a/pkg/indexer/parser/version/v0/invoke_v0.go b/pkg/indexer/parser/version/v0/invoke_v0.go index 67f461b..d3f1a6e 100644 --- a/pkg/indexer/parser/version/v0/invoke_v0.go +++ b/pkg/indexer/parser/version/v0/invoke_v0.go @@ -13,10 +13,11 @@ import ( "github.com/dipdup-io/starknet-indexer/pkg/indexer/decode" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/helpers" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" ) // ParseInvokeV0 - -func (parser Parser) ParseInvokeV0(ctx context.Context, raw *data.Invoke, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Invoke, *storage.Fee, error) { +func (parser Parser) ParseInvokeV0(ctx context.Context, raw *data.Invoke, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Invoke, *storage.Fee, error) { tx := storage.Invoke{ ID: parser.Resolver.NextTxId(), Height: block.Height, @@ -161,7 +162,7 @@ func (parser Parser) ParseInvokeV0(ctx context.Context, raw *data.Invoke, block return tx, fee, nil } } else { - transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receipts.ActualFee) + transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receiverTx.ActualFee) if err != nil { return tx, nil, nil } diff --git a/pkg/indexer/parser/version/v0/invoke_v1.go b/pkg/indexer/parser/version/v0/invoke_v1.go index 2d4010b..a771787 100644 --- a/pkg/indexer/parser/version/v0/invoke_v1.go +++ b/pkg/indexer/parser/version/v0/invoke_v1.go @@ -10,11 +10,12 @@ import ( "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/pkg/indexer/decode" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" "github.com/pkg/errors" ) // ParseInvokeV1 - -func (parser Parser) ParseInvokeV1(ctx context.Context, raw *data.Invoke, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Invoke, *storage.Fee, error) { +func (parser Parser) ParseInvokeV1(ctx context.Context, raw *data.Invoke, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Invoke, *storage.Fee, error) { tx := storage.Invoke{ ID: parser.Resolver.NextTxId(), Height: block.Height, @@ -138,7 +139,7 @@ func (parser Parser) ParseInvokeV1(ctx context.Context, raw *data.Invoke, block return tx, fee, nil } } else { - transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receipts.ActualFee) + transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receiverTx.ActualFee) if err != nil { return tx, nil, nil } diff --git a/pkg/indexer/parser/version/v0/invoke_v3.go b/pkg/indexer/parser/version/v0/invoke_v3.go index 34359de..e24808c 100644 --- a/pkg/indexer/parser/version/v0/invoke_v3.go +++ b/pkg/indexer/parser/version/v0/invoke_v3.go @@ -10,11 +10,12 @@ import ( "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/pkg/indexer/decode" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" "github.com/pkg/errors" ) // ParseInvokeV1 - -func (parser Parser) ParseInvokeV3(ctx context.Context, raw *data.Invoke, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Invoke, *storage.Fee, error) { +func (parser Parser) ParseInvokeV3(ctx context.Context, raw *data.Invoke, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Invoke, *storage.Fee, error) { tx := storage.Invoke{ ID: parser.Resolver.NextTxId(), Height: block.Height, @@ -138,7 +139,7 @@ func (parser Parser) ParseInvokeV3(ctx context.Context, raw *data.Invoke, block return tx, fee, nil } } else { - transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receipts.ActualFee) + transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receiverTx.ActualFee) if err != nil { return tx, nil, nil } diff --git a/pkg/indexer/parser/version/v0/l1_handler.go b/pkg/indexer/parser/version/v0/l1_handler.go index d6d5902..0ca9890 100644 --- a/pkg/indexer/parser/version/v0/l1_handler.go +++ b/pkg/indexer/parser/version/v0/l1_handler.go @@ -12,10 +12,11 @@ import ( "github.com/dipdup-io/starknet-indexer/pkg/indexer/decode" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/helpers" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" ) // ParseL1Handler - -func (parser Parser) ParseL1Handler(ctx context.Context, raw *data.L1Handler, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.L1Handler, *storage.Fee, error) { +func (parser Parser) ParseL1Handler(ctx context.Context, raw *data.L1Handler, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.L1Handler, *storage.Fee, error) { tx := storage.L1Handler{ ID: parser.Resolver.NextTxId(), Height: block.Height, @@ -109,7 +110,7 @@ func (parser Parser) ParseL1Handler(ctx context.Context, raw *data.L1Handler, bl return tx, fee, nil } } else { - transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receipts.ActualFee) + transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receiverTx.ActualFee) if err != nil { return tx, nil, nil } diff --git a/pkg/indexer/parser/version/v0/message.go b/pkg/indexer/parser/version/v0/message.go index 59ef11e..00cc394 100644 --- a/pkg/indexer/parser/version/v0/message.go +++ b/pkg/indexer/parser/version/v0/message.go @@ -42,7 +42,7 @@ func (parser MessageParser) Parse(ctx context.Context, txCtx parserData.TxContex FeeID: txCtx.FeeID, InternalID: txCtx.InternalID, } - for i := range msg.Payload { + for i := 0; i < len(msg.Payload); i++ { message.Payload[i] = msg.Payload[i].String() } if txCtx.ProxyId > 0 { diff --git a/pkg/indexer/receiver/api.go b/pkg/indexer/receiver/api.go new file mode 100644 index 0000000..b82091d --- /dev/null +++ b/pkg/indexer/receiver/api.go @@ -0,0 +1,44 @@ +package receiver + +import ( + "context" + "time" + + "github.com/dipdup-io/starknet-go-api/pkg/data" + starknetData "github.com/dipdup-io/starknet-go-api/pkg/data" + starknet "github.com/dipdup-io/starknet-go-api/pkg/sequencer" + "github.com/dipdup-io/starknet-indexer/internal/storage" +) + +type API interface { + GetBlock(ctx context.Context, blockId starknetData.BlockID) (Block, error) + TraceBlock(ctx context.Context, blockId starknetData.BlockID) ([]starknet.Trace, error) + GetStateUpdate(ctx context.Context, blockId starknetData.BlockID) (starknetData.StateUpdate, error) + GetBlockStatus(ctx context.Context, height uint64) (storage.Status, error) + TransactionStatus(ctx context.Context, hash string) (storage.Status, error) + GetClass(ctx context.Context, hash string) (starknetData.Class, error) + Head(ctx context.Context) (uint64, error) +} + +type Block struct { + Height uint64 + Time time.Time + Version *string + Status storage.Status + Hash []byte + ParentHash []byte + NewRoot []byte + SequencerAddress []byte + + Transactions []Transaction + Receipts []starknet.Receipt +} + +type Transaction struct { + Type string + Version data.Felt + Hash data.Felt + ActualFee data.Felt + + Body any +} diff --git a/pkg/indexer/receiver/feeder.go b/pkg/indexer/receiver/feeder.go new file mode 100644 index 0000000..74e9fc6 --- /dev/null +++ b/pkg/indexer/receiver/feeder.go @@ -0,0 +1,109 @@ +package receiver + +import ( + "context" + "time" + + "github.com/dipdup-io/starknet-go-api/pkg/data" + starknetData "github.com/dipdup-io/starknet-go-api/pkg/data" + "github.com/dipdup-io/starknet-go-api/pkg/encoding" + starknet "github.com/dipdup-io/starknet-go-api/pkg/sequencer" + "github.com/dipdup-io/starknet-indexer/internal/storage" + "github.com/dipdup-net/go-lib/config" + "github.com/pkg/errors" +) + +type Feeder struct { + api starknet.API +} + +func NewFeeder(cfg config.DataSource) *Feeder { + opts := make([]starknet.ApiOption, 0) + if cfg.RequestsPerSecond > 0 { + opts = append(opts, starknet.WithRateLimit(cfg.RequestsPerSecond)) + } + + return &Feeder{ + api: starknet.NewAPI("", cfg.URL, opts...), + } +} + +func (f *Feeder) GetBlock(ctx context.Context, blockId starknetData.BlockID) (block Block, err error) { + response, err := f.api.GetBlock(ctx, blockId, false) + if err != nil { + return block, err + } + + block.Height = response.BlockNumber + block.Time = time.Unix(response.Timestamp, 0).UTC() + block.Hash = data.Felt(response.BlockHash).Bytes() + block.ParentHash = data.Felt(response.ParentHash).Bytes() + block.NewRoot = encoding.MustDecodeHex(response.NewRoot) + block.SequencerAddress = encoding.MustDecodeHex(response.SequencerAddress) + block.Version = response.StarknetVersion + block.Status = storage.NewStatus(response.Status) + block.Receipts = response.Receipts + + if len(response.Transactions) != len(response.Receipts) { + return block, errors.Errorf("length arrays of txs and receipts are differ") + } + + block.Transactions = make([]Transaction, len(response.Transactions)) + + for i := range response.Transactions { + block.Transactions[i].Hash = response.Transactions[i].TransactionHash + block.Transactions[i].Type = response.Transactions[i].Type + block.Transactions[i].Version = response.Transactions[i].Version + block.Transactions[i].Body = response.Transactions[i].Body + block.Transactions[i].ActualFee = response.Receipts[i].ActualFee + } + + return +} + +func (f *Feeder) TraceBlock(ctx context.Context, blockId starknetData.BlockID) (traces []starknet.Trace, err error) { + response, err := f.api.TraceBlock(ctx, blockId) + if err != nil { + return + } + return response.Traces, nil +} + +func (f *Feeder) GetStateUpdate(ctx context.Context, blockId starknetData.BlockID) (response starknetData.StateUpdate, err error) { + return f.api.GetStateUpdate(ctx, blockId) +} + +func (f *Feeder) GetBlockStatus(ctx context.Context, height uint64) (storage.Status, error) { + response, err := f.api.GetBlock(ctx, starknetData.BlockID{Number: &height}, false) + if err != nil { + return storage.StatusUnknown, err + } + return storage.NewStatus(response.Status), nil +} + +func (f *Feeder) TransactionStatus(ctx context.Context, hash string) (storage.Status, error) { + response, err := f.api.GetTransactionStatus(ctx, hash) + if err != nil { + return storage.StatusUnknown, err + } + + return storage.NewStatus(response.Status), nil +} + +func (f *Feeder) GetClass(ctx context.Context, hash string) (starknetData.Class, error) { + blockId := starknetData.BlockID{ + String: starknetData.Latest, + } + + return f.api.GetClassByHash(ctx, blockId, hash) +} + +func (f *Feeder) Head(ctx context.Context) (uint64, error) { + response, err := f.api.GetBlock(ctx, starknetData.BlockID{ + String: starknetData.Latest, + }, true) + if err != nil { + return 0, err + } + return response.BlockNumber, nil +} diff --git a/pkg/indexer/receiver/node.go b/pkg/indexer/receiver/node.go new file mode 100644 index 0000000..469aa5e --- /dev/null +++ b/pkg/indexer/receiver/node.go @@ -0,0 +1,180 @@ +package receiver + +import ( + "context" + "os" + "time" + + "github.com/dipdup-io/starknet-go-api/pkg/data" + starknetData "github.com/dipdup-io/starknet-go-api/pkg/data" + "github.com/dipdup-io/starknet-go-api/pkg/encoding" + starknet "github.com/dipdup-io/starknet-go-api/pkg/rpc" + "github.com/dipdup-io/starknet-go-api/pkg/sequencer" + "github.com/dipdup-io/starknet-indexer/internal/storage" + "github.com/dipdup-net/go-lib/config" + "github.com/pkg/errors" +) + +type Node struct { + api starknet.API +} + +func NewNode(cfg config.DataSource) *Node { + apiKey := os.Getenv("NODE_APIKEY") + headerName := os.Getenv("NODE_HEADER_APIKEY") + return &Node{ + api: starknet.NewAPI( + cfg.URL, + starknet.WithRateLimit(cfg.RequestsPerSecond), + starknet.WithApiKey(headerName, apiKey), + ), + } +} + +func (n *Node) GetBlock(ctx context.Context, blockId starknetData.BlockID) (block Block, err error) { + response, err := n.api.GetBlockWithReceipts(ctx, blockId) + if err != nil { + return + } + + block.Height = response.Result.BlockNumber + block.Time = time.Unix(response.Result.Timestamp, 0).UTC() + block.Hash = data.Felt(response.Result.BlockHash).Bytes() + block.ParentHash = data.Felt(response.Result.ParentHash).Bytes() + block.NewRoot = encoding.MustDecodeHex(response.Result.NewRoot) + block.SequencerAddress = encoding.MustDecodeHex(response.Result.SequencerAddress) + block.Version = response.Result.Version + block.Status = storage.NewStatus(response.Result.Status) + block.Transactions = make([]Transaction, len(response.Result.Transactions)) + + for i := range response.Result.Transactions { + block.Transactions[i].Hash = response.Result.Transactions[i].Transaction.TransactionHash + block.Transactions[i].Type = response.Result.Transactions[i].Transaction.Type + block.Transactions[i].Version = response.Result.Transactions[i].Transaction.Version + block.Transactions[i].Body = response.Result.Transactions[i].Transaction.Body + block.Transactions[i].ActualFee = response.Result.Transactions[i].Receipt.ActualFee.Amount + + switch block.Transactions[i].Type { + case starknetData.TransactionTypeDeploy: + if deploy, ok := block.Transactions[i].Body.(*starknetData.Deploy); ok { + deploy.ContractAddress = starknetData.Felt(response.Result.Transactions[i].Receipt.ContractAddress) + } else { + return block, errors.Errorf("invalid invoke transaction type: expected Deploy (non-pointer)") + } + case starknetData.TransactionTypeDeployAccount: + if deploy, ok := block.Transactions[i].Body.(*starknetData.DeployAccount); ok { + deploy.ContractAddress = starknetData.Felt(response.Result.Transactions[i].Receipt.ContractAddress) + } else { + return block, errors.Errorf("invalid invoke transaction type: expected DeployAccount (non-pointer)") + } + default: + continue + } + } + + return +} + +func (n *Node) TraceBlock(ctx context.Context, block starknetData.BlockID) (traces []sequencer.Trace, err error) { + response, err := n.api.Trace(ctx, block) + if err != nil { + return + } + + traces = make([]sequencer.Trace, len(response.Result)) + for i := range response.Result { + if inv := response.Result[i].TraceRoot.ExecuteInvocation; inv != nil { + if inv.RevertReason != "" { + traces[i].RevertedError = inv.RevertReason + } else { + traces[i].FunctionInvocation = makeSeqInvocationFromNodeCall(inv) + } + } + + if inv := response.Result[i].TraceRoot.ConstructorInvocation; inv != nil { + if inv.RevertReason != "" { + traces[i].RevertedError = inv.RevertReason + } else { + traces[i].FunctionInvocation = makeSeqInvocationFromNodeCall(inv) + } + } + + traces[i].ValidateInvocation = makeSeqInvocationFromNodeCall(response.Result[i].TraceRoot.ValidateInvocation) + traces[i].FeeTransferInvocation = makeSeqInvocationFromNodeCall(response.Result[i].TraceRoot.FeeTransferInvocation) + traces[i].TransactionHash = response.Result[i].TransactionHash + } + + return +} + +func makeSeqInvocationFromNodeCall(call *starknet.Call) *sequencer.Invocation { + if call == nil { + return nil + } + + inv := &sequencer.Invocation{ + CallerAddress: call.CallerAddress, + ContractAddress: call.ContractAddress, + Calldata: call.Calldata, + CallType: call.CallType, + ClassHash: call.ClassHash, + Selector: call.EntryPointSelector, + EntrypointType: call.EntryPointType, + Result: call.Result, + Events: call.Events, + Messages: call.Messages, + InternalCalls: make([]sequencer.Invocation, len(call.Calls)), + } + + for i := range call.Calls { + internalCall := makeSeqInvocationFromNodeCall(&call.Calls[i]) + inv.InternalCalls[i] = *internalCall + } + + return inv +} + +func (n *Node) GetStateUpdate(ctx context.Context, block starknetData.BlockID) (starknetData.StateUpdate, error) { + response, err := n.api.GetStateUpdate(ctx, block) + if err != nil { + return starknetData.StateUpdate{}, err + } + return response.Result.ToStateUpdate(), nil +} + +func (n *Node) GetBlockStatus(ctx context.Context, height uint64) (storage.Status, error) { + response, err := n.api.GetBlockWithTxHashes(ctx, starknetData.BlockID{Number: &height}) + if err != nil { + return storage.StatusUnknown, err + } + return storage.NewStatus(response.Result.Status), nil +} + +func (n *Node) TransactionStatus(ctx context.Context, hash string) (storage.Status, error) { + response, err := n.api.GetTransactionStatus(ctx, hash) + if err != nil { + return storage.StatusUnknown, err + } + + return storage.NewStatus(response.Result.Finality), nil +} + +func (n *Node) GetClass(ctx context.Context, hash string) (starknetData.Class, error) { + blockId := starknetData.BlockID{ + String: starknetData.Latest, + } + + response, err := n.api.GetClass(ctx, blockId, hash) + if err != nil { + return starknetData.Class{}, err + } + return response.Result, nil +} + +func (n *Node) Head(ctx context.Context) (uint64, error) { + response, err := n.api.BlockNumber(ctx) + if err != nil { + return 0, err + } + return response.Result, nil +} diff --git a/pkg/indexer/receiver/receiver.go b/pkg/indexer/receiver/receiver.go index bc29461..15fe4c2 100644 --- a/pkg/indexer/receiver/receiver.go +++ b/pkg/indexer/receiver/receiver.go @@ -2,31 +2,30 @@ package receiver import ( "context" - "errors" "sync" "time" starknetData "github.com/dipdup-io/starknet-go-api/pkg/data" - starknetRpc "github.com/dipdup-io/starknet-go-api/pkg/rpc" starknet "github.com/dipdup-io/starknet-go-api/pkg/sequencer" "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/pkg/indexer/config" "github.com/dipdup-io/workerpool" + ddConfig "github.com/dipdup-net/go-lib/config" + "github.com/pkg/errors" "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) // Result - type Result struct { - Block starknet.Block - Trace starknet.TraceResponse + Block Block + Traces []starknet.Trace StateUpdate starknetData.StateUpdate } // Receiver - type Receiver struct { - api starknet.API - rpc *starknetRpc.API + api API result chan Result pool *workerpool.Pool[uint64] processing map[uint64]struct{} @@ -37,19 +36,20 @@ type Receiver struct { } // NewReceiver - -func NewReceiver(cfg config.Config) *Receiver { - opts := make([]starknet.ApiOption, 0) - if cfg.Sequencer.Rps > 0 { - opts = append(opts, starknet.WithRateLimit(cfg.Sequencer.Rps)) +func NewReceiver(cfg config.Config, ds map[string]ddConfig.DataSource) (*Receiver, error) { + dsCfg, ok := ds[cfg.Datasource] + if !ok { + return nil, errors.Errorf("unknown datasource name: %s", cfg.Datasource) } - log.Info().Bool("enabled", cfg.Cache).Str("dir", cfg.CacheDir).Msg("rpc response caching") - if cfg.Cache && cfg.CacheDir != "" { - opts = append(opts, starknet.WithCacheInFS(cfg.CacheDir)) + var api API + switch cfg.Datasource { + case "node": + api = NewNode(dsCfg) + case "sequencer": + api = NewFeeder(dsCfg) } - api := starknet.NewAPI(cfg.Sequencer.Gateway, cfg.Sequencer.FeederGateway, opts...) - receiver := &Receiver{ api: api, result: make(chan Result, cfg.ThreadsCount*2), @@ -60,18 +60,13 @@ func NewReceiver(cfg config.Config) *Receiver { wg: new(sync.WaitGroup), } - if cfg.Node != nil && cfg.Node.Url != "" { - rpc := starknetRpc.NewAPI(cfg.Node.Url, starknetRpc.WithRateLimit(cfg.Node.Rps)) - receiver.rpc = &rpc - } - if receiver.timeout == 0 { receiver.timeout = 10 * time.Second } receiver.pool = workerpool.NewPool(receiver.worker, cfg.ThreadsCount) - return receiver + return receiver, nil } // Close - @@ -105,7 +100,7 @@ func (r *Receiver) worker(ctx context.Context, height uint64) { default: } - response, err := r.api.GetBlock(ctx, blockId, false) + response, err := r.api.GetBlock(ctx, blockId) if err != nil { if errors.Is(err, context.Canceled) { return @@ -134,7 +129,7 @@ func (r *Receiver) worker(ctx context.Context, height uint64) { time.Sleep(time.Second) continue } - result.Trace = response + result.Traces = response break } @@ -185,21 +180,7 @@ func (r *Receiver) Head(ctx context.Context) (uint64, error) { requestCtx, cancel := context.WithTimeout(ctx, r.timeout) defer cancel() - if r.rpc != nil { - response, err := r.rpc.BlockNumber(requestCtx) - if err != nil { - return 0, err - } - return response.Result, nil - } - - response, err := r.api.GetBlock(requestCtx, starknetData.BlockID{ - String: starknetData.Latest, - }, true) - if err != nil { - return 0, err - } - return response.BlockNumber, nil + return r.api.Head(requestCtx) } // GetClass - @@ -207,11 +188,7 @@ func (r *Receiver) GetClass(ctx context.Context, hash string) (starknetData.Clas requestCtx, cancel := context.WithTimeout(ctx, r.timeout) defer cancel() - blockId := starknetData.BlockID{ - String: starknetData.Latest, - } - - return r.api.GetClassByHash(requestCtx, blockId, hash) + return r.api.GetClass(requestCtx, hash) } // TransactionStatus - @@ -219,12 +196,7 @@ func (r *Receiver) TransactionStatus(ctx context.Context, hash string) (storage. requestCtx, cancel := context.WithTimeout(ctx, r.timeout) defer cancel() - response, err := r.api.GetTransactionStatus(requestCtx, hash) - if err != nil { - return storage.StatusUnknown, err - } - - return storage.NewStatus(response.Status), nil + return r.api.TransactionStatus(requestCtx, hash) } // GetBlockStatus - @@ -232,23 +204,7 @@ func (r *Receiver) GetBlockStatus(ctx context.Context, height uint64) (storage.S requestCtx, cancel := context.WithTimeout(ctx, r.timeout) defer cancel() - blockId := starknetData.BlockID{ - Number: &height, - } - - if r.rpc != nil { - response, err := r.rpc.GetBlockWithTxHashes(requestCtx, blockId) - if err != nil { - return storage.StatusUnknown, err - } - return storage.NewStatus(response.Result.Status), nil - } - - response, err := r.api.GetBlock(requestCtx, blockId, false) - if err != nil { - return storage.StatusUnknown, err - } - return storage.NewStatus(response.Status), nil + return r.api.GetBlockStatus(requestCtx, height) } // Results - @@ -265,14 +221,6 @@ func (r *Receiver) getStateUpdate(ctx context.Context, blockId starknetData.Bloc requestCtx, cancel := context.WithTimeout(ctx, r.timeout) defer cancel() - if r.rpc != nil { - response, err := r.rpc.GetStateUpdate(requestCtx, blockId) - if err != nil { - return starknetData.StateUpdate{}, err - } - return response.Result.ToStateUpdate(), nil - } - return r.api.GetStateUpdate(requestCtx, blockId) } diff --git a/pkg/indexer/store/sub_models.go b/pkg/indexer/store/sub_models.go index 76aafb8..d65f64b 100644 --- a/pkg/indexer/store/sub_models.go +++ b/pkg/indexer/store/sub_models.go @@ -6,6 +6,7 @@ import ( models "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/internal/storage/postgres" "github.com/dipdup-net/indexer-sdk/pkg/storage" + "github.com/lib/pq" ) const copyThreashold = 25 @@ -106,17 +107,26 @@ func bulkSaveWithCopy[M models.CopiableModel](ctx context.Context, tx storage.Tr case len(arr) == 0: return nil case len(arr) < copyThreashold: - data := make([]any, len(arr)) - for i := range arr { - data[i] = &arr[i] - } - return tx.BulkSave(ctx, data) + _, err := tx.Tx().NewInsert().Model(&arr).Exec(ctx) + return err default: tableName := arr[0].TableName() - data := make([]storage.Copiable, len(arr)) + + stmt, err := tx.Tx().PrepareContext(ctx, pq.CopyIn(tableName, arr[0].Columns()...)) + if err != nil { + return err + } + for i := range arr { - data[i] = arr[i] + if _, err := stmt.ExecContext(ctx, arr[i].Flat()...); err != nil { + return err + } } - return tx.CopyFrom(ctx, tableName, data) + + if _, err := stmt.ExecContext(ctx); err != nil { + return err + } + + return stmt.Close() } }