From ddf7e900e248d94b00b4c88c6ebbd6e2fbfd220c Mon Sep 17 00:00:00 2001 From: Jerry Wong Date: Sun, 22 May 2022 15:11:08 +0800 Subject: [PATCH 1/5] add cryptoswap pools for Curve.fi --- cmd/exchange-scrapers/collector/go.mod | 8 +++-- cmd/exchange-scrapers/collector/go.sum | 2 ++ cmd/services/pairDiscoveryService/go.sum | 2 ++ .../local/exchange-scraper/docker-compose.yml | 2 +- deployments/local/exchange-scraper/local.env | 4 +-- .../exchange-scrapers/CurvefiScraper.go | 30 +++++++++++++++++-- 6 files changed, 40 insertions(+), 8 deletions(-) diff --git a/cmd/exchange-scrapers/collector/go.mod b/cmd/exchange-scrapers/collector/go.mod index 0e8ff90c2..0e45dfdcd 100644 --- a/cmd/exchange-scrapers/collector/go.mod +++ b/cmd/exchange-scrapers/collector/go.mod @@ -5,8 +5,12 @@ go 1.14 require ( github.com/adshao/go-binance/v2 v2.3.5 // indirect github.com/cryptwire/go-binance/v2 v2.2.3 // indirect - github.com/diadata-org/diadata v1.4.1-rc-180 + github.com/diadata-org/diadata v1.4.1-rc-180 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/segmentio/kafka-go v0.3.7 github.com/sirupsen/logrus v1.8.1 -) \ No newline at end of file +) + +// replace ( +// github.com/diadata-org/diadata => ../../../ +// ) diff --git a/cmd/exchange-scrapers/collector/go.sum b/cmd/exchange-scrapers/collector/go.sum index b37842112..28736421f 100644 --- a/cmd/exchange-scrapers/collector/go.sum +++ b/cmd/exchange-scrapers/collector/go.sum @@ -263,6 +263,8 @@ github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMa github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/diadata-org/diadata v1.4.1-rc-177 h1:k6s0nvZFJjaY8/nD6IjWpl6iVkLgZOiz9EYpdQEB/ZI= github.com/diadata-org/diadata v1.4.1-rc-177/go.mod h1:O26QF3fP9+IPPQ+OCP+tDLEaE9Zu0sbZBgskPwGvAWc= +github.com/diadata-org/diadata v1.4.1-rc-180 h1:8qAuCVJd6nz7F7t00/fubpA2Ywyf2VwdgYelfyV87rs= +github.com/diadata-org/diadata v1.4.1-rc-180/go.mod h1:O26QF3fP9+IPPQ+OCP+tDLEaE9Zu0sbZBgskPwGvAWc= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= diff --git a/cmd/services/pairDiscoveryService/go.sum b/cmd/services/pairDiscoveryService/go.sum index 4bf4a1202..590b2fc71 100644 --- a/cmd/services/pairDiscoveryService/go.sum +++ b/cmd/services/pairDiscoveryService/go.sum @@ -271,6 +271,8 @@ github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMa github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/diadata-org/diadata v1.3.4-rc-1 h1:W/joRBrON2f/8g1SJufglb1fi3zqV3tflh3B6FJyHd4= github.com/diadata-org/diadata v1.3.4-rc-1/go.mod h1:V/VEMr61h9MQFvz4ZCrpH1Y/ikFG/+K2gS+V4J/EIs4= +github.com/diadata-org/diadata v1.4.1-rc-179 h1:zGOgQL9oIuERYfug2wlWHN71JzF1PBLXkPMiJm6AbPw= +github.com/diadata-org/diadata v1.4.1-rc-179/go.mod h1:O26QF3fP9+IPPQ+OCP+tDLEaE9Zu0sbZBgskPwGvAWc= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= diff --git a/deployments/local/exchange-scraper/docker-compose.yml b/deployments/local/exchange-scraper/docker-compose.yml index 1efe9a83d..ae3cff7db 100644 --- a/deployments/local/exchange-scraper/docker-compose.yml +++ b/deployments/local/exchange-scraper/docker-compose.yml @@ -12,7 +12,7 @@ services: links: - influxdb image: influxdb:2.0 - entrypoint: influx setup --bucket test_bucket -t test_token -o test_org --username=test --password=test --host=http://influxdb:8086 -f + entrypoint: influx setup --bucket test_bucket -t test_token -o test_org --username test --password testtest --host http://127.0.0.1:8086 -f restart: on-failure:20 depends_on: - influxdb diff --git a/deployments/local/exchange-scraper/local.env b/deployments/local/exchange-scraper/local.env index d8e5c40ac..19b9edaa6 100644 --- a/deployments/local/exchange-scraper/local.env +++ b/deployments/local/exchange-scraper/local.env @@ -1,9 +1,9 @@ export USE_ENV=true export INFLUXURL=http://localhost:8086 export INFLUXUSER=test -export INFLUXPASSWORD=test +export INFLUXPASSWORD=testtest export POSTGRES_USER=postgres export POSTGRES_PASSWORD=password export POSTGRES_HOST=localhost export POSTGRES_DB=postgres -export REDISURL=localhost:6379 \ No newline at end of file +export REDISURL=localhost:6379 diff --git a/pkg/dia/scraper/exchange-scrapers/CurvefiScraper.go b/pkg/dia/scraper/exchange-scrapers/CurvefiScraper.go index c7c80ebb2..55413b673 100644 --- a/pkg/dia/scraper/exchange-scrapers/CurvefiScraper.go +++ b/pkg/dia/scraper/exchange-scrapers/CurvefiScraper.go @@ -139,6 +139,20 @@ func NewCurveFIScraper(exchange dia.Exchange, scrape bool) *CurveFIScraper { } log.Infof("loaded standard pools. Now %v pools.", len(scraper.pools.pools)) + // Load crypto pools. + // err = scraper.loadPoolsAndCoins(common.HexToAddress("0xF18056Bbd320E96A48e3Fbf8bC061322531aac99")) + // if err != nil { + // log.Error("loadPoolsAndCoins: ", err) + // } + // log.Infof("loaded crypto pools. Now %v pools.", len(scraper.pools.pools)) + + // Load cryptoswap pools. + err = scraper.loadPoolsAndCoins(common.HexToAddress("0x8F942C20D02bEfc377D41445793068908E2250D0")) + if err != nil { + log.Error("loadPoolsAndCoins: ", err) + } + log.Infof("loaded cryptoswap pools. Now %v pools.", len(scraper.pools.pools)) + if scrape { go scraper.mainLoop() } @@ -364,7 +378,7 @@ func (scraper *CurveFIScraper) watchNewPools() { // contract.poolList.map(contract.GetPoolCoins(pool).) func (scraper *CurveFIScraper) loadPoolsAndCoins(factoryAddress common.Address) (err error) { - if factoryAddress == common.HexToAddress(curveFiMetaPoolsFactory) { + if factoryAddress == common.HexToAddress(curveFiMetaPoolsFactory) || factoryAddress == common.HexToAddress("0xF18056Bbd320E96A48e3Fbf8bC061322531aac99") { log.Info("load meta pools...") var contract *curvefimeta.CurvefimetaCaller var poolCount *big.Int @@ -424,8 +438,10 @@ func (scraper *CurveFIScraper) loadPoolsAndCoins(factoryAddress common.Address) func (scraper *CurveFIScraper) loadPoolData(pool string, factoryContract common.Address) error { var poolCoins [8]common.Address var poolCoinsMap = make(map[int]*CurveCoin) - if factoryContract == common.HexToAddress(curveFiMetaPoolsFactory) { + + if factoryContract == common.HexToAddress(curveFiMetaPoolsFactory) || factoryContract == common.HexToAddress("0xF18056Bbd320E96A48e3Fbf8bC061322531aac99") { contract, err := curvefimeta.NewCurvefimetaCaller(factoryContract, scraper.RestClient) + if err != nil { log.Error("loadPoolData - NewCurvefiCaller: ", err) } @@ -450,6 +466,11 @@ func (scraper *CurveFIScraper) loadPoolData(pool string, factoryContract common. log.Error("loadPoolData - GetCoins: ", err) } log.Info("pool coins: ", poolCoins) + poolName, err := contract.GetPoolName(&bind.CallOpts{}, common.HexToAddress(pool)) + if err != nil { + log.Error("loadPoolData - GetPoolName: ", err) + } + log.Info("pool name: ", poolName) } var err error @@ -458,7 +479,9 @@ func (scraper *CurveFIScraper) loadPoolData(pool string, factoryContract common. var symbol string var decimals *big.Int var name string - if c == common.HexToAddress("0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE") { + if c == common.HexToAddress("0x0000000000000000000000000000000000000000") { + continue + } else if c == common.HexToAddress("0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE") { symbol = "ETH" decimals = big.NewInt(int64(18)) name = "Ether" @@ -485,6 +508,7 @@ func (scraper *CurveFIScraper) loadPoolData(pool string, factoryContract common. continue } } + log.Info(symbol, " ", decimals, " ", "'", name, "'", " ", c) poolCoinsMap[cIdx] = &CurveCoin{ Symbol: symbol, From ea66ad8b0f9787dfa1ccdc6e28c1b6bee4653cd1 Mon Sep 17 00:00:00 2001 From: Jerry Wong Date: Sat, 11 Jun 2022 13:53:33 +0800 Subject: [PATCH 2/5] improve exchange scraper tutorial --- documentation/tutorials/exchangescrapers.md | 48 +++++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/documentation/tutorials/exchangescrapers.md b/documentation/tutorials/exchangescrapers.md index d348e6afc..74da2ae8f 100644 --- a/documentation/tutorials/exchangescrapers.md +++ b/documentation/tutorials/exchangescrapers.md @@ -8,15 +8,15 @@ Now, let's assume you want to scrape a data source that provides trade informati ```go type APIScraper interface { - io.Closer - // ScrapePair returns a PairScraper that continuously scrapes trades for a - // single pair from this APIScraper - ScrapePair(pair dia.Pair) (PairScraper, error) - // FetchAvailablePairs returns a list with all available trade pairs (usually - // fetched from an exchange's API) - FetchAvailablePairs() (pairs []dia.Pair, err error) - // Channel returns a channel that can be used to receive trades - Channel() chan *dia.Trade + io.Closer + // ScrapePair returns a PairScraper that continuously scrapes trades for a + // single pair from this APIScraper + ScrapePair(pair dia.Pair) (PairScraper, error) + // FetchAvailablePairs returns a list with all available trade pairs (usually + // fetched from an exchange's API) + FetchAvailablePairs() (pairs []dia.Pair, err error) + // Channel returns a channel that can be used to receive trades + Channel() chan *dia.Trade } ``` @@ -27,15 +27,27 @@ Also, please take care of proper error handling and cleanup. More precisely, you Furthermore, in order for our system to see your scraper, add a reference to it in `Config.go` in the dia package, and to the switch statement in `APIScraper.go` in the scrapers package: ```go +# pkg/dia/scraper/exchange-scraper/APIScraper.go func NewAPIScraper(exchange string, key string, secret string) APIScraper { - switch exchange { - case dia.MySourceExchange: - return NewMySourceScraper(key, secret, dia.MySourceExchange) - } + switch exchange { + case dia.MySourceExchange: + return NewMySourceScraper(key, secret, dia.MySourceExchange) + } } ``` +Also, if your want to get data from contract, install `abigen` and generate the code from exchanger provided abi. + +```sh +go install github.com/ethereum/go-ethereum/cmd/abigen@latest + +abigen --abi myexchange/myexchange.abi --pkg myexchange --type MyExchange --out myexchange/myexchange.go +``` + +Please put your abi file and generated code into a folder under the `exchange-scrapers/` and name it with the exchange name. + ## Steps to run a scraper locally + 1. Navigate to the `deployments/local/exchange-scraper` directory of the project. 2. Run the required services using `docker-compose up -d`, this will run and prepare Redis, PostgreSQL, and InfluxDB databases. 3. Set the required environment variables using the following commands: @@ -54,6 +66,15 @@ export REDISURL=localhost:6379 Or simple by sourcing the `local.env` inside the `deployments/local/exchange-scraper` directory. +Also don't forget to set your Ethereum Node API Key. + +```sh +export ETHEREUM_URL_REST=${YOUR_API_REST_ENDPOINT} +export ETHEREUM_URL_WS=${YOUR_API_WS_ENDPOINT} +``` + +Tips: Find you favourite node api provider at [EthereumNodes](https://ethereumnodes.com) + 4. Execute `main.go` from `cmd/services/pairDiscoveryServices` for fetching the available pairs and setting them in the Redis database. 5. Finally, run the scraping executable flagged as follows: @@ -63,4 +84,3 @@ go run collector.go -exchange MySource ``` For an illustration you can have a look at the `KrakenScraper.go`. - From 67f2fe204acc6d2681dac23e79fe769b3bc58462 Mon Sep 17 00:00:00 2001 From: Jerry Wong Date: Mon, 13 Jun 2022 18:06:44 +0800 Subject: [PATCH 3/5] add some links and small improvement --- documentation/tutorials/exchangescrapers.md | 25 ++++++++++++++++--- documentation/tutorials/ratescrapers.md | 14 +++++------ .../tutorials/write-your-own-rate-scraper.md | 3 +-- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/documentation/tutorials/exchangescrapers.md b/documentation/tutorials/exchangescrapers.md index 74da2ae8f..2d9b63485 100644 --- a/documentation/tutorials/exchangescrapers.md +++ b/documentation/tutorials/exchangescrapers.md @@ -66,19 +66,38 @@ export REDISURL=localhost:6379 Or simple by sourcing the `local.env` inside the `deployments/local/exchange-scraper` directory. -Also don't forget to set your Ethereum Node API Key. +Also don't forget to set your Ethereum Node API Key. (If you are working on ethereum chain.) ```sh export ETHEREUM_URL_REST=${YOUR_API_REST_ENDPOINT} export ETHEREUM_URL_WS=${YOUR_API_WS_ENDPOINT} ``` -Tips: Find you favourite node api provider at [EthereumNodes](https://ethereumnodes.com) +Tips: Find you favourite ethereum node api provider at [EthereumNodes](https://ethereumnodes.com) + +If you are working on another ethereum-compatible chain, replace `ETHEREUM` prefix with your chain name and get them from env. + +```go +func NewAPIScraper() *APIScraper { + // some initial stuff... + log.Infof("Init rest and ws client for %s.", exchange.BlockChain.Name) + restClient, err := ethclient.Dial(utils.Getenv(strings.ToUpper(exchange.BlockChain.Name)+"_URI_REST", curveRestDial)) + if err != nil { + log.Fatal("init rest client: ", err) + } + wsClient, err := ethclient.Dial(utils.Getenv(strings.ToUpper(exchange.BlockChain.Name)+"_URI_WS", curveWsDial)) + if err != nil { + log.Fatal("init ws client: ", err) + } + // other stuff here... +} + +``` 4. Execute `main.go` from `cmd/services/pairDiscoveryServices` for fetching the available pairs and setting them in the Redis database. 5. Finally, run the scraping executable flagged as follows: -```text +```sh cd cmd/exchange-scrapers/collector go run collector.go -exchange MySource ``` diff --git a/documentation/tutorials/ratescrapers.md b/documentation/tutorials/ratescrapers.md index c45c6d944..51d35ba23 100644 --- a/documentation/tutorials/ratescrapers.md +++ b/documentation/tutorials/ratescrapers.md @@ -1,10 +1,10 @@ # Write your own rate scraper -These instructions concern writing scrapers for single units characterised by a (floating point) number. For scrapers describing the relation between pairs of units, i.e. exchange rates see the instructions in exchangescrapers.md. +These instructions concern writing scrapers for single units characterised by a (floating point) number. For scrapers describing the relation between pairs of units, i.e. exchange rates see the instructions in [exchangescrapers.md](exchangescrapers.md). ## Instructions for the addition of a rate scraper -In order to add your own scraper for a new data source, you must adhere to our format. Create the package file `UpdateMYRATE.go` in the package `/internal/pkg/ratescrapers`. The central method is ` UpdateMYRATE()`. This method acts on a RateScraper struct which is defined in RateScraper.go in the ratescrapers package. For instance, for the the Euro Short-Term Rate (ESTER) issued by the ECB, `UpdateESTER.go` would look like +In order to add your own scraper for a new data source, you must adhere to our format. Create the package file `UpdateMYRATE.go` in the package `/internal/pkg/ratescrapers`. The central method is `UpdateMYRATE()`. This method acts on a RateScraper struct which is defined in RateScraper.go in the ratescrapers package. For instance, for the the Euro Short-Term Rate (ESTER) issued by the ECB, `UpdateESTER.go` would look like ```go func (s *RateScraper) UpdateESTER() error { @@ -16,14 +16,14 @@ The scraped data has to be written into a struct of type InterestRate from `pkg/ ```go type InterestRate struct { - Symbol string - Value float64 - Time time.Time - Source string + Symbol string + Value float64 + Time time.Time + Source string } ``` + and sent to the channel chanInterestRate of the RateScraper s. In order to write a new scraper, it is not imperative to understand the architecture of the pathway from top to bottom, but it might be helpful. For a first impression you can have a look at the following [diagram](github.com/diadata-org/diadata/documentation/tutorials/rate_scraper_diagram_down.pdf). **Remark**: Parsing xml in go is not always straightforward. A useful resource for parsing can be found here: github.com/gnewton/chidley - diff --git a/documentation/tutorials/write-your-own-rate-scraper.md b/documentation/tutorials/write-your-own-rate-scraper.md index 1caded342..d8cccebdf 100644 --- a/documentation/tutorials/write-your-own-rate-scraper.md +++ b/documentation/tutorials/write-your-own-rate-scraper.md @@ -1,6 +1,6 @@ # Write your own rate scraper -These instructions concern writing scrapers for single units characterised by a \(floating point\) number. For scrapers describing the relation between pairs of units, i.e. exchange rates see the instructions in exchangescrapers.md. +These instructions concern writing scrapers for single units characterised by a \(floating point\) number. For scrapers describing the relation between pairs of units, i.e. exchange rates see the instructions in [exchangescrapers.md](exchangescrapers.md). ## Instructions for the addition of a rate scraper @@ -46,4 +46,3 @@ type InterestRate struct { ``` and sent to the channel `chanInterestRate` of `s`. In order to write a new scraper, it is not imperative to understand the architecture of the pathway from top to bottom, but it might be helpful. For a first impression you can have a look at the following [diagram](https://github.com/diadata-org/diadata/tree/master/documentation/tutorials/rate_scraper_diagram_down.pdf). - From 2a128ff9667de57cb3646b3752d516ed59eacf91 Mon Sep 17 00:00:00 2001 From: Jerry Wong Date: Mon, 13 Jun 2022 22:44:49 +0800 Subject: [PATCH 4/5] re-improve exchange scraper tutorial --- documentation/tutorials/exchangescrapers.md | 47 ++++++++++++--------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/documentation/tutorials/exchangescrapers.md b/documentation/tutorials/exchangescrapers.md index 2d9b63485..323f1b3fb 100644 --- a/documentation/tutorials/exchangescrapers.md +++ b/documentation/tutorials/exchangescrapers.md @@ -36,6 +36,30 @@ func NewAPIScraper(exchange string, key string, secret string) APIScraper { } ``` +If you are working on ethereum chain with the decentralized exchange, you can get your node api key from environment variant. + +```go +func NewAPIScraper() *APIScraper { + // some initial stuff... + + log.Infof("Init rest and ws client for %s.", exchange.BlockChain.Name) + restClient, err := ethclient.Dial(utils.Getenv(strings.ToUpper(exchange.BlockChain.Name)+"_URI_REST", curveRestDial)) + if err != nil { + log.Fatal("init rest client: ", err) + } + wsClient, err := ethclient.Dial(utils.Getenv(strings.ToUpper(exchange.BlockChain.Name)+"_URI_WS", curveWsDial)) + if err != nil { + log.Fatal("init ws client: ", err) + } + // other stuff here... +} + +``` + +There's no limit to use your own method to connect with blockchain, you can make your own connection. + +For centralized exchange you should check the provider document. Then do your own connection with that. + Also, if your want to get data from contract, install `abigen` and generate the code from exchanger provided abi. ```sh @@ -44,7 +68,7 @@ go install github.com/ethereum/go-ethereum/cmd/abigen@latest abigen --abi myexchange/myexchange.abi --pkg myexchange --type MyExchange --out myexchange/myexchange.go ``` -Please put your abi file and generated code into a folder under the `exchange-scrapers/` and name it with the exchange name. +Put your abi file and generated code into a folder under the `exchange-scrapers/` and name it with the exchange name. ## Steps to run a scraper locally @@ -66,7 +90,7 @@ export REDISURL=localhost:6379 Or simple by sourcing the `local.env` inside the `deployments/local/exchange-scraper` directory. -Also don't forget to set your Ethereum Node API Key. (If you are working on ethereum chain.) +If you are working on ethereum chain, export your node api key from terminal. ```sh export ETHEREUM_URL_REST=${YOUR_API_REST_ENDPOINT} @@ -75,24 +99,7 @@ export ETHEREUM_URL_WS=${YOUR_API_WS_ENDPOINT} Tips: Find you favourite ethereum node api provider at [EthereumNodes](https://ethereumnodes.com) -If you are working on another ethereum-compatible chain, replace `ETHEREUM` prefix with your chain name and get them from env. - -```go -func NewAPIScraper() *APIScraper { - // some initial stuff... - log.Infof("Init rest and ws client for %s.", exchange.BlockChain.Name) - restClient, err := ethclient.Dial(utils.Getenv(strings.ToUpper(exchange.BlockChain.Name)+"_URI_REST", curveRestDial)) - if err != nil { - log.Fatal("init rest client: ", err) - } - wsClient, err := ethclient.Dial(utils.Getenv(strings.ToUpper(exchange.BlockChain.Name)+"_URI_WS", curveWsDial)) - if err != nil { - log.Fatal("init ws client: ", err) - } - // other stuff here... -} - -``` +Also if you are working on another ethereum-compatible chain, simply replace `ETHEREUM` prefix with your chain name and get them from code. 4. Execute `main.go` from `cmd/services/pairDiscoveryServices` for fetching the available pairs and setting them in the Redis database. 5. Finally, run the scraping executable flagged as follows: From 3cab0b53abfd71a3f14460b88a6405bad3cf9f59 Mon Sep 17 00:00:00 2001 From: Jerry Wong Date: Tue, 14 Jun 2022 13:28:44 +0800 Subject: [PATCH 5/5] update some parts in exchange scraper tutorial --- documentation/tutorials/exchangescrapers.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/documentation/tutorials/exchangescrapers.md b/documentation/tutorials/exchangescrapers.md index 323f1b3fb..35eb88a91 100644 --- a/documentation/tutorials/exchangescrapers.md +++ b/documentation/tutorials/exchangescrapers.md @@ -39,9 +39,8 @@ func NewAPIScraper(exchange string, key string, secret string) APIScraper { If you are working on ethereum chain with the decentralized exchange, you can get your node api key from environment variant. ```go -func NewAPIScraper() *APIScraper { +func NewMySourceScraper(exchangeName string) *MySourceScraper { // some initial stuff... - log.Infof("Init rest and ws client for %s.", exchange.BlockChain.Name) restClient, err := ethclient.Dial(utils.Getenv(strings.ToUpper(exchange.BlockChain.Name)+"_URI_REST", curveRestDial)) if err != nil { @@ -58,7 +57,9 @@ func NewAPIScraper() *APIScraper { There's no limit to use your own method to connect with blockchain, you can make your own connection. -For centralized exchange you should check the provider document. Then do your own connection with that. +For a working example, check out `CurvefiScraper.go` or other decentralized exchange scrapers. + +For centralized exchange, you should follow the CEX provider documents. Then do your own connection. For an illustration you can have a look at the `KrakenScraper.go`. Also, if your want to get data from contract, install `abigen` and generate the code from exchanger provided abi. @@ -108,5 +109,3 @@ Also if you are working on another ethereum-compatible chain, simply replace `ET cd cmd/exchange-scrapers/collector go run collector.go -exchange MySource ``` - -For an illustration you can have a look at the `KrakenScraper.go`.