From 61f28a755f8c50cf466ce14289a54660afc003ac Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Thu, 10 May 2018 10:14:33 +0800 Subject: [PATCH 01/37] add update_time for candidate and delegator document --- model/store/document/stake_role_candidate.go | 17 ++--------------- model/store/document/stake_role_delegator.go | 2 ++ sync/handler.go | 5 +++++ 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/model/store/document/stake_role_candidate.go b/model/store/document/stake_role_candidate.go index 1632847..729e225 100644 --- a/model/store/document/stake_role_candidate.go +++ b/model/store/document/stake_role_candidate.go @@ -6,6 +6,7 @@ import ( "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "log" + "time" ) const ( @@ -18,6 +19,7 @@ type Candidate struct { Shares int64 `bson:"shares"` VotingPower uint64 `bson:"voting_power"` // Voting power if pubKey is a considered a validator Description Description `bson:"description"` // Description terms for the candidate + UpdateTime time.Time `bson:"update_time"` } func (d Candidate) Name() string { @@ -45,21 +47,6 @@ func (d Candidate) Index() []mgo.Index { } } -//func QueryCandidateByAddressAndPubkey(address string, pubKey string) (Candidate, error) { -// var result Candidate -// query := func(c *mgo.Collection) error { -// err := c.Find(bson.M{"address": address, "pub_key": pubKey}).Sort("-shares").One(&result) -// return err -// } -// -// if store.ExecCollection(CollectionNmStakeRoleDelegator, query) != nil { -// log.Printf("delegator is Empty") -// return result, errors.New("delegator is Empty") -// } -// -// return result, nil -//} - func QueryCandidateByPubkey(pubKey string) (Candidate, error) { var result Candidate query := func(c *mgo.Collection) error { diff --git a/model/store/document/stake_role_delegator.go b/model/store/document/stake_role_delegator.go index 37d565f..84198f6 100644 --- a/model/store/document/stake_role_delegator.go +++ b/model/store/document/stake_role_delegator.go @@ -6,6 +6,7 @@ import ( "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "log" + "time" ) const ( @@ -16,6 +17,7 @@ type Delegator struct { Address string `bson:"address"` PubKey string `bson:"pub_key"` Shares int64 `bson:"shares"` + UpdateTime time.Time `bson:"update_time"` } func (d Delegator) Name() string { diff --git a/sync/handler.go b/sync/handler.go index f0f8249..ba192ec 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -60,6 +60,7 @@ func saveTx(tx store.Docs) { // TODO: in further share not equal amount candidate.Shares += stakeTxDeclareCandidacy.Amount.Amount candidate.VotingPower += uint64(stakeTxDeclareCandidacy.Amount.Amount) + candidate.UpdateTime = stakeTxDeclareCandidacy.Time store.SaveOrUpdate(candidate) break case stake.TypeTxDelegate: @@ -82,10 +83,12 @@ func saveTx(tx store.Docs) { } // TODO: in further share not equal amount delegator.Shares += stakeTx.Amount.Amount + delegator.UpdateTime = stakeTx.Time store.SaveOrUpdate(delegator) candidate.Shares += stakeTx.Amount.Amount candidate.VotingPower += uint64(stakeTx.Amount.Amount) + candidate.UpdateTime = stakeTx.Time store.SaveOrUpdate(candidate) break case stake.TypeTxUnbond: @@ -98,6 +101,7 @@ func saveTx(tx store.Docs) { return } delegator.Shares -= stakeTx.Amount.Amount + delegator.UpdateTime = stakeTx.Time store.Update(delegator) candidate, err2 := document.QueryCandidateByPubkey(stakeTx.PubKey) @@ -109,6 +113,7 @@ func saveTx(tx store.Docs) { } candidate.Shares -= stakeTx.Amount.Amount candidate.VotingPower -= uint64(stakeTx.Amount.Amount) + candidate.UpdateTime = stakeTx.Time store.Update(candidate) break } From 08c6528afccda660d1ea41bb04593ffb7cbddf6a Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Fri, 11 May 2018 18:17:12 +0800 Subject: [PATCH 02/37] add makefile and review code --- Makefile | 32 +++++++ README.md | 4 +- conf/server/types.go | 6 +- main.go | 13 +-- model/store/document/stake_role_candidate.go | 4 +- model/store/document/stake_role_delegator.go | 4 +- model/store/store.go | 15 +-- sync/sync.go | 51 ++++------- sync/sync_test.go | 96 ++++++++++++++++++++ 9 files changed, 169 insertions(+), 56 deletions(-) create mode 100644 Makefile create mode 100644 sync/sync_test.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0714b3e --- /dev/null +++ b/Makefile @@ -0,0 +1,32 @@ +GOCMD=go +GOBUILD=$(GOCMD) build +GOCLEAN=$(GOCMD) clean +GOTEST=$(GOCMD) test +GOGET=$(GOCMD) get +BINARY_NAME=sync-iris +BINARY_UNIX=$(BINARY_NAME)_unix + +all: get_vendor build + +get_vendor: + @echo "--> Running glide install" + @glide install -v + +build: + $(GOBUILD) -o $(BINARY_NAME) -v + +clean: + $(GOCLEAN) + rm -f $(BINARY_NAME) + rm -f $(BINARY_UNIX) + +run: + $(GOBUILD) -o $(BINARY_NAME) -v ./... + ./$(BINARY_NAME) + + +# Cross compilation +build-linux: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) -v +docker-build: + docker run --rm -it -v "$(GOPATH)":/go -w /go/src/github.com/irisnet/iris-sync-server golang:latest go build -o "$(BINARY_UNIX)" -v \ No newline at end of file diff --git a/README.md b/README.md index 53abf46..9b8c6fc 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ -# iris-sync-server -A server that synchronize IRIS blockchain data into a relational database +# IRIS-sync-server +A server that synchronize IRIS blockChain data into a database \ No newline at end of file diff --git a/conf/server/types.go b/conf/server/types.go index 4f96102..6f738b9 100644 --- a/conf/server/types.go +++ b/conf/server/types.go @@ -1,10 +1,10 @@ package server const ( - BlockChainMonitorUrl = "tcp://localhost:46657" + BlockChainMonitorUrl = "tcp://47.104.155.125:46757" Token = "iris" - InitConnectionNum = 100 - MaxConnectionNum = 200 + InitConnectionNum = 1000 + MaxConnectionNum = 2000 ChainId = "test" SyncCron = "1 * * * * *" diff --git a/main.go b/main.go index a4d141d..a1a4a4b 100644 --- a/main.go +++ b/main.go @@ -1,14 +1,15 @@ package main import ( - "fmt" - + "github.com/irisnet/iris-sync-server/sync" + "time" "github.com/irisnet/iris-sync-server/module/logger" ) func main() { - i := 5 - j := 2 - logger.Info.Printf("result of j / i is %v", i/j) - fmt.Println("This is test") + sync.Start() + for true { + time.Sleep(time.Minute) + logger.Info.Println("wait") + } } diff --git a/model/store/document/stake_role_candidate.go b/model/store/document/stake_role_candidate.go index 729e225..2280c44 100644 --- a/model/store/document/stake_role_candidate.go +++ b/model/store/document/stake_role_candidate.go @@ -5,8 +5,8 @@ import ( "github.com/irisnet/iris-sync-server/model/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" - "log" "time" + "github.com/irisnet/iris-sync-server/module/logger" ) const ( @@ -55,7 +55,7 @@ func QueryCandidateByPubkey(pubKey string) (Candidate, error) { } if store.ExecCollection(CollectionNmStakeRoleCandidate, query) != nil { - log.Printf("candidate is Empty") + logger.Info.Println("candidate is Empty") return result, errors.New("candidate is Empty") } diff --git a/model/store/document/stake_role_delegator.go b/model/store/document/stake_role_delegator.go index 84198f6..f0ff9e4 100644 --- a/model/store/document/stake_role_delegator.go +++ b/model/store/document/stake_role_delegator.go @@ -5,8 +5,8 @@ import ( "github.com/irisnet/iris-sync-server/model/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" - "log" "time" + "github.com/irisnet/iris-sync-server/module/logger" ) const ( @@ -59,7 +59,7 @@ func QueryDelegatorByAddressAndPubkey(address string, pubKey string) (Delegator, } if store.ExecCollection(CollectionNmStakeRoleDelegator, query) != nil { - log.Printf("delegator is Empty") + logger.Info.Println("delegator is Empty") return result, errors.New("delegator is Empty") } diff --git a/model/store/store.go b/model/store/store.go index 42177e9..6d5a550 100644 --- a/model/store/store.go +++ b/model/store/store.go @@ -35,7 +35,7 @@ func Init() { } session.SetMode(mgo.Monotonic, true) - index() + //index() } } @@ -105,7 +105,7 @@ func Save(h Docs) error { logger.Info.Println("db: record existed while save data") return nil } - logger.Info.Printf("insert %s %+v\n", h.Name(), h) + //logger.Info.Printf("insert %s %+v\n", h.Name(), h) return c.Insert(h) } @@ -116,11 +116,14 @@ func SaveOrUpdate(h Docs) error { save := func(c *mgo.Collection) error { //先按照关键字查询，如果存在，直接返回 n, err := c.Find(h.PkKvPair()).Count() - logger.Info.Printf("Count:%d err:%+v\n", n, err) + if err != nil { + logger.Error.Printf("Count:%d err:%+v\n", n, err) + } + if n >= 1 { return Update(h) } - logger.Info.Printf("insert %s %+v\n", h.Name(), h) + //logger.Info.Printf("insert %s %+v\n", h.Name(), h) return c.Insert(h) } @@ -130,8 +133,8 @@ func SaveOrUpdate(h Docs) error { func Update(h Docs) error { update := func(c *mgo.Collection) error { key := h.PkKvPair() - logger.Info.Printf("update %s set %+v where %+v\n", h.Name(), h, key) - return c.Update(h.PkKvPair(), h) + //logger.Info.Printf("update %s set %+v where %+v\n", h.Name(), h, key) + return c.Update(key, h) } return ExecCollection(h.Name(), update) } diff --git a/sync/sync.go b/sync/sync.go index 0c398b5..9b9b7f1 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -33,7 +33,6 @@ func Start() { logger.Error.Fatalf("sync block failed,%v\n", err) } startCron(c) - //go watch(c) 监控的方式在启动同步过程中容易丢失区块 } func InitServer() { @@ -68,7 +67,7 @@ func startCron(client rpcClient.Client) { } func watchBlock(c rpcClient.Client) error { - b, _ := document.QuerySyncTask() + syncTask, _ := document.QuerySyncTask() status, _ := c.Status() latestBlockHeight := status.LatestBlockHeight @@ -79,23 +78,24 @@ func watchBlock(c rpcClient.Client) error { ch := make(chan int64) limitChan <- 1 - go syncBlock(b.Height+1, latestBlockHeight, funcChain, ch, 0) + go syncBlock(syncTask.Height+1, latestBlockHeight, funcChain, ch, 0) select { case <-ch: - //更新同步记录 + logger.Info.Printf("watch block, current height is %v \n", latestBlockHeight) block, _ := c.Block(&latestBlockHeight) - b.Height = block.Block.Height - b.Time = block.Block.Time - return store.Update(b) + syncTask.Height = block.Block.Height + syncTask.Time = block.Block.Time + return store.Update(syncTask) } } // fast sync data from blockChain func fastSync(c rpcClient.Client) error { - b, _ := document.QuerySyncTask() + syncTaskDoc, _ := document.QuerySyncTask() status, _ := c.Status() latestBlockHeight := status.LatestBlockHeight + funcChain := []func(tx store.Docs){ saveTx, saveOrUpdateAccount, updateAccountBalance, } @@ -103,7 +103,7 @@ func fastSync(c rpcClient.Client) error { ch := make(chan int64) activeThreadNum := int64(0) - goRoutineNum := (latestBlockHeight - b.Height) / syncBlockNumFastSync + goRoutineNum := (latestBlockHeight - syncTaskDoc.Height) / syncBlockNumFastSync if goRoutineNum == 0 { goRoutineNum = 10 @@ -114,8 +114,8 @@ func fastSync(c rpcClient.Client) error { activeThreadNum++ limitChan <- i var ( - start = b.Height + (i-1)*syncBlockNumFastSync + 1 - end = b.Height + i*syncBlockNumFastSync + start = syncTaskDoc.Height + (i-1)*syncBlockNumFastSync + 1 + end = syncTaskDoc.Height + i*syncBlockNumFastSync ) if i == goRoutineNum { end = latestBlockHeight @@ -123,25 +123,6 @@ func fastSync(c rpcClient.Client) error { go syncBlock(start, end, funcChain, ch, i) } - //threadNum := (latestBlockHeight - b.Height) / maxBatchNum - //// 单线程处理 - //if threadNum == 0 { - // go syncBlock(b.Height, latestBlockHeight, funcChain, ch, 0) - //} else { - // // 开启多线程处理 - // for i := int64(1); i <= threadNum; i++ { - // activeThreadNum ++ - // var start = b.Height + (i-1)*maxBatchNum + 1 - // var end = b.Height + i*maxBatchNum - // if i == threadNum { - // end = latestBlockHeight - // } - // - // go syncBlock(start, end, funcChain, ch, i) - // } - // - //} - for { select { case threadNo := <-ch: @@ -155,11 +136,12 @@ func fastSync(c rpcClient.Client) error { end: { - //更新同步记录 + logger.Info.Println("fast sync block, complete sync task") + // update syncTask document block, _ := c.Block(&latestBlockHeight) - b.Height = block.Block.Height - b.Time = block.Block.Time - store.Update(b) + syncTaskDoc.Height = block.Block.Height + syncTaskDoc.Time = block.Block.Time + store.Update(syncTaskDoc) return nil } } @@ -177,7 +159,6 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs), ch chan for j := start; j <= end; j++ { logger.Info.Printf("===========threadNo[%d] sync block,height:%d===========\n", threadNum, j) - // TODO 使用client.Client.BlockChainInfo block, err := client.Client.Block(&j) if err != nil { // try again diff --git a/sync/sync_test.go b/sync/sync_test.go new file mode 100644 index 0000000..ca37e37 --- /dev/null +++ b/sync/sync_test.go @@ -0,0 +1,96 @@ +package sync + +import ( + "testing" + + "github.com/irisnet/iris-sync-server/model/store" + rpcClient "github.com/tendermint/tendermint/rpc/client" + "fmt" + "time" +) + +func TestStart(t *testing.T) { + Start() + for true { + time.Sleep(time.Minute) + fmt.Printf("wait\n") + } +} + +func Test_startCron(t *testing.T) { + type args struct { + client rpcClient.Client + } + tests := []struct { + name string + args args + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + startCron(tt.args.client) + }) + } +} + +func Test_watchBlock(t *testing.T) { + type args struct { + c rpcClient.Client + } + tests := []struct { + name string + args args + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := watchBlock(tt.args.c); (err != nil) != tt.wantErr { + t.Errorf("watchBlock() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_fastSync(t *testing.T) { + type args struct { + c rpcClient.Client + } + tests := []struct { + name string + args args + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := fastSync(tt.args.c); (err != nil) != tt.wantErr { + t.Errorf("fastSync() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_syncBlock(t *testing.T) { + type args struct { + start int64 + end int64 + funcChain []func(tx store.Docs) + ch chan int64 + threadNum int64 + } + tests := []struct { + name string + args args + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + syncBlock(tt.args.start, tt.args.end, tt.args.funcChain, tt.args.ch, tt.args.threadNum) + }) + } +} From 4676c0511359a69bf7c982ef261871d320d7e62d Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Mon, 14 May 2018 15:59:34 +0800 Subject: [PATCH 03/37] review code and optimize code --- Makefile | 2 +- conf/db/types.go | 6 ++-- conf/server/types.go | 6 ++-- main.go | 16 +++++----- sync/handler.go | 25 +++++++++------ sync/handler_test.go | 75 +++++++++++++++++++++++++++----------------- sync/sync.go | 37 +++++++++++++--------- sync/sync_test.go | 58 +++++++++++----------------------- util/helper/tx.go | 2 +- 9 files changed, 119 insertions(+), 108 deletions(-) diff --git a/Makefile b/Makefile index 0714b3e..282c9f2 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ GOCLEAN=$(GOCMD) clean GOTEST=$(GOCMD) test GOGET=$(GOCMD) get BINARY_NAME=sync-iris -BINARY_UNIX=$(BINARY_NAME)_unix +BINARY_UNIX=$(BINARY_NAME)-unix all: get_vendor build diff --git a/conf/db/types.go b/conf/db/types.go index 93bee57..4f9b50a 100644 --- a/conf/db/types.go +++ b/conf/db/types.go @@ -1,7 +1,7 @@ package db const ( - Host = "" - Port = "" - Database = "" + Host = "127.0.0.1" + Port = "27117" + Database = "sync_iris" ) diff --git a/conf/server/types.go b/conf/server/types.go index 6f738b9..3b6c173 100644 --- a/conf/server/types.go +++ b/conf/server/types.go @@ -1,12 +1,12 @@ package server const ( - BlockChainMonitorUrl = "tcp://47.104.155.125:46757" + BlockChainMonitorUrl = "tcp://127.0.0.1:46757" Token = "iris" InitConnectionNum = 1000 MaxConnectionNum = 2000 - ChainId = "test" - SyncCron = "1 * * * * *" + ChainId = "pangu" + SyncCron = "0-59 * * * * *" SyncMaxGoroutine = 2000 SyncBlockNumFastSync = 2000 diff --git a/main.go b/main.go index a1a4a4b..c972a94 100644 --- a/main.go +++ b/main.go @@ -1,15 +1,15 @@ package main import ( - "github.com/irisnet/iris-sync-server/sync" - "time" - "github.com/irisnet/iris-sync-server/module/logger" + "sync" + syncTask "github.com/irisnet/iris-sync-server/sync" ) func main() { - sync.Start() - for true { - time.Sleep(time.Minute) - logger.Info.Println("wait") - } + var wg sync.WaitGroup + wg.Add(2) + + syncTask.Start() + + wg.Wait() } diff --git a/sync/handler.go b/sync/handler.go index ba192ec..3b46c3e 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -1,7 +1,6 @@ package sync import ( - "reflect" "time" "sync" @@ -17,18 +16,17 @@ import ( var ( delay = false - mutex sync.Mutex ) -func handle(tx store.Docs, funChains []func(tx store.Docs)) { +func handle(tx store.Docs, mutex sync.Mutex, funChains []func(tx store.Docs, mutex sync.Mutex)) { for _, fun := range funChains { - fun(tx) + fun(tx, mutex) } } // save Tx document into collection -func saveTx(tx store.Docs) { +func saveTx(tx store.Docs, mutex sync.Mutex) { err := store.Save(tx) if err != nil { @@ -36,6 +34,7 @@ func saveTx(tx store.Docs) { } mutex.Lock() + logger.Info.Println("saveTx method get lock") { if tx.Name() == document.CollectionNmStakeTx { @@ -44,6 +43,7 @@ func saveTx(tx store.Docs) { return } stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() + logger.Info.Printf("saveTx: type of stakeTx is %v", stakeType) switch stakeType { case stake.TypeTxDeclareCandidacy: @@ -123,9 +123,11 @@ func saveTx(tx store.Docs) { } mutex.Unlock() + + logger.Info.Println("saveTx method release lock") } -func saveOrUpdateAccount(tx store.Docs) { +func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { var ( address string updateTime time.Time @@ -145,6 +147,7 @@ func saveOrUpdateAccount(tx store.Docs) { } mutex.Lock() + logger.Info.Println("saveOrUpdateAccpunt method get lock") { switch tx.Name() { @@ -162,7 +165,8 @@ func saveOrUpdateAccount(tx store.Docs) { return } stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - + + logger.Info.Printf("saveOrUpdateAccpunt: type of stakeTx is %v", stakeType) switch stakeType { case stake.TypeTxDeclareCandidacy: stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) @@ -185,10 +189,11 @@ func saveOrUpdateAccount(tx store.Docs) { } mutex.Unlock() + logger.Info.Println("saveOrUpdateAccpunt method release lock") } -func updateAccountBalance(tx store.Docs) { +func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { var ( address string ) @@ -206,7 +211,7 @@ func updateAccountBalance(tx store.Docs) { } mutex.Lock() - + logger.Info.Println("updateAccountBalance method get lock") { switch tx.Name() { case document.CollectionNmCoinTx: @@ -239,6 +244,6 @@ func updateAccountBalance(tx store.Docs) { } mutex.Unlock() - + logger.Info.Println("updateAccountBalance method release lock") } diff --git a/sync/handler_test.go b/sync/handler_test.go index a31456d..87586da 100644 --- a/sync/handler_test.go +++ b/sync/handler_test.go @@ -8,6 +8,7 @@ import ( "github.com/irisnet/iris-sync-server/util/constant" "github.com/irisnet/iris-sync-server/model/store/document" "github.com/irisnet/iris-sync-server/module/stake" + "sync" ) func init() { @@ -65,6 +66,7 @@ func Test_saveTx(t *testing.T) { type args struct { tx store.Docs + mutex sync.Mutex } tests := []struct { name string @@ -72,31 +74,39 @@ func Test_saveTx(t *testing.T) { }{ { name:"save tx_coin", - args: struct{ tx store.Docs }{ - tx: docTxCoin,}, + args: args{ + tx:docTxCoin, + mutex:sync.Mutex{}, + }, }, { name:"save tx_stake_declareCandidacy", - args: struct{ tx store.Docs }{ - tx: docTxStakeDeclareCandidacy,}, + args: args{ + tx:docTxStakeDeclareCandidacy, + mutex:sync.Mutex{}, + }, }, { name:"save tx_stake_delegate", - args: struct{ tx store.Docs }{ - tx: docTxStakeDelegate,}, + args:args{ + tx:docTxStakeDelegate, + mutex:sync.Mutex{}, + }, }, { name:"save tx_stake_unBond", - args: struct{ tx store.Docs }{ - tx: docTxStakeUnBond,}, + args: args{ + tx: docTxStakeUnBond, + mutex: sync.Mutex{}, + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - saveTx(tt.args.tx) + saveTx(tt.args.tx, tt.args.mutex) }) } } @@ -110,6 +120,7 @@ func Test_saveOrUpdateAccount(t *testing.T) { type args struct { tx store.Docs + mutex sync.Mutex } tests := []struct { name string @@ -117,31 +128,35 @@ func Test_saveOrUpdateAccount(t *testing.T) { }{ { name:"save tx_coin", - args: struct{ tx store.Docs }{ - tx: docTxCoin,}, + args: args{ + tx: docTxCoin, + }, }, { name:"save tx_stake_declareCandidacy", - args: struct{ tx store.Docs }{ - tx: docTxStakeDeclareCandidacy,}, + args: args{ + tx: docTxStakeDeclareCandidacy, + }, }, { name:"save tx_stake_delegate", - args: struct{ tx store.Docs }{ - tx: docTxStakeDelegate,}, + args: args{ + tx:docTxStakeDelegate, + }, }, { name:"save tx_stake_unBond", - args: struct{ tx store.Docs }{ - tx: docTxStakeUnBond,}, + args:args{ + tx:docTxStakeUnBond, + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - saveOrUpdateAccount(tt.args.tx) + saveOrUpdateAccount(tt.args.tx, tt.args.mutex) }) } } @@ -155,6 +170,7 @@ func Test_updateAccountBalance(t *testing.T) { type args struct { tx store.Docs + mutex sync.Mutex } tests := []struct { name string @@ -162,32 +178,35 @@ func Test_updateAccountBalance(t *testing.T) { }{ { name:"tx_coin", - args: struct{ tx store.Docs }{ - tx: docTxCoin,}, + args: args{ + tx:docTxCoin, + }, }, { name:"tx_stake_declareCandidacy", - args: struct{ tx store.Docs }{ - tx: docTxStakeDeclareCandidacy,}, + args: args{ + tx:docTxStakeDeclareCandidacy, + }, }, { name:"tx_stake_delegate", - args: struct{ tx store.Docs }{ - tx: docTxStakeDelegate,}, + args: args{ + tx:docTxStakeDelegate, + }, }, { name:"tx_stake_unBond", - args: struct{ tx store.Docs }{ - tx: docTxStakeUnBond,}, - + args: args{ + tx:docTxStakeUnBond, + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - updateAccountBalance(tt.args.tx) + updateAccountBalance(tt.args.tx, tt.args.mutex) }) } } diff --git a/sync/sync.go b/sync/sync.go index 9b9b7f1..338ccbc 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -14,7 +14,8 @@ import ( "github.com/robfig/cron" rpcClient "github.com/tendermint/tendermint/rpc/client" "github.com/irisnet/iris-sync-server/model/store/document" - + + "sync" ) var ( @@ -23,6 +24,8 @@ var ( // limit max goroutine limitChan = make(chan int64, conf.SyncMaxGoroutine) + + mutex sync.Mutex ) // start sync server @@ -43,7 +46,7 @@ func InitServer() { if err != nil { if chainId == "" { - logger.Error.Fatalln("sync process start failed,chainId is empty\n") + logger.Error.Fatalln("sync process start failed,chainId is empty") } syncTask = document.SyncTask{ Height: 0, @@ -70,8 +73,11 @@ func watchBlock(c rpcClient.Client) error { syncTask, _ := document.QuerySyncTask() status, _ := c.Status() latestBlockHeight := status.LatestBlockHeight + + // for test + // latestBlockHeight := int64(60010) - funcChain := []func(tx store.Docs){ + funcChain := []func(tx store.Docs, mutex sync.Mutex){ saveTx, saveOrUpdateAccount, updateAccountBalance, } @@ -82,7 +88,7 @@ func watchBlock(c rpcClient.Client) error { select { case <-ch: - logger.Info.Printf("watch block, current height is %v \n", latestBlockHeight) + logger.Info.Printf("Watch block, current height is %v \n", latestBlockHeight) block, _ := c.Block(&latestBlockHeight) syncTask.Height = block.Block.Height syncTask.Time = block.Block.Time @@ -95,8 +101,11 @@ func fastSync(c rpcClient.Client) error { syncTaskDoc, _ := document.QuerySyncTask() status, _ := c.Status() latestBlockHeight := status.LatestBlockHeight + + // for test + // latestBlockHeight := int64(60000) - funcChain := []func(tx store.Docs){ + funcChain := []func(tx store.Docs, mutex sync.Mutex){ saveTx, saveOrUpdateAccount, updateAccountBalance, } @@ -126,8 +135,8 @@ func fastSync(c rpcClient.Client) error { for { select { case threadNo := <-ch: - logger.Info.Printf("threadNo[%d] is over\n", threadNo) activeThreadNum = activeThreadNum - 1 + logger.Info.Printf("ThreadNo[%d] is over and active thread num is %d\n", threadNo, activeThreadNum) if activeThreadNum == 0 { goto end } @@ -136,7 +145,7 @@ func fastSync(c rpcClient.Client) error { end: { - logger.Info.Println("fast sync block, complete sync task") + logger.Info.Println("Fast sync block, complete sync task") // update syncTask document block, _ := c.Block(&latestBlockHeight) syncTaskDoc.Height = block.Block.Height @@ -146,8 +155,8 @@ end: } } -func syncBlock(start int64, end int64, funcChain []func(tx store.Docs), ch chan int64, threadNum int64) { - logger.Info.Printf("threadNo[%d] begin sync block from %d to %d\n", threadNum, start, end) +func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex sync.Mutex), ch chan int64, threadNum int64) { + logger.Info.Printf("ThreadNo[%d] begin sync block from %d to %d\n", threadNum, start, end) client := helper.GetClient() // release client defer client.Release() @@ -157,15 +166,13 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs), ch chan }() for j := start; j <= end; j++ { - logger.Info.Printf("===========threadNo[%d] sync block,height:%d===========\n", threadNum, j) - block, err := client.Client.Block(&j) if err != nil { // try again client2 := helper.GetClient() block, err = client2.Client.Block(&j) if err != nil { - logger.Error.Fatalf("invalid block height %d\n", j) + logger.Error.Fatalf("invalid block height %d and err is %v\n", j, err.Error()) } } if block.BlockMeta.Header.NumTxs > 0 { @@ -180,13 +187,13 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs), ch chan coinTx, _ := tx.(document.CoinTx) coinTx.Height = block.Block.Height coinTx.Time = block.Block.Time - handle(coinTx, funcChain) + handle(coinTx, mutex, funcChain) break case stake.TypeTxDeclareCandidacy: stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) stakeTxDeclareCandidacy.Height = block.Block.Height stakeTxDeclareCandidacy.Time = block.Block.Time - handle(stakeTxDeclareCandidacy, funcChain) + handle(stakeTxDeclareCandidacy, mutex, funcChain) break case stake.TypeTxEditCandidacy: break @@ -194,7 +201,7 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs), ch chan stakeTx, _ := tx.(document.StakeTx) stakeTx.Height = block.Block.Height stakeTx.Time = block.Block.Time - handle(stakeTx, funcChain) + handle(stakeTx, mutex, funcChain) break } } diff --git a/sync/sync_test.go b/sync/sync_test.go index ca37e37..4b1d975 100644 --- a/sync/sync_test.go +++ b/sync/sync_test.go @@ -2,11 +2,15 @@ package sync import ( "testing" - - "github.com/irisnet/iris-sync-server/model/store" - rpcClient "github.com/tendermint/tendermint/rpc/client" - "fmt" "time" + "fmt" + "github.com/robfig/cron" + + conf "github.com/irisnet/iris-sync-server/conf/server" + + rpcClient "github.com/tendermint/tendermint/rpc/client" + "sync" + "github.com/irisnet/iris-sync-server/module/logger" ) func TestStart(t *testing.T) { @@ -18,20 +22,17 @@ func TestStart(t *testing.T) { } func Test_startCron(t *testing.T) { - type args struct { - client rpcClient.Client - } - tests := []struct { - name string - args args - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - startCron(tt.args.client) - }) - } + var wg sync.WaitGroup + wg.Add(2) + + spec := conf.SyncCron + c := cron.New() + c.AddFunc(spec, func() { + logger.Info.Println("print word every second") + }) + go c.Start() + + wg.Wait() } func Test_watchBlock(t *testing.T) { @@ -73,24 +74,3 @@ func Test_fastSync(t *testing.T) { }) } } - -func Test_syncBlock(t *testing.T) { - type args struct { - start int64 - end int64 - funcChain []func(tx store.Docs) - ch chan int64 - threadNum int64 - } - tests := []struct { - name string - args args - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - syncBlock(tt.args.start, tt.args.end, tt.args.funcChain, tt.args.ch, tt.args.threadNum) - }) - } -} diff --git a/util/helper/tx.go b/util/helper/tx.go index fe3c5b7..6cced0c 100644 --- a/util/helper/tx.go +++ b/util/helper/tx.go @@ -97,7 +97,7 @@ func ParseTx(txByte types.Tx) (string, interface{}) { } return kind, stakeTx default: - logger.Info.Printf("unsupported tx type, %+v\n", txi.Unwrap()) + // logger.Info.Printf("unsupported tx type, %+v\n", txi.Unwrap()) } txl, ok = txi.Unwrap().(sdk.TxLayer) From 260be034c23653beb3a1ba45f4ab2cdb39b39bda Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Mon, 14 May 2018 16:45:02 +0800 Subject: [PATCH 04/37] modify config --- .gitignore | 2 +- conf/db/types.go.example | 7 +++++++ conf/server/types.go.example | 13 +++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 conf/db/types.go.example create mode 100644 conf/server/types.go.example diff --git a/.gitignore b/.gitignore index f9a0987..dd30048 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ .idea/ vendor/ -*.conf +conf/**/type.go diff --git a/conf/db/types.go.example b/conf/db/types.go.example new file mode 100644 index 0000000..91b62c6 --- /dev/null +++ b/conf/db/types.go.example @@ -0,0 +1,7 @@ +package db + +const ( + Host = "127.0.0.1" + Port = "27017" + Database = "sync_iris" +) diff --git a/conf/server/types.go.example b/conf/server/types.go.example new file mode 100644 index 0000000..1c76d30 --- /dev/null +++ b/conf/server/types.go.example @@ -0,0 +1,13 @@ +package server + +const ( + BlockChainMonitorUrl = "tcp://127.0.0.1:46657" + Token = "iris" + InitConnectionNum = 1000 + MaxConnectionNum = 2000 + ChainId = "pangu" + SyncCron = "0-59 * * * * *" + + SyncMaxGoroutine = 2000 + SyncBlockNumFastSync = 2000 +) From f5c378296cb2f331c3542feca82fcac609299d91 Mon Sep 17 00:00:00 2001 From: kaifei Hu Date: Mon, 14 May 2018 16:47:21 +0800 Subject: [PATCH 05/37] Delete types.go --- conf/server/types.go | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 conf/server/types.go diff --git a/conf/server/types.go b/conf/server/types.go deleted file mode 100644 index 3b6c173..0000000 --- a/conf/server/types.go +++ /dev/null @@ -1,13 +0,0 @@ -package server - -const ( - BlockChainMonitorUrl = "tcp://127.0.0.1:46757" - Token = "iris" - InitConnectionNum = 1000 - MaxConnectionNum = 2000 - ChainId = "pangu" - SyncCron = "0-59 * * * * *" - - SyncMaxGoroutine = 2000 - SyncBlockNumFastSync = 2000 -) From 27b64f25821c1aa92bc7ec0a20410324f4d5401e Mon Sep 17 00:00:00 2001 From: kaifei Hu Date: Mon, 14 May 2018 16:47:47 +0800 Subject: [PATCH 06/37] Delete types.go --- conf/db/types.go | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 conf/db/types.go diff --git a/conf/db/types.go b/conf/db/types.go deleted file mode 100644 index 4f9b50a..0000000 --- a/conf/db/types.go +++ /dev/null @@ -1,7 +0,0 @@ -package db - -const ( - Host = "127.0.0.1" - Port = "27117" - Database = "sync_iris" -) From 406aca55011149b4da6062b5337ad8f9f0161472 Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Mon, 14 May 2018 16:57:05 +0800 Subject: [PATCH 07/37] .gitignore add config file --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index dd30048..d45a039 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ .idea/ vendor/ -conf/**/type.go +conf/**/types.go From ad1e2544157e6861dbf606fd9f9cc45ae6eb63da Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Tue, 15 May 2018 13:21:02 +0800 Subject: [PATCH 08/37] fix not release lock during sync data --- README.md | 19 +++- model/store/document/stake_role_candidate.go | 2 +- model/store/document/stake_role_delegator.go | 3 +- model/store/store.go | 12 ++- module/logger/logger.go | 13 +++ sync/handler.go | 91 +++++++++++++------- sync/sync.go | 6 ++ util/constant/types.go | 2 + util/helper/account.go | 2 + util/helper/tx.go | 4 +- 10 files changed, 108 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 9b8c6fc..b911029 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,17 @@ -# IRIS-sync-server -A server that synchronize IRIS blockChain data into a database \ No newline at end of file +# IRIS-SYNC-SERVER +A server that synchronize IRIS blockChain data into a database + +# Structure + +- `conf`: config of project +- `model`: database model +- `module`: project module +- `sync`: main logic of sync-server, sync data from blockChain and write to database +- `util`: common constants and helper functions +- `main.go`: bootstrap project + +# Build And Run + +- Build: `make all` +- Run: `make run` +- Cross compilation: `make build-linux` or `make docker-build` \ No newline at end of file diff --git a/model/store/document/stake_role_candidate.go b/model/store/document/stake_role_candidate.go index 2280c44..0a103d8 100644 --- a/model/store/document/stake_role_candidate.go +++ b/model/store/document/stake_role_candidate.go @@ -17,7 +17,7 @@ type Candidate struct { Address string `bson:"address"` // owner PubKey string `bson:"pub_key"` Shares int64 `bson:"shares"` - VotingPower uint64 `bson:"voting_power"` // Voting power if pubKey is a considered a validator + VotingPower int64 `bson:"voting_power"` // Voting power if pubKey is a considered a validator Description Description `bson:"description"` // Description terms for the candidate UpdateTime time.Time `bson:"update_time"` } diff --git a/model/store/document/stake_role_delegator.go b/model/store/document/stake_role_delegator.go index f0ff9e4..529302f 100644 --- a/model/store/document/stake_role_delegator.go +++ b/model/store/document/stake_role_delegator.go @@ -6,7 +6,6 @@ import ( "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" - "github.com/irisnet/iris-sync-server/module/logger" ) const ( @@ -59,7 +58,7 @@ func QueryDelegatorByAddressAndPubkey(address string, pubKey string) (Delegator, } if store.ExecCollection(CollectionNmStakeRoleDelegator, query) != nil { - logger.Info.Println("delegator is Empty") + // logger.Info.Println("delegator is Empty") return result, errors.New("delegator is Empty") } diff --git a/model/store/store.go b/model/store/store.go index 6d5a550..bd74e84 100644 --- a/model/store/store.go +++ b/model/store/store.go @@ -35,13 +35,13 @@ func Init() { } session.SetMode(mgo.Monotonic, true) - //index() + // index() } } func InitWithAuth(addrs []string, username, password string) { dialInfo := &mgo.DialInfo{ - Addrs: addrs, //[]string{"192.168.6.122"} + Addrs: addrs, // []string{"192.168.6.122"} Direct: false, Timeout: time.Second * 1, Database: conf.Database, @@ -99,13 +99,12 @@ func index() { func Save(h Docs) error { save := func(c *mgo.Collection) error { - //先按照关键字查询，如果存在，直接返回 n, _ := c.Find(h.PkKvPair()).Count() if n >= 1 { logger.Info.Println("db: record existed while save data") return nil } - //logger.Info.Printf("insert %s %+v\n", h.Name(), h) + // logger.Info.Printf("insert %s %+v\n", h.Name(), h) return c.Insert(h) } @@ -114,7 +113,6 @@ func Save(h Docs) error { func SaveOrUpdate(h Docs) error { save := func(c *mgo.Collection) error { - //先按照关键字查询，如果存在，直接返回 n, err := c.Find(h.PkKvPair()).Count() if err != nil { logger.Error.Printf("Count:%d err:%+v\n", n, err) @@ -123,7 +121,7 @@ func SaveOrUpdate(h Docs) error { if n >= 1 { return Update(h) } - //logger.Info.Printf("insert %s %+v\n", h.Name(), h) + // logger.Info.Printf("insert %s %+v\n", h.Name(), h) return c.Insert(h) } @@ -133,7 +131,7 @@ func SaveOrUpdate(h Docs) error { func Update(h Docs) error { update := func(c *mgo.Collection) error { key := h.PkKvPair() - //logger.Info.Printf("update %s set %+v where %+v\n", h.Name(), h, key) + // logger.Info.Printf("update %s set %+v where %+v\n", h.Name(), h, key) return c.Update(key, h) } return ExecCollection(h.Name(), update) diff --git a/module/logger/logger.go b/module/logger/logger.go index d574889..9902fca 100644 --- a/module/logger/logger.go +++ b/module/logger/logger.go @@ -8,11 +8,13 @@ import ( var ( Info *log.Logger // Important information + Warning *log.Logger // Warning information Error *log.Logger // Critical problem ) const ( errFile = ".sync_server_err.log" + warningFile = ".sync_server_warning.log" ) func init() { @@ -21,10 +23,20 @@ func init() { if err != nil { log.Fatalln("Failed to open error log file:", err) } + + warningFile, err := os.OpenFile(os.ExpandEnv("$HOME/" + warningFile), + os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + log.Fatalln("Failed to open warning log file:", err) + } Info = log.New(os.Stdout, "INFO: ", log.Ldate|log.Ltime|log.Lshortfile) + + Warning = log.New(io.MultiWriter(warningFile, os.Stderr), + "Warning: ", + log.Ldate|log.Ltime|log.Lshortfile) Error = log.New(io.MultiWriter(errFile, os.Stderr), "ERROR: ", @@ -33,5 +45,6 @@ func init() { func test() { Info.Println("This is info info...") + Warning.Println("This is warning info...") Error.Println("This is err info...") } diff --git a/sync/handler.go b/sync/handler.go index 3b46c3e..551e4be 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -16,6 +16,7 @@ import ( var ( delay = false + methodName string ) @@ -27,40 +28,48 @@ func handle(tx store.Docs, mutex sync.Mutex, funChains []func(tx store.Docs, mut // save Tx document into collection func saveTx(tx store.Docs, mutex sync.Mutex) { + methodName = "SaveTx: " + err := store.Save(tx) if err != nil { - logger.Error.Println(err) + logger.Error.Printf("%v Save failed. tx is %+v, err is %v", + tx, err.Error()) } mutex.Lock() - logger.Info.Println("saveTx method get lock") + logger.Info.Printf("%v get lock\n", methodName) { if tx.Name() == document.CollectionNmStakeTx { if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { - logger.Error.Println("type which is field name of stake tx is missed") + logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) + mutex.Unlock() + logger.Info.Printf("%v release lock\n", methodName) return } stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - logger.Info.Printf("saveTx: type of stakeTx is %v", stakeType) + logger.Info.Printf("%v type of stakeTx is %v\n", methodName, stakeType) switch stakeType { case stake.TypeTxDeclareCandidacy: stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) - candidate, err := document.QueryCandidateByPubkey(stakeTxDeclareCandidacy.PubKey) - // new candidate - if err != nil { - candidate = document.Candidate { - Address: stakeTxDeclareCandidacy.From, - PubKey: stakeTxDeclareCandidacy.PubKey, - Description: stakeTxDeclareCandidacy.Description, - } + cd, err := document.QueryCandidateByPubkey(stakeTxDeclareCandidacy.PubKey) + + candidate := document.Candidate { + Address: stakeTxDeclareCandidacy.From, + PubKey: stakeTxDeclareCandidacy.PubKey, + Description: stakeTxDeclareCandidacy.Description, } // TODO: in further share not equal amount candidate.Shares += stakeTxDeclareCandidacy.Amount.Amount - candidate.VotingPower += uint64(stakeTxDeclareCandidacy.Amount.Amount) + candidate.VotingPower += int64(stakeTxDeclareCandidacy.Amount.Amount) candidate.UpdateTime = stakeTxDeclareCandidacy.Time + + if err != nil && cd.Address == "" { + logger.Warning.Printf("%v Replace candidate from %+v to %+v\n", methodName, cd, candidate) + } + store.SaveOrUpdate(candidate) break case stake.TypeTxDelegate: @@ -68,9 +77,11 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { candidate, err := document.QueryCandidateByPubkey(stakeTx.PubKey) // candidate is not exist if err != nil { - logger.Error.Printf("candidate is not exist while delegate, add = %s ,pub_key = %s\n", - stakeTx.From, stakeTx.PubKey) - return + logger.Warning.Printf("%v candidate is not exist while delegate, add = %s ,pub_key = %s\n", + methodName, stakeTx.From, stakeTx.PubKey) + candidate = document.Candidate{ + PubKey: stakeTx.PubKey, + } } delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) @@ -87,7 +98,7 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { store.SaveOrUpdate(delegator) candidate.Shares += stakeTx.Amount.Amount - candidate.VotingPower += uint64(stakeTx.Amount.Amount) + candidate.VotingPower += int64(stakeTx.Amount.Amount) candidate.UpdateTime = stakeTx.Time store.SaveOrUpdate(candidate) break @@ -96,9 +107,12 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) // delegator is not exist if err != nil { - logger.Info.Printf("delegator is not exist while unBond,add = %s,pub_key=%s\n", - stakeTx.From, stakeTx.PubKey) - return + logger.Warning.Printf("%v delegator is not exist while unBond,add = %s,pub_key=%s\n", + methodName, stakeTx.From, stakeTx.PubKey) + delegator = document.Delegator{ + Address: stakeTx.From, + PubKey: stakeTx.PubKey, + } } delegator.Shares -= stakeTx.Amount.Amount delegator.UpdateTime = stakeTx.Time @@ -107,12 +121,14 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { candidate, err2 := document.QueryCandidateByPubkey(stakeTx.PubKey) // candidate is not exist if err2 != nil { - logger.Info.Printf("candidate is not exist while unBond,add = %s,pub_key=%s\n", - stakeTx.From, stakeTx.PubKey) - return + logger.Warning.Printf("%v candidate is not exist while unBond,add = %s,pub_key=%s\n", + methodName, stakeTx.From, stakeTx.PubKey) + candidate = document.Candidate{ + PubKey: stakeTx.PubKey, + } } candidate.Shares -= stakeTx.Amount.Amount - candidate.VotingPower -= uint64(stakeTx.Amount.Amount) + candidate.VotingPower -= int64(stakeTx.Amount.Amount) candidate.UpdateTime = stakeTx.Time store.Update(candidate) break @@ -124,10 +140,12 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { mutex.Unlock() - logger.Info.Println("saveTx method release lock") + logger.Info.Printf("%v method release lock\n", methodName) } func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { + methodName = "SaveOrUpdateAccount: " + var ( address string updateTime time.Time @@ -142,12 +160,13 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { } if err := store.SaveOrUpdate(account); err != nil { - logger.Error.Printf("account:[%q] balance update failed,%s\n", account.Address, err) + logger.Error.Printf("%v saveOrUpdateAccount failed, account is %v, err is %s\n", + methodName, account.Address, err) } } mutex.Lock() - logger.Info.Println("saveOrUpdateAccpunt method get lock") + logger.Info.Printf("%v get lock\n", methodName) { switch tx.Name() { @@ -161,12 +180,14 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { break case document.CollectionNmStakeTx: if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { - logger.Error.Println("type which is field name of stake tx is missed") + logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) + mutex.Unlock() + logger.Info.Printf("%v release lock\n", methodName) return } stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - logger.Info.Printf("saveOrUpdateAccpunt: type of stakeTx is %v", stakeType) + logger.Info.Printf("%v type of stakeTx is %v", methodName, stakeType) switch stakeType { case stake.TypeTxDeclareCandidacy: stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) @@ -189,11 +210,12 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { } mutex.Unlock() - logger.Info.Println("saveOrUpdateAccpunt method release lock") + logger.Info.Printf("%v release lock\n", methodName) } func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { + methodName = "UpdateAccountBalance: " var ( address string ) @@ -206,12 +228,13 @@ func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { ac := helper.QueryAccountBalance(address, delay) account.Amount = ac.Coins if err := store.Update(account); err != nil { - logger.Error.Printf("account:[%q] balance update failed,%s\n", account.Address, err) + logger.Error.Printf("%v account:[%q] balance update failed,%s\n", + methodName, account.Address, err) } } mutex.Lock() - logger.Info.Println("updateAccountBalance method get lock") + logger.Info.Printf("%v get lock\n", methodName) { switch tx.Name() { case document.CollectionNmCoinTx: @@ -221,7 +244,9 @@ func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { break case document.CollectionNmStakeTx: if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { - logger.Error.Println("type which is field name of stake tx is missed") + logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) + mutex.Unlock() + logger.Info.Printf("%v release lock\n", methodName) return } stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() diff --git a/sync/sync.go b/sync/sync.go index 338ccbc..24dc42b 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -180,6 +180,12 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex syn for _, txByte := range txs { txType, tx := helper.ParseTx(txByte) txHash := strings.ToUpper(hex.EncodeToString(txByte.Hash())) + if txHash == "" { + logger.Warning.Printf("Tx has no hash, skip this tx." + + "" + "type of tx is %v, value of tx is %v\n", + txType, tx) + continue + } logger.Info.Printf("===========threadNo[%d] find tx,txType=%s;txHash=%s\n", threadNum, txType, txHash) switch txType { diff --git a/util/constant/types.go b/util/constant/types.go index 2b95211..c3549cc 100644 --- a/util/constant/types.go +++ b/util/constant/types.go @@ -1,3 +1,5 @@ +// package for define constants + package constant const ( diff --git a/util/helper/account.go b/util/helper/account.go index 88ae1d6..35eccee 100644 --- a/util/helper/account.go +++ b/util/helper/account.go @@ -1,3 +1,5 @@ +// This package is used for query balance of account + package helper import ( diff --git a/util/helper/tx.go b/util/helper/tx.go index 6cced0c..30cace1 100644 --- a/util/helper/tx.go +++ b/util/helper/tx.go @@ -1,3 +1,5 @@ +// package for parse tx struct from binary data + package helper import ( @@ -81,7 +83,7 @@ func ParseTx(txByte types.Tx) (string, interface{}) { return kind, StakeTxDeclareCandidacy case stake.TypeTxEditCandidacy: // TODO：record edit candidacy tx if necessary - //ctx, _ := txi.Unwrap().(stake.TxEditCandidacy) + // ctx, _ := txi.Unwrap().(stake.TxEditCandidacy) break case stake.TypeTxDelegate: ctx, _ := txi.Unwrap().(stake.TxDelegate) From e833f94bb201f8335198be3bc64fe3c0491872de Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Wed, 16 May 2018 11:25:14 +0800 Subject: [PATCH 09/37] fix bug: can't save stakeTx into database --- model/store/document/account.go | 8 +-- model/store/document/tx_stake.go | 2 +- model/store/store.go | 7 +- sync/handler.go | 115 +++++++++++++++++-------------- sync/handler_test.go | 52 ++++++++++---- util/helper/account.go | 2 +- util/helper/account_test.go | 2 +- 7 files changed, 116 insertions(+), 72 deletions(-) diff --git a/model/store/document/account.go b/model/store/document/account.go index f808f35..5cd8884 100644 --- a/model/store/document/account.go +++ b/model/store/document/account.go @@ -6,7 +6,6 @@ import ( "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" - "github.com/irisnet/iris-sync-server/module/logger" ) const ( @@ -45,10 +44,11 @@ func QueryAccount(address string) (Account, error) { err := c.Find(bson.M{"address": address}).Sort("-amount.amount").One(&result) return err } + + err := store.ExecCollection(CollectionNmAccount, query) - if store.ExecCollection(CollectionNmAccount, query) != nil { - logger.Info.Println("Account is Empty") - return result, nil + if err != nil { + return result, err } return result, nil diff --git a/model/store/document/tx_stake.go b/model/store/document/tx_stake.go index 4167101..39a4bde 100644 --- a/model/store/document/tx_stake.go +++ b/model/store/document/tx_stake.go @@ -34,7 +34,7 @@ func (c StakeTx) Index() []mgo.Index { return []mgo.Index{ { Key: []string{"tx_hash"}, - Unique: true, + Unique: false, DropDups: false, Background: true, }, diff --git a/model/store/store.go b/model/store/store.go index bd74e84..319fab2 100644 --- a/model/store/store.go +++ b/model/store/store.go @@ -39,6 +39,10 @@ func Init() { } } +func AddIndex() { + index() +} + func InitWithAuth(addrs []string, username, password string) { dialInfo := &mgo.DialInfo{ Addrs: addrs, // []string{"192.168.6.122"} @@ -99,7 +103,8 @@ func index() { func Save(h Docs) error { save := func(c *mgo.Collection) error { - n, _ := c.Find(h.PkKvPair()).Count() + pk := h.PkKvPair() + n, _ := c.Find(pk).Count() if n >= 1 { logger.Info.Println("db: record existed while save data") return nil diff --git a/sync/handler.go b/sync/handler.go index 551e4be..7049dab 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -29,31 +29,40 @@ func handle(tx store.Docs, mutex sync.Mutex, funChains []func(tx store.Docs, mut // save Tx document into collection func saveTx(tx store.Docs, mutex sync.Mutex) { methodName = "SaveTx: " + txCollectionName := tx.Name() - err := store.Save(tx) - - if err != nil { - logger.Error.Printf("%v Save failed. tx is %+v, err is %v", - tx, err.Error()) + // save tx document into database + storeTxDocFunc := func(doc store.Docs) { + err := store.Save(doc) + if err != nil { + logger.Error.Printf("%v Save failed. doc is %+v, err is %v", + methodName, doc, err.Error()) + } } - - mutex.Lock() - logger.Info.Printf("%v get lock\n", methodName) - - { - if tx.Name() == document.CollectionNmStakeTx { - if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { - logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) - mutex.Unlock() - logger.Info.Printf("%v release lock\n", methodName) - return - } - stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - logger.Info.Printf("%v type of stakeTx is %v\n", methodName, stakeType) - + + switch txCollectionName { + case document.CollectionNmCoinTx: + storeTxDocFunc(tx) + break + case document.CollectionNmStakeTx: + if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { + logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) + break + } + stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() + + logger.Info.Printf("%v handle %v and type is %v", + methodName, txCollectionName, stakeType) + + mutex.Lock() + logger.Info.Printf("%v get lock\n", methodName) + + { switch stakeType { case stake.TypeTxDeclareCandidacy: stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) + storeTxDocFunc(stakeTxDeclareCandidacy) + cd, err := document.QueryCandidateByPubkey(stakeTxDeclareCandidacy.PubKey) candidate := document.Candidate { @@ -74,6 +83,8 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { break case stake.TypeTxDelegate: stakeTx, _ := tx.(document.StakeTx) + storeTxDocFunc(stakeTx) + candidate, err := document.QueryCandidateByPubkey(stakeTx.PubKey) // candidate is not exist if err != nil { @@ -83,7 +94,7 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { PubKey: stakeTx.PubKey, } } - + delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) // delegator is not exist if err != nil { @@ -96,7 +107,7 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { delegator.Shares += stakeTx.Amount.Amount delegator.UpdateTime = stakeTx.Time store.SaveOrUpdate(delegator) - + candidate.Shares += stakeTx.Amount.Amount candidate.VotingPower += int64(stakeTx.Amount.Amount) candidate.UpdateTime = stakeTx.Time @@ -104,6 +115,8 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { break case stake.TypeTxUnbond: stakeTx, _ := tx.(document.StakeTx) + storeTxDocFunc(stakeTx) + delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) // delegator is not exist if err != nil { @@ -116,8 +129,8 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { } delegator.Shares -= stakeTx.Amount.Amount delegator.UpdateTime = stakeTx.Time - store.Update(delegator) - + store.SaveOrUpdate(delegator) + candidate, err2 := document.QueryCandidateByPubkey(stakeTx.PubKey) // candidate is not exist if err2 != nil { @@ -130,27 +143,24 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { candidate.Shares -= stakeTx.Amount.Amount candidate.VotingPower -= int64(stakeTx.Amount.Amount) candidate.UpdateTime = stakeTx.Time - store.Update(candidate) + store.SaveOrUpdate(candidate) break } - } - + + mutex.Unlock() + logger.Info.Printf("%v method release lock\n", methodName) } - - mutex.Unlock() - - logger.Info.Printf("%v method release lock\n", methodName) } func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { - methodName = "SaveOrUpdateAccount: " - var ( address string updateTime time.Time height int64 + methodName = "SaveOrUpdateAccount: " ) + txCollectionName := tx.Name() fun := func(address string, updateTime time.Time, height int64) { account := document.Account{ @@ -164,30 +174,30 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { methodName, account.Address, err) } } - + mutex.Lock() logger.Info.Printf("%v get lock\n", methodName) - + { - switch tx.Name() { + switch txCollectionName { case document.CollectionNmCoinTx: coinTx, _ := tx.(document.CoinTx) updateTime = coinTx.Time height = coinTx.Height - + fun(coinTx.From, updateTime, height) fun(coinTx.To, updateTime, height) break case document.CollectionNmStakeTx: if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) - mutex.Unlock() - logger.Info.Printf("%v release lock\n", methodName) - return + break } stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - logger.Info.Printf("%v type of stakeTx is %v", methodName, stakeType) + logger.Info.Printf("%v handle %v and type is %v", + methodName, txCollectionName, stakeType) + switch stakeType { case stake.TypeTxDeclareCandidacy: stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) @@ -204,26 +214,30 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { height = stakeTx.Height break } + fun(address, updateTime, height) } - } - + mutex.Unlock() logger.Info.Printf("%v release lock\n", methodName) - } func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { - methodName = "UpdateAccountBalance: " var ( address string + methodName = "UpdateAccountBalance: " ) + txCollectionName := tx.Name() + fun := func(address string) { account, err := document.QueryAccount(address) if err != nil { + logger.Error.Printf("%v updateAccountBalance failed, account is %v and err is %v", + methodName, account, err.Error()) return } + // query balance of account ac := helper.QueryAccountBalance(address, delay) account.Amount = ac.Coins @@ -236,7 +250,7 @@ func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { mutex.Lock() logger.Info.Printf("%v get lock\n", methodName) { - switch tx.Name() { + switch txCollectionName { case document.CollectionNmCoinTx: coinTx, _ := tx.(document.CoinTx) fun(coinTx.From) @@ -245,12 +259,13 @@ func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { case document.CollectionNmStakeTx: if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) - mutex.Unlock() - logger.Info.Printf("%v release lock\n", methodName) - return + break } stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - + + logger.Info.Printf("%v handle %v and type is %v", + methodName, txCollectionName, stakeType) + switch stakeType { case stake.TypeTxDeclareCandidacy: stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) diff --git a/sync/handler_test.go b/sync/handler_test.go index 87586da..a73c3dc 100644 --- a/sync/handler_test.go +++ b/sync/handler_test.go @@ -59,10 +59,11 @@ func buildDocData(blockHeight int64) store.Docs { func Test_saveTx(t *testing.T) { - docTxCoin := buildDocData(12453) - docTxStakeDeclareCandidacy := buildDocData(19073) - docTxStakeDelegate := buildDocData(13725) - docTxStakeUnBond := buildDocData(14260) + docTxCoin := buildDocData(1756) + docTxStakeDeclareCandidacy := buildDocData(1921) + docTxStakeDeclareCandidacy2 := buildDocData(1927) + docTxStakeDelegate := buildDocData(115135) + docTxStakeUnBond := buildDocData(397201) type args struct { tx store.Docs @@ -87,6 +88,13 @@ func Test_saveTx(t *testing.T) { }, }, + { + name:"save tx_stake_declareCandidacy2", + args: args{ + tx:docTxStakeDeclareCandidacy2, + mutex:sync.Mutex{}, + }, + }, { name:"save tx_stake_delegate", args:args{ @@ -112,11 +120,12 @@ func Test_saveTx(t *testing.T) { } func Test_saveOrUpdateAccount(t *testing.T) { - - docTxCoin := buildDocData(12453) - docTxStakeDeclareCandidacy := buildDocData(19073) - docTxStakeDelegate := buildDocData(13725) - docTxStakeUnBond := buildDocData(14260) + + docTxCoin := buildDocData(1756) + docTxStakeDeclareCandidacy := buildDocData(1921) + docTxStakeDeclareCandidacy2 := buildDocData(1927) + docTxStakeDelegate := buildDocData(115135) + docTxStakeUnBond := buildDocData(397201) type args struct { tx store.Docs @@ -138,6 +147,13 @@ func Test_saveOrUpdateAccount(t *testing.T) { tx: docTxStakeDeclareCandidacy, }, + }, + { + name:"save tx_stake_declareCandidacy2", + args: args{ + tx: docTxStakeDeclareCandidacy2, + }, + }, { name:"save tx_stake_delegate", @@ -162,11 +178,12 @@ func Test_saveOrUpdateAccount(t *testing.T) { } func Test_updateAccountBalance(t *testing.T) { - - docTxCoin := buildDocData(12453) - docTxStakeDeclareCandidacy := buildDocData(19073) - docTxStakeDelegate := buildDocData(13725) - docTxStakeUnBond := buildDocData(14260) + + docTxCoin := buildDocData(1756) + docTxStakeDeclareCandidacy := buildDocData(1921) + docTxStakeDeclareCandidacy2 := buildDocData(1927) + docTxStakeDelegate := buildDocData(115135) + docTxStakeUnBond := buildDocData(397201) type args struct { tx store.Docs @@ -188,6 +205,13 @@ func Test_updateAccountBalance(t *testing.T) { tx:docTxStakeDeclareCandidacy, }, + }, + { + name:"tx_stake_declareCandidacy2", + args: args{ + tx:docTxStakeDeclareCandidacy2, + }, + }, { name:"tx_stake_delegate", diff --git a/util/helper/account.go b/util/helper/account.go index 35eccee..c7aceb6 100644 --- a/util/helper/account.go +++ b/util/helper/account.go @@ -39,7 +39,7 @@ func QueryAccountBalance(address string, delay bool) *coin.Account { } _, err2 := GetParsed(key, account, query.GetHeight(), false) if err2 != nil { - logger.Info.Printf("account bytes are empty for address: %q\n", address) + logger.Info.Printf("QueryAccountBalance failed, account bytes are empty for address: %q\n", address) } return account } diff --git a/util/helper/account_test.go b/util/helper/account_test.go index d0c9324..2076d9d 100644 --- a/util/helper/account_test.go +++ b/util/helper/account_test.go @@ -27,7 +27,7 @@ func TestQueryAccountBalance(t *testing.T) { address string delay bool }{ - address: "ADBC4AAB3A089BDC8A8155AB97E64CD2CF4A0E9F", + address: "37D2C544F9D2CF811108B56A496520129B1F80CC", delay: false}, }, { From bc2504af35ad0ee996598d71ecb66d590dc1218b Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Thu, 17 May 2018 11:13:09 +0800 Subject: [PATCH 10/37] fix: go routine blocked --- model/store/document/stake_role_delegator.go | 7 +- model/store/store.go | 20 ++-- module/logger/logger.go | 32 +++++-- sync/handler.go | 96 ++++++++++---------- sync/sync.go | 35 +++---- util/helper/pool_client.go | 31 +++---- 6 files changed, 117 insertions(+), 104 deletions(-) diff --git a/model/store/document/stake_role_delegator.go b/model/store/document/stake_role_delegator.go index 529302f..4e95cfa 100644 --- a/model/store/document/stake_role_delegator.go +++ b/model/store/document/stake_role_delegator.go @@ -13,9 +13,9 @@ const ( ) type Delegator struct { - Address string `bson:"address"` - PubKey string `bson:"pub_key"` - Shares int64 `bson:"shares"` + Address string `bson:"address"` + PubKey string `bson:"pub_key"` + Shares int64 `bson:"shares"` UpdateTime time.Time `bson:"update_time"` } @@ -58,7 +58,6 @@ func QueryDelegatorByAddressAndPubkey(address string, pubKey string) (Delegator, } if store.ExecCollection(CollectionNmStakeRoleDelegator, query) != nil { - // logger.Info.Println("delegator is Empty") return result, errors.New("delegator is Empty") } diff --git a/model/store/store.go b/model/store/store.go index 319fab2..cc0322e 100644 --- a/model/store/store.go +++ b/model/store/store.go @@ -3,12 +3,13 @@ package store import ( - "fmt" "time" conf "github.com/irisnet/iris-sync-server/conf/db" "github.com/irisnet/iris-sync-server/module/logger" + "errors" + "fmt" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" ) @@ -26,20 +27,17 @@ func Init() { if session == nil { url := fmt.Sprintf("mongodb://%s:%s", conf.Host, conf.Port) - logger.Info.Printf("Mgo start on %s\n", url) - var err error session, err = mgo.Dial(url) if err != nil { logger.Error.Fatalln(err) } + logger.Info.Printf("Mgo start on %s\n", url) session.SetMode(mgo.Monotonic, true) - - // index() } } -func AddIndex() { +func AddIndex() { index() } @@ -106,13 +104,13 @@ func Save(h Docs) error { pk := h.PkKvPair() n, _ := c.Find(pk).Count() if n >= 1 { - logger.Info.Println("db: record existed while save data") - return nil + errMsg := fmt.Sprintf("Record existed while save %v, data is %+v\n", + h.Name(), h) + return errors.New(errMsg) } // logger.Info.Printf("insert %s %+v\n", h.Name(), h) return c.Insert(h) } - return ExecCollection(h.Name(), save) } @@ -126,7 +124,7 @@ func SaveOrUpdate(h Docs) error { if n >= 1 { return Update(h) } - // logger.Info.Printf("insert %s %+v\n", h.Name(), h) + // logger.Trace.Printf("insert %s %+v\n", h.Name(), h) return c.Insert(h) } @@ -136,7 +134,7 @@ func SaveOrUpdate(h Docs) error { func Update(h Docs) error { update := func(c *mgo.Collection) error { key := h.PkKvPair() - // logger.Info.Printf("update %s set %+v where %+v\n", h.Name(), h, key) + // logger.Trace.Printf("update %s set %+v where %+v\n", h.Name(), h, key) return c.Update(key, h) } return ExecCollection(h.Name(), update) diff --git a/module/logger/logger.go b/module/logger/logger.go index 9902fca..d98b473 100644 --- a/module/logger/logger.go +++ b/module/logger/logger.go @@ -7,35 +7,47 @@ import ( ) var ( - Info *log.Logger // Important information + Trace *log.Logger // Trace db log + Info *log.Logger // Important information Warning *log.Logger // Warning information - Error *log.Logger // Critical problem + Error *log.Logger // Critical problem ) const ( - errFile = ".sync_server_err.log" + traceDbFile = ".sync_server_db.log" + errFile = ".sync_server_err.log" warningFile = ".sync_server_warning.log" ) func init() { - errFile, err := os.OpenFile(os.ExpandEnv("$HOME/" + errFile), + errFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+errFile), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err != nil { log.Fatalln("Failed to open error log file:", err) } - - warningFile, err := os.OpenFile(os.ExpandEnv("$HOME/" + warningFile), + + warningFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+warningFile), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err != nil { log.Fatalln("Failed to open warning log file:", err) } + traceDbFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+traceDbFile), + os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + log.Fatalln("Failed to open trace db log file:", err) + } + + Trace = log.New(io.MultiWriter(traceDbFile), + "TRACE: ", + log.Ldate|log.Ltime|log.Lshortfile) + Info = log.New(os.Stdout, "INFO: ", log.Ldate|log.Ltime|log.Lshortfile) - + Warning = log.New(io.MultiWriter(warningFile, os.Stderr), - "Warning: ", + "WARNING: ", log.Ldate|log.Ltime|log.Lshortfile) Error = log.New(io.MultiWriter(errFile, os.Stderr), @@ -44,7 +56,11 @@ func init() { } func test() { + Trace.Println("This is trace info...") Info.Println("This is info info...") Warning.Println("This is warning info...") Error.Println("This is err info...") + + Error.Fatalf("This is err") + Info.Println("test") } diff --git a/sync/handler.go b/sync/handler.go index 7049dab..1232a79 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -2,24 +2,22 @@ package sync import ( "reflect" - "time" "sync" + "time" "github.com/irisnet/iris-sync-server/model/store" + "github.com/irisnet/iris-sync-server/model/store/document" "github.com/irisnet/iris-sync-server/module/logger" "github.com/irisnet/iris-sync-server/module/stake" - "github.com/irisnet/iris-sync-server/util/helper" - "github.com/irisnet/iris-sync-server/model/store/document" "github.com/irisnet/iris-sync-server/util/constant" - + "github.com/irisnet/iris-sync-server/util/helper" ) var ( - delay = false + delay = false methodName string ) - func handle(tx store.Docs, mutex sync.Mutex, funChains []func(tx store.Docs, mutex sync.Mutex)) { for _, fun := range funChains { fun(tx, mutex) @@ -30,7 +28,7 @@ func handle(tx store.Docs, mutex sync.Mutex, funChains []func(tx store.Docs, mut func saveTx(tx store.Docs, mutex sync.Mutex) { methodName = "SaveTx: " txCollectionName := tx.Name() - + // save tx document into database storeTxDocFunc := func(doc store.Docs) { err := store.Save(doc) @@ -39,7 +37,7 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { methodName, doc, err.Error()) } } - + switch txCollectionName { case document.CollectionNmCoinTx: storeTxDocFunc(tx) @@ -50,22 +48,22 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { break } stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - + logger.Info.Printf("%v handle %v and type is %v", methodName, txCollectionName, stakeType) - + mutex.Lock() logger.Info.Printf("%v get lock\n", methodName) - + { switch stakeType { case stake.TypeTxDeclareCandidacy: stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) storeTxDocFunc(stakeTxDeclareCandidacy) - + cd, err := document.QueryCandidateByPubkey(stakeTxDeclareCandidacy.PubKey) - - candidate := document.Candidate { + + candidate := document.Candidate{ Address: stakeTxDeclareCandidacy.From, PubKey: stakeTxDeclareCandidacy.PubKey, Description: stakeTxDeclareCandidacy.Description, @@ -74,17 +72,17 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { candidate.Shares += stakeTxDeclareCandidacy.Amount.Amount candidate.VotingPower += int64(stakeTxDeclareCandidacy.Amount.Amount) candidate.UpdateTime = stakeTxDeclareCandidacy.Time - + if err != nil && cd.Address == "" { logger.Warning.Printf("%v Replace candidate from %+v to %+v\n", methodName, cd, candidate) } - + store.SaveOrUpdate(candidate) break case stake.TypeTxDelegate: stakeTx, _ := tx.(document.StakeTx) storeTxDocFunc(stakeTx) - + candidate, err := document.QueryCandidateByPubkey(stakeTx.PubKey) // candidate is not exist if err != nil { @@ -94,20 +92,20 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { PubKey: stakeTx.PubKey, } } - + delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) // delegator is not exist if err != nil { delegator = document.Delegator{ Address: stakeTx.From, - PubKey: stakeTx.PubKey, + PubKey: stakeTx.PubKey, } } // TODO: in further share not equal amount delegator.Shares += stakeTx.Amount.Amount delegator.UpdateTime = stakeTx.Time store.SaveOrUpdate(delegator) - + candidate.Shares += stakeTx.Amount.Amount candidate.VotingPower += int64(stakeTx.Amount.Amount) candidate.UpdateTime = stakeTx.Time @@ -116,7 +114,7 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { case stake.TypeTxUnbond: stakeTx, _ := tx.(document.StakeTx) storeTxDocFunc(stakeTx) - + delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) // delegator is not exist if err != nil { @@ -124,13 +122,13 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { methodName, stakeTx.From, stakeTx.PubKey) delegator = document.Delegator{ Address: stakeTx.From, - PubKey: stakeTx.PubKey, + PubKey: stakeTx.PubKey, } } delegator.Shares -= stakeTx.Amount.Amount delegator.UpdateTime = stakeTx.Time store.SaveOrUpdate(delegator) - + candidate, err2 := document.QueryCandidateByPubkey(stakeTx.PubKey) // candidate is not exist if err2 != nil { @@ -147,7 +145,7 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { break } } - + mutex.Unlock() logger.Info.Printf("%v method release lock\n", methodName) } @@ -155,9 +153,9 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { var ( - address string + address string updateTime time.Time - height int64 + height int64 methodName = "SaveOrUpdateAccount: " ) txCollectionName := tx.Name() @@ -169,22 +167,22 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { Height: height, } - if err := store.SaveOrUpdate(account); err != nil { - logger.Error.Printf("%v saveOrUpdateAccount failed, account is %v, err is %s\n", - methodName, account.Address, err) + if err := store.Save(account); err != nil { + logger.Warning.Printf("%v Record exists, account is %v, err is %s\n", + methodName, account.Address, err.Error()) } } - - mutex.Lock() - logger.Info.Printf("%v get lock\n", methodName) - + + // mutex.Lock() + // logger.Info.Printf("%v get lock\n", methodName) + { switch txCollectionName { case document.CollectionNmCoinTx: coinTx, _ := tx.(document.CoinTx) updateTime = coinTx.Time height = coinTx.Height - + fun(coinTx.From, updateTime, height) fun(coinTx.To, updateTime, height) break @@ -194,10 +192,10 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { break } stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - + logger.Info.Printf("%v handle %v and type is %v", methodName, txCollectionName, stakeType) - + switch stakeType { case stake.TypeTxDeclareCandidacy: stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) @@ -214,22 +212,22 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { height = stakeTx.Height break } - + fun(address, updateTime, height) } } - - mutex.Unlock() - logger.Info.Printf("%v release lock\n", methodName) + + // mutex.Unlock() + // logger.Info.Printf("%v release lock\n", methodName) } func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { var ( - address string + address string methodName = "UpdateAccountBalance: " ) txCollectionName := tx.Name() - + fun := func(address string) { account, err := document.QueryAccount(address) if err != nil { @@ -237,7 +235,7 @@ func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { methodName, account, err.Error()) return } - + // query balance of account ac := helper.QueryAccountBalance(address, delay) account.Amount = ac.Coins @@ -247,8 +245,8 @@ func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { } } - mutex.Lock() - logger.Info.Printf("%v get lock\n", methodName) + // mutex.Lock() + // logger.Info.Printf("%v get lock\n", methodName) { switch txCollectionName { case document.CollectionNmCoinTx: @@ -262,10 +260,10 @@ func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { break } stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - + logger.Info.Printf("%v handle %v and type is %v", methodName, txCollectionName, stakeType) - + switch stakeType { case stake.TypeTxDeclareCandidacy: stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) @@ -283,7 +281,7 @@ func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { } } - mutex.Unlock() - logger.Info.Println("updateAccountBalance method release lock") + // mutex.Unlock() + // logger.Info.Println("updateAccountBalance method release lock") } diff --git a/sync/sync.go b/sync/sync.go index 24dc42b..dbccf1b 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -1,20 +1,20 @@ package sync import ( - "strings" "encoding/hex" + "strings" conf "github.com/irisnet/iris-sync-server/conf/server" "github.com/irisnet/iris-sync-server/model/store" "github.com/irisnet/iris-sync-server/module/logger" - "github.com/irisnet/iris-sync-server/util/helper" - "github.com/irisnet/iris-sync-server/util/constant" "github.com/irisnet/iris-sync-server/module/stake" + "github.com/irisnet/iris-sync-server/util/constant" + "github.com/irisnet/iris-sync-server/util/helper" + "github.com/irisnet/iris-sync-server/model/store/document" "github.com/robfig/cron" rpcClient "github.com/tendermint/tendermint/rpc/client" - "github.com/irisnet/iris-sync-server/model/store/document" - + "sync" ) @@ -23,8 +23,8 @@ var ( syncBlockNumFastSync = int64(conf.SyncBlockNumFastSync) // limit max goroutine - limitChan = make(chan int64, conf.SyncMaxGoroutine) - + // limitChan = make(chan int64, conf.SyncMaxGoroutine) + mutex sync.Mutex ) @@ -73,7 +73,7 @@ func watchBlock(c rpcClient.Client) error { syncTask, _ := document.QuerySyncTask() status, _ := c.Status() latestBlockHeight := status.LatestBlockHeight - + // for test // latestBlockHeight := int64(60010) @@ -82,7 +82,7 @@ func watchBlock(c rpcClient.Client) error { } ch := make(chan int64) - limitChan <- 1 + // limitChan <- 1 go syncBlock(syncTask.Height+1, latestBlockHeight, funcChain, ch, 0) @@ -101,7 +101,7 @@ func fastSync(c rpcClient.Client) error { syncTaskDoc, _ := document.QuerySyncTask() status, _ := c.Status() latestBlockHeight := status.LatestBlockHeight - + // for test // latestBlockHeight := int64(60000) @@ -121,7 +121,7 @@ func fastSync(c rpcClient.Client) error { for i := int64(1); i <= goRoutineNum; i++ { activeThreadNum++ - limitChan <- i + // limitChan <- i var ( start = syncTaskDoc.Height + (i-1)*syncBlockNumFastSync + 1 end = syncTaskDoc.Height + i*syncBlockNumFastSync @@ -160,18 +160,22 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex syn client := helper.GetClient() // release client defer client.Release() - // release buffer chain + // release unBuffer chain and buffer chain defer func() { - <-limitChan + ch <- threadNum + logger.Info.Printf("Send threadNum into channel: %v\n", threadNum) + // <- limitChan }() for j := start; j <= end; j++ { block, err := client.Client.Block(&j) if err != nil { + logger.Error.Printf("Invalid block height %d and err is %v, try again\n", j, err.Error()) // try again client2 := helper.GetClient() block, err = client2.Client.Block(&j) if err != nil { + ch <- threadNum logger.Error.Fatalf("invalid block height %d and err is %v\n", j, err.Error()) } } @@ -181,8 +185,8 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex syn txType, tx := helper.ParseTx(txByte) txHash := strings.ToUpper(hex.EncodeToString(txByte.Hash())) if txHash == "" { - logger.Warning.Printf("Tx has no hash, skip this tx." + - "" + "type of tx is %v, value of tx is %v\n", + logger.Warning.Printf("Tx has no hash, skip this tx."+ + ""+"type of tx is %v, value of tx is %v\n", txType, tx) continue } @@ -222,5 +226,4 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex syn store.SaveOrUpdate(bk) } - ch <- threadNum } diff --git a/util/helper/pool_client.go b/util/helper/pool_client.go index a52136b..e453698 100644 --- a/util/helper/pool_client.go +++ b/util/helper/pool_client.go @@ -5,11 +5,10 @@ package helper import ( "errors" - "log" - conf "github.com/irisnet/iris-sync-server/conf/server" rpcClient "github.com/cosmos/cosmos-sdk/client" + "github.com/irisnet/iris-sync-server/module/logger" "github.com/tendermint/tendermint/rpc/client" ) @@ -47,7 +46,7 @@ func InitClientPool() { func GetClient() Client { c, err := getClient() if err != nil { - log.Fatal(err) + logger.Error.Fatalln(err.Error()) } return c } @@ -61,38 +60,38 @@ func (n Client) Release() { } func createConnection(id int64) Client { - client := Client{ + tmClient := Client{ Client: rpcClient.GetNode(conf.BlockChainMonitorUrl), used: false, id: id, } - pool.clients[id] = client + pool.clients[id] = tmClient pool.available++ - return client + return tmClient } func getClient() (Client, error) { if pool.available == 0 { maxConnNum := int64(conf.MaxConnectionNum) if pool.used < maxConnNum { - var client Client + var tmClient Client for i := int64(len(pool.clients)); i < maxConnNum; i++ { - client = createConnection(i) + tmClient = createConnection(i) } - return client, nil + return tmClient, nil } else { - log.Fatal("client pool has no available connection") + logger.Error.Fatalln("client pool has no available connection") } } - for _, client := range pool.clients { - if !client.used { - client.used = true - pool.clients[client.id] = client + for _, tmClient := range pool.clients { + if !tmClient.used { + tmClient.used = true + pool.clients[tmClient.id] = tmClient pool.available-- pool.used++ - log.Printf("current available coonection ：%d", pool.available) - return client, nil + logger.Info.Printf("current available coonection ：%d\n", pool.available) + return tmClient, nil } } return Client{}, errors.New("pool is empty") From 5c3e6c9e0f3dad56a71c3420bec213448660b63d Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Fri, 18 May 2018 11:51:51 +0800 Subject: [PATCH 11/37] optimize code: add log --- module/logger/logger.go | 2 +- sync/handler.go | 70 +++++++++++++++++++++++++++++++++++------ sync/sync.go | 16 +++++++--- sync/sync_test.go | 21 +++++++++---- 4 files changed, 88 insertions(+), 21 deletions(-) diff --git a/module/logger/logger.go b/module/logger/logger.go index d98b473..568d4f9 100644 --- a/module/logger/logger.go +++ b/module/logger/logger.go @@ -38,7 +38,7 @@ func init() { log.Fatalln("Failed to open trace db log file:", err) } - Trace = log.New(io.MultiWriter(traceDbFile), + Trace = log.New(io.MultiWriter(traceDbFile, os.Stdout), "TRACE: ", log.Ldate|log.Ltime|log.Lshortfile) diff --git a/sync/handler.go b/sync/handler.go index 1232a79..714df04 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -58,7 +58,12 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { { switch stakeType { case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) + stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, stakeType) + break + } storeTxDocFunc(stakeTxDeclareCandidacy) cd, err := document.QueryCandidateByPubkey(stakeTxDeclareCandidacy.PubKey) @@ -80,7 +85,12 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { store.SaveOrUpdate(candidate) break case stake.TypeTxDelegate: - stakeTx, _ := tx.(document.StakeTx) + stakeTx, r := tx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, stakeType) + break + } storeTxDocFunc(stakeTx) candidate, err := document.QueryCandidateByPubkey(stakeTx.PubKey) @@ -112,7 +122,12 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { store.SaveOrUpdate(candidate) break case stake.TypeTxUnbond: - stakeTx, _ := tx.(document.StakeTx) + stakeTx, r := tx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, stakeType) + break + } storeTxDocFunc(stakeTx) delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) @@ -159,6 +174,7 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { methodName = "SaveOrUpdateAccount: " ) txCollectionName := tx.Name() + logger.Info.Printf("Start %v\n", methodName) fun := func(address string, updateTime time.Time, height int64) { account := document.Account{ @@ -168,7 +184,7 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { } if err := store.Save(account); err != nil { - logger.Warning.Printf("%v Record exists, account is %v, err is %s\n", + logger.Trace.Printf("%v Record exists, account is %v, err is %s\n", methodName, account.Address, err.Error()) } } @@ -179,7 +195,12 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { { switch txCollectionName { case document.CollectionNmCoinTx: - coinTx, _ := tx.(document.CoinTx) + coinTx, r := tx.(document.CoinTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, txCollectionName) + break + } updateTime = coinTx.Time height = coinTx.Height @@ -198,7 +219,12 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { switch stakeType { case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) + stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, stakeType) + break + } address = stakeTxDeclareCandidacy.From updateTime = stakeTxDeclareCandidacy.Time height = stakeTxDeclareCandidacy.Height @@ -206,7 +232,12 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { case stake.TypeTxEditCandidacy: break case stake.TypeTxDelegate, stake.TypeTxUnbond: - stakeTx, _ := tx.(document.StakeTx) + stakeTx, r := tx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, stakeType) + break + } address = stakeTx.From updateTime = stakeTx.Time height = stakeTx.Height @@ -217,6 +248,7 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { } } + logger.Info.Printf("End %v\n", methodName) // mutex.Unlock() // logger.Info.Printf("%v release lock\n", methodName) } @@ -227,6 +259,7 @@ func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { methodName = "UpdateAccountBalance: " ) txCollectionName := tx.Name() + logger.Info.Printf("Start %v\n", methodName) fun := func(address string) { account, err := document.QueryAccount(address) @@ -250,7 +283,12 @@ func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { { switch txCollectionName { case document.CollectionNmCoinTx: - coinTx, _ := tx.(document.CoinTx) + coinTx, r := tx.(document.CoinTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, txCollectionName) + break + } fun(coinTx.From) fun(coinTx.To) break @@ -266,13 +304,23 @@ func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { switch stakeType { case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) + stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, stakeType) + break + } address = stakeTxDeclareCandidacy.From break case stake.TypeTxEditCandidacy: break case stake.TypeTxDelegate, stake.TypeTxUnbond: - stakeTx, _ := tx.(document.StakeTx) + stakeTx, r := tx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, stakeType) + break + } address = stakeTx.From break } @@ -280,6 +328,8 @@ func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { break } } + + logger.Info.Printf("End %v\n", methodName) // mutex.Unlock() // logger.Info.Println("updateAccountBalance method release lock") diff --git a/sync/sync.go b/sync/sync.go index dbccf1b..55e1f38 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -156,7 +156,9 @@ end: } func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex sync.Mutex), ch chan int64, threadNum int64) { - logger.Info.Printf("ThreadNo[%d] begin sync block from %d to %d\n", threadNum, start, end) + logger.Info.Printf("ThreadNo[%d] begin sync block from %d to %d\n", + threadNum, start, end) + client := helper.GetClient() // release client defer client.Release() @@ -176,7 +178,7 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex syn block, err = client2.Client.Block(&j) if err != nil { ch <- threadNum - logger.Error.Fatalf("invalid block height %d and err is %v\n", j, err.Error()) + logger.Error.Fatalf("Invalid block height %d and err is %v\n", j, err.Error()) } } if block.BlockMeta.Header.NumTxs > 0 { @@ -223,7 +225,13 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex syn Time: block.Block.Time, TxNum: block.BlockMeta.Header.NumTxs, } - store.SaveOrUpdate(bk) - + if err := store.Save(bk); err != nil { + logger.Error.Printf("Save block info failed, err is %v", + err.Error()) + } } + + logger.Info.Printf("ThreadNo[%d] finish sync block from %d to %d\n", + threadNum, start, end) + } diff --git a/sync/sync_test.go b/sync/sync_test.go index 4b1d975..04ac262 100644 --- a/sync/sync_test.go +++ b/sync/sync_test.go @@ -2,8 +2,6 @@ package sync import ( "testing" - "time" - "fmt" "github.com/robfig/cron" conf "github.com/irisnet/iris-sync-server/conf/server" @@ -14,10 +12,21 @@ import ( ) func TestStart(t *testing.T) { - Start() - for true { - time.Sleep(time.Minute) - fmt.Printf("wait\n") + var ( + num int + ) + num = 2 + switch num { + case 1: + + break + case 2: + if num > 1 { + logger.Info.Println("num gt 1") + break + } + logger.Info.Println("num is 2") + break } } From 4994728f8f8946b5520b529dd2f7b58538915f04 Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Wed, 23 May 2018 11:31:58 +0800 Subject: [PATCH 12/37] fix problem: main thread will be blocked when open too many sub thread --- .gitignore | 4 ++ module/logger/logger.go | 24 ++++++---- module/stake/tx.go | 4 +- sync/sync.go | 100 +++++++++++++++++++++++++++------------- sync/sync_test.go | 92 ++++++++++++++++-------------------- util/helper/account.go | 6 +-- util/helper/tx.go | 1 - 7 files changed, 130 insertions(+), 101 deletions(-) diff --git a/.gitignore b/.gitignore index d45a039..c53a361 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ .idea/ vendor/ conf/**/types.go +.DS_Store +sync-iris +sync-iris-unix + diff --git a/module/logger/logger.go b/module/logger/logger.go index 568d4f9..9ae2b6a 100644 --- a/module/logger/logger.go +++ b/module/logger/logger.go @@ -15,34 +15,41 @@ var ( const ( traceDbFile = ".sync_server_db.log" + debugFile = ".sync_server_debug.log" errFile = ".sync_server_err.log" warningFile = ".sync_server_warning.log" ) func init() { - errFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+errFile), + traceDbFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+traceDbFile), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err != nil { - log.Fatalln("Failed to open error log file:", err) + log.Fatalln("Failed to open trace db log file:", err) } - + + debugFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+debugFile), + os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + log.Fatalln("Failed to open debug log file:", err) + } + warningFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+warningFile), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err != nil { log.Fatalln("Failed to open warning log file:", err) } - - traceDbFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+traceDbFile), + + errFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+errFile), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err != nil { - log.Fatalln("Failed to open trace db log file:", err) + log.Fatalln("Failed to open error log file:", err) } Trace = log.New(io.MultiWriter(traceDbFile, os.Stdout), "TRACE: ", log.Ldate|log.Ltime|log.Lshortfile) - Info = log.New(os.Stdout, + Info = log.New(io.MultiWriter(debugFile, os.Stdout), "INFO: ", log.Ldate|log.Ltime|log.Lshortfile) @@ -60,7 +67,4 @@ func test() { Info.Println("This is info info...") Warning.Println("This is warning info...") Error.Println("This is err info...") - - Error.Fatalf("This is err") - Info.Println("test") } diff --git a/module/stake/tx.go b/module/stake/tx.go index 2202687..e5f51e3 100644 --- a/module/stake/tx.go +++ b/module/stake/tx.go @@ -8,7 +8,7 @@ import ( crypto "github.com/tendermint/go-crypto" ) -//TODO 当cosmos-sdk加入stake模块时需要删除该文件，防止在启动时，重复加载ByteTxDeclareCandidacy，ByteTxEditCandidacy等类型，导致启动失败 +// TODO 当cosmos-sdk加入stake模块时需要删除该文件，防止在启动时，重复加载ByteTxDeclareCandidacy，ByteTxEditCandidacy等类型，导致启动失败 const stakingModuleName = "stake" @@ -49,7 +49,7 @@ func init() { sdk.TxMapper.RegisterImplementation(TxUnbond{}, TypeTxUnbond, ByteTxUnbond) } -//Verify interface at compile time +// Verify interface at compile time var _, _, _, _ sdk.TxInner = &TxDeclareCandidacy{}, &TxEditCandidacy{}, &TxDelegate{}, &TxUnbond{} // BondUpdate - struct for bonding or unbonding transactions diff --git a/sync/sync.go b/sync/sync.go index 55e1f38..6df2d63 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -14,6 +14,7 @@ import ( "github.com/irisnet/iris-sync-server/model/store/document" "github.com/robfig/cron" rpcClient "github.com/tendermint/tendermint/rpc/client" + ctypes "github.com/tendermint/tendermint/rpc/core/types" "sync" ) @@ -23,18 +24,43 @@ var ( syncBlockNumFastSync = int64(conf.SyncBlockNumFastSync) // limit max goroutine - // limitChan = make(chan int64, conf.SyncMaxGoroutine) + limitChan = make(chan int64, conf.SyncMaxGoroutine) mutex sync.Mutex + mutexWatchBlock sync.Mutex ) // start sync server func Start() { + var ( + status *ctypes.ResultStatus + err error + i = 1 + ) InitServer() c := helper.GetClient().Client - if err := fastSync(c); err != nil { - logger.Error.Fatalf("sync block failed,%v\n", err) + + for { + logger.Info.Printf("Begin %v time fast sync task", i) + syncLatestHeight := fastSync(c) + status, err = c.Status() + if err != nil { + logger.Error.Printf("TmClient err and try again, %v\n", err.Error()) + c := helper.GetClient().Client + status, err = c.Status() + if err != nil { + logger.Error.Fatalf("TmClient err and exit, %v\n", err.Error()) + } + } + latestHeight := status.LatestBlockHeight + if syncLatestHeight >= latestHeight - 60 { + logger.Info.Println("All fast sync task complete!") + break + } + logger.Info.Printf("End %v time fast sync task", i) + i++ } + startCron(c) } @@ -69,20 +95,19 @@ func startCron(client rpcClient.Client) { go c.Start() } -func watchBlock(c rpcClient.Client) error { +func watchBlock(c rpcClient.Client) { + mutexWatchBlock.Lock() + syncTask, _ := document.QuerySyncTask() status, _ := c.Status() latestBlockHeight := status.LatestBlockHeight - // for test - // latestBlockHeight := int64(60010) - funcChain := []func(tx store.Docs, mutex sync.Mutex){ saveTx, saveOrUpdateAccount, updateAccountBalance, } ch := make(chan int64) - // limitChan <- 1 + limitChan <- 1 go syncBlock(syncTask.Height+1, latestBlockHeight, funcChain, ch, 0) @@ -92,41 +117,44 @@ func watchBlock(c rpcClient.Client) error { block, _ := c.Block(&latestBlockHeight) syncTask.Height = block.Block.Height syncTask.Time = block.Block.Time - return store.Update(syncTask) + err := store.Update(syncTask) + if err != nil { + logger.Error.Printf("Update syncTask fail, err is %v", + err.Error()) + } } + + mutexWatchBlock.Unlock() } // fast sync data from blockChain -func fastSync(c rpcClient.Client) error { +func fastSync(c rpcClient.Client) int64 { syncTaskDoc, _ := document.QuerySyncTask() status, _ := c.Status() latestBlockHeight := status.LatestBlockHeight - // for test - // latestBlockHeight := int64(60000) - funcChain := []func(tx store.Docs, mutex sync.Mutex){ saveTx, saveOrUpdateAccount, updateAccountBalance, } ch := make(chan int64) - activeThreadNum := int64(0) + - goRoutineNum := (latestBlockHeight - syncTaskDoc.Height) / syncBlockNumFastSync + goroutineNum := (latestBlockHeight - syncTaskDoc.Height) / syncBlockNumFastSync - if goRoutineNum == 0 { - goRoutineNum = 10 - syncBlockNumFastSync = 100 + if goroutineNum == 0 { + goroutineNum = 20 + syncBlockNumFastSync = (latestBlockHeight - syncTaskDoc.Height) / goroutineNum } + activeGoroutineNum := goroutineNum - for i := int64(1); i <= goRoutineNum; i++ { - activeThreadNum++ - // limitChan <- i + for i := int64(1); i <= goroutineNum; i++ { + limitChan <- i var ( start = syncTaskDoc.Height + (i-1)*syncBlockNumFastSync + 1 end = syncTaskDoc.Height + i*syncBlockNumFastSync ) - if i == goRoutineNum { + if i == goroutineNum { end = latestBlockHeight } go syncBlock(start, end, funcChain, ch, i) @@ -135,9 +163,9 @@ func fastSync(c rpcClient.Client) error { for { select { case threadNo := <-ch: - activeThreadNum = activeThreadNum - 1 - logger.Info.Printf("ThreadNo[%d] is over and active thread num is %d\n", threadNo, activeThreadNum) - if activeThreadNum == 0 { + activeGoroutineNum = activeGoroutineNum - 1 + logger.Info.Printf("ThreadNo[%d] is over and active thread num is %d\n", threadNo, activeGoroutineNum) + if activeGoroutineNum == 0 { goto end } } @@ -145,13 +173,17 @@ func fastSync(c rpcClient.Client) error { end: { - logger.Info.Println("Fast sync block, complete sync task") + logger.Info.Println("This fastSync task complete!") // update syncTask document block, _ := c.Block(&latestBlockHeight) syncTaskDoc.Height = block.Block.Height syncTaskDoc.Time = block.Block.Time - store.Update(syncTaskDoc) - return nil + err := store.Update(syncTaskDoc) + if err != nil { + logger.Error.Printf("Update syncTask fail, err is %v", + err.Error()) + } + return syncTaskDoc.Height } } @@ -163,11 +195,9 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex syn // release client defer client.Release() // release unBuffer chain and buffer chain - defer func() { - ch <- threadNum - logger.Info.Printf("Send threadNum into channel: %v\n", threadNum) - // <- limitChan - }() + // defer func() { + // + // }() for j := start; j <= end; j++ { block, err := client.Client.Block(&j) @@ -234,4 +264,8 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex syn logger.Info.Printf("ThreadNo[%d] finish sync block from %d to %d\n", threadNum, start, end) + <- limitChan + ch <- threadNum + logger.Info.Printf("Send threadNum into channel: %v\n", threadNum) + } diff --git a/sync/sync_test.go b/sync/sync_test.go index 04ac262..d507a76 100644 --- a/sync/sync_test.go +++ b/sync/sync_test.go @@ -6,80 +6,68 @@ import ( conf "github.com/irisnet/iris-sync-server/conf/server" - rpcClient "github.com/tendermint/tendermint/rpc/client" "sync" "github.com/irisnet/iris-sync-server/module/logger" + "time" ) func TestStart(t *testing.T) { var ( - num int + limitChan chan int + unBufferChan chan int ) - num = 2 - switch num { - case 1: - - break - case 2: - if num > 1 { - logger.Info.Println("num gt 1") - break + limitChan = make(chan int, 3) + unBufferChan = make(chan int) + goroutineNum := 5 + activeGoroutineNum := goroutineNum + for i := 1; i <= goroutineNum; i++ { + limitChan <- i + go func(goroutineNum int, ch chan int) { + logger.Info.Println("release limitChan") + <- limitChan + defer func() { + logger.Info.Printf("%v goroutine send data to channel\n", + goroutineNum) + ch <- goroutineNum + }() + + }(i, nil) + } + + for + { + select { + case <-unBufferChan: + activeGoroutineNum = activeGoroutineNum - 1 + logger.Info.Printf("active goroutine num is %v", activeGoroutineNum) + if activeGoroutineNum == 0 { + logger.Info.Println("All goroutine complete") + break + } } - logger.Info.Println("num is 2") - break } + + } func Test_startCron(t *testing.T) { var wg sync.WaitGroup + var mutex sync.Mutex wg.Add(2) spec := conf.SyncCron c := cron.New() c.AddFunc(spec, func() { - logger.Info.Println("print word every second") + mutex.Lock() + var sleepSecond time.Duration + sleepSecond = 3 + time.Sleep(time.Second * sleepSecond) + logger.Info.Printf("awake up after %v second\n", sleepSecond) + mutex.Unlock() }) go c.Start() wg.Wait() } -func Test_watchBlock(t *testing.T) { - type args struct { - c rpcClient.Client - } - tests := []struct { - name string - args args - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := watchBlock(tt.args.c); (err != nil) != tt.wantErr { - t.Errorf("watchBlock() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} -func Test_fastSync(t *testing.T) { - type args struct { - c rpcClient.Client - } - tests := []struct { - name string - args args - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := fastSync(tt.args.c); (err != nil) != tt.wantErr { - t.Errorf("fastSync() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/util/helper/account.go b/util/helper/account.go index c7aceb6..ba8b4a7 100644 --- a/util/helper/account.go +++ b/util/helper/account.go @@ -3,12 +3,11 @@ package helper import ( - "fmt" "time" "github.com/irisnet/iris-sync-server/module/logger" - wire "github.com/tendermint/go-wire" + "github.com/tendermint/go-wire" "github.com/cosmos/cosmos-sdk/modules/coin" "github.com/cosmos/cosmos-sdk/stack" "github.com/cosmos/cosmos-sdk/client/commands" @@ -17,6 +16,7 @@ import ( "github.com/tendermint/iavl" "github.com/cosmos/cosmos-sdk/client" rpcclient "github.com/tendermint/tendermint/rpc/client" + "github.com/pkg/errors" ) var delay = false @@ -68,7 +68,7 @@ func GetParsed(key []byte, data interface{}, height int64, prove bool) (int64, e // and it is localhost or you have a secure connection (not HTTP) func Get(key []byte, height int64, prove bool) (data.Bytes, int64, error) { if height < 0 { - return nil, 0, fmt.Errorf("Height cannot be negative\n") + return nil, 0, errors.New("Height cannot be negative\n") } if !prove { diff --git a/util/helper/tx.go b/util/helper/tx.go index 30cace1..a58f854 100644 --- a/util/helper/tx.go +++ b/util/helper/tx.go @@ -55,7 +55,6 @@ func ParseTx(txByte types.Tx) (string, interface{}) { case nonce.Tx: ctx, _ := txi.Unwrap().(nonce.Tx) nonceAddr = ctx.Signers[0].Address - fmt.Println(nonceAddr) break case stake.TxUnbond, stake.TxDelegate, stake.TxDeclareCandidacy: kind, _ := txi.GetKind() From e37e359046c38c1139cc6a5420f69303c38793f6 Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Wed, 23 May 2018 17:28:31 +0800 Subject: [PATCH 13/37] read value from env var and add dockerfile(IRISSYNCS-2) --- Dockerfile | 33 +++++++++++++++++++++++++++++++++ Makefile | 21 ++++++++++++++++----- conf/db/types.go.example | 7 ------- conf/server/types.go.example | 13 ------------- tools/Makefile | 33 +++++++++++++++++++++++++++++++++ util/constant/types.go | 12 ++++++++++++ 6 files changed, 94 insertions(+), 25 deletions(-) create mode 100644 Dockerfile delete mode 100644 conf/db/types.go.example delete mode 100644 conf/server/types.go.example create mode 100644 tools/Makefile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..706f0cd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM alpine:edge + +# Set up dependencies +ENV PACKAGES go make git libc-dev bash + +# Set up GOPATH & PATH + +ENV GOPATH /root/go +ENV BASE_PATH $GOPATH/src/github.com/irisnet +ENV REPO_PATH $BASE_PATH/iris-sync-server +ENV LOG_DIR /sync-iris/log +ENV PATH $GOPATH/bin:$PATH + +# Set volumes + +VOLUME $LOG_DIR:sync-iris-log + +# Link expected Go repo path + +RUN mkdir -p $LOG_DIR $GOPATH/pkg $GOPATH/bin $BASE_PATH $REPO_PATH + +# Add source files + +COPY . $REPO_PATH + +# Install minimum necessary dependencies, build iris-sync-server +RUN apk add --no-cache $PACKAGES && \ + cd $REPO_PATH && make all && \ + mv $REPO_PATH/sync-iris $GOPATH/bin && \ + rm -rf $REPO_PATH/vendor + apk del $PACKAGES + +CMD sync-iris > $LOG_DIR/debug.log && tail -f $LOG_DIR/debug.log \ No newline at end of file diff --git a/Makefile b/Makefile index 282c9f2..e3fd065 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,9 @@ GOGET=$(GOCMD) get BINARY_NAME=sync-iris BINARY_UNIX=$(BINARY_NAME)-unix -all: get_vendor build +all: get_deps build -get_vendor: +get_deps: @echo "--> Running glide install" @glide install -v @@ -21,12 +21,23 @@ clean: rm -f $(BINARY_UNIX) run: - $(GOBUILD) -o $(BINARY_NAME) -v ./... + $(GOBUILD) -o $(BINARY_NAME) -v ./ ./$(BINARY_NAME) # Cross compilation build-linux: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) -v -docker-build: - docker run --rm -it -v "$(GOPATH)":/go -w /go/src/github.com/irisnet/iris-sync-server golang:latest go build -o "$(BINARY_UNIX)" -v \ No newline at end of file + + +###################################### +## Tools + +check_tools: + cd tools && $(MAKE) check_tools + +get_tools: + cd tools && $(MAKE) get_tools + +update_tools: + cd tools && $(MAKE) update_tools \ No newline at end of file diff --git a/conf/db/types.go.example b/conf/db/types.go.example deleted file mode 100644 index 91b62c6..0000000 --- a/conf/db/types.go.example +++ /dev/null @@ -1,7 +0,0 @@ -package db - -const ( - Host = "127.0.0.1" - Port = "27017" - Database = "sync_iris" -) diff --git a/conf/server/types.go.example b/conf/server/types.go.example deleted file mode 100644 index 1c76d30..0000000 --- a/conf/server/types.go.example +++ /dev/null @@ -1,13 +0,0 @@ -package server - -const ( - BlockChainMonitorUrl = "tcp://127.0.0.1:46657" - Token = "iris" - InitConnectionNum = 1000 - MaxConnectionNum = 2000 - ChainId = "pangu" - SyncCron = "0-59 * * * * *" - - SyncMaxGoroutine = 2000 - SyncBlockNumFastSync = 2000 -) diff --git a/tools/Makefile b/tools/Makefile new file mode 100644 index 0000000..08969c1 --- /dev/null +++ b/tools/Makefile @@ -0,0 +1,33 @@ +######################################## +### GLIDE + +GLIDE = github.com/Masterminds/glide +GLIDE_CHECK := $(shell command -v glide 2> /dev/null) + +check_tools: +ifndef GLIDE_CHECK + @echo "No glide in path. Install with 'make get_tools'." +else + @echo "Found glide in path." +endif + +get_tools: +ifdef GLIDE_CHECK + @echo "glide is already installed. Run 'make update_tools' to update." +else + @echo "$(ansi_grn)Installing glide$(ansi_end)" + go get -v $(GLIDE) +endif + +update_tools: + @echo "$(ansi_grn)Updating glide$(ansi_end)" + go get -u -v $(GLIDE) + + +######################################## +# ANSI colors + +ansi_red=\033[0;31m +ansi_grn=\033[0;32m +ansi_yel=\033[0;33m +ansi_end=\033[0m \ No newline at end of file diff --git a/util/constant/types.go b/util/constant/types.go index c3549cc..23f9d3e 100644 --- a/util/constant/types.go +++ b/util/constant/types.go @@ -5,4 +5,16 @@ package constant const ( TxTypeCoin = "coin" TxTypeStake = "stake" + + EnvNameDbHost = "DB_HOST" + EnvNameDbPort = "DB_PORT" + EnvNameDbUser = "DB_USER" + EnvNameDbPassWd = "DB_PASSWD" + EnvNameDbDataBase = "DB_DATABASE" + + EnvNameSerNetworkNodeUrl = "SER_BC_NODE_URL" + EnvNameSerNetworkChainId = "SER_BC_CHAIN_ID" + EnvNameSerNetworkToken = "SER_BC_TOKEN" + EnvNameSerMaxGoRoutine = "SER_MAX_GOROUTINE" + EnvNameSerSyncBlockNum = "SER_SYNC_BLOCK_NUM" ) From 5af3f0f8db6952cbe4c519435a8a066b5fc806f0 Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Wed, 23 May 2018 18:39:40 +0800 Subject: [PATCH 14/37] fix err in dockerfile(IRISSYNCS-2) --- Dockerfile | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 706f0cd..74ef5f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ COPY . $REPO_PATH RUN apk add --no-cache $PACKAGES && \ cd $REPO_PATH && make all && \ mv $REPO_PATH/sync-iris $GOPATH/bin && \ - rm -rf $REPO_PATH/vendor + rm -rf $REPO_PATH/vendor && \ apk del $PACKAGES CMD sync-iris > $LOG_DIR/debug.log && tail -f $LOG_DIR/debug.log \ No newline at end of file diff --git a/Makefile b/Makefile index e3fd065..74e0a7e 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ GOGET=$(GOCMD) get BINARY_NAME=sync-iris BINARY_UNIX=$(BINARY_NAME)-unix -all: get_deps build +all: get_tools get_deps build get_deps: @echo "--> Running glide install" From 8af04d95c428023ee16a108810996d67e059cb48 Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Wed, 23 May 2018 18:50:31 +0800 Subject: [PATCH 15/37] add mongodb script --- mongodb/mongodb.js | 67 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 mongodb/mongodb.js diff --git a/mongodb/mongodb.js b/mongodb/mongodb.js new file mode 100644 index 0000000..5e1ddb5 --- /dev/null +++ b/mongodb/mongodb.js @@ -0,0 +1,67 @@ +// create collections +db.createCollection("account"); +db.createCollection("block"); +db.createCollection("stake_role_candidate"); +db.createCollection("stake_role_delegator"); +db.createCollection("sync_task"); +db.createCollection("tx_coin"); +db.createCollection("tx_stake"); + +// create index +db.account.createIndex({"address": 1}, {"unique": true}); +db.block.createIndex({"height": -1}, {"unique": true}); + +db.stake_role_candidate.createIndex({"pub_key": 1}, {"unique": true}); +db.stake_role_candidate.createIndex({"address": 1}); + +db.stake_role_delegator.createIndex({"pub_key": 1}); +db.stake_role_delegator.createIndex({"address": 1}); +db.stake_role_delegator.createIndex({"address": 1, "pub_key": 1}, {"unique": true}); + +db.sync_task.createIndex({"chain_id": 1}, {"unique": true}); + +db.tx_coin.createIndex({"tx_hash": 1}, {"unique": true}); +db.tx_coin.createIndex({"from": 1}); +db.tx_coin.createIndex({"to": 1}); +db.tx_coin.createIndex({"height": -1}); +db.tx_coin.createIndex({"time": -1}); +db.tx_coin.createIndex({"from": 1, "to": 1, "time": 1}); + +db.tx_stake.createIndex({"tx_hash": 1}); +db.tx_stake.createIndex({"stake_tx.tx_hash": 1}); +db.tx_stake.createIndex({"description.moniker": 1}); +db.tx_stake.createIndex({"from": 1}); +db.tx_stake.createIndex({"pub_key": 1}); +db.tx_stake.createIndex({"height": -1}); +db.tx_stake.createIndex({"type": 1}); +db.tx_stake.createIndex({"time": -1}); +db.tx_stake.createIndex({"from": 1, "to": 1, "type": 1, "time": -1}); + + +// drop collection +db.account.drop(); +db.block.drop(); +db.stake_role_candidate.drop(); +db.stake_role_delegator.drop(); +db.sync_task.drop(); +db.tx_coin.drop(); +db.tx_stake.drop(); + +// remove collection data +db.account.remove({}); +db.block.remove({}); +db.stake_role_candidate.remove({}); +db.stake_role_delegator.remove({}); +db.sync_task.remove({}); +db.tx_coin.remove({}); +db.tx_stake.remove({}); + + + + + + + + + + From 61580e35aa2d69b924da12a0b8b06599eea0edfa Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Wed, 23 May 2018 19:08:46 +0800 Subject: [PATCH 16/37] add conf and modify readme --- README.md | 14 +++++++- conf/db/types.go.example | 53 +++++++++++++++++++++++++++ conf/server/types.go.example | 70 ++++++++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 conf/db/types.go.example create mode 100644 conf/server/types.go.example diff --git a/README.md b/README.md index b911029..e7cb3ec 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,24 @@ A server that synchronize IRIS blockChain data into a database - `conf`: config of project - `model`: database model - `module`: project module +- `mongodb`: mongodb script to create database - `sync`: main logic of sync-server, sync data from blockChain and write to database - `util`: common constants and helper functions - `main.go`: bootstrap project +# SetUp + +## Rewrite config file + +1. rename `/conf/db/type.go.example` to `/conf/db/type.go`, `/conf/server/type.go.example` to `/conf/db/type.go` +2. write your own config into `/conf/db/type.go` and `/conf/db/type.go` + +## Create mongodb database + +run script in `mongodb` to create database before run project + # Build And Run - Build: `make all` - Run: `make run` -- Cross compilation: `make build-linux` or `make docker-build` \ No newline at end of file +- Cross compilation: `make build-linux` \ No newline at end of file diff --git a/conf/db/types.go.example b/conf/db/types.go.example new file mode 100644 index 0000000..456b792 --- /dev/null +++ b/conf/db/types.go.example @@ -0,0 +1,53 @@ +package db + +import ( + "os" + "github.com/irisnet/iris-sync-server/util/constant" + "github.com/irisnet/iris-sync-server/module/logger" +) + +var ( + Host = "127.0.0.1" + Port = "27017" + User = "" + Passwd = "" + Database = "test" +) + +// get value of env var +func init() { + host, found := os.LookupEnv(constant.EnvNameDbHost) + if found { + Host = host + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbHost, host) + } + + port, found := os.LookupEnv(constant.EnvNameDbPort) + if found { + Port = port + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbPort, port) + } + + user, found := os.LookupEnv(constant.EnvNameDbUser) + if found { + User = user + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbUser, user) + } + + passwd, found := os.LookupEnv(constant.EnvNameDbPassWd) + if found { + Passwd = passwd + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbPassWd, passwd) + } + + database, found := os.LookupEnv(constant.EnvNameDbDataBase) + if found { + Database = database + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbDataBase, database) + } +} diff --git a/conf/server/types.go.example b/conf/server/types.go.example new file mode 100644 index 0000000..95529a2 --- /dev/null +++ b/conf/server/types.go.example @@ -0,0 +1,70 @@ +package server + +import ( + "os" + "strconv" + + "github.com/irisnet/iris-sync-server/util/constant" + "github.com/irisnet/iris-sync-server/module/logger" +) + +var ( + BlockChainMonitorUrl = "tcp://127.0.0.1:46757" + ChainId = "test" + Token = "iris" + + InitConnectionNum = 100 // fast init num of tendermint client pool + MaxConnectionNum = 1000 // max size of tendermint client pool + SyncCron = "0-59 * * * * *" + + SyncMaxGoroutine = 60 // max go routine in server + SyncBlockNumFastSync = 8000 // sync block num each goroutine +) + +// get value of env var +func init() { + nodeUrl, found := os.LookupEnv(constant.EnvNameSerNetworkNodeUrl) + if found { + BlockChainMonitorUrl = nodeUrl + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerNetworkNodeUrl, nodeUrl) + } + + chainId, found := os.LookupEnv(constant.EnvNameSerNetworkChainId) + if found { + ChainId = chainId + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerNetworkChainId, chainId) + } + + token, found := os.LookupEnv(constant.EnvNameSerNetworkToken) + if found { + Token = token + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerNetworkToken, token) + } + + maxGoroutine, found := os.LookupEnv(constant.EnvNameSerMaxGoRoutine) + if found { + var err error + SyncMaxGoroutine, err = strconv.Atoi(maxGoroutine) + if err != nil { + logger.Error.Fatalf("Convert str to int failed, env var is %v\n", + constant.EnvNameSerMaxGoRoutine) + } + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerMaxGoRoutine, maxGoroutine) + } + + syncBlockNum, found := os.LookupEnv(constant.EnvNameSerSyncBlockNum) + if found { + var err error + SyncBlockNumFastSync, err = strconv.Atoi(syncBlockNum) + if err != nil { + logger.Error.Fatalf("Convert str to int failed, env var is %v\n", + constant.EnvNameSerSyncBlockNum) + } + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerSyncBlockNum, SyncBlockNumFastSync) + } +} From e6a9732930591797994399661be686e1a91fd173 Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Wed, 23 May 2018 19:12:30 +0800 Subject: [PATCH 17/37] modify readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e7cb3ec..bb2d7c4 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ A server that synchronize IRIS blockChain data into a database ## Create mongodb database -run script in `mongodb` to create database before run project +run script `mongodb.js` in `mongodb` folder to create database before run project # Build And Run From 804f0e63b028fcc44d59a5a0539863227a345bd3 Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Wed, 23 May 2018 23:15:14 +0800 Subject: [PATCH 18/37] fix bug: client pool can't add length when used equal initnum --- util/helper/pool_client.go | 11 ++++++- util/helper/pool_client_test.go | 57 +++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 util/helper/pool_client_test.go diff --git a/util/helper/pool_client.go b/util/helper/pool_client.go index e453698..15f2644 100644 --- a/util/helper/pool_client.go +++ b/util/helper/pool_client.go @@ -65,6 +65,14 @@ func createConnection(id int64) Client { used: false, id: id, } + + if id == int64(len(pool.clients)) { + newSlice := make([]Client, pool.maxConnection) + for i, v := range pool.clients { + newSlice[i] = v + } + pool.clients = newSlice + } pool.clients[id] = tmClient pool.available++ return tmClient @@ -75,7 +83,8 @@ func getClient() (Client, error) { maxConnNum := int64(conf.MaxConnectionNum) if pool.used < maxConnNum { var tmClient Client - for i := int64(len(pool.clients)); i < maxConnNum; i++ { + length := len(pool.clients) + for i := int64(length); i < maxConnNum; i++ { tmClient = createConnection(i) } return tmClient, nil diff --git a/util/helper/pool_client_test.go b/util/helper/pool_client_test.go new file mode 100644 index 0000000..495fa88 --- /dev/null +++ b/util/helper/pool_client_test.go @@ -0,0 +1,57 @@ +// init client from clientPool. +// client is httpClient of tendermint + +package helper + +import ( + "testing" + + "github.com/tendermint/tendermint/rpc/client" + + conf "github.com/irisnet/iris-sync-server/conf/server" + "github.com/irisnet/iris-sync-server/module/logger" +) + +func TestInitClientPool(t *testing.T) { + a := []int{1, 2, 3} + b := make([]int, 6, 6) + for index, value := range a { + b[index] = value + } + b[3] = 4 + logger.Info.Println(b) +} + +func TestGetClient(t *testing.T) { + InitClientPool() + + for i := 0; i < conf.InitConnectionNum + 10; i++ { + client := GetClient() + logger.Info.Println(client) + } + +} + +func TestClient_Release(t *testing.T) { + type fields struct { + Client client.Client + used bool + id int64 + } + tests := []struct { + name string + fields fields + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + n := Client{ + Client: tt.fields.Client, + used: tt.fields.used, + id: tt.fields.id, + } + n.Release() + }) + } +} From bc5104800f18242c44c1f39a409604a1f7ea2a52 Mon Sep 17 00:00:00 2001 From: zhangyelong Date: Thu, 24 May 2018 10:09:28 +0800 Subject: [PATCH 19/37] Merge from kaifei --- .gitignore | 6 +- Dockerfile | 33 +++ Makefile | 21 +- README.md | 31 ++- conf/db/types.go | 7 - conf/db/types.go.example | 53 ++++ conf/server/types.go | 13 - conf/server/types.go.example | 70 ++++++ model/store/document/account.go | 8 +- model/store/document/stake_role_candidate.go | 2 +- model/store/document/stake_role_delegator.go | 8 +- model/store/document/tx_stake.go | 2 +- model/store/store.go | 31 +-- module/logger/logger.go | 43 +++- module/stake/tx.go | 4 +- mongodb/mongodb.js | 67 +++++ sync/handler.go | 248 +++++++++++++------ sync/handler_test.go | 52 ++-- sync/sync.go | 129 +++++++--- sync/sync_test.go | 89 ++++--- tools/Makefile | 33 +++ util/constant/types.go | 14 ++ util/helper/account.go | 10 +- util/helper/account_test.go | 2 +- util/helper/pool_client.go | 42 ++-- util/helper/pool_client_test.go | 57 +++++ util/helper/tx.go | 5 +- 27 files changed, 816 insertions(+), 264 deletions(-) create mode 100644 Dockerfile delete mode 100644 conf/db/types.go create mode 100644 conf/db/types.go.example delete mode 100644 conf/server/types.go create mode 100644 conf/server/types.go.example create mode 100644 mongodb/mongodb.js create mode 100644 tools/Makefile create mode 100644 util/helper/pool_client_test.go diff --git a/.gitignore b/.gitignore index f9a0987..c53a361 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ .idea/ vendor/ -*.conf +conf/**/types.go +.DS_Store +sync-iris +sync-iris-unix + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..74ef5f9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM alpine:edge + +# Set up dependencies +ENV PACKAGES go make git libc-dev bash + +# Set up GOPATH & PATH + +ENV GOPATH /root/go +ENV BASE_PATH $GOPATH/src/github.com/irisnet +ENV REPO_PATH $BASE_PATH/iris-sync-server +ENV LOG_DIR /sync-iris/log +ENV PATH $GOPATH/bin:$PATH + +# Set volumes + +VOLUME $LOG_DIR:sync-iris-log + +# Link expected Go repo path + +RUN mkdir -p $LOG_DIR $GOPATH/pkg $GOPATH/bin $BASE_PATH $REPO_PATH + +# Add source files + +COPY . $REPO_PATH + +# Install minimum necessary dependencies, build iris-sync-server +RUN apk add --no-cache $PACKAGES && \ + cd $REPO_PATH && make all && \ + mv $REPO_PATH/sync-iris $GOPATH/bin && \ + rm -rf $REPO_PATH/vendor && \ + apk del $PACKAGES + +CMD sync-iris > $LOG_DIR/debug.log && tail -f $LOG_DIR/debug.log \ No newline at end of file diff --git a/Makefile b/Makefile index 282c9f2..74e0a7e 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,9 @@ GOGET=$(GOCMD) get BINARY_NAME=sync-iris BINARY_UNIX=$(BINARY_NAME)-unix -all: get_vendor build +all: get_tools get_deps build -get_vendor: +get_deps: @echo "--> Running glide install" @glide install -v @@ -21,12 +21,23 @@ clean: rm -f $(BINARY_UNIX) run: - $(GOBUILD) -o $(BINARY_NAME) -v ./... + $(GOBUILD) -o $(BINARY_NAME) -v ./ ./$(BINARY_NAME) # Cross compilation build-linux: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) -v -docker-build: - docker run --rm -it -v "$(GOPATH)":/go -w /go/src/github.com/irisnet/iris-sync-server golang:latest go build -o "$(BINARY_UNIX)" -v \ No newline at end of file + + +###################################### +## Tools + +check_tools: + cd tools && $(MAKE) check_tools + +get_tools: + cd tools && $(MAKE) get_tools + +update_tools: + cd tools && $(MAKE) update_tools \ No newline at end of file diff --git a/README.md b/README.md index f1ab8db..2334e46 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,29 @@ -# iris-sync-server -A server that synchronize IRIS blockchain data into database \ No newline at end of file +# IRIS-SYNC-SERVER +A server that synchronize IRIS blockChain data into a database + +# Structure + +- `conf`: config of project +- `model`: database model +- `module`: project module +- `mongodb`: mongodb script to create database +- `sync`: main logic of sync-server, sync data from blockChain and write to database +- `util`: common constants and helper functions +- `main.go`: bootstrap project + +# SetUp + +## Rewrite config file + +1. rename `/conf/db/type.go.example` to `/conf/db/type.go`, `/conf/server/type.go.example` to `/conf/db/type.go` +2. write your own config into `/conf/db/type.go` and `/conf/db/type.go` + +## Create mongodb database + +run script `mongodb.js` in `mongodb` folder to create database before run project + +# Build And Run + +- Build: `make all` +- Run: `make run` +- Cross compilation: `make build-linux` diff --git a/conf/db/types.go b/conf/db/types.go deleted file mode 100644 index 4f9b50a..0000000 --- a/conf/db/types.go +++ /dev/null @@ -1,7 +0,0 @@ -package db - -const ( - Host = "127.0.0.1" - Port = "27117" - Database = "sync_iris" -) diff --git a/conf/db/types.go.example b/conf/db/types.go.example new file mode 100644 index 0000000..456b792 --- /dev/null +++ b/conf/db/types.go.example @@ -0,0 +1,53 @@ +package db + +import ( + "os" + "github.com/irisnet/iris-sync-server/util/constant" + "github.com/irisnet/iris-sync-server/module/logger" +) + +var ( + Host = "127.0.0.1" + Port = "27017" + User = "" + Passwd = "" + Database = "test" +) + +// get value of env var +func init() { + host, found := os.LookupEnv(constant.EnvNameDbHost) + if found { + Host = host + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbHost, host) + } + + port, found := os.LookupEnv(constant.EnvNameDbPort) + if found { + Port = port + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbPort, port) + } + + user, found := os.LookupEnv(constant.EnvNameDbUser) + if found { + User = user + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbUser, user) + } + + passwd, found := os.LookupEnv(constant.EnvNameDbPassWd) + if found { + Passwd = passwd + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbPassWd, passwd) + } + + database, found := os.LookupEnv(constant.EnvNameDbDataBase) + if found { + Database = database + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameDbDataBase, database) + } +} diff --git a/conf/server/types.go b/conf/server/types.go deleted file mode 100644 index 3b6c173..0000000 --- a/conf/server/types.go +++ /dev/null @@ -1,13 +0,0 @@ -package server - -const ( - BlockChainMonitorUrl = "tcp://127.0.0.1:46757" - Token = "iris" - InitConnectionNum = 1000 - MaxConnectionNum = 2000 - ChainId = "pangu" - SyncCron = "0-59 * * * * *" - - SyncMaxGoroutine = 2000 - SyncBlockNumFastSync = 2000 -) diff --git a/conf/server/types.go.example b/conf/server/types.go.example new file mode 100644 index 0000000..95529a2 --- /dev/null +++ b/conf/server/types.go.example @@ -0,0 +1,70 @@ +package server + +import ( + "os" + "strconv" + + "github.com/irisnet/iris-sync-server/util/constant" + "github.com/irisnet/iris-sync-server/module/logger" +) + +var ( + BlockChainMonitorUrl = "tcp://127.0.0.1:46757" + ChainId = "test" + Token = "iris" + + InitConnectionNum = 100 // fast init num of tendermint client pool + MaxConnectionNum = 1000 // max size of tendermint client pool + SyncCron = "0-59 * * * * *" + + SyncMaxGoroutine = 60 // max go routine in server + SyncBlockNumFastSync = 8000 // sync block num each goroutine +) + +// get value of env var +func init() { + nodeUrl, found := os.LookupEnv(constant.EnvNameSerNetworkNodeUrl) + if found { + BlockChainMonitorUrl = nodeUrl + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerNetworkNodeUrl, nodeUrl) + } + + chainId, found := os.LookupEnv(constant.EnvNameSerNetworkChainId) + if found { + ChainId = chainId + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerNetworkChainId, chainId) + } + + token, found := os.LookupEnv(constant.EnvNameSerNetworkToken) + if found { + Token = token + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerNetworkToken, token) + } + + maxGoroutine, found := os.LookupEnv(constant.EnvNameSerMaxGoRoutine) + if found { + var err error + SyncMaxGoroutine, err = strconv.Atoi(maxGoroutine) + if err != nil { + logger.Error.Fatalf("Convert str to int failed, env var is %v\n", + constant.EnvNameSerMaxGoRoutine) + } + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerMaxGoRoutine, maxGoroutine) + } + + syncBlockNum, found := os.LookupEnv(constant.EnvNameSerSyncBlockNum) + if found { + var err error + SyncBlockNumFastSync, err = strconv.Atoi(syncBlockNum) + if err != nil { + logger.Error.Fatalf("Convert str to int failed, env var is %v\n", + constant.EnvNameSerSyncBlockNum) + } + logger.Info.Printf("The value of env var %v is %v\n", + constant.EnvNameSerSyncBlockNum, SyncBlockNumFastSync) + } +} diff --git a/model/store/document/account.go b/model/store/document/account.go index f808f35..5cd8884 100644 --- a/model/store/document/account.go +++ b/model/store/document/account.go @@ -6,7 +6,6 @@ import ( "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" - "github.com/irisnet/iris-sync-server/module/logger" ) const ( @@ -45,10 +44,11 @@ func QueryAccount(address string) (Account, error) { err := c.Find(bson.M{"address": address}).Sort("-amount.amount").One(&result) return err } + + err := store.ExecCollection(CollectionNmAccount, query) - if store.ExecCollection(CollectionNmAccount, query) != nil { - logger.Info.Println("Account is Empty") - return result, nil + if err != nil { + return result, err } return result, nil diff --git a/model/store/document/stake_role_candidate.go b/model/store/document/stake_role_candidate.go index 2280c44..0a103d8 100644 --- a/model/store/document/stake_role_candidate.go +++ b/model/store/document/stake_role_candidate.go @@ -17,7 +17,7 @@ type Candidate struct { Address string `bson:"address"` // owner PubKey string `bson:"pub_key"` Shares int64 `bson:"shares"` - VotingPower uint64 `bson:"voting_power"` // Voting power if pubKey is a considered a validator + VotingPower int64 `bson:"voting_power"` // Voting power if pubKey is a considered a validator Description Description `bson:"description"` // Description terms for the candidate UpdateTime time.Time `bson:"update_time"` } diff --git a/model/store/document/stake_role_delegator.go b/model/store/document/stake_role_delegator.go index f0ff9e4..4e95cfa 100644 --- a/model/store/document/stake_role_delegator.go +++ b/model/store/document/stake_role_delegator.go @@ -6,7 +6,6 @@ import ( "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" - "github.com/irisnet/iris-sync-server/module/logger" ) const ( @@ -14,9 +13,9 @@ const ( ) type Delegator struct { - Address string `bson:"address"` - PubKey string `bson:"pub_key"` - Shares int64 `bson:"shares"` + Address string `bson:"address"` + PubKey string `bson:"pub_key"` + Shares int64 `bson:"shares"` UpdateTime time.Time `bson:"update_time"` } @@ -59,7 +58,6 @@ func QueryDelegatorByAddressAndPubkey(address string, pubKey string) (Delegator, } if store.ExecCollection(CollectionNmStakeRoleDelegator, query) != nil { - logger.Info.Println("delegator is Empty") return result, errors.New("delegator is Empty") } diff --git a/model/store/document/tx_stake.go b/model/store/document/tx_stake.go index 4167101..39a4bde 100644 --- a/model/store/document/tx_stake.go +++ b/model/store/document/tx_stake.go @@ -34,7 +34,7 @@ func (c StakeTx) Index() []mgo.Index { return []mgo.Index{ { Key: []string{"tx_hash"}, - Unique: true, + Unique: false, DropDups: false, Background: true, }, diff --git a/model/store/store.go b/model/store/store.go index 6d5a550..cc0322e 100644 --- a/model/store/store.go +++ b/model/store/store.go @@ -3,12 +3,13 @@ package store import ( - "fmt" "time" conf "github.com/irisnet/iris-sync-server/conf/db" "github.com/irisnet/iris-sync-server/module/logger" + "errors" + "fmt" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" ) @@ -26,22 +27,23 @@ func Init() { if session == nil { url := fmt.Sprintf("mongodb://%s:%s", conf.Host, conf.Port) - logger.Info.Printf("Mgo start on %s\n", url) - var err error session, err = mgo.Dial(url) if err != nil { logger.Error.Fatalln(err) } + logger.Info.Printf("Mgo start on %s\n", url) session.SetMode(mgo.Monotonic, true) - - //index() } } +func AddIndex() { + index() +} + func InitWithAuth(addrs []string, username, password string) { dialInfo := &mgo.DialInfo{ - Addrs: addrs, //[]string{"192.168.6.122"} + Addrs: addrs, // []string{"192.168.6.122"} Direct: false, Timeout: time.Second * 1, Database: conf.Database, @@ -99,22 +101,21 @@ func index() { func Save(h Docs) error { save := func(c *mgo.Collection) error { - //先按照关键字查询，如果存在，直接返回 - n, _ := c.Find(h.PkKvPair()).Count() + pk := h.PkKvPair() + n, _ := c.Find(pk).Count() if n >= 1 { - logger.Info.Println("db: record existed while save data") - return nil + errMsg := fmt.Sprintf("Record existed while save %v, data is %+v\n", + h.Name(), h) + return errors.New(errMsg) } - //logger.Info.Printf("insert %s %+v\n", h.Name(), h) + // logger.Info.Printf("insert %s %+v\n", h.Name(), h) return c.Insert(h) } - return ExecCollection(h.Name(), save) } func SaveOrUpdate(h Docs) error { save := func(c *mgo.Collection) error { - //先按照关键字查询，如果存在，直接返回 n, err := c.Find(h.PkKvPair()).Count() if err != nil { logger.Error.Printf("Count:%d err:%+v\n", n, err) @@ -123,7 +124,7 @@ func SaveOrUpdate(h Docs) error { if n >= 1 { return Update(h) } - //logger.Info.Printf("insert %s %+v\n", h.Name(), h) + // logger.Trace.Printf("insert %s %+v\n", h.Name(), h) return c.Insert(h) } @@ -133,7 +134,7 @@ func SaveOrUpdate(h Docs) error { func Update(h Docs) error { update := func(c *mgo.Collection) error { key := h.PkKvPair() - //logger.Info.Printf("update %s set %+v where %+v\n", h.Name(), h, key) + // logger.Trace.Printf("update %s set %+v where %+v\n", h.Name(), h, key) return c.Update(key, h) } return ExecCollection(h.Name(), update) diff --git a/module/logger/logger.go b/module/logger/logger.go index d574889..9ae2b6a 100644 --- a/module/logger/logger.go +++ b/module/logger/logger.go @@ -7,31 +7,64 @@ import ( ) var ( - Info *log.Logger // Important information - Error *log.Logger // Critical problem + Trace *log.Logger // Trace db log + Info *log.Logger // Important information + Warning *log.Logger // Warning information + Error *log.Logger // Critical problem ) const ( - errFile = ".sync_server_err.log" + traceDbFile = ".sync_server_db.log" + debugFile = ".sync_server_debug.log" + errFile = ".sync_server_err.log" + warningFile = ".sync_server_warning.log" ) func init() { - errFile, err := os.OpenFile(os.ExpandEnv("$HOME/" + errFile), + traceDbFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+traceDbFile), + os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + log.Fatalln("Failed to open trace db log file:", err) + } + + debugFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+debugFile), + os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + log.Fatalln("Failed to open debug log file:", err) + } + + warningFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+warningFile), + os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + log.Fatalln("Failed to open warning log file:", err) + } + + errFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+errFile), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err != nil { log.Fatalln("Failed to open error log file:", err) } - Info = log.New(os.Stdout, + Trace = log.New(io.MultiWriter(traceDbFile, os.Stdout), + "TRACE: ", + log.Ldate|log.Ltime|log.Lshortfile) + + Info = log.New(io.MultiWriter(debugFile, os.Stdout), "INFO: ", log.Ldate|log.Ltime|log.Lshortfile) + Warning = log.New(io.MultiWriter(warningFile, os.Stderr), + "WARNING: ", + log.Ldate|log.Ltime|log.Lshortfile) + Error = log.New(io.MultiWriter(errFile, os.Stderr), "ERROR: ", log.Ldate|log.Ltime|log.Lshortfile) } func test() { + Trace.Println("This is trace info...") Info.Println("This is info info...") + Warning.Println("This is warning info...") Error.Println("This is err info...") } diff --git a/module/stake/tx.go b/module/stake/tx.go index 2202687..e5f51e3 100644 --- a/module/stake/tx.go +++ b/module/stake/tx.go @@ -8,7 +8,7 @@ import ( crypto "github.com/tendermint/go-crypto" ) -//TODO 当cosmos-sdk加入stake模块时需要删除该文件，防止在启动时，重复加载ByteTxDeclareCandidacy，ByteTxEditCandidacy等类型，导致启动失败 +// TODO 当cosmos-sdk加入stake模块时需要删除该文件，防止在启动时，重复加载ByteTxDeclareCandidacy，ByteTxEditCandidacy等类型，导致启动失败 const stakingModuleName = "stake" @@ -49,7 +49,7 @@ func init() { sdk.TxMapper.RegisterImplementation(TxUnbond{}, TypeTxUnbond, ByteTxUnbond) } -//Verify interface at compile time +// Verify interface at compile time var _, _, _, _ sdk.TxInner = &TxDeclareCandidacy{}, &TxEditCandidacy{}, &TxDelegate{}, &TxUnbond{} // BondUpdate - struct for bonding or unbonding transactions diff --git a/mongodb/mongodb.js b/mongodb/mongodb.js new file mode 100644 index 0000000..5e1ddb5 --- /dev/null +++ b/mongodb/mongodb.js @@ -0,0 +1,67 @@ +// create collections +db.createCollection("account"); +db.createCollection("block"); +db.createCollection("stake_role_candidate"); +db.createCollection("stake_role_delegator"); +db.createCollection("sync_task"); +db.createCollection("tx_coin"); +db.createCollection("tx_stake"); + +// create index +db.account.createIndex({"address": 1}, {"unique": true}); +db.block.createIndex({"height": -1}, {"unique": true}); + +db.stake_role_candidate.createIndex({"pub_key": 1}, {"unique": true}); +db.stake_role_candidate.createIndex({"address": 1}); + +db.stake_role_delegator.createIndex({"pub_key": 1}); +db.stake_role_delegator.createIndex({"address": 1}); +db.stake_role_delegator.createIndex({"address": 1, "pub_key": 1}, {"unique": true}); + +db.sync_task.createIndex({"chain_id": 1}, {"unique": true}); + +db.tx_coin.createIndex({"tx_hash": 1}, {"unique": true}); +db.tx_coin.createIndex({"from": 1}); +db.tx_coin.createIndex({"to": 1}); +db.tx_coin.createIndex({"height": -1}); +db.tx_coin.createIndex({"time": -1}); +db.tx_coin.createIndex({"from": 1, "to": 1, "time": 1}); + +db.tx_stake.createIndex({"tx_hash": 1}); +db.tx_stake.createIndex({"stake_tx.tx_hash": 1}); +db.tx_stake.createIndex({"description.moniker": 1}); +db.tx_stake.createIndex({"from": 1}); +db.tx_stake.createIndex({"pub_key": 1}); +db.tx_stake.createIndex({"height": -1}); +db.tx_stake.createIndex({"type": 1}); +db.tx_stake.createIndex({"time": -1}); +db.tx_stake.createIndex({"from": 1, "to": 1, "type": 1, "time": -1}); + + +// drop collection +db.account.drop(); +db.block.drop(); +db.stake_role_candidate.drop(); +db.stake_role_delegator.drop(); +db.sync_task.drop(); +db.tx_coin.drop(); +db.tx_stake.drop(); + +// remove collection data +db.account.remove({}); +db.block.remove({}); +db.stake_role_candidate.remove({}); +db.stake_role_delegator.remove({}); +db.sync_task.remove({}); +db.tx_coin.remove({}); +db.tx_stake.remove({}); + + + + + + + + + + diff --git a/sync/handler.go b/sync/handler.go index 3b46c3e..714df04 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -2,23 +2,22 @@ package sync import ( "reflect" - "time" "sync" + "time" "github.com/irisnet/iris-sync-server/model/store" + "github.com/irisnet/iris-sync-server/model/store/document" "github.com/irisnet/iris-sync-server/module/logger" "github.com/irisnet/iris-sync-server/module/stake" - "github.com/irisnet/iris-sync-server/util/helper" - "github.com/irisnet/iris-sync-server/model/store/document" "github.com/irisnet/iris-sync-server/util/constant" - + "github.com/irisnet/iris-sync-server/util/helper" ) var ( - delay = false + delay = false + methodName string ) - func handle(tx store.Docs, mutex sync.Mutex, funChains []func(tx store.Docs, mutex sync.Mutex)) { for _, fun := range funChains { fun(tx, mutex) @@ -27,50 +26,81 @@ func handle(tx store.Docs, mutex sync.Mutex, funChains []func(tx store.Docs, mut // save Tx document into collection func saveTx(tx store.Docs, mutex sync.Mutex) { - err := store.Save(tx) + methodName = "SaveTx: " + txCollectionName := tx.Name() - if err != nil { - logger.Error.Println(err) + // save tx document into database + storeTxDocFunc := func(doc store.Docs) { + err := store.Save(doc) + if err != nil { + logger.Error.Printf("%v Save failed. doc is %+v, err is %v", + methodName, doc, err.Error()) + } } - mutex.Lock() - logger.Info.Println("saveTx method get lock") + switch txCollectionName { + case document.CollectionNmCoinTx: + storeTxDocFunc(tx) + break + case document.CollectionNmStakeTx: + if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { + logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) + break + } + stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - { - if tx.Name() == document.CollectionNmStakeTx { - if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { - logger.Error.Println("type which is field name of stake tx is missed") - return - } - stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - logger.Info.Printf("saveTx: type of stakeTx is %v", stakeType) + logger.Info.Printf("%v handle %v and type is %v", + methodName, txCollectionName, stakeType) + mutex.Lock() + logger.Info.Printf("%v get lock\n", methodName) + + { switch stakeType { case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) - candidate, err := document.QueryCandidateByPubkey(stakeTxDeclareCandidacy.PubKey) - // new candidate - if err != nil { - candidate = document.Candidate { - Address: stakeTxDeclareCandidacy.From, - PubKey: stakeTxDeclareCandidacy.PubKey, - Description: stakeTxDeclareCandidacy.Description, - } + stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, stakeType) + break + } + storeTxDocFunc(stakeTxDeclareCandidacy) + + cd, err := document.QueryCandidateByPubkey(stakeTxDeclareCandidacy.PubKey) + + candidate := document.Candidate{ + Address: stakeTxDeclareCandidacy.From, + PubKey: stakeTxDeclareCandidacy.PubKey, + Description: stakeTxDeclareCandidacy.Description, } // TODO: in further share not equal amount candidate.Shares += stakeTxDeclareCandidacy.Amount.Amount - candidate.VotingPower += uint64(stakeTxDeclareCandidacy.Amount.Amount) + candidate.VotingPower += int64(stakeTxDeclareCandidacy.Amount.Amount) candidate.UpdateTime = stakeTxDeclareCandidacy.Time + + if err != nil && cd.Address == "" { + logger.Warning.Printf("%v Replace candidate from %+v to %+v\n", methodName, cd, candidate) + } + store.SaveOrUpdate(candidate) break case stake.TypeTxDelegate: - stakeTx, _ := tx.(document.StakeTx) + stakeTx, r := tx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, stakeType) + break + } + storeTxDocFunc(stakeTx) + candidate, err := document.QueryCandidateByPubkey(stakeTx.PubKey) // candidate is not exist if err != nil { - logger.Error.Printf("candidate is not exist while delegate, add = %s ,pub_key = %s\n", - stakeTx.From, stakeTx.PubKey) - return + logger.Warning.Printf("%v candidate is not exist while delegate, add = %s ,pub_key = %s\n", + methodName, stakeTx.From, stakeTx.PubKey) + candidate = document.Candidate{ + PubKey: stakeTx.PubKey, + } } delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) @@ -78,7 +108,7 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { if err != nil { delegator = document.Delegator{ Address: stakeTx.From, - PubKey: stakeTx.PubKey, + PubKey: stakeTx.PubKey, } } // TODO: in further share not equal amount @@ -87,52 +117,64 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { store.SaveOrUpdate(delegator) candidate.Shares += stakeTx.Amount.Amount - candidate.VotingPower += uint64(stakeTx.Amount.Amount) + candidate.VotingPower += int64(stakeTx.Amount.Amount) candidate.UpdateTime = stakeTx.Time store.SaveOrUpdate(candidate) break case stake.TypeTxUnbond: - stakeTx, _ := tx.(document.StakeTx) + stakeTx, r := tx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, stakeType) + break + } + storeTxDocFunc(stakeTx) + delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) // delegator is not exist if err != nil { - logger.Info.Printf("delegator is not exist while unBond,add = %s,pub_key=%s\n", - stakeTx.From, stakeTx.PubKey) - return + logger.Warning.Printf("%v delegator is not exist while unBond,add = %s,pub_key=%s\n", + methodName, stakeTx.From, stakeTx.PubKey) + delegator = document.Delegator{ + Address: stakeTx.From, + PubKey: stakeTx.PubKey, + } } delegator.Shares -= stakeTx.Amount.Amount delegator.UpdateTime = stakeTx.Time - store.Update(delegator) + store.SaveOrUpdate(delegator) candidate, err2 := document.QueryCandidateByPubkey(stakeTx.PubKey) // candidate is not exist if err2 != nil { - logger.Info.Printf("candidate is not exist while unBond,add = %s,pub_key=%s\n", - stakeTx.From, stakeTx.PubKey) - return + logger.Warning.Printf("%v candidate is not exist while unBond,add = %s,pub_key=%s\n", + methodName, stakeTx.From, stakeTx.PubKey) + candidate = document.Candidate{ + PubKey: stakeTx.PubKey, + } } candidate.Shares -= stakeTx.Amount.Amount - candidate.VotingPower -= uint64(stakeTx.Amount.Amount) + candidate.VotingPower -= int64(stakeTx.Amount.Amount) candidate.UpdateTime = stakeTx.Time - store.Update(candidate) + store.SaveOrUpdate(candidate) break } - } + mutex.Unlock() + logger.Info.Printf("%v method release lock\n", methodName) } - - mutex.Unlock() - - logger.Info.Println("saveTx method release lock") } func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { var ( - address string + address string updateTime time.Time - height int64 + height int64 + methodName = "SaveOrUpdateAccount: " ) + txCollectionName := tx.Name() + logger.Info.Printf("Start %v\n", methodName) fun := func(address string, updateTime time.Time, height int64) { account := document.Account{ @@ -141,18 +183,24 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { Height: height, } - if err := store.SaveOrUpdate(account); err != nil { - logger.Error.Printf("account:[%q] balance update failed,%s\n", account.Address, err) + if err := store.Save(account); err != nil { + logger.Trace.Printf("%v Record exists, account is %v, err is %s\n", + methodName, account.Address, err.Error()) } } - mutex.Lock() - logger.Info.Println("saveOrUpdateAccpunt method get lock") + // mutex.Lock() + // logger.Info.Printf("%v get lock\n", methodName) { - switch tx.Name() { + switch txCollectionName { case document.CollectionNmCoinTx: - coinTx, _ := tx.(document.CoinTx) + coinTx, r := tx.(document.CoinTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, txCollectionName) + break + } updateTime = coinTx.Time height = coinTx.Height @@ -161,15 +209,22 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { break case document.CollectionNmStakeTx: if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { - logger.Error.Println("type which is field name of stake tx is missed") - return + logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) + break } stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - - logger.Info.Printf("saveOrUpdateAccpunt: type of stakeTx is %v", stakeType) + + logger.Info.Printf("%v handle %v and type is %v", + methodName, txCollectionName, stakeType) + switch stakeType { case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) + stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, stakeType) + break + } address = stakeTxDeclareCandidacy.From updateTime = stakeTxDeclareCandidacy.Time height = stakeTxDeclareCandidacy.Height @@ -177,64 +232,95 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { case stake.TypeTxEditCandidacy: break case stake.TypeTxDelegate, stake.TypeTxUnbond: - stakeTx, _ := tx.(document.StakeTx) + stakeTx, r := tx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, stakeType) + break + } address = stakeTx.From updateTime = stakeTx.Time height = stakeTx.Height break } + fun(address, updateTime, height) } - } - mutex.Unlock() - logger.Info.Println("saveOrUpdateAccpunt method release lock") - + logger.Info.Printf("End %v\n", methodName) + // mutex.Unlock() + // logger.Info.Printf("%v release lock\n", methodName) } func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { var ( - address string + address string + methodName = "UpdateAccountBalance: " ) + txCollectionName := tx.Name() + logger.Info.Printf("Start %v\n", methodName) + fun := func(address string) { account, err := document.QueryAccount(address) if err != nil { + logger.Error.Printf("%v updateAccountBalance failed, account is %v and err is %v", + methodName, account, err.Error()) return } + // query balance of account ac := helper.QueryAccountBalance(address, delay) account.Amount = ac.Coins if err := store.Update(account); err != nil { - logger.Error.Printf("account:[%q] balance update failed,%s\n", account.Address, err) + logger.Error.Printf("%v account:[%q] balance update failed,%s\n", + methodName, account.Address, err) } } - mutex.Lock() - logger.Info.Println("updateAccountBalance method get lock") + // mutex.Lock() + // logger.Info.Printf("%v get lock\n", methodName) { - switch tx.Name() { + switch txCollectionName { case document.CollectionNmCoinTx: - coinTx, _ := tx.(document.CoinTx) + coinTx, r := tx.(document.CoinTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, txCollectionName) + break + } fun(coinTx.From) fun(coinTx.To) break case document.CollectionNmStakeTx: if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { - logger.Error.Println("type which is field name of stake tx is missed") - return + logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) + break } stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() + logger.Info.Printf("%v handle %v and type is %v", + methodName, txCollectionName, stakeType) + switch stakeType { case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) + stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, stakeType) + break + } address = stakeTxDeclareCandidacy.From break case stake.TypeTxEditCandidacy: break case stake.TypeTxDelegate, stake.TypeTxUnbond: - stakeTx, _ := tx.(document.StakeTx) + stakeTx, r := tx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, stakeType) + break + } address = stakeTx.From break } @@ -242,8 +328,10 @@ func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { break } } + + logger.Info.Printf("End %v\n", methodName) - mutex.Unlock() - logger.Info.Println("updateAccountBalance method release lock") + // mutex.Unlock() + // logger.Info.Println("updateAccountBalance method release lock") } diff --git a/sync/handler_test.go b/sync/handler_test.go index 87586da..a73c3dc 100644 --- a/sync/handler_test.go +++ b/sync/handler_test.go @@ -59,10 +59,11 @@ func buildDocData(blockHeight int64) store.Docs { func Test_saveTx(t *testing.T) { - docTxCoin := buildDocData(12453) - docTxStakeDeclareCandidacy := buildDocData(19073) - docTxStakeDelegate := buildDocData(13725) - docTxStakeUnBond := buildDocData(14260) + docTxCoin := buildDocData(1756) + docTxStakeDeclareCandidacy := buildDocData(1921) + docTxStakeDeclareCandidacy2 := buildDocData(1927) + docTxStakeDelegate := buildDocData(115135) + docTxStakeUnBond := buildDocData(397201) type args struct { tx store.Docs @@ -87,6 +88,13 @@ func Test_saveTx(t *testing.T) { }, }, + { + name:"save tx_stake_declareCandidacy2", + args: args{ + tx:docTxStakeDeclareCandidacy2, + mutex:sync.Mutex{}, + }, + }, { name:"save tx_stake_delegate", args:args{ @@ -112,11 +120,12 @@ func Test_saveTx(t *testing.T) { } func Test_saveOrUpdateAccount(t *testing.T) { - - docTxCoin := buildDocData(12453) - docTxStakeDeclareCandidacy := buildDocData(19073) - docTxStakeDelegate := buildDocData(13725) - docTxStakeUnBond := buildDocData(14260) + + docTxCoin := buildDocData(1756) + docTxStakeDeclareCandidacy := buildDocData(1921) + docTxStakeDeclareCandidacy2 := buildDocData(1927) + docTxStakeDelegate := buildDocData(115135) + docTxStakeUnBond := buildDocData(397201) type args struct { tx store.Docs @@ -138,6 +147,13 @@ func Test_saveOrUpdateAccount(t *testing.T) { tx: docTxStakeDeclareCandidacy, }, + }, + { + name:"save tx_stake_declareCandidacy2", + args: args{ + tx: docTxStakeDeclareCandidacy2, + }, + }, { name:"save tx_stake_delegate", @@ -162,11 +178,12 @@ func Test_saveOrUpdateAccount(t *testing.T) { } func Test_updateAccountBalance(t *testing.T) { - - docTxCoin := buildDocData(12453) - docTxStakeDeclareCandidacy := buildDocData(19073) - docTxStakeDelegate := buildDocData(13725) - docTxStakeUnBond := buildDocData(14260) + + docTxCoin := buildDocData(1756) + docTxStakeDeclareCandidacy := buildDocData(1921) + docTxStakeDeclareCandidacy2 := buildDocData(1927) + docTxStakeDelegate := buildDocData(115135) + docTxStakeUnBond := buildDocData(397201) type args struct { tx store.Docs @@ -188,6 +205,13 @@ func Test_updateAccountBalance(t *testing.T) { tx:docTxStakeDeclareCandidacy, }, + }, + { + name:"tx_stake_declareCandidacy2", + args: args{ + tx:docTxStakeDeclareCandidacy2, + }, + }, { name:"tx_stake_delegate", diff --git a/sync/sync.go b/sync/sync.go index 338ccbc..6df2d63 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -1,20 +1,21 @@ package sync import ( - "strings" "encoding/hex" + "strings" conf "github.com/irisnet/iris-sync-server/conf/server" "github.com/irisnet/iris-sync-server/model/store" "github.com/irisnet/iris-sync-server/module/logger" - "github.com/irisnet/iris-sync-server/util/helper" - "github.com/irisnet/iris-sync-server/util/constant" "github.com/irisnet/iris-sync-server/module/stake" + "github.com/irisnet/iris-sync-server/util/constant" + "github.com/irisnet/iris-sync-server/util/helper" + "github.com/irisnet/iris-sync-server/model/store/document" "github.com/robfig/cron" rpcClient "github.com/tendermint/tendermint/rpc/client" - "github.com/irisnet/iris-sync-server/model/store/document" - + ctypes "github.com/tendermint/tendermint/rpc/core/types" + "sync" ) @@ -24,17 +25,42 @@ var ( // limit max goroutine limitChan = make(chan int64, conf.SyncMaxGoroutine) - + mutex sync.Mutex + mutexWatchBlock sync.Mutex ) // start sync server func Start() { + var ( + status *ctypes.ResultStatus + err error + i = 1 + ) InitServer() c := helper.GetClient().Client - if err := fastSync(c); err != nil { - logger.Error.Fatalf("sync block failed,%v\n", err) + + for { + logger.Info.Printf("Begin %v time fast sync task", i) + syncLatestHeight := fastSync(c) + status, err = c.Status() + if err != nil { + logger.Error.Printf("TmClient err and try again, %v\n", err.Error()) + c := helper.GetClient().Client + status, err = c.Status() + if err != nil { + logger.Error.Fatalf("TmClient err and exit, %v\n", err.Error()) + } + } + latestHeight := status.LatestBlockHeight + if syncLatestHeight >= latestHeight - 60 { + logger.Info.Println("All fast sync task complete!") + break + } + logger.Info.Printf("End %v time fast sync task", i) + i++ } + startCron(c) } @@ -69,13 +95,12 @@ func startCron(client rpcClient.Client) { go c.Start() } -func watchBlock(c rpcClient.Client) error { +func watchBlock(c rpcClient.Client) { + mutexWatchBlock.Lock() + syncTask, _ := document.QuerySyncTask() status, _ := c.Status() latestBlockHeight := status.LatestBlockHeight - - // for test - // latestBlockHeight := int64(60010) funcChain := []func(tx store.Docs, mutex sync.Mutex){ saveTx, saveOrUpdateAccount, updateAccountBalance, @@ -92,41 +117,44 @@ func watchBlock(c rpcClient.Client) error { block, _ := c.Block(&latestBlockHeight) syncTask.Height = block.Block.Height syncTask.Time = block.Block.Time - return store.Update(syncTask) + err := store.Update(syncTask) + if err != nil { + logger.Error.Printf("Update syncTask fail, err is %v", + err.Error()) + } } + + mutexWatchBlock.Unlock() } // fast sync data from blockChain -func fastSync(c rpcClient.Client) error { +func fastSync(c rpcClient.Client) int64 { syncTaskDoc, _ := document.QuerySyncTask() status, _ := c.Status() latestBlockHeight := status.LatestBlockHeight - - // for test - // latestBlockHeight := int64(60000) funcChain := []func(tx store.Docs, mutex sync.Mutex){ saveTx, saveOrUpdateAccount, updateAccountBalance, } ch := make(chan int64) - activeThreadNum := int64(0) + - goRoutineNum := (latestBlockHeight - syncTaskDoc.Height) / syncBlockNumFastSync + goroutineNum := (latestBlockHeight - syncTaskDoc.Height) / syncBlockNumFastSync - if goRoutineNum == 0 { - goRoutineNum = 10 - syncBlockNumFastSync = 100 + if goroutineNum == 0 { + goroutineNum = 20 + syncBlockNumFastSync = (latestBlockHeight - syncTaskDoc.Height) / goroutineNum } + activeGoroutineNum := goroutineNum - for i := int64(1); i <= goRoutineNum; i++ { - activeThreadNum++ + for i := int64(1); i <= goroutineNum; i++ { limitChan <- i var ( start = syncTaskDoc.Height + (i-1)*syncBlockNumFastSync + 1 end = syncTaskDoc.Height + i*syncBlockNumFastSync ) - if i == goRoutineNum { + if i == goroutineNum { end = latestBlockHeight } go syncBlock(start, end, funcChain, ch, i) @@ -135,9 +163,9 @@ func fastSync(c rpcClient.Client) error { for { select { case threadNo := <-ch: - activeThreadNum = activeThreadNum - 1 - logger.Info.Printf("ThreadNo[%d] is over and active thread num is %d\n", threadNo, activeThreadNum) - if activeThreadNum == 0 { + activeGoroutineNum = activeGoroutineNum - 1 + logger.Info.Printf("ThreadNo[%d] is over and active thread num is %d\n", threadNo, activeGoroutineNum) + if activeGoroutineNum == 0 { goto end } } @@ -145,34 +173,42 @@ func fastSync(c rpcClient.Client) error { end: { - logger.Info.Println("Fast sync block, complete sync task") + logger.Info.Println("This fastSync task complete!") // update syncTask document block, _ := c.Block(&latestBlockHeight) syncTaskDoc.Height = block.Block.Height syncTaskDoc.Time = block.Block.Time - store.Update(syncTaskDoc) - return nil + err := store.Update(syncTaskDoc) + if err != nil { + logger.Error.Printf("Update syncTask fail, err is %v", + err.Error()) + } + return syncTaskDoc.Height } } func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex sync.Mutex), ch chan int64, threadNum int64) { - logger.Info.Printf("ThreadNo[%d] begin sync block from %d to %d\n", threadNum, start, end) + logger.Info.Printf("ThreadNo[%d] begin sync block from %d to %d\n", + threadNum, start, end) + client := helper.GetClient() // release client defer client.Release() - // release buffer chain - defer func() { - <-limitChan - }() + // release unBuffer chain and buffer chain + // defer func() { + // + // }() for j := start; j <= end; j++ { block, err := client.Client.Block(&j) if err != nil { + logger.Error.Printf("Invalid block height %d and err is %v, try again\n", j, err.Error()) // try again client2 := helper.GetClient() block, err = client2.Client.Block(&j) if err != nil { - logger.Error.Fatalf("invalid block height %d and err is %v\n", j, err.Error()) + ch <- threadNum + logger.Error.Fatalf("Invalid block height %d and err is %v\n", j, err.Error()) } } if block.BlockMeta.Header.NumTxs > 0 { @@ -180,6 +216,12 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex syn for _, txByte := range txs { txType, tx := helper.ParseTx(txByte) txHash := strings.ToUpper(hex.EncodeToString(txByte.Hash())) + if txHash == "" { + logger.Warning.Printf("Tx has no hash, skip this tx."+ + ""+"type of tx is %v, value of tx is %v\n", + txType, tx) + continue + } logger.Info.Printf("===========threadNo[%d] find tx,txType=%s;txHash=%s\n", threadNum, txType, txHash) switch txType { @@ -213,8 +255,17 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex syn Time: block.Block.Time, TxNum: block.BlockMeta.Header.NumTxs, } - store.SaveOrUpdate(bk) - + if err := store.Save(bk); err != nil { + logger.Error.Printf("Save block info failed, err is %v", + err.Error()) + } } + + logger.Info.Printf("ThreadNo[%d] finish sync block from %d to %d\n", + threadNum, start, end) + + <- limitChan ch <- threadNum + logger.Info.Printf("Send threadNum into channel: %v\n", threadNum) + } diff --git a/sync/sync_test.go b/sync/sync_test.go index 4b1d975..d507a76 100644 --- a/sync/sync_test.go +++ b/sync/sync_test.go @@ -2,75 +2,72 @@ package sync import ( "testing" - "time" - "fmt" "github.com/robfig/cron" conf "github.com/irisnet/iris-sync-server/conf/server" - rpcClient "github.com/tendermint/tendermint/rpc/client" "sync" "github.com/irisnet/iris-sync-server/module/logger" + "time" ) func TestStart(t *testing.T) { - Start() - for true { - time.Sleep(time.Minute) - fmt.Printf("wait\n") + var ( + limitChan chan int + unBufferChan chan int + ) + limitChan = make(chan int, 3) + unBufferChan = make(chan int) + goroutineNum := 5 + activeGoroutineNum := goroutineNum + for i := 1; i <= goroutineNum; i++ { + limitChan <- i + go func(goroutineNum int, ch chan int) { + logger.Info.Println("release limitChan") + <- limitChan + defer func() { + logger.Info.Printf("%v goroutine send data to channel\n", + goroutineNum) + ch <- goroutineNum + }() + + }(i, nil) + } + + for + { + select { + case <-unBufferChan: + activeGoroutineNum = activeGoroutineNum - 1 + logger.Info.Printf("active goroutine num is %v", activeGoroutineNum) + if activeGoroutineNum == 0 { + logger.Info.Println("All goroutine complete") + break + } + } } + + } func Test_startCron(t *testing.T) { var wg sync.WaitGroup + var mutex sync.Mutex wg.Add(2) spec := conf.SyncCron c := cron.New() c.AddFunc(spec, func() { - logger.Info.Println("print word every second") + mutex.Lock() + var sleepSecond time.Duration + sleepSecond = 3 + time.Sleep(time.Second * sleepSecond) + logger.Info.Printf("awake up after %v second\n", sleepSecond) + mutex.Unlock() }) go c.Start() wg.Wait() } -func Test_watchBlock(t *testing.T) { - type args struct { - c rpcClient.Client - } - tests := []struct { - name string - args args - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := watchBlock(tt.args.c); (err != nil) != tt.wantErr { - t.Errorf("watchBlock() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} -func Test_fastSync(t *testing.T) { - type args struct { - c rpcClient.Client - } - tests := []struct { - name string - args args - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := fastSync(tt.args.c); (err != nil) != tt.wantErr { - t.Errorf("fastSync() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/tools/Makefile b/tools/Makefile new file mode 100644 index 0000000..08969c1 --- /dev/null +++ b/tools/Makefile @@ -0,0 +1,33 @@ +######################################## +### GLIDE + +GLIDE = github.com/Masterminds/glide +GLIDE_CHECK := $(shell command -v glide 2> /dev/null) + +check_tools: +ifndef GLIDE_CHECK + @echo "No glide in path. Install with 'make get_tools'." +else + @echo "Found glide in path." +endif + +get_tools: +ifdef GLIDE_CHECK + @echo "glide is already installed. Run 'make update_tools' to update." +else + @echo "$(ansi_grn)Installing glide$(ansi_end)" + go get -v $(GLIDE) +endif + +update_tools: + @echo "$(ansi_grn)Updating glide$(ansi_end)" + go get -u -v $(GLIDE) + + +######################################## +# ANSI colors + +ansi_red=\033[0;31m +ansi_grn=\033[0;32m +ansi_yel=\033[0;33m +ansi_end=\033[0m \ No newline at end of file diff --git a/util/constant/types.go b/util/constant/types.go index 2b95211..23f9d3e 100644 --- a/util/constant/types.go +++ b/util/constant/types.go @@ -1,6 +1,20 @@ +// package for define constants + package constant const ( TxTypeCoin = "coin" TxTypeStake = "stake" + + EnvNameDbHost = "DB_HOST" + EnvNameDbPort = "DB_PORT" + EnvNameDbUser = "DB_USER" + EnvNameDbPassWd = "DB_PASSWD" + EnvNameDbDataBase = "DB_DATABASE" + + EnvNameSerNetworkNodeUrl = "SER_BC_NODE_URL" + EnvNameSerNetworkChainId = "SER_BC_CHAIN_ID" + EnvNameSerNetworkToken = "SER_BC_TOKEN" + EnvNameSerMaxGoRoutine = "SER_MAX_GOROUTINE" + EnvNameSerSyncBlockNum = "SER_SYNC_BLOCK_NUM" ) diff --git a/util/helper/account.go b/util/helper/account.go index 88ae1d6..ba8b4a7 100644 --- a/util/helper/account.go +++ b/util/helper/account.go @@ -1,12 +1,13 @@ +// This package is used for query balance of account + package helper import ( - "fmt" "time" "github.com/irisnet/iris-sync-server/module/logger" - wire "github.com/tendermint/go-wire" + "github.com/tendermint/go-wire" "github.com/cosmos/cosmos-sdk/modules/coin" "github.com/cosmos/cosmos-sdk/stack" "github.com/cosmos/cosmos-sdk/client/commands" @@ -15,6 +16,7 @@ import ( "github.com/tendermint/iavl" "github.com/cosmos/cosmos-sdk/client" rpcclient "github.com/tendermint/tendermint/rpc/client" + "github.com/pkg/errors" ) var delay = false @@ -37,7 +39,7 @@ func QueryAccountBalance(address string, delay bool) *coin.Account { } _, err2 := GetParsed(key, account, query.GetHeight(), false) if err2 != nil { - logger.Info.Printf("account bytes are empty for address: %q\n", address) + logger.Info.Printf("QueryAccountBalance failed, account bytes are empty for address: %q\n", address) } return account } @@ -66,7 +68,7 @@ func GetParsed(key []byte, data interface{}, height int64, prove bool) (int64, e // and it is localhost or you have a secure connection (not HTTP) func Get(key []byte, height int64, prove bool) (data.Bytes, int64, error) { if height < 0 { - return nil, 0, fmt.Errorf("Height cannot be negative\n") + return nil, 0, errors.New("Height cannot be negative\n") } if !prove { diff --git a/util/helper/account_test.go b/util/helper/account_test.go index d0c9324..2076d9d 100644 --- a/util/helper/account_test.go +++ b/util/helper/account_test.go @@ -27,7 +27,7 @@ func TestQueryAccountBalance(t *testing.T) { address string delay bool }{ - address: "ADBC4AAB3A089BDC8A8155AB97E64CD2CF4A0E9F", + address: "37D2C544F9D2CF811108B56A496520129B1F80CC", delay: false}, }, { diff --git a/util/helper/pool_client.go b/util/helper/pool_client.go index a52136b..15f2644 100644 --- a/util/helper/pool_client.go +++ b/util/helper/pool_client.go @@ -5,11 +5,10 @@ package helper import ( "errors" - "log" - conf "github.com/irisnet/iris-sync-server/conf/server" rpcClient "github.com/cosmos/cosmos-sdk/client" + "github.com/irisnet/iris-sync-server/module/logger" "github.com/tendermint/tendermint/rpc/client" ) @@ -47,7 +46,7 @@ func InitClientPool() { func GetClient() Client { c, err := getClient() if err != nil { - log.Fatal(err) + logger.Error.Fatalln(err.Error()) } return c } @@ -61,38 +60,47 @@ func (n Client) Release() { } func createConnection(id int64) Client { - client := Client{ + tmClient := Client{ Client: rpcClient.GetNode(conf.BlockChainMonitorUrl), used: false, id: id, } - pool.clients[id] = client + + if id == int64(len(pool.clients)) { + newSlice := make([]Client, pool.maxConnection) + for i, v := range pool.clients { + newSlice[i] = v + } + pool.clients = newSlice + } + pool.clients[id] = tmClient pool.available++ - return client + return tmClient } func getClient() (Client, error) { if pool.available == 0 { maxConnNum := int64(conf.MaxConnectionNum) if pool.used < maxConnNum { - var client Client - for i := int64(len(pool.clients)); i < maxConnNum; i++ { - client = createConnection(i) + var tmClient Client + length := len(pool.clients) + for i := int64(length); i < maxConnNum; i++ { + tmClient = createConnection(i) } - return client, nil + return tmClient, nil } else { - log.Fatal("client pool has no available connection") + logger.Error.Fatalln("client pool has no available connection") } } - for _, client := range pool.clients { - if !client.used { - client.used = true - pool.clients[client.id] = client + for _, tmClient := range pool.clients { + if !tmClient.used { + tmClient.used = true + pool.clients[tmClient.id] = tmClient pool.available-- pool.used++ - log.Printf("current available coonection ：%d", pool.available) - return client, nil + logger.Info.Printf("current available coonection ：%d\n", pool.available) + return tmClient, nil } } return Client{}, errors.New("pool is empty") diff --git a/util/helper/pool_client_test.go b/util/helper/pool_client_test.go new file mode 100644 index 0000000..495fa88 --- /dev/null +++ b/util/helper/pool_client_test.go @@ -0,0 +1,57 @@ +// init client from clientPool. +// client is httpClient of tendermint + +package helper + +import ( + "testing" + + "github.com/tendermint/tendermint/rpc/client" + + conf "github.com/irisnet/iris-sync-server/conf/server" + "github.com/irisnet/iris-sync-server/module/logger" +) + +func TestInitClientPool(t *testing.T) { + a := []int{1, 2, 3} + b := make([]int, 6, 6) + for index, value := range a { + b[index] = value + } + b[3] = 4 + logger.Info.Println(b) +} + +func TestGetClient(t *testing.T) { + InitClientPool() + + for i := 0; i < conf.InitConnectionNum + 10; i++ { + client := GetClient() + logger.Info.Println(client) + } + +} + +func TestClient_Release(t *testing.T) { + type fields struct { + Client client.Client + used bool + id int64 + } + tests := []struct { + name string + fields fields + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + n := Client{ + Client: tt.fields.Client, + used: tt.fields.used, + id: tt.fields.id, + } + n.Release() + }) + } +} diff --git a/util/helper/tx.go b/util/helper/tx.go index 6cced0c..a58f854 100644 --- a/util/helper/tx.go +++ b/util/helper/tx.go @@ -1,3 +1,5 @@ +// package for parse tx struct from binary data + package helper import ( @@ -53,7 +55,6 @@ func ParseTx(txByte types.Tx) (string, interface{}) { case nonce.Tx: ctx, _ := txi.Unwrap().(nonce.Tx) nonceAddr = ctx.Signers[0].Address - fmt.Println(nonceAddr) break case stake.TxUnbond, stake.TxDelegate, stake.TxDeclareCandidacy: kind, _ := txi.GetKind() @@ -81,7 +82,7 @@ func ParseTx(txByte types.Tx) (string, interface{}) { return kind, StakeTxDeclareCandidacy case stake.TypeTxEditCandidacy: // TODO：record edit candidacy tx if necessary - //ctx, _ := txi.Unwrap().(stake.TxEditCandidacy) + // ctx, _ := txi.Unwrap().(stake.TxEditCandidacy) break case stake.TypeTxDelegate: ctx, _ := txi.Unwrap().(stake.TxDelegate) From be6a8a3075801e2a4acc198f3970d31ea5e4f2b9 Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Fri, 25 May 2018 17:11:08 +0800 Subject: [PATCH 20/37] merge coinTx and stakeTx into commonTx#IRISSYNCS-3 --- model/store/document/tx_coin.go | 2 +- model/store/document/tx_common.go | 34 ++ mongodb/mongodb.js | 9 + sync/handler.go | 550 +++++++++++++++++------------- 4 files changed, 364 insertions(+), 231 deletions(-) create mode 100644 model/store/document/tx_common.go diff --git a/model/store/document/tx_coin.go b/model/store/document/tx_coin.go index fb401d2..8bc8597 100644 --- a/model/store/document/tx_coin.go +++ b/model/store/document/tx_coin.go @@ -11,7 +11,7 @@ const ( CollectionNmCoinTx = "tx_coin" ) -//Coin交易 +// Coin tx type CoinTx struct { TxHash string `bson:"tx_hash"` Time time.Time `bson:"time"` diff --git a/model/store/document/tx_common.go b/model/store/document/tx_common.go new file mode 100644 index 0000000..1448ce8 --- /dev/null +++ b/model/store/document/tx_common.go @@ -0,0 +1,34 @@ +package document + +import ( + "time" + "github.com/cosmos/cosmos-sdk/modules/coin" + "gopkg.in/mgo.v2/bson" + "gopkg.in/mgo.v2" +) + +const ( + CollectionNmCommonTx = "tx_common" +) + +type CommonTx struct { + TxHash string `bson:"tx_hash"` + Time time.Time `bson:"time"` + Height int64 `bson:"height"` + From string `bson:"from"` + To string `bson:"to"` + Amount coin.Coins `bson:"amount"` + Type string `bson:"type"` +} + +func (d CommonTx) Name() string { + return CollectionNmCommonTx +} + +func (d CommonTx) PkKvPair() map[string]interface{} { + return bson.M{"tx_hash": d.TxHash} +} + +func (d CommonTx) Index() []mgo.Index { + return []mgo.Index{} +} diff --git a/mongodb/mongodb.js b/mongodb/mongodb.js index 5e1ddb5..f1a2cb8 100644 --- a/mongodb/mongodb.js +++ b/mongodb/mongodb.js @@ -6,6 +6,7 @@ db.createCollection("stake_role_delegator"); db.createCollection("sync_task"); db.createCollection("tx_coin"); db.createCollection("tx_stake"); +db.createCollection("tx_common"); // create index db.account.createIndex({"address": 1}, {"unique": true}); @@ -37,6 +38,12 @@ db.tx_stake.createIndex({"type": 1}); db.tx_stake.createIndex({"time": -1}); db.tx_stake.createIndex({"from": 1, "to": 1, "type": 1, "time": -1}); +db.tx_common.createIndex({"tx_hash": 1}, {"unique": true}); +db.tx_common.createIndex({"from": 1}); +db.tx_common.createIndex({"to": 1}); +db.tx_common.createIndex({"height": -1}); +db.tx_common.createIndex({"time": -1}); +db.tx_common.createIndex({"type": 1}); // drop collection db.account.drop(); @@ -46,6 +53,7 @@ db.stake_role_delegator.drop(); db.sync_task.drop(); db.tx_coin.drop(); db.tx_stake.drop(); +db.tx_common.drop(); // remove collection data db.account.remove({}); @@ -55,6 +63,7 @@ db.stake_role_delegator.remove({}); db.sync_task.remove({}); db.tx_coin.remove({}); db.tx_stake.remove({}); +db.tx_common.remove({}); diff --git a/sync/handler.go b/sync/handler.go index 714df04..6536a13 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -11,6 +11,7 @@ import ( "github.com/irisnet/iris-sync-server/module/stake" "github.com/irisnet/iris-sync-server/util/constant" "github.com/irisnet/iris-sync-server/util/helper" + "github.com/cosmos/cosmos-sdk/modules/coin" ) var ( @@ -27,7 +28,6 @@ func handle(tx store.Docs, mutex sync.Mutex, funChains []func(tx store.Docs, mut // save Tx document into collection func saveTx(tx store.Docs, mutex sync.Mutex) { methodName = "SaveTx: " - txCollectionName := tx.Name() // save tx document into database storeTxDocFunc := func(doc store.Docs) { @@ -37,135 +37,160 @@ func saveTx(tx store.Docs, mutex sync.Mutex) { methodName, doc, err.Error()) } } - - switch txCollectionName { - case document.CollectionNmCoinTx: + + // save common tx document + saveCommonTx := func(commonTx document.CommonTx) { + err := store.Save(commonTx) + if err != nil { + logger.Error.Printf("%v Save commonTx failed. doc is %+v, err is %v", + methodName, commonTx, err.Error()) + } + } + + txType := GetTxType(tx) + if txType == "" { + logger.Error.Printf("%v get tx type failed, tx is %v\n", + methodName, tx) + return + } + + saveCommonTx(buildCommonTxData(tx, txType)) + + switch txType { + case constant.TxTypeCoin: storeTxDocFunc(tx) break - case document.CollectionNmStakeTx: - if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { - logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) + case stake.TypeTxDeclareCandidacy: + stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, txType) break } - stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - - logger.Info.Printf("%v handle %v and type is %v", - methodName, txCollectionName, stakeType) - + storeTxDocFunc(stakeTxDeclareCandidacy) + mutex.Lock() - logger.Info.Printf("%v get lock\n", methodName) - - { - switch stakeType { - case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, stakeType) - break - } - storeTxDocFunc(stakeTxDeclareCandidacy) - - cd, err := document.QueryCandidateByPubkey(stakeTxDeclareCandidacy.PubKey) - - candidate := document.Candidate{ - Address: stakeTxDeclareCandidacy.From, - PubKey: stakeTxDeclareCandidacy.PubKey, - Description: stakeTxDeclareCandidacy.Description, - } - // TODO: in further share not equal amount - candidate.Shares += stakeTxDeclareCandidacy.Amount.Amount - candidate.VotingPower += int64(stakeTxDeclareCandidacy.Amount.Amount) - candidate.UpdateTime = stakeTxDeclareCandidacy.Time - - if err != nil && cd.Address == "" { - logger.Warning.Printf("%v Replace candidate from %+v to %+v\n", methodName, cd, candidate) - } - - store.SaveOrUpdate(candidate) - break - case stake.TypeTxDelegate: - stakeTx, r := tx.(document.StakeTx) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, stakeType) - break - } - storeTxDocFunc(stakeTx) - - candidate, err := document.QueryCandidateByPubkey(stakeTx.PubKey) - // candidate is not exist - if err != nil { - logger.Warning.Printf("%v candidate is not exist while delegate, add = %s ,pub_key = %s\n", - methodName, stakeTx.From, stakeTx.PubKey) - candidate = document.Candidate{ - PubKey: stakeTx.PubKey, - } - } - - delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) - // delegator is not exist - if err != nil { - delegator = document.Delegator{ - Address: stakeTx.From, - PubKey: stakeTx.PubKey, - } - } - // TODO: in further share not equal amount - delegator.Shares += stakeTx.Amount.Amount - delegator.UpdateTime = stakeTx.Time - store.SaveOrUpdate(delegator) - - candidate.Shares += stakeTx.Amount.Amount - candidate.VotingPower += int64(stakeTx.Amount.Amount) - candidate.UpdateTime = stakeTx.Time - store.SaveOrUpdate(candidate) - break - case stake.TypeTxUnbond: - stakeTx, r := tx.(document.StakeTx) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, stakeType) - break - } - storeTxDocFunc(stakeTx) - - delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) - // delegator is not exist - if err != nil { - logger.Warning.Printf("%v delegator is not exist while unBond,add = %s,pub_key=%s\n", - methodName, stakeTx.From, stakeTx.PubKey) - delegator = document.Delegator{ - Address: stakeTx.From, - PubKey: stakeTx.PubKey, - } - } - delegator.Shares -= stakeTx.Amount.Amount - delegator.UpdateTime = stakeTx.Time - store.SaveOrUpdate(delegator) - - candidate, err2 := document.QueryCandidateByPubkey(stakeTx.PubKey) - // candidate is not exist - if err2 != nil { - logger.Warning.Printf("%v candidate is not exist while unBond,add = %s,pub_key=%s\n", - methodName, stakeTx.From, stakeTx.PubKey) - candidate = document.Candidate{ - PubKey: stakeTx.PubKey, - } - } - candidate.Shares -= stakeTx.Amount.Amount - candidate.VotingPower -= int64(stakeTx.Amount.Amount) - candidate.UpdateTime = stakeTx.Time - store.SaveOrUpdate(candidate) - break + logger.Info.Printf("%v saveOrUpdate cndidates get lock\n", methodName) + + cd, err := document.QueryCandidateByPubkey(stakeTxDeclareCandidacy.PubKey) + + candidate := document.Candidate{ + Address: stakeTxDeclareCandidacy.From, + PubKey: stakeTxDeclareCandidacy.PubKey, + Description: stakeTxDeclareCandidacy.Description, + } + // TODO: in further share not equal amount + candidate.Shares += stakeTxDeclareCandidacy.Amount.Amount + candidate.VotingPower += int64(stakeTxDeclareCandidacy.Amount.Amount) + candidate.UpdateTime = stakeTxDeclareCandidacy.Time + + if err != nil && cd.Address == "" { + logger.Warning.Printf("%v Replace candidate from %+v to %+v\n", methodName, cd, candidate) + } + + store.SaveOrUpdate(candidate) + + mutex.Unlock() + logger.Info.Printf("%v saveOrUpdate cndidates release lock\n", methodName) + break + + case stake.TypeTxEditCandidacy: + break + case stake.TypeTxDelegate: + stakeTx, r := tx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, txType) + break + } + storeTxDocFunc(stakeTx) + + mutex.Lock() + logger.Info.Printf("%v saveOrUpdate tx type %v get lock\n", + methodName, txType) + + candidate, err := document.QueryCandidateByPubkey(stakeTx.PubKey) + // candidate is not exist + if err != nil { + logger.Warning.Printf("%v candidate is not exist while delegate, add = %s ,pub_key = %s\n", + methodName, stakeTx.From, stakeTx.PubKey) + candidate = document.Candidate{ + PubKey: stakeTx.PubKey, } } - + + delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) + // delegator is not exist + if err != nil { + delegator = document.Delegator{ + Address: stakeTx.From, + PubKey: stakeTx.PubKey, + } + } + // TODO: in further share not equal amount + delegator.Shares += stakeTx.Amount.Amount + delegator.UpdateTime = stakeTx.Time + store.SaveOrUpdate(delegator) + + candidate.Shares += stakeTx.Amount.Amount + candidate.VotingPower += int64(stakeTx.Amount.Amount) + candidate.UpdateTime = stakeTx.Time + store.SaveOrUpdate(candidate) + + mutex.Unlock() + logger.Info.Printf("%v saveOrUpdate tx type %v release lock\n", + methodName, txType) + break + + case stake.TypeTxUnbond: + stakeTx, r := tx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, txType) + break + } + storeTxDocFunc(stakeTx) + + mutex.Lock() + logger.Info.Printf("%v saveOrUpdate tx type %v get lock\n", + methodName, txType) + + delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) + // delegator is not exist + if err != nil { + logger.Warning.Printf("%v delegator is not exist while unBond,add = %s,pub_key=%s\n", + methodName, stakeTx.From, stakeTx.PubKey) + delegator = document.Delegator{ + Address: stakeTx.From, + PubKey: stakeTx.PubKey, + } + } + delegator.Shares -= stakeTx.Amount.Amount + delegator.UpdateTime = stakeTx.Time + store.SaveOrUpdate(delegator) + + candidate, err2 := document.QueryCandidateByPubkey(stakeTx.PubKey) + // candidate is not exist + if err2 != nil { + logger.Warning.Printf("%v candidate is not exist while unBond,add = %s,pub_key=%s\n", + methodName, stakeTx.From, stakeTx.PubKey) + candidate = document.Candidate{ + PubKey: stakeTx.PubKey, + } + } + candidate.Shares -= stakeTx.Amount.Amount + candidate.VotingPower -= int64(stakeTx.Amount.Amount) + candidate.UpdateTime = stakeTx.Time + store.SaveOrUpdate(candidate) + mutex.Unlock() - logger.Info.Printf("%v method release lock\n", methodName) + logger.Info.Printf("%v saveOrUpdate tx type %v release lock\n", + methodName, txType) + break } } +// save account func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { var ( address string @@ -173,9 +198,9 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { height int64 methodName = "SaveOrUpdateAccount: " ) - txCollectionName := tx.Name() logger.Info.Printf("Start %v\n", methodName) + // save account fun := func(address string, updateTime time.Time, height int64) { account := document.Account{ Address: address, @@ -188,77 +213,67 @@ func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { methodName, account.Address, err.Error()) } } - - // mutex.Lock() - // logger.Info.Printf("%v get lock\n", methodName) - - { - switch txCollectionName { - case document.CollectionNmCoinTx: - coinTx, r := tx.(document.CoinTx) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, txCollectionName) - break - } - updateTime = coinTx.Time - height = coinTx.Height - - fun(coinTx.From, updateTime, height) - fun(coinTx.To, updateTime, height) + + txType := GetTxType(tx) + if txType == "" { + logger.Error.Printf("%v get tx type failed, tx is %v\n", + methodName, tx) + return + } + + switch txType { + case constant.TxTypeCoin: + coinTx, r := tx.(document.CoinTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, txType) + break + } + updateTime = coinTx.Time + height = coinTx.Height + + fun(coinTx.From, updateTime, height) + fun(coinTx.To, updateTime, height) + break + case stake.TypeTxDeclareCandidacy: + stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, txType) + break + } + address = stakeTxDeclareCandidacy.From + updateTime = stakeTxDeclareCandidacy.Time + height = stakeTxDeclareCandidacy.Height + + fun(address, updateTime, height) + break + case stake.TypeTxEditCandidacy: + break + case stake.TypeTxDelegate, stake.TypeTxUnbond: + stakeTx, r := tx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, txType) break - case document.CollectionNmStakeTx: - if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { - logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) - break - } - stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - - logger.Info.Printf("%v handle %v and type is %v", - methodName, txCollectionName, stakeType) - - switch stakeType { - case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, stakeType) - break - } - address = stakeTxDeclareCandidacy.From - updateTime = stakeTxDeclareCandidacy.Time - height = stakeTxDeclareCandidacy.Height - break - case stake.TypeTxEditCandidacy: - break - case stake.TypeTxDelegate, stake.TypeTxUnbond: - stakeTx, r := tx.(document.StakeTx) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, stakeType) - break - } - address = stakeTx.From - updateTime = stakeTx.Time - height = stakeTx.Height - break - } - - fun(address, updateTime, height) } + address = stakeTx.From + updateTime = stakeTx.Time + height = stakeTx.Height + + fun(address, updateTime, height) + break } - + logger.Info.Printf("End %v\n", methodName) - // mutex.Unlock() - // logger.Info.Printf("%v release lock\n", methodName) } +// update account balance func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { var ( address string methodName = "UpdateAccountBalance: " ) - txCollectionName := tx.Name() logger.Info.Printf("Start %v\n", methodName) fun := func(address string) { @@ -277,61 +292,136 @@ func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { methodName, account.Address, err) } } - - // mutex.Lock() - // logger.Info.Printf("%v get lock\n", methodName) - { - switch txCollectionName { - case document.CollectionNmCoinTx: - coinTx, r := tx.(document.CoinTx) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, txCollectionName) - break - } - fun(coinTx.From) - fun(coinTx.To) + + txType := GetTxType(tx) + if txType == "" { + logger.Error.Printf("%v get tx type failed, tx is %v\n", + methodName, tx) + return + } + + switch txType { + case constant.TxTypeCoin: + coinTx, r := tx.(document.CoinTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, txType) break - case document.CollectionNmStakeTx: - if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { - logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) - break - } - stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - - logger.Info.Printf("%v handle %v and type is %v", - methodName, txCollectionName, stakeType) - - switch stakeType { - case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, stakeType) - break - } - address = stakeTxDeclareCandidacy.From - break - case stake.TypeTxEditCandidacy: - break - case stake.TypeTxDelegate, stake.TypeTxUnbond: - stakeTx, r := tx.(document.StakeTx) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, stakeType) - break - } - address = stakeTx.From - break - } - fun(address) + } + fun(coinTx.From) + fun(coinTx.To) + break + case stake.TypeTxDeclareCandidacy: + stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, txType) + break + } + address = stakeTxDeclareCandidacy.From + fun(address) + break + case stake.TypeTxEditCandidacy: + break + case stake.TypeTxDelegate, stake.TypeTxUnbond: + stakeTx, r := tx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", + methodName, txType) break } + address = stakeTx.From + fun(address) + break } logger.Info.Printf("End %v\n", methodName) +} - // mutex.Unlock() - // logger.Info.Println("updateAccountBalance method release lock") - +// build common tx data through parse tx +func buildCommonTxData(tx store.Docs, txType string) document.CommonTx { + var commonTx document.CommonTx + + if txType == "" { + txType = GetTxType(tx) + } + switch txType { + // tx coin + case constant.TxTypeCoin: + txCoin := tx.(document.CoinTx) + commonTx = document.CommonTx{ + TxHash: txCoin.TxHash, + Time: txCoin.Time, + Height: txCoin.Height, + From: txCoin.From, + To: txCoin.To, + Amount: txCoin.Amount, + Type: txType, + } + break + case stake.TypeTxDeclareCandidacy: + txDeclare := tx.(document.StakeTxDeclareCandidacy) + commonTx = document.CommonTx{ + TxHash: txDeclare.TxHash, + Time: txDeclare.Time, + Height: txDeclare.Height, + From: txDeclare.From, + To: txDeclare.PubKey, + Amount: []coin.Coin{txDeclare.Amount}, + Type: txType, + } + break + case stake.TypeTxEditCandidacy: + break + case stake.TypeTxDelegate, stake.TypeTxUnbond: + txStake := tx.(document.StakeTx) + commonTx = document.CommonTx{ + TxHash: txStake.TxHash, + Time: txStake.Time, + Height: txStake.Height, + From: txStake.From, + To: txStake.PubKey, + Amount: []coin.Coin{txStake.Amount}, + Type: txType, + } + break + } + + return commonTx } + +// get tx type +func GetTxType(tx store.Docs) string { + txCollectionName := tx.Name() + var txType string + + switch txCollectionName { + case document.CollectionNmCoinTx: + txType = constant.TxTypeCoin + break + case document.CollectionNmStakeTx: + if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { + logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) + break + } + stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() + + switch stakeType { + case stake.TypeTxDeclareCandidacy: + txType = stake.TypeTxDeclareCandidacy + break + case stake.TypeTxEditCandidacy: + txType = stake.TypeTxEditCandidacy + break + case stake.TypeTxDelegate: + txType = stake.TypeTxDelegate + break + case stake.TypeTxUnbond: + txType = stake.TypeTxUnbond + break + } + break + } + + return txType +} \ No newline at end of file From c2d7cb63c2200856c4353e644a0fd368f39fb612 Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Fri, 25 May 2018 19:15:26 +0800 Subject: [PATCH 21/37] modify tx_type's value when write to db#IRISSYNCS-3 --- sync/handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/handler.go b/sync/handler.go index 6536a13..38d6cba 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -368,7 +368,7 @@ func buildCommonTxData(tx store.Docs, txType string) document.CommonTx { From: txDeclare.From, To: txDeclare.PubKey, Amount: []coin.Coin{txDeclare.Amount}, - Type: txType, + Type: txDeclare.Type, } break case stake.TypeTxEditCandidacy: @@ -382,7 +382,7 @@ func buildCommonTxData(tx store.Docs, txType string) document.CommonTx { From: txStake.From, To: txStake.PubKey, Amount: []coin.Coin{txStake.Amount}, - Type: txType, + Type: txStake.Type, } break } From 1db0cd8587491921da76262fa31b03c30c88d4c7 Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Wed, 13 Jun 2018 14:31:16 +0800 Subject: [PATCH 22/37] rename project from iris-sync-server to irishub-sync and modify gaia repository(#IRISSYNCS-4) --- Dockerfile | 21 +++++++------ Makefile | 2 +- README.md | 2 +- conf/db/types.go.example | 4 +-- conf/server/types.go.example | 4 +-- glide.lock | 4 +-- glide.yaml | 2 +- main.go | 2 +- model/store/document/account.go | 2 +- model/store/document/doc.go | 2 +- model/store/document/stake_role_candidate.go | 4 +-- model/store/document/stake_role_delegator.go | 2 +- model/store/document/sync_task.go | 2 +- model/store/store.go | 4 +-- module/logger/logger.go | 31 ++++++++++---------- sync/handler.go | 12 ++++---- sync/handler_test.go | 12 ++++---- sync/sync.go | 14 ++++----- sync/sync_test.go | 4 +-- util/helper/account.go | 2 +- util/helper/account_test.go | 2 +- util/helper/pool_client.go | 4 +-- util/helper/pool_client_test.go | 4 +-- util/helper/tx.go | 8 ++--- util/helper/tx_test.go | 2 +- 25 files changed, 78 insertions(+), 74 deletions(-) diff --git a/Dockerfile b/Dockerfile index 74ef5f9..622541e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,14 +7,10 @@ ENV PACKAGES go make git libc-dev bash ENV GOPATH /root/go ENV BASE_PATH $GOPATH/src/github.com/irisnet -ENV REPO_PATH $BASE_PATH/iris-sync-server +ENV REPO_PATH $BASE_PATH/irishub-sync ENV LOG_DIR /sync-iris/log ENV PATH $GOPATH/bin:$PATH -# Set volumes - -VOLUME $LOG_DIR:sync-iris-log - # Link expected Go repo path RUN mkdir -p $LOG_DIR $GOPATH/pkg $GOPATH/bin $BASE_PATH $REPO_PATH @@ -23,11 +19,18 @@ RUN mkdir -p $LOG_DIR $GOPATH/pkg $GOPATH/bin $BASE_PATH $REPO_PATH COPY . $REPO_PATH -# Install minimum necessary dependencies, build iris-sync-server +# Install minimum necessary dependencies, build irishub-sync RUN apk add --no-cache $PACKAGES && \ - cd $REPO_PATH && make all && \ - mv $REPO_PATH/sync-iris $GOPATH/bin && \ + mv $REPO_PATH/conf/db/types.go.example $REPO_PATH/conf/db/types.go && \ + mv $REPO_PATH/conf/server/types.go.example $REPO_PATH/conf/server/types.go && \ + cd $REPO_PATH && \ + make all && \ + mv $REPO_PATH/sync-irishub $GOPATH/bin && \ rm -rf $REPO_PATH/vendor && \ + rm -rf $GOPATH/src/github.com/golang $GOPATH/bin/dep $GOPATH/pkg/* && \ apk del $PACKAGES -CMD sync-iris > $LOG_DIR/debug.log && tail -f $LOG_DIR/debug.log \ No newline at end of file +# Set volumes +VOLUME ["$LOG_DIR"] + +CMD sync-irishub > $LOG_DIR/debug.log && tail -f $LOG_DIR/debug.log \ No newline at end of file diff --git a/Makefile b/Makefile index 74e0a7e..3cd07ca 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GOBUILD=$(GOCMD) build GOCLEAN=$(GOCMD) clean GOTEST=$(GOCMD) test GOGET=$(GOCMD) get -BINARY_NAME=sync-iris +BINARY_NAME=sync-irishub BINARY_UNIX=$(BINARY_NAME)-unix all: get_tools get_deps build diff --git a/README.md b/README.md index bb2d7c4..eb92d23 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# IRIS-SYNC-SERVER +# IRISHUB-SYNC A server that synchronize IRIS blockChain data into a database # Structure diff --git a/conf/db/types.go.example b/conf/db/types.go.example index 456b792..cc6524a 100644 --- a/conf/db/types.go.example +++ b/conf/db/types.go.example @@ -2,8 +2,8 @@ package db import ( "os" - "github.com/irisnet/iris-sync-server/util/constant" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/module/logger" ) var ( diff --git a/conf/server/types.go.example b/conf/server/types.go.example index 95529a2..a4c89ee 100644 --- a/conf/server/types.go.example +++ b/conf/server/types.go.example @@ -4,8 +4,8 @@ import ( "os" "strconv" - "github.com/irisnet/iris-sync-server/util/constant" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/module/logger" ) var ( diff --git a/glide.lock b/glide.lock index d0a8da6..1c1156e 100644 --- a/glide.lock +++ b/glide.lock @@ -46,8 +46,8 @@ imports: - stack - state - version -- name: github.com/cosmos/gaia - version: 2ac7ed920446ef8edf582c2b3e890cb7676a7d77 +- name: github.com/MrXJC/gaia + version: develop subpackages: - modules/stake - modules/stake/commands diff --git a/glide.yaml b/glide.yaml index 4a1d605..6de341d 100644 --- a/glide.yaml +++ b/glide.yaml @@ -1,4 +1,4 @@ -package: github.com/irisnet/iris-sync-server +package: github.com/irisnet/irishub-sync import: - package: github.com/gorilla/websocket - package: github.com/pkg/errors diff --git a/main.go b/main.go index c972a94..c9d2821 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,7 @@ package main import ( "sync" - syncTask "github.com/irisnet/iris-sync-server/sync" + syncTask "github.com/irisnet/irishub-sync/sync" ) func main() { diff --git a/model/store/document/account.go b/model/store/document/account.go index 5cd8884..da474f9 100644 --- a/model/store/document/account.go +++ b/model/store/document/account.go @@ -2,7 +2,7 @@ package document import ( "github.com/cosmos/cosmos-sdk/modules/coin" - "github.com/irisnet/iris-sync-server/model/store" + "github.com/irisnet/irishub-sync/model/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" diff --git a/model/store/document/doc.go b/model/store/document/doc.go index 27705a9..5d11c97 100644 --- a/model/store/document/doc.go +++ b/model/store/document/doc.go @@ -1,7 +1,7 @@ package document import ( - "github.com/irisnet/iris-sync-server/model/store" + "github.com/irisnet/irishub-sync/model/store" ) func init() { diff --git a/model/store/document/stake_role_candidate.go b/model/store/document/stake_role_candidate.go index 0a103d8..d29e2e2 100644 --- a/model/store/document/stake_role_candidate.go +++ b/model/store/document/stake_role_candidate.go @@ -2,11 +2,11 @@ package document import ( "errors" - "github.com/irisnet/iris-sync-server/model/store" + "github.com/irisnet/irishub-sync/model/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/module/logger" ) const ( diff --git a/model/store/document/stake_role_delegator.go b/model/store/document/stake_role_delegator.go index 4e95cfa..b867620 100644 --- a/model/store/document/stake_role_delegator.go +++ b/model/store/document/stake_role_delegator.go @@ -2,7 +2,7 @@ package document import ( "errors" - "github.com/irisnet/iris-sync-server/model/store" + "github.com/irisnet/irishub-sync/model/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" diff --git a/model/store/document/sync_task.go b/model/store/document/sync_task.go index 61c8a7f..16c40cc 100644 --- a/model/store/document/sync_task.go +++ b/model/store/document/sync_task.go @@ -1,7 +1,7 @@ package document import ( - "github.com/irisnet/iris-sync-server/model/store" + "github.com/irisnet/irishub-sync/model/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" diff --git a/model/store/store.go b/model/store/store.go index cc0322e..fec6019 100644 --- a/model/store/store.go +++ b/model/store/store.go @@ -5,8 +5,8 @@ package store import ( "time" - conf "github.com/irisnet/iris-sync-server/conf/db" - "github.com/irisnet/iris-sync-server/module/logger" + conf "github.com/irisnet/irishub-sync/conf/db" + "github.com/irisnet/irishub-sync/module/logger" "errors" "fmt" diff --git a/module/logger/logger.go b/module/logger/logger.go index 9ae2b6a..f75f11a 100644 --- a/module/logger/logger.go +++ b/module/logger/logger.go @@ -4,6 +4,7 @@ import ( "io" "log" "os" + "io/ioutil" ) var ( @@ -14,24 +15,24 @@ var ( ) const ( - traceDbFile = ".sync_server_db.log" - debugFile = ".sync_server_debug.log" + // traceDbFile = ".sync_server_db.log" + // debugFile = ".sync_server_debug.log" errFile = ".sync_server_err.log" warningFile = ".sync_server_warning.log" ) func init() { - traceDbFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+traceDbFile), - os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - log.Fatalln("Failed to open trace db log file:", err) - } - - debugFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+debugFile), - os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - log.Fatalln("Failed to open debug log file:", err) - } + // traceDbFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+traceDbFile), + // os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + // if err != nil { + // log.Fatalln("Failed to open trace db log file:", err) + // } + // + // debugFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+debugFile), + // os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + // if err != nil { + // log.Fatalln("Failed to open debug log file:", err) + // } warningFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+warningFile), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) @@ -45,11 +46,11 @@ func init() { log.Fatalln("Failed to open error log file:", err) } - Trace = log.New(io.MultiWriter(traceDbFile, os.Stdout), + Trace = log.New(ioutil.Discard, "TRACE: ", log.Ldate|log.Ltime|log.Lshortfile) - Info = log.New(io.MultiWriter(debugFile, os.Stdout), + Info = log.New(os.Stdout, "INFO: ", log.Ldate|log.Ltime|log.Lshortfile) diff --git a/sync/handler.go b/sync/handler.go index 38d6cba..9d663c9 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -5,12 +5,12 @@ import ( "sync" "time" - "github.com/irisnet/iris-sync-server/model/store" - "github.com/irisnet/iris-sync-server/model/store/document" - "github.com/irisnet/iris-sync-server/module/logger" - "github.com/irisnet/iris-sync-server/module/stake" - "github.com/irisnet/iris-sync-server/util/constant" - "github.com/irisnet/iris-sync-server/util/helper" + "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/model/store/document" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/module/stake" + "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/util/helper" "github.com/cosmos/cosmos-sdk/modules/coin" ) diff --git a/sync/handler_test.go b/sync/handler_test.go index a73c3dc..9959c58 100644 --- a/sync/handler_test.go +++ b/sync/handler_test.go @@ -2,12 +2,12 @@ package sync import ( "testing" - "github.com/irisnet/iris-sync-server/model/store" - "github.com/irisnet/iris-sync-server/util/helper" - "github.com/irisnet/iris-sync-server/module/logger" - "github.com/irisnet/iris-sync-server/util/constant" - "github.com/irisnet/iris-sync-server/model/store/document" - "github.com/irisnet/iris-sync-server/module/stake" + "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/util/helper" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/model/store/document" + "github.com/irisnet/irishub-sync/module/stake" "sync" ) diff --git a/sync/sync.go b/sync/sync.go index 6df2d63..8899820 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -4,14 +4,14 @@ import ( "encoding/hex" "strings" - conf "github.com/irisnet/iris-sync-server/conf/server" - "github.com/irisnet/iris-sync-server/model/store" - "github.com/irisnet/iris-sync-server/module/logger" - "github.com/irisnet/iris-sync-server/module/stake" - "github.com/irisnet/iris-sync-server/util/constant" - "github.com/irisnet/iris-sync-server/util/helper" + conf "github.com/irisnet/irishub-sync/conf/server" + "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/module/stake" + "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/util/helper" - "github.com/irisnet/iris-sync-server/model/store/document" + "github.com/irisnet/irishub-sync/model/store/document" "github.com/robfig/cron" rpcClient "github.com/tendermint/tendermint/rpc/client" ctypes "github.com/tendermint/tendermint/rpc/core/types" diff --git a/sync/sync_test.go b/sync/sync_test.go index d507a76..49a713a 100644 --- a/sync/sync_test.go +++ b/sync/sync_test.go @@ -4,10 +4,10 @@ import ( "testing" "github.com/robfig/cron" - conf "github.com/irisnet/iris-sync-server/conf/server" + conf "github.com/irisnet/irishub-sync/conf/server" "sync" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/module/logger" "time" ) diff --git a/util/helper/account.go b/util/helper/account.go index ba8b4a7..b4e9a1b 100644 --- a/util/helper/account.go +++ b/util/helper/account.go @@ -5,7 +5,7 @@ package helper import ( "time" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/module/logger" "github.com/tendermint/go-wire" "github.com/cosmos/cosmos-sdk/modules/coin" diff --git a/util/helper/account_test.go b/util/helper/account_test.go index 2076d9d..9026a54 100644 --- a/util/helper/account_test.go +++ b/util/helper/account_test.go @@ -3,7 +3,7 @@ package helper import ( "testing" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/module/logger" ) func setUp() { diff --git a/util/helper/pool_client.go b/util/helper/pool_client.go index 15f2644..ef4cf67 100644 --- a/util/helper/pool_client.go +++ b/util/helper/pool_client.go @@ -5,10 +5,10 @@ package helper import ( "errors" - conf "github.com/irisnet/iris-sync-server/conf/server" + conf "github.com/irisnet/irishub-sync/conf/server" rpcClient "github.com/cosmos/cosmos-sdk/client" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/module/logger" "github.com/tendermint/tendermint/rpc/client" ) diff --git a/util/helper/pool_client_test.go b/util/helper/pool_client_test.go index 495fa88..e2f8d9d 100644 --- a/util/helper/pool_client_test.go +++ b/util/helper/pool_client_test.go @@ -8,8 +8,8 @@ import ( "github.com/tendermint/tendermint/rpc/client" - conf "github.com/irisnet/iris-sync-server/conf/server" - "github.com/irisnet/iris-sync-server/module/logger" + conf "github.com/irisnet/irishub-sync/conf/server" + "github.com/irisnet/irishub-sync/module/logger" ) func TestInitClientPool(t *testing.T) { diff --git a/util/helper/tx.go b/util/helper/tx.go index a58f854..d96b3ea 100644 --- a/util/helper/tx.go +++ b/util/helper/tx.go @@ -7,8 +7,8 @@ import ( "fmt" "strings" - "github.com/irisnet/iris-sync-server/module/logger" - "github.com/irisnet/iris-sync-server/util/constant" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/util/constant" sdk "github.com/cosmos/cosmos-sdk" "github.com/tendermint/go-wire/data" @@ -20,8 +20,8 @@ import ( _ "github.com/cosmos/cosmos-sdk/modules/base" "github.com/cosmos/cosmos-sdk/modules/coin" "github.com/cosmos/cosmos-sdk/modules/nonce" - "github.com/irisnet/iris-sync-server/module/stake" - "github.com/irisnet/iris-sync-server/model/store/document" + "github.com/irisnet/irishub-sync/module/stake" + "github.com/irisnet/irishub-sync/model/store/document" ) // parse tx struct from binary data diff --git a/util/helper/tx_test.go b/util/helper/tx_test.go index 913596c..0cf6ad9 100644 --- a/util/helper/tx_test.go +++ b/util/helper/tx_test.go @@ -3,7 +3,7 @@ package helper import ( "testing" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/module/logger" _ "github.com/cosmos/cosmos-sdk/modules/auth" _ "github.com/cosmos/cosmos-sdk/modules/base" From f13e3c6997d0fd5980c621507319dd3767a4e2d0 Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Wed, 13 Jun 2018 14:31:16 +0800 Subject: [PATCH 23/37] rename project from iris-sync-server to irishub-sync and modify gaia repository(#IRISSYNCS-4) --- Dockerfile | 21 +++++++------ Makefile | 2 +- README.md | 2 +- conf/db/types.go.example | 4 +-- conf/server/types.go.example | 4 +-- glide.lock | 4 +-- glide.yaml | 2 +- main.go | 2 +- model/store/document/account.go | 2 +- model/store/document/doc.go | 2 +- model/store/document/stake_role_candidate.go | 4 +-- model/store/document/stake_role_delegator.go | 2 +- model/store/document/sync_task.go | 2 +- model/store/store.go | 4 +-- module/logger/logger.go | 31 ++++++++++---------- sync/handler.go | 12 ++++---- sync/handler_test.go | 12 ++++---- sync/sync.go | 14 ++++----- sync/sync_test.go | 4 +-- util/helper/account.go | 2 +- util/helper/account_test.go | 2 +- util/helper/pool_client.go | 4 +-- util/helper/pool_client_test.go | 4 +-- util/helper/tx.go | 8 ++--- util/helper/tx_test.go | 2 +- 25 files changed, 78 insertions(+), 74 deletions(-) diff --git a/Dockerfile b/Dockerfile index 74ef5f9..ee0b10b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,14 +7,10 @@ ENV PACKAGES go make git libc-dev bash ENV GOPATH /root/go ENV BASE_PATH $GOPATH/src/github.com/irisnet -ENV REPO_PATH $BASE_PATH/iris-sync-server +ENV REPO_PATH $BASE_PATH/irishub-sync ENV LOG_DIR /sync-iris/log ENV PATH $GOPATH/bin:$PATH -# Set volumes - -VOLUME $LOG_DIR:sync-iris-log - # Link expected Go repo path RUN mkdir -p $LOG_DIR $GOPATH/pkg $GOPATH/bin $BASE_PATH $REPO_PATH @@ -23,11 +19,18 @@ RUN mkdir -p $LOG_DIR $GOPATH/pkg $GOPATH/bin $BASE_PATH $REPO_PATH COPY . $REPO_PATH -# Install minimum necessary dependencies, build iris-sync-server +# Install minimum necessary dependencies, build irishub-sync RUN apk add --no-cache $PACKAGES && \ - cd $REPO_PATH && make all && \ - mv $REPO_PATH/sync-iris $GOPATH/bin && \ + mv $REPO_PATH/conf/db/types.go.example $REPO_PATH/conf/db/types.go && \ + mv $REPO_PATH/conf/server/types.go.example $REPO_PATH/conf/server/types.go && \ + cd $REPO_PATH && \ + make all && \ + mv $REPO_PATH/sync-irishub $GOPATH/bin && \ rm -rf $REPO_PATH/vendor && \ + rm -rf $GOPATH/src/github.com/golang $GOPATH/bin/glide $GOPATH/pkg/* && \ apk del $PACKAGES -CMD sync-iris > $LOG_DIR/debug.log && tail -f $LOG_DIR/debug.log \ No newline at end of file +# Set volumes +VOLUME ["$LOG_DIR"] + +CMD sync-irishub > $LOG_DIR/debug.log && tail -f $LOG_DIR/debug.log \ No newline at end of file diff --git a/Makefile b/Makefile index 74e0a7e..3cd07ca 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GOBUILD=$(GOCMD) build GOCLEAN=$(GOCMD) clean GOTEST=$(GOCMD) test GOGET=$(GOCMD) get -BINARY_NAME=sync-iris +BINARY_NAME=sync-irishub BINARY_UNIX=$(BINARY_NAME)-unix all: get_tools get_deps build diff --git a/README.md b/README.md index bb2d7c4..eb92d23 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# IRIS-SYNC-SERVER +# IRISHUB-SYNC A server that synchronize IRIS blockChain data into a database # Structure diff --git a/conf/db/types.go.example b/conf/db/types.go.example index 456b792..cc6524a 100644 --- a/conf/db/types.go.example +++ b/conf/db/types.go.example @@ -2,8 +2,8 @@ package db import ( "os" - "github.com/irisnet/iris-sync-server/util/constant" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/module/logger" ) var ( diff --git a/conf/server/types.go.example b/conf/server/types.go.example index 95529a2..a4c89ee 100644 --- a/conf/server/types.go.example +++ b/conf/server/types.go.example @@ -4,8 +4,8 @@ import ( "os" "strconv" - "github.com/irisnet/iris-sync-server/util/constant" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/module/logger" ) var ( diff --git a/glide.lock b/glide.lock index d0a8da6..1c1156e 100644 --- a/glide.lock +++ b/glide.lock @@ -46,8 +46,8 @@ imports: - stack - state - version -- name: github.com/cosmos/gaia - version: 2ac7ed920446ef8edf582c2b3e890cb7676a7d77 +- name: github.com/MrXJC/gaia + version: develop subpackages: - modules/stake - modules/stake/commands diff --git a/glide.yaml b/glide.yaml index 4a1d605..6de341d 100644 --- a/glide.yaml +++ b/glide.yaml @@ -1,4 +1,4 @@ -package: github.com/irisnet/iris-sync-server +package: github.com/irisnet/irishub-sync import: - package: github.com/gorilla/websocket - package: github.com/pkg/errors diff --git a/main.go b/main.go index c972a94..c9d2821 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,7 @@ package main import ( "sync" - syncTask "github.com/irisnet/iris-sync-server/sync" + syncTask "github.com/irisnet/irishub-sync/sync" ) func main() { diff --git a/model/store/document/account.go b/model/store/document/account.go index 5cd8884..da474f9 100644 --- a/model/store/document/account.go +++ b/model/store/document/account.go @@ -2,7 +2,7 @@ package document import ( "github.com/cosmos/cosmos-sdk/modules/coin" - "github.com/irisnet/iris-sync-server/model/store" + "github.com/irisnet/irishub-sync/model/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" diff --git a/model/store/document/doc.go b/model/store/document/doc.go index 27705a9..5d11c97 100644 --- a/model/store/document/doc.go +++ b/model/store/document/doc.go @@ -1,7 +1,7 @@ package document import ( - "github.com/irisnet/iris-sync-server/model/store" + "github.com/irisnet/irishub-sync/model/store" ) func init() { diff --git a/model/store/document/stake_role_candidate.go b/model/store/document/stake_role_candidate.go index 0a103d8..d29e2e2 100644 --- a/model/store/document/stake_role_candidate.go +++ b/model/store/document/stake_role_candidate.go @@ -2,11 +2,11 @@ package document import ( "errors" - "github.com/irisnet/iris-sync-server/model/store" + "github.com/irisnet/irishub-sync/model/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/module/logger" ) const ( diff --git a/model/store/document/stake_role_delegator.go b/model/store/document/stake_role_delegator.go index 4e95cfa..b867620 100644 --- a/model/store/document/stake_role_delegator.go +++ b/model/store/document/stake_role_delegator.go @@ -2,7 +2,7 @@ package document import ( "errors" - "github.com/irisnet/iris-sync-server/model/store" + "github.com/irisnet/irishub-sync/model/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" diff --git a/model/store/document/sync_task.go b/model/store/document/sync_task.go index 61c8a7f..16c40cc 100644 --- a/model/store/document/sync_task.go +++ b/model/store/document/sync_task.go @@ -1,7 +1,7 @@ package document import ( - "github.com/irisnet/iris-sync-server/model/store" + "github.com/irisnet/irishub-sync/model/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" diff --git a/model/store/store.go b/model/store/store.go index cc0322e..fec6019 100644 --- a/model/store/store.go +++ b/model/store/store.go @@ -5,8 +5,8 @@ package store import ( "time" - conf "github.com/irisnet/iris-sync-server/conf/db" - "github.com/irisnet/iris-sync-server/module/logger" + conf "github.com/irisnet/irishub-sync/conf/db" + "github.com/irisnet/irishub-sync/module/logger" "errors" "fmt" diff --git a/module/logger/logger.go b/module/logger/logger.go index 9ae2b6a..f75f11a 100644 --- a/module/logger/logger.go +++ b/module/logger/logger.go @@ -4,6 +4,7 @@ import ( "io" "log" "os" + "io/ioutil" ) var ( @@ -14,24 +15,24 @@ var ( ) const ( - traceDbFile = ".sync_server_db.log" - debugFile = ".sync_server_debug.log" + // traceDbFile = ".sync_server_db.log" + // debugFile = ".sync_server_debug.log" errFile = ".sync_server_err.log" warningFile = ".sync_server_warning.log" ) func init() { - traceDbFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+traceDbFile), - os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - log.Fatalln("Failed to open trace db log file:", err) - } - - debugFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+debugFile), - os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - log.Fatalln("Failed to open debug log file:", err) - } + // traceDbFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+traceDbFile), + // os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + // if err != nil { + // log.Fatalln("Failed to open trace db log file:", err) + // } + // + // debugFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+debugFile), + // os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + // if err != nil { + // log.Fatalln("Failed to open debug log file:", err) + // } warningFile, err := os.OpenFile(os.ExpandEnv("$HOME/"+warningFile), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) @@ -45,11 +46,11 @@ func init() { log.Fatalln("Failed to open error log file:", err) } - Trace = log.New(io.MultiWriter(traceDbFile, os.Stdout), + Trace = log.New(ioutil.Discard, "TRACE: ", log.Ldate|log.Ltime|log.Lshortfile) - Info = log.New(io.MultiWriter(debugFile, os.Stdout), + Info = log.New(os.Stdout, "INFO: ", log.Ldate|log.Ltime|log.Lshortfile) diff --git a/sync/handler.go b/sync/handler.go index 38d6cba..9d663c9 100644 --- a/sync/handler.go +++ b/sync/handler.go @@ -5,12 +5,12 @@ import ( "sync" "time" - "github.com/irisnet/iris-sync-server/model/store" - "github.com/irisnet/iris-sync-server/model/store/document" - "github.com/irisnet/iris-sync-server/module/logger" - "github.com/irisnet/iris-sync-server/module/stake" - "github.com/irisnet/iris-sync-server/util/constant" - "github.com/irisnet/iris-sync-server/util/helper" + "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/model/store/document" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/module/stake" + "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/util/helper" "github.com/cosmos/cosmos-sdk/modules/coin" ) diff --git a/sync/handler_test.go b/sync/handler_test.go index a73c3dc..9959c58 100644 --- a/sync/handler_test.go +++ b/sync/handler_test.go @@ -2,12 +2,12 @@ package sync import ( "testing" - "github.com/irisnet/iris-sync-server/model/store" - "github.com/irisnet/iris-sync-server/util/helper" - "github.com/irisnet/iris-sync-server/module/logger" - "github.com/irisnet/iris-sync-server/util/constant" - "github.com/irisnet/iris-sync-server/model/store/document" - "github.com/irisnet/iris-sync-server/module/stake" + "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/util/helper" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/model/store/document" + "github.com/irisnet/irishub-sync/module/stake" "sync" ) diff --git a/sync/sync.go b/sync/sync.go index 6df2d63..8899820 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -4,14 +4,14 @@ import ( "encoding/hex" "strings" - conf "github.com/irisnet/iris-sync-server/conf/server" - "github.com/irisnet/iris-sync-server/model/store" - "github.com/irisnet/iris-sync-server/module/logger" - "github.com/irisnet/iris-sync-server/module/stake" - "github.com/irisnet/iris-sync-server/util/constant" - "github.com/irisnet/iris-sync-server/util/helper" + conf "github.com/irisnet/irishub-sync/conf/server" + "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/module/stake" + "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/util/helper" - "github.com/irisnet/iris-sync-server/model/store/document" + "github.com/irisnet/irishub-sync/model/store/document" "github.com/robfig/cron" rpcClient "github.com/tendermint/tendermint/rpc/client" ctypes "github.com/tendermint/tendermint/rpc/core/types" diff --git a/sync/sync_test.go b/sync/sync_test.go index d507a76..49a713a 100644 --- a/sync/sync_test.go +++ b/sync/sync_test.go @@ -4,10 +4,10 @@ import ( "testing" "github.com/robfig/cron" - conf "github.com/irisnet/iris-sync-server/conf/server" + conf "github.com/irisnet/irishub-sync/conf/server" "sync" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/module/logger" "time" ) diff --git a/util/helper/account.go b/util/helper/account.go index ba8b4a7..b4e9a1b 100644 --- a/util/helper/account.go +++ b/util/helper/account.go @@ -5,7 +5,7 @@ package helper import ( "time" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/module/logger" "github.com/tendermint/go-wire" "github.com/cosmos/cosmos-sdk/modules/coin" diff --git a/util/helper/account_test.go b/util/helper/account_test.go index 2076d9d..9026a54 100644 --- a/util/helper/account_test.go +++ b/util/helper/account_test.go @@ -3,7 +3,7 @@ package helper import ( "testing" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/module/logger" ) func setUp() { diff --git a/util/helper/pool_client.go b/util/helper/pool_client.go index 15f2644..ef4cf67 100644 --- a/util/helper/pool_client.go +++ b/util/helper/pool_client.go @@ -5,10 +5,10 @@ package helper import ( "errors" - conf "github.com/irisnet/iris-sync-server/conf/server" + conf "github.com/irisnet/irishub-sync/conf/server" rpcClient "github.com/cosmos/cosmos-sdk/client" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/module/logger" "github.com/tendermint/tendermint/rpc/client" ) diff --git a/util/helper/pool_client_test.go b/util/helper/pool_client_test.go index 495fa88..e2f8d9d 100644 --- a/util/helper/pool_client_test.go +++ b/util/helper/pool_client_test.go @@ -8,8 +8,8 @@ import ( "github.com/tendermint/tendermint/rpc/client" - conf "github.com/irisnet/iris-sync-server/conf/server" - "github.com/irisnet/iris-sync-server/module/logger" + conf "github.com/irisnet/irishub-sync/conf/server" + "github.com/irisnet/irishub-sync/module/logger" ) func TestInitClientPool(t *testing.T) { diff --git a/util/helper/tx.go b/util/helper/tx.go index a58f854..d96b3ea 100644 --- a/util/helper/tx.go +++ b/util/helper/tx.go @@ -7,8 +7,8 @@ import ( "fmt" "strings" - "github.com/irisnet/iris-sync-server/module/logger" - "github.com/irisnet/iris-sync-server/util/constant" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/util/constant" sdk "github.com/cosmos/cosmos-sdk" "github.com/tendermint/go-wire/data" @@ -20,8 +20,8 @@ import ( _ "github.com/cosmos/cosmos-sdk/modules/base" "github.com/cosmos/cosmos-sdk/modules/coin" "github.com/cosmos/cosmos-sdk/modules/nonce" - "github.com/irisnet/iris-sync-server/module/stake" - "github.com/irisnet/iris-sync-server/model/store/document" + "github.com/irisnet/irishub-sync/module/stake" + "github.com/irisnet/irishub-sync/model/store/document" ) // parse tx struct from binary data diff --git a/util/helper/tx_test.go b/util/helper/tx_test.go index 913596c..0cf6ad9 100644 --- a/util/helper/tx_test.go +++ b/util/helper/tx_test.go @@ -3,7 +3,7 @@ package helper import ( "testing" - "github.com/irisnet/iris-sync-server/module/logger" + "github.com/irisnet/irishub-sync/module/logger" _ "github.com/cosmos/cosmos-sdk/modules/auth" _ "github.com/cosmos/cosmos-sdk/modules/base" From e97e285aa940e3172eb7cbec86ea72acbd0562be Mon Sep 17 00:00:00 2001 From: "kaifei@bianjie.ai" Date: Wed, 13 Jun 2018 16:35:23 +0800 Subject: [PATCH 24/37] modify go-crypto repository --- glide.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glide.lock b/glide.lock index 9bf39f7..1c1156e 100644 --- a/glide.lock +++ b/glide.lock @@ -171,8 +171,8 @@ imports: subpackages: - edwards25519 - extra25519 -- name: github.com/kaifei-bianjie/go-crypto - version: 24495c106407be16701180f6948e9007037d677c +- name: github.com/tendermint/go-crypto + version: 2017856384589234024cda848a6332157d1f527c subpackages: - keys - keys/cryptostore From 3d7e480c0d21f897a5aa236e49432a9105484d26 Mon Sep 17 00:00:00 2001 From: kaifei Date: Thu, 5 Jul 2018 19:04:48 +0800 Subject: [PATCH 25/37] upgrade irishub-sync to adapt new version of cosmos-sdk and tendermint(#IRISSYNCS-5, #IRISSYNCS-6) --- .gitignore | 1 - Dockerfile | 38 -- Gopkg.lock | 504 ++++++++++++++++++ Gopkg.toml | 58 ++ Makefile | 43 -- conf/db/{types.go.example => types.go} | 4 +- conf/server/{types.go.example => types.go} | 2 +- glide.lock | 309 ----------- glide.yaml | 53 -- model/store/document/account.go | 14 +- model/store/document/block.go | 93 +++- model/store/document/doc.go | 2 +- model/store/document/stake_role_candidate.go | 29 +- model/store/document/stake_role_delegator.go | 37 +- model/store/document/sync_task.go | 11 - model/store/document/tx_coin.go | 66 --- model/store/document/tx_common.go | 11 +- model/store/document/tx_stake.go | 98 ++-- .../document/tx_stake_declare_candidacy.go | 42 -- model/store/store.go | 34 -- model/store/types.go | 18 +- module/codec/codec.go | 30 ++ module/stake/errors.go | 53 -- module/stake/state.go | 22 - module/stake/tx.go | 168 ------ mongodb/mongodb.js | 27 +- sync/handler.go | 427 --------------- sync/handler/account.go | 174 ++++++ sync/handler/account_test.go | 125 +++++ sync/handler/block.go | 96 ++++ sync/handler/block_test.go | 58 ++ sync/handler/tx.go | 287 ++++++++++ sync/handler/tx_test.go | 98 ++++ sync/handler/types.go | 27 + sync/handler_test.go | 236 -------- sync/sync.go | 67 +-- tools/Makefile | 33 -- util/constant/types.go | 9 +- util/helper/account.go | 114 ++-- util/helper/account_test.go | 38 +- util/helper/common.go | 22 + util/helper/pool_client.go | 3 +- util/helper/tx.go | 247 ++++++--- util/helper/tx_test.go | 73 +-- version.info | 0 45 files changed, 1933 insertions(+), 1968 deletions(-) delete mode 100644 Dockerfile create mode 100644 Gopkg.lock create mode 100644 Gopkg.toml delete mode 100644 Makefile rename conf/db/{types.go.example => types.go} (95%) rename conf/server/{types.go.example => types.go} (97%) delete mode 100644 glide.lock delete mode 100644 glide.yaml delete mode 100644 model/store/document/tx_coin.go delete mode 100644 model/store/document/tx_stake_declare_candidacy.go create mode 100644 module/codec/codec.go delete mode 100644 module/stake/errors.go delete mode 100644 module/stake/state.go delete mode 100644 module/stake/tx.go delete mode 100644 sync/handler.go create mode 100644 sync/handler/account.go create mode 100644 sync/handler/account_test.go create mode 100644 sync/handler/block.go create mode 100644 sync/handler/block_test.go create mode 100644 sync/handler/tx.go create mode 100644 sync/handler/tx_test.go create mode 100644 sync/handler/types.go delete mode 100644 sync/handler_test.go delete mode 100644 tools/Makefile create mode 100644 util/helper/common.go create mode 100644 version.info diff --git a/.gitignore b/.gitignore index c53a361..102cf0b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ .idea/ vendor/ -conf/**/types.go .DS_Store sync-iris sync-iris-unix diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 843c424..0000000 --- a/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# docker usage -# docker build -t sync-irishub:dev . -FROM alpine:edge - -# Set up dependencies -ENV PACKAGES go make git libc-dev bash - -# Set up GOPATH & PATH - -ENV GOPATH /root/go -ENV BASE_PATH $GOPATH/src/github.com/irisnet -ENV REPO_PATH $BASE_PATH/irishub-sync -ENV LOG_DIR /sync-iris/log -ENV PATH $GOPATH/bin:$PATH - -# Link expected Go repo path - -RUN mkdir -p $LOG_DIR $GOPATH/pkg $GOPATH/bin $BASE_PATH $REPO_PATH - -# Add source files - -COPY . $REPO_PATH - -# Install minimum necessary dependencies, build irishub-sync -RUN apk add --no-cache $PACKAGES && \ - mv $REPO_PATH/conf/db/types.go.example $REPO_PATH/conf/db/types.go && \ - mv $REPO_PATH/conf/server/types.go.example $REPO_PATH/conf/server/types.go && \ - cd $REPO_PATH && \ - make all && \ - mv $REPO_PATH/sync-irishub $GOPATH/bin && \ - rm -rf $REPO_PATH/vendor && \ - rm -rf $GOPATH/src/github.com/golang $GOPATH/bin/glide $GOPATH/pkg/* && \ - apk del $PACKAGES - -# Set volumes -VOLUME ["$LOG_DIR"] - -CMD sync-irishub > $LOG_DIR/debug.log && tail -f $LOG_DIR/debug.log \ No newline at end of file diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..666b0e2 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,504 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/bgentry/speakeasy" + packages = ["."] + revision = "4aabc24848ce5fd31929f7d1e4ea74d3709c14cd" + version = "v0.1.0" + +[[projects]] + branch = "master" + name = "github.com/btcsuite/btcd" + packages = ["btcec"] + revision = "86fed781132ac890ee03e906e4ecd5d6fa180c64" + +[[projects]] + branch = "master" + name = "github.com/btcsuite/btcutil" + packages = ["bech32"] + revision = "d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4" + +[[projects]] + name = "github.com/cosmos/cosmos-sdk" + packages = [ + "client", + "client/context", + "client/keys", + "store", + "types", + "wire", + "x/auth", + "x/auth/client/cli", + "x/bank", + "x/ibc", + "x/slashing", + "x/stake" + ] + revision = "c6711810a86f09457481a8dadae899681a9d77ab" + version = "v0.19.0" + +[[projects]] + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + branch = "master" + name = "github.com/ebuchman/fail-test" + packages = ["."] + revision = "95f809107225be108efcf10a3509e4ea6ceef3c4" + +[[projects]] + name = "github.com/fsnotify/fsnotify" + packages = ["."] + revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" + version = "v1.4.7" + +[[projects]] + name = "github.com/go-kit/kit" + packages = [ + "log", + "log/level", + "log/term" + ] + revision = "4dc7be5d2d12881735283bcab7352178e190fc71" + version = "v0.6.0" + +[[projects]] + name = "github.com/go-logfmt/logfmt" + packages = ["."] + revision = "390ab7935ee28ec6b286364bba9b4dd6410cb3d5" + version = "v0.3.0" + +[[projects]] + name = "github.com/go-stack/stack" + packages = ["."] + revision = "259ab82a6cad3992b4e21ff5cac294ccb06474bc" + version = "v1.7.0" + +[[projects]] + name = "github.com/gogo/protobuf" + packages = [ + "gogoproto", + "jsonpb", + "proto", + "protoc-gen-gogo/descriptor", + "sortkeys", + "types" + ] + revision = "1adfc126b41513cc696b209667c8656ea7aac67c" + version = "v1.0.0" + +[[projects]] + name = "github.com/golang/protobuf" + packages = [ + "proto", + "ptypes", + "ptypes/any", + "ptypes/duration", + "ptypes/timestamp" + ] + revision = "925541529c1fa6821df4e44ce2723319eb2be768" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/golang/snappy" + packages = ["."] + revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a" + +[[projects]] + name = "github.com/gorilla/context" + packages = ["."] + revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42" + version = "v1.1.1" + +[[projects]] + name = "github.com/gorilla/mux" + packages = ["."] + revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf" + version = "v1.6.2" + +[[projects]] + name = "github.com/gorilla/websocket" + packages = ["."] + revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b" + version = "v1.2.0" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/hcl" + packages = [ + ".", + "hcl/ast", + "hcl/parser", + "hcl/printer", + "hcl/scanner", + "hcl/strconv", + "hcl/token", + "json/parser", + "json/scanner", + "json/token" + ] + revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" + +[[projects]] + branch = "master" + name = "github.com/howeyc/crc16" + packages = ["."] + revision = "2b2a61e366a66d3efb279e46176e7291001e0354" + +[[projects]] + name = "github.com/inconshreveable/mousetrap" + packages = ["."] + revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" + version = "v1.0" + +[[projects]] + branch = "master" + name = "github.com/jmhodges/levigo" + packages = ["."] + revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9" + +[[projects]] + branch = "master" + name = "github.com/kr/logfmt" + packages = ["."] + revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0" + +[[projects]] + name = "github.com/magiconair/properties" + packages = ["."] + revision = "c2353362d570a7bfa228149c62842019201cfb71" + version = "v1.8.0" + +[[projects]] + name = "github.com/mattn/go-isatty" + packages = ["."] + revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" + version = "v0.0.3" + +[[projects]] + branch = "master" + name = "github.com/mitchellh/mapstructure" + packages = ["."] + revision = "bb74f1db0675b241733089d5a1faa5dd8b0ef57b" + +[[projects]] + name = "github.com/pelletier/go-toml" + packages = ["."] + revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194" + version = "v1.2.0" + +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + +[[projects]] + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/rcrowley/go-metrics" + packages = ["."] + revision = "e2704e165165ec55d062f5919b4b29494e9fa790" + +[[projects]] + name = "github.com/robfig/cron" + packages = ["."] + revision = "b41be1df696709bb6395fe435af20370037c0b4c" + version = "v1.1" + +[[projects]] + name = "github.com/spf13/afero" + packages = [ + ".", + "mem" + ] + revision = "787d034dfe70e44075ccc060d346146ef53270ad" + version = "v1.1.1" + +[[projects]] + name = "github.com/spf13/cast" + packages = ["."] + revision = "8965335b8c7107321228e3e3702cab9832751bac" + version = "v1.2.0" + +[[projects]] + name = "github.com/spf13/cobra" + packages = ["."] + revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" + version = "v0.0.3" + +[[projects]] + branch = "master" + name = "github.com/spf13/jwalterweatherman" + packages = ["."] + revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" + +[[projects]] + name = "github.com/spf13/pflag" + packages = ["."] + revision = "583c0c0531f06d5278b7d917446061adc344b5cd" + version = "v1.0.1" + +[[projects]] + name = "github.com/spf13/viper" + packages = ["."] + revision = "b5e8006cbee93ec955a89ab31e0e3ce3204f3736" + version = "v1.0.2" + +[[projects]] + name = "github.com/stretchr/testify" + packages = [ + "assert", + "require" + ] + revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" + version = "v1.2.2" + +[[projects]] + branch = "master" + name = "github.com/syndtr/goleveldb" + packages = [ + "leveldb", + "leveldb/cache", + "leveldb/comparer", + "leveldb/errors", + "leveldb/filter", + "leveldb/iterator", + "leveldb/journal", + "leveldb/memdb", + "leveldb/opt", + "leveldb/storage", + "leveldb/table", + "leveldb/util" + ] + revision = "0d5a0ceb10cf9ab89fdd744cc8c50a83134f6697" + +[[projects]] + name = "github.com/tendermint/abci" + packages = [ + "client", + "example/code", + "example/kvstore", + "types" + ] + revision = "ebee2fe114020aa49c70bbbae50b7079fc7e7b90" + version = "v0.11.0" + +[[projects]] + branch = "master" + name = "github.com/tendermint/ed25519" + packages = [ + ".", + "edwards25519", + "extra25519" + ] + revision = "d8387025d2b9d158cf4efb07e7ebf814bcce2057" + +[[projects]] + name = "github.com/tendermint/go-amino" + packages = ["."] + revision = "ed62928576cfcaf887209dc96142cd79cdfff389" + version = "0.9.9" + +[[projects]] + name = "github.com/tendermint/go-crypto" + packages = [ + ".", + "keys", + "keys/bcrypt", + "keys/words", + "keys/words/wordlist" + ] + revision = "915416979bf70efa4bcbf1c6cd5d64c5fff9fc19" + version = "v0.6.2" + +[[projects]] + name = "github.com/tendermint/iavl" + packages = [ + ".", + "sha256truncated" + ] + revision = "c9206995e8f948e99927f5084a88a7e94ca256da" + version = "v0.8.0-rc0" + +[[projects]] + name = "github.com/tendermint/tendermint" + packages = [ + "blockchain", + "cmd/tendermint/commands", + "config", + "consensus", + "consensus/types", + "evidence", + "libs/events", + "libs/pubsub", + "libs/pubsub/query", + "lite", + "lite/client", + "lite/errors", + "lite/files", + "lite/proxy", + "mempool", + "node", + "p2p", + "p2p/conn", + "p2p/pex", + "p2p/upnp", + "privval", + "proxy", + "rpc/client", + "rpc/core", + "rpc/core/types", + "rpc/grpc", + "rpc/lib", + "rpc/lib/client", + "rpc/lib/server", + "rpc/lib/types", + "state", + "state/txindex", + "state/txindex/kv", + "state/txindex/null", + "types", + "version" + ] + revision = "27bd1deabe4ba6a2d9b463b8f3e3f1e31b993e61" + version = "v0.20.0" + +[[projects]] + name = "github.com/tendermint/tmlibs" + packages = [ + "autofile", + "bech32", + "cli", + "cli/flags", + "clist", + "common", + "db", + "flowrate", + "log", + "merkle", + "merkle/tmhash" + ] + revision = "0c98d10b4ffbd87978d79c160e835b3d3df241ec" + +[[projects]] + branch = "master" + name = "golang.org/x/crypto" + packages = [ + "blowfish", + "curve25519", + "internal/subtle", + "nacl/box", + "nacl/secretbox", + "openpgp/armor", + "openpgp/errors", + "poly1305", + "ripemd160", + "salsa20/salsa" + ] + revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = [ + "context", + "http/httpguts", + "http2", + "http2/hpack", + "idna", + "internal/timeseries", + "trace" + ] + revision = "ed29d75add3d7c4bf7ca65aac0c6df3d1420216f" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = ["unix"] + revision = "7138fd3d9dc8335c567ca206f4333fb75eb05d56" + +[[projects]] + name = "golang.org/x/text" + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable" + ] + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + +[[projects]] + name = "google.golang.org/genproto" + packages = ["googleapis/rpc/status"] + revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200" + +[[projects]] + name = "google.golang.org/grpc" + packages = [ + ".", + "balancer", + "codes", + "connectivity", + "credentials", + "grpclb/grpc_lb_v1/messages", + "grpclog", + "internal", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "stats", + "status", + "tap", + "transport" + ] + revision = "5b3c4e850e90a4cf6a20ebd46c8b32a0a3afcb9e" + version = "v1.7.5" + +[[projects]] + branch = "v2" + name = "gopkg.in/mgo.v2" + packages = [ + ".", + "bson", + "internal/json", + "internal/sasl", + "internal/scram" + ] + revision = "3f83fa5005286a7fe593b055f0d7771a7dce4655" + +[[projects]] + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "52551a8a6af4cebfff972e9f1fbcf2e71200436239d0f89004fa1ae9896f85a1" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..24e9e22 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,58 @@ +# Gopkg.toml example +# +# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[[constraint]] + name = "github.com/robfig/cron" + version = "1.1.0" + +[[constraint]] + name = "github.com/cosmos/cosmos-sdk" + version = "v0.19.0" + +[[constraint]] + name = "github.com/tendermint/tendermint" + version = "=0.20.0" + +[[override]] + name = "github.com/tendermint/tmlibs" + revision = "0c98d10b4ffbd87978d79c160e835b3d3df241ec" + +[[override]] + name = "google.golang.org/genproto" + revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200" + +[[constraint]] + branch = "v2" + name = "gopkg.in/mgo.v2" + +[[constraint]] + branch = "v2" + name = "github.com/stretchr/testify" + +[prune] + go-tests = true + unused-packages = true diff --git a/Makefile b/Makefile deleted file mode 100644 index 3cd07ca..0000000 --- a/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -GOCMD=go -GOBUILD=$(GOCMD) build -GOCLEAN=$(GOCMD) clean -GOTEST=$(GOCMD) test -GOGET=$(GOCMD) get -BINARY_NAME=sync-irishub -BINARY_UNIX=$(BINARY_NAME)-unix - -all: get_tools get_deps build - -get_deps: - @echo "--> Running glide install" - @glide install -v - -build: - $(GOBUILD) -o $(BINARY_NAME) -v - -clean: - $(GOCLEAN) - rm -f $(BINARY_NAME) - rm -f $(BINARY_UNIX) - -run: - $(GOBUILD) -o $(BINARY_NAME) -v ./ - ./$(BINARY_NAME) - - -# Cross compilation -build-linux: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) -v - - -###################################### -## Tools - -check_tools: - cd tools && $(MAKE) check_tools - -get_tools: - cd tools && $(MAKE) get_tools - -update_tools: - cd tools && $(MAKE) update_tools \ No newline at end of file diff --git a/conf/db/types.go.example b/conf/db/types.go similarity index 95% rename from conf/db/types.go.example rename to conf/db/types.go index cc6524a..4fa96e9 100644 --- a/conf/db/types.go.example +++ b/conf/db/types.go @@ -8,10 +8,10 @@ import ( var ( Host = "127.0.0.1" - Port = "27017" + Port = "27117" User = "" Passwd = "" - Database = "test" + Database = "sync-iris-dev" ) // get value of env var diff --git a/conf/server/types.go.example b/conf/server/types.go similarity index 97% rename from conf/server/types.go.example rename to conf/server/types.go index a4c89ee..538589d 100644 --- a/conf/server/types.go.example +++ b/conf/server/types.go @@ -9,7 +9,7 @@ import ( ) var ( - BlockChainMonitorUrl = "tcp://127.0.0.1:46757" + BlockChainMonitorUrl = "tcp://127.0.0.1:46657" ChainId = "test" Token = "iris" diff --git a/glide.lock b/glide.lock deleted file mode 100644 index 1c1156e..0000000 --- a/glide.lock +++ /dev/null @@ -1,309 +0,0 @@ -hash: 195b6bc507f3be7f05cbfeab8a493ee8188892f8fc266853ea43d2090b718298 -updated: 2018-03-20T09:31:27.390489+08:00 -imports: -- name: github.com/bgentry/speakeasy - version: 4aabc24848ce5fd31929f7d1e4ea74d3709c14cd -- name: github.com/btcsuite/btcd - version: 2e60448ffcc6bf78332d1fe590260095f554dd78 - subpackages: - - btcec -- name: github.com/BurntSushi/toml - version: b26d9c308763d68093482582cea63d69be07a0f0 -- name: github.com/cosmos/cosmos-sdk - version: f1b1d36324812dc221da87f56734a5c25d8754aa - subpackages: - - app - - client - - client/commands - - client/commands/commits - - client/commands/keys - - client/commands/proxy - - client/commands/query - - client/commands/rpc - - client/commands/search - - client/commands/txs - - client/rest - - errors - - genesis - - modules/auth - - modules/auth/commands - - modules/base - - modules/base/commands - - modules/coin - - modules/coin/commands - - modules/coin/rest - - modules/fee - - modules/fee/commands - - modules/ibc - - modules/ibc/commands - - modules/nonce - - modules/nonce/commands - - modules/nonce/rest - - modules/roles - - modules/roles/commands - - modules/roles/rest - - server/commands - - stack - - state - - version -- name: github.com/MrXJC/gaia - version: develop - subpackages: - - modules/stake - - modules/stake/commands - - modules/stake/rest - - version -- name: github.com/ebuchman/fail-test - version: 95f809107225be108efcf10a3509e4ea6ceef3c4 -- name: github.com/fsnotify/fsnotify - version: 4da3e2cfbabc9f751898f250b49f2439785783a1 -- name: github.com/go-kit/kit - version: 4dc7be5d2d12881735283bcab7352178e190fc71 - subpackages: - - log - - log/level - - log/term -- name: github.com/go-logfmt/logfmt - version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5 -- name: github.com/go-playground/locales - version: e4cbcb5d0652150d40ad0646651076b6bd2be4f6 - subpackages: - - currency -- name: github.com/go-playground/universal-translator - version: 71201497bace774495daed26a3874fd339e0b538 -- name: github.com/go-stack/stack - version: 259ab82a6cad3992b4e21ff5cac294ccb06474bc -- name: github.com/gogo/protobuf - version: 1adfc126b41513cc696b209667c8656ea7aac67c - subpackages: - - gogoproto - - jsonpb - - proto - - protoc-gen-gogo/descriptor - - sortkeys - - types -- name: github.com/golang/protobuf - version: 925541529c1fa6821df4e44ce2723319eb2be768 - subpackages: - - proto - - ptypes - - ptypes/any - - ptypes/duration - - ptypes/timestamp -- name: github.com/golang/snappy - version: 553a641470496b2327abcac10b36396bd98e45c9 -- name: github.com/gorilla/context - version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42 -- name: github.com/gorilla/mux - version: 53c1911da2b537f792e7cafcb446b05ffe33b996 -- name: github.com/gorilla/websocket - version: ea4d1f681babbce9545c9c5f3d5194a789c89f5b -- name: github.com/hashicorp/hcl - version: 23c074d0eceb2b8a5bfdbb271ab780cde70f05a8 - subpackages: - - hcl/ast - - hcl/parser - - hcl/scanner - - hcl/strconv - - hcl/token - - json/parser - - json/scanner - - json/token -- name: github.com/howeyc/crc16 - version: 2b2a61e366a66d3efb279e46176e7291001e0354 -- name: github.com/inconshreveable/mousetrap - version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 -- name: github.com/jmhodges/levigo - version: c42d9e0ca023e2198120196f842701bb4c55d7b9 -- name: github.com/kr/logfmt - version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0 -- name: github.com/magiconair/properties - version: 49d762b9817ba1c2e9d0c69183c2b4a8b8f1d934 -- name: github.com/mattn/go-isatty - version: 0360b2af4f38e8d38c7fce2a9f4e702702d73a39 -- name: github.com/mitchellh/mapstructure - version: 06020f85339e21b2478f756a78e295255ffa4d6a -- name: github.com/pelletier/go-toml - version: 4e9e0ee19b60b13eb79915933f44d8ed5f268bdd -- name: github.com/pkg/errors - version: 645ef00459ed84a119197bfb8d8205042c6df63d -- name: github.com/rcrowley/go-metrics - version: e181e095bae94582363434144c61a9653aff6e50 -- name: github.com/spf13/afero - version: 8d919cbe7e2627e417f3e45c3c0e489a5b7e2536 - subpackages: - - mem -- name: github.com/spf13/cast - version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4 -- name: github.com/spf13/cobra - version: 7b2c5ac9fc04fc5efafb60700713d4fa609b777b -- name: github.com/spf13/jwalterweatherman - version: 12bd96e66386c1960ab0f74ced1362f66f552f7b -- name: github.com/spf13/pflag - version: e57e3eeb33f795204c1ca35f56c44f83227c6e66 -- name: github.com/spf13/viper - version: 25b30aa063fc18e48662b86996252eabdcf2f0c7 -- name: github.com/syndtr/goleveldb - version: adf24ef3f94bd13ec4163060b21a5678f22b429b - subpackages: - - leveldb - - leveldb/cache - - leveldb/comparer - - leveldb/errors - - leveldb/filter - - leveldb/iterator - - leveldb/journal - - leveldb/memdb - - leveldb/opt - - leveldb/storage - - leveldb/table - - leveldb/util -- name: github.com/tendermint/abci - version: 3d5f0a8b94bbbfa70ecd4072fc55854f8210425b - subpackages: - - client - - example/code - - example/kvstore - - server - - types -- name: github.com/tendermint/ed25519 - version: d8387025d2b9d158cf4efb07e7ebf814bcce2057 - subpackages: - - edwards25519 - - extra25519 -- name: github.com/tendermint/go-crypto - version: 2017856384589234024cda848a6332157d1f527c - subpackages: - - keys - - keys/cryptostore - - keys/storage/filestorage - - keys/wordlist -- name: github.com/tendermint/go-wire - version: b6fc872b42d41158a60307db4da051dd6f179415 - subpackages: - - data -- name: github.com/tendermint/iavl - version: 39de8f0b4ee758fdd5bb3a9afc6dd9bf42c04785 -- name: github.com/tendermint/tendermint - version: 9c5937df969de23e9087f183c52b5088d4b6f268 - subpackages: - - blockchain - - cmd/tendermint/commands - - config - - consensus - - consensus/types - - evidence - - lite - - lite/client - - lite/errors - - lite/files - - lite/proxy - - mempool - - client - - p2p - - p2p/conn - - p2p/pex - - p2p/trust - - p2p/upnp - - proxy - - rpc/client - - rpc/core - - rpc/core/types - - rpc/grpc - - rpc/lib - - rpc/lib/client - - rpc/lib/server - - rpc/lib/types - - state - - state/txindex - - state/txindex/kv - - state/txindex/null - - types - - version -- name: github.com/tendermint/tmlibs - version: 26f2ab65f82cfc6873c312e8030104c47c05f10e - subpackages: - - autofile - - cli - - cli/flags - - clist - - common - - db - - flowrate - - log - - merkle - - pubsub - - pubsub/query -- name: golang.org/x/crypto - version: 94eea52f7b742c7cbe0b03b22f0c4c8631ece122 - subpackages: - - curve25519 - - nacl/box - - nacl/secretbox - - openpgp/armor - - openpgp/errors - - poly1305 - - ripemd160 - - salsa20/salsa -- name: golang.org/x/net - version: 5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec - subpackages: - - context - - http2 - - http2/hpack - - idna - - internal/timeseries - - lex/httplex - - trace -- name: golang.org/x/sys - version: 8b4580aae2a0dd0c231a45d3ccb8434ff533b840 - subpackages: - - unix -- name: golang.org/x/text - version: 57961680700a5336d15015c8c50686ca5ba362a4 - subpackages: - - secure/bidirule - - transform - - unicode/bidi - - unicode/norm -- name: google.golang.org/genproto - version: a8101f21cf983e773d0c1133ebc5424792003214 - repo: https://github.com/google/go-genproto - vcs: git - subpackages: - - googleapis/rpc/status -- name: google.golang.org/grpc - version: 401e0e00e4bb830a10496d64cd95e068c5bf50de - repo: https://github.com/grpc/grpc-go - vcs: git - subpackages: - - balancer - - codes - - connectivity - - credentials - - grpclb/grpc_lb_v1/messages - - grpclog - - internal - - keepalive - - metadata - - naming - - peer - - resolver - - stats - - status - - tap - - transport -- name: gopkg.in/go-playground/validator.v9 - version: 1b8c8e19cd250435025214492d9a08411d760fdd -- name: gopkg.in/yaml.v2 - version: 287cf08546ab5e7e37d55a84f7ed3fd1db036de5 -testImports: -- name: github.com/stretchr/testify - version: 2aa2c176b9dab406a6970f6a55f513e8a8c8b18f - subpackages: - - assert - - require -- name: gopkg.in/mgo.v2 - version: master -- name: github.com/robfig/cron -version: master \ No newline at end of file diff --git a/glide.yaml b/glide.yaml deleted file mode 100644 index 6de341d..0000000 --- a/glide.yaml +++ /dev/null @@ -1,53 +0,0 @@ -package: github.com/irisnet/irishub-sync -import: -- package: github.com/gorilla/websocket -- package: github.com/pkg/errors - version: ^0.8.0 -- package: github.com/spf13/cobra -- package: github.com/spf13/pflag -- package: github.com/spf13/viper -- package: github.com/tendermint/abci - version: develop - subpackages: - - server - - types -- package: github.com/cosmos/cosmos-sdk - version: develop -- package: github.com/tendermint/iavl - version: develop -- package: github.com/tendermint/go-crypto - version: remove_ledger - subpackages: - - cmd - - keys -- package: github.com/tendermint/go-wire - version: develop - subpackages: - - data -- package: github.com/tendermint/tendermint - version: v0.15.0 - subpackages: - - config - - node - - proxy - - rpc/client - - rpc/core/types - - rpc/lib/client - - rpc/lib/types - - types -- package: github.com/tendermint/tmlibs - version: develop - subpackages: - - cli - - cli/flags - - common - - events - - log - - logger -- package: github.com/gorilla/mux - version: ^1.5.0 -testImport: -- package: github.com/stretchr/testify - subpackages: - - assert - - require diff --git a/model/store/document/account.go b/model/store/document/account.go index da474f9..7a9c3e9 100644 --- a/model/store/document/account.go +++ b/model/store/document/account.go @@ -1,7 +1,6 @@ package document import ( - "github.com/cosmos/cosmos-sdk/modules/coin" "github.com/irisnet/irishub-sync/model/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" @@ -14,7 +13,7 @@ const ( type Account struct { Address string `bson:"address"` - Amount coin.Coins `bson:"amount"` + Amount store.Coins `bson:"amount"` Time time.Time `bson:"time"` Height int64 `bson:"height"` } @@ -27,17 +26,6 @@ func (a Account) PkKvPair() map[string]interface{} { return bson.M{"address": a.Address} } -func (a Account) Index() []mgo.Index { - return []mgo.Index{ - { - Key: []string{"address"}, - Unique: true, - DropDups: false, - Background: true, - }, - } -} - func QueryAccount(address string) (Account, error) { var result Account query := func(c *mgo.Collection) error { diff --git a/model/store/document/block.go b/model/store/document/block.go index 44f4520..d5ad321 100644 --- a/model/store/document/block.go +++ b/model/store/document/block.go @@ -1,7 +1,6 @@ package document import ( - "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" ) @@ -11,26 +10,88 @@ const ( ) type Block struct { - Height int64 `bson:"height"` - Time time.Time `bson:"time"` - TxNum int64 `bson:"tx_num"` + Height int64 `bson:"height"` + Hash string `bson:"hash"` + Time time.Time `bson:"time"` + NumTxs int64 `bson:"num_txs"` + Meta BlockMeta `bson:"meta"` + Block BlockContent `bson:"block"` } +type BlockMeta struct { + BlockID BlockID `bson:"block_id"` + Header Header `bson:"header"` +} + +type BlockID struct { + Hash string `bson:"hash"` + PartsHeader PartSetHeader `bson:"parts"` +} + +type PartSetHeader struct { + Total int `bson:"total"` + Hash string `bson:"hash"` +} + +type Header struct { + // basic block info + ChainID string `bson:"chain_id"` + Height int64 `bson:"height"` + Time time.Time `bson:"time"` + NumTxs int64 `bson:"num_txs"` + + // prev block info + LastBlockID BlockID `bson:"last_block_id"` + TotalTxs int64 `bson:"total_txs"` + + // hashes of block data + LastCommitHash string `bson:"last_commit_hash"` // commit from validators from the last block + DataHash string `bson:"data_hash"` // transactions + + // hashes from the app output from the prev block + ValidatorsHash string `bson:"validators_hash"` // validators for the current block + ConsensusHash string `bson:"consensus_hash"` // consensus params for current block + AppHash string `bson:"app_hash"` // state after txs from the previous block + LastResultsHash string `bson:"last_results_hash"` // root hash of all results from the txs from the previous block + + // consensus info + EvidenceHash string `bson:"evidence_hash"` // evidence included in the block +} + +type BlockContent struct { + LastCommit Commit `bson:"last_commit"` +} + +type Commit struct { + // NOTE: The Precommits are in order of address to preserve the bonded ValidatorSet order. + // Any peer with a block can gossip precommits by index with a peer without recalculating the + // active ValidatorSet. + BlockID BlockID `bson:"block_id"` + Precommits []Vote `bson:"precommits"` +} + +// Represents a prevote, precommit, or commit vote from validators for consensus. +type Vote struct { + ValidatorAddress string `bson:"validator_address"` + ValidatorIndex int `bson:"validator_index"` + Height int64 `bson:"height"` + Round int `bson:"round"` + Timestamp time.Time `bson:"timestamp"` + Type byte `bson:"type"` + BlockID BlockID `bson:"block_id"` // zero if vote is nil. + Signature Signature `bson:"signature"` +} + +type Signature struct { + Type string `bson:"type"` + Value string `bson:"value"` +} + + func (d Block) Name() string { return CollectionNmBlock } func (d Block) PkKvPair() map[string]interface{} { return bson.M{"height": d.Height} -} - -func (d Block) Index() []mgo.Index { - return []mgo.Index{ - { - Key: []string{"height"}, - Unique: true, - DropDups: true, - Background: true, - }, - } -} +} \ No newline at end of file diff --git a/model/store/document/doc.go b/model/store/document/doc.go index 5d11c97..2e49d38 100644 --- a/model/store/document/doc.go +++ b/model/store/document/doc.go @@ -6,9 +6,9 @@ import ( func init() { store.RegisterDocs(new(Account)) - store.RegisterDocs(new(CoinTx)) store.RegisterDocs(new(StakeTx)) store.RegisterDocs(new(StakeTxDeclareCandidacy)) + store.RegisterDocs(new(StakeTxEditCandidacy)) store.RegisterDocs(new(Candidate)) store.RegisterDocs(new(Delegator)) store.RegisterDocs(new(Block)) diff --git a/model/store/document/stake_role_candidate.go b/model/store/document/stake_role_candidate.go index d29e2e2..ee71a80 100644 --- a/model/store/document/stake_role_candidate.go +++ b/model/store/document/stake_role_candidate.go @@ -14,12 +14,12 @@ const ( ) type Candidate struct { - Address string `bson:"address"` // owner + Address string `bson:"address"` // owner, identity key PubKey string `bson:"pub_key"` Shares int64 `bson:"shares"` - VotingPower int64 `bson:"voting_power"` // Voting power if pubKey is a considered a validator + VotingPower int64 `bson:"voting_power"` // Voting power if pubKey is a considered a validator Description Description `bson:"description"` // Description terms for the candidate - UpdateTime time.Time `bson:"update_time"` + UpdateTime time.Time `bson:"update_time"` } func (d Candidate) Name() string { @@ -27,30 +27,13 @@ func (d Candidate) Name() string { } func (d Candidate) PkKvPair() map[string]interface{} { - return bson.M{"pub_key": d.PubKey} + return bson.M{"address": d.Address} } -func (d Candidate) Index() []mgo.Index { - return []mgo.Index { - { - Key: []string{"pub_key"}, - Unique: true, - DropDups: false, - Background: true, - }, - { - Key: []string{"address"}, - Unique: false, - DropDups: false, - Background: true, - }, - } -} - -func QueryCandidateByPubkey(pubKey string) (Candidate, error) { +func QueryCandidateByAddress(address string) (Candidate, error) { var result Candidate query := func(c *mgo.Collection) error { - err := c.Find(bson.M{"pub_key": pubKey}).One(&result) + err := c.Find(bson.M{"address": address}).One(&result) return err } diff --git a/model/store/document/stake_role_delegator.go b/model/store/document/stake_role_delegator.go index b867620..a5bd1f2 100644 --- a/model/store/document/stake_role_delegator.go +++ b/model/store/document/stake_role_delegator.go @@ -13,10 +13,10 @@ const ( ) type Delegator struct { - Address string `bson:"address"` - PubKey string `bson:"pub_key"` - Shares int64 `bson:"shares"` - UpdateTime time.Time `bson:"update_time"` + Address string `bson:"address"` + ValidatorAddr string `bson:"pub_key"` // validatorAddr + Shares int64 `bson:"shares"` + UpdateTime time.Time `bson:"update_time"` } func (d Delegator) Name() string { @@ -24,36 +24,13 @@ func (d Delegator) Name() string { } func (d Delegator) PkKvPair() map[string]interface{} { - return bson.M{"address": d.Address, "pub_key": d.PubKey} + return bson.M{"address": d.Address, "pub_key": d.ValidatorAddr} } -func (d Delegator) Index() []mgo.Index { - return []mgo.Index{ - { - Key: []string{"address"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"pub_key"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"address", "pub_key"}, - Unique: true, - DropDups: false, - Background: true, - }, - } -} - -func QueryDelegatorByAddressAndPubkey(address string, pubKey string) (Delegator, error) { +func QueryDelegatorByAddressAndValAddr(address string, valAddr string) (Delegator, error) { var result Delegator query := func(c *mgo.Collection) error { - err := c.Find(bson.M{"address": address, "pub_key": pubKey}).Sort("-shares").One(&result) + err := c.Find(bson.M{"address": address, "pub_key": valAddr}).Sort("-shares").One(&result) return err } diff --git a/model/store/document/sync_task.go b/model/store/document/sync_task.go index 16c40cc..d0b0ce9 100644 --- a/model/store/document/sync_task.go +++ b/model/store/document/sync_task.go @@ -26,17 +26,6 @@ func (c SyncTask) PkKvPair() map[string]interface{} { return bson.M{"chain_id": c.ChainID} } -func (c SyncTask) Index() []mgo.Index { - return []mgo.Index{ - { - Key: []string{"chain_id"}, - Unique: true, - DropDups: false, - Background: true, - }, - } -} - func QuerySyncTask() (SyncTask, error) { result := SyncTask{} diff --git a/model/store/document/tx_coin.go b/model/store/document/tx_coin.go deleted file mode 100644 index 8bc8597..0000000 --- a/model/store/document/tx_coin.go +++ /dev/null @@ -1,66 +0,0 @@ -package document - -import ( - "github.com/cosmos/cosmos-sdk/modules/coin" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "time" -) - -const ( - CollectionNmCoinTx = "tx_coin" -) - -// Coin tx -type CoinTx struct { - TxHash string `bson:"tx_hash"` - Time time.Time `bson:"time"` - Height int64 `bson:"height"` - From string `bson:"from"` - To string `bson:"to"` - Amount coin.Coins `bson:"amount"` -} - -func (c CoinTx) Name() string { - return CollectionNmCoinTx -} - -func (c CoinTx) PkKvPair() map[string]interface{} { - return bson.M{"tx_hash": c.TxHash} -} - -func (c CoinTx) Index() []mgo.Index { - return []mgo.Index{ - { - Key: []string{"tx_hash"}, - Unique: true, - DropDups: false, - Background: true, - }, - { - Key: []string{"from"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"to"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"-height"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"from", "to"}, - Unique: false, - DropDups: false, - Background: true, - }, - - } -} diff --git a/model/store/document/tx_common.go b/model/store/document/tx_common.go index 1448ce8..b239ad1 100644 --- a/model/store/document/tx_common.go +++ b/model/store/document/tx_common.go @@ -2,9 +2,8 @@ package document import ( "time" - "github.com/cosmos/cosmos-sdk/modules/coin" + "github.com/irisnet/irishub-sync/model/store" "gopkg.in/mgo.v2/bson" - "gopkg.in/mgo.v2" ) const ( @@ -17,8 +16,10 @@ type CommonTx struct { Height int64 `bson:"height"` From string `bson:"from"` To string `bson:"to"` - Amount coin.Coins `bson:"amount"` + Amount store.Coins `bson:"amount"` Type string `bson:"type"` + Fee store.Fee `bson:"fee"` + Status string `bson:"status"` } func (d CommonTx) Name() string { @@ -28,7 +29,3 @@ func (d CommonTx) Name() string { func (d CommonTx) PkKvPair() map[string]interface{} { return bson.M{"tx_hash": d.TxHash} } - -func (d CommonTx) Index() []mgo.Index { - return []mgo.Index{} -} diff --git a/model/store/document/tx_stake.go b/model/store/document/tx_stake.go index 39a4bde..9971d91 100644 --- a/model/store/document/tx_stake.go +++ b/model/store/document/tx_stake.go @@ -1,25 +1,29 @@ package document import ( - "github.com/cosmos/cosmos-sdk/modules/coin" - "gopkg.in/mgo.v2" + "github.com/irisnet/irishub-sync/model/store" "gopkg.in/mgo.v2/bson" "time" ) const ( CollectionNmStakeTx = "tx_stake" + CollectionNmStakeTxDeclareCandidacy = CollectionNmStakeTx + CollectionNmStakeTxEditCandidacy = CollectionNmStakeTx ) // StakeTx type StakeTx struct { - TxHash string `bson:"tx_hash"` - Time time.Time `bson:"time"` - Height int64 `bson:"height"` - From string `bson:"from"` - PubKey string `bson:"pub_key"` - Type string `bson:"type"` - Amount coin.Coin `bson:"amount"` + TxHash string `bson:"tx_hash"` + Time time.Time `bson:"time"` + Height int64 `bson:"height"` + DelegatorAddr string `bson:"from"` + ValidatorAddr string `bson:"to"` + PubKey string `bson:"pub_key"` + Type string `bson:"type"` + Amount store.Coin `bson:"amount"` + Fee store.Fee `bson:"fee"` + Status string `bson:"status"` } func (c StakeTx) Name() string { @@ -30,43 +34,41 @@ func (c StakeTx) PkKvPair() map[string]interface{} { return bson.M{"tx_hash": c.TxHash} } -func (c StakeTx) Index() []mgo.Index { - return []mgo.Index{ - { - Key: []string{"tx_hash"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"from"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"pub_key"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"-height"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"type"}, - Unique: false, - DropDups: false, - Background: true, - }, - { - Key: []string{"from", "pub_key", "type"}, - Unique: false, - DropDups: false, - Background: true, - }, - } +// ====== + + +// Description +type Description struct { + Moniker string `bson:"moniker"` + Identity string `bson:"identity"` + Website string `bson:"website"` + Details string `bson:"details"` +} + +type StakeTxDeclareCandidacy struct { + StakeTx `bson:"stake_tx"` + Description `bson:"description"` +} + +func (s StakeTxDeclareCandidacy) Name() string { + return CollectionNmStakeTxDeclareCandidacy +} + +func (s StakeTxDeclareCandidacy) PkKvPair() map[string]interface{} { + return bson.M{"stake_tx.tx_hash": s.TxHash} +} + +// ====== + +type StakeTxEditCandidacy struct { + StakeTx `bson:"stake_tx"` + Description `bson:"description"` +} + +func (s StakeTxEditCandidacy) Name() string { + return CollectionNmStakeTxEditCandidacy +} + +func (s StakeTxEditCandidacy) PkKvPair() map[string]interface{} { + return bson.M{"stake_tx.tx_hash": s.TxHash} } diff --git a/model/store/document/tx_stake_declare_candidacy.go b/model/store/document/tx_stake_declare_candidacy.go deleted file mode 100644 index dc0163e..0000000 --- a/model/store/document/tx_stake_declare_candidacy.go +++ /dev/null @@ -1,42 +0,0 @@ -package document - -import ( - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" -) - -const ( - CollectionNmStakeTxDeclareCandidacy = "tx_stake" -) - -// Description -type Description struct { - Moniker string `bson:"moniker"` - Identity string `bson:"identity"` - Website string `bson:"website"` - Details string `bson:"details"` -} - -type StakeTxDeclareCandidacy struct { - StakeTx `bson:"stake_tx"` - Description `bson:"description"` -} - -func (s StakeTxDeclareCandidacy) Name() string { - return CollectionNmStakeTxDeclareCandidacy -} - -func (s StakeTxDeclareCandidacy) PkKvPair() map[string]interface{} { - return bson.M{"stake_tx.tx_hash": s.TxHash} -} - -func (s StakeTxDeclareCandidacy) Index() []mgo.Index { - return []mgo.Index{ - { - Key: []string{"description.moniker"}, - Unique: false, - DropDups: false, - Background: true, - }, - } -} \ No newline at end of file diff --git a/model/store/store.go b/model/store/store.go index fec6019..f0f222f 100644 --- a/model/store/store.go +++ b/model/store/store.go @@ -37,10 +37,6 @@ func Init() { } } -func AddIndex() { - index() -} - func InitWithAuth(addrs []string, username, password string) { dialInfo := &mgo.DialInfo{ Addrs: addrs, // []string{"192.168.6.122"} @@ -57,7 +53,6 @@ func InitWithAuth(addrs []string, username, password string) { if nil != err { panic(err) } - index() } func getSession() *mgo.Session { @@ -80,25 +75,6 @@ func Find(collection string, query interface{}) *mgo.Query { return c.Find(query) } -func index() { - if len(docs) == 0 { - return - } - for _, h := range docs { - indexKey := func(c *mgo.Collection) error { - for _, i := range h.Index() { - err := c.EnsureIndex(i) - if err != nil { - logger.Error.Println(err) - return err - } - } - return nil - } - ExecCollection(h.Name(), indexKey) - } -} - func Save(h Docs) error { save := func(c *mgo.Collection) error { pk := h.PkKvPair() @@ -140,16 +116,6 @@ func Update(h Docs) error { return ExecCollection(h.Name(), update) } -/** - * 执行查询，此方法可拆分做为公共方法 - * [SearchPerson description] - * @param {[type]} collectionName string [description] - * @param {[type]} query bson.M [description] - * @param {[type]} sort bson.M [description] - * @param {[type]} fields bson.M [description] - * @param {[type]} skip int [description] - * @param {[type]} limit int) (results []interface{}, err error [description] - */ func Query(collectionName string, query bson.M, sort string, fields bson.M, skip int, limit int) (results []interface{}, err error) { exop := func(c *mgo.Collection) error { return c.Find(query).Sort(sort).Select(fields).Skip(skip).Limit(limit).All(&results) diff --git a/model/store/types.go b/model/store/types.go index dabb41f..a7d46da 100644 --- a/model/store/types.go +++ b/model/store/types.go @@ -2,15 +2,21 @@ package store -import ( - "gopkg.in/mgo.v2" -) - type Docs interface { // collection name Name() string // primary key pair(used to find a unique record) PkKvPair() map[string]interface{} - // add index for collection - Index() []mgo.Index } + +type Coin struct { + Denom string `json:"denom"` + Amount int64 `json:"amount"` +} + +type Coins []Coin + +type Fee struct { + Amount Coins + Gas int64 +} \ No newline at end of file diff --git a/module/codec/codec.go b/module/codec/codec.go new file mode 100644 index 0000000..47efc98 --- /dev/null +++ b/module/codec/codec.go @@ -0,0 +1,30 @@ +package codec + +import ( + "github.com/cosmos/cosmos-sdk/wire" + "github.com/cosmos/cosmos-sdk/x/ibc" + "github.com/cosmos/cosmos-sdk/x/bank" + "github.com/cosmos/cosmos-sdk/x/stake" + "github.com/cosmos/cosmos-sdk/x/slashing" + "github.com/cosmos/cosmos-sdk/x/auth" + sdktypes "github.com/cosmos/cosmos-sdk/types" +) + +var ( + Cdc *wire.Codec +) + +func init() { + Cdc = wire.NewCodec() + + ibc.RegisterWire(Cdc) + bank.RegisterWire(Cdc) + stake.RegisterWire(Cdc) + slashing.RegisterWire(Cdc) + auth.RegisterWire(Cdc) + + sdktypes.RegisterWire(Cdc) + //ctypes.RegisterAmino(Cdc) + + wire.RegisterCrypto(Cdc) +} diff --git a/module/stake/errors.go b/module/stake/errors.go deleted file mode 100644 index 900077b..0000000 --- a/module/stake/errors.go +++ /dev/null @@ -1,53 +0,0 @@ -// nolint -package stake - -import ( - "fmt" - - "github.com/cosmos/cosmos-sdk/errors" -) - -var ( - errCandidateEmpty = fmt.Errorf("Cannot bond to an empty candidate") - errBadBondingDenom = fmt.Errorf("Invalid coin denomination") - errBadBondingAmount = fmt.Errorf("Amount must be > 0") - errNoBondingAcct = fmt.Errorf("No bond account for this (address, validator) pair") - errCommissionNegative = fmt.Errorf("Commission must be positive") - errCommissionHuge = fmt.Errorf("Commission cannot be more than 100%") - - errBadValidatorAddr = fmt.Errorf("Validator does not exist for that address") - errCandidateExistsAddr = fmt.Errorf("Candidate already exist, cannot re-declare candidacy") - errMissingSignature = fmt.Errorf("Missing signature") - errBondNotNominated = fmt.Errorf("Cannot bond to non-nominated account") - errNoCandidateForAddress = fmt.Errorf("Validator does not exist for that address") - errNoDelegatorForAddress = fmt.Errorf("Delegator does not contain validator bond") - errInsufficientFunds = fmt.Errorf("Insufficient bond shares") - errBadRemoveValidator = fmt.Errorf("Error removing validator") - - invalidInput = errors.CodeTypeBaseInvalidInput -) - -func ErrBadValidatorAddr() error { - return errors.WithCode(errBadValidatorAddr, errors.CodeTypeBaseUnknownAddress) -} -func ErrCandidateExistsAddr() error { - return errors.WithCode(errCandidateExistsAddr, errors.CodeTypeBaseInvalidInput) -} -func ErrMissingSignature() error { - return errors.WithCode(errMissingSignature, errors.CodeTypeUnauthorized) -} -func ErrBondNotNominated() error { - return errors.WithCode(errBondNotNominated, errors.CodeTypeBaseInvalidOutput) -} -func ErrNoCandidateForAddress() error { - return errors.WithCode(errNoCandidateForAddress, errors.CodeTypeBaseUnknownAddress) -} -func ErrNoDelegatorForAddress() error { - return errors.WithCode(errNoDelegatorForAddress, errors.CodeTypeBaseInvalidInput) -} -func ErrInsufficientFunds() error { - return errors.WithCode(errInsufficientFunds, errors.CodeTypeBaseInvalidInput) -} -func ErrBadRemoveValidator() error { - return errors.WithCode(errBadRemoveValidator, errors.CodeTypeInternalErr) -} diff --git a/module/stake/state.go b/module/stake/state.go deleted file mode 100644 index f26e989..0000000 --- a/module/stake/state.go +++ /dev/null @@ -1,22 +0,0 @@ -package stake - -import "github.com/tendermint/go-crypto" - -var ( - // Keys for store prefixes - CandidatesPubKeysKey = []byte{0x01} // key for all candidates' pubkeys - ParamKey = []byte{0x02} // key for global parameters relating to staking - - // Key prefixes - CandidateKeyPrefix = []byte{0x03} // prefix for each key to a candidate - DelegatorBondKeyPrefix = []byte{0x04} // prefix for each key to a delegator's bond - DelegatorBondsKeyPrefix = []byte{0x05} // prefix for each key to a delegator's bond -) - -// DelegatorBond represents the bond with tokens held by an account. It is -// owned by one delegator, and is associated with the voting power of one -// pubKey. -type DelegatorBond struct { - PubKey crypto.PubKey - Shares uint64 -} diff --git a/module/stake/tx.go b/module/stake/tx.go deleted file mode 100644 index e5f51e3..0000000 --- a/module/stake/tx.go +++ /dev/null @@ -1,168 +0,0 @@ -package stake - -import ( - "fmt" - - sdk "github.com/cosmos/cosmos-sdk" - "github.com/cosmos/cosmos-sdk/modules/coin" - crypto "github.com/tendermint/go-crypto" -) - -// TODO 当cosmos-sdk加入stake模块时需要删除该文件，防止在启动时，重复加载ByteTxDeclareCandidacy，ByteTxEditCandidacy等类型，导致启动失败 - -const stakingModuleName = "stake" - -// Description - description fields for a candidate -type Description struct { - Moniker string `json:"moniker"` - Identity string `json:"identity"` - Website string `json:"website"` - Details string `json:"details"` -} - -// Name is the name of the modules. -func Name() string { - return stakingModuleName -} - -// Tx -//-------------------------------------------------------------------------------- - -// register the tx type with its validation logic -// make sure to use the name of the handler as the prefix in the tx type, -// so it gets routed properly -const ( - ByteTxDeclareCandidacy = 0x55 - ByteTxEditCandidacy = 0x56 - ByteTxDelegate = 0x57 - ByteTxUnbond = 0x58 - TypeTxDeclareCandidacy = stakingModuleName + "/declareCandidacy" - TypeTxEditCandidacy = stakingModuleName + "/editCandidacy" - TypeTxDelegate = stakingModuleName + "/delegate" - TypeTxUnbond = stakingModuleName + "/unbond" -) - -func init() { - sdk.TxMapper.RegisterImplementation(TxDeclareCandidacy{}, TypeTxDeclareCandidacy, ByteTxDeclareCandidacy) - sdk.TxMapper.RegisterImplementation(TxEditCandidacy{}, TypeTxEditCandidacy, ByteTxEditCandidacy) - sdk.TxMapper.RegisterImplementation(TxDelegate{}, TypeTxDelegate, ByteTxDelegate) - sdk.TxMapper.RegisterImplementation(TxUnbond{}, TypeTxUnbond, ByteTxUnbond) -} - -// Verify interface at compile time -var _, _, _, _ sdk.TxInner = &TxDeclareCandidacy{}, &TxEditCandidacy{}, &TxDelegate{}, &TxUnbond{} - -// BondUpdate - struct for bonding or unbonding transactions -type BondUpdate struct { - PubKey crypto.PubKey `json:"pub_key"` - Bond coin.Coin `json:"amount"` -} - -// ValidateBasic - Check for non-empty candidate, and valid coins -func (tx BondUpdate) ValidateBasic() error { - if tx.PubKey.Empty() { - return errCandidateEmpty - } - - coins := coin.Coins{tx.Bond} - if !coins.IsValid() { - return coin.ErrInvalidCoins() - } - if !coins.IsPositive() { - return fmt.Errorf("Amount must be > 0") - } - return nil -} - -// TxDeclareCandidacy - struct for unbonding transactions -type TxDeclareCandidacy struct { - BondUpdate - Description -} - -// NewTxDeclareCandidacy - new TxDeclareCandidacy -func NewTxDeclareCandidacy(bond coin.Coin, pubKey crypto.PubKey, description Description) sdk.Tx { - return TxDeclareCandidacy{ - BondUpdate{ - PubKey: pubKey, - Bond: bond, - }, - description, - }.Wrap() -} - -// Wrap - Wrap a Tx as a Basecoin Tx -func (tx TxDeclareCandidacy) Wrap() sdk.Tx { return sdk.Tx{tx} } - -// TxEditCandidacy - struct for editing a candidate -type TxEditCandidacy struct { - PubKey crypto.PubKey `json:"pub_key"` - Description -} - -// NewTxEditCandidacy - new TxEditCandidacy -func NewTxEditCandidacy(pubKey crypto.PubKey, description Description) sdk.Tx { - return TxEditCandidacy{ - PubKey: pubKey, - Description: description, - }.Wrap() -} - -// Wrap - Wrap a Tx as a Basecoin Tx -func (tx TxEditCandidacy) Wrap() sdk.Tx { return sdk.Tx{tx} } - -// ValidateBasic - Check for non-empty candidate, -func (tx TxEditCandidacy) ValidateBasic() error { - if tx.PubKey.Empty() { - return errCandidateEmpty - } - - empty := Description{} - if tx.Description == empty { - return fmt.Errorf("Transaction must include some information to modify") - } - return nil -} - -// TxDelegate - struct for bonding transactions -type TxDelegate struct{ BondUpdate } - -// NewTxDelegate - new TxDelegate -func NewTxDelegate(bond coin.Coin, pubKey crypto.PubKey) sdk.Tx { - return TxDelegate{BondUpdate{ - PubKey: pubKey, - Bond: bond, - }}.Wrap() -} - -// Wrap - Wrap a Tx as a Basecoin Tx -func (tx TxDelegate) Wrap() sdk.Tx { return sdk.Tx{tx} } - -// TxUnbond - struct for unbonding transactions -type TxUnbond struct { - PubKey crypto.PubKey `json:"pub_key"` - Shares uint64 `json:"amount"` -} - -// NewTxUnbond - new TxUnbond -func NewTxUnbond(shares uint64, pubKey crypto.PubKey) sdk.Tx { - return TxUnbond{ - PubKey: pubKey, - Shares: shares, - }.Wrap() -} - -// Wrap - Wrap a Tx as a Basecoin Tx -func (tx TxUnbond) Wrap() sdk.Tx { return sdk.Tx{tx} } - -// ValidateBasic - Check for non-empty candidate, positive shares -func (tx TxUnbond) ValidateBasic() error { - if tx.PubKey.Empty() { - return errCandidateEmpty - } - - if tx.Shares == 0 { - return fmt.Errorf("Shares must be > 0") - } - return nil -} diff --git a/mongodb/mongodb.js b/mongodb/mongodb.js index f1a2cb8..5a88204 100644 --- a/mongodb/mongodb.js +++ b/mongodb/mongodb.js @@ -4,7 +4,6 @@ db.createCollection("block"); db.createCollection("stake_role_candidate"); db.createCollection("stake_role_delegator"); db.createCollection("sync_task"); -db.createCollection("tx_coin"); db.createCollection("tx_stake"); db.createCollection("tx_common"); @@ -12,8 +11,8 @@ db.createCollection("tx_common"); db.account.createIndex({"address": 1}, {"unique": true}); db.block.createIndex({"height": -1}, {"unique": true}); -db.stake_role_candidate.createIndex({"pub_key": 1}, {"unique": true}); -db.stake_role_candidate.createIndex({"address": 1}); +db.stake_role_candidate.createIndex({"address": 1}, {"unique": true}); +db.stake_role_candidate.createIndex({"pub_key": 1}); db.stake_role_delegator.createIndex({"pub_key": 1}); db.stake_role_delegator.createIndex({"address": 1}); @@ -21,29 +20,25 @@ db.stake_role_delegator.createIndex({"address": 1, "pub_key": 1}, {"unique": tru db.sync_task.createIndex({"chain_id": 1}, {"unique": true}); -db.tx_coin.createIndex({"tx_hash": 1}, {"unique": true}); -db.tx_coin.createIndex({"from": 1}); -db.tx_coin.createIndex({"to": 1}); -db.tx_coin.createIndex({"height": -1}); -db.tx_coin.createIndex({"time": -1}); -db.tx_coin.createIndex({"from": 1, "to": 1, "time": 1}); - +db.tx_stake.createIndex({"height": -1}); +db.tx_stake.createIndex({"time": -1}); db.tx_stake.createIndex({"tx_hash": 1}); db.tx_stake.createIndex({"stake_tx.tx_hash": 1}); db.tx_stake.createIndex({"description.moniker": 1}); db.tx_stake.createIndex({"from": 1}); +db.tx_stake.createIndex({"to": 1}); db.tx_stake.createIndex({"pub_key": 1}); -db.tx_stake.createIndex({"height": -1}); db.tx_stake.createIndex({"type": 1}); -db.tx_stake.createIndex({"time": -1}); -db.tx_stake.createIndex({"from": 1, "to": 1, "type": 1, "time": -1}); +db.tx_stake.createIndex({"status": 1}); +db.tx_stake.createIndex({"from": 1, "to": 1, "type": 1, "status": 1, "time": -1}); +db.tx_common.createIndex({"height": -1}); +db.tx_common.createIndex({"time": -1}); db.tx_common.createIndex({"tx_hash": 1}, {"unique": true}); db.tx_common.createIndex({"from": 1}); db.tx_common.createIndex({"to": 1}); -db.tx_common.createIndex({"height": -1}); -db.tx_common.createIndex({"time": -1}); db.tx_common.createIndex({"type": 1}); +db.tx_common.createIndex({"status": 1}); // drop collection db.account.drop(); @@ -51,7 +46,6 @@ db.block.drop(); db.stake_role_candidate.drop(); db.stake_role_delegator.drop(); db.sync_task.drop(); -db.tx_coin.drop(); db.tx_stake.drop(); db.tx_common.drop(); @@ -61,7 +55,6 @@ db.block.remove({}); db.stake_role_candidate.remove({}); db.stake_role_delegator.remove({}); db.sync_task.remove({}); -db.tx_coin.remove({}); db.tx_stake.remove({}); db.tx_common.remove({}); diff --git a/sync/handler.go b/sync/handler.go deleted file mode 100644 index 9d663c9..0000000 --- a/sync/handler.go +++ /dev/null @@ -1,427 +0,0 @@ -package sync - -import ( - "reflect" - "sync" - "time" - - "github.com/irisnet/irishub-sync/model/store" - "github.com/irisnet/irishub-sync/model/store/document" - "github.com/irisnet/irishub-sync/module/logger" - "github.com/irisnet/irishub-sync/module/stake" - "github.com/irisnet/irishub-sync/util/constant" - "github.com/irisnet/irishub-sync/util/helper" - "github.com/cosmos/cosmos-sdk/modules/coin" -) - -var ( - delay = false - methodName string -) - -func handle(tx store.Docs, mutex sync.Mutex, funChains []func(tx store.Docs, mutex sync.Mutex)) { - for _, fun := range funChains { - fun(tx, mutex) - } -} - -// save Tx document into collection -func saveTx(tx store.Docs, mutex sync.Mutex) { - methodName = "SaveTx: " - - // save tx document into database - storeTxDocFunc := func(doc store.Docs) { - err := store.Save(doc) - if err != nil { - logger.Error.Printf("%v Save failed. doc is %+v, err is %v", - methodName, doc, err.Error()) - } - } - - // save common tx document - saveCommonTx := func(commonTx document.CommonTx) { - err := store.Save(commonTx) - if err != nil { - logger.Error.Printf("%v Save commonTx failed. doc is %+v, err is %v", - methodName, commonTx, err.Error()) - } - } - - txType := GetTxType(tx) - if txType == "" { - logger.Error.Printf("%v get tx type failed, tx is %v\n", - methodName, tx) - return - } - - saveCommonTx(buildCommonTxData(tx, txType)) - - switch txType { - case constant.TxTypeCoin: - storeTxDocFunc(tx) - break - case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, txType) - break - } - storeTxDocFunc(stakeTxDeclareCandidacy) - - mutex.Lock() - logger.Info.Printf("%v saveOrUpdate cndidates get lock\n", methodName) - - cd, err := document.QueryCandidateByPubkey(stakeTxDeclareCandidacy.PubKey) - - candidate := document.Candidate{ - Address: stakeTxDeclareCandidacy.From, - PubKey: stakeTxDeclareCandidacy.PubKey, - Description: stakeTxDeclareCandidacy.Description, - } - // TODO: in further share not equal amount - candidate.Shares += stakeTxDeclareCandidacy.Amount.Amount - candidate.VotingPower += int64(stakeTxDeclareCandidacy.Amount.Amount) - candidate.UpdateTime = stakeTxDeclareCandidacy.Time - - if err != nil && cd.Address == "" { - logger.Warning.Printf("%v Replace candidate from %+v to %+v\n", methodName, cd, candidate) - } - - store.SaveOrUpdate(candidate) - - mutex.Unlock() - logger.Info.Printf("%v saveOrUpdate cndidates release lock\n", methodName) - break - - case stake.TypeTxEditCandidacy: - break - case stake.TypeTxDelegate: - stakeTx, r := tx.(document.StakeTx) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, txType) - break - } - storeTxDocFunc(stakeTx) - - mutex.Lock() - logger.Info.Printf("%v saveOrUpdate tx type %v get lock\n", - methodName, txType) - - candidate, err := document.QueryCandidateByPubkey(stakeTx.PubKey) - // candidate is not exist - if err != nil { - logger.Warning.Printf("%v candidate is not exist while delegate, add = %s ,pub_key = %s\n", - methodName, stakeTx.From, stakeTx.PubKey) - candidate = document.Candidate{ - PubKey: stakeTx.PubKey, - } - } - - delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) - // delegator is not exist - if err != nil { - delegator = document.Delegator{ - Address: stakeTx.From, - PubKey: stakeTx.PubKey, - } - } - // TODO: in further share not equal amount - delegator.Shares += stakeTx.Amount.Amount - delegator.UpdateTime = stakeTx.Time - store.SaveOrUpdate(delegator) - - candidate.Shares += stakeTx.Amount.Amount - candidate.VotingPower += int64(stakeTx.Amount.Amount) - candidate.UpdateTime = stakeTx.Time - store.SaveOrUpdate(candidate) - - mutex.Unlock() - logger.Info.Printf("%v saveOrUpdate tx type %v release lock\n", - methodName, txType) - break - - case stake.TypeTxUnbond: - stakeTx, r := tx.(document.StakeTx) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, txType) - break - } - storeTxDocFunc(stakeTx) - - mutex.Lock() - logger.Info.Printf("%v saveOrUpdate tx type %v get lock\n", - methodName, txType) - - delegator, err := document.QueryDelegatorByAddressAndPubkey(stakeTx.From, stakeTx.PubKey) - // delegator is not exist - if err != nil { - logger.Warning.Printf("%v delegator is not exist while unBond,add = %s,pub_key=%s\n", - methodName, stakeTx.From, stakeTx.PubKey) - delegator = document.Delegator{ - Address: stakeTx.From, - PubKey: stakeTx.PubKey, - } - } - delegator.Shares -= stakeTx.Amount.Amount - delegator.UpdateTime = stakeTx.Time - store.SaveOrUpdate(delegator) - - candidate, err2 := document.QueryCandidateByPubkey(stakeTx.PubKey) - // candidate is not exist - if err2 != nil { - logger.Warning.Printf("%v candidate is not exist while unBond,add = %s,pub_key=%s\n", - methodName, stakeTx.From, stakeTx.PubKey) - candidate = document.Candidate{ - PubKey: stakeTx.PubKey, - } - } - candidate.Shares -= stakeTx.Amount.Amount - candidate.VotingPower -= int64(stakeTx.Amount.Amount) - candidate.UpdateTime = stakeTx.Time - store.SaveOrUpdate(candidate) - - mutex.Unlock() - logger.Info.Printf("%v saveOrUpdate tx type %v release lock\n", - methodName, txType) - break - } -} - -// save account -func saveOrUpdateAccount(tx store.Docs, mutex sync.Mutex) { - var ( - address string - updateTime time.Time - height int64 - methodName = "SaveOrUpdateAccount: " - ) - logger.Info.Printf("Start %v\n", methodName) - - // save account - fun := func(address string, updateTime time.Time, height int64) { - account := document.Account{ - Address: address, - Time: updateTime, - Height: height, - } - - if err := store.Save(account); err != nil { - logger.Trace.Printf("%v Record exists, account is %v, err is %s\n", - methodName, account.Address, err.Error()) - } - } - - txType := GetTxType(tx) - if txType == "" { - logger.Error.Printf("%v get tx type failed, tx is %v\n", - methodName, tx) - return - } - - switch txType { - case constant.TxTypeCoin: - coinTx, r := tx.(document.CoinTx) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, txType) - break - } - updateTime = coinTx.Time - height = coinTx.Height - - fun(coinTx.From, updateTime, height) - fun(coinTx.To, updateTime, height) - break - case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, txType) - break - } - address = stakeTxDeclareCandidacy.From - updateTime = stakeTxDeclareCandidacy.Time - height = stakeTxDeclareCandidacy.Height - - fun(address, updateTime, height) - break - case stake.TypeTxEditCandidacy: - break - case stake.TypeTxDelegate, stake.TypeTxUnbond: - stakeTx, r := tx.(document.StakeTx) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, txType) - break - } - address = stakeTx.From - updateTime = stakeTx.Time - height = stakeTx.Height - - fun(address, updateTime, height) - break - } - - logger.Info.Printf("End %v\n", methodName) -} - -// update account balance -func updateAccountBalance(tx store.Docs, mutex sync.Mutex) { - var ( - address string - methodName = "UpdateAccountBalance: " - ) - logger.Info.Printf("Start %v\n", methodName) - - fun := func(address string) { - account, err := document.QueryAccount(address) - if err != nil { - logger.Error.Printf("%v updateAccountBalance failed, account is %v and err is %v", - methodName, account, err.Error()) - return - } - - // query balance of account - ac := helper.QueryAccountBalance(address, delay) - account.Amount = ac.Coins - if err := store.Update(account); err != nil { - logger.Error.Printf("%v account:[%q] balance update failed,%s\n", - methodName, account.Address, err) - } - } - - txType := GetTxType(tx) - if txType == "" { - logger.Error.Printf("%v get tx type failed, tx is %v\n", - methodName, tx) - return - } - - switch txType { - case constant.TxTypeCoin: - coinTx, r := tx.(document.CoinTx) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, txType) - break - } - fun(coinTx.From) - fun(coinTx.To) - break - case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, r := tx.(document.StakeTxDeclareCandidacy) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, txType) - break - } - address = stakeTxDeclareCandidacy.From - fun(address) - break - case stake.TypeTxEditCandidacy: - break - case stake.TypeTxDelegate, stake.TypeTxUnbond: - stakeTx, r := tx.(document.StakeTx) - if !r { - logger.Error.Printf("%v get docuemnt from tx failed. tx type is %v\n", - methodName, txType) - break - } - address = stakeTx.From - fun(address) - break - } - - logger.Info.Printf("End %v\n", methodName) -} - -// build common tx data through parse tx -func buildCommonTxData(tx store.Docs, txType string) document.CommonTx { - var commonTx document.CommonTx - - if txType == "" { - txType = GetTxType(tx) - } - switch txType { - // tx coin - case constant.TxTypeCoin: - txCoin := tx.(document.CoinTx) - commonTx = document.CommonTx{ - TxHash: txCoin.TxHash, - Time: txCoin.Time, - Height: txCoin.Height, - From: txCoin.From, - To: txCoin.To, - Amount: txCoin.Amount, - Type: txType, - } - break - case stake.TypeTxDeclareCandidacy: - txDeclare := tx.(document.StakeTxDeclareCandidacy) - commonTx = document.CommonTx{ - TxHash: txDeclare.TxHash, - Time: txDeclare.Time, - Height: txDeclare.Height, - From: txDeclare.From, - To: txDeclare.PubKey, - Amount: []coin.Coin{txDeclare.Amount}, - Type: txDeclare.Type, - } - break - case stake.TypeTxEditCandidacy: - break - case stake.TypeTxDelegate, stake.TypeTxUnbond: - txStake := tx.(document.StakeTx) - commonTx = document.CommonTx{ - TxHash: txStake.TxHash, - Time: txStake.Time, - Height: txStake.Height, - From: txStake.From, - To: txStake.PubKey, - Amount: []coin.Coin{txStake.Amount}, - Type: txStake.Type, - } - break - } - - return commonTx -} - -// get tx type -func GetTxType(tx store.Docs) string { - txCollectionName := tx.Name() - var txType string - - switch txCollectionName { - case document.CollectionNmCoinTx: - txType = constant.TxTypeCoin - break - case document.CollectionNmStakeTx: - if !reflect.ValueOf(tx).FieldByName("Type").IsValid() { - logger.Error.Printf("%v type which is field name of stake tx is missed\n", methodName) - break - } - stakeType := constant.TxTypeStake + "/" + reflect.ValueOf(tx).FieldByName("Type").String() - - switch stakeType { - case stake.TypeTxDeclareCandidacy: - txType = stake.TypeTxDeclareCandidacy - break - case stake.TypeTxEditCandidacy: - txType = stake.TypeTxEditCandidacy - break - case stake.TypeTxDelegate: - txType = stake.TypeTxDelegate - break - case stake.TypeTxUnbond: - txType = stake.TypeTxUnbond - break - } - break - } - - return txType -} \ No newline at end of file diff --git a/sync/handler/account.go b/sync/handler/account.go new file mode 100644 index 0000000..64717c5 --- /dev/null +++ b/sync/handler/account.go @@ -0,0 +1,174 @@ +package handler + +import ( + "github.com/irisnet/irishub-sync/util/constant" + "github.com/irisnet/irishub-sync/model/store/document" + "github.com/irisnet/irishub-sync/model/store" + "sync" + "time" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/util/helper" +) + +// save account +func SaveAccount(docTx store.Docs, mutex sync.Mutex) { + var ( + address string + updateTime time.Time + height int64 + methodName = "SaveAccount: " + ) + logger.Info.Printf("Start %v\n", methodName) + + // save account + fun := func(address string, updateTime time.Time, height int64) { + account := document.Account{ + Address: address, + Time: updateTime, + Height: height, + } + + if err := store.Save(account); err != nil { + logger.Trace.Printf("%v Record exists, account is %v, err is %s\n", + methodName, account.Address, err.Error()) + } + } + + txType := GetTxType(docTx) + if txType == "" { + logger.Error.Printf("%v get docTx type failed, docTx is %v\n", + methodName, docTx) + return + } + + switch txType { + case constant.TxTypeBank: + docTx, r := docTx.(document.CommonTx) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + updateTime = docTx.Time + height = docTx.Height + + fun(docTx.From, updateTime, height) + fun(docTx.To, updateTime, height) + break + case constant.TxTypeStakeCreate: + docTx, r := docTx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + address = docTx.ValidatorAddr + updateTime = docTx.Time + height = docTx.Height + + fun(address, updateTime, height) + break + case constant.TxTypeStakeEdit: + docTx, r := docTx.(document.StakeTxEditCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + address = docTx.ValidatorAddr + updateTime = docTx.Time + height = docTx.Height + + fun(address, updateTime, height) + break + case constant.TxTypeStakeDelegate, constant.TxTypeStakeUnbond: + stakeTx, r := docTx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + updateTime = stakeTx.Time + height = stakeTx.Height + + fun(stakeTx.ValidatorAddr, updateTime, height) + fun(stakeTx.DelegatorAddr, updateTime, height) + break + } + + logger.Info.Printf("End %v\n", methodName) +} + +// update account balance +func UpdateBalance(docTx store.Docs, mutex sync.Mutex) { + var ( + methodName = "UpdateBalance: " + ) + logger.Info.Printf("Start %v\n", methodName) + + fun := func(address string) { + account, err := document.QueryAccount(address) + if err != nil { + logger.Error.Printf("%v updateAccountBalance failed, account is %v and err is %v", + methodName, account, err.Error()) + return + } + + // query balance of account + account.Amount = helper.QueryAccountBalance(address) + if err := store.Update(account); err != nil { + logger.Error.Printf("%v account:[%q] balance update failed,%s\n", + methodName, account.Address, err) + } + } + + txType := GetTxType(docTx) + if txType == "" { + logger.Error.Printf("%v get docTx type failed, docTx is %v\n", + methodName, docTx) + return + } + + switch txType { + case constant.TxTypeBank: + docTx, r := docTx.(document.CommonTx) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + fun(docTx.From) + fun(docTx.To) + break + case constant.TxTypeStakeCreate: + docTx, r := docTx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + fun(docTx.ValidatorAddr) + break + case constant.TxTypeStakeEdit: + docTx, r := docTx.(document.StakeTxEditCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + fun(docTx.ValidatorAddr) + break + case constant.TxTypeStakeDelegate, constant.TxTypeStakeUnbond: + docTx, r := docTx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + fun(docTx.ValidatorAddr) + fun(docTx.DelegatorAddr) + break + } + + logger.Info.Printf("End %v\n", methodName) +} diff --git a/sync/handler/account_test.go b/sync/handler/account_test.go new file mode 100644 index 0000000..bf4fd16 --- /dev/null +++ b/sync/handler/account_test.go @@ -0,0 +1,125 @@ +package handler + +import ( + "sync" + "testing" + + "github.com/irisnet/irishub-sync/model/store" +) + +func TestSaveAccount(t *testing.T) { + docTxBank := buildDocData(1762) + docTxStakeCreate := buildDocData(46910) + docTxStakeEdit := buildDocData(49388) + docTxStakeDelegate := buildDocData(47349) + docTxStakeUnBond := buildDocData(34241) + + type args struct { + docTx store.Docs + mutex sync.Mutex + } + tests := []struct { + name string + args args + }{ + { + name: "tx bank", + args: args{ + docTx: docTxBank, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/create", + args: args{ + docTx: docTxStakeCreate, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/edit", + args: args{ + docTx: docTxStakeEdit, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/delegate", + args: args{ + docTx: docTxStakeDelegate, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/unbond", + args: args{ + docTx: docTxStakeUnBond, + mutex: sync.Mutex{}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + SaveAccount(tt.args.docTx, tt.args.mutex) + }) + } +} + +func TestUpdateBalance(t *testing.T) { + docTxBank := buildDocData(1762) + docTxStakeCreate := buildDocData(46910) + docTxStakeEdit := buildDocData(49388) + docTxStakeDelegate := buildDocData(47349) + docTxStakeUnBond := buildDocData(34241) + + + type args struct { + docTx store.Docs + mutex sync.Mutex + } + tests := []struct { + name string + args args + }{ + { + name: "tx bank", + args: args{ + docTx: docTxBank, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/create", + args: args{ + docTx: docTxStakeCreate, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/edit", + args: args{ + docTx: docTxStakeEdit, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/delegate", + args: args{ + docTx: docTxStakeDelegate, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/unbond", + args: args{ + docTx: docTxStakeUnBond, + mutex: sync.Mutex{}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + UpdateBalance(tt.args.docTx, tt.args.mutex) + }) + } +} diff --git a/sync/handler/block.go b/sync/handler/block.go new file mode 100644 index 0000000..9083948 --- /dev/null +++ b/sync/handler/block.go @@ -0,0 +1,96 @@ +package handler + +import ( + "github.com/irisnet/irishub-sync/util/helper" + "github.com/irisnet/irishub-sync/model/store/document" + "github.com/tendermint/tendermint/types" + "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/module/codec" + "encoding/json" + "github.com/irisnet/irishub-sync/module/logger" +) + +func SaveBlock(meta *types.BlockMeta, block *types.Block) { + + hexFunc := func(bytes []byte) string { + return helper.BuildHex(bytes) + } + + docBlock := document.Block{ + Height: meta.Header.Height, + Hash: hexFunc(meta.BlockID.Hash), + Time: meta.Header.Time, + NumTxs: meta.Header.NumTxs, + } + + lastBlockId := document.BlockID{ + Hash: hexFunc(meta.Header.LastBlockID.Hash), + PartsHeader: document.PartSetHeader{ + Total: meta.Header.LastBlockID.PartsHeader.Total, + Hash: hexFunc(meta.Header.LastBlockID.PartsHeader.Hash), + }, + } + + blockMeta := document.BlockMeta{ + BlockID: document.BlockID{ + Hash: hexFunc(meta.BlockID.Hash), + PartsHeader: document.PartSetHeader{ + Total: meta.BlockID.PartsHeader.Total, + Hash: hexFunc(meta.BlockID.PartsHeader.Hash), + }, + }, + Header: document.Header{ + ChainID: meta.Header.ChainID, + Height: meta.Header.Height, + Time: meta.Header.Time, + NumTxs: meta.Header.NumTxs, + LastBlockID: lastBlockId, + TotalTxs: meta.Header.TotalTxs, + LastCommitHash: hexFunc(meta.Header.LastCommitHash), + DataHash: hexFunc(meta.Header.DataHash), + ValidatorsHash: hexFunc(meta.Header.ValidatorsHash), + ConsensusHash: hexFunc(meta.Header.ConsensusHash), + AppHash: hexFunc(meta.Header.AppHash), + LastResultsHash: hexFunc(meta.Header.LastResultsHash), + EvidenceHash: hexFunc(meta.Header.EvidenceHash), + }, + } + + var ( + preCommits []document.Vote + ) + + if len(block.LastCommit.Precommits) > 0 { + for _, v := range block.LastCommit.Precommits { + var sig document.Signature + out, _ := codec.Cdc.MarshalJSON(v.Signature) + json.Unmarshal(out, &sig) + preCommit := document.Vote{ + ValidatorAddress: v.ValidatorAddress.String(), + ValidatorIndex: v.ValidatorIndex, + Height: v.Height, + Round: v.Round, + Timestamp: v.Timestamp, + Type: v.Type, + BlockID: lastBlockId, + Signature: sig, + } + preCommits = append(preCommits, preCommit) + } + } + + blockContent := document.BlockContent{ + LastCommit: document.Commit{ + BlockID: lastBlockId, + Precommits: preCommits, + }, + } + + docBlock.Meta = blockMeta + docBlock.Block = blockContent + + err := store.Save(docBlock) + if err != nil { + logger.Error.Println(err) + } +} diff --git a/sync/handler/block_test.go b/sync/handler/block_test.go new file mode 100644 index 0000000..88631b6 --- /dev/null +++ b/sync/handler/block_test.go @@ -0,0 +1,58 @@ +package handler + +import ( + "testing" + + "github.com/tendermint/tendermint/types" + "github.com/irisnet/irishub-sync/util/helper" + "github.com/irisnet/irishub-sync/module/logger" +) + +func buildBlock(blockHeight int64) (*types.BlockMeta, *types.Block) { + + client := helper.GetClient() + // release client + defer client.Release() + + block, err := client.Client.Block(&blockHeight) + + if err != nil { + logger.Error.Fatalln(err) + } + + return block.BlockMeta, block.Block +} + +func TestSaveBlock(t *testing.T) { + meta1, block1 := buildBlock(28558) + meta2, block2 := buildBlock(47349) + + type args struct { + meta *types.BlockMeta + block *types.Block + } + tests := []struct { + name string + args args + }{ + { + name: "test save block", + args: args{ + meta: meta1, + block: block1, + }, + }, + { + name: "test save block", + args: args{ + meta: meta2, + block: block2, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + SaveBlock(tt.args.meta, tt.args.block) + }) + } +} diff --git a/sync/handler/tx.go b/sync/handler/tx.go new file mode 100644 index 0000000..c6516ec --- /dev/null +++ b/sync/handler/tx.go @@ -0,0 +1,287 @@ +package handler + +import ( + "github.com/irisnet/irishub-sync/model/store/document" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/model/store" + "sync" + "github.com/irisnet/irishub-sync/util/constant" +) + + + +// save Tx document into collection +func SaveTx(docTx store.Docs, mutex sync.Mutex) { + var ( + methodName = "SaveTx: " + ) + + // save docTx document into database + storeTxDocFunc := func(doc store.Docs) { + err := store.Save(doc) + if err != nil { + logger.Error.Printf("%v Save failed. doc is %+v, err is %v", + methodName, doc, err.Error()) + } + } + + // save common docTx document + saveCommonTx := func(commonTx document.CommonTx) { + err := store.Save(commonTx) + if err != nil { + logger.Error.Printf("%v Save commonTx failed. doc is %+v, err is %v", + methodName, commonTx, err.Error()) + } + } + + txType := GetTxType(docTx) + if txType == "" { + logger.Error.Printf("%v get docTx type failed, docTx is %v\n", + methodName, docTx) + return + } + + saveCommonTx(buildCommonTxData(docTx, txType)) + + switch txType { + case constant.TxTypeBank: + break + case constant.TxTypeStakeCreate: + docTx, r := docTx.(document.StakeTxDeclareCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + storeTxDocFunc(docTx) + + mutex.Lock() + logger.Info.Printf("%v saveOrUpdate cndidates get lock\n", methodName) + + cd, err := document.QueryCandidateByAddress(docTx.ValidatorAddr) + + candidate := document.Candidate{ + Address: docTx.ValidatorAddr, + PubKey: docTx.PubKey, + } + + if err == nil && cd.PubKey == "" { + // candidate exist + logger.Warning.Printf("%v Replace candidate from %+v to %+v\n", methodName, cd, candidate) + // TODO: in further share not equal amount + candidate.Shares = cd.Shares + docTx.Amount.Amount + candidate.VotingPower = int64(candidate.Shares) + + // description of candidate is empty + if cd.Description.Moniker == "" { + candidate.Description = docTx.Description + } + } else { + // candidate not exist + candidate.Shares = docTx.Amount.Amount + candidate.VotingPower = int64(candidate.Shares) + candidate.Description = docTx.Description + } + candidate.UpdateTime = docTx.Time + + store.SaveOrUpdate(candidate) + + mutex.Unlock() + logger.Info.Printf("%v saveOrUpdate cndidates release lock\n", methodName) + break + + case constant.TxTypeStakeEdit: + docTx, r := docTx.(document.StakeTxEditCandidacy) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + storeTxDocFunc(docTx) + + mutex.Lock() + logger.Info.Printf("%v saveOrUpdate cndidates get lock\n", methodName) + + cd, err := document.QueryCandidateByAddress(docTx.ValidatorAddr) + + var candidate document.Candidate + + if err != nil { + // candidate not exist + candidate = document.Candidate{ + Address: docTx.ValidatorAddr, + Description: docTx.Description, + } + } else { + // candidate exist + cd.Description = docTx.Description + candidate = cd + } + + store.SaveOrUpdate(candidate) + + mutex.Unlock() + logger.Info.Printf("%v saveOrUpdate cndidates release lock\n", methodName) + break + case constant.TxTypeStakeDelegate: + docTx, r := docTx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + storeTxDocFunc(docTx) + + mutex.Lock() + logger.Info.Printf("%v saveOrUpdate docTx type %v get lock\n", + methodName, txType) + + candidate, err := document.QueryCandidateByAddress(docTx.ValidatorAddr) + // candidate is not exist + if err != nil { + logger.Warning.Printf("%v candidate is not exist while delegate, addrdelegator = %s ,addrvalidator = %s\n", + methodName, docTx.DelegatorAddr, docTx.ValidatorAddr) + candidate = document.Candidate{ + Address: docTx.ValidatorAddr, + } + } + + delegator, err := document.QueryDelegatorByAddressAndValAddr(docTx.DelegatorAddr, docTx.ValidatorAddr) + // delegator is not exist + if err != nil { + delegator = document.Delegator{ + Address: docTx.DelegatorAddr, + ValidatorAddr: docTx.ValidatorAddr, + } + } + // TODO: in further share not equal amount + delegator.Shares += docTx.Amount.Amount + delegator.UpdateTime = docTx.Time + store.SaveOrUpdate(delegator) + + candidate.Shares += docTx.Amount.Amount + candidate.VotingPower += int64(docTx.Amount.Amount) + candidate.UpdateTime = docTx.Time + store.SaveOrUpdate(candidate) + + mutex.Unlock() + logger.Info.Printf("%v saveOrUpdate docTx type %v release lock\n", + methodName, txType) + break + + case constant.TxTypeStakeUnbond: + docTx, r := docTx.(document.StakeTx) + if !r { + logger.Error.Printf("%v get docuemnt from docTx failed. docTx type is %v\n", + methodName, txType) + break + } + storeTxDocFunc(docTx) + + mutex.Lock() + logger.Info.Printf("%v saveOrUpdate docTx type %v get lock\n", + methodName, txType) + + delegator, err := document.QueryDelegatorByAddressAndValAddr(docTx.DelegatorAddr, docTx.ValidatorAddr) + // delegator is not exist + if err != nil { + logger.Warning.Printf("%v delegator is not exist while unBond,add = %s,valAddr=%s\n", + methodName, docTx.DelegatorAddr, docTx.ValidatorAddr) + delegator = document.Delegator{ + Address: docTx.DelegatorAddr, + ValidatorAddr: docTx.ValidatorAddr, + } + } + delegator.Shares -= docTx.Amount.Amount + delegator.UpdateTime = docTx.Time + store.SaveOrUpdate(delegator) + + candidate, err2 := document.QueryCandidateByAddress(docTx.ValidatorAddr) + // candidate is not exist + if err2 != nil { + logger.Warning.Printf("%v candidate is not exist while unBond,add = %s,valAddr=%s\n", + methodName, docTx.DelegatorAddr, docTx.ValidatorAddr) + candidate = document.Candidate{ + Address: docTx.ValidatorAddr, + } + } + candidate.Shares -= docTx.Amount.Amount + candidate.VotingPower -= int64(docTx.Amount.Amount) + candidate.UpdateTime = docTx.Time + store.SaveOrUpdate(candidate) + + mutex.Unlock() + logger.Info.Printf("%v saveOrUpdate docTx type %v release lock\n", + methodName, txType) + break + } +} + +// build common tx data through parse tx +func buildCommonTxData(docTx store.Docs, txType string) document.CommonTx { + var commonTx document.CommonTx + + if txType == "" { + txType = GetTxType(docTx) + } + switch txType { + case constant.TxTypeBank: + doc := docTx.(document.CommonTx) + commonTx = document.CommonTx{ + TxHash: doc.TxHash, + Time: doc.Time, + Height: doc.Height, + From: doc.From, + To: doc.To, + Amount: doc.Amount, + Type: doc.Type, + Fee: doc.Fee, + Status: doc.Status, + } + break + case constant.TxTypeStakeCreate: + doc := docTx.(document.StakeTxDeclareCandidacy) + commonTx = document.CommonTx{ + TxHash: doc.TxHash, + Time: doc.Time, + Height: doc.Height, + From: doc.ValidatorAddr, + To: "", + Amount: []store.Coin{doc.Amount}, + Type: doc.Type, + Fee: doc.Fee, + Status: doc.Status, + } + break + case constant.TxTypeStakeEdit: + doc := docTx.(document.StakeTxEditCandidacy) + commonTx = document.CommonTx{ + TxHash: doc.TxHash, + Time: doc.Time, + Height: doc.Height, + From: doc.ValidatorAddr, + To: "", + Amount: []store.Coin{doc.Amount}, + Type: doc.Type, + Fee: doc.Fee, + Status: doc.Status, + } + break + case constant.TxTypeStakeDelegate, constant.TxTypeStakeUnbond: + doc := docTx.(document.StakeTx) + commonTx = document.CommonTx{ + TxHash: doc.TxHash, + Time: doc.Time, + Height: doc.Height, + From: doc.ValidatorAddr, + To: doc.DelegatorAddr, + Amount: []store.Coin{doc.Amount}, + Type: doc.Type, + Fee: doc.Fee, + Status: doc.Status, + } + break + } + + return commonTx +} diff --git a/sync/handler/tx_test.go b/sync/handler/tx_test.go new file mode 100644 index 0000000..6762105 --- /dev/null +++ b/sync/handler/tx_test.go @@ -0,0 +1,98 @@ +package handler + +import ( + "sync" + "testing" + + "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/util/helper" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/module/codec" +) + +func init() { + helper.InitClientPool() + store.Init() +} + +func buildDocData(blockHeight int64) store.Docs { + + client := helper.GetClient() + // release client + defer client.Release() + + block, err := client.Client.Block(&blockHeight) + + if err != nil { + logger.Error.Panic(err) + } + + if block.BlockMeta.Header.NumTxs > 0 { + txs := block.Block.Data.Txs + txByte := txs[0] + docTx := helper.ParseTx(codec.Cdc, txByte, block.Block) + + return docTx + + } + return nil +} + + +func TestSaveTx(t *testing.T) { + docTxBank := buildDocData(1762) + docTxStakeCreate := buildDocData(46910) + docTxStakeEdit := buildDocData(49388) + docTxStakeDelegate := buildDocData(47349) + docTxStakeUnBond := buildDocData(34241) + + type args struct { + docTx store.Docs + mutex sync.Mutex + } + tests := []struct { + name string + args args + }{ + { + name: "tx bank", + args: args{ + docTx: docTxBank, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/create", + args: args{ + docTx: docTxStakeCreate, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/edit", + args: args{ + docTx: docTxStakeEdit, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/delegate", + args: args{ + docTx: docTxStakeDelegate, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/unbond", + args: args{ + docTx: docTxStakeUnBond, + mutex: sync.Mutex{}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + SaveTx(tt.args.docTx, tt.args.mutex) + }) + } +} diff --git a/sync/handler/types.go b/sync/handler/types.go new file mode 100644 index 0000000..1e88017 --- /dev/null +++ b/sync/handler/types.go @@ -0,0 +1,27 @@ +package handler + +import ( + "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/module/logger" + "reflect" + "sync" + "github.com/irisnet/irishub-sync/util/helper" +) + +// get tx type +func GetTxType(docTx store.Docs) string { + if !reflect.ValueOf(docTx).FieldByName("Type").IsValid() { + logger.Error.Printf("type which is field name of stake docTx is missed, docTx is %+v\n", + helper.ToJson(docTx)) + return "" + } + txType := reflect.ValueOf(docTx).FieldByName("Type").String() + + return txType +} + +func Handle(docTx store.Docs, mutex sync.Mutex, funChains []func(tx store.Docs, mutex sync.Mutex)) { + for _, fun := range funChains { + fun(docTx, mutex) + } +} diff --git a/sync/handler_test.go b/sync/handler_test.go deleted file mode 100644 index 9959c58..0000000 --- a/sync/handler_test.go +++ /dev/null @@ -1,236 +0,0 @@ -package sync - -import ( - "testing" - "github.com/irisnet/irishub-sync/model/store" - "github.com/irisnet/irishub-sync/util/helper" - "github.com/irisnet/irishub-sync/module/logger" - "github.com/irisnet/irishub-sync/util/constant" - "github.com/irisnet/irishub-sync/model/store/document" - "github.com/irisnet/irishub-sync/module/stake" - "sync" -) - -func init() { - helper.InitClientPool() - store.Init() -} - -func buildDocData(blockHeight int64) store.Docs { - - client := helper.GetClient() - // release client - defer client.Release() - - block, err := client.Client.Block(&blockHeight) - - if err != nil { - logger.Error.Panic(err) - } - - if block.BlockMeta.Header.NumTxs > 0 { - txs := block.Block.Data.Txs - txByte := txs[0] - txType, tx := helper.ParseTx(txByte) - - switch txType { - case constant.TxTypeCoin: - coinTx, _ := tx.(document.CoinTx) - coinTx.Height = block.Block.Height - coinTx.Time = block.Block.Time - return coinTx - case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) - stakeTxDeclareCandidacy.Height = block.Block.Height - stakeTxDeclareCandidacy.Time = block.Block.Time - return stakeTxDeclareCandidacy - case stake.TypeTxEditCandidacy: - break - case stake.TypeTxDelegate, stake.TypeTxUnbond: - stakeTx, _ := tx.(document.StakeTx) - stakeTx.Height = block.Block.Height - stakeTx.Time = block.Block.Time - return stakeTx - } - - } - return nil -} - -func Test_saveTx(t *testing.T) { - - docTxCoin := buildDocData(1756) - docTxStakeDeclareCandidacy := buildDocData(1921) - docTxStakeDeclareCandidacy2 := buildDocData(1927) - docTxStakeDelegate := buildDocData(115135) - docTxStakeUnBond := buildDocData(397201) - - type args struct { - tx store.Docs - mutex sync.Mutex - } - tests := []struct { - name string - args args - }{ - { - name:"save tx_coin", - args: args{ - tx:docTxCoin, - mutex:sync.Mutex{}, - }, - }, - { - name:"save tx_stake_declareCandidacy", - args: args{ - tx:docTxStakeDeclareCandidacy, - mutex:sync.Mutex{}, - }, - - }, - { - name:"save tx_stake_declareCandidacy2", - args: args{ - tx:docTxStakeDeclareCandidacy2, - mutex:sync.Mutex{}, - }, - }, - { - name:"save tx_stake_delegate", - args:args{ - tx:docTxStakeDelegate, - mutex:sync.Mutex{}, - }, - - }, - { - name:"save tx_stake_unBond", - args: args{ - tx: docTxStakeUnBond, - mutex: sync.Mutex{}, - }, - - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - saveTx(tt.args.tx, tt.args.mutex) - }) - } -} - -func Test_saveOrUpdateAccount(t *testing.T) { - - docTxCoin := buildDocData(1756) - docTxStakeDeclareCandidacy := buildDocData(1921) - docTxStakeDeclareCandidacy2 := buildDocData(1927) - docTxStakeDelegate := buildDocData(115135) - docTxStakeUnBond := buildDocData(397201) - - type args struct { - tx store.Docs - mutex sync.Mutex - } - tests := []struct { - name string - args args - }{ - { - name:"save tx_coin", - args: args{ - tx: docTxCoin, - }, - }, - { - name:"save tx_stake_declareCandidacy", - args: args{ - tx: docTxStakeDeclareCandidacy, - }, - - }, - { - name:"save tx_stake_declareCandidacy2", - args: args{ - tx: docTxStakeDeclareCandidacy2, - }, - - }, - { - name:"save tx_stake_delegate", - args: args{ - tx:docTxStakeDelegate, - }, - - }, - { - name:"save tx_stake_unBond", - args:args{ - tx:docTxStakeUnBond, - }, - - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - saveOrUpdateAccount(tt.args.tx, tt.args.mutex) - }) - } -} - -func Test_updateAccountBalance(t *testing.T) { - - docTxCoin := buildDocData(1756) - docTxStakeDeclareCandidacy := buildDocData(1921) - docTxStakeDeclareCandidacy2 := buildDocData(1927) - docTxStakeDelegate := buildDocData(115135) - docTxStakeUnBond := buildDocData(397201) - - type args struct { - tx store.Docs - mutex sync.Mutex - } - tests := []struct { - name string - args args - }{ - { - name:"tx_coin", - args: args{ - tx:docTxCoin, - }, - }, - { - name:"tx_stake_declareCandidacy", - args: args{ - tx:docTxStakeDeclareCandidacy, - }, - - }, - { - name:"tx_stake_declareCandidacy2", - args: args{ - tx:docTxStakeDeclareCandidacy2, - }, - - }, - { - name:"tx_stake_delegate", - args: args{ - tx:docTxStakeDelegate, - }, - - }, - { - name:"tx_stake_unBond", - args: args{ - tx:docTxStakeUnBond, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - updateAccountBalance(tt.args.tx, tt.args.mutex) - }) - } -} diff --git a/sync/sync.go b/sync/sync.go index 8899820..2146144 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -1,15 +1,12 @@ package sync import ( - "encoding/hex" - "strings" - conf "github.com/irisnet/irishub-sync/conf/server" "github.com/irisnet/irishub-sync/model/store" "github.com/irisnet/irishub-sync/module/logger" - "github.com/irisnet/irishub-sync/module/stake" - "github.com/irisnet/irishub-sync/util/constant" "github.com/irisnet/irishub-sync/util/helper" + "github.com/irisnet/irishub-sync/module/codec" + "github.com/irisnet/irishub-sync/sync/handler" "github.com/irisnet/irishub-sync/model/store/document" "github.com/robfig/cron" @@ -37,7 +34,7 @@ func Start() { err error i = 1 ) - InitServer() + Init() c := helper.GetClient().Client for { @@ -52,7 +49,7 @@ func Start() { logger.Error.Fatalf("TmClient err and exit, %v\n", err.Error()) } } - latestHeight := status.LatestBlockHeight + latestHeight := status.SyncInfo.LatestBlockHeight if syncLatestHeight >= latestHeight - 60 { logger.Info.Println("All fast sync task complete!") break @@ -64,7 +61,7 @@ func Start() { startCron(c) } -func InitServer() { +func Init() { store.Init() chainId := conf.ChainId @@ -100,10 +97,10 @@ func watchBlock(c rpcClient.Client) { syncTask, _ := document.QuerySyncTask() status, _ := c.Status() - latestBlockHeight := status.LatestBlockHeight + latestBlockHeight := status.SyncInfo.LatestBlockHeight funcChain := []func(tx store.Docs, mutex sync.Mutex){ - saveTx, saveOrUpdateAccount, updateAccountBalance, + handler.SaveTx, handler.SaveAccount, handler.UpdateBalance, } ch := make(chan int64) @@ -131,10 +128,10 @@ func watchBlock(c rpcClient.Client) { func fastSync(c rpcClient.Client) int64 { syncTaskDoc, _ := document.QuerySyncTask() status, _ := c.Status() - latestBlockHeight := status.LatestBlockHeight + latestBlockHeight := status.SyncInfo.LatestBlockHeight funcChain := []func(tx store.Docs, mutex sync.Mutex){ - saveTx, saveOrUpdateAccount, updateAccountBalance, + handler.SaveTx, handler.SaveAccount, handler.UpdateBalance, } ch := make(chan int64) @@ -194,10 +191,6 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex syn client := helper.GetClient() // release client defer client.Release() - // release unBuffer chain and buffer chain - // defer func() { - // - // }() for j := start; j <= end; j++ { block, err := client.Client.Block(&j) @@ -214,51 +207,21 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex syn if block.BlockMeta.Header.NumTxs > 0 { txs := block.Block.Data.Txs for _, txByte := range txs { - txType, tx := helper.ParseTx(txByte) - txHash := strings.ToUpper(hex.EncodeToString(txByte.Hash())) + docTx := helper.ParseTx(codec.Cdc, txByte, block.Block) + txHash := helper.BuildHex(txByte.Hash()) if txHash == "" { logger.Warning.Printf("Tx has no hash, skip this tx."+ - ""+"type of tx is %v, value of tx is %v\n", - txType, tx) + ""+"tx is %v\n", helper.ToJson(docTx)) continue } - logger.Info.Printf("===========threadNo[%d] find tx,txType=%s;txHash=%s\n", threadNum, txType, txHash) + logger.Info.Printf("===========threadNo[%d] find tx, txHash=%s\n", threadNum, txHash) - switch txType { - case constant.TxTypeCoin: - coinTx, _ := tx.(document.CoinTx) - coinTx.Height = block.Block.Height - coinTx.Time = block.Block.Time - handle(coinTx, mutex, funcChain) - break - case stake.TypeTxDeclareCandidacy: - stakeTxDeclareCandidacy, _ := tx.(document.StakeTxDeclareCandidacy) - stakeTxDeclareCandidacy.Height = block.Block.Height - stakeTxDeclareCandidacy.Time = block.Block.Time - handle(stakeTxDeclareCandidacy, mutex, funcChain) - break - case stake.TypeTxEditCandidacy: - break - case stake.TypeTxDelegate, stake.TypeTxUnbond: - stakeTx, _ := tx.(document.StakeTx) - stakeTx.Height = block.Block.Height - stakeTx.Time = block.Block.Time - handle(stakeTx, mutex, funcChain) - break - } + handler.Handle(docTx, mutex, funcChain) } } // save block info - bk := document.Block{ - Height: block.Block.Height, - Time: block.Block.Time, - TxNum: block.BlockMeta.Header.NumTxs, - } - if err := store.Save(bk); err != nil { - logger.Error.Printf("Save block info failed, err is %v", - err.Error()) - } + handler.SaveBlock(block.BlockMeta, block.Block) } logger.Info.Printf("ThreadNo[%d] finish sync block from %d to %d\n", diff --git a/tools/Makefile b/tools/Makefile deleted file mode 100644 index 08969c1..0000000 --- a/tools/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -######################################## -### GLIDE - -GLIDE = github.com/Masterminds/glide -GLIDE_CHECK := $(shell command -v glide 2> /dev/null) - -check_tools: -ifndef GLIDE_CHECK - @echo "No glide in path. Install with 'make get_tools'." -else - @echo "Found glide in path." -endif - -get_tools: -ifdef GLIDE_CHECK - @echo "glide is already installed. Run 'make update_tools' to update." -else - @echo "$(ansi_grn)Installing glide$(ansi_end)" - go get -v $(GLIDE) -endif - -update_tools: - @echo "$(ansi_grn)Updating glide$(ansi_end)" - go get -u -v $(GLIDE) - - -######################################## -# ANSI colors - -ansi_red=\033[0;31m -ansi_grn=\033[0;32m -ansi_yel=\033[0;33m -ansi_end=\033[0m \ No newline at end of file diff --git a/util/constant/types.go b/util/constant/types.go index 23f9d3e..360e2c9 100644 --- a/util/constant/types.go +++ b/util/constant/types.go @@ -3,8 +3,13 @@ package constant const ( - TxTypeCoin = "coin" - TxTypeStake = "stake" + TxTypeBank = "coin" + TxTypeStakeCreate = "declareCandidacy" + TxTypeStakeEdit = "editCandidacy" + TxTypeStakeDelegate = "delegate" + TxTypeStakeUnbond = "unbond" + + TxStatusSuccess = "success" EnvNameDbHost = "DB_HOST" EnvNameDbPort = "DB_PORT" diff --git a/util/helper/account.go b/util/helper/account.go index b4e9a1b..00d397d 100644 --- a/util/helper/account.go +++ b/util/helper/account.go @@ -3,98 +3,68 @@ package helper import ( - "time" + "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/module/codec" - "github.com/irisnet/irishub-sync/module/logger" - - "github.com/tendermint/go-wire" - "github.com/cosmos/cosmos-sdk/modules/coin" - "github.com/cosmos/cosmos-sdk/stack" - "github.com/cosmos/cosmos-sdk/client/commands" - "github.com/cosmos/cosmos-sdk/client/commands/query" - "github.com/tendermint/go-wire/data" - "github.com/tendermint/iavl" - "github.com/cosmos/cosmos-sdk/client" rpcclient "github.com/tendermint/tendermint/rpc/client" + cmn "github.com/tendermint/tmlibs/common" + sdk "github.com/cosmos/cosmos-sdk/types" "github.com/pkg/errors" + "github.com/cosmos/cosmos-sdk/x/auth" + "fmt" + "github.com/irisnet/irishub-sync/module/logger" + authcmd "github.com/cosmos/cosmos-sdk/x/auth/client/cli" ) -var delay = false +func QueryAccountBalance(address string) store.Coins { + addr, err := sdk.GetValAddressHex(address) + if err != nil { + logger.Error.Printf("get addr from hex failed, %+v\n", err) + return nil + } -func setDelay(d bool) { - delay = d -} + res, err := query(auth.AddressStoreKey(addr), "acc", "key") -func QueryAccountBalance(address string, delay bool) *coin.Account { - account := new(coin.Account) - actor, err := commands.ParseActor(address) if err != nil { - return account + logger.Error.Printf("query balance from tendermint failed, %+v\n", err) + return nil } - actor = coin.ChainAddr(actor) - key := stack.PrefixedKey(coin.NameCoin, actor.Bytes()) - if delay { - time.Sleep(1 * time.Second) + // balance is empty + if len(res) <= 0 { + return nil } - _, err2 := GetParsed(key, account, query.GetHeight(), false) - if err2 != nil { - logger.Info.Printf("QueryAccountBalance failed, account bytes are empty for address: %q\n", address) - } - return account -} - -// argument (so pass in a pointer to the appropriate struct) -func GetParsed(key []byte, data interface{}, height int64, prove bool) (int64, error) { - bs, h, err := Get(key, height, prove) - if err != nil { - return 0, err - } - err = wire.ReadBinaryBytes(bs, data) + decoder := authcmd.GetAccountDecoder(codec.Cdc) + account, err := decoder(res) if err != nil { - return 0, err + logger.Error.Printf("decode account failed, %+v\n", err) + return nil } - return h, nil + + return BuildCoins(account.GetCoins()) } -// Get queries the given key and returns the value stored there and the -// height we checked at. -// -// If prove is true (and why shouldn't it be?), -// the data is fully verified before returning. If prove is false, -// we just repeat whatever any (potentially malicious) node gives us. -// Only use that if you are running the full node yourself, -// and it is localhost or you have a secure connection (not HTTP) -func Get(key []byte, height int64, prove bool) (data.Bytes, int64, error) { - if height < 0 { - return nil, 0, errors.New("Height cannot be negative\n") +// Query from Tendermint with the provided storename and path +func query(key cmn.HexBytes, storeName string, endPath string) (res []byte, err error) { + path := fmt.Sprintf("/store/%s/%s", storeName, endPath) + rpcClient := GetClient().Client + if err != nil { + return res, err } - if !prove { - tmClient := GetClient() - defer tmClient.Release() - resp, err := tmClient.Client.ABCIQueryWithOptions("/key", key, - rpcclient.ABCIQueryOptions{Trusted: true, Height: int64(height)}) - if resp == nil { - return nil, height, err - } - return data.Bytes(resp.Response.Value), resp.Response.Height, err + opts := rpcclient.ABCIQueryOptions{ + Height: 0, + Trusted: true, } - val, h, _, err := GetWithProof(key, height) - return val, h, err -} - -// GetWithProof returns the values stored under a given key at the named -// height as in Get. Additionally, it will return a validated merkle -// proof for the key-value pair if it exists, and all checks pass. -func GetWithProof(key []byte, height int64) (data.Bytes, int64, iavl.KeyProof, error) { - tmClient := GetClient() - defer tmClient.Release() - cert, err := commands.GetCertifier() + result, err := rpcClient.ABCIQueryWithOptions(path, key, opts) if err != nil { - return nil, 0, nil, err + return res, err + } + resp := result.Response + if resp.Code != uint32(0) { + return res, errors.Errorf("Query failed: (%d) %s", resp.Code, resp.Log) } - return client.GetWithProof(key, height, tmClient.Client, cert) + return resp.Value, nil } diff --git a/util/helper/account_test.go b/util/helper/account_test.go index 9026a54..3cfadf7 100644 --- a/util/helper/account_test.go +++ b/util/helper/account_test.go @@ -1,3 +1,5 @@ +// This package is used for query balance of account + package helper import ( @@ -6,45 +8,33 @@ import ( "github.com/irisnet/irishub-sync/module/logger" ) -func setUp() { - InitClientPool() -} - func TestQueryAccountBalance(t *testing.T) { - setUp() + InitClientPool() type args struct { address string - delay bool } tests := []struct { name string args args }{ { - name:"the balance of address is not empty", - args: struct { - address string - delay bool - }{ - address: "37D2C544F9D2CF811108B56A496520129B1F80CC", - delay: false}, + name: "test balance not nil", + args: args{ + address: "D770D45DEA7548076F8A27F9C9749B200934F1B4", + }, }, { - name:"the balance of address is empty", - args: struct { - address string - delay bool - }{ - address: "ADBC4AAB3A089BDC8A8155AB97E64CD2CF4A0E9E", - delay: false}, + name: "test balance is nil", + args: args{ + address: "D770D45DEA7548076F8A27F9C9749B200934F1B9", + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - account := QueryAccountBalance(tt.args.address, tt.args.delay) - logger.Info.Printf("the balance of %s is %+v\n", tt.args.address, account) + got := QueryAccountBalance(tt.args.address) + logger.Info.Println(ToJson(got)) }) - } -} \ No newline at end of file +} diff --git a/util/helper/common.go b/util/helper/common.go new file mode 100644 index 0000000..5367301 --- /dev/null +++ b/util/helper/common.go @@ -0,0 +1,22 @@ +package helper + +import ( + "encoding/json" + "github.com/irisnet/irishub-sync/module/logger" + "encoding/binary" +) + + +// convert object to json +func ToJson(v interface{}) string { + data, err := json.Marshal(v) + if err != nil { + logger.Error.Println(err) + } + return string(data) +} + +// byte to int +func BytesToInt(bytes []byte) uint64 { + return binary.BigEndian.Uint64(bytes) +} diff --git a/util/helper/pool_client.go b/util/helper/pool_client.go index ef4cf67..633ed92 100644 --- a/util/helper/pool_client.go +++ b/util/helper/pool_client.go @@ -7,7 +7,6 @@ import ( "errors" conf "github.com/irisnet/irishub-sync/conf/server" - rpcClient "github.com/cosmos/cosmos-sdk/client" "github.com/irisnet/irishub-sync/module/logger" "github.com/tendermint/tendermint/rpc/client" ) @@ -61,7 +60,7 @@ func (n Client) Release() { func createConnection(id int64) Client { tmClient := Client{ - Client: rpcClient.GetNode(conf.BlockChainMonitorUrl), + Client: client.NewHTTP(conf.BlockChainMonitorUrl, "/websocket"), used: false, id: id, } diff --git a/util/helper/tx.go b/util/helper/tx.go index d96b3ea..5fb926d 100644 --- a/util/helper/tx.go +++ b/util/helper/tx.go @@ -3,105 +3,182 @@ package helper import ( - "encoding/hex" - "fmt" - "strings" + "github.com/irisnet/irishub-sync/model/store/document" + "github.com/tendermint/tendermint/types" + "github.com/cosmos/cosmos-sdk/wire" + "github.com/cosmos/cosmos-sdk/x/auth" + "github.com/cosmos/cosmos-sdk/x/bank" + "github.com/cosmos/cosmos-sdk/x/stake" + sdktypes "github.com/cosmos/cosmos-sdk/types" "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/model/store" + "strings" + "encoding/hex" "github.com/irisnet/irishub-sync/util/constant" + "strconv" +) - sdk "github.com/cosmos/cosmos-sdk" - "github.com/tendermint/go-wire/data" - "github.com/tendermint/tendermint/types" - - // 需要将 cosmos-sdk module 中的 txInner 注册到内存 - // 中，才能解析 tx 结构 - _ "github.com/cosmos/cosmos-sdk/modules/auth" - _ "github.com/cosmos/cosmos-sdk/modules/base" - "github.com/cosmos/cosmos-sdk/modules/coin" - "github.com/cosmos/cosmos-sdk/modules/nonce" - "github.com/irisnet/irishub-sync/module/stake" - "github.com/irisnet/irishub-sync/model/store/document" +type ( + msgBankSend = bank.MsgSend + msgStakeCreate = stake.MsgCreateValidator + msgStakeEdit = stake.MsgEditValidator + msgStakeDelegate = stake.MsgDelegate + msgStakeUnbond = stake.MsgUnbond ) -// parse tx struct from binary data -func ParseTx(txByte types.Tx) (string, interface{}) { - txb, err := sdk.LoadTx(txByte) +func ParseTx(cdc *wire.Codec, txBytes types.Tx, block *types.Block) store.Docs { + var ( + authTx auth.StdTx + ) + + err := cdc.UnmarshalBinary(txBytes, &authTx) if err != nil { logger.Error.Println(err) + return nil + } + + height := block.Height + time := block.Time + txHash := BuildHex(txBytes.Hash()) + fee := buildFee(authTx.Fee) + status := constant.TxStatusSuccess + + switch authTx.GetMsg().(type) { + case msgBankSend: + msg := authTx.Msg.(msgBankSend) + docTx := document.CommonTx { + Height: height, + Time: time, + TxHash: txHash, + Fee: fee, + Status: status, + } + docTx.From = msg.Inputs[0].Address.String() + docTx.To = msg.Outputs[0].Address.String() + docTx.Amount = BuildCoins(msg.Inputs[0].Coins) + docTx.Type = constant.TxTypeBank + return docTx + case msgStakeCreate: + msg := authTx.Msg.(msgStakeCreate) + stakeTx := document.StakeTx{ + Height: height, + Time: time, + TxHash: txHash, + Fee: fee, + Status: status, + } + stakeTx.ValidatorAddr = msg.ValidatorAddr.String() + stakeTx.PubKey = BuildHex(msg.PubKey.Bytes()) + stakeTx.Amount = buildCoin(msg.Bond) + + description := document.Description{ + Moniker: msg.Moniker, + Identity: msg.Identity, + Website: msg.Website, + Details: msg.Details, + } + + docTx := document.StakeTxDeclareCandidacy{ + StakeTx: stakeTx, + Description: description, + } + docTx.Type = constant.TxTypeStakeCreate + return docTx + case msgStakeEdit: + msg := authTx.Msg.(msgStakeEdit) + stakeTx := document.StakeTx{ + Height: height, + Time: time, + TxHash: txHash, + Fee: fee, + Status: status, + } + stakeTx.ValidatorAddr = msg.ValidatorAddr.String() + + description := document.Description{ + Moniker: msg.Moniker, + Identity: msg.Identity, + Website: msg.Website, + Details: msg.Details, + } + + docTx := document.StakeTxEditCandidacy{ + StakeTx: stakeTx, + Description: description, + } + docTx.Type = constant.TxTypeStakeEdit + return docTx + case msgStakeDelegate: + msg := authTx.Msg.(msgStakeDelegate) + docTx := document.StakeTx{ + Height: height, + Time: time, + TxHash: txHash, + Fee: fee, + Status: status, + } + docTx.DelegatorAddr = msg.DelegatorAddr.String() + docTx.ValidatorAddr = msg.ValidatorAddr.String() + docTx.Amount = buildCoin(msg.Bond) + docTx.Type = constant.TxTypeStakeDelegate + return docTx + case msgStakeUnbond: + msg := authTx.Msg.(msgStakeUnbond) + shares, err := strconv.Atoi(msg.Shares) + if err != nil { + logger.Error.Println(err) + } + docTx := document.StakeTx{ + Height: height, + Time: time, + TxHash: txHash, + Fee: fee, + Status: status, + } + docTx.DelegatorAddr = msg.DelegatorAddr.String() + docTx.ValidatorAddr = msg.ValidatorAddr.String() + docTx.Amount = store.Coin{ + Amount: int64(shares), + } + docTx.Type = constant.TxTypeStakeUnbond + return docTx + default: + logger.Info.Println("unknown msg type") } - txl, ok := txb.Unwrap().(sdk.TxLayer) + return nil +} + + +func BuildCoins(coins sdktypes.Coins) store.Coins { var ( - txi sdk.Tx - coinTx document.CoinTx - stakeTx document.StakeTx - StakeTxDeclareCandidacy document.StakeTxDeclareCandidacy - nonceAddr data.Bytes + localCoins store.Coins ) - for ok { - txi = txl.Next() - - switch txi.Unwrap().(type) { - - case coin.SendTx: - ctx, _ := txi.Unwrap().(coin.SendTx) - coinTx.From = fmt.Sprintf("%s", ctx.Inputs[0].Address.Address) - coinTx.To = fmt.Sprintf("%s", ctx.Outputs[0].Address.Address) - coinTx.Amount = ctx.Inputs[0].Coins - coinTx.TxHash = strings.ToUpper(hex.EncodeToString(txByte.Hash())) - return constant.TxTypeCoin, coinTx - case nonce.Tx: - ctx, _ := txi.Unwrap().(nonce.Tx) - nonceAddr = ctx.Signers[0].Address - break - case stake.TxUnbond, stake.TxDelegate, stake.TxDeclareCandidacy: - kind, _ := txi.GetKind() - stakeTx.From = fmt.Sprintf("%s", nonceAddr) - stakeTx.Type = strings.Replace(kind, constant.TxTypeStake + "/", "", -1) - stakeTx.TxHash = strings.ToUpper(hex.EncodeToString(txByte.Hash())) - - switch kind { - case stake.TypeTxDeclareCandidacy: - ctx, _ := txi.Unwrap().(stake.TxDeclareCandidacy) - stakeTx.Amount.Denom = ctx.BondUpdate.Bond.Denom - stakeTx.Amount.Amount = ctx.BondUpdate.Bond.Amount - stakeTx.PubKey = fmt.Sprintf("%s", ctx.PubKey.KeyString()) - - description := document.Description{ - Moniker: ctx.Description.Moniker, - Identity: ctx.Description.Identity, - Website: ctx.Description.Website, - Details: ctx.Description.Details, - } - - StakeTxDeclareCandidacy.StakeTx = stakeTx - StakeTxDeclareCandidacy.Description = description - - return kind, StakeTxDeclareCandidacy - case stake.TypeTxEditCandidacy: - // TODO：record edit candidacy tx if necessary - // ctx, _ := txi.Unwrap().(stake.TxEditCandidacy) - break - case stake.TypeTxDelegate: - ctx, _ := txi.Unwrap().(stake.TxDelegate) - stakeTx.Amount.Denom = ctx.Bond.Denom - stakeTx.Amount.Amount = ctx.Bond.Amount - stakeTx.PubKey = fmt.Sprintf("%s", ctx.PubKey.KeyString()) - break - case stake.TypeTxUnbond: - ctx, _ := txi.Unwrap().(stake.TxUnbond) - stakeTx.Amount.Amount = int64(ctx.Shares) - stakeTx.PubKey = fmt.Sprintf("%s", ctx.PubKey.KeyString()) - break - } - return kind, stakeTx - default: - // logger.Info.Printf("unsupported tx type, %+v\n", txi.Unwrap()) + if len(coins) > 0 { + for _, coin := range coins { + localCoins = append(localCoins, buildCoin(coin)) } + } + + return localCoins +} - txl, ok = txi.Unwrap().(sdk.TxLayer) +func buildCoin(coin sdktypes.Coin) store.Coin { + return store.Coin{ + Denom: coin.Denom, + Amount: coin.Amount, } - return "", nil +} + +func buildFee(fee auth.StdFee) store.Fee { + return store.Fee{ + Amount: BuildCoins(fee.Amount), + Gas: fee.Gas, + } +} + +func BuildHex(bytes []byte) string { + return strings.ToUpper(hex.EncodeToString(bytes)) } diff --git a/util/helper/tx_test.go b/util/helper/tx_test.go index 0cf6ad9..5aacb80 100644 --- a/util/helper/tx_test.go +++ b/util/helper/tx_test.go @@ -1,17 +1,16 @@ +// package for parse tx struct from binary data + package helper import ( "testing" - "github.com/irisnet/irishub-sync/module/logger" - - _ "github.com/cosmos/cosmos-sdk/modules/auth" - _ "github.com/cosmos/cosmos-sdk/modules/base" - + "github.com/irisnet/irishub-sync/module/codec" "github.com/tendermint/tendermint/types" + "github.com/irisnet/irishub-sync/module/logger" ) -func buildTxByte(blockHeight int64) types.Tx { +func buildTxByte(blockHeight int64) (types.Tx, *types.Block) { InitClientPool() client := GetClient() @@ -26,57 +25,69 @@ func buildTxByte(blockHeight int64) types.Tx { if block.BlockMeta.Header.NumTxs > 0 { txs := block.Block.Data.Txs - return txs[0] + return txs[0], block.Block } - return nil + return nil, nil } func TestParseTx(t *testing.T) { - - txCoinByte := buildTxByte(12453) - txStakeDeclareCandidacyByte := buildTxByte(19073) - txStakeDelegateByte := buildTxByte(13725) - txStakeUnBondByte := buildTxByte(14260) - + coinByte, coinBlock := buildTxByte(1762) + scByte, scBlock := buildTxByte(46910) + seByte, seBlock := buildTxByte(49388) + sdByte, sdBlock := buildTxByte(47349) + suByte, suBlock := buildTxByte(34241) type args struct { txByte types.Tx + block *types.Block } tests := []struct { name string args args + want string + want1 interface{} }{ { - name: "tx_coin_send", - args: struct{ txByte types.Tx }{ - txByte: txCoinByte, - }, + name: "test tx coin", + args: args{ + txByte: coinByte, + block: coinBlock, + }, }, { - name: "tx_stake_declareCandidacy", - args: struct{ txByte types.Tx }{ - txByte: txStakeDeclareCandidacyByte, - }, + name: "test tx stake/create", + args: args{ + txByte: scByte, + block: scBlock, + }, }, { - name: "tx_stake_delegate", - args: struct{ txByte types.Tx }{ - txByte: txStakeDelegateByte, + name: "test tx stake/edit", + args: args{ + txByte: seByte, + block: seBlock, + }, + }, + { + name: "test tx stake/delegate", + args: args{ + txByte: sdByte, + block: sdBlock, }, }, - { - name: "tx_stake_unbond", - args: struct{ txByte types.Tx }{ - txByte: txStakeUnBondByte, + name: "test tx stake/unbond", + args: args{ + txByte: suByte, + block: suBlock, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - txType, txContent := ParseTx(tt.args.txByte) - logger.Info.Printf("%s: tx type is %s, and struct is %+v\n", tt.name, txType, txContent) + res := ParseTx(codec.Cdc, tt.args.txByte, tt.args.block) + logger.Info.Printf("Tx is %v\n", ToJson(res)) }) } } diff --git a/version.info b/version.info new file mode 100644 index 0000000..e69de29 From 7f9cc17d6ec2b33656501ddf9785f43f7f2322bd Mon Sep 17 00:00:00 2001 From: kaifei Date: Fri, 6 Jul 2018 15:32:08 +0800 Subject: [PATCH 26/37] store validatorSet at given height --- model/store/document/block.go | 8 +++++ sync/handler/block.go | 20 ++++++++++- sync/handler/block_test.go | 27 +++++++++++--- sync/handler/tx_test.go | 66 +++++++++++++++++------------------ sync/sync.go | 13 ++++++- 5 files changed, 94 insertions(+), 40 deletions(-) diff --git a/model/store/document/block.go b/model/store/document/block.go index d5ad321..66ecca1 100644 --- a/model/store/document/block.go +++ b/model/store/document/block.go @@ -16,6 +16,7 @@ type Block struct { NumTxs int64 `bson:"num_txs"` Meta BlockMeta `bson:"meta"` Block BlockContent `bson:"block"` + Validators []Validator `bson:"validators"` } type BlockMeta struct { @@ -87,6 +88,13 @@ type Signature struct { Value string `bson:"value"` } +type Validator struct { + Address string `bson:"address"` + PubKey string `bson:"pub_key"` + VotingPower int64 `bson:"voting_power"` + Accum int64 `bson:"accum"` +} + func (d Block) Name() string { return CollectionNmBlock diff --git a/sync/handler/block.go b/sync/handler/block.go index 9083948..5893b33 100644 --- a/sync/handler/block.go +++ b/sync/handler/block.go @@ -10,7 +10,7 @@ import ( "github.com/irisnet/irishub-sync/module/logger" ) -func SaveBlock(meta *types.BlockMeta, block *types.Block) { +func SaveBlock(meta *types.BlockMeta, block *types.Block, validators []*types.Validator) { hexFunc := func(bytes []byte) string { return helper.BuildHex(bytes) @@ -31,6 +31,7 @@ func SaveBlock(meta *types.BlockMeta, block *types.Block) { }, } + // blockMeta blockMeta := document.BlockMeta{ BlockID: document.BlockID{ Hash: hexFunc(meta.BlockID.Hash), @@ -56,6 +57,7 @@ func SaveBlock(meta *types.BlockMeta, block *types.Block) { }, } + // block var ( preCommits []document.Vote ) @@ -86,8 +88,24 @@ func SaveBlock(meta *types.BlockMeta, block *types.Block) { }, } + // validators + var vals []document.Validator + if len(validators) > 0 { + for _, v := range validators { + validator := document.Validator{ + Address: v.Address.String(), + VotingPower: v.VotingPower, + Accum: v.Accum, + PubKey: hexFunc(v.PubKey.Bytes()), + } + vals = append(vals, validator) + } + } + + docBlock.Meta = blockMeta docBlock.Block = blockContent + docBlock.Validators = vals err := store.Save(docBlock) if err != nil { diff --git a/sync/handler/block_test.go b/sync/handler/block_test.go index 88631b6..cc07d3b 100644 --- a/sync/handler/block_test.go +++ b/sync/handler/block_test.go @@ -8,7 +8,7 @@ import ( "github.com/irisnet/irishub-sync/module/logger" ) -func buildBlock(blockHeight int64) (*types.BlockMeta, *types.Block) { +func buildBlock(blockHeight int64) (*types.BlockMeta, *types.Block, []*types.Validator) { client := helper.GetClient() // release client @@ -20,16 +20,23 @@ func buildBlock(blockHeight int64) (*types.BlockMeta, *types.Block) { logger.Error.Fatalln(err) } - return block.BlockMeta, block.Block + validators, err := client.Client.Validators(&blockHeight) + if err != nil { + logger.Error.Fatalln(err) + } + + return block.BlockMeta, block.Block, validators.Validators } func TestSaveBlock(t *testing.T) { - meta1, block1 := buildBlock(28558) - meta2, block2 := buildBlock(47349) + meta1, block1, vals1 := buildBlock(28558) + meta2, block2, vals2 := buildBlock(96319) + meta3, block3, vals3 := buildBlock(34241) type args struct { meta *types.BlockMeta block *types.Block + vals []*types.Validator } tests := []struct { name string @@ -40,6 +47,7 @@ func TestSaveBlock(t *testing.T) { args: args{ meta: meta1, block: block1, + vals: vals1, }, }, { @@ -47,12 +55,21 @@ func TestSaveBlock(t *testing.T) { args: args{ meta: meta2, block: block2, + vals: vals2, + }, + }, + { + name: "test save block", + args: args{ + meta: meta3, + block: block3, + vals: vals3, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - SaveBlock(tt.args.meta, tt.args.block) + SaveBlock(tt.args.meta, tt.args.block, tt.args.vals) }) } } diff --git a/sync/handler/tx_test.go b/sync/handler/tx_test.go index 6762105..3398e7d 100644 --- a/sync/handler/tx_test.go +++ b/sync/handler/tx_test.go @@ -40,11 +40,11 @@ func buildDocData(blockHeight int64) store.Docs { func TestSaveTx(t *testing.T) { - docTxBank := buildDocData(1762) - docTxStakeCreate := buildDocData(46910) - docTxStakeEdit := buildDocData(49388) - docTxStakeDelegate := buildDocData(47349) - docTxStakeUnBond := buildDocData(34241) + //docTxBank := buildDocData(1762) + //docTxStakeCreate := buildDocData(46910) + //docTxStakeEdit := buildDocData(49388) + //docTxStakeDelegate := buildDocData(47349) + docTxStakeUnBond := buildDocData(96319) type args struct { docTx store.Docs @@ -54,34 +54,34 @@ func TestSaveTx(t *testing.T) { name string args args }{ - { - name: "tx bank", - args: args{ - docTx: docTxBank, - mutex: sync.Mutex{}, - }, - }, - { - name: "tx stake/create", - args: args{ - docTx: docTxStakeCreate, - mutex: sync.Mutex{}, - }, - }, - { - name: "tx stake/edit", - args: args{ - docTx: docTxStakeEdit, - mutex: sync.Mutex{}, - }, - }, - { - name: "tx stake/delegate", - args: args{ - docTx: docTxStakeDelegate, - mutex: sync.Mutex{}, - }, - }, + //{ + // name: "tx bank", + // args: args{ + // docTx: docTxBank, + // mutex: sync.Mutex{}, + // }, + //}, + //{ + // name: "tx stake/create", + // args: args{ + // docTx: docTxStakeCreate, + // mutex: sync.Mutex{}, + // }, + //}, + //{ + // name: "tx stake/edit", + // args: args{ + // docTx: docTxStakeEdit, + // mutex: sync.Mutex{}, + // }, + //}, + //{ + // name: "tx stake/delegate", + // args: args{ + // docTx: docTxStakeDelegate, + // mutex: sync.Mutex{}, + // }, + //}, { name: "tx stake/unbond", args: args{ diff --git a/sync/sync.go b/sync/sync.go index 2146144..1671663 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -14,6 +14,7 @@ import ( ctypes "github.com/tendermint/tendermint/rpc/core/types" "sync" + "github.com/tendermint/tendermint/types" ) var ( @@ -220,8 +221,18 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex syn } } + // get validatorSet at given height + var validators []*types.Validator + res, err := client.Client.Validators(&j) + if err != nil { + logger.Error.Printf("Can't get validatorSet at %v\n", j) + } else { + validators = res.Validators + } + + // save block info - handler.SaveBlock(block.BlockMeta, block.Block) + handler.SaveBlock(block.BlockMeta, block.Block, validators) } logger.Info.Printf("ThreadNo[%d] finish sync block from %d to %d\n", From 2d1fa30f8df9b3980157fd6a6549fc9cd85568a5 Mon Sep 17 00:00:00 2001 From: kaifei Date: Fri, 6 Jul 2018 18:23:30 +0800 Subject: [PATCH 27/37] refactor project structure --- main.go | 8 +-- {sync => service}/handler/account.go | 8 +-- {sync => service}/handler/account_test.go | 3 +- {sync => service}/handler/block.go | 63 +++++++++---------- {sync => service}/handler/block_test.go | 18 +++--- {sync => service}/handler/tx.go | 10 ++- {sync => service}/handler/tx_test.go | 9 ++- {sync => service}/handler/types.go | 4 +- {sync => service}/sync.go | 40 ++++++------ {sync => service}/sync_test.go | 28 ++++----- {model/store => store}/document/account.go | 10 +-- {model/store => store}/document/block.go | 49 +++++++-------- {model/store => store}/document/doc.go | 2 +- .../document/stake_role_candidate.go | 4 +- .../document/stake_role_delegator.go | 2 +- {model/store => store}/document/sync_task.go | 2 +- {model/store => store}/document/tx_common.go | 20 +++--- {model/store => store}/document/tx_stake.go | 19 +++--- {model/store => store}/store.go | 0 {model/store => store}/types.go | 4 +- util/helper/account.go | 13 ++-- util/helper/tx.go | 63 +++++++++---------- 22 files changed, 182 insertions(+), 197 deletions(-) rename {sync => service}/handler/account.go (97%) rename {sync => service}/handler/account_test.go (97%) rename {sync => service}/handler/block.go (62%) rename {sync => service}/handler/block_test.go (90%) rename {sync => service}/handler/tx.go (98%) rename {sync => service}/handler/tx_test.go (96%) rename {sync => service}/handler/types.go (93%) rename {sync => service}/sync.go (95%) rename {sync => service}/sync_test.go (95%) rename {model/store => store}/document/account.go (81%) rename {model/store => store}/document/block.go (67%) rename {model/store => store}/document/doc.go (88%) rename {model/store => store}/document/stake_role_candidate.go (96%) rename {model/store => store}/document/stake_role_delegator.go (95%) rename {model/store => store}/document/sync_task.go (93%) rename {model/store => store}/document/tx_common.go (50%) rename {model/store => store}/document/tx_stake.go (81%) rename {model/store => store}/store.go (100%) rename {model/store => store}/types.go (95%) diff --git a/main.go b/main.go index c9d2821..0d58df8 100644 --- a/main.go +++ b/main.go @@ -1,15 +1,15 @@ package main import ( + "github.com/irisnet/irishub-sync/service" "sync" - syncTask "github.com/irisnet/irishub-sync/sync" ) func main() { var wg sync.WaitGroup wg.Add(2) - - syncTask.Start() - + + service.Start() + wg.Wait() } diff --git a/sync/handler/account.go b/service/handler/account.go similarity index 97% rename from sync/handler/account.go rename to service/handler/account.go index 64717c5..19285e6 100644 --- a/sync/handler/account.go +++ b/service/handler/account.go @@ -1,13 +1,13 @@ package handler import ( + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/store" + "github.com/irisnet/irishub-sync/store/document" "github.com/irisnet/irishub-sync/util/constant" - "github.com/irisnet/irishub-sync/model/store/document" - "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/util/helper" "sync" "time" - "github.com/irisnet/irishub-sync/module/logger" - "github.com/irisnet/irishub-sync/util/helper" ) // save account diff --git a/sync/handler/account_test.go b/service/handler/account_test.go similarity index 97% rename from sync/handler/account_test.go rename to service/handler/account_test.go index bf4fd16..1dd6229 100644 --- a/sync/handler/account_test.go +++ b/service/handler/account_test.go @@ -4,7 +4,7 @@ import ( "sync" "testing" - "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/store" ) func TestSaveAccount(t *testing.T) { @@ -72,7 +72,6 @@ func TestUpdateBalance(t *testing.T) { docTxStakeDelegate := buildDocData(47349) docTxStakeUnBond := buildDocData(34241) - type args struct { docTx store.Docs mutex sync.Mutex diff --git a/sync/handler/block.go b/service/handler/block.go similarity index 62% rename from sync/handler/block.go rename to service/handler/block.go index 5893b33..66133fd 100644 --- a/sync/handler/block.go +++ b/service/handler/block.go @@ -1,16 +1,16 @@ package handler import ( - "github.com/irisnet/irishub-sync/util/helper" - "github.com/irisnet/irishub-sync/model/store/document" - "github.com/tendermint/tendermint/types" - "github.com/irisnet/irishub-sync/model/store" - "github.com/irisnet/irishub-sync/module/codec" "encoding/json" + "github.com/irisnet/irishub-sync/module/codec" "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/store" + "github.com/irisnet/irishub-sync/store/document" + "github.com/irisnet/irishub-sync/util/helper" + "github.com/tendermint/tendermint/types" ) -func SaveBlock(meta *types.BlockMeta, block *types.Block, validators []*types.Validator) { +func SaveBlock(meta *types.BlockMeta, block *types.Block, validators []*types.Validator) { hexFunc := func(bytes []byte) string { return helper.BuildHex(bytes) @@ -27,7 +27,7 @@ func SaveBlock(meta *types.BlockMeta, block *types.Block, validators []*types.Va Hash: hexFunc(meta.Header.LastBlockID.Hash), PartsHeader: document.PartSetHeader{ Total: meta.Header.LastBlockID.PartsHeader.Total, - Hash: hexFunc(meta.Header.LastBlockID.PartsHeader.Hash), + Hash: hexFunc(meta.Header.LastBlockID.PartsHeader.Hash), }, } @@ -37,23 +37,23 @@ func SaveBlock(meta *types.BlockMeta, block *types.Block, validators []*types.Va Hash: hexFunc(meta.BlockID.Hash), PartsHeader: document.PartSetHeader{ Total: meta.BlockID.PartsHeader.Total, - Hash: hexFunc(meta.BlockID.PartsHeader.Hash), + Hash: hexFunc(meta.BlockID.PartsHeader.Hash), }, }, Header: document.Header{ - ChainID: meta.Header.ChainID, - Height: meta.Header.Height, - Time: meta.Header.Time, - NumTxs: meta.Header.NumTxs, - LastBlockID: lastBlockId, - TotalTxs: meta.Header.TotalTxs, - LastCommitHash: hexFunc(meta.Header.LastCommitHash), - DataHash: hexFunc(meta.Header.DataHash), - ValidatorsHash: hexFunc(meta.Header.ValidatorsHash), - ConsensusHash: hexFunc(meta.Header.ConsensusHash), - AppHash: hexFunc(meta.Header.AppHash), + ChainID: meta.Header.ChainID, + Height: meta.Header.Height, + Time: meta.Header.Time, + NumTxs: meta.Header.NumTxs, + LastBlockID: lastBlockId, + TotalTxs: meta.Header.TotalTxs, + LastCommitHash: hexFunc(meta.Header.LastCommitHash), + DataHash: hexFunc(meta.Header.DataHash), + ValidatorsHash: hexFunc(meta.Header.ValidatorsHash), + ConsensusHash: hexFunc(meta.Header.ConsensusHash), + AppHash: hexFunc(meta.Header.AppHash), LastResultsHash: hexFunc(meta.Header.LastResultsHash), - EvidenceHash: hexFunc(meta.Header.EvidenceHash), + EvidenceHash: hexFunc(meta.Header.EvidenceHash), }, } @@ -69,13 +69,13 @@ func SaveBlock(meta *types.BlockMeta, block *types.Block, validators []*types.Va json.Unmarshal(out, &sig) preCommit := document.Vote{ ValidatorAddress: v.ValidatorAddress.String(), - ValidatorIndex: v.ValidatorIndex, - Height: v.Height, - Round: v.Round, - Timestamp: v.Timestamp, - Type: v.Type, - BlockID: lastBlockId, - Signature: sig, + ValidatorIndex: v.ValidatorIndex, + Height: v.Height, + Round: v.Round, + Timestamp: v.Timestamp, + Type: v.Type, + BlockID: lastBlockId, + Signature: sig, } preCommits = append(preCommits, preCommit) } @@ -83,7 +83,7 @@ func SaveBlock(meta *types.BlockMeta, block *types.Block, validators []*types.Va blockContent := document.BlockContent{ LastCommit: document.Commit{ - BlockID: lastBlockId, + BlockID: lastBlockId, Precommits: preCommits, }, } @@ -93,16 +93,15 @@ func SaveBlock(meta *types.BlockMeta, block *types.Block, validators []*types.Va if len(validators) > 0 { for _, v := range validators { validator := document.Validator{ - Address: v.Address.String(), + Address: v.Address.String(), VotingPower: v.VotingPower, - Accum: v.Accum, - PubKey: hexFunc(v.PubKey.Bytes()), + Accum: v.Accum, + PubKey: hexFunc(v.PubKey.Bytes()), } vals = append(vals, validator) } } - docBlock.Meta = blockMeta docBlock.Block = blockContent docBlock.Validators = vals diff --git a/sync/handler/block_test.go b/service/handler/block_test.go similarity index 90% rename from sync/handler/block_test.go rename to service/handler/block_test.go index cc07d3b..32928af 100644 --- a/sync/handler/block_test.go +++ b/service/handler/block_test.go @@ -3,9 +3,9 @@ package handler import ( "testing" - "github.com/tendermint/tendermint/types" - "github.com/irisnet/irishub-sync/util/helper" "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/util/helper" + "github.com/tendermint/tendermint/types" ) func buildBlock(blockHeight int64) (*types.BlockMeta, *types.Block, []*types.Validator) { @@ -36,7 +36,7 @@ func TestSaveBlock(t *testing.T) { type args struct { meta *types.BlockMeta block *types.Block - vals []*types.Validator + vals []*types.Validator } tests := []struct { name string @@ -45,25 +45,25 @@ func TestSaveBlock(t *testing.T) { { name: "test save block", args: args{ - meta: meta1, + meta: meta1, block: block1, - vals: vals1, + vals: vals1, }, }, { name: "test save block", args: args{ - meta: meta2, + meta: meta2, block: block2, - vals: vals2, + vals: vals2, }, }, { name: "test save block", args: args{ - meta: meta3, + meta: meta3, block: block3, - vals: vals3, + vals: vals3, }, }, } diff --git a/sync/handler/tx.go b/service/handler/tx.go similarity index 98% rename from sync/handler/tx.go rename to service/handler/tx.go index c6516ec..a5634a2 100644 --- a/sync/handler/tx.go +++ b/service/handler/tx.go @@ -1,15 +1,13 @@ package handler import ( - "github.com/irisnet/irishub-sync/model/store/document" "github.com/irisnet/irishub-sync/module/logger" - "github.com/irisnet/irishub-sync/model/store" - "sync" + "github.com/irisnet/irishub-sync/store" + "github.com/irisnet/irishub-sync/store/document" "github.com/irisnet/irishub-sync/util/constant" + "sync" ) - - // save Tx document into collection func SaveTx(docTx store.Docs, mutex sync.Mutex) { var ( @@ -109,7 +107,7 @@ func SaveTx(docTx store.Docs, mutex sync.Mutex) { if err != nil { // candidate not exist candidate = document.Candidate{ - Address: docTx.ValidatorAddr, + Address: docTx.ValidatorAddr, Description: docTx.Description, } } else { diff --git a/sync/handler/tx_test.go b/service/handler/tx_test.go similarity index 96% rename from sync/handler/tx_test.go rename to service/handler/tx_test.go index 3398e7d..ca78c10 100644 --- a/sync/handler/tx_test.go +++ b/service/handler/tx_test.go @@ -4,13 +4,13 @@ import ( "sync" "testing" - "github.com/irisnet/irishub-sync/model/store" - "github.com/irisnet/irishub-sync/util/helper" - "github.com/irisnet/irishub-sync/module/logger" "github.com/irisnet/irishub-sync/module/codec" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/store" + "github.com/irisnet/irishub-sync/util/helper" ) -func init() { +func init() { helper.InitClientPool() store.Init() } @@ -38,7 +38,6 @@ func buildDocData(blockHeight int64) store.Docs { return nil } - func TestSaveTx(t *testing.T) { //docTxBank := buildDocData(1762) //docTxStakeCreate := buildDocData(46910) diff --git a/sync/handler/types.go b/service/handler/types.go similarity index 93% rename from sync/handler/types.go rename to service/handler/types.go index 1e88017..8607136 100644 --- a/sync/handler/types.go +++ b/service/handler/types.go @@ -1,11 +1,11 @@ package handler import ( - "github.com/irisnet/irishub-sync/model/store" "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/store" + "github.com/irisnet/irishub-sync/util/helper" "reflect" "sync" - "github.com/irisnet/irishub-sync/util/helper" ) // get tx type diff --git a/sync/sync.go b/service/sync.go similarity index 95% rename from sync/sync.go rename to service/sync.go index 1671663..6ac27b4 100644 --- a/sync/sync.go +++ b/service/sync.go @@ -1,20 +1,20 @@ -package sync +package service import ( conf "github.com/irisnet/irishub-sync/conf/server" - "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/module/codec" "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/service/handler" + "github.com/irisnet/irishub-sync/store" "github.com/irisnet/irishub-sync/util/helper" - "github.com/irisnet/irishub-sync/module/codec" - "github.com/irisnet/irishub-sync/sync/handler" - "github.com/irisnet/irishub-sync/model/store/document" + "github.com/irisnet/irishub-sync/store/document" "github.com/robfig/cron" rpcClient "github.com/tendermint/tendermint/rpc/client" ctypes "github.com/tendermint/tendermint/rpc/core/types" - "sync" "github.com/tendermint/tendermint/types" + "sync" ) var ( @@ -24,7 +24,7 @@ var ( // limit max goroutine limitChan = make(chan int64, conf.SyncMaxGoroutine) - mutex sync.Mutex + mutex sync.Mutex mutexWatchBlock sync.Mutex ) @@ -32,12 +32,12 @@ var ( func Start() { var ( status *ctypes.ResultStatus - err error - i = 1 + err error + i = 1 ) Init() c := helper.GetClient().Client - + for { logger.Info.Printf("Begin %v time fast sync task", i) syncLatestHeight := fastSync(c) @@ -51,14 +51,14 @@ func Start() { } } latestHeight := status.SyncInfo.LatestBlockHeight - if syncLatestHeight >= latestHeight - 60 { + if syncLatestHeight >= latestHeight-60 { logger.Info.Println("All fast sync task complete!") break } logger.Info.Printf("End %v time fast sync task", i) i++ } - + startCron(c) } @@ -95,7 +95,7 @@ func startCron(client rpcClient.Client) { func watchBlock(c rpcClient.Client) { mutexWatchBlock.Lock() - + syncTask, _ := document.QuerySyncTask() status, _ := c.Status() latestBlockHeight := status.SyncInfo.LatestBlockHeight @@ -121,7 +121,7 @@ func watchBlock(c rpcClient.Client) { err.Error()) } } - + mutexWatchBlock.Unlock() } @@ -136,7 +136,6 @@ func fastSync(c rpcClient.Client) int64 { } ch := make(chan int64) - goroutineNum := (latestBlockHeight - syncTaskDoc.Height) / syncBlockNumFastSync @@ -188,7 +187,7 @@ end: func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex sync.Mutex), ch chan int64, threadNum int64) { logger.Info.Printf("ThreadNo[%d] begin sync block from %d to %d\n", threadNum, start, end) - + client := helper.GetClient() // release client defer client.Release() @@ -230,16 +229,15 @@ func syncBlock(start int64, end int64, funcChain []func(tx store.Docs, mutex syn validators = res.Validators } - // save block info handler.SaveBlock(block.BlockMeta, block.Block, validators) } - + logger.Info.Printf("ThreadNo[%d] finish sync block from %d to %d\n", threadNum, start, end) - - <- limitChan + + <-limitChan ch <- threadNum logger.Info.Printf("Send threadNum into channel: %v\n", threadNum) - + } diff --git a/sync/sync_test.go b/service/sync_test.go similarity index 95% rename from sync/sync_test.go rename to service/sync_test.go index 49a713a..949da08 100644 --- a/sync/sync_test.go +++ b/service/sync_test.go @@ -1,13 +1,13 @@ -package sync +package service import ( - "testing" "github.com/robfig/cron" - + "testing" + conf "github.com/irisnet/irishub-sync/conf/server" - - "sync" + "github.com/irisnet/irishub-sync/module/logger" + "sync" "time" ) @@ -24,18 +24,17 @@ func TestStart(t *testing.T) { limitChan <- i go func(goroutineNum int, ch chan int) { logger.Info.Println("release limitChan") - <- limitChan + <-limitChan defer func() { logger.Info.Printf("%v goroutine send data to channel\n", goroutineNum) ch <- goroutineNum }() - + }(i, nil) } - - for - { + + for { select { case <-unBufferChan: activeGoroutineNum = activeGoroutineNum - 1 @@ -46,15 +45,14 @@ func TestStart(t *testing.T) { } } } - - + } func Test_startCron(t *testing.T) { var wg sync.WaitGroup var mutex sync.Mutex wg.Add(2) - + spec := conf.SyncCron c := cron.New() c.AddFunc(spec, func() { @@ -66,8 +64,6 @@ func Test_startCron(t *testing.T) { mutex.Unlock() }) go c.Start() - + wg.Wait() } - - diff --git a/model/store/document/account.go b/store/document/account.go similarity index 81% rename from model/store/document/account.go rename to store/document/account.go index 7a9c3e9..a980159 100644 --- a/model/store/document/account.go +++ b/store/document/account.go @@ -1,7 +1,7 @@ package document import ( - "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" @@ -12,10 +12,10 @@ const ( ) type Account struct { - Address string `bson:"address"` + Address string `bson:"address"` Amount store.Coins `bson:"amount"` - Time time.Time `bson:"time"` - Height int64 `bson:"height"` + Time time.Time `bson:"time"` + Height int64 `bson:"height"` } func (a Account) Name() string { @@ -32,7 +32,7 @@ func QueryAccount(address string) (Account, error) { err := c.Find(bson.M{"address": address}).Sort("-amount.amount").One(&result) return err } - + err := store.ExecCollection(CollectionNmAccount, query) if err != nil { diff --git a/model/store/document/block.go b/store/document/block.go similarity index 67% rename from model/store/document/block.go rename to store/document/block.go index 66ecca1..5e373a8 100644 --- a/model/store/document/block.go +++ b/store/document/block.go @@ -10,22 +10,22 @@ const ( ) type Block struct { - Height int64 `bson:"height"` - Hash string `bson:"hash"` - Time time.Time `bson:"time"` - NumTxs int64 `bson:"num_txs"` - Meta BlockMeta `bson:"meta"` - Block BlockContent `bson:"block"` - Validators []Validator `bson:"validators"` + Height int64 `bson:"height"` + Hash string `bson:"hash"` + Time time.Time `bson:"time"` + NumTxs int64 `bson:"num_txs"` + Meta BlockMeta `bson:"meta"` + Block BlockContent `bson:"block"` + Validators []Validator `bson:"validators"` } type BlockMeta struct { BlockID BlockID `bson:"block_id"` - Header Header `bson:"header"` + Header Header `bson:"header"` } type BlockID struct { - Hash string `bson:"hash"` + Hash string `bson:"hash"` PartsHeader PartSetHeader `bson:"parts"` } @@ -60,7 +60,7 @@ type Header struct { } type BlockContent struct { - LastCommit Commit `bson:"last_commit"` + LastCommit Commit `bson:"last_commit"` } type Commit struct { @@ -68,33 +68,32 @@ type Commit struct { // Any peer with a block can gossip precommits by index with a peer without recalculating the // active ValidatorSet. BlockID BlockID `bson:"block_id"` - Precommits []Vote `bson:"precommits"` + Precommits []Vote `bson:"precommits"` } // Represents a prevote, precommit, or commit vote from validators for consensus. type Vote struct { - ValidatorAddress string `bson:"validator_address"` - ValidatorIndex int `bson:"validator_index"` - Height int64 `bson:"height"` - Round int `bson:"round"` - Timestamp time.Time `bson:"timestamp"` - Type byte `bson:"type"` - BlockID BlockID `bson:"block_id"` // zero if vote is nil. + ValidatorAddress string `bson:"validator_address"` + ValidatorIndex int `bson:"validator_index"` + Height int64 `bson:"height"` + Round int `bson:"round"` + Timestamp time.Time `bson:"timestamp"` + Type byte `bson:"type"` + BlockID BlockID `bson:"block_id"` // zero if vote is nil. Signature Signature `bson:"signature"` } type Signature struct { - Type string `bson:"type"` + Type string `bson:"type"` Value string `bson:"value"` } type Validator struct { - Address string `bson:"address"` + Address string `bson:"address"` PubKey string `bson:"pub_key"` - VotingPower int64 `bson:"voting_power"` - Accum int64 `bson:"accum"` -} - + VotingPower int64 `bson:"voting_power"` + Accum int64 `bson:"accum"` +} func (d Block) Name() string { return CollectionNmBlock @@ -102,4 +101,4 @@ func (d Block) Name() string { func (d Block) PkKvPair() map[string]interface{} { return bson.M{"height": d.Height} -} \ No newline at end of file +} diff --git a/model/store/document/doc.go b/store/document/doc.go similarity index 88% rename from model/store/document/doc.go rename to store/document/doc.go index 2e49d38..04ac999 100644 --- a/model/store/document/doc.go +++ b/store/document/doc.go @@ -1,7 +1,7 @@ package document import ( - "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/store" ) func init() { diff --git a/model/store/document/stake_role_candidate.go b/store/document/stake_role_candidate.go similarity index 96% rename from model/store/document/stake_role_candidate.go rename to store/document/stake_role_candidate.go index ee71a80..650107b 100644 --- a/model/store/document/stake_role_candidate.go +++ b/store/document/stake_role_candidate.go @@ -2,11 +2,11 @@ package document import ( "errors" - "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/irisnet/irishub-sync/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" - "github.com/irisnet/irishub-sync/module/logger" ) const ( diff --git a/model/store/document/stake_role_delegator.go b/store/document/stake_role_delegator.go similarity index 95% rename from model/store/document/stake_role_delegator.go rename to store/document/stake_role_delegator.go index a5bd1f2..9f29e60 100644 --- a/model/store/document/stake_role_delegator.go +++ b/store/document/stake_role_delegator.go @@ -2,7 +2,7 @@ package document import ( "errors" - "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" diff --git a/model/store/document/sync_task.go b/store/document/sync_task.go similarity index 93% rename from model/store/document/sync_task.go rename to store/document/sync_task.go index d0b0ce9..62052ba 100644 --- a/model/store/document/sync_task.go +++ b/store/document/sync_task.go @@ -1,7 +1,7 @@ package document import ( - "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/store" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" "time" diff --git a/model/store/document/tx_common.go b/store/document/tx_common.go similarity index 50% rename from model/store/document/tx_common.go rename to store/document/tx_common.go index b239ad1..dd49f71 100644 --- a/model/store/document/tx_common.go +++ b/store/document/tx_common.go @@ -1,9 +1,9 @@ package document import ( - "time" - "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/store" "gopkg.in/mgo.v2/bson" + "time" ) const ( @@ -11,15 +11,15 @@ const ( ) type CommonTx struct { - TxHash string `bson:"tx_hash"` - Time time.Time `bson:"time"` - Height int64 `bson:"height"` - From string `bson:"from"` - To string `bson:"to"` + TxHash string `bson:"tx_hash"` + Time time.Time `bson:"time"` + Height int64 `bson:"height"` + From string `bson:"from"` + To string `bson:"to"` Amount store.Coins `bson:"amount"` - Type string `bson:"type"` - Fee store.Fee `bson:"fee"` - Status string `bson:"status"` + Type string `bson:"type"` + Fee store.Fee `bson:"fee"` + Status string `bson:"status"` } func (d CommonTx) Name() string { diff --git a/model/store/document/tx_stake.go b/store/document/tx_stake.go similarity index 81% rename from model/store/document/tx_stake.go rename to store/document/tx_stake.go index 9971d91..8494d70 100644 --- a/model/store/document/tx_stake.go +++ b/store/document/tx_stake.go @@ -1,15 +1,15 @@ package document import ( - "github.com/irisnet/irishub-sync/model/store" + "github.com/irisnet/irishub-sync/store" "gopkg.in/mgo.v2/bson" "time" ) const ( - CollectionNmStakeTx = "tx_stake" + CollectionNmStakeTx = "tx_stake" CollectionNmStakeTxDeclareCandidacy = CollectionNmStakeTx - CollectionNmStakeTxEditCandidacy = CollectionNmStakeTx + CollectionNmStakeTxEditCandidacy = CollectionNmStakeTx ) // StakeTx @@ -36,7 +36,6 @@ func (c StakeTx) PkKvPair() map[string]interface{} { // ====== - // Description type Description struct { Moniker string `bson:"moniker"` @@ -46,29 +45,29 @@ type Description struct { } type StakeTxDeclareCandidacy struct { - StakeTx `bson:"stake_tx"` + StakeTx `bson:"stake_tx"` Description `bson:"description"` } -func (s StakeTxDeclareCandidacy) Name() string { +func (s StakeTxDeclareCandidacy) Name() string { return CollectionNmStakeTxDeclareCandidacy } -func (s StakeTxDeclareCandidacy) PkKvPair() map[string]interface{} { +func (s StakeTxDeclareCandidacy) PkKvPair() map[string]interface{} { return bson.M{"stake_tx.tx_hash": s.TxHash} } // ====== type StakeTxEditCandidacy struct { - StakeTx `bson:"stake_tx"` + StakeTx `bson:"stake_tx"` Description `bson:"description"` } -func (s StakeTxEditCandidacy) Name() string { +func (s StakeTxEditCandidacy) Name() string { return CollectionNmStakeTxEditCandidacy } -func (s StakeTxEditCandidacy) PkKvPair() map[string]interface{} { +func (s StakeTxEditCandidacy) PkKvPair() map[string]interface{} { return bson.M{"stake_tx.tx_hash": s.TxHash} } diff --git a/model/store/store.go b/store/store.go similarity index 100% rename from model/store/store.go rename to store/store.go diff --git a/model/store/types.go b/store/types.go similarity index 95% rename from model/store/types.go rename to store/types.go index a7d46da..eaea9c1 100644 --- a/model/store/types.go +++ b/store/types.go @@ -18,5 +18,5 @@ type Coins []Coin type Fee struct { Amount Coins - Gas int64 -} \ No newline at end of file + Gas int64 +} diff --git a/util/helper/account.go b/util/helper/account.go index 00d397d..6913db8 100644 --- a/util/helper/account.go +++ b/util/helper/account.go @@ -3,17 +3,17 @@ package helper import ( - "github.com/irisnet/irishub-sync/model/store" "github.com/irisnet/irishub-sync/module/codec" + "github.com/irisnet/irishub-sync/store" - rpcclient "github.com/tendermint/tendermint/rpc/client" - cmn "github.com/tendermint/tmlibs/common" + "fmt" sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/pkg/errors" "github.com/cosmos/cosmos-sdk/x/auth" - "fmt" - "github.com/irisnet/irishub-sync/module/logger" authcmd "github.com/cosmos/cosmos-sdk/x/auth/client/cli" + "github.com/irisnet/irishub-sync/module/logger" + "github.com/pkg/errors" + rpcclient "github.com/tendermint/tendermint/rpc/client" + cmn "github.com/tendermint/tmlibs/common" ) func QueryAccountBalance(address string) store.Coins { @@ -67,4 +67,3 @@ func query(key cmn.HexBytes, storeName string, endPath string) (res []byte, err } return resp.Value, nil } - diff --git a/util/helper/tx.go b/util/helper/tx.go index 5fb926d..4bdc00d 100644 --- a/util/helper/tx.go +++ b/util/helper/tx.go @@ -3,28 +3,28 @@ package helper import ( - "github.com/irisnet/irishub-sync/model/store/document" + "github.com/irisnet/irishub-sync/store/document" - "github.com/tendermint/tendermint/types" + "encoding/hex" + sdktypes "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/wire" "github.com/cosmos/cosmos-sdk/x/auth" "github.com/cosmos/cosmos-sdk/x/bank" "github.com/cosmos/cosmos-sdk/x/stake" - sdktypes "github.com/cosmos/cosmos-sdk/types" "github.com/irisnet/irishub-sync/module/logger" - "github.com/irisnet/irishub-sync/model/store" - "strings" - "encoding/hex" + "github.com/irisnet/irishub-sync/store" "github.com/irisnet/irishub-sync/util/constant" + "github.com/tendermint/tendermint/types" "strconv" + "strings" ) type ( - msgBankSend = bank.MsgSend - msgStakeCreate = stake.MsgCreateValidator - msgStakeEdit = stake.MsgEditValidator + msgBankSend = bank.MsgSend + msgStakeCreate = stake.MsgCreateValidator + msgStakeEdit = stake.MsgEditValidator msgStakeDelegate = stake.MsgDelegate - msgStakeUnbond = stake.MsgUnbond + msgStakeUnbond = stake.MsgUnbond ) func ParseTx(cdc *wire.Codec, txBytes types.Tx, block *types.Block) store.Docs { @@ -47,11 +47,11 @@ func ParseTx(cdc *wire.Codec, txBytes types.Tx, block *types.Block) store.Docs { switch authTx.GetMsg().(type) { case msgBankSend: msg := authTx.Msg.(msgBankSend) - docTx := document.CommonTx { + docTx := document.CommonTx{ Height: height, - Time: time, + Time: time, TxHash: txHash, - Fee: fee, + Fee: fee, Status: status, } docTx.From = msg.Inputs[0].Address.String() @@ -63,9 +63,9 @@ func ParseTx(cdc *wire.Codec, txBytes types.Tx, block *types.Block) store.Docs { msg := authTx.Msg.(msgStakeCreate) stakeTx := document.StakeTx{ Height: height, - Time: time, + Time: time, TxHash: txHash, - Fee: fee, + Fee: fee, Status: status, } stakeTx.ValidatorAddr = msg.ValidatorAddr.String() @@ -73,14 +73,14 @@ func ParseTx(cdc *wire.Codec, txBytes types.Tx, block *types.Block) store.Docs { stakeTx.Amount = buildCoin(msg.Bond) description := document.Description{ - Moniker: msg.Moniker, + Moniker: msg.Moniker, Identity: msg.Identity, - Website: msg.Website, - Details: msg.Details, + Website: msg.Website, + Details: msg.Details, } docTx := document.StakeTxDeclareCandidacy{ - StakeTx: stakeTx, + StakeTx: stakeTx, Description: description, } docTx.Type = constant.TxTypeStakeCreate @@ -89,22 +89,22 @@ func ParseTx(cdc *wire.Codec, txBytes types.Tx, block *types.Block) store.Docs { msg := authTx.Msg.(msgStakeEdit) stakeTx := document.StakeTx{ Height: height, - Time: time, + Time: time, TxHash: txHash, - Fee: fee, + Fee: fee, Status: status, } stakeTx.ValidatorAddr = msg.ValidatorAddr.String() description := document.Description{ - Moniker: msg.Moniker, + Moniker: msg.Moniker, Identity: msg.Identity, - Website: msg.Website, - Details: msg.Details, + Website: msg.Website, + Details: msg.Details, } docTx := document.StakeTxEditCandidacy{ - StakeTx: stakeTx, + StakeTx: stakeTx, Description: description, } docTx.Type = constant.TxTypeStakeEdit @@ -113,9 +113,9 @@ func ParseTx(cdc *wire.Codec, txBytes types.Tx, block *types.Block) store.Docs { msg := authTx.Msg.(msgStakeDelegate) docTx := document.StakeTx{ Height: height, - Time: time, + Time: time, TxHash: txHash, - Fee: fee, + Fee: fee, Status: status, } docTx.DelegatorAddr = msg.DelegatorAddr.String() @@ -131,9 +131,9 @@ func ParseTx(cdc *wire.Codec, txBytes types.Tx, block *types.Block) store.Docs { } docTx := document.StakeTx{ Height: height, - Time: time, + Time: time, TxHash: txHash, - Fee: fee, + Fee: fee, Status: status, } docTx.DelegatorAddr = msg.DelegatorAddr.String() @@ -150,7 +150,6 @@ func ParseTx(cdc *wire.Codec, txBytes types.Tx, block *types.Block) store.Docs { return nil } - func BuildCoins(coins sdktypes.Coins) store.Coins { var ( localCoins store.Coins @@ -167,7 +166,7 @@ func BuildCoins(coins sdktypes.Coins) store.Coins { func buildCoin(coin sdktypes.Coin) store.Coin { return store.Coin{ - Denom: coin.Denom, + Denom: coin.Denom, Amount: coin.Amount, } } @@ -179,6 +178,6 @@ func buildFee(fee auth.StdFee) store.Fee { } } -func BuildHex(bytes []byte) string { +func BuildHex(bytes []byte) string { return strings.ToUpper(hex.EncodeToString(bytes)) } From 1724fc4d020f5e7b064f919b9e73d7e25993d860 Mon Sep 17 00:00:00 2001 From: kaifei Date: Mon, 9 Jul 2018 15:29:51 +0800 Subject: [PATCH 28/37] add dockerfile support --- Dockerfile | 38 +++++++++++++++ Makefile | 43 +++++++++++++++++ README.md | 9 +--- docker/docker-compose.yml | 18 ++++++++ module/codec/codec.go | 1 - mongodb/mongodb.js | 4 +- service/handler/tx_test.go | 64 +++++++++++++------------- store/document/stake_role_delegator.go | 6 +-- tools/Makefile | 33 +++++++++++++ version.info | 0 10 files changed, 171 insertions(+), 45 deletions(-) create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 docker/docker-compose.yml create mode 100644 tools/Makefile delete mode 100644 version.info diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ff4f202 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +FROM alpine:edge + +# Set up dependencies +ENV PACKAGES go make git libc-dev bash + +# Set up GOPATH & PATH + +ENV PROJECT_NAME irishub-sync +ENV GOPATH /root/go +ENV BASE_PATH $GOPATH/src/github.com/irisnet +ENV REPO_PATH $BASE_PATH/$PROJECT_NAME +ENV LOG_DIR /$PROJECT_NAME/log +ENV PATH $GOPATH/bin:$PATH + +# Set volumes + +VOLUME $LOG_DIR + +# Link expected Go repo path + +RUN mkdir -p $GOPATH/pkg $GOPATH/bin $BASE_PATH $REPO_PATH $LOG_DIR + +# Add source files + +COPY . $REPO_PATH + +# Install minimum necessary dependencies, build irishub-server +RUN apk add --no-cache $PACKAGES && \ + cd $REPO_PATH && make all && \ + mv $REPO_PATH/$PROJECT_NAME $GOPATH/bin && \ + rm -rf $REPO_PATH/vendor && \ + rm -rf $GOPATH/src/github.com/golang $GOPATH/bin/dep $GOPATH/pkg/* && \ + apk del $PACKAGES + +VOLUME ["$LOG_DIR"] + + +CMD irishub-sync > $LOG_DIR/debug.log && tail -f $LOG_DIR/debug.log \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5fe6801 --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +GOCMD=go +GOBUILD=$(GOCMD) build +GOCLEAN=$(GOCMD) clean +GOTEST=$(GOCMD) test +GOGET=$(GOCMD) get +BINARY_NAME=irishub-sync +BINARY_UNIX=$(BINARY_NAME)-unix + +all: get_tools get_deps build + +get_deps: + @rm -rf vendor/ + @echo "--> Running dep ensure" + @dep ensure -v + +build: + $(GOBUILD) -o $(BINARY_NAME) -v + +clean: + $(GOCLEAN) + rm -f $(BINARY_NAME) + rm -f $(BINARY_UNIX) + +run: + $(GOBUILD) -o $(BINARY_NAME) -v + ./$(BINARY_NAME) + + +# Cross compilation +build-linux: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) -v + +###################################### +## Tools + +check_tools: + cd tools && $(MAKE) check_tools + +get_tools: + cd tools && $(MAKE) get_tools + +update_tools: + cd tools && $(MAKE) update_tools \ No newline at end of file diff --git a/README.md b/README.md index eb92d23..4905b6e 100644 --- a/README.md +++ b/README.md @@ -4,20 +4,15 @@ A server that synchronize IRIS blockChain data into a database # Structure - `conf`: config of project -- `model`: database model - `module`: project module - `mongodb`: mongodb script to create database -- `sync`: main logic of sync-server, sync data from blockChain and write to database +- `service`: main logic of sync-server, sync data from blockChain and write to database +- `store`: database model - `util`: common constants and helper functions - `main.go`: bootstrap project # SetUp -## Rewrite config file - -1. rename `/conf/db/type.go.example` to `/conf/db/type.go`, `/conf/server/type.go.example` to `/conf/db/type.go` -2. write your own config into `/conf/db/type.go` and `/conf/db/type.go` - ## Create mongodb database run script `mongodb.js` in `mongodb` folder to create database before run project diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..e6c2689 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,18 @@ +version: "2" +services: + irishub-server: + image: irisnet/irishub-sync:develop + container_name: c_irishub-sync-develop + volumes: + - /mnt/data/irishub-sync/log:/irishub-sync/log + environment: + ENV: dev + DB_HOST: 127.0.0.1 + DB_PORT: 27117 + DB_DATABASE: sync_iris + + SER_BC_NODE_URL: tcp://127.0.0.1:46657 + SER_BC_CHAIN_ID: test + SER_BC_TOKEN: iris + SER_MAX_GOROUTINE: 60 + SER_SYNC_BLOCK_NUM: 8000 \ No newline at end of file diff --git a/module/codec/codec.go b/module/codec/codec.go index 47efc98..cd171ee 100644 --- a/module/codec/codec.go +++ b/module/codec/codec.go @@ -24,7 +24,6 @@ func init() { auth.RegisterWire(Cdc) sdktypes.RegisterWire(Cdc) - //ctypes.RegisterAmino(Cdc) wire.RegisterCrypto(Cdc) } diff --git a/mongodb/mongodb.js b/mongodb/mongodb.js index 5a88204..60f96e0 100644 --- a/mongodb/mongodb.js +++ b/mongodb/mongodb.js @@ -14,9 +14,9 @@ db.block.createIndex({"height": -1}, {"unique": true}); db.stake_role_candidate.createIndex({"address": 1}, {"unique": true}); db.stake_role_candidate.createIndex({"pub_key": 1}); -db.stake_role_delegator.createIndex({"pub_key": 1}); +db.stake_role_delegator.createIndex({"validator_addr": 1}); db.stake_role_delegator.createIndex({"address": 1}); -db.stake_role_delegator.createIndex({"address": 1, "pub_key": 1}, {"unique": true}); +db.stake_role_delegator.createIndex({"address": 1, "validator_addr": 1}, {"unique": true}); db.sync_task.createIndex({"chain_id": 1}, {"unique": true}); diff --git a/service/handler/tx_test.go b/service/handler/tx_test.go index ca78c10..0080daf 100644 --- a/service/handler/tx_test.go +++ b/service/handler/tx_test.go @@ -39,10 +39,10 @@ func buildDocData(blockHeight int64) store.Docs { } func TestSaveTx(t *testing.T) { - //docTxBank := buildDocData(1762) - //docTxStakeCreate := buildDocData(46910) - //docTxStakeEdit := buildDocData(49388) - //docTxStakeDelegate := buildDocData(47349) + docTxBank := buildDocData(1762) + docTxStakeCreate := buildDocData(46910) + docTxStakeEdit := buildDocData(49388) + docTxStakeDelegate := buildDocData(47349) docTxStakeUnBond := buildDocData(96319) type args struct { @@ -53,34 +53,34 @@ func TestSaveTx(t *testing.T) { name string args args }{ - //{ - // name: "tx bank", - // args: args{ - // docTx: docTxBank, - // mutex: sync.Mutex{}, - // }, - //}, - //{ - // name: "tx stake/create", - // args: args{ - // docTx: docTxStakeCreate, - // mutex: sync.Mutex{}, - // }, - //}, - //{ - // name: "tx stake/edit", - // args: args{ - // docTx: docTxStakeEdit, - // mutex: sync.Mutex{}, - // }, - //}, - //{ - // name: "tx stake/delegate", - // args: args{ - // docTx: docTxStakeDelegate, - // mutex: sync.Mutex{}, - // }, - //}, + { + name: "tx bank", + args: args{ + docTx: docTxBank, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/create", + args: args{ + docTx: docTxStakeCreate, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/edit", + args: args{ + docTx: docTxStakeEdit, + mutex: sync.Mutex{}, + }, + }, + { + name: "tx stake/delegate", + args: args{ + docTx: docTxStakeDelegate, + mutex: sync.Mutex{}, + }, + }, { name: "tx stake/unbond", args: args{ diff --git a/store/document/stake_role_delegator.go b/store/document/stake_role_delegator.go index 9f29e60..d6ac316 100644 --- a/store/document/stake_role_delegator.go +++ b/store/document/stake_role_delegator.go @@ -14,7 +14,7 @@ const ( type Delegator struct { Address string `bson:"address"` - ValidatorAddr string `bson:"pub_key"` // validatorAddr + ValidatorAddr string `bson:"validator_addr"` // validatorAddr Shares int64 `bson:"shares"` UpdateTime time.Time `bson:"update_time"` } @@ -24,13 +24,13 @@ func (d Delegator) Name() string { } func (d Delegator) PkKvPair() map[string]interface{} { - return bson.M{"address": d.Address, "pub_key": d.ValidatorAddr} + return bson.M{"address": d.Address, "validator_addr": d.ValidatorAddr} } func QueryDelegatorByAddressAndValAddr(address string, valAddr string) (Delegator, error) { var result Delegator query := func(c *mgo.Collection) error { - err := c.Find(bson.M{"address": address, "pub_key": valAddr}).Sort("-shares").One(&result) + err := c.Find(bson.M{"address": address, "validator_addr": valAddr}).Sort("-shares").One(&result) return err } diff --git a/tools/Makefile b/tools/Makefile new file mode 100644 index 0000000..08e0cbc --- /dev/null +++ b/tools/Makefile @@ -0,0 +1,33 @@ +######################################## +### DEP + +DEP = github.com/golang/dep/cmd/dep +DEP_CHECK := $(shell command -v dep 2> /dev/null) + +check_tools: +ifndef DEP_CHECK + @echo "No dep in path. Install with 'make get_tools'." +else + @echo "Found dep in path." +endif + +get_tools: +ifdef DEP_CHECK + @echo "Dep is already installed. Run 'make update_tools' to update." +else + @echo "$(ansi_grn)Installing dep$(ansi_end)" + go get -v $(DEP) +endif + +update_tools: + @echo "$(ansi_grn)Updating dep$(ansi_end)" + go get -u -v $(DEP) + + +######################################## +# ANSI colors + +ansi_red=\033[0;31m +ansi_grn=\033[0;32m +ansi_yel=\033[0;33m +ansi_end=\033[0m \ No newline at end of file diff --git a/version.info b/version.info deleted file mode 100644 index e69de29..0000000 From c920ed8cad2904d7063808116b27500900d82d0e Mon Sep 17 00:00:00 2001 From: kaifei Date: Mon, 9 Jul 2018 17:24:18 +0800 Subject: [PATCH 29/37] change db name --- conf/db/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/db/types.go b/conf/db/types.go index 4fa96e9..04fe281 100644 --- a/conf/db/types.go +++ b/conf/db/types.go @@ -11,7 +11,7 @@ var ( Port = "27117" User = "" Passwd = "" - Database = "sync-iris-dev" + Database = "sync_iris" ) // get value of env var From 2d077def8e045cd0d5adfc688e89f3682c5ebeaf Mon Sep 17 00:00:00 2001 From: kaifei Date: Tue, 10 Jul 2018 11:22:11 +0800 Subject: [PATCH 30/37] add authInfo when connect mongodb --- conf/db/types.go | 10 ++++----- service/handler/tx_test.go | 2 +- service/sync.go | 2 +- store/store.go | 46 +++++++++++++++++++++----------------- store/store_test.go | 20 +++++++++++++++++ 5 files changed, 52 insertions(+), 28 deletions(-) create mode 100644 store/store_test.go diff --git a/conf/db/types.go b/conf/db/types.go index 04fe281..5bb2290 100644 --- a/conf/db/types.go +++ b/conf/db/types.go @@ -7,11 +7,11 @@ import ( ) var ( - Host = "127.0.0.1" - Port = "27117" - User = "" - Passwd = "" - Database = "sync_iris" + Host = "116.62.62.39" + Port = "27217" + User = "irishub" + Passwd = "bianjie.ai" + Database = "sync_irishub" ) // get value of env var diff --git a/service/handler/tx_test.go b/service/handler/tx_test.go index 0080daf..b949c4a 100644 --- a/service/handler/tx_test.go +++ b/service/handler/tx_test.go @@ -12,7 +12,7 @@ import ( func init() { helper.InitClientPool() - store.Init() + store.InitWithAuth() } func buildDocData(blockHeight int64) store.Docs { diff --git a/service/sync.go b/service/sync.go index 6ac27b4..e1ff783 100644 --- a/service/sync.go +++ b/service/sync.go @@ -63,7 +63,7 @@ func Start() { } func Init() { - store.Init() + store.InitWithAuth() chainId := conf.ChainId syncTask, err := document.QuerySyncTask() diff --git a/store/store.go b/store/store.go index f0f222f..2fd7188 100644 --- a/store/store.go +++ b/store/store.go @@ -23,35 +23,39 @@ func RegisterDocs(d Docs) { docs = append(docs, d) } -func Init() { - if session == nil { - url := fmt.Sprintf("mongodb://%s:%s", conf.Host, conf.Port) +//func Init() { +// if session == nil { +// url := fmt.Sprintf("mongodb://%s:%s", conf.Host, conf.Port) +// +// var err error +// session, err = mgo.Dial(url) +// if err != nil { +// logger.Error.Fatalln(err) +// } +// logger.Info.Printf("Mgo start on %s\n", url) +// session.SetMode(mgo.Monotonic, true) +// } +//} + +func InitWithAuth() { + addr := fmt.Sprintf("%s:%s", conf.Host, conf.Port) + addrs := []string{addr} - var err error - session, err = mgo.Dial(url) - if err != nil { - logger.Error.Fatalln(err) - } - logger.Info.Printf("Mgo start on %s\n", url) - session.SetMode(mgo.Monotonic, true) - } -} - -func InitWithAuth(addrs []string, username, password string) { dialInfo := &mgo.DialInfo{ Addrs: addrs, // []string{"192.168.6.122"} - Direct: false, - Timeout: time.Second * 1, Database: conf.Database, - Username: username, - Password: password, + Username: conf.User, + Password: conf.Passwd, + Direct: false, + Timeout: time.Second * 10, PoolLimit: 4096, // Session.SetPoolLimit } - session, err := mgo.DialWithInfo(dialInfo) + var err error + session, err = mgo.DialWithInfo(dialInfo) session.SetMode(mgo.Monotonic, true) - if nil != err { - panic(err) + if err != nil { + logger.Error.Panicln(err) } } diff --git a/store/store_test.go b/store/store_test.go new file mode 100644 index 0000000..bd1d857 --- /dev/null +++ b/store/store_test.go @@ -0,0 +1,20 @@ +// init mongodb session and provide common functions + +package store + +import "testing" + +func TestInitWithAuth(t *testing.T) { + tests := []struct { + name string + }{ + { + name: "tets initWithAuth", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + InitWithAuth() + }) + } +} From 62546b59364f9455602f85ea7948bccbc50fc109 Mon Sep 17 00:00:00 2001 From: kaifei Date: Tue, 10 Jul 2018 11:22:11 +0800 Subject: [PATCH 31/37] add authInfo when connect mongodb --- conf/db/types.go | 8 +++---- service/handler/tx_test.go | 2 +- service/sync.go | 2 +- store/store.go | 46 +++++++++++++++++++++----------------- store/store_test.go | 20 +++++++++++++++++ 5 files changed, 51 insertions(+), 27 deletions(-) create mode 100644 store/store_test.go diff --git a/conf/db/types.go b/conf/db/types.go index 04fe281..d7fe50c 100644 --- a/conf/db/types.go +++ b/conf/db/types.go @@ -8,10 +8,10 @@ import ( var ( Host = "127.0.0.1" - Port = "27117" - User = "" - Passwd = "" - Database = "sync_iris" + Port = "27217" + User = "user" + Passwd = "passwd" + Database = "sync_irishub" ) // get value of env var diff --git a/service/handler/tx_test.go b/service/handler/tx_test.go index 0080daf..b949c4a 100644 --- a/service/handler/tx_test.go +++ b/service/handler/tx_test.go @@ -12,7 +12,7 @@ import ( func init() { helper.InitClientPool() - store.Init() + store.InitWithAuth() } func buildDocData(blockHeight int64) store.Docs { diff --git a/service/sync.go b/service/sync.go index 6ac27b4..e1ff783 100644 --- a/service/sync.go +++ b/service/sync.go @@ -63,7 +63,7 @@ func Start() { } func Init() { - store.Init() + store.InitWithAuth() chainId := conf.ChainId syncTask, err := document.QuerySyncTask() diff --git a/store/store.go b/store/store.go index f0f222f..2fd7188 100644 --- a/store/store.go +++ b/store/store.go @@ -23,35 +23,39 @@ func RegisterDocs(d Docs) { docs = append(docs, d) } -func Init() { - if session == nil { - url := fmt.Sprintf("mongodb://%s:%s", conf.Host, conf.Port) +//func Init() { +// if session == nil { +// url := fmt.Sprintf("mongodb://%s:%s", conf.Host, conf.Port) +// +// var err error +// session, err = mgo.Dial(url) +// if err != nil { +// logger.Error.Fatalln(err) +// } +// logger.Info.Printf("Mgo start on %s\n", url) +// session.SetMode(mgo.Monotonic, true) +// } +//} + +func InitWithAuth() { + addr := fmt.Sprintf("%s:%s", conf.Host, conf.Port) + addrs := []string{addr} - var err error - session, err = mgo.Dial(url) - if err != nil { - logger.Error.Fatalln(err) - } - logger.Info.Printf("Mgo start on %s\n", url) - session.SetMode(mgo.Monotonic, true) - } -} - -func InitWithAuth(addrs []string, username, password string) { dialInfo := &mgo.DialInfo{ Addrs: addrs, // []string{"192.168.6.122"} - Direct: false, - Timeout: time.Second * 1, Database: conf.Database, - Username: username, - Password: password, + Username: conf.User, + Password: conf.Passwd, + Direct: false, + Timeout: time.Second * 10, PoolLimit: 4096, // Session.SetPoolLimit } - session, err := mgo.DialWithInfo(dialInfo) + var err error + session, err = mgo.DialWithInfo(dialInfo) session.SetMode(mgo.Monotonic, true) - if nil != err { - panic(err) + if err != nil { + logger.Error.Panicln(err) } } diff --git a/store/store_test.go b/store/store_test.go new file mode 100644 index 0000000..bd1d857 --- /dev/null +++ b/store/store_test.go @@ -0,0 +1,20 @@ +// init mongodb session and provide common functions + +package store + +import "testing" + +func TestInitWithAuth(t *testing.T) { + tests := []struct { + name string + }{ + { + name: "tets initWithAuth", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + InitWithAuth() + }) + } +} From 837b578d3693c0f36e98fefe5755abe946f4a9ff Mon Sep 17 00:00:00 2001 From: kaifei Date: Tue, 10 Jul 2018 15:14:48 +0800 Subject: [PATCH 32/37] update cosmos-sdk version to v0.19.1-rc1 --- Gopkg.lock | 32 +++++++++++++++++--------------- Gopkg.toml | 28 ++++++++++++++++++++-------- docker/docker-compose.yml | 3 +++ 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 666b0e2..3290a8b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -11,13 +11,13 @@ branch = "master" name = "github.com/btcsuite/btcd" packages = ["btcec"] - revision = "86fed781132ac890ee03e906e4ecd5d6fa180c64" + revision = "fdfc19097e7ac6b57035062056f5b7b4638b8898" [[projects]] branch = "master" name = "github.com/btcsuite/btcutil" packages = ["bech32"] - revision = "d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4" + revision = "ab6388e0c60ae4834a1f57511e20c17b5f78be4b" [[projects]] name = "github.com/cosmos/cosmos-sdk" @@ -35,8 +35,8 @@ "x/slashing", "x/stake" ] - revision = "c6711810a86f09457481a8dadae899681a9d77ab" - version = "v0.19.0" + revision = "1e6d26ad3dce18fd1dde9bbee7d2aa192c196310" + version = "v0.19.1-rc1" [[projects]] name = "github.com/davecgh/go-spew" @@ -281,7 +281,7 @@ "leveldb/table", "leveldb/util" ] - revision = "0d5a0ceb10cf9ab89fdd744cc8c50a83134f6697" + revision = "c4c61651e9e37fa117f53c5a906d3b63090d8445" [[projects]] name = "github.com/tendermint/abci" @@ -291,8 +291,8 @@ "example/kvstore", "types" ] - revision = "ebee2fe114020aa49c70bbbae50b7079fc7e7b90" - version = "v0.11.0" + revision = "198dccf0ddfd1bb176f87657e3286a05a6ed9540" + version = "v0.12.0" [[projects]] branch = "master" @@ -371,8 +371,8 @@ "types", "version" ] - revision = "27bd1deabe4ba6a2d9b463b8f3e3f1e31b993e61" - version = "v0.20.0" + revision = "46369a1ab76f274ab47179c4176221842b8207b4" + version = "v0.21.0" [[projects]] name = "github.com/tendermint/tmlibs" @@ -389,7 +389,8 @@ "merkle", "merkle/tmhash" ] - revision = "0c98d10b4ffbd87978d79c160e835b3d3df241ec" + revision = "49596e0a1f48866603813df843c9409fc19805c6" + version = "v0.9.0" [[projects]] branch = "master" @@ -420,13 +421,13 @@ "internal/timeseries", "trace" ] - revision = "ed29d75add3d7c4bf7ca65aac0c6df3d1420216f" + revision = "292b43bbf7cb8d35ddf40f8d5100ef3837cced3f" [[projects]] branch = "master" name = "golang.org/x/sys" packages = ["unix"] - revision = "7138fd3d9dc8335c567ca206f4333fb75eb05d56" + revision = "1b2967e3c290b7c545b3db0deeda16e9be4f98a2" [[projects]] name = "golang.org/x/text" @@ -450,9 +451,10 @@ version = "v0.3.0" [[projects]] + branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200" + revision = "e92b116572682a5b432ddd840aeaba2a559eeff1" [[projects]] name = "google.golang.org/grpc" @@ -488,7 +490,7 @@ "internal/sasl", "internal/scram" ] - revision = "3f83fa5005286a7fe593b055f0d7771a7dce4655" + revision = "9856a29383ce1c59f308dd1cf0363a79b5bef6b5" [[projects]] name = "gopkg.in/yaml.v2" @@ -499,6 +501,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "52551a8a6af4cebfff972e9f1fbcf2e71200436239d0f89004fa1ae9896f85a1" + inputs-digest = "058e6efc878a71337af7122715fe77e78f545a8add40243f00d755950c2afed9" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 24e9e22..906dfb7 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -31,19 +31,31 @@ [[constraint]] name = "github.com/cosmos/cosmos-sdk" - version = "v0.19.0" + version = "v0.19.1-rc1" -[[constraint]] - name = "github.com/tendermint/tendermint" - version = "=0.20.0" +[[override]] + name = "github.com/tendermint/abci" + version = "=0.12.0" [[override]] - name = "github.com/tendermint/tmlibs" - revision = "0c98d10b4ffbd87978d79c160e835b3d3df241ec" + name = "github.com/tendermint/go-crypto" + version = "=0.6.2" + +[[override]] + name = "github.com/tendermint/go-amino" + version = "=0.9.9" [[override]] - name = "google.golang.org/genproto" - revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200" + name = "github.com/tendermint/iavl" + version = "=0.8.0-rc0" + +[[override]] + name = "github.com/tendermint/tendermint" + version = "=0.21.0" + +[[override]] + name = "github.com/tendermint/tmlibs" + version = "=v0.9.0" [[constraint]] branch = "v2" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e6c2689..254c9f5 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -9,8 +9,11 @@ services: ENV: dev DB_HOST: 127.0.0.1 DB_PORT: 27117 + DB_USER: user + DB_PASSWD: passwd DB_DATABASE: sync_iris + SER_BC_NODE_URL: tcp://127.0.0.1:46657 SER_BC_CHAIN_ID: test SER_BC_TOKEN: iris From 8477d34527fb8e29f4ff180afac51d135b9f019e Mon Sep 17 00:00:00 2001 From: kaifei Date: Thu, 12 Jul 2018 10:23:53 +0800 Subject: [PATCH 33/37] update sdk to 0.19.1-rc1 --- Gopkg.lock | 32 +++++++++++++++++--------------- Gopkg.toml | 28 ++++++++++++++++++++-------- docker/docker-compose.yml | 4 +++- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 666b0e2..1fad626 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -11,13 +11,13 @@ branch = "master" name = "github.com/btcsuite/btcd" packages = ["btcec"] - revision = "86fed781132ac890ee03e906e4ecd5d6fa180c64" + revision = "fdfc19097e7ac6b57035062056f5b7b4638b8898" [[projects]] branch = "master" name = "github.com/btcsuite/btcutil" packages = ["bech32"] - revision = "d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4" + revision = "ab6388e0c60ae4834a1f57511e20c17b5f78be4b" [[projects]] name = "github.com/cosmos/cosmos-sdk" @@ -35,8 +35,8 @@ "x/slashing", "x/stake" ] - revision = "c6711810a86f09457481a8dadae899681a9d77ab" - version = "v0.19.0" + revision = "1e6d26ad3dce18fd1dde9bbee7d2aa192c196310" + version = "v0.19.1-rc1" [[projects]] name = "github.com/davecgh/go-spew" @@ -281,7 +281,7 @@ "leveldb/table", "leveldb/util" ] - revision = "0d5a0ceb10cf9ab89fdd744cc8c50a83134f6697" + revision = "c4c61651e9e37fa117f53c5a906d3b63090d8445" [[projects]] name = "github.com/tendermint/abci" @@ -291,8 +291,8 @@ "example/kvstore", "types" ] - revision = "ebee2fe114020aa49c70bbbae50b7079fc7e7b90" - version = "v0.11.0" + revision = "198dccf0ddfd1bb176f87657e3286a05a6ed9540" + version = "v0.12.0" [[projects]] branch = "master" @@ -371,8 +371,8 @@ "types", "version" ] - revision = "27bd1deabe4ba6a2d9b463b8f3e3f1e31b993e61" - version = "v0.20.0" + revision = "46369a1ab76f274ab47179c4176221842b8207b4" + version = "v0.21.0" [[projects]] name = "github.com/tendermint/tmlibs" @@ -389,7 +389,8 @@ "merkle", "merkle/tmhash" ] - revision = "0c98d10b4ffbd87978d79c160e835b3d3df241ec" + revision = "49596e0a1f48866603813df843c9409fc19805c6" + version = "v0.9.0" [[projects]] branch = "master" @@ -420,13 +421,13 @@ "internal/timeseries", "trace" ] - revision = "ed29d75add3d7c4bf7ca65aac0c6df3d1420216f" + revision = "039a4258aec0ad3c79b905677cceeab13b296a77" [[projects]] branch = "master" name = "golang.org/x/sys" packages = ["unix"] - revision = "7138fd3d9dc8335c567ca206f4333fb75eb05d56" + revision = "1b2967e3c290b7c545b3db0deeda16e9be4f98a2" [[projects]] name = "golang.org/x/text" @@ -450,9 +451,10 @@ version = "v0.3.0" [[projects]] + branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200" + revision = "e92b116572682a5b432ddd840aeaba2a559eeff1" [[projects]] name = "google.golang.org/grpc" @@ -488,7 +490,7 @@ "internal/sasl", "internal/scram" ] - revision = "3f83fa5005286a7fe593b055f0d7771a7dce4655" + revision = "9856a29383ce1c59f308dd1cf0363a79b5bef6b5" [[projects]] name = "gopkg.in/yaml.v2" @@ -499,6 +501,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "52551a8a6af4cebfff972e9f1fbcf2e71200436239d0f89004fa1ae9896f85a1" + inputs-digest = "058e6efc878a71337af7122715fe77e78f545a8add40243f00d755950c2afed9" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 24e9e22..906dfb7 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -31,19 +31,31 @@ [[constraint]] name = "github.com/cosmos/cosmos-sdk" - version = "v0.19.0" + version = "v0.19.1-rc1" -[[constraint]] - name = "github.com/tendermint/tendermint" - version = "=0.20.0" +[[override]] + name = "github.com/tendermint/abci" + version = "=0.12.0" [[override]] - name = "github.com/tendermint/tmlibs" - revision = "0c98d10b4ffbd87978d79c160e835b3d3df241ec" + name = "github.com/tendermint/go-crypto" + version = "=0.6.2" + +[[override]] + name = "github.com/tendermint/go-amino" + version = "=0.9.9" [[override]] - name = "google.golang.org/genproto" - revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200" + name = "github.com/tendermint/iavl" + version = "=0.8.0-rc0" + +[[override]] + name = "github.com/tendermint/tendermint" + version = "=0.21.0" + +[[override]] + name = "github.com/tendermint/tmlibs" + version = "=v0.9.0" [[constraint]] branch = "v2" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e6c2689..fdbd91a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -9,7 +9,9 @@ services: ENV: dev DB_HOST: 127.0.0.1 DB_PORT: 27117 - DB_DATABASE: sync_iris + DB_USER: user + DB_PASSWD: passwd + DB_DATABASE: sync_irishub SER_BC_NODE_URL: tcp://127.0.0.1:46657 SER_BC_CHAIN_ID: test From 101f5d6b3c91c2fc4e27fc9f3f06f9ecbf4a0b1d Mon Sep 17 00:00:00 2001 From: kaifei Date: Thu, 12 Jul 2018 13:42:59 +0800 Subject: [PATCH 34/37] add denpendences --- Gopkg.lock | 7 +++---- Gopkg.toml | 5 +++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 3290a8b..52d18d6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -421,7 +421,7 @@ "internal/timeseries", "trace" ] - revision = "292b43bbf7cb8d35ddf40f8d5100ef3837cced3f" + revision = "039a4258aec0ad3c79b905677cceeab13b296a77" [[projects]] branch = "master" @@ -451,10 +451,9 @@ version = "v0.3.0" [[projects]] - branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "e92b116572682a5b432ddd840aeaba2a559eeff1" + revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200" [[projects]] name = "google.golang.org/grpc" @@ -501,6 +500,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "058e6efc878a71337af7122715fe77e78f545a8add40243f00d755950c2afed9" + inputs-digest = "165c8f6c97ce24d4415c9721fa0e86eccd01edd9a77268153465ddbc15ef378a" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 906dfb7..e051a91 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -57,6 +57,11 @@ name = "github.com/tendermint/tmlibs" version = "=v0.9.0" +# this got updated and broke, so locked to an old working commit ... +[[override]] + name = "google.golang.org/genproto" + revision = "7fd901a49ba6a7f87732eb344f6e3c5b19d1b200" + [[constraint]] branch = "v2" name = "gopkg.in/mgo.v2" From 602fc5ece0eee6be0bc2403e11a88cf5595c16c4 Mon Sep 17 00:00:00 2001 From: kaifei Date: Thu, 12 Jul 2018 14:11:46 +0800 Subject: [PATCH 35/37] delete drop scripts --- mongodb/mongodb.js | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/mongodb/mongodb.js b/mongodb/mongodb.js index 60f96e0..152e3bb 100644 --- a/mongodb/mongodb.js +++ b/mongodb/mongodb.js @@ -41,22 +41,22 @@ db.tx_common.createIndex({"type": 1}); db.tx_common.createIndex({"status": 1}); // drop collection -db.account.drop(); -db.block.drop(); -db.stake_role_candidate.drop(); -db.stake_role_delegator.drop(); -db.sync_task.drop(); -db.tx_stake.drop(); -db.tx_common.drop(); - +// db.account.drop(); +// db.block.drop(); +// db.stake_role_candidate.drop(); +// db.stake_role_delegator.drop(); +// db.sync_task.drop(); +// db.tx_stake.drop(); +// db.tx_common.drop(); +// // remove collection data -db.account.remove({}); -db.block.remove({}); -db.stake_role_candidate.remove({}); -db.stake_role_delegator.remove({}); -db.sync_task.remove({}); -db.tx_stake.remove({}); -db.tx_common.remove({}); +// db.account.remove({}); +// db.block.remove({}); +// db.stake_role_candidate.remove({}); +// db.stake_role_delegator.remove({}); +// db.sync_task.remove({}); +// db.tx_stake.remove({}); +// db.tx_common.remove({}); From 2ac7705d2fc73277092286a38ca7e94df52a4247 Mon Sep 17 00:00:00 2001 From: kaifei Date: Fri, 13 Jul 2018 17:51:12 +0800 Subject: [PATCH 36/37] fix address of from and to --- service/handler/tx.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/handler/tx.go b/service/handler/tx.go index a5634a2..743e16c 100644 --- a/service/handler/tx.go +++ b/service/handler/tx.go @@ -271,8 +271,8 @@ func buildCommonTxData(docTx store.Docs, txType string) document.CommonTx { TxHash: doc.TxHash, Time: doc.Time, Height: doc.Height, - From: doc.ValidatorAddr, - To: doc.DelegatorAddr, + From: doc.DelegatorAddr, + To: doc.ValidatorAddr, Amount: []store.Coin{doc.Amount}, Type: doc.Type, Fee: doc.Fee, From 8ea32eab4d5840711eab90ab66bd9cbf90398a55 Mon Sep 17 00:00:00 2001 From: Yelong Zhang Date: Thu, 19 Jul 2018 15:26:46 +0800 Subject: [PATCH 37/37] Create CHANGELOG.md --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6d335b5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,15 @@ +## 0.2.0 + +*July 19th, 2018* + +BREAKING CHANGES: + +- Switched from glide to dep internally for package management + +- Upgrade dependencies of cosmos-sdk to v0.19.1-rc1 and tendermint to 0.21.0 + +- [service/handler] Breaking changes to cosmos-sdk and tendermint will be reflected in the resorving and storing of account, block and tx b + +FEATURES: + +- [service/handler] Sync more information about each block and validator set