From 3ba209977a3dc33dae98861acd9ee42ea1e85936 Mon Sep 17 00:00:00 2001 From: Denis Hananein Date: Sun, 21 May 2023 13:03:04 +0000 Subject: [PATCH 01/38] Init lite client from config file --- liteclient/config.go | 15 +++++++++++++++ liteclient/connection.go | 11 ++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/liteclient/config.go b/liteclient/config.go index dbe9fada..670d26dd 100644 --- a/liteclient/config.go +++ b/liteclient/config.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "net/http" + "os" "strconv" ) @@ -37,6 +38,20 @@ func GetConfigFromUrl(ctx context.Context, url string) (*GlobalConfig, error) { return config, nil } +func GetConfigFromFile(filepath string) (*GlobalConfig, error) { + config := &GlobalConfig{} + + configData, err := os.ReadFile(filepath) + if err != nil { + return nil, err + } + if err := json.Unmarshal(configData, config); err != nil { + return nil, err + } + + return config, nil +} + func intToIP4(ipInt int64) string { b0 := strconv.FormatInt((ipInt>>24)&0xff, 10) b1 := strconv.FormatInt((ipInt>>16)&0xff, 10) diff --git a/liteclient/connection.go b/liteclient/connection.go index 25a6dab1..a2400ffc 100644 --- a/liteclient/connection.go +++ b/liteclient/connection.go @@ -12,7 +12,6 @@ import ( "encoding/hex" "errors" "fmt" - "github.com/xssnick/tonutils-go/adnl" "hash/crc32" "io" "log" @@ -23,6 +22,7 @@ import ( "sync/atomic" "time" + "github.com/xssnick/tonutils-go/adnl" "github.com/xssnick/tonutils-go/tl" ) @@ -86,6 +86,15 @@ func (c *ConnectionPool) AddConnectionsFromConfig(ctx context.Context, config *G return <-result } +func (c *ConnectionPool) AddConnectionsFromConfigFile(configPath string) error { + config, err := GetConfigFromFile(configPath) + if err != nil { + return err + } + + return c.AddConnectionsFromConfig(context.Background(), config) +} + func (c *ConnectionPool) AddConnectionsFromConfigUrl(ctx context.Context, configUrl string) error { config, err := GetConfigFromUrl(ctx, configUrl) if err != nil { From 1cc5332bf1e497a803276201dadd4f2105e8ac0c Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 29 May 2023 10:31:28 +0400 Subject: [PATCH 02/38] Moved storage impl to dff repo & hash levels lazy recalc --- adnl/overlay/manager-adnl.go | 1 + adnl/overlay/manager-rldp.go | 1 + adnl/rldp/client.go | 5 + adnl/storage/client.go | 747 ----------------------------------- adnl/storage/storage.go | 169 -------- tvm/cell/proof.go | 6 +- 6 files changed, 10 insertions(+), 919 deletions(-) delete mode 100644 adnl/storage/client.go delete mode 100644 adnl/storage/storage.go diff --git a/adnl/overlay/manager-adnl.go b/adnl/overlay/manager-adnl.go index 67523d57..15ed6082 100644 --- a/adnl/overlay/manager-adnl.go +++ b/adnl/overlay/manager-adnl.go @@ -27,6 +27,7 @@ type ADNL interface { Query(ctx context.Context, req, result tl.Serializable) error Answer(ctx context.Context, queryID []byte, result tl.Serializable) error RemoteAddr() string + GetID() []byte Close() } diff --git a/adnl/overlay/manager-rldp.go b/adnl/overlay/manager-rldp.go index b951ab04..09b69146 100644 --- a/adnl/overlay/manager-rldp.go +++ b/adnl/overlay/manager-rldp.go @@ -10,6 +10,7 @@ import ( ) type RLDP interface { + GetADNL() rldp.ADNL Close() DoQuery(ctx context.Context, maxAnswerSize int64, query, result tl.Serializable) error SetOnQuery(handler func(transferId []byte, query *rldp.Query) error) diff --git a/adnl/rldp/client.go b/adnl/rldp/client.go index 3f7b1280..27adfcf2 100644 --- a/adnl/rldp/client.go +++ b/adnl/rldp/client.go @@ -16,6 +16,7 @@ import ( ) type ADNL interface { + GetID() []byte SetCustomMessageHandler(handler func(msg *adnl.MessageCustom) error) SetDisconnectHandler(handler func(addr string, key ed25519.PublicKey)) SendCustomMessage(ctx context.Context, req tl.Serializable) error @@ -77,6 +78,10 @@ func NewClientV2(a ADNL) *RLDP { return c } +func (r *RLDP) GetADNL() ADNL { + return r.adnl +} + func (r *RLDP) SetOnQuery(handler func(transferId []byte, query *Query) error) { r.onQuery = handler } diff --git a/adnl/storage/client.go b/adnl/storage/client.go deleted file mode 100644 index f8300b1e..00000000 --- a/adnl/storage/client.go +++ /dev/null @@ -1,747 +0,0 @@ -package storage - -import ( - "bytes" - "context" - "crypto/ed25519" - "crypto/sha256" - "encoding/hex" - "fmt" - "github.com/xssnick/tonutils-go/adnl" - "github.com/xssnick/tonutils-go/adnl/address" - "github.com/xssnick/tonutils-go/adnl/dht" - "github.com/xssnick/tonutils-go/adnl/overlay" - "github.com/xssnick/tonutils-go/adnl/rldp" - "github.com/xssnick/tonutils-go/tl" - "github.com/xssnick/tonutils-go/tlb" - "github.com/xssnick/tonutils-go/tvm/cell" - "math" - "reflect" - "sync" - "sync/atomic" - "time" -) - -var Logger = func(a ...any) {} - -type DHT interface { - StoreAddress(ctx context.Context, addresses address.List, ttl time.Duration, ownerKey ed25519.PrivateKey, copies int) (int, []byte, error) - FindAddresses(ctx context.Context, key []byte) (*address.List, ed25519.PublicKey, error) - FindOverlayNodes(ctx context.Context, overlayId []byte, continuation ...*dht.Continuation) (*overlay.NodesList, *dht.Continuation, error) - Close() -} - -type FileInfo struct { - Size uint64 - FromPiece uint32 - ToPiece uint32 - FromPieceOffset uint32 - ToPieceOffset uint32 -} - -type TorrentDownloader interface { - GetDescription() string - GetDirName() string - ListFiles() []string - GetFileOffsets(name string) *FileInfo - DownloadPiece(ctx context.Context, pieceIndex uint32) (_ []byte, err error) - SetDesiredMinNodesNum(num int) - Close() -} - -type Client struct { - dht DHT -} - -func NewClient(dht DHT) *Client { - return &Client{ - dht: dht, - } -} - -type torrentDownloader struct { - bagId []byte - piecesNum uint32 - filesIndex map[string]uint32 - dirName string - - desiredMinPeersNum int - threadsPerPeer int - - info *TorrentInfo - header *TorrentHeader - client *Client - knownNodes map[string]*overlay.Node - activeNodes map[string]*storageNode - - mx sync.RWMutex - - pieceQueue chan *pieceRequest - - globalCtx context.Context - downloadCancel func() -} - -type pieceResponse struct { - index int32 - data []byte - err error -} - -type pieceRequest struct { - index int32 - ctx context.Context - result chan<- pieceResponse -} - -type storageNode struct { - torrent *torrentDownloader - sessionId int64 - rawAdnl overlay.ADNL - rldp *overlay.RLDPOverlayWrapper - - globalCtx context.Context -} - -// fec_info_none#c82a1964 = FecInfo; -// -// torrent_header#9128aab7 -// files_count:uint32 -// tot_name_size:uint64 -// tot_data_size:uint64 -// fec:FecInfo -// dir_name_size:uint32 -// dir_name:(dir_name_size * [uint8]) -// name_index:(files_count * [uint64]) -// data_index:(files_count * [uint64]) -// names:(file_names_size * [uint8]) -// data:(tot_data_size * [uint8]) -// = TorrentHeader; -// -// Filename rules: -// 1) Name can't be empty -// 2) Names in a torrent should be unique -// 3) Name can't start or end with '/' or contain two consequitive '/' -// 4) Components of name can't be equal to "." or ".." -// 5) If there's a name aaa/bbb/ccc, no other name can start with aaa/bbb/ccc/ - -// torrent_info piece_size:uint32 file_size:uint64 root_hash:(## 256) header_size:uint64 header_hash:(## 256) -// description:Text = TorrentInfo; - -type TorrentInfo struct { - PieceSize uint32 `tlb:"## 32"` - FileSize uint64 `tlb:"## 64"` - RootHash []byte `tlb:"bits 256"` - HeaderSize uint64 `tlb:"## 64"` - HeaderHash []byte `tlb:"bits 256"` - Description tlb.Text `tlb:"."` -} - -func (c *Client) CreateDownloader(ctx context.Context, bagId []byte, desiredMinPeersNum, threadsPerPeer int, attempts ...int) (_ TorrentDownloader, err error) { - - globalCtx, downloadCancel := context.WithCancel(context.Background()) - var dow = &torrentDownloader{ - client: c, - bagId: bagId, - activeNodes: map[string]*storageNode{}, - knownNodes: map[string]*overlay.Node{}, - globalCtx: globalCtx, - downloadCancel: downloadCancel, - pieceQueue: make(chan *pieceRequest, 50), - desiredMinPeersNum: desiredMinPeersNum, - threadsPerPeer: threadsPerPeer, - } - defer func() { - if err != nil { - downloadCancel() - } - }() - - // connect to first node - err = dow.scale(ctx, 1, 2) - if err != nil { - err = fmt.Errorf("failed to find storage nodes for this bag, err: %w", err) - return nil, err - } - - if dow.info.PieceSize == 0 || dow.info.HeaderSize == 0 { - err = fmt.Errorf("incorrect torrent info sizes") - return nil, err - } - if dow.info.HeaderSize > 20*1024*1024 { - err = fmt.Errorf("too big header > 20 MB, looks dangerous") - return nil, err - } - go dow.scaleController() - - hdrPieces := dow.info.HeaderSize / uint64(dow.info.PieceSize) - if dow.info.HeaderSize%uint64(dow.info.PieceSize) > 0 { - // add not full piece - hdrPieces += 1 - } - - dow.piecesNum = uint32(dow.info.FileSize / uint64(dow.info.PieceSize)) - if dow.info.FileSize%uint64(dow.info.PieceSize) > 0 { - dow.piecesNum += 1 - } - - data := make([]byte, 0, hdrPieces*uint64(dow.info.PieceSize)) - for i := uint32(0); i < uint32(hdrPieces); i++ { - piece, pieceErr := dow.DownloadPiece(ctx, i) - if pieceErr != nil { - err = fmt.Errorf("failed to get header piece %d, err: %w", i, pieceErr) - return nil, err - } - data = append(data, piece...) - // TODO: data piece part save - } - - var header TorrentHeader - data, err = tl.Parse(&header, data, true) - if err != nil { - err = fmt.Errorf("failed to load header from cell, err: %w", err) - return nil, err - } - dow.header = &header - - dow.dirName = string(header.DirName) - if header.FilesCount > 1_000_000 { - return nil, fmt.Errorf("bag has > 1_000_000 files, looks dangerous") - } - if uint32(len(header.NameIndex)) != header.FilesCount || - uint32(len(header.DataIndex)) != header.FilesCount { - err = fmt.Errorf("corrupted header, lack of files info") - return nil, err - } - - dow.filesIndex = map[string]uint32{} - for i := uint32(0); i < header.FilesCount; i++ { - if uint64(len(header.Names)) < header.NameIndex[i] { - err = fmt.Errorf("corrupted header, too short names data") - return nil, err - } - if dow.info.FileSize < header.DataIndex[i]+dow.info.HeaderSize { - err = fmt.Errorf("corrupted header, data out of range") - return nil, err - } - - nameFrom := uint64(0) - if i > 0 { - nameFrom = header.NameIndex[i-1] - } - name := header.Names[nameFrom:header.NameIndex[i]] - dow.filesIndex[string(name)] = i - } - - return dow, nil -} - -func (s *storageNode) Close() { - s.rawAdnl.Close() -} - -func (s *storageNode) loop() { - defer s.Close() - - fails := 0 - for { - var req *pieceRequest - select { - case <-s.globalCtx.Done(): - return - case req = <-s.torrent.pieceQueue: - } - - resp := pieceResponse{ - index: req.index, - } - - var piece Piece - resp.err = func() error { - reqCtx, cancel := context.WithTimeout(req.ctx, 7*time.Second) - err := s.rldp.DoQuery(reqCtx, 4096+int64(s.torrent.info.PieceSize)*3, &GetPiece{req.index}, &piece) - cancel() - if err != nil { - return fmt.Errorf("failed to query piece %d. err: %w", req.index, err) - } - - proof, err := cell.FromBOC(piece.Proof) - if err != nil { - return fmt.Errorf("failed to parse BoC of piece %d, err: %w", req.index, err) - } - - err = cell.CheckProof(proof, s.torrent.info.RootHash) - if err != nil { - return fmt.Errorf("proof check of piece %d failed: %w", req.index, err) - } - - err = s.torrent.checkProofBranch(proof, piece.Data, uint32(req.index)) - if err != nil { - time.Sleep(50 * time.Millisecond) - return fmt.Errorf("proof branch check of piece %d failed: %w", req.index, err) - } - return nil - }() - if resp.err == nil { - fails = 0 - resp.data = piece.Data - } else { - Logger("[DOWNLOADER] LOAD PIECE FROM", s.rawAdnl.RemoteAddr(), "ERR:", resp.err.Error()) - - fails++ - } - req.result <- resp - - if fails > 3 { - // something wrong, close connection, we should reconnect after it - return - } - - if resp.err != nil { - select { - case <-s.globalCtx.Done(): - return - case <-time.After(500 * time.Millisecond): - // TODO: take down all loops - // take loop down for some time, to allow other nodes to pickup piece - } - } - } -} - -func (t *torrentDownloader) connectToNode(ctx context.Context, adnlID []byte, node *overlay.Node, onDisconnect func()) (*storageNode, error) { - addrs, keyN, err := t.client.dht.FindAddresses(ctx, adnlID) - if err != nil { - return nil, fmt.Errorf("failed to find node address: %w", err) - } - - ax, err := adnl.Connect(ctx, addrs.Addresses[0].IP.String()+":"+fmt.Sprint(addrs.Addresses[0].Port), keyN, nil) - if err != nil { - return nil, fmt.Errorf("failed to connnect to node: %w", err) - } - extADNL := overlay.CreateExtendedADNL(ax) - rl := overlay.CreateExtendedRLDP(rldp.NewClientV2(extADNL)).CreateOverlay(node.Overlay) - - var sessionReady = make(chan int64, 1) - var ready bool - var readyMx sync.Mutex - rl.SetOnQuery(func(transferId []byte, query *rldp.Query) error { - ctx, cancel := context.WithTimeout(t.globalCtx, 30*time.Second) - defer cancel() - - switch q := query.Data.(type) { - case Ping: - err = rl.SendAnswer(ctx, query.MaxAnswerSize, query.ID, transferId, &Pong{}) - if err != nil { - return err - } - - readyMx.Lock() - if !ready { - var status Ok - err = rl.DoQuery(ctx, 1<<25, &AddUpdate{ - SessionID: q.SessionID, - Seqno: 1, - Update: UpdateInit{ - HavePieces: nil, - HavePiecesOffset: 0, - State: State{ - WillUpload: false, - WantDownload: true, - }, - }, - }, &status) - if err == nil { // if err - we will try again on next ping - ready = true - sessionReady <- q.SessionID - } - } - readyMx.Unlock() - case AddUpdate: - // do nothing with this info for now, just ok - err = rl.SendAnswer(ctx, query.MaxAnswerSize, query.ID, transferId, &Ok{}) - if err != nil { - return err - } - default: - return fmt.Errorf("unexpected rldp query received by storage cliet: %s", reflect.ValueOf(q).String()) - } - return nil - }) - - var res TorrentInfoContainer - err = rl.DoQuery(ctx, 1<<25, &GetTorrentInfo{}, &res) - if err != nil { - return nil, fmt.Errorf("failed to get torrent info: %w", err) - } - - cl, err := cell.FromBOC(res.Data) - if err != nil { - return nil, fmt.Errorf("failed to parse torrent info boc: %w", err) - } - - if !bytes.Equal(cl.Hash(), t.bagId) { - return nil, fmt.Errorf("incorrect torrent info") - } - - nodeCtx, cancel := context.WithCancel(t.globalCtx) - stNode := &storageNode{ - rawAdnl: ax, - rldp: rl, - globalCtx: nodeCtx, - torrent: t, - } - rl.SetOnDisconnect(func() { - cancel() - onDisconnect() - }) - - t.mx.Lock() - if t.info == nil { - var info TorrentInfo - err = tlb.LoadFromCell(&info, cl.BeginParse()) - if err != nil { - t.mx.Unlock() - ax.Close() - return nil, fmt.Errorf("invalid torrent info cell") - } - t.info = &info - } - t.mx.Unlock() - - // query first piece to be sure node is ready for downloading - for { - Logger("[SCALER] TRY LOAD PIECE FROM", addrs.Addresses[0].IP.String()) - qCtx, cancelC := context.WithTimeout(ctx, 7*time.Second) - var piece Piece - err = rl.DoQuery(qCtx, 4096+int64(t.info.PieceSize)*3, &GetPiece{0}, &piece) - cancelC() - if err != nil { - select { - case <-ctx.Done(): - ax.Close() - return nil, fmt.Errorf("failed to query first peice, err: %w", err) - case <-time.After(1 * time.Second): - } - continue - } - - Logger("[SCALER] GOT PIECE FROM", addrs.Addresses[0].IP.String(), "CONNECTED!") - - break - } - - select { - case id := <-sessionReady: - stNode.sessionId = id - return stNode, nil - case <-ctx.Done(): - // close connection and all related overlays - ax.Close() - return nil, ctx.Err() - } -} - -func (t *torrentDownloader) GetFileOffsets(name string) *FileInfo { - i, ok := t.filesIndex[name] - if !ok { - return nil - } - info := &FileInfo{} - - var end = t.header.DataIndex[i] - var start uint64 = 0 - if i > 0 { - start = t.header.DataIndex[i-1] - } - info.FromPiece = uint32((t.info.HeaderSize + start) / uint64(t.info.PieceSize)) - info.ToPiece = uint32((t.info.HeaderSize + end) / uint64(t.info.PieceSize)) - info.FromPieceOffset = uint32((t.info.HeaderSize + start) - uint64(info.FromPiece)*uint64(t.info.PieceSize)) - info.ToPieceOffset = uint32((t.info.HeaderSize + end) - uint64(info.ToPiece)*uint64(t.info.PieceSize)) - info.Size = (uint64(info.ToPiece-info.FromPiece)*uint64(t.info.PieceSize) + uint64(info.ToPieceOffset)) - uint64(info.FromPieceOffset) - return info -} - -// DownloadPiece - downloads piece from one of available nodes. -// Can be used concurrently to download from multiple nodes in the same time -func (t *torrentDownloader) DownloadPiece(ctx context.Context, pieceIndex uint32) (_ []byte, err error) { - resp := make(chan pieceResponse, 1) - req := pieceRequest{ - index: int32(pieceIndex), - ctx: ctx, - result: resp, - } - - for { - select { - case <-ctx.Done(): - return nil, ctx.Err() - case t.pieceQueue <- &req: - } - - select { - case <-ctx.Done(): - return nil, ctx.Err() - case result := <-resp: - if result.err != nil { - continue - // return nil, fmt.Errorf("failed to query piece %d after retries: %w", pieceIndex, err) - } - return result.data, nil - } - } -} - -func (t *torrentDownloader) checkProofBranch(proof *cell.Cell, data []byte, piece uint32) error { - if piece >= t.piecesNum { - return fmt.Errorf("too big piece") - } - - tree, err := proof.BeginParse().LoadRef() - if err != nil { - return err - } - - // calc tree depth - depth := int(math.Log2(float64(t.piecesNum))) - if t.piecesNum > uint32(math.Pow(2, float64(depth))) { - // add 1 if pieces num is not exact log2 - depth++ - } - - // check bits from left to right and load branches - for i := depth - 1; i >= 0; i-- { - isLeft := piece&(1< 0 { - timer := time.After(40 * time.Second) - waiter: - for { - select { - case connected := <-connections: - if connected { - num-- - } - - if num <= 0 { - // we scaled enough - return nil - } - case <-scaleDone: - break waiter - // all connection attempts finished - case <-timer: - // timeout for connections, lets try to find more nodes - break waiter - } - } - } - - var err error - var nodes *overlay.NodesList - - ctxFind, cancel := context.WithTimeout(ctx, 15*time.Second) - nodes, nodesDhtCont, err = t.client.dht.FindOverlayNodes(ctxFind, t.bagId, nodesDhtCont) - cancel() - if err != nil { - attempts-- - if attempts == 0 { - return fmt.Errorf("no nodes found") - } - - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(1 * time.Second): - continue - } - } - - for i := range nodes.List { - id, err := adnl.ToKeyID(nodes.List[i].ID) - if err != nil { - continue - } - // add known nodes in case we will need them in future to scale - t.knownNodes[hex.EncodeToString(id)] = &nodes.List[i] - } - } -} - -func (t *torrentDownloader) scaleController() { - for { - select { - case <-t.globalCtx.Done(): - return - case <-time.After(2 * time.Second): - } - - t.mx.RLock() - peersNum := len(t.activeNodes) - t.mx.RUnlock() - - if peersNum < t.desiredMinPeersNum { - _ = t.scale(t.globalCtx, t.desiredMinPeersNum-peersNum, 3) - } - } -} - -func (t *torrentDownloader) SetDesiredMinNodesNum(num int) { - t.desiredMinPeersNum = num -} - -func (t *torrentDownloader) ListFiles() []string { - files := make([]string, 0, len(t.filesIndex)) - for s := range t.filesIndex { - files = append(files, s) - } - return files -} - -func (t *torrentDownloader) Close() { - t.downloadCancel() -} - -func (t *torrentDownloader) GetDirName() string { - return string(t.header.DirName) -} - -func (t *torrentDownloader) GetDescription() string { - return t.info.Description.Value -} diff --git a/adnl/storage/storage.go b/adnl/storage/storage.go deleted file mode 100644 index 937ee539..00000000 --- a/adnl/storage/storage.go +++ /dev/null @@ -1,169 +0,0 @@ -package storage - -import ( - "encoding/binary" - "fmt" - "github.com/xssnick/tonutils-go/tl" -) - -func init() { - tl.Register(TorrentInfoContainer{}, "storage.torrentInfo data:bytes = storage.TorrentInfo") - tl.Register(GetTorrentInfo{}, "storage.getTorrentInfo = storage.TorrentInfo") - tl.Register(Piece{}, "storage.piece proof:bytes data:bytes = storage.Piece") - tl.Register(GetPiece{}, "storage.getPiece piece_id:int = storage.Piece") - tl.Register(Ping{}, "storage.ping session_id:long = storage.Pong") - tl.Register(Pong{}, "storage.pong = storage.Pong") - tl.Register(AddUpdate{}, "storage.addUpdate session_id:long seqno:int update:storage.Update = Ok") - tl.Register(State{}, "storage.state will_upload:Bool want_download:Bool = storage.State") - tl.Register(UpdateInit{}, "storage.updateInit have_pieces:bytes have_pieces_offset:int state:storage.State = storage.Update") - tl.Register(UpdateHavePieces{}, "storage.updateHavePieces piece_id:(vector int) = storage.Update") - tl.Register(UpdateState{}, "storage.updateState state:storage.State = storage.Update") - tl.Register(Ok{}, "storage.ok = Ok") - - tl.Register(FECInfoNone{}, "fec_info_none#c82a1964 = FecInfo") - tl.Register(TorrentHeader{}, "torrent_header#9128aab7 files_count:uint32 "+ - "tot_name_size:uint64 tot_data_size:uint64 fec:FecInfo "+ - "dir_name_size:uint32 dir_name:(dir_name_size * [uint8]) "+ - "name_index:(files_count * [uint64]) data_index:(files_count * [uint64]) "+ - "names:(file_names_size * [uint8]) data:(tot_data_size * [uint8]) "+ - "= TorrentHeader") -} - -type AddUpdate struct { - SessionID int64 `tl:"long"` - Seqno int64 `tl:"int"` - Update any `tl:"struct boxed [storage.updateInit,storage.updateHavePieces,storage.updateState]"` -} - -type TorrentInfoContainer struct { - Data []byte `tl:"bytes"` -} - -type GetTorrentInfo struct{} - -type Piece struct { - Proof []byte `tl:"bytes"` - Data []byte `tl:"bytes"` -} - -type GetPiece struct { - PieceID int32 `tl:"int"` -} - -type Ping struct { - SessionID int64 `tl:"long"` -} - -type Pong struct{} - -type State struct { - WillUpload bool `tl:"bool"` - WantDownload bool `tl:"bool"` -} - -type UpdateInit struct { - HavePieces []byte `tl:"bytes"` - HavePiecesOffset int32 `tl:"int"` - State State `tl:"struct boxed"` -} - -type UpdateHavePieces struct { - PieceIDs []int32 `tl:"vector int"` -} - -type UpdateState struct { - State State `tl:"struct boxed"` -} - -type Ok struct{} - -type FECInfoNone struct{} - -type TorrentHeader struct { - FilesCount uint32 - TotalNameSize uint64 - TotalDataSize uint64 - FEC FECInfoNone - DirNameSize uint32 - DirName []byte - NameIndex []uint64 - DataIndex []uint64 - Names []byte - Data []byte -} - -func (t *TorrentHeader) Parse(data []byte) (_ []byte, err error) { - // Manual parse because of not standard array definition - if len(data) < 28 { - return nil, fmt.Errorf("too short sizes data to parse") - } - t.FilesCount = binary.LittleEndian.Uint32(data) - data = data[4:] - t.TotalNameSize = binary.LittleEndian.Uint64(data) - data = data[8:] - t.TotalDataSize = binary.LittleEndian.Uint64(data) - data = data[8:] - data, err = tl.Parse(&t.FEC, data, true) - if err != nil { - return nil, fmt.Errorf("failed to parse fec: %w", err) - } - t.DirNameSize = binary.LittleEndian.Uint32(data) - data = data[4:] - - if uint64(len(data)) < uint64(t.DirNameSize)+uint64(t.FilesCount*8*2)+t.TotalNameSize+t.TotalDataSize { - return nil, fmt.Errorf("too short arrays data to parse") - } - - t.DirName = data[:t.DirNameSize] - data = data[t.DirNameSize:] - - for i := uint32(0); i < t.FilesCount; i++ { - t.NameIndex = append(t.NameIndex, binary.LittleEndian.Uint64(data[i*8:])) - t.DataIndex = append(t.DataIndex, binary.LittleEndian.Uint64(data[t.FilesCount*8+i*8:])) - } - data = data[t.FilesCount*8*2:] - - t.Names = data[:t.TotalNameSize] - data = data[t.TotalNameSize:] - t.Data = data[:t.TotalDataSize] - data = data[t.TotalDataSize:] - return data, nil -} - -func (t *TorrentHeader) Serialize() ([]byte, error) { - data := make([]byte, 20) - binary.LittleEndian.PutUint32(data[0:], t.FilesCount) - binary.LittleEndian.PutUint64(data[4:], t.TotalNameSize) - binary.LittleEndian.PutUint64(data[12:], t.TotalDataSize) - - fecData, err := tl.Serialize(t.FEC, true) - if err != nil { - return nil, err - } - data = append(data, fecData...) - - if t.DirNameSize != uint32(len(t.DirName)) { - return nil, fmt.Errorf("incorrect dir name size") - } - - dataDirNameSz := make([]byte, 4) - binary.LittleEndian.PutUint32(dataDirNameSz, t.DirNameSize) - data = append(data, dataDirNameSz...) - data = append(data, t.DirName...) - - for _, ni := range t.NameIndex { - iData := make([]byte, 8) - binary.LittleEndian.PutUint64(iData, ni) - data = append(data, iData...) - } - - for _, ni := range t.DataIndex { - iData := make([]byte, 8) - binary.LittleEndian.PutUint64(iData, ni) - data = append(data, iData...) - } - data = append(data, t.Names...) - data = append(data, t.Data...) - - return data, nil -} diff --git a/tvm/cell/proof.go b/tvm/cell/proof.go index 0b463c18..02517477 100644 --- a/tvm/cell/proof.go +++ b/tvm/cell/proof.go @@ -10,7 +10,7 @@ import ( type cellHash = []byte func (c *Cell) CreateProof(forHashes [][]byte) (*Cell, error) { - proofBody := c.copy() // TODO: optimize + proofBody := c.copy() hasParts, err := proofBody.toProof(forHashes) if err != nil { return nil, fmt.Errorf("failed to build proof for cell: %w", err) @@ -159,7 +159,7 @@ func (c *Cell) getHash(level int) []byte { } // lazy hash calc - if c.hashes == nil { + if len(c.hashes) <= hashIndex*32 { c.calculateHashes() } @@ -273,7 +273,7 @@ func (c *Cell) getDepth(level int) uint16 { } // lazy hash calc - if c.hashes == nil { + if len(c.depthLevels) <= hashIndex { c.calculateHashes() } From b7bd1ee14a16b1bb6e9e827afc8f02bae64b7d1f Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Tue, 30 May 2023 10:36:38 +0400 Subject: [PATCH 03/38] ADNL Query packet retry experiment --- adnl/adnl.go | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/adnl/adnl.go b/adnl/adnl.go index 3b13f876..4b8706ee 100644 --- a/adnl/adnl.go +++ b/adnl/adnl.go @@ -397,23 +397,26 @@ func (a *ADNL) query(ctx context.Context, ch *Channel, req, result tl.Serializab a.activeQueries[reqID] = res a.mx.Unlock() - if err = a.sendRequestMaySplit(ctx, ch, q); err != nil { - a.mx.Lock() - delete(a.activeQueries, reqID) - a.mx.Unlock() + for { + if err = a.sendRequestMaySplit(ctx, ch, q); err != nil { + a.mx.Lock() + delete(a.activeQueries, reqID) + a.mx.Unlock() - return fmt.Errorf("request failed: %w", err) - } + return fmt.Errorf("request failed: %w", err) + } - select { - case resp := <-res: - if err, ok := resp.(error); ok { - return err + select { + case resp := <-res: + if err, ok := resp.(error); ok { + return err + } + reflect.ValueOf(result).Elem().Set(reflect.ValueOf(resp)) + return nil + case <-ctx.Done(): + return fmt.Errorf("deadline exceeded, addr %s %s, err: %w", a.addr, hex.EncodeToString(a.peerKey), ctx.Err()) + case <-time.After(100 * time.Millisecond): } - reflect.ValueOf(result).Elem().Set(reflect.ValueOf(resp)) - return nil - case <-ctx.Done(): - return fmt.Errorf("deadline exceeded, addr %s %s, err: %w", a.addr, hex.EncodeToString(a.peerKey), ctx.Err()) } } From 173750422c910a117e189969c42e8bbef9dc0df5 Mon Sep 17 00:00:00 2001 From: iam047801 Date: Wed, 31 May 2023 22:03:58 +0800 Subject: [PATCH 04/38] tlb: add ShardDescB and FutureSplitMerge --- tlb/shard.go | 118 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 101 insertions(+), 17 deletions(-) diff --git a/tlb/shard.go b/tlb/shard.go index 00c0e606..a764b436 100644 --- a/tlb/shard.go +++ b/tlb/shard.go @@ -2,6 +2,7 @@ package tlb import ( "fmt" + "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/tvm/cell" ) @@ -49,24 +50,72 @@ type ShardIdent struct { ShardPrefix uint64 `tlb:"## 64"` } +type FutureSplitMergeNone struct { + _ Magic `tlb:"$00"` +} + +type FutureSplit struct { + _ Magic `tlb:"$10"` + SplitUtime uint32 `tlb:"## 32"` + Interval uint32 `tlb:"## 32"` +} + +type FutureMerge struct { + _ Magic `tlb:"$11"` + MergeUtime uint32 `tlb:"## 32"` + Interval uint32 `tlb:"## 32"` +} + +type FutureSplitMerge struct { + FSM any `tlb:"."` +} + type ShardDesc struct { - _ Magic `tlb:"#a"` - SeqNo uint32 `tlb:"## 32"` - RegMcSeqno uint32 `tlb:"## 32"` - StartLT uint64 `tlb:"## 64"` - EndLT uint64 `tlb:"## 64"` - RootHash []byte `tlb:"bits 256"` - FileHash []byte `tlb:"bits 256"` - BeforeSplit bool `tlb:"bool"` - BeforeMerge bool `tlb:"bool"` - WantSplit bool `tlb:"bool"` - WantMerge bool `tlb:"bool"` - NXCCUpdated bool `tlb:"bool"` - Flags uint8 `tlb:"## 3"` - NextCatchainSeqNo uint32 `tlb:"## 32"` - NextValidatorShard int64 `tlb:"## 64"` - MinRefMcSeqNo uint32 `tlb:"## 32"` - GenUTime uint32 `tlb:"## 32"` + _ Magic `tlb:"#a"` + SeqNo uint32 `tlb:"## 32"` + RegMcSeqno uint32 `tlb:"## 32"` + StartLT uint64 `tlb:"## 64"` + EndLT uint64 `tlb:"## 64"` + RootHash []byte `tlb:"bits 256"` + FileHash []byte `tlb:"bits 256"` + BeforeSplit bool `tlb:"bool"` + BeforeMerge bool `tlb:"bool"` + WantSplit bool `tlb:"bool"` + WantMerge bool `tlb:"bool"` + NXCCUpdated bool `tlb:"bool"` + Flags uint8 `tlb:"## 3"` + NextCatchainSeqNo uint32 `tlb:"## 32"` + NextValidatorShard int64 `tlb:"## 64"` + MinRefMcSeqNo uint32 `tlb:"## 32"` + GenUTime uint32 `tlb:"## 32"` + SplitMergeAt FutureSplitMerge `tlb:"."` + Currencies struct { + FeesCollected CurrencyCollection `tlb:"."` + FundsCreated CurrencyCollection `tlb:"."` + } `tlb:"^"` +} + +type ShardDescB struct { + _ Magic `tlb:"#b"` + SeqNo uint32 `tlb:"## 32"` + RegMcSeqno uint32 `tlb:"## 32"` + StartLT uint64 `tlb:"## 64"` + EndLT uint64 `tlb:"## 64"` + RootHash []byte `tlb:"bits 256"` + FileHash []byte `tlb:"bits 256"` + BeforeSplit bool `tlb:"bool"` + BeforeMerge bool `tlb:"bool"` + WantSplit bool `tlb:"bool"` + WantMerge bool `tlb:"bool"` + NXCCUpdated bool `tlb:"bool"` + Flags uint8 `tlb:"## 3"` + NextCatchainSeqNo uint32 `tlb:"## 32"` + NextValidatorShard int64 `tlb:"## 64"` + MinRefMcSeqNo uint32 `tlb:"## 32"` + GenUTime uint32 `tlb:"## 32"` + SplitMergeAt FutureSplitMerge `tlb:"."` + FeesCollected CurrencyCollection `tlb:"."` + FundsCreated CurrencyCollection `tlb:"."` } func (s *ShardState) LoadFromCell(loader *cell.Slice) error { @@ -130,3 +179,38 @@ func (p *ConfigParams) LoadFromCell(loader *cell.Slice) error { return nil } + +func (s *FutureSplitMerge) LoadFromCell(loader *cell.Slice) error { + isNotEmpty, err := loader.LoadBoolBit() + if err != nil { + return fmt.Errorf("load bool bit is fsm none: %w", err) + } + + if !isNotEmpty { + s.FSM = FutureSplitMergeNone{} + return nil + } + + isMerge, err := loader.LoadBoolBit() + if err != nil { + return fmt.Errorf("load bool bit is fsm merge: %w", err) + } + + if !isMerge { + var split FutureSplit + err = LoadFromCell(&split, loader, true) + if err != nil { + return fmt.Errorf("failed to parse FutureSplit: %w", err) + } + s.FSM = split + } else { + var merge FutureMerge + err = LoadFromCell(&merge, loader, true) + if err != nil { + return fmt.Errorf("failed to parse FutureMerge: %w", err) + } + s.FSM = merge + } + + return nil +} From 5f0600456318fa45fe9dbb7aecfba57ee9b6b55e Mon Sep 17 00:00:00 2001 From: iam047801 Date: Wed, 31 May 2023 22:04:59 +0800 Subject: [PATCH 05/38] ton api client: GetBlockShardsInfo update with ShardDescB --- ton/block.go | 46 ++++++++++++++++++++++++++--------- ton/integration_test.go | 53 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 11 deletions(-) diff --git a/ton/block.go b/ton/block.go index 4b5075f5..8c6e3701 100644 --- a/ton/block.go +++ b/ton/block.go @@ -4,9 +4,10 @@ import ( "context" "errors" "fmt" - "github.com/xssnick/tonutils-go/tl" "time" + "github.com/xssnick/tonutils-go/tl" + "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/tvm/cell" ) @@ -359,18 +360,41 @@ func (c *APIClient) GetBlockShardsInfo(ctx context.Context, master *BlockIDExt) } for _, bk := range binTree.All() { - var shardDesc tlb.ShardDesc - if err = tlb.LoadFromCell(&shardDesc, bk.Value.BeginParse()); err != nil { - return nil, fmt.Errorf("load ShardDesc err: %w", err) + loader := bk.Value.BeginParse() + + ab, err := loader.LoadUInt(4) + if err != nil { + return nil, fmt.Errorf("load ShardDesc magic err: %w", err) } - shards = append(shards, &BlockIDExt{ - Workchain: int32(workchain), - Shard: shardDesc.NextValidatorShard, - SeqNo: shardDesc.SeqNo, - RootHash: shardDesc.RootHash, - FileHash: shardDesc.FileHash, - }) + switch ab { + case 0xa: + var shardDesc tlb.ShardDesc + if err = tlb.LoadFromCell(&shardDesc, loader, true); err != nil { + return nil, fmt.Errorf("load ShardDesc err: %w", err) + } + shards = append(shards, &BlockIDExt{ + Workchain: int32(workchain), + Shard: shardDesc.NextValidatorShard, + SeqNo: shardDesc.SeqNo, + RootHash: shardDesc.RootHash, + FileHash: shardDesc.FileHash, + }) + case 0xb: + var shardDesc tlb.ShardDescB + if err = tlb.LoadFromCell(&shardDesc, loader, true); err != nil { + return nil, fmt.Errorf("load ShardDescB err: %w", err) + } + shards = append(shards, &BlockIDExt{ + Workchain: int32(workchain), + Shard: shardDesc.NextValidatorShard, + SeqNo: shardDesc.SeqNo, + RootHash: shardDesc.RootHash, + FileHash: shardDesc.FileHash, + }) + default: + return nil, fmt.Errorf("wrong ShardDesc magic: %x", ab) + } } } diff --git a/ton/integration_test.go b/ton/integration_test.go index 061ba2c7..428d6501 100644 --- a/ton/integration_test.go +++ b/ton/integration_test.go @@ -111,6 +111,59 @@ func TestAPIClient_GetBlockData(t *testing.T) { // TODO: data check } +func TestAPIClient_GetOldBlockData(t *testing.T) { + client := liteclient.NewConnectionPool() + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + err := client.AddConnection(ctx, "135.181.177.59:53312", "aF91CuUHuuOv9rm2W5+O/4h38M3sRm40DtSdRxQhmtQ=") + if err != nil { + panic(err) + } + + api := NewAPIClient(client) + + b, err := api.CurrentMasterchainInfo(ctx) + if err != nil { + t.Fatal("get block err:", err.Error()) + return + } + + b, err = api.LookupBlock(ctx, b.Workchain, b.Shard, 3) + if err != nil { + t.Fatal("lookup err:", err.Error()) + return + } + + shards, err := api.GetBlockShardsInfo(ctx, b) + if err != nil { + log.Fatalln("get shards err:", err.Error()) + return + } + + for _, shard := range shards { + data, err := api.GetBlockData(ctx, shard) + if err != nil { + t.Fatal("Get shard block data err:", err.Error()) + return + } + _, err = data.BlockInfo.GetParentBlocks() + if err != nil { + t.Fatal("Get block parents err:", err.Error()) + return + } + } + + _, err = api.GetBlockData(ctx, b) + if err != nil { + t.Fatal("Get master block data err:", err.Error()) + return + } + + // TODO: data check +} + func Test_RunMethod(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() From 78435f838f3bccebddc6de1b3329e619feae8f17 Mon Sep 17 00:00:00 2001 From: iam047801 Date: Wed, 31 May 2023 23:26:26 +0800 Subject: [PATCH 06/38] tlb.ShardStateUnsplit: McStateExtra as a cell --- tlb/shard.go | 7 ++++--- ton/getconfig.go | 13 ++++++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tlb/shard.go b/tlb/shard.go index a764b436..fac43a6a 100644 --- a/tlb/shard.go +++ b/tlb/shard.go @@ -21,8 +21,8 @@ type ShardStateUnsplit struct { Accounts struct { ShardAccounts *cell.Dictionary `tlb:"dict 256"` } `tlb:"^"` - Stats *cell.Cell `tlb:"^"` - McStateExtra *McStateExtra `tlb:"maybe ^"` + Stats *cell.Cell `tlb:"^"` + McStateExtra *cell.Cell `tlb:"maybe ^"` } type McStateExtra struct { @@ -148,8 +148,9 @@ func (s *ShardState) LoadFromCell(loader *cell.Slice) error { s.Right = &right case 0x9023afe2: var state ShardStateUnsplit - err = LoadFromCell(&state, loader) + err = LoadFromCell(&state, preloader, true) if err != nil { + fmt.Println("ShardStateUnsplit error", err.Error()) return err } s.Left = state diff --git a/ton/getconfig.go b/ton/getconfig.go index f9293678..7045553d 100644 --- a/ton/getconfig.go +++ b/ton/getconfig.go @@ -4,10 +4,11 @@ import ( "context" "errors" "fmt" + "math/big" + "github.com/xssnick/tonutils-go/tl" "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/tvm/cell" - "math/big" ) func init() { @@ -82,12 +83,18 @@ func (c *APIClient) GetBlockchainConfig(ctx context.Context, block *BlockIDExt, return nil, errors.New("no mc extra state found, something went wrong") } + var stateExtra tlb.McStateExtra + err = tlb.LoadFromCell(&stateExtra, state.McStateExtra.BeginParse()) + if err != nil { + return nil, fmt.Errorf("load masterchain state extra: %w", err) + } + result := &BlockchainConfig{data: map[int32]*cell.Cell{}} if len(onlyParams) > 0 { // we need it because lite server may add some unwanted keys for _, param := range onlyParams { - res := state.McStateExtra.ConfigParams.Config.GetByIntKey(big.NewInt(int64(param))) + res := stateExtra.ConfigParams.Config.GetByIntKey(big.NewInt(int64(param))) if res == nil { return nil, fmt.Errorf("config param %d not found", param) } @@ -100,7 +107,7 @@ func (c *APIClient) GetBlockchainConfig(ctx context.Context, block *BlockIDExt, result.data[param] = v.MustToCell() } } else { - for _, kv := range state.McStateExtra.ConfigParams.Config.All() { + for _, kv := range stateExtra.ConfigParams.Config.All() { v, err := kv.Value.BeginParse().LoadRef() if err != nil { return nil, fmt.Errorf("failed to load config param %d, err: %w", kv.Key.BeginParse().MustLoadInt(32), err) From c11f7ed326b50b5bedba0405637b2896c34b8a7d Mon Sep 17 00:00:00 2001 From: iam047801 Date: Wed, 31 May 2023 23:27:37 +0800 Subject: [PATCH 07/38] tlb.Block: StateUpdate as a cell --- tlb/block.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tlb/block.go b/tlb/block.go index 0bc3fb10..b0d31849 100644 --- a/tlb/block.go +++ b/tlb/block.go @@ -2,6 +2,7 @@ package tlb import ( "fmt" + "github.com/xssnick/tonutils-go/tvm/cell" ) @@ -52,7 +53,7 @@ type Block struct { GlobalID int32 `tlb:"## 32"` BlockInfo BlockHeader `tlb:"^"` ValueFlow *cell.Cell `tlb:"^"` - StateUpdate StateUpdate `tlb:"^"` + StateUpdate *cell.Cell `tlb:"^"` Extra *BlockExtra `tlb:"^"` } From 96108830bf64b24cef8e4f28f27f96cc68874d7a Mon Sep 17 00:00:00 2001 From: iam047801 Date: Fri, 2 Jun 2023 14:54:14 +0800 Subject: [PATCH 08/38] tlb.AccountStorage: add ExtraCurrencies dict --- tlb/account.go | 13 +++++-------- ton/integration_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/tlb/account.go b/tlb/account.go index 2474a24e..d6ce5c2c 100644 --- a/tlb/account.go +++ b/tlb/account.go @@ -2,11 +2,11 @@ package tlb import ( "bytes" - "errors" "fmt" - "github.com/sigurn/crc16" "math/big" + "github.com/sigurn/crc16" + "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/tvm/cell" ) @@ -43,6 +43,7 @@ type AccountStorage struct { Status AccountStatus LastTransactionLT uint64 Balance Coins + ExtraCurrencies *cell.Dictionary `tlb:"dict 32"` // has value when active StateInit *StateInit @@ -221,13 +222,9 @@ func (s *AccountStorage) LoadFromCell(loader *cell.Slice) error { return fmt.Errorf("failed to load coins balance: %w", err) } - extraExists, err := loader.LoadBoolBit() + s.ExtraCurrencies, err = loader.LoadDict(32) if err != nil { - return fmt.Errorf("failed to load extra exists bit: %w", err) - } - - if extraExists { - return errors.New("extra currency info is not supported for AccountStorage") + return fmt.Errorf("failed to load extra currencies: %w", err) } isStatusActive, err := loader.LoadBoolBit() diff --git a/ton/integration_test.go b/ton/integration_test.go index 061ba2c7..9920c509 100644 --- a/ton/integration_test.go +++ b/ton/integration_test.go @@ -478,3 +478,33 @@ func Test_LSErrorCase(t *testing.T) { } } } + +func TestAccountStorage_LoadFromCell_ExtraCurrencies(t *testing.T) { + client := liteclient.NewConnectionPool() + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + err := client.AddConnection(context.Background(), "135.181.177.59:53312", "aF91CuUHuuOv9rm2W5+O/4h38M3sRm40DtSdRxQhmtQ=") + if err != nil { + t.Fatal(err) + } + + mainnetAPI := NewAPIClient(client) + + shard := uint64(0xa000000000000000) + + b, err := mainnetAPI.LookupBlock(ctx, 0, int64(shard), 3328952) + if err != nil { + t.Fatal(err) + } + + a, err := mainnetAPI.GetAccount(ctx, b, address.MustParseAddr("EQCYv992KVNNCKZHSLLJgM2GGzsgL0UgWP24BCQBaAdqSE2I")) + if err != nil { + t.Fatal(err) + } + + if a.State.ExtraCurrencies == nil { + t.Fatal("expected extra currencies dict") + } +} From a86793de15de8f332d48c3899ceb221c2161f947 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Sat, 3 Jun 2023 11:07:56 +0400 Subject: [PATCH 09/38] ADNL improvements --- adnl/adnl.go | 15 +++++--- adnl/adnl_test.go | 89 +++++++++++++++++++++++++++++++++++++++++++++ adnl/gateway.go | 10 +++++ adnl/rldp/client.go | 6 +-- 4 files changed, 111 insertions(+), 9 deletions(-) diff --git a/adnl/adnl.go b/adnl/adnl.go index 4b8706ee..fd93492d 100644 --- a/adnl/adnl.go +++ b/adnl/adnl.go @@ -98,9 +98,9 @@ func initADNL(key ed25519.PrivateKey) *ADNL { } func (a *ADNL) Close() { - a.mx.Lock() - defer a.mx.Unlock() + trigger := false + a.mx.Lock() if !a.closed { a.closed = true @@ -111,10 +111,13 @@ func (a *ADNL) Close() { con.Close() } - if a.onDisconnect != nil { - // do it async to not get accidental deadlock - go a.onDisconnect(a.addr, a.peerKey) - } + trigger = true + } + a.mx.Unlock() + + disc := a.onDisconnect + if trigger && disc != nil { + disc(a.addr, a.peerKey) } } diff --git a/adnl/adnl_test.go b/adnl/adnl_test.go index 8b85d300..4823b736 100644 --- a/adnl/adnl_test.go +++ b/adnl/adnl_test.go @@ -236,3 +236,92 @@ func TestADNL_Connect(t *testing.T) { } adnl.Close() } + +func TestADNL_ClientServerStartStop(t *testing.T) { + _, aPriv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + + bPub, bPriv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + + a := NewGateway(aPriv) + err = a.StartServer("127.0.0.1:9055") + if err != nil { + t.Fatal(err) + } + a.SetConnectionHandler(connHandler) + + b := NewGateway(bPriv) + err = b.StartServer("127.0.0.1:9065") + if err != nil { + t.Fatal(err) + } + b.SetConnectionHandler(connHandler) + + p, err := a.RegisterClient("127.0.0.1:9065", bPub) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var res MessagePong + err = p.Query(ctx, &MessagePing{7755}, &res) + if err != nil { + t.Fatal(err) + } + + if res.Value != 7755 { + t.Fatal("value not eq") + } + + _ = b.Close() + b = NewGateway(bPriv) + err = b.StartServer("127.0.0.1:9065") + if err != nil { + t.Fatal(err) + } + b.SetConnectionHandler(connHandler) + + p.Close() + p, err = a.RegisterClient("127.0.0.1:9065", bPub) + if err != nil { + t.Fatal(err) + } + + err = p.Query(ctx, &MessagePing{1111}, &res) + if err != nil { + t.Fatal(err) + } +} + +func connHandler(client Peer) error { + client.SetQueryHandler(func(msg *MessageQuery) error { + switch m := msg.Data.(type) { + case MessagePing: + if m.Value == 9999 { + client.Close() + return fmt.Errorf("handle mock err") + } + + err := client.Answer(context.Background(), msg.ID, MessagePong{ + Value: m.Value, + }) + if err != nil { + panic(err) + } + } + return nil + }) + client.SetCustomMessageHandler(func(msg *MessageCustom) error { + return client.SendCustomMessage(context.Background(), TestMsg{Data: make([]byte, 1280)}) + }) + client.SetDisconnectHandler(func(addr string, key ed25519.PublicKey) { + }) + return nil +} diff --git a/adnl/gateway.go b/adnl/gateway.go index c6b8bea2..03930f66 100644 --- a/adnl/gateway.go +++ b/adnl/gateway.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/ed25519" + "encoding/hex" "fmt" "github.com/xssnick/tonutils-go/adnl/address" "github.com/xssnick/tonutils-go/tl" @@ -191,6 +192,12 @@ func (g *Gateway) listen(rootId []byte) { buf := make([]byte, 4096) n, addr, err := g.conn.ReadFrom(buf) if err != nil { + select { + case <-g.globalCtx.Done(): + return + default: + } + Logger("failed to read packet:", err) continue } @@ -267,6 +274,7 @@ func (g *Gateway) listen(rootId []byte) { g.mx.RUnlock() if proc == nil { + Logger("no processor for ADNL packet from", hex.EncodeToString(id)) continue } @@ -410,6 +418,8 @@ func (g *Gateway) Close() error { g.mx.Lock() defer g.mx.Unlock() + g.globalCtxCancel() + if g.conn == nil { return nil } diff --git a/adnl/rldp/client.go b/adnl/rldp/client.go index 27adfcf2..b6379ed8 100644 --- a/adnl/rldp/client.go +++ b/adnl/rldp/client.go @@ -159,7 +159,7 @@ func (r *RLDP) handleMessage(msg *adnl.MessageCustom) error { defer stream.mx.Unlock() if stream.finishedAt != nil { - if stream.lastCompleteAt.Add(2 * time.Millisecond).Before(time.Now()) { // we not send completions too often, to not get socket buffer overflow + if stream.lastCompleteAt.Add(5 * time.Millisecond).Before(time.Now()) { // we not send completions too often, to not get socket buffer overflow var complete tl.Serializable = Complete{ TransferID: m.TransferID, @@ -356,8 +356,8 @@ func (r *RLDP) sendMessageParts(ctx context.Context, transferId, data []byte) er default: } - if symbolsSent > enc.BaseSymbolsNum()+enc.BaseSymbolsNum()*2 { //+enc.BaseSymbolsNum()/2 - x := symbolsSent - (enc.BaseSymbolsNum() + enc.BaseSymbolsNum()*2) + if symbolsSent > enc.BaseSymbolsNum()+enc.BaseSymbolsNum()/2 { //+enc.BaseSymbolsNum()/2 + x := symbolsSent - (enc.BaseSymbolsNum() + enc.BaseSymbolsNum()/2) select { case <-ctx.Done(): From e14073a0805c820e469ac7502328d207c585aa91 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Sat, 3 Jun 2023 18:23:07 +0400 Subject: [PATCH 10/38] Async close adnl --- adnl/adnl.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/adnl/adnl.go b/adnl/adnl.go index fd93492d..a0d9a393 100644 --- a/adnl/adnl.go +++ b/adnl/adnl.go @@ -117,7 +117,8 @@ func (a *ADNL) Close() { disc := a.onDisconnect if trigger && disc != nil { - disc(a.addr, a.peerKey) + // TODO: check where lock is possible, refactor and remove goroutine here + go disc(a.addr, a.peerKey) } } From 15afb889c790d9f1809392bf927d11862d021b72 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 5 Jun 2023 10:13:24 +0400 Subject: [PATCH 11/38] ADNL confirm seqno reset on reinit --- adnl/adnl.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/adnl/adnl.go b/adnl/adnl.go index a0d9a393..7e4bc4d8 100644 --- a/adnl/adnl.go +++ b/adnl/adnl.go @@ -163,6 +163,11 @@ func (a *ADNL) processPacket(packet *PacketContent, ch *Channel) (err error) { } if packet.ReinitDate != nil && *packet.ReinitDate > a.dstReinit { + // reset their seqno even if it is lower, + // because other side could lose counter + a.confirmSeqno = seqno + a.loss = 0 + // a.dstReinit = *packet.ReinitDate // a.seqno = 0 // a.channel = nil From bef2d77f7c0d5fb60d062c87177e501b117f97f4 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 5 Jun 2023 22:03:36 +0400 Subject: [PATCH 12/38] Added GetID for gateway --- adnl/gateway.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/adnl/gateway.go b/adnl/gateway.go index 03930f66..7b7a6169 100644 --- a/adnl/gateway.go +++ b/adnl/gateway.go @@ -448,6 +448,11 @@ func (g *Gateway) write(deadline time.Time, addr net.Addr, buf []byte) error { return nil } +func (g *Gateway) GetID() []byte { + id, _ := ToKeyID(PublicKeyED25519{Key: g.key.Public().(ed25519.PublicKey)}) + return id +} + func (p *peerConn) GetID() []byte { return p.client.GetID() } From 0067733b467defcbf7bcf35212ddace5543b8d92 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Tue, 6 Jun 2023 10:13:27 +0400 Subject: [PATCH 13/38] DHT wait for add node during find value --- adnl/adnl.go | 2 +- adnl/dht/client.go | 42 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/adnl/adnl.go b/adnl/adnl.go index 7e4bc4d8..2d15877a 100644 --- a/adnl/adnl.go +++ b/adnl/adnl.go @@ -370,7 +370,7 @@ func (a *ADNL) processAnswer(id string, query any) { res <- query } } else { - Logger("unknown response with id", id, a.addr, reflect.TypeOf(query).String()) + // Logger("unknown response with id", id, a.addr, reflect.TypeOf(query).String()) } } diff --git a/adnl/dht/client.go b/adnl/dht/client.go index 1e861e77..e7fd22b2 100644 --- a/adnl/dht/client.go +++ b/adnl/dht/client.go @@ -34,9 +34,14 @@ type Gateway interface { RegisterClient(addr string, key ed25519.PublicKey) (adnl.Peer, error) } +type knownNode struct { + node *Node + mx sync.Mutex +} + type Client struct { activeNodes map[string]*dhtNode - knownNodesInfo map[string]*Node + knownNodesInfo map[string]*knownNode queryTimeout time.Duration mx sync.RWMutex minNodeMx sync.Mutex @@ -122,7 +127,7 @@ func NewClient(connectTimeout time.Duration, gateway Gateway, nodes []*Node) (*C globalCtx, cancel := context.WithCancel(context.Background()) c := &Client{ activeNodes: map[string]*dhtNode{}, - knownNodesInfo: map[string]*Node{}, + knownNodesInfo: map[string]*knownNode{}, globalCtx: globalCtx, globalCtxCancel: cancel, gateway: gateway, @@ -228,15 +233,36 @@ func (c *Client) addNode(ctx context.Context, node *Node) (_ *dhtNode, err error c.mx.Lock() // check again under lock to guarantee that only one connection will be made if c.knownNodesInfo[keyID] == nil { - c.knownNodesInfo[keyID] = node + kNode = &knownNode{node: node} + c.knownNodesInfo[keyID] = kNode } else { kNode = c.knownNodesInfo[keyID] } c.mx.Unlock() } - if kNode != nil { - return nil, fmt.Errorf("node is known, but no active connection yet") + for { + if kNode.mx.TryLock() { + break + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + runtime.Gosched() + } + } + + defer kNode.mx.Unlock() + + c.mx.RLock() + aNode = c.activeNodes[keyID] + c.mx.RUnlock() + + if aNode != nil { + // we already connected to this node, just return it + return aNode, nil } // connect to first available address of node @@ -566,9 +592,9 @@ func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Conti case *Value: result <- &foundResult{value: v, node: node} case []*Node: - if len(v) > 12 { - // max 12 nodes to check - v = v[:12] + if len(v) > 24 { + // max 24 nodes to check + v = v[:24] } wg := sync.WaitGroup{} From ae5a45458d3abac19e53a856a6505037d0c78516 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Tue, 6 Jun 2023 10:34:40 +0400 Subject: [PATCH 14/38] DHT node conn wait only for not tried --- adnl/dht/client.go | 55 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/adnl/dht/client.go b/adnl/dht/client.go index e7fd22b2..929ac713 100644 --- a/adnl/dht/client.go +++ b/adnl/dht/client.go @@ -140,7 +140,7 @@ func NewClient(connectTimeout time.Duration, gateway Gateway, nodes []*Node) (*C ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) defer cancel() - _, err := c.addNode(ctx, node) + _, err := c.addNode(ctx, node, false) if err != nil { Logger("failed to add DHT node", node.AddrList.Addresses[0].IP.String(), node.AddrList.Addresses[0].Port, " from config, err:", err.Error()) return @@ -200,7 +200,7 @@ func (c *Client) nodeStateHandler(id string) func(node *dhtNode, state int) { } } -func (c *Client) addNode(ctx context.Context, node *Node) (_ *dhtNode, err error) { +func (c *Client) addNode(ctx context.Context, node *Node, waitConnection bool) (_ *dhtNode, err error) { pub, ok := node.ID.(adnl.PublicKeyED25519) if !ok { return nil, fmt.Errorf("unsupported id type %s", reflect.TypeOf(node.ID).String()) @@ -241,16 +241,22 @@ func (c *Client) addNode(ctx context.Context, node *Node) (_ *dhtNode, err error c.mx.Unlock() } - for { - if kNode.mx.TryLock() { - break - } + if waitConnection { + for { + if kNode.mx.TryLock() { + break + } - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - runtime.Gosched() + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + runtime.Gosched() + } + } + } else { + if !kNode.mx.TryLock() { + return nil, fmt.Errorf("connection already in progress") } } @@ -416,6 +422,7 @@ func (c *Client) Store(ctx context.Context, id any, name []byte, index int32, va var checkedMx sync.RWMutex checked := map[string]bool{} + triedToAdd := map[string]bool{} plist := c.buildPriorityList(kid) @@ -462,8 +469,15 @@ func (c *Client) Store(ctx context.Context, id any, name []byte, index int32, va priority := leadingZeroBits(xor(kid, nid)) if priority > currentPriority { + checkedMx.Lock() + tried := triedToAdd[hex.EncodeToString(nid)] + if !tried { + triedToAdd[hex.EncodeToString(nid)] = true + } + checkedMx.Unlock() + addCtx, cancel := context.WithTimeout(storeCtx, queryTimeout) - dNode, err := c.addNode(addCtx, n) + dNode, err := c.addNode(addCtx, n, !tried) cancel() if err != nil { continue @@ -544,6 +558,9 @@ func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Conti threadCtx, stopThreads := context.WithCancel(ctx) defer stopThreads() + var checkedMx sync.RWMutex + triedToAdd := map[string]bool{} + const threads = 12 result := make(chan *foundResult, threads) var numNoTasks int64 @@ -605,7 +622,19 @@ func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Conti go func(n *Node) { defer wg.Done() - newNode, err := c.addNode(connectCtx, n) + nid, err := adnl.ToKeyID(n.ID) + if err != nil { + return + } + + checkedMx.Lock() + tried := triedToAdd[hex.EncodeToString(nid)] + if !tried { + triedToAdd[hex.EncodeToString(nid)] = true + } + checkedMx.Unlock() + + newNode, err := c.addNode(connectCtx, n, !tried) if err != nil { return } From 665842dea4d45e3190d1e36f7be17d76b91948d4 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Tue, 6 Jun 2023 12:15:00 +0400 Subject: [PATCH 15/38] DHT new find value --- adnl/dht/client.go | 113 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/adnl/dht/client.go b/adnl/dht/client.go index 929ac713..0eba3859 100644 --- a/adnl/dht/client.go +++ b/adnl/dht/client.go @@ -558,6 +558,119 @@ func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Conti threadCtx, stopThreads := context.WithCancel(ctx) defer stopThreads() + result := make(chan *foundResult) + + checked := map[string]bool{} + checkedMx := sync.Mutex{} + go func() { + wg := sync.WaitGroup{} + for { + n, _ := plist.getNode() + if n == nil { + break + } + + wg.Add(1) + go func() { + c.searchVal(threadCtx, n, id, result, checked, &checkedMx) + wg.Done() + }() + } + wg.Wait() + + select { + case <-threadCtx.Done(): + case result <- nil: + } + }() + + select { + case val := <-result: + if val == nil { + return nil, cont, ErrDHTValueIsNotFound + } + cont.checkedNodes = append(cont.checkedNodes, val.node) + return val.value, cont, nil + case <-ctx.Done(): + return nil, nil, ctx.Err() + } +} + +func (c *Client) searchVal(ctx context.Context, n *dhtNode, id []byte, result chan<- *foundResult, checked map[string]bool, mx *sync.Mutex) { + val, err := n.findValue(ctx, id, _K) + if err != nil { + return + } + + switch v := val.(type) { + case *Value: + select { + case <-ctx.Done(): + case result <- &foundResult{value: v, node: n}: + } + return + case []*Node: + if len(v) > 16 { + // max 16 nodes to check + v = v[:16] + } + + wg := sync.WaitGroup{} + wg.Add(len(v)) + + for _, node := range v { + nid, keyErr := adnl.ToKeyID(node.ID) + if keyErr != nil { + wg.Done() + continue + } + + mx.Lock() + if checked[string(nid)] { + mx.Unlock() + wg.Done() + continue + } + checked[string(nid)] = true + mx.Unlock() + + go func(node *Node) { + defer wg.Done() + + connectCtx, connectCancel := context.WithTimeout(ctx, 40*time.Second) + newNode, err := c.addNode(connectCtx, node, true) + connectCancel() + if err != nil { + return + } + + c.searchVal(ctx, newNode, id, result, checked, mx) + }(node) + } + wg.Wait() + } +} + +func (c *Client) FindValueOld(ctx context.Context, key *Key, continuation ...*Continuation) (*Value, *Continuation, error) { + id, keyErr := adnl.ToKeyID(key) + if keyErr != nil { + return nil, nil, keyErr + } + + plist := c.buildPriorityList(id) + + cont := &Continuation{} + if len(continuation) > 0 && continuation[0] != nil { + cont = continuation[0] + for _, n := range cont.checkedNodes { + // mark nodes as used to not get a value from them again + plist.markUsed(n, true) + } + } + + threadCtx, stopThreads := context.WithCancel(ctx) + defer stopThreads() + var checkedMx sync.RWMutex triedToAdd := map[string]bool{} From bd68ec84df55ee373bfae047fc3dc4c6f2213d7d Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Wed, 7 Jun 2023 19:03:21 +0400 Subject: [PATCH 16/38] Experimental aggressive DHT --- adnl/adnl.go | 2 +- adnl/dht/client.go | 12 +++++++++--- adnl/dht/node.go | 6 ++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/adnl/adnl.go b/adnl/adnl.go index 2d15877a..ded4b7b4 100644 --- a/adnl/adnl.go +++ b/adnl/adnl.go @@ -424,7 +424,7 @@ func (a *ADNL) query(ctx context.Context, ch *Channel, req, result tl.Serializab return nil case <-ctx.Done(): return fmt.Errorf("deadline exceeded, addr %s %s, err: %w", a.addr, hex.EncodeToString(a.peerKey), ctx.Err()) - case <-time.After(100 * time.Millisecond): + case <-time.After(250 * time.Millisecond): } } } diff --git a/adnl/dht/client.go b/adnl/dht/client.go index 0eba3859..f8fa17e6 100644 --- a/adnl/dht/client.go +++ b/adnl/dht/client.go @@ -570,6 +570,10 @@ func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Conti break } + checkedMx.Lock() + checked[string(n.id)] = true + checkedMx.Unlock() + wg.Add(1) go func() { c.searchVal(threadCtx, n, id, result, checked, &checkedMx) @@ -597,7 +601,9 @@ func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Conti } func (c *Client) searchVal(ctx context.Context, n *dhtNode, id []byte, result chan<- *foundResult, checked map[string]bool, mx *sync.Mutex) { - val, err := n.findValue(ctx, id, _K) + findCtx, cancel := context.WithTimeout(ctx, queryTimeout) + val, err := n.findValue(findCtx, id, _K) + cancel() if err != nil { return } @@ -637,7 +643,7 @@ func (c *Client) searchVal(ctx context.Context, n *dhtNode, id []byte, result ch go func(node *Node) { defer wg.Done() - connectCtx, connectCancel := context.WithTimeout(ctx, 40*time.Second) + connectCtx, connectCancel := context.WithTimeout(ctx, queryTimeout) newNode, err := c.addNode(connectCtx, node, true) connectCancel() if err != nil { @@ -827,7 +833,7 @@ func (c *Client) nodesPinger() { } func (c *Client) buildPriorityList(id []byte) *priorityList { - plist := newPriorityList(_K, id) + plist := newPriorityList(_K*3, id) added := 0 c.mx.RLock() diff --git a/adnl/dht/node.go b/adnl/dht/node.go index 5b9493e2..5033683e 100644 --- a/adnl/dht/node.go +++ b/adnl/dht/node.go @@ -48,10 +48,12 @@ func (c *Client) connectToNode(ctx context.Context, id []byte, addr string, serv client: c, } - err := n.checkPing(ctx) + n.changeState(_StateThrottle) + + /*err := n.checkPing(ctx) if err != nil { return nil, fmt.Errorf("failed to ping node, err: %w", err) - } + }*/ return n, nil } From d8a7351ae96bdae52d71767a8935bf892ce128f2 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Sat, 10 Jun 2023 17:50:56 +0400 Subject: [PATCH 17/38] DHT Tunning --- adnl/dht/client.go | 81 ++++++++++++++++++++++++++++++++++++++++++++-- adnl/dht/node.go | 6 ++-- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/adnl/dht/client.go b/adnl/dht/client.go index f8fa17e6..ea0ede2f 100644 --- a/adnl/dht/client.go +++ b/adnl/dht/client.go @@ -538,7 +538,7 @@ type foundResult struct { node *dhtNode } -func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Continuation) (*Value, *Continuation, error) { +func (c *Client) FindValueAggressive(ctx context.Context, key *Key, continuation ...*Continuation) (*Value, *Continuation, error) { id, keyErr := adnl.ToKeyID(key) if keyErr != nil { return nil, nil, keyErr @@ -657,7 +657,7 @@ func (c *Client) searchVal(ctx context.Context, n *dhtNode, id []byte, result ch } } -func (c *Client) FindValueOld(ctx context.Context, key *Key, continuation ...*Continuation) (*Value, *Continuation, error) { +func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Continuation) (*Value, *Continuation, error) { id, keyErr := adnl.ToKeyID(key) if keyErr != nil { return nil, nil, keyErr @@ -680,7 +680,7 @@ func (c *Client) FindValueOld(ctx context.Context, key *Key, continuation ...*Co var checkedMx sync.RWMutex triedToAdd := map[string]bool{} - const threads = 12 + const threads = 8 result := make(chan *foundResult, threads) var numNoTasks int64 for i := 0; i < threads; i++ { @@ -867,3 +867,78 @@ func (c *Client) buildPriorityList(id []byte) *priorityList { return plist } + +func TstsNodesFind(c *Client) { + var n *Node + for s := range c.activeNodes { + n = c.knownNodesInfo[s].node + println(n.AddrList.Addresses[0].IP.String()) + break + } + + hg, _ := hex.DecodeString("7906710747daaad26b08df75dcfb695b44d9d0a5657ba91aeb04de97d11dba6b") + nodes, _, err := c.FindOverlayNodes(context.Background(), hg, nil) + if err != nil { + println("OVER ERR") + } else { + println(nodes.List) + } + + /* kkk, _ := hex.DecodeString("14bbe0fe397b143b7bb6f259de2ee2f4d0196dfd1b34486d6ec550ba0cf1ecc1") + list, _, err := c.FindAddresses(context.Background(), kkk) + if err != nil { + println("FIND ERR") + } else { + println(list.Addresses) + }*/ + + err = search(1, n, c) + panic(err) +} + +func search(i int, n *Node, c *Client) error { + addr := n.AddrList.Addresses[0].IP.String() + ":" + fmt.Sprint(n.AddrList.Addresses[0].Port) + + println("search", i, addr) + + pub, _ := n.ID.(adnl.PublicKeyED25519) + kid, _ := adnl.ToKeyID(pub) + + kk, _ := hex.DecodeString("14bbe0fe397b143b7bb6f259de2ee2f4d0196dfd1b34486d6ec550ba0cf1ecc1") + key, _ := adnl.ToKeyID(&Key{ + ID: kk, + Name: []byte("address"), + Index: 0, + }) + + node := &dhtNode{ + id: kid, + addr: addr, + serverKey: pub.Key, + onStateChange: func(node *dhtNode, state int) { + }, + client: c, + } + + ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond) + val, err := node.findValue(ctx, key, _K) + cancel() + if err != nil { + return err + } + println("O", i, reflect.ValueOf(val).String()) + + switch t := val.(type) { + case []*Node: + for _, n2 := range t { + if search(i+1, n2, c) == nil { + return nil + } else { + println("fail", n2.AddrList.Addresses[0].IP.String()) + } + } + default: + println("OK!") + } + return nil +} diff --git a/adnl/dht/node.go b/adnl/dht/node.go index 5033683e..5b9493e2 100644 --- a/adnl/dht/node.go +++ b/adnl/dht/node.go @@ -48,12 +48,10 @@ func (c *Client) connectToNode(ctx context.Context, id []byte, addr string, serv client: c, } - n.changeState(_StateThrottle) - - /*err := n.checkPing(ctx) + err := n.checkPing(ctx) if err != nil { return nil, fmt.Errorf("failed to ping node, err: %w", err) - }*/ + } return n, nil } From a4289c5e12dfec5a2ca508c0574c71f1f113b6f5 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Sat, 10 Jun 2023 21:37:06 +0400 Subject: [PATCH 18/38] Added addr to adnl rldp interface --- adnl/rldp/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/adnl/rldp/client.go b/adnl/rldp/client.go index b6379ed8..b9c1af86 100644 --- a/adnl/rldp/client.go +++ b/adnl/rldp/client.go @@ -16,6 +16,7 @@ import ( ) type ADNL interface { + RemoteAddr() string GetID() []byte SetCustomMessageHandler(handler func(msg *adnl.MessageCustom) error) SetDisconnectHandler(handler func(addr string, key ed25519.PublicKey)) From 72855155de7b94e2dcdc704e5feed226ea90cd6a Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Sun, 11 Jun 2023 14:56:04 +0400 Subject: [PATCH 19/38] DHT dbg, disabled pinger --- adnl/dht/client.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/adnl/dht/client.go b/adnl/dht/client.go index ea0ede2f..35c7fe9a 100644 --- a/adnl/dht/client.go +++ b/adnl/dht/client.go @@ -159,13 +159,15 @@ func NewClient(connectTimeout time.Duration, gateway Gateway, nodes []*Node) (*C return nil, fmt.Errorf("no available nodes in the given list %v", nodes) } - go c.nodesPinger() + // go c.nodesPinger() return c, nil } const _K = 10 func (c *Client) Close() { + c.globalCtxCancel() + c.mx.Lock() var toClose []*dhtNode // doing this way to not get deadlock with nodeStateHandler @@ -175,8 +177,6 @@ func (c *Client) Close() { c.activeNodes = nil c.mx.Unlock() - c.globalCtxCancel() - for _, node := range toClose { node.Close() } @@ -788,6 +788,8 @@ func (c *Client) nodesPinger() { case <-time.After(1 * time.Second): } + println("[DHT] Pinger cycle", time.Now().String()) + now := time.Now() c.mx.RLock() if len(c.activeNodes) == 0 { From 6e2e242900faf6d6c4f052860d445df5ebefdec0 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Sun, 11 Jun 2023 19:18:39 +0400 Subject: [PATCH 20/38] Simplified DHT --- adnl/dht/client.go | 492 ++++--------------------------------------- adnl/dht/node.go | 127 +++-------- adnl/dht/priority.go | 6 +- 3 files changed, 79 insertions(+), 546 deletions(-) diff --git a/adnl/dht/client.go b/adnl/dht/client.go index 35c7fe9a..79b165f6 100644 --- a/adnl/dht/client.go +++ b/adnl/dht/client.go @@ -31,20 +31,13 @@ type ADNL interface { } type Gateway interface { + Close() error RegisterClient(addr string, key ed25519.PublicKey) (adnl.Peer, error) } -type knownNode struct { - node *Node - mx sync.Mutex -} - type Client struct { - activeNodes map[string]*dhtNode - knownNodesInfo map[string]*knownNode - queryTimeout time.Duration - mx sync.RWMutex - minNodeMx sync.Mutex + knownNodes map[string]*dhtNode + mx sync.RWMutex gateway Gateway @@ -72,15 +65,10 @@ func NewClientFromConfigUrl(ctx context.Context, gateway Gateway, cfgUrl string) return nil, err } - return NewClientFromConfig(ctx, gateway, cfg) + return NewClientFromConfig(gateway, cfg) } -func NewClientFromConfig(ctx context.Context, gateway Gateway, cfg *liteclient.GlobalConfig) (*Client, error) { - dl, ok := ctx.Deadline() - if !ok { - dl = time.Now().Add(10 * time.Second) - } - +func NewClientFromConfig(gateway Gateway, cfg *liteclient.GlobalConfig) (*Client, error) { var nodes []*Node for _, node := range cfg.DHT.StaticNodes.Nodes { key, err := base64.StdEncoding.DecodeString(node.ID.Key) @@ -120,46 +108,29 @@ func NewClientFromConfig(ctx context.Context, gateway Gateway, cfg *liteclient.G nodes = append(nodes, n) } - return NewClient(dl.Sub(time.Now()), gateway, nodes) + return NewClient(gateway, nodes) } -func NewClient(connectTimeout time.Duration, gateway Gateway, nodes []*Node) (*Client, error) { +func NewClient(gateway Gateway, nodes []*Node) (*Client, error) { globalCtx, cancel := context.WithCancel(context.Background()) c := &Client{ - activeNodes: map[string]*dhtNode{}, - knownNodesInfo: map[string]*knownNode{}, + knownNodes: map[string]*dhtNode{}, globalCtx: globalCtx, globalCtxCancel: cancel, gateway: gateway, } - ch := make(chan bool, len(nodes)) - for _, node := range nodes { - go func(node *Node) { - ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) - defer cancel() - - _, err := c.addNode(ctx, node, false) - if err != nil { - Logger("failed to add DHT node", node.AddrList.Addresses[0].IP.String(), node.AddrList.Addresses[0].Port, " from config, err:", err.Error()) - return - } - - ch <- true - }(node) - } - - select { - case <-ch: - case <-time.After(connectTimeout): + _, err := c.addNode(node) + if err != nil { + Logger("failed to add DHT node", node.AddrList.Addresses[0].IP.String(), node.AddrList.Addresses[0].Port, " from config, err:", err.Error()) + continue + } } - if len(c.activeNodes) == 0 { - return nil, fmt.Errorf("no available nodes in the given list %v", nodes) + if len(c.knownNodes) == 0 { + return nil, errors.New("0 nodes was added") } - - // go c.nodesPinger() return c, nil } @@ -167,40 +138,10 @@ const _K = 10 func (c *Client) Close() { c.globalCtxCancel() - - c.mx.Lock() - var toClose []*dhtNode - // doing this way to not get deadlock with nodeStateHandler - for _, v := range c.activeNodes { - toClose = append(toClose, v) - } - c.activeNodes = nil - c.mx.Unlock() - - for _, node := range toClose { - node.Close() - } -} - -func (c *Client) nodeStateHandler(id string) func(node *dhtNode, state int) { - return func(node *dhtNode, state int) { - c.mx.Lock() - defer c.mx.Unlock() - - if c.activeNodes == nil { - return - } - - switch state { - case _StateFail: - // delete(c.activeNodes, id) - case _StateThrottle, _StateActive: // TODO: handle throttle in a diff list - c.activeNodes[id] = node - } - } + _ = c.gateway.Close() } -func (c *Client) addNode(ctx context.Context, node *Node, waitConnection bool) (_ *dhtNode, err error) { +func (c *Client) addNode(node *Node) (_ *dhtNode, err error) { pub, ok := node.ID.(adnl.PublicKeyED25519) if !ok { return nil, fmt.Errorf("unsupported id type %s", reflect.TypeOf(node.ID).String()) @@ -210,16 +151,14 @@ func (c *Client) addNode(ctx context.Context, node *Node, waitConnection bool) ( if err != nil { return nil, err } - keyID := hex.EncodeToString(kid) - c.mx.RLock() - kNode := c.knownNodesInfo[keyID] - aNode := c.activeNodes[keyID] - c.mx.RUnlock() - if aNode != nil { - // we already connected to this node, just return it - return aNode, nil + c.mx.Lock() + defer c.mx.Unlock() + + kNode := c.knownNodes[keyID] + if kNode != nil { + return kNode, nil } if len(node.AddrList.Addresses) == 0 { @@ -229,71 +168,13 @@ func (c *Client) addNode(ctx context.Context, node *Node, waitConnection bool) ( node.AddrList.Addresses = node.AddrList.Addresses[:8] } - if kNode == nil { - c.mx.Lock() - // check again under lock to guarantee that only one connection will be made - if c.knownNodesInfo[keyID] == nil { - kNode = &knownNode{node: node} - c.knownNodesInfo[keyID] = kNode - } else { - kNode = c.knownNodesInfo[keyID] - } - c.mx.Unlock() - } - - if waitConnection { - for { - if kNode.mx.TryLock() { - break - } - - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - runtime.Gosched() - } - } - } else { - if !kNode.mx.TryLock() { - return nil, fmt.Errorf("connection already in progress") - } - } - - defer kNode.mx.Unlock() - - c.mx.RLock() - aNode = c.activeNodes[keyID] - c.mx.RUnlock() - - if aNode != nil { - // we already connected to this node, just return it - return aNode, nil - } + // TODO: maybe use other addresses too + addr := node.AddrList.Addresses[0].IP.String() + ":" + fmt.Sprint(node.AddrList.Addresses[0].Port) - // connect to first available address of node - for _, udp := range node.AddrList.Addresses { - addr := udp.IP.String() + ":" + fmt.Sprint(udp.Port) + kNode = c.connectToNode(kid, addr, pub.Key) + c.knownNodes[keyID] = kNode - aNode, err = c.connectToNode(ctx, kid, addr, pub.Key, c.nodeStateHandler(keyID)) - if err != nil { - // failed to connect, we will try next addr - continue - } - // connected successfully - break - } - - if err != nil { - c.mx.Lock() - // connection was unsuccessful, so we remove node from known, to be able to retry later - delete(c.knownNodesInfo, keyID) - c.mx.Unlock() - - return nil, fmt.Errorf("failed to connect to node: %w", err) - } - - return aNode, nil + return kNode, nil } func (c *Client) FindOverlayNodes(ctx context.Context, overlayKey []byte, continuation ...*Continuation) (*overlay.NodesList, *Continuation, error) { @@ -422,7 +303,6 @@ func (c *Client) Store(ctx context.Context, id any, name []byte, index int32, va var checkedMx sync.RWMutex checked := map[string]bool{} - triedToAdd := map[string]bool{} plist := c.buildPriorityList(kid) @@ -439,7 +319,7 @@ func (c *Client) Store(ctx context.Context, id any, name []byte, index int32, va break } - strId := hex.EncodeToString(node.id) + strId := node.id() checkedMx.RLock() isChecked := checked[strId] checkedMx.RUnlock() @@ -451,7 +331,7 @@ func (c *Client) Store(ctx context.Context, id any, name []byte, index int32, va } hasBetter := false - currentPriority := leadingZeroBits(xor(kid, node.id)) + currentPriority := leadingZeroBits(xor(kid, node.adnlId)) for _, n := range nodes { var nid []byte nid, err = adnl.ToKeyID(n.ID) @@ -469,16 +349,7 @@ func (c *Client) Store(ctx context.Context, id any, name []byte, index int32, va priority := leadingZeroBits(xor(kid, nid)) if priority > currentPriority { - checkedMx.Lock() - tried := triedToAdd[hex.EncodeToString(nid)] - if !tried { - triedToAdd[hex.EncodeToString(nid)] = true - } - checkedMx.Unlock() - - addCtx, cancel := context.WithTimeout(storeCtx, queryTimeout) - dNode, err := c.addNode(addCtx, n, !tried) - cancel() + dNode, err := c.addNode(n) if err != nil { continue } @@ -538,125 +409,6 @@ type foundResult struct { node *dhtNode } -func (c *Client) FindValueAggressive(ctx context.Context, key *Key, continuation ...*Continuation) (*Value, *Continuation, error) { - id, keyErr := adnl.ToKeyID(key) - if keyErr != nil { - return nil, nil, keyErr - } - - plist := c.buildPriorityList(id) - - cont := &Continuation{} - if len(continuation) > 0 && continuation[0] != nil { - cont = continuation[0] - for _, n := range cont.checkedNodes { - // mark nodes as used to not get a value from them again - plist.markUsed(n, true) - } - } - - threadCtx, stopThreads := context.WithCancel(ctx) - defer stopThreads() - - result := make(chan *foundResult) - - checked := map[string]bool{} - checkedMx := sync.Mutex{} - go func() { - wg := sync.WaitGroup{} - for { - n, _ := plist.getNode() - if n == nil { - break - } - - checkedMx.Lock() - checked[string(n.id)] = true - checkedMx.Unlock() - - wg.Add(1) - go func() { - c.searchVal(threadCtx, n, id, result, checked, &checkedMx) - wg.Done() - }() - } - wg.Wait() - - select { - case <-threadCtx.Done(): - case result <- nil: - } - }() - - select { - case val := <-result: - if val == nil { - return nil, cont, ErrDHTValueIsNotFound - } - cont.checkedNodes = append(cont.checkedNodes, val.node) - return val.value, cont, nil - case <-ctx.Done(): - return nil, nil, ctx.Err() - } -} - -func (c *Client) searchVal(ctx context.Context, n *dhtNode, id []byte, result chan<- *foundResult, checked map[string]bool, mx *sync.Mutex) { - findCtx, cancel := context.WithTimeout(ctx, queryTimeout) - val, err := n.findValue(findCtx, id, _K) - cancel() - if err != nil { - return - } - - switch v := val.(type) { - case *Value: - select { - case <-ctx.Done(): - case result <- &foundResult{value: v, node: n}: - } - return - case []*Node: - if len(v) > 16 { - // max 16 nodes to check - v = v[:16] - } - - wg := sync.WaitGroup{} - wg.Add(len(v)) - - for _, node := range v { - nid, keyErr := adnl.ToKeyID(node.ID) - if keyErr != nil { - wg.Done() - continue - } - - mx.Lock() - if checked[string(nid)] { - mx.Unlock() - wg.Done() - continue - } - checked[string(nid)] = true - mx.Unlock() - - go func(node *Node) { - defer wg.Done() - - connectCtx, connectCancel := context.WithTimeout(ctx, queryTimeout) - newNode, err := c.addNode(connectCtx, node, true) - connectCancel() - if err != nil { - return - } - - c.searchVal(ctx, newNode, id, result, checked, mx) - }(node) - } - wg.Wait() - } -} - func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Continuation) (*Value, *Continuation, error) { id, keyErr := adnl.ToKeyID(key) if keyErr != nil { @@ -677,9 +429,6 @@ func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Conti threadCtx, stopThreads := context.WithCancel(ctx) defer stopThreads() - var checkedMx sync.RWMutex - triedToAdd := map[string]bool{} - const threads = 8 result := make(chan *foundResult, threads) var numNoTasks int64 @@ -729,40 +478,18 @@ func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Conti result <- &foundResult{value: v, node: node} case []*Node: if len(v) > 24 { - // max 24 nodes to check + // max 24 nodes to add v = v[:24] } - wg := sync.WaitGroup{} - wg.Add(len(v)) - - connectCtx, connectCancel := context.WithTimeout(threadCtx, queryTimeout) for _, n := range v { - go func(n *Node) { - defer wg.Done() - - nid, err := adnl.ToKeyID(n.ID) - if err != nil { - return - } - - checkedMx.Lock() - tried := triedToAdd[hex.EncodeToString(nid)] - if !tried { - triedToAdd[hex.EncodeToString(nid)] = true - } - checkedMx.Unlock() - - newNode, err := c.addNode(connectCtx, n, !tried) - if err != nil { - return - } + newNode, err := c.addNode(n) + if err != nil { + continue + } - plist.addNode(newNode) - }(n) + plist.addNode(newNode) } - wg.Wait() - connectCancel() } } }() @@ -780,76 +507,25 @@ func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Conti } } -func (c *Client) nodesPinger() { - for { - select { - case <-c.globalCtx.Done(): - return - case <-time.After(1 * time.Second): - } - - println("[DHT] Pinger cycle", time.Now().String()) - - now := time.Now() - c.mx.RLock() - if len(c.activeNodes) == 0 { - c.mx.RUnlock() - continue - } - - ch := make(chan *dhtNode, len(c.activeNodes)+1) - for _, node := range c.activeNodes { - // add check task for nodes that were not queried for > 8 seconds - if atomic.LoadInt64(&node.lastQueryAt)+8 < now.Unix() { - ch <- node - } - } - close(ch) - c.mx.RUnlock() - - var wg sync.WaitGroup - wg.Add(8) - for i := 0; i < 8; i++ { - go func() { - defer wg.Done() - for { - var node *dhtNode - select { - case <-c.globalCtx.Done(): - return - case node = <-ch: - if node == nil { - // everything is checked - return - } - } - - ctx, cancel := context.WithTimeout(c.globalCtx, queryTimeout) - _ = node.checkPing(ctx) // we don't need the result, it will report new state to callback - cancel() - } - }() - } - wg.Wait() - } -} - func (c *Client) buildPriorityList(id []byte) *priorityList { plist := newPriorityList(_K*3, id) added := 0 + c.mx.RLock() + defer c.mx.RUnlock() + // add fastest nodes first - for _, node := range c.activeNodes { - if node.getState() == _StateActive { + for _, node := range c.knownNodes { + if node.currentState == _StateActive { plist.addNode(node) added++ } } // if we have not enough fast nodes, add slow if added < 15 { - for _, node := range c.activeNodes { - if node.getState() == _StateThrottle { + for _, node := range c.knownNodes { + if node.currentState == _StateThrottle { plist.addNode(node) added++ } @@ -858,89 +534,13 @@ func (c *Client) buildPriorityList(id []byte) *priorityList { // if not enough active nodes, add failed, hope they will accept connection and become active // they may be failed due to our connection problems if added < 15 { - for _, node := range c.activeNodes { - if node.getState() == _StateFail { + for _, node := range c.knownNodes { + if node.currentState == _StateFail { plist.addNode(node) added++ } } } - c.mx.RUnlock() return plist } - -func TstsNodesFind(c *Client) { - var n *Node - for s := range c.activeNodes { - n = c.knownNodesInfo[s].node - println(n.AddrList.Addresses[0].IP.String()) - break - } - - hg, _ := hex.DecodeString("7906710747daaad26b08df75dcfb695b44d9d0a5657ba91aeb04de97d11dba6b") - nodes, _, err := c.FindOverlayNodes(context.Background(), hg, nil) - if err != nil { - println("OVER ERR") - } else { - println(nodes.List) - } - - /* kkk, _ := hex.DecodeString("14bbe0fe397b143b7bb6f259de2ee2f4d0196dfd1b34486d6ec550ba0cf1ecc1") - list, _, err := c.FindAddresses(context.Background(), kkk) - if err != nil { - println("FIND ERR") - } else { - println(list.Addresses) - }*/ - - err = search(1, n, c) - panic(err) -} - -func search(i int, n *Node, c *Client) error { - addr := n.AddrList.Addresses[0].IP.String() + ":" + fmt.Sprint(n.AddrList.Addresses[0].Port) - - println("search", i, addr) - - pub, _ := n.ID.(adnl.PublicKeyED25519) - kid, _ := adnl.ToKeyID(pub) - - kk, _ := hex.DecodeString("14bbe0fe397b143b7bb6f259de2ee2f4d0196dfd1b34486d6ec550ba0cf1ecc1") - key, _ := adnl.ToKeyID(&Key{ - ID: kk, - Name: []byte("address"), - Index: 0, - }) - - node := &dhtNode{ - id: kid, - addr: addr, - serverKey: pub.Key, - onStateChange: func(node *dhtNode, state int) { - }, - client: c, - } - - ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond) - val, err := node.findValue(ctx, key, _K) - cancel() - if err != nil { - return err - } - println("O", i, reflect.ValueOf(val).String()) - - switch t := val.(type) { - case []*Node: - for _, n2 := range t { - if search(i+1, n2, c) == nil { - return nil - } else { - println("fail", n2.AddrList.Addresses[0].IP.String()) - } - } - default: - println("OK!") - } - return nil -} diff --git a/adnl/dht/node.go b/adnl/dht/node.go index 5b9493e2..c93d3df9 100644 --- a/adnl/dht/node.go +++ b/adnl/dht/node.go @@ -4,8 +4,8 @@ import ( "bytes" "context" "crypto/ed25519" + "encoding/hex" "fmt" - "math/rand" "reflect" "sync" "sync/atomic" @@ -23,92 +23,29 @@ const ( ) type dhtNode struct { - id []byte - adnl ADNL + adnlId []byte client *Client ping int64 addr string serverKey ed25519.PublicKey - onStateChange func(node *dhtNode, state int) - currentState int + currentState int - lastQueryAt int64 + lastQueryAt int64 + inFlyQueries int32 mx sync.Mutex } -func (c *Client) connectToNode(ctx context.Context, id []byte, addr string, serverKey ed25519.PublicKey, onStateChange func(node *dhtNode, state int)) (*dhtNode, error) { +func (c *Client) connectToNode(id []byte, addr string, serverKey ed25519.PublicKey) *dhtNode { n := &dhtNode{ - id: id, - addr: addr, - serverKey: serverKey, - onStateChange: onStateChange, - client: c, + adnlId: id, + addr: addr, + serverKey: serverKey, + client: c, } - - err := n.checkPing(ctx) - if err != nil { - return nil, fmt.Errorf("failed to ping node, err: %w", err) - } - - return n, nil -} - -func (n *dhtNode) Close() { - n.mx.Lock() - defer n.mx.Unlock() - - if n.adnl != nil { - n.adnl.Close() - } -} - -func (n *dhtNode) changeState(state int) { - n.mx.Lock() - defer n.mx.Unlock() - - if state == _StateFail { - // in case of fail - close connection - if n.adnl != nil { - n.adnl.Close() - n.adnl = nil - } - } - - if n.currentState == state { - return - } - n.currentState = state - - n.onStateChange(n, state) -} - -func (n *dhtNode) getState() int { - n.mx.Lock() - defer n.mx.Unlock() - return n.currentState -} - -func (n *dhtNode) prepare() (ADNL, error) { - n.mx.Lock() - defer n.mx.Unlock() - - if n.adnl != nil { - return n.adnl, nil - } - - a, err := n.client.gateway.RegisterClient(n.addr, n.serverKey) - if err != nil { - return nil, err - } - a.SetDisconnectHandler(func(addr string, key ed25519.PublicKey) { - n.changeState(_StateFail) - }) - n.adnl = a - - return n.adnl, nil + return n } func (n *dhtNode) findNodes(ctx context.Context, id []byte, K int32) (result []*Node, err error) { @@ -271,35 +208,29 @@ func checkValue(id []byte, value *Value) error { return nil } -func (n *dhtNode) checkPing(ctx context.Context) error { - ping := Ping{ - ID: int64(rand.Uint64()), - } +func (n *dhtNode) query(ctx context.Context, req, res tl.Serializable) error { + atomic.AddInt32(&n.inFlyQueries, 1) - var res Pong - err := n.query(ctx, ping, &res) + a, err := n.client.gateway.RegisterClient(n.addr, n.serverKey) if err != nil { - n.changeState(_StateFail) + atomic.AddInt32(&n.inFlyQueries, -1) + n.currentState = _StateFail return err } - if res.ID != ping.ID { - return fmt.Errorf("wrong pong id") - } - - return nil -} - -func (n *dhtNode) query(ctx context.Context, req, res tl.Serializable) error { - a, err := n.prepare() - if err != nil { - return fmt.Errorf("failed to prepare dht node: %w", err) - } + defer func() { + if atomic.AddInt32(&n.inFlyQueries, -1) == 0 && n.currentState == _StateFail { + a.Close() + } + }() t := time.Now() atomic.StoreInt64(&n.lastQueryAt, t.Unix()) err = a.Query(ctx, req, res) if err != nil { + if ctx.Err() == nil || time.Since(t) > 3*time.Second { + n.currentState = _StateFail + } return err } ping := time.Since(t) @@ -307,19 +238,23 @@ func (n *dhtNode) query(ctx context.Context, req, res tl.Serializable) error { switch { case ping > queryTimeout/3: - n.changeState(_StateThrottle) + n.currentState = _StateThrottle default: - n.changeState(_StateActive) + n.currentState = _StateActive } return nil } +func (n *dhtNode) id() string { + return hex.EncodeToString(n.adnlId) +} + func (n *dhtNode) weight(id []byte) int { n.mx.Lock() defer n.mx.Unlock() - w := leadingZeroBits(xor(id, n.id)) + w := leadingZeroBits(xor(id, n.adnlId)) if n.currentState == _StateFail { w -= 3 // less priority for failed if w < 0 { diff --git a/adnl/dht/priority.go b/adnl/dht/priority.go index 82080913..2867a59f 100644 --- a/adnl/dht/priority.go +++ b/adnl/dht/priority.go @@ -1,7 +1,6 @@ package dht import ( - "encoding/hex" "sync" ) @@ -25,12 +24,11 @@ func newPriorityList(maxLen int, targetId []byte) *priorityList { targetId: targetId, maxLen: maxLen, } - return p } func (p *priorityList) addNode(node *dhtNode) bool { - id := hex.EncodeToString(node.id) + id := node.id() item := &nodePriority{ id: id, @@ -121,7 +119,7 @@ func (p *priorityList) markUsed(node *dhtNode, used bool) { p.mx.Lock() defer p.mx.Unlock() - id := hex.EncodeToString(node.id) + id := node.id() curNode := p.list for curNode != nil { From bfa3f0fb7dbcf830fe4ef21b5b713eb342588ec4 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Thu, 15 Jun 2023 20:52:35 +0400 Subject: [PATCH 21/38] Unsafe cell methods added --- tvm/cell/cell.go | 20 ++++++++++++++++++-- tvm/cell/level.go | 16 ++++++++-------- tvm/cell/parse.go | 2 +- tvm/cell/proof.go | 30 +++++++++++++++--------------- tvm/cell/serialize.go | 2 +- 5 files changed, 43 insertions(+), 27 deletions(-) diff --git a/tvm/cell/cell.go b/tvm/cell/cell.go index 670fc123..ad90a2c7 100644 --- a/tvm/cell/cell.go +++ b/tvm/cell/cell.go @@ -85,6 +85,22 @@ func (c *Cell) RefsNum() uint { return uint(len(c.refs)) } +func (c *Cell) MustPeekRef(i int) *Cell { + return c.refs[i] +} + +func (c *Cell) UnsafeModify(levelMask LevelMask, special bool) { + c.special = special + c.levelMask = levelMask +} + +func (c *Cell) PeekRef(i int) (*Cell, error) { + if i > len(c.refs) { + return nil, ErrNoMoreRefs + } + return c.refs[i], nil +} + func (c *Cell) Dump(limitLength ...int) string { var lim = (1024 << 20) * 16 if len(limitLength) > 0 { @@ -123,8 +139,8 @@ func (c *Cell) dump(deep int, bin bool, limitLength int) string { } str := strings.Repeat(" ", deep) + fmt.Sprint(sz) + "[" + val + "]" - if c.levelMask.getLevel() > 0 { - str += fmt.Sprintf("{%d}", c.levelMask.getLevel()) + if c.levelMask.GetLevel() > 0 { + str += fmt.Sprintf("{%d}", c.levelMask.GetLevel()) } if c.special { str += "*" diff --git a/tvm/cell/level.go b/tvm/cell/level.go index a148f651..bc1bc553 100644 --- a/tvm/cell/level.go +++ b/tvm/cell/level.go @@ -3,21 +3,21 @@ package cell import "math/bits" type LevelMask struct { - mask byte + Mask byte } -func (m LevelMask) getLevel() int { - return bits.Len8(m.mask) +func (m LevelMask) GetLevel() int { + return bits.Len8(m.Mask) } func (m LevelMask) getHashIndex() int { - return bits.OnesCount8(m.mask) + return bits.OnesCount8(m.Mask) } -func (m LevelMask) apply(level int) LevelMask { - return LevelMask{m.mask & ((1 << level) - 1)} +func (m LevelMask) Apply(level int) LevelMask { + return LevelMask{m.Mask & ((1 << level) - 1)} } -func (m LevelMask) isSignificant(level int) bool { - return level == 0 || ((m.mask>>(level-1))%2 != 0) +func (m LevelMask) IsSignificant(level int) bool { + return level == 0 || ((m.Mask>>(level-1))%2 != 0) } diff --git a/tvm/cell/parse.go b/tvm/cell/parse.go index 7702f11d..c9eefe2f 100644 --- a/tvm/cell/parse.go +++ b/tvm/cell/parse.go @@ -143,7 +143,7 @@ func parseCells(rootsNum, cellsNum, refSzBytes int, data []byte, index []int) ([ } if withHashes { - maskBits := int(math.Ceil(math.Log2(float64(levelMask.mask) + 1))) + maskBits := int(math.Ceil(math.Log2(float64(levelMask.Mask) + 1))) hashesNum := maskBits + 1 offset += hashesNum*hashSize + hashesNum*depthSize diff --git a/tvm/cell/proof.go b/tvm/cell/proof.go index 02517477..be67e08b 100644 --- a/tvm/cell/proof.go +++ b/tvm/cell/proof.go @@ -23,8 +23,8 @@ func (c *Cell) CreateProof(forHashes [][]byte) (*Cell, error) { // build root Merkle Proof cell data := make([]byte, 1+32+2) data[0] = _MerkleProofType - copy(data[1:], c.getHash(c.levelMask.getLevel())) - binary.BigEndian.PutUint16(data[1+32:], c.getDepth(c.levelMask.getLevel())) + copy(data[1:], c.getHash(c.levelMask.GetLevel())) + binary.BigEndian.PutUint16(data[1+32:], c.getDepth(c.levelMask.GetLevel())) proof := &Cell{ special: true, @@ -77,15 +77,15 @@ func (c *Cell) toProof(parts []cellHash) ([]cellHash, error) { if len(hasPartsRefs) > 0 && len(toPruneRefs) > 0 { // contains some useful and unuseful refs, pune unuseful for i, ref := range toPruneRefs { - if ref.levelMask.getLevel() >= 3 { + if ref.levelMask.GetLevel() >= 3 { return nil, fmt.Errorf("child level is to big to prune") } - ourLvl := ref.levelMask.getLevel() + ourLvl := ref.levelMask.GetLevel() prunedData := make([]byte, 2+(ourLvl+1)*(32+2)) prunedData[0] = _PrunedType - prunedData[1] = byte(ref.levelMask.getLevel()) + 1 + prunedData[1] = byte(ref.levelMask.GetLevel()) + 1 for lvl := 0; lvl <= ourLvl; lvl++ { copy(prunedData[2+(lvl*32):], ref.getHash(lvl)) @@ -94,7 +94,7 @@ func (c *Cell) toProof(parts []cellHash) ([]cellHash, error) { c.refs[toPruneIdx[i]] = &Cell{ special: true, - levelMask: LevelMask{ref.levelMask.mask + 1}, + levelMask: LevelMask{ref.levelMask.Mask + 1}, bitsSz: uint(len(prunedData) * 8), data: prunedData, } @@ -103,10 +103,10 @@ func (c *Cell) toProof(parts []cellHash) ([]cellHash, error) { typ := c.getType() for _, ref := range c.refs { - if ref.levelMask.getLevel() > c.levelMask.getLevel() { + if ref.levelMask.GetLevel() > c.levelMask.GetLevel() { if typ == _MerkleProofType { // proof should be 1 level less than child - c.levelMask = LevelMask{ref.levelMask.mask - 1} + c.levelMask = LevelMask{ref.levelMask.Mask - 1} } else { c.levelMask = ref.levelMask } @@ -122,12 +122,12 @@ func CheckProof(proof *Cell, hash []byte) error { return fmt.Errorf("not a merkle proof cell") } - needLvl := proof.refs[0].levelMask.getLevel() + needLvl := proof.refs[0].levelMask.GetLevel() if needLvl > 0 { needLvl -= 1 } - if needLvl != proof.levelMask.getLevel() { + if needLvl != proof.levelMask.GetLevel() { return fmt.Errorf("incorrect level of child") } if !bytes.Equal(hash, proof.data[1:33]) { @@ -147,7 +147,7 @@ func (c *Cell) getLevelMask() LevelMask { } func (c *Cell) getHash(level int) []byte { - hashIndex := c.getLevelMask().apply(level).getHashIndex() + hashIndex := c.getLevelMask().Apply(level).getHashIndex() if c.getType() == _PrunedType { prunedHashIndex := c.getLevelMask().getHashIndex() @@ -179,9 +179,9 @@ func (c *Cell) calculateHashes() { hashIndexOffset := totalHashCount - hashCount hashIndex := 0 - level := c.levelMask.getLevel() + level := c.levelMask.GetLevel() for levelIndex := 0; levelIndex <= level; levelIndex++ { - if !c.levelMask.isSignificant(levelIndex) { + if !c.levelMask.IsSignificant(levelIndex) { continue } @@ -195,7 +195,7 @@ func (c *Cell) calculateHashes() { } dsc := make([]byte, 2) - dsc[0], dsc[1] = c.descriptors(c.levelMask.apply(levelIndex)) + dsc[0], dsc[1] = c.descriptors(c.levelMask.Apply(levelIndex)) hash := sha256.New() hash.Write(dsc) @@ -261,7 +261,7 @@ func (c *Cell) calculateHashes() { } func (c *Cell) getDepth(level int) uint16 { - hashIndex := c.getLevelMask().apply(level).getHashIndex() + hashIndex := c.getLevelMask().Apply(level).getHashIndex() if c.getType() == _PrunedType { prunedHashIndex := c.getLevelMask().getHashIndex() if hashIndex != prunedHashIndex { diff --git a/tvm/cell/serialize.go b/tvm/cell/serialize.go index 6806039f..eb0de1be 100644 --- a/tvm/cell/serialize.go +++ b/tvm/cell/serialize.go @@ -113,7 +113,7 @@ func (c *Cell) descriptors(lvl LevelMask) (byte, byte) { specBit = 8 } - return byte(len(c.refs)) + specBit + lvl.mask*32, byte(ln) + return byte(len(c.refs)) + specBit + lvl.Mask*32, byte(ln) } func dynamicIntBytes(val uint64, sz uint) []byte { From 5549e796f8cd03b3e766737992205c70320f8f81 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Thu, 22 Jun 2023 10:31:39 +0400 Subject: [PATCH 22/38] Fixed rldp compilation --- adnl/rldp/client_test.go | 10 ++++++++++ adnl/rldp/http/client.go | 2 ++ adnl/rldp/http/client_test.go | 10 ++++++++++ example/external-message/main.go | 10 ++++++++-- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/adnl/rldp/client_test.go b/adnl/rldp/client_test.go index 56835d67..50bceb77 100644 --- a/adnl/rldp/client_test.go +++ b/adnl/rldp/client_test.go @@ -31,6 +31,16 @@ type MockADNL struct { close func() } +func (m MockADNL) GetID() []byte { + //TODO implement me + panic("implement me") +} + +func (m MockADNL) RemoteAddr() string { + //TODO implement me + panic("implement me") +} + func (m MockADNL) SetCustomMessageHandler(handler func(msg *adnl.MessageCustom) error) { } diff --git a/adnl/rldp/http/client.go b/adnl/rldp/http/client.go index 1efe03e4..87487e3c 100644 --- a/adnl/rldp/http/client.go +++ b/adnl/rldp/http/client.go @@ -45,6 +45,8 @@ type RLDP interface { } type ADNL interface { + GetID() []byte + RemoteAddr() string Query(ctx context.Context, req, result tl.Serializable) error SetDisconnectHandler(handler func(addr string, key ed25519.PublicKey)) SetCustomMessageHandler(handler func(msg *adnl.MessageCustom) error) diff --git a/adnl/rldp/http/client_test.go b/adnl/rldp/http/client_test.go index 26f279dd..8ced9e30 100644 --- a/adnl/rldp/http/client_test.go +++ b/adnl/rldp/http/client_test.go @@ -46,6 +46,16 @@ type MockADNL struct { close func() } +func (m MockADNL) GetID() []byte { + //TODO implement me + panic("implement me") +} + +func (m MockADNL) RemoteAddr() string { + //TODO implement me + panic("implement me") +} + func (m MockADNL) GetQueryHandler() func(msg *adnl.MessageQuery) error { return nil } diff --git a/example/external-message/main.go b/example/external-message/main.go index 748c2a4f..b197d8c3 100644 --- a/example/external-message/main.go +++ b/example/external-message/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/hex" "log" "github.com/xssnick/tonutils-go/address" @@ -79,10 +80,15 @@ func main() { MustStoreUInt(1, 16). // add 1 to total EndCell() - err = api.SendExternalMessage(ctx, &tlb.ExternalMessage{ + msg := &tlb.ExternalMessage{ DstAddr: address.MustParseAddr("kQBL2_3lMiyywU17g-or8N7v9hDmPCpttzBPE2isF2GTziky"), Body: data, - }) + } + + msgCell, _ := tlb.ToCell(msg) + println("HASH", hex.EncodeToString(msgCell.Hash())) + + err = api.SendExternalMessage(ctx, msg) if err != nil { // FYI: it can fail if not enough balance on contract log.Printf("send err: %s", err.Error()) From 26698fcb71f7a351284f1227921ab43f0a9d8a01 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Thu, 29 Jun 2023 09:18:15 +0400 Subject: [PATCH 23/38] Liteclient Stop method --- example/account-state/main.go | 2 +- liteclient/connection.go | 31 ++++++++++++++++++++++++++----- liteclient/pool.go | 19 +++++++++++++++++++ 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/example/account-state/main.go b/example/account-state/main.go index 211f1020..58efad59 100644 --- a/example/account-state/main.go +++ b/example/account-state/main.go @@ -33,7 +33,7 @@ func main() { return } - addr := address.MustParseAddr("EQCVRJ-RqeZWcDqgTzzcxUIrChFYs0SyKGUvye9kGOuEWndQ") + addr := address.MustParseAddr("EQCD39VS5jcptHL8vMjEXrzGaRcCVYto7HUn4bpAOg8xqB2N") // we use WaitForBlock to make sure block is ready, // it is optional but escapes us from liteserver block not ready errors diff --git a/liteclient/connection.go b/liteclient/connection.go index 25a6dab1..a29c22f3 100644 --- a/liteclient/connection.go +++ b/liteclient/connection.go @@ -95,7 +95,17 @@ func (c *ConnectionPool) AddConnectionsFromConfigUrl(ctx context.Context, config return c.AddConnectionsFromConfig(ctx, config) } +var ErrStopped = errors.New("connection pool is closed") + func (c *ConnectionPool) AddConnection(ctx context.Context, addr, serverKey string, clientKey ...ed25519.PrivateKey) error { + select { + case <-c.globalCtx.Done(): + return ErrStopped + case <-ctx.Done(): + return ctx.Err() + default: + } + sKey, err := base64.StdEncoding.DecodeString(serverKey) if err != nil { return err @@ -170,12 +180,16 @@ func (c *ConnectionPool) AddConnection(ctx context.Context, addr, serverKey stri select { case <-conn.authEvt: // auth completed + case <-c.globalCtx.Done(): + return ErrStopped case <-ctx.Done(): return ctx.Err() } } select { + case <-c.globalCtx.Done(): + return ErrStopped case err = <-connResult: if err != nil { return err @@ -290,12 +304,17 @@ func (n *connection) listen(connResult chan<- error) { } n.pool.nodesMx.Unlock() - n.pool.reqMx.RLock() - dis := n.pool.onDisconnect - n.pool.reqMx.RUnlock() + select { + case <-n.pool.globalCtx.Done(): + return + default: + n.pool.reqMx.RLock() + dis := n.pool.onDisconnect + n.pool.reqMx.RUnlock() - if dis != nil { - go dis(n.addr, n.serverKey) + if dis != nil { + go dis(n.addr, n.serverKey) + } } } } @@ -303,6 +322,8 @@ func (n *connection) listen(connResult chan<- error) { func (n *connection) startPings(every time.Duration) { for { select { + case <-n.pool.globalCtx.Done(): + return case <-time.After(every): } diff --git a/liteclient/pool.go b/liteclient/pool.go index 8375e6d9..cd74d364 100644 --- a/liteclient/pool.go +++ b/liteclient/pool.go @@ -40,6 +40,9 @@ type ConnectionPool struct { roundRobinOffset uint64 authKey ed25519.PrivateKey + + globalCtx context.Context + stop func() } var ErrNoActiveConnections = errors.New("no active connections") @@ -52,6 +55,7 @@ func NewConnectionPool() *ConnectionPool { // default reconnect policy c.SetOnDisconnect(c.DefaultReconnect(3*time.Second, -1)) + c.globalCtx, c.stop = context.WithCancel(context.Background()) return c } @@ -89,11 +93,26 @@ func (c *ConnectionPool) StickyContext(ctx context.Context) context.Context { return context.WithValue(ctx, _StickyCtxKey, id) } +func (c *ConnectionPool) StickyContextWithNodeID(ctx context.Context, nodeId uint32) context.Context { + return context.WithValue(ctx, _StickyCtxKey, nodeId) +} + func (c *ConnectionPool) StickyNodeID(ctx context.Context) uint32 { nodeID, _ := ctx.Value(_StickyCtxKey).(uint32) return nodeID } +func (c *ConnectionPool) Stop() { + c.stop() + + c.nodesMx.RLock() + defer c.nodesMx.RUnlock() + + for _, node := range c.activeNodes { + _ = node.tcp.Close() + } +} + // QueryLiteserver - sends request to liteserver func (c *ConnectionPool) QueryLiteserver(ctx context.Context, request tl.Serializable, result tl.Serializable) error { return c.QueryADNL(ctx, LiteServerQuery{Data: request}, result) From 78c7c335cb531cb9e797a5a250adecffb7587fec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 Jun 2023 10:25:12 +0000 Subject: [PATCH 24/38] Bump golang.org/x/sys from 0.0.0-20220325203850-36772127a21f to 0.1.0 Bumps [golang.org/x/sys](https://github.com/golang/sys) from 0.0.0-20220325203850-36772127a21f to 0.1.0. - [Commits](https://github.com/golang/sys/commits/v0.1.0) --- updated-dependencies: - dependency-name: golang.org/x/sys dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 57e5348f..1a9efd24 100644 --- a/go.mod +++ b/go.mod @@ -8,4 +8,4 @@ require ( golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 ) -require golang.org/x/sys v0.0.0-20220325203850-36772127a21f // indirect +require golang.org/x/sys v0.1.0 // indirect diff --git a/go.sum b/go.sum index b0524db4..1e0ab72b 100644 --- a/go.sum +++ b/go.sum @@ -4,5 +4,5 @@ github.com/sigurn/crc16 v0.0.0-20211026045750-20ab5afb07e3 h1:aQKxg3+2p+IFXXg97M github.com/sigurn/crc16 v0.0.0-20211026045750-20ab5afb07e3/go.mod h1:9/etS5gpQq9BJsJMWg1wpLbfuSnkm8dPF6FdW2JXVhA= golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 h1:S25/rfnfsMVgORT4/J61MJ7rdyseOZOyvLIrZEZ7s6s= golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/sys v0.0.0-20220325203850-36772127a21f h1:TrmogKRsSOxRMJbLYGrB4SBbW+LJcEllYBLME5Zk5pU= -golang.org/x/sys v0.0.0-20220325203850-36772127a21f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From f8d8e82c53c3570d0b715da963136b4ddbe99aef Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Tue, 1 Aug 2023 20:12:58 +0400 Subject: [PATCH 25/38] Proofs check for liteserver responses & imporvements --- address/addr.go | 2 + adnl/adnl.go | 3 +- adnl/adnl_test.go | 10 +- adnl/dht/client_test.go | 81 +++--- adnl/dht/node_test.go | 397 +++--------------------------- adnl/dht/priority_test.go | 24 +- example/send-to-contract/main.go | 18 +- example/wallet-cold-alike/main.go | 91 +++++++ tlb/account.go | 6 + tlb/block.go | 8 +- tlb/loader.go | 43 +++- ton/api.go | 5 + ton/block.go | 182 ++++++++++---- ton/dns/integration_test.go | 2 +- ton/getconfig.go | 13 +- ton/getstate.go | 72 +++--- ton/integration_test.go | 102 ++++++-- ton/jetton/integration_test.go | 2 +- ton/proof.go | 186 ++++++++++++++ ton/runmethod.go | 54 +++- ton/transactions.go | 29 ++- ton/wallet/address.go | 6 +- ton/wallet/highloadv2r2.go | 3 + ton/wallet/integration_test.go | 9 +- ton/wallet/wallet.go | 109 ++++---- tvm/cell/cell.go | 50 ++-- tvm/cell/cell_test.go | 13 + tvm/cell/dict.go | 8 + tvm/cell/proof.go | 41 +-- tvm/cell/slice.go | 10 +- 30 files changed, 934 insertions(+), 645 deletions(-) create mode 100644 example/wallet-cold-alike/main.go create mode 100644 ton/proof.go diff --git a/address/addr.go b/address/addr.go index 1a537ace..df290649 100644 --- a/address/addr.go +++ b/address/addr.go @@ -18,6 +18,8 @@ const ( VarAddress AddrType = 3 ) +const MasterchainID int32 = -1 + type Address struct { flags flags addrType AddrType diff --git a/adnl/adnl.go b/adnl/adnl.go index ded4b7b4..243e8523 100644 --- a/adnl/adnl.go +++ b/adnl/adnl.go @@ -117,8 +117,7 @@ func (a *ADNL) Close() { disc := a.onDisconnect if trigger && disc != nil { - // TODO: check where lock is possible, refactor and remove goroutine here - go disc(a.addr, a.peerKey) + disc(a.addr, a.peerKey) } } diff --git a/adnl/adnl_test.go b/adnl/adnl_test.go index 4823b736..51923424 100644 --- a/adnl/adnl_test.go +++ b/adnl/adnl_test.go @@ -31,7 +31,7 @@ func TestADNL_ClientServer(t *testing.T) { gotSrvDiscon := make(chan any, 1) s := NewGateway(srvKey) - err = s.StartServer("127.0.0.1:9055") + err = s.StartServer("127.0.0.1:9155") if err != nil { t.Fatal(err) } @@ -66,7 +66,7 @@ func TestADNL_ClientServer(t *testing.T) { time.Sleep(1 * time.Second) - cli, err := Connect(context.Background(), "127.0.0.1:9055", srvPub, nil) + cli, err := Connect(context.Background(), "127.0.0.1:9155", srvPub, nil) if err != nil { t.Fatal(err) } @@ -109,7 +109,7 @@ func TestADNL_ClientServer(t *testing.T) { t.Fatal(err) } - cliBad, err := Connect(context.Background(), "127.0.0.1:9055", rndPub, nil) + cliBad, err := Connect(context.Background(), "127.0.0.1:9155", rndPub, nil) if err != nil { t.Fatal(err) } @@ -130,7 +130,7 @@ func TestADNL_ClientServer(t *testing.T) { t.Fatal(err) } - cliBadQuery, err := Connect(context.Background(), "127.0.0.1:9055", srvPub, rndOur) + cliBadQuery, err := Connect(context.Background(), "127.0.0.1:9155", srvPub, rndOur) if err != nil { t.Fatal(err) } @@ -173,7 +173,7 @@ func TestADNL_ClientServer(t *testing.T) { }) t.Run("custom msg channel reinited", func(t *testing.T) { - cli, err = Connect(context.Background(), "127.0.0.1:9055", srvPub, nil) + cli, err = Connect(context.Background(), "127.0.0.1:9155", srvPub, nil) if err != nil { t.Fatal(err) } diff --git a/adnl/dht/client_test.go b/adnl/dht/client_test.go index 0f9525d7..f909f9a6 100644 --- a/adnl/dht/client_test.go +++ b/adnl/dht/client_test.go @@ -26,6 +26,23 @@ type MockGateway struct { reg func(addr string, key ed25519.PublicKey) (adnl.Peer, error) } +func (m *MockGateway) GetAddressList() address.List { + //TODO implement me + panic("implement me") +} + +func (m *MockGateway) Close() error { + return nil +} + +func (m *MockGateway) SetConnectionHandler(f func(client adnl.Peer) error) {} + +func (m *MockGateway) SetExternalIP(ip net.IP) {} + +func (m *MockGateway) StartServer(listenAddr string) error { + return nil +} + func (m *MockGateway) RegisterClient(addr string, key ed25519.PublicKey) (adnl.Peer, error) { return m.reg(addr, key) } @@ -253,10 +270,7 @@ func TestClient_FindValue(t *testing.T) { }, nil } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - dhtCli, err := NewClientFromConfig(ctx, gateway, cnf) + dhtCli, err := NewClientFromConfig(gateway, cnf) if err != nil { t.Fatal(err) } @@ -305,7 +319,7 @@ func TestClient_NewClientFromConfig(t *testing.T) { tAddr1 := makeStrAddress(-1185526007, 22096) node1 := dhtNode{ - id: tKeyId1, + adnlId: tKeyId1, addr: tAddr1, serverKey: pubKey1, } @@ -328,7 +342,7 @@ func TestClient_NewClientFromConfig(t *testing.T) { tAddr2 := makeStrAddress(-1307380860, 15888) node2 := dhtNode{ - id: tKeyId2, + adnlId: tKeyId2, addr: tAddr2, serverKey: pubKey2, } @@ -345,7 +359,7 @@ func TestClient_NewClientFromConfig(t *testing.T) { "positive case (all nodes valid)", node1, node2, 2, true, true, }, { - "negative case (one of two nodes with bad sign)", node1, node2, 1, true, false, + "negative case (one of two nodes with bad sign)", node1, node2, 2, true, true, }, } @@ -374,26 +388,21 @@ func TestClient_NewClientFromConfig(t *testing.T) { }, nil } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - cli, err := NewClientFromConfig(ctx, gateway, cnf) + cli, err := NewClientFromConfig(gateway, cnf) if err != nil { t.Fatal(err) } - time.Sleep(1 * time.Second) - - if len(cli.activeNodes) != test.wantLenNodes || len(cli.knownNodesInfo) != test.wantLenNodes { - t.Errorf("added nodes count (active'%d', known'%d') but expected(%d)", len(cli.activeNodes), len(cli.knownNodesInfo), test.wantLenNodes) + if len(cli.knownNodes) != test.wantLenNodes { + t.Errorf("added nodes count (known'%d') but expected(%d)", len(cli.knownNodes), test.wantLenNodes) } - resDhtNode1, ok1 := cli.activeNodes[hexTKeyId1] + resDhtNode1, ok1 := cli.knownNodes[hexTKeyId1] if ok1 != test.checkAdd1 { t.Errorf("invalid active nodes addition") } if ok1 { - if !bytes.Equal(resDhtNode1.id, test.tNode1.id) { + if !bytes.Equal(resDhtNode1.adnlId, test.tNode1.adnlId) { t.Errorf("invalid active node id") } if resDhtNode1.addr != test.tNode1.addr { @@ -404,13 +413,13 @@ func TestClient_NewClientFromConfig(t *testing.T) { } } - resDhtNode2, ok2 := cli.activeNodes[hexTKeyId2] + resDhtNode2, ok2 := cli.knownNodes[hexTKeyId2] if ok2 != test.checkAdd2 { t.Errorf("invalid active nodes addition") } if ok2 { - if !bytes.Equal(resDhtNode2.id, test.tNode2.id) { + if !bytes.Equal(resDhtNode2.adnlId, test.tNode2.adnlId) { t.Errorf("invalid active node id") } if resDhtNode2.addr != test.tNode2.addr { @@ -489,10 +498,7 @@ func TestClient_FindAddressesUnit(t *testing.T) { }, nil } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - cli, err := NewClientFromConfig(ctx, gateway, cnf) + cli, err := NewClientFromConfig(gateway, cnf) if err != nil { t.Fatal("failed to prepare test client, err:", err) } @@ -565,30 +571,20 @@ func TestClient_Close(t *testing.T) { }, nil } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - cli, err := NewClientFromConfig(ctx, gateway, cnf) + cli, err := NewClientFromConfig(gateway, cnf) if err != nil { t.Fatal("failed to prepare test client, err: ", err) } t.Run("close client test", func(t *testing.T) { cli.Close() - if cli.activeNodes != nil { - t.Error("found active nodes in client after 'Close' operation") - } - for _, node := range cli.activeNodes { - if node.adnl != nil { - t.Errorf("found connected node (id: %s) after 'Close' operation", hex.EncodeToString(node.id)) - } + if cli.globalCtx.Err() == nil { + t.Error("global context was not canceled") } - }) } func TestClient_StoreAddress(t *testing.T) { for i := 0; i < 15; i++ { - addrList := address.List{ Addresses: []*address.UDP{ { @@ -659,13 +655,11 @@ func TestClient_StoreAddress(t *testing.T) { case FindNode: if addr == "185.86.79.9:22096" { reflect.ValueOf(result).Elem().Set(reflect.ValueOf(NodesList{[]*Node{testNode}})) - } else if addr == "" { - } else { reflect.ValueOf(result).Elem().Set(reflect.ValueOf(NodesList{nil})) } case Store: - if addr != "6.6.6.6:65432" && addr != "178.18.243.132:15888" { + if addr != "185.86.79.9:22096" && addr != "178.18.243.132:15888" { t.Fatalf("invalid node to store: check priority list %s", addr) } sign := rowReqType.Value.Signature @@ -693,10 +687,7 @@ func TestClient_StoreAddress(t *testing.T) { }, nil } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - cli, err := NewClientFromConfig(ctx, gateway, cnf) + cli, err := NewClientFromConfig(gateway, cnf) if err != nil { t.Fatal("failed to prepare test client, err: ", err) } @@ -754,9 +745,7 @@ func TestClient_StoreOverlayNodesIntegration(t *testing.T) { t.Fatal(err) } - println("NUM", len(list.List)) - - if len(list.List) > 1 { + if len(list.List) == 0 { t.Fatal("list len") } diff --git a/adnl/dht/node_test.go b/adnl/dht/node_test.go index 49dd2f8d..a04cf4c4 100644 --- a/adnl/dht/node_test.go +++ b/adnl/dht/node_test.go @@ -27,191 +27,13 @@ func newCorrectDhtNode(a byte, b byte, c byte, d byte, port string) (*dhtNode, e return nil, err } resDhtNode := &dhtNode{ - id: kId, - adnl: nil, - ping: 0, - addr: net.IPv4(a, b, c, d).To4().String() + ":" + port, - serverKey: tPubKey, - onStateChange: func(node *dhtNode, state int) {}, - currentState: 0, + adnlId: kId, + addr: net.IPv4(a, b, c, d).To4().String() + ":" + port, + serverKey: tPubKey, } return resDhtNode, nil } -func TestNode_connectToNode(t *testing.T) { - corNode, err := newCorrectNode(1, 2, 3, 4, 5678) - if err != nil { - t.Fatal("failed to prepare correct test node, err: ", err) - } - tKeyId, err := adnl.ToKeyID(corNode.ID.(adnl.PublicKeyED25519)) - if err != nil { - t.Fatal("failed to prepare correct test key id, err: ", err) - } - addr := corNode.AddrList.Addresses[0].IP.String() + ":" + fmt.Sprint(corNode.AddrList.Addresses[0].Port) - - t.Run("everything correct case", func(t *testing.T) { - gateway := &MockGateway{} - gateway.reg = func(addr string, peerKey ed25519.PublicKey) (adnl.Peer, error) { - return MockADNL{ - query: func(ctx context.Context, req, result tl.Serializable) error { - switch request := req.(type) { - case Ping: - reflect.ValueOf(result).Elem().Set(reflect.ValueOf(Pong{ID: request.ID})) - default: - return fmt.Errorf("mock err: unsupported request type '%s'", reflect.TypeOf(request).String()) - } - return nil - }, - }, nil - } - cli := Client{ - gateway: gateway, - } - _, err = cli.connectToNode(context.Background(), tKeyId, addr, corNode.ID.(adnl.PublicKeyED25519).Key, func(node *dhtNode, state int) {}) - if err != nil { - t.Fatal("failed to execute 'connectToNode' func, err: ", err) - } - }) - t.Run("incorrect pong id", func(t *testing.T) { - gateway := &MockGateway{} - gateway.reg = func(addr string, peerKey ed25519.PublicKey) (adnl.Peer, error) { - return MockADNL{ - query: func(ctx context.Context, req, result tl.Serializable) error { - switch request := req.(type) { - case Ping: - reflect.ValueOf(result).Elem().Set(reflect.ValueOf(Pong{ID: request.ID + 1})) - default: - return fmt.Errorf("mock err: unsupported request type '%s'", reflect.TypeOf(request).String()) - } - return nil - }, - }, nil - } - cli := Client{ - gateway: gateway, - } - _, err = cli.connectToNode(context.Background(), tKeyId, addr, corNode.ID.(adnl.PublicKeyED25519).Key, func(node *dhtNode, state int) {}) - if !strings.Contains(err.Error(), "wrong pong id") { - t.Errorf("got '%s', want 'wrong pong id'", err.Error()) - } - }) -} - -func TestNode_changeState(t *testing.T) { - tNode, err := newCorrectDhtNode(1, 2, 3, 4, "5678") - if err != nil { - t.Fatal("failed to prepare correct test node, err: ", err) - } - - t.Run("give state = 0, node not initialized", func(t *testing.T) { - gateway := &MockGateway{} - gateway.reg = func(addr string, peerKey ed25519.PublicKey) (adnl.Peer, error) { - return MockADNL{ - query: func(ctx context.Context, req, result tl.Serializable) error { - switch request := req.(type) { - case Ping: - reflect.ValueOf(result).Elem().Set(reflect.ValueOf(Pong{ID: request.ID})) - default: - return fmt.Errorf("mock err: unsupported request type '%s'", reflect.TypeOf(request).String()) - } - return nil - }, - }, nil - } - tNode.adnl = MockADNL{} - tNode.currentState = 1 - tNode.changeState(0) - if err != nil { - t.Fatal("failed to execute 'changeState' func, err: ", err) - } - if tNode.adnl != nil { - t.Errorf("got not nil adnl after seting '_StateOffline' whit uncconnected node") - } - if tNode.currentState != 0 { - t.Errorf("got not '0' currentState after seting '_StateOffline' whit uncconnected node") - } - }) - - t.Run("give state = 0, node initialized", func(t *testing.T) { - gateway := &MockGateway{} - gateway.reg = func(addr string, peerKey ed25519.PublicKey) (adnl.Peer, error) { - return MockADNL{ - query: func(ctx context.Context, req, result tl.Serializable) error { - switch request := req.(type) { - case Ping: - reflect.ValueOf(result).Elem().Set(reflect.ValueOf(Pong{ID: request.ID})) - default: - return fmt.Errorf("mock err: unsupported request type '%s'", reflect.TypeOf(request).String()) - } - return nil - }, - }, nil - } - tNode.adnl = MockADNL{ - query: func(ctx context.Context, req, result tl.Serializable) error { - switch request := req.(type) { - case Ping: - reflect.ValueOf(result).Elem().Set(reflect.ValueOf(Pong{ID: request.ID})) - default: - return fmt.Errorf("mock err: unsupported request type '%s'", reflect.TypeOf(request).String()) - } - return nil - }, - } - tNode.currentState = _StateActive - - tNode.changeState(0) - if err != nil { - t.Fatal("failed to execute 'changeState' func, err: ", err) - } - if tNode.adnl != nil { - t.Errorf("got not nil adnl in case of disconnected node") - } - if tNode.currentState != 0 { - t.Errorf("got '%d' currentState in case of connected node want 0", tNode.currentState) - } - }) - t.Run("give state = 0, node initialized but throttle", func(t *testing.T) { - gateway := &MockGateway{} - gateway.reg = func(addr string, peerKey ed25519.PublicKey) (adnl.Peer, error) { - return MockADNL{ - query: func(ctx context.Context, req, result tl.Serializable) error { - switch request := req.(type) { - case Ping: - time.Sleep(2 * time.Second) - reflect.ValueOf(result).Elem().Set(reflect.ValueOf(Pong{ID: request.ID})) - default: - return fmt.Errorf("mock err: unsupported request type '%s'", reflect.TypeOf(request).String()) - } - return nil - }, - }, nil - } - tNode.adnl = MockADNL{ - query: func(ctx context.Context, req, result tl.Serializable) error { - switch request := req.(type) { - case Ping: - reflect.ValueOf(result).Elem().Set(reflect.ValueOf(Pong{ID: request.ID})) - default: - return fmt.Errorf("mock err: unsupported request type '%s'", reflect.TypeOf(request).String()) - } - return nil - }, - } - tNode.currentState = 1 - tNode.changeState(0) - if err != nil { - t.Fatal("failed to execute 'changeState' func, err: ", err) - } - if tNode.adnl != nil { - t.Errorf("got not nil adnl in case of disconnected node") - } - if tNode.currentState != 0 { - t.Errorf("got '%d' currentState in case of connected node want 0", tNode.currentState) - } - }) -} - func TestNode_findNodes(t *testing.T) { tDhtNode, err := newCorrectDhtNode(1, 2, 3, 4, "12356") if err != nil { @@ -222,7 +44,7 @@ func TestNode_findNodes(t *testing.T) { if err != nil { t.Fatal("failed to prepare test pub key, err: ", err) } - pubKeyAdnl := adnl.PublicKeyED25519{pubKey} + pubKeyAdnl := adnl.PublicKeyED25519{Key: pubKey} idKey, err := adnl.ToKeyID(pubKeyAdnl) if err != nil { @@ -244,6 +66,9 @@ func TestNode_findNodes(t *testing.T) { } t.Run("good response", func(t *testing.T) { gateway := &MockGateway{} + client := &Client{ + gateway: gateway, + } gateway.reg = func(addr string, peerKey ed25519.PublicKey) (adnl.Peer, error) { return MockADNL{ query: func(ctx context.Context, req, result tl.Serializable) error { @@ -268,11 +93,7 @@ func TestNode_findNodes(t *testing.T) { }, nil } - tDhtNode.adnl, err = gateway.RegisterClient(tDhtNode.addr, tDhtNode.serverKey) - if err != nil { - t.Fatal("failed to prepare test adnl connection, err: ", err) - } - + tDhtNode.client = client nodesL, err := tDhtNode.findNodes(context.Background(), kId, 10) if err != nil { t.Fatal("failed to execute findNodes func, err: ", err) @@ -284,6 +105,9 @@ func TestNode_findNodes(t *testing.T) { t.Run("bad response", func(t *testing.T) { gateway := &MockGateway{} + client := &Client{ + gateway: gateway, + } gateway.reg = func(addr string, peerKey ed25519.PublicKey) (adnl.Peer, error) { return MockADNL{ query: func(ctx context.Context, req, result tl.Serializable) error { @@ -307,10 +131,7 @@ func TestNode_findNodes(t *testing.T) { }, }, nil } - tDhtNode.adnl, err = gateway.RegisterClient(tDhtNode.addr, tDhtNode.serverKey) - if err != nil { - t.Fatal("failed to prepare test adnl connection, err: ", err) - } + tDhtNode.client = client _, err := tDhtNode.findNodes(context.Background(), kId, 10) if err == nil { @@ -348,6 +169,9 @@ func TestNode_storeValue(t *testing.T) { t.Run("good response", func(t *testing.T) { gateway := &MockGateway{} + client := &Client{ + gateway: gateway, + } gateway.reg = func(addr string, peerKey ed25519.PublicKey) (adnl.Peer, error) { return MockADNL{ query: func(ctx context.Context, req, result tl.Serializable) error { @@ -371,11 +195,7 @@ func TestNode_storeValue(t *testing.T) { }, }, nil } - - tDhtNode.adnl, err = gateway.RegisterClient(tDhtNode.addr, tDhtNode.serverKey) - if err != nil { - t.Fatal("failed to prepare test adnl connection, err: ", err) - } + tDhtNode.client = client err := tDhtNode.storeValue(context.Background(), kId, &val) if err != nil { @@ -385,6 +205,9 @@ func TestNode_storeValue(t *testing.T) { t.Run("bad response", func(t *testing.T) { gateway := &MockGateway{} + client := &Client{ + gateway: gateway, + } gateway.reg = func(addr string, peerKey ed25519.PublicKey) (adnl.Peer, error) { return MockADNL{ query: func(ctx context.Context, req, result tl.Serializable) error { @@ -408,10 +231,7 @@ func TestNode_storeValue(t *testing.T) { }, }, nil } - tDhtNode.adnl, err = gateway.RegisterClient(tDhtNode.addr, tDhtNode.serverKey) - if err != nil { - t.Fatal("failed to prepare test adnl connection, err: ", err) - } + tDhtNode.client = client err := tDhtNode.storeValue(context.Background(), kId, &val) if err == nil { @@ -492,10 +312,7 @@ func TestNode_findValue(t *testing.T) { }, nil } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - cli, err := NewClientFromConfig(ctx, gateway, cnf) + cli, err := NewClientFromConfig(gateway, cnf) if err != nil { t.Fatal(err) } @@ -503,7 +320,7 @@ func TestNode_findValue(t *testing.T) { time.Sleep(2 * time.Second) var testNode *dhtNode - for _, val := range cli.activeNodes { + for _, val := range cli.knownNodes { testNode = val } @@ -595,134 +412,6 @@ func TestNode_checkValue(t *testing.T) { //}) } -func TestNode_checkPing(t *testing.T) { - tDhtNode, err := newCorrectDhtNode(1, 2, 3, 4, "5678") - if err != nil { - t.Fatal("failed to prepare correct test node, err: ", err) - } - - t.Run("connected node", func(t *testing.T) { - gateway := &MockGateway{} - gateway.reg = func(addr string, peerKey ed25519.PublicKey) (adnl.Peer, error) { - return MockADNL{ - query: func(ctx context.Context, req, result tl.Serializable) error { - switch request := req.(type) { - case Ping: - reflect.ValueOf(result).Elem().Set(reflect.ValueOf(Pong{ID: request.ID})) - default: - return fmt.Errorf("mock err: unsupported request type '%s'", reflect.TypeOf(request).String()) - } - return nil - }, - }, nil - } - tDhtNode.adnl, err = gateway.RegisterClient(tDhtNode.addr, tDhtNode.serverKey) - if err != nil { - t.Fatal("failed to prepare test adnl connection, err: ", err) - } - err := tDhtNode.checkPing(context.Background()) - if err != nil { - t.Fatal("failed to execute 'checkPing' func, err: ", err) - } - if tDhtNode.currentState != 2 { - t.Errorf("got node state '%d', want 2(stateActive))", tDhtNode.currentState) - } - }) - - t.Run("throttle node", func(t *testing.T) { - gateway := &MockGateway{} - gateway.reg = func(addr string, peerKey ed25519.PublicKey) (adnl.Peer, error) { - return MockADNL{ - query: func(ctx context.Context, req, result tl.Serializable) error { - switch request := req.(type) { - case Ping: - time.Sleep(2 * time.Second) - reflect.ValueOf(result).Elem().Set(reflect.ValueOf(Pong{ID: request.ID})) - default: - return fmt.Errorf("mock err: unsupported request type '%s'", reflect.TypeOf(request).String()) - } - return nil - }, - }, nil - } - tDhtNode.adnl, err = gateway.RegisterClient(tDhtNode.addr, tDhtNode.serverKey) - if err != nil { - t.Fatal("failed to prepare test adnl connection, err: ", err) - } - err := tDhtNode.checkPing(context.Background()) - if err != nil { - t.Fatal("failed to execute 'checkPing' func, err: ", err) - } - if tDhtNode.currentState != 1 { - t.Errorf("got node state '%d', want 2(state throttle))", tDhtNode.currentState) - } - }) - - t.Run("disconnected node", func(t *testing.T) { - gateway := &MockGateway{} - gateway.reg = func(addr string, peerKey ed25519.PublicKey) (adnl.Peer, error) { - return MockADNL{ - query: func(ctx context.Context, req, result tl.Serializable) error { - switch request := req.(type) { - case Ping: - time.Sleep(2 * time.Second) - return ctx.Err() - default: - return fmt.Errorf("mock err: unsupported request type '%s'", reflect.TypeOf(request).String()) - } - }, - }, nil - } - tDhtNode.adnl, err = gateway.RegisterClient(tDhtNode.addr, tDhtNode.serverKey) - if err != nil { - t.Fatal("failed to prepare test adnl connection, err: ", err) - } - - ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) - err := tDhtNode.checkPing(ctx) - if err != nil { - if strings.Contains(err.Error(), "deadline exceeded") != true { - t.Fatal("failed to execute 'checkPing' func, err: ", err) - } - if tDhtNode.currentState != 0 { - t.Errorf("got node state '%d', want 0(state fail))", tDhtNode.currentState) - } - } - }) - - t.Run("wrong pong", func(t *testing.T) { - gateway := &MockGateway{} - gateway.reg = func(addr string, peerKey ed25519.PublicKey) (adnl.Peer, error) { - return MockADNL{ - query: func(ctx context.Context, req, result tl.Serializable) error { - switch request := req.(type) { - case Ping: - reflect.ValueOf(result).Elem().Set(reflect.ValueOf(Pong{ID: request.ID + 1})) - default: - return fmt.Errorf("mock err: unsupported request type '%s'", reflect.TypeOf(request).String()) - } - return nil - }, - }, nil - } - tDhtNode.adnl, err = gateway.RegisterClient(tDhtNode.addr, tDhtNode.serverKey) - if err != nil { - t.Fatal("failed to prepare test adnl connection, err: ", err) - } - - ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) - err := tDhtNode.checkPing(ctx) - if err != nil { - if strings.Contains(err.Error(), "wrong pong id") != true { - t.Fatal("failed to execute 'checkPing' func, err: ", err) - } - if tDhtNode.currentState != _StateActive { - t.Errorf("got node state '%d', want 0(state fail))", tDhtNode.currentState) - } - } - }) -} - func TestNode_weight(t *testing.T) { tPubKey, err := hex.DecodeString("75b9507dc58a931ea6e860d444987e82d8501e09191264c35b95f6956d8debe4") if err != nil { @@ -734,14 +423,12 @@ func TestNode_weight(t *testing.T) { t.Fatal("failed to prepare test key id, err: ", err) } tNode1 := &dhtNode{ - id: kId, - adnl: nil, - ping: 0, - addr: net.IPv4(1, 2, 3, 4).To4().String() + ":" + "35465", - serverKey: tPubKey, - onStateChange: func(node *dhtNode, state int) {}, - currentState: _StateActive, - mx: sync.Mutex{}, + adnlId: kId, + ping: 0, + addr: net.IPv4(1, 2, 3, 4).To4().String() + ":" + "35465", + serverKey: tPubKey, + currentState: _StateActive, + mx: sync.Mutex{}, } tPubKey, err = hex.DecodeString("4680cd40ea26311fe68a6ca0a3dd48aae19561b915ca870b2412d846ae8f53ae") @@ -754,14 +441,12 @@ func TestNode_weight(t *testing.T) { t.Fatal("failed to prepare test key id, err: ", err) } tNode2 := &dhtNode{ - id: kId, - adnl: nil, - ping: 0, - addr: net.IPv4(1, 2, 3, 4).To4().String() + ":" + "35465", - serverKey: tPubKey, - onStateChange: func(node *dhtNode, state int) {}, - currentState: _StateFail, - mx: sync.Mutex{}, + adnlId: kId, + ping: 0, + addr: net.IPv4(1, 2, 3, 4).To4().String() + ":" + "35465", + serverKey: tPubKey, + currentState: _StateFail, + mx: sync.Mutex{}, } tPubKey, err = hex.DecodeString("63c92be0faffbda7dcc32a4380a19c98a75a6d58b9aceadb02cc0bc0bfd6b7d3") @@ -774,14 +459,12 @@ func TestNode_weight(t *testing.T) { t.Fatal("failed to prepare test key id, err: ", err) } tNode3 := &dhtNode{ - id: kId, - adnl: nil, - ping: 0, - addr: net.IPv4(1, 2, 3, 4).To4().String() + ":" + "35465", - serverKey: tPubKey, - onStateChange: func(node *dhtNode, state int) {}, - currentState: _StateActive, - mx: sync.Mutex{}, + adnlId: kId, + ping: 0, + addr: net.IPv4(1, 2, 3, 4).To4().String() + ":" + "35465", + serverKey: tPubKey, + currentState: _StateActive, + mx: sync.Mutex{}, } tests := []struct { diff --git a/adnl/dht/priority_test.go b/adnl/dht/priority_test.go index 9ade30f2..92d7a6b2 100644 --- a/adnl/dht/priority_test.go +++ b/adnl/dht/priority_test.go @@ -57,20 +57,17 @@ func TestPriorityList_addNode(t *testing.T) { } node1 := dhtNode{ - id: kId1, - adnl: nil, + adnlId: kId1, currentState: _StateActive, } node2 := dhtNode{ - id: kId2, - adnl: nil, + adnlId: kId2, currentState: _StateFail, } node3 := dhtNode{ - id: kId3, - adnl: nil, + adnlId: kId3, currentState: _StateActive, } @@ -95,8 +92,8 @@ func TestPriorityList_addNode(t *testing.T) { t.Run("nodes priority order", func(t *testing.T) { for nodeNo := 1; nodeNo < 4; nodeNo++ { node, _ := pList.getNode() - if nodeOrder[hex.EncodeToString(node.id)] != uint8(nodeNo) { - t.Errorf("want node index '%d', got '%d'", nodeOrder[hex.EncodeToString(node.id)], uint8(nodeNo)) + if nodeOrder[node.id()] != uint8(nodeNo) { + t.Errorf("want node index '%d', got '%d'", nodeOrder[node.id()], uint8(nodeNo)) } } }) @@ -152,18 +149,15 @@ func TestPriorityList_markNotUsed(t *testing.T) { } node1 := dhtNode{ - id: kId1, - adnl: nil, + adnlId: kId1, } node2 := dhtNode{ - id: kId2, - adnl: nil, + adnlId: kId2, } node3 := dhtNode{ - id: kId3, - adnl: nil, + adnlId: kId3, } pList := newPriorityList(12, keyId) @@ -194,7 +188,7 @@ func TestPriorityList_markNotUsed(t *testing.T) { curNode = pList.list for curNode != nil { - if bytes.Equal(curNode.node.id, usedNode.id) { + if bytes.Equal(curNode.node.adnlId, usedNode.adnlId) { if curNode.used != false { t.Errorf("want 'false' use status, got '%v'", curNode.used) } diff --git a/example/send-to-contract/main.go b/example/send-to-contract/main.go index 74b45bdb..9868422f 100644 --- a/example/send-to-contract/main.go +++ b/example/send-to-contract/main.go @@ -53,7 +53,7 @@ func main() { if balance.NanoTON().Uint64() >= 3000000 { // create transaction body cell, depends on what contract needs, just random example here body := cell.BeginCell(). - MustStoreUInt(0x123abc55, 32). // op code + MustStoreUInt(0x123abc55, 32). // op code MustStoreUInt(rand.Uint64(), 64). // query id // payload: MustStoreAddr(address.MustParseAddr("EQAbMQzuuGiCne0R7QEj9nrXsjM7gNjeVmrlBZouyC-SCLlO")). @@ -63,6 +63,22 @@ func main() { EndCell(), ).EndCell() + /* + // alternative, more high level way to serialize cell; see tlb.LoadFromCell method for doc + type ContractRequest struct { + _ tlb.Magic `tlb:"#123abc55"` + QueryID uint64 `tlb:"## 64"` + Addr *address.Address `tlb:"addr"` + RefMoney tlb.Coins `tlb:"^"` + } + + body, err := tlb.ToCell(ContractRequest{ + QueryID: rand.Uint64(), + Addr: address.MustParseAddr("EQAbMQzuuGiCne0R7QEj9nrXsjM7gNjeVmrlBZouyC-SCLlO"), + RefMoney: tlb.MustFromTON("1.521"), + }) + */ + log.Println("sending transaction and waiting for confirmation...") tx, block, err := w.SendWaitTransaction(context.Background(), &wallet.Message{ diff --git a/example/wallet-cold-alike/main.go b/example/wallet-cold-alike/main.go new file mode 100644 index 00000000..9d1cbac1 --- /dev/null +++ b/example/wallet-cold-alike/main.go @@ -0,0 +1,91 @@ +package main + +import ( + "context" + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/liteclient" + "github.com/xssnick/tonutils-go/tlb" + "github.com/xssnick/tonutils-go/ton" + "github.com/xssnick/tonutils-go/ton/wallet" + "log" + "strings" +) + +func main() { + client := liteclient.NewConnectionPool() + + // connect to mainnet lite server + err := client.AddConnection(context.Background(), "135.181.140.212:13206", "K0t3+IWLOXHYMvMcrGZDPs+pn58a17LFbnXoQkKc2xw=") + if err != nil { + log.Fatalln("connection err: ", err.Error()) + return + } + + api := ton.NewAPIClient(client) + // bound all requests to single ton node + ctx := client.StickyContext(context.Background()) + + // seed words of account, you can generate them with any wallet or using wallet.NewSeed() method + words := strings.Split("birth pattern then forest walnut then phrase walnut fan pumpkin pattern then cluster blossom verify then forest velvet pond fiction pattern collect then then", " ") + + w, err := wallet.FromSeed(api, words, wallet.V3) + if err != nil { + log.Fatalln("FromSeed err:", err.Error()) + return + } + + log.Println("wallet address:", w.Address()) + + block, err := api.CurrentMasterchainInfo(ctx) + if err != nil { + log.Fatalln("CurrentMasterchainInfo err:", err.Error()) + return + } + + balance, err := w.GetBalance(ctx, block) + if err != nil { + log.Fatalln("GetBalance err:", err.Error()) + return + } + + if balance.NanoTON().Uint64() >= 3000000 { + addr := address.MustParseAddr("EQCD39VS5jcptHL8vMjEXrzGaRcCVYto7HUn4bpAOg8xqB2N") + + log.Println("sending transaction and waiting for confirmation...") + + // default message ttl is 3 minutes, it is time during which you can send it to blockchain + // if you need to set longer TTL, you could use this method + // w.GetSpec().(*wallet.SpecV3).SetMessagesTTL(uint32((10 * time.Minute) / time.Second)) + + // if destination wallet is not initialized you should set bounce = true + msg, err := w.BuildTransfer(addr, tlb.MustFromTON("0.003"), false, "Hello from tonutils-go!") + if err != nil { + log.Fatalln("BuildTransfer err:", err.Error()) + return + } + + // pack message to send later or from other place + ext, err := w.BuildExternalMessage(ctx, msg) + if err != nil { + log.Fatalln("BuildExternalMessage err:", err.Error()) + return + } + + // if you wish to send it from diff source, or later, you could serialize it to BoC + // msgCell, _ := ext.ToCell() + // log.Println(base64.StdEncoding.EncodeToString(msgCell.ToBOC())) + + // send message to blockchain + err = api.SendExternalMessage(ctx, ext) + if err != nil { + log.Fatalln("Failed to send external message:", err.Error()) + return + } + + log.Println("transaction sent") + + return + } + + log.Println("not enough balance:", balance.TON()) +} diff --git a/tlb/account.go b/tlb/account.go index d6ce5c2c..8b03a001 100644 --- a/tlb/account.go +++ b/tlb/account.go @@ -39,6 +39,12 @@ type DepthBalanceInfo struct { Currencies CurrencyCollection `tlb:"."` } +type ShardAccount struct { + Account *cell.Cell `tlb:"^"` + LastTransHash []byte `tlb:"bits 256"` + LastTransLT uint64 `tlb:"## 64"` +} + type AccountStorage struct { Status AccountStatus LastTransactionLT uint64 diff --git a/tlb/block.go b/tlb/block.go index b0d31849..5c142e23 100644 --- a/tlb/block.go +++ b/tlb/block.go @@ -1,6 +1,7 @@ package tlb import ( + "bytes" "fmt" "github.com/xssnick/tonutils-go/tvm/cell" @@ -44,7 +45,7 @@ type ShardAccountBlocks struct { type AccountBlock struct { _ Magic `tlb:"#5"` Addr []byte `tlb:"bits 256"` - Transactions *cell.Dictionary `tlb:"dict 64"` + Transactions *cell.Dictionary `tlb:"dict inline 64"` StateUpdate *cell.Cell `tlb:"^"` } @@ -111,6 +112,11 @@ type BlkPrevInfo struct { Prev2 *ExtBlkRef } +func (h *BlockInfo) Equals(h2 *BlockInfo) bool { + return h.Shard == h2.Shard && h.SeqNo == h2.SeqNo && h.Workchain == h2.Workchain && + bytes.Equal(h.FileHash, h2.FileHash) && bytes.Equal(h.RootHash, h2.RootHash) +} + func (h *BlockHeader) LoadFromCell(loader *cell.Slice) error { var infoPart blockInfoPart err := LoadFromCell(&infoPart, loader) diff --git a/tlb/loader.go b/tlb/loader.go index d9c34586..e0c5cebc 100644 --- a/tlb/loader.go +++ b/tlb/loader.go @@ -37,6 +37,14 @@ type manualStore interface { // _ Magic `tlb:"#deadbeef" // _ Magic `tlb:"$1101" func LoadFromCell(v any, loader *cell.Slice, skipMagic ...bool) error { + return loadFromCell(v, loader, false, len(skipMagic) > 0 && skipMagic[0]) +} + +func LoadFromCellAsProof(v any, loader *cell.Slice, skipMagic ...bool) error { + return loadFromCell(v, loader, true, len(skipMagic) > 0 && skipMagic[0]) +} + +func loadFromCell(v any, loader *cell.Slice, skipProofBranches, skipMagic bool) error { rv := reflect.ValueOf(v) if rv.Kind() != reflect.Pointer || rv.IsNil() { return fmt.Errorf("v should be a pointer and not nil") @@ -137,7 +145,7 @@ func LoadFromCell(v any, loader *cell.Slice, skipMagic ...bool) error { case reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8, reflect.Int: x, err = loader.LoadInt(uint(num)) if err != nil { - return fmt.Errorf("failed to load int %d, err: %w", num, err) + return fmt.Errorf("failed to load %s int %d, err: %w", structField.Name, num, err) } switch parseType.Kind() { @@ -153,7 +161,7 @@ func LoadFromCell(v any, loader *cell.Slice, skipMagic ...bool) error { case reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uint: x, err = loader.LoadUInt(uint(num)) if err != nil { - return fmt.Errorf("failed to load uint %d, err: %w", num, err) + return fmt.Errorf("failed to load %s uint %d, err: %w", structField.Name, num, err) } switch parseType.Kind() { @@ -222,11 +230,16 @@ func LoadFromCell(v any, loader *cell.Slice, skipMagic ...bool) error { next := loader if settings[0] == "^" { - ref, err := loader.LoadRef() + ref, err := loader.LoadRefCell() if err != nil { return fmt.Errorf("failed to load ref for %s, err: %w", structField.Name, err) } - next = ref + + if skipProofBranches && ref.GetType() == cell.PrunedCellType { + continue + } + + next = ref.BeginParse() } switch parseType { @@ -248,7 +261,7 @@ func LoadFromCell(v any, loader *cell.Slice, skipMagic ...bool) error { continue } } else if parseType == reflect.TypeOf(Magic{}) { - if len(skipMagic) > 0 && skipMagic[0] { + if skipMagic { // it can be skipped if parsed before in parent type, to determine child type continue } @@ -283,14 +296,28 @@ func LoadFromCell(v any, loader *cell.Slice, skipMagic ...bool) error { } continue } else if settings[0] == "dict" { + inline := false + if settings[1] == "inline" { + settings = settings[1:] + inline = true + } + sz, err := strconv.ParseUint(settings[1], 10, 64) if err != nil { panic(fmt.Sprintf("cannot deserialize field '%s' as dict, bad size '%s'", structField.Name, settings[1])) } - dict, err := loader.LoadDict(uint(sz)) - if err != nil { - return fmt.Errorf("failed to load ref for %s, err: %w", structField.Name, err) + var dict *cell.Dictionary + if inline { + dict, err = loader.ToDict(uint(sz)) + if err != nil { + return fmt.Errorf("failed to load dict for %s, err: %w", structField.Name, err) + } + } else { + dict, err = loader.LoadDict(uint(sz)) + if err != nil { + return fmt.Errorf("failed to load ref for %s, err: %w", structField.Name, err) + } } if len(settings) >= 4 { diff --git a/ton/api.go b/ton/api.go index 6ea0d69f..d0f526d1 100644 --- a/ton/api.go +++ b/ton/api.go @@ -55,6 +55,7 @@ type APIClient struct { curMasters map[uint32]*masterInfo curMastersLock sync.RWMutex + skipProofCheck bool } type masterInfo struct { @@ -70,6 +71,10 @@ func NewAPIClient(client LiteClient) *APIClient { } } +func (c *APIClient) SetSkipProofCheck(skip bool) { + c.skipProofCheck = skip +} + func (c *APIClient) WaitForBlock(seqno uint32) APIClientWaiter { return &APIClient{ parent: c, diff --git a/ton/block.go b/ton/block.go index 8c6e3701..c499ed1a 100644 --- a/ton/block.go +++ b/ton/block.go @@ -1,7 +1,9 @@ package ton import ( + "bytes" "context" + "encoding/hex" "errors" "fmt" "time" @@ -105,8 +107,8 @@ type ListBlockTransactions struct { Mode uint32 `tl:"flags"` Count uint32 `tl:"int"` After *TransactionID3 `tl:"?7 struct"` - ReverseOrder *True `tl:"?6 struct boxed"` - WantProof *True `tl:"?5 struct boxed"` + ReverseOrder *True `tl:"?6 struct"` + WantProof *True `tl:"?5 struct"` } type TransactionShortInfo struct { @@ -197,6 +199,7 @@ func (c *APIClient) GetMasterchainInfo(ctx context.Context) (*BlockIDExt, error) switch t := resp.(type) { case MasterchainInfo: + return t.Last, nil case LSError: return nil, t @@ -247,6 +250,10 @@ func (c *APIClient) GetBlockData(ctx context.Context, block *BlockIDExt) (*tlb.B return nil, fmt.Errorf("failed to parse block boc: %w", err) } + if !bytes.Equal(cl.Hash(), block.RootHash) { + return nil, fmt.Errorf("incorrect block") + } + var bData tlb.Block if err = tlb.LoadFromCell(&bData, cl.BeginParse()); err != nil { return nil, fmt.Errorf("failed to parse block data: %w", err) @@ -293,12 +300,18 @@ func (c *APIClient) GetBlockTransactionsV2(ctx context.Context, block *BlockIDEx withAfter = 1 } + mode := 0b111 | (withAfter << 7) + if !c.skipProofCheck { + mode |= 1 << 5 + } + var resp tl.Serializable err := c.client.QueryLiteserver(ctx, ListBlockTransactions{ - Mode: 0b111 | (withAfter << 7), - ID: block, - Count: count, - After: afterTx, + Mode: mode, + ID: block, + Count: count, + After: afterTx, + WantProof: &True{}, }, &resp) if err != nil { return nil, false, err @@ -306,11 +319,37 @@ func (c *APIClient) GetBlockTransactionsV2(ctx context.Context, block *BlockIDEx switch t := resp.(type) { case BlockTransactions: + var shardAccounts tlb.ShardAccountBlocks + + if !c.skipProofCheck { + proof, err := cell.FromBOC(t.Proof) + if err != nil { + return nil, false, fmt.Errorf("failed to parse proof boc: %w", err) + } + + blockProof, err := CheckBlockProof(proof, block.RootHash) + if err != nil { + return nil, false, fmt.Errorf("failed to check block proof: %w", err) + } + + err = tlb.LoadFromCellAsProof(&shardAccounts, blockProof.Extra.ShardAccountBlocks.BeginParse()) + if err != nil { + return nil, false, fmt.Errorf("failed to load shard accounts from proof: %w", err) + } + } + txIds := make([]TransactionShortInfo, 0, len(t.TransactionIds)) for _, id := range t.TransactionIds { if id.LT == 0 || id.Hash == nil || id.Account == nil { return nil, false, fmt.Errorf("invalid ls response, fields are nil") } + + if !c.skipProofCheck { + if err = CheckTransactionProof(id.Hash, id.LT, id.Account, &shardAccounts); err != nil { + return nil, false, fmt.Errorf("incorrect tx %s proof: %w", hex.EncodeToString(id.Hash), err) + } + } + txIds = append(txIds, TransactionShortInfo{ Account: id.Account, LT: id.LT, @@ -334,77 +373,118 @@ func (c *APIClient) GetBlockShardsInfo(ctx context.Context, master *BlockIDExt) switch t := resp.(type) { case AllShardsInfo: - c, err := cell.FromBOC(t.Data) + shardsInfo, err := cell.FromBOC(t.Data) if err != nil { return nil, err } var inf tlb.AllShardsInfo - err = tlb.LoadFromCell(&inf, c.BeginParse()) + err = tlb.LoadFromCell(&inf, shardsInfo.BeginParse()) if err != nil { return nil, err } - var shards []*BlockIDExt - - for _, kv := range inf.ShardHashes.All() { - workchain, err := kv.Key.BeginParse().LoadInt(32) + if !c.skipProofCheck { + proof, err := cell.FromBOCMultiRoot(t.Proof) if err != nil { - return nil, fmt.Errorf("load workchain err: %w", err) + return nil, fmt.Errorf("failed to parse proof boc: %w", err) } - var binTree tlb.BinTree - err = binTree.LoadFromCell(kv.Value.BeginParse().MustLoadRef()) + shardState, err := CheckBlockShardStateProof(proof, master.RootHash) if err != nil { - return nil, fmt.Errorf("load BinTree err: %w", err) + return nil, fmt.Errorf("failed to check proof: %w", err) } - for _, bk := range binTree.All() { - loader := bk.Value.BeginParse() + mcShort := shardState.McStateExtra.BeginParse() + if v, err := mcShort.LoadUInt(16); err != nil || v != 0xcc26 { + return nil, fmt.Errorf("invalic mc extra in proof") + } - ab, err := loader.LoadUInt(4) - if err != nil { - return nil, fmt.Errorf("load ShardDesc magic err: %w", err) - } + dictProof, err := mcShort.LoadMaybeRef() + if err != nil { + return nil, fmt.Errorf("failed to load dict proof: %w", err) + } - switch ab { - case 0xa: - var shardDesc tlb.ShardDesc - if err = tlb.LoadFromCell(&shardDesc, loader, true); err != nil { - return nil, fmt.Errorf("load ShardDesc err: %w", err) - } - shards = append(shards, &BlockIDExt{ - Workchain: int32(workchain), - Shard: shardDesc.NextValidatorShard, - SeqNo: shardDesc.SeqNo, - RootHash: shardDesc.RootHash, - FileHash: shardDesc.FileHash, - }) - case 0xb: - var shardDesc tlb.ShardDescB - if err = tlb.LoadFromCell(&shardDesc, loader, true); err != nil { - return nil, fmt.Errorf("load ShardDescB err: %w", err) - } - shards = append(shards, &BlockIDExt{ - Workchain: int32(workchain), - Shard: shardDesc.NextValidatorShard, - SeqNo: shardDesc.SeqNo, - RootHash: shardDesc.RootHash, - FileHash: shardDesc.FileHash, - }) - default: - return nil, fmt.Errorf("wrong ShardDesc magic: %x", ab) - } + if dictProof == nil && inf.ShardHashes.Size() == 0 { + return []*BlockIDExt{}, nil + } + + if (dictProof == nil) != (inf.ShardHashes.Size() == 0) || + !bytes.Equal(dictProof.MustToCell().Hash(0), shardsInfo.MustPeekRef(0).Hash()) { + return nil, fmt.Errorf("incorrect proof") } } - return shards, nil + return LoadShardsFromHashes(inf.ShardHashes) case LSError: return nil, t } return nil, errUnexpectedResponse(resp) } +func LoadShardsFromHashes(shardHashes *cell.Dictionary) (shards []*BlockIDExt, err error) { + if shardHashes == nil { + return []*BlockIDExt{}, nil + } + + for _, kv := range shardHashes.All() { + workchain, err := kv.Key.BeginParse().LoadInt(32) + if err != nil { + return nil, fmt.Errorf("failed to load workchain: %w", err) + } + + binTreeRef, err := kv.Value.BeginParse().LoadRef() + if err != nil { + return nil, fmt.Errorf("failed to load bin tree ref: %w", err) + } + + var binTree tlb.BinTree + err = binTree.LoadFromCell(binTreeRef) + if err != nil { + return nil, fmt.Errorf("load BinTree err: %w", err) + } + + for _, bk := range binTree.All() { + loader := bk.Value.BeginParse() + + ab, err := loader.LoadUInt(4) + if err != nil { + return nil, fmt.Errorf("load ShardDesc magic err: %w", err) + } + + switch ab { + case 0xa: + var shardDesc tlb.ShardDesc + if err = tlb.LoadFromCell(&shardDesc, loader, true); err != nil { + return nil, fmt.Errorf("load ShardDesc err: %w", err) + } + shards = append(shards, &BlockIDExt{ + Workchain: int32(workchain), + Shard: shardDesc.NextValidatorShard, + SeqNo: shardDesc.SeqNo, + RootHash: shardDesc.RootHash, + FileHash: shardDesc.FileHash, + }) + case 0xb: + var shardDesc tlb.ShardDescB + if err = tlb.LoadFromCell(&shardDesc, loader, true); err != nil { + return nil, fmt.Errorf("load ShardDescB err: %w", err) + } + shards = append(shards, &BlockIDExt{ + Workchain: int32(workchain), + Shard: shardDesc.NextValidatorShard, + SeqNo: shardDesc.SeqNo, + RootHash: shardDesc.RootHash, + FileHash: shardDesc.FileHash, + }) + default: + return nil, fmt.Errorf("wrong ShardDesc magic: %x", ab) + } + } + } + return +} + // WaitNextMasterBlock - wait for the next block of master chain func (c *APIClient) waitMasterBlock(ctx context.Context, seqno uint32) (*BlockIDExt, error) { var timeout = 10 * time.Second diff --git a/ton/dns/integration_test.go b/ton/dns/integration_test.go index 8a2717d3..df218f19 100644 --- a/ton/dns/integration_test.go +++ b/ton/dns/integration_test.go @@ -11,7 +11,7 @@ import ( var client = liteclient.NewConnectionPool() var api = func() *ton.APIClient { - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() err := client.AddConnectionsFromConfigUrl(ctx, "https://ton-blockchain.github.io/global.config.json") diff --git a/ton/getconfig.go b/ton/getconfig.go index 7045553d..70aacec5 100644 --- a/ton/getconfig.go +++ b/ton/getconfig.go @@ -63,20 +63,19 @@ func (c *APIClient) GetBlockchainConfig(ctx context.Context, block *BlockIDExt, switch t := resp.(type) { case ConfigAll: - c, err := cell.FromBOC(t.ConfigProof) + cfgProof, err := cell.FromBOC(t.ConfigProof) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to parse config proof boc: %w", err) } - ref, err := c.BeginParse().LoadRef() + stateProof, err := cell.FromBOC(t.StateProof) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to parse state proof boc: %w", err) } - var state tlb.ShardStateUnsplit - err = tlb.LoadFromCell(&state, ref) + state, err := CheckBlockShardStateProof([]*cell.Cell{cfgProof, stateProof}, block.RootHash) if err != nil { - return nil, err + return nil, fmt.Errorf("incorrect proof: %w", err) } if state.McStateExtra == nil { diff --git a/ton/getstate.go b/ton/getstate.go index bf82a44b..be0518b8 100644 --- a/ton/getstate.go +++ b/ton/getstate.go @@ -1,8 +1,8 @@ package ton import ( + "bytes" "context" - "errors" "fmt" "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/tl" @@ -48,6 +48,10 @@ func (c *APIClient) GetAccount(ctx context.Context, block *BlockIDExt, addr *add switch t := resp.(type) { case AccountState: + if !t.ID.Equals(block) { + return nil, fmt.Errorf("response with incorrect master block") + } + if len(t.State) == 0 { return &tlb.Account{ IsActive: false, @@ -58,53 +62,33 @@ func (c *APIClient) GetAccount(ctx context.Context, block *BlockIDExt, addr *add IsActive: true, } - cls, err := cell.FromBOCMultiRoot(t.Proof) + proof, err := cell.FromBOCMultiRoot(t.Proof) if err != nil { return nil, fmt.Errorf("failed to parse proof boc: %w", err) } - bp := cls[0].BeginParse() - - merkle, err := bp.LoadRef() - if err != nil { - return nil, fmt.Errorf("failed to load ref ShardStateUnsplit: %w", err) - } - - _, err = merkle.LoadRef() - if err != nil { - return nil, fmt.Errorf("failed to load ref ShardState: %w", err) - } - - shardAccounts, err := merkle.LoadRef() - if err != nil { - return nil, fmt.Errorf("failed to load ref ShardState: %w", err) - } - shardAccountsDict, err := shardAccounts.LoadDict(256) - - if shardAccountsDict != nil { - addrKey := cell.BeginCell().MustStoreSlice(addr.Data(), 256).EndCell() - val := shardAccountsDict.Get(addrKey) - if val == nil { - return nil, errors.New("no addr info in proof hashmap") + var shardProof []*cell.Cell + var shardHash []byte + if !c.skipProofCheck && addr.Workchain() != address.MasterchainID { + if len(t.ShardProof) == 0 { + return nil, ErrNoProof } - loadVal := val.BeginParse() - - // skip it - err = tlb.LoadFromCell(new(tlb.DepthBalanceInfo), loadVal) + shardProof, err = cell.FromBOCMultiRoot(t.ShardProof) if err != nil { - return nil, fmt.Errorf("failed to load DepthBalanceInfo: %w", err) + return nil, fmt.Errorf("failed to parse shard proof boc: %w", err) } - acc.LastTxHash, err = loadVal.LoadSlice(256) - if err != nil { - return nil, fmt.Errorf("failed to load LastTxHash: %w", err) + if t.Shard == nil || len(t.Shard.RootHash) != 32 { + return nil, fmt.Errorf("shard block not passed") } - acc.LastTxLT, err = loadVal.LoadUInt(64) - if err != nil { - return nil, fmt.Errorf("failed to load LastTxLT: %w", err) - } + shardHash = t.Shard.RootHash + } + + shardAcc, balanceInfo, err := CheckAccountStateProof(addr, block, proof, shardProof, shardHash, c.skipProofCheck) + if err != nil { + return nil, fmt.Errorf("failed to check acc state proof: %w", err) } stateCell, err := cell.FromBOC(t.State) @@ -112,12 +96,22 @@ func (c *APIClient) GetAccount(ctx context.Context, block *BlockIDExt, addr *add return nil, fmt.Errorf("failed to parse state boc: %w", err) } + if !bytes.Equal(shardAcc.Account.Hash(0), stateCell.Hash()) { + return nil, fmt.Errorf("proof hash not match state account hash") + } + var st tlb.AccountState - err = st.LoadFromCell(stateCell.BeginParse()) - if err != nil { + if err = st.LoadFromCell(stateCell.BeginParse()); err != nil { return nil, fmt.Errorf("failed to load account state: %w", err) } + if st.Balance.NanoTON().Cmp(balanceInfo.Currencies.Coins.NanoTON()) != 0 { + return nil, fmt.Errorf("proof balance not match state balance") + } + + acc.LastTxHash = shardAcc.LastTransHash + acc.LastTxLT = shardAcc.LastTransLT + if st.Status == tlb.AccountStatusActive { acc.Code = st.StateInit.Code acc.Data = st.StateInit.Data diff --git a/ton/integration_test.go b/ton/integration_test.go index 74124c1c..becddd82 100644 --- a/ton/integration_test.go +++ b/ton/integration_test.go @@ -18,7 +18,7 @@ import ( var apiTestNet = func() *APIClient { client := liteclient.NewConnectionPool() - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() err := client.AddConnectionsFromConfigUrl(ctx, "https://ton-blockchain.github.io/testnet-global.config.json") @@ -32,7 +32,7 @@ var apiTestNet = func() *APIClient { var api = func() *APIClient { client := liteclient.NewConnectionPool() - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() err := client.AddConnectionsFromConfigUrl(ctx, "https://ton-blockchain.github.io/global.config.json") @@ -114,7 +114,7 @@ func TestAPIClient_GetBlockData(t *testing.T) { func TestAPIClient_GetOldBlockData(t *testing.T) { client := liteclient.NewConnectionPool() - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() err := client.AddConnection(ctx, "135.181.177.59:53312", "aF91CuUHuuOv9rm2W5+O/4h38M3sRm40DtSdRxQhmtQ=") @@ -165,7 +165,7 @@ func TestAPIClient_GetOldBlockData(t *testing.T) { } func Test_RunMethod(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() b, err := api.CurrentMasterchainInfo(ctx) @@ -195,7 +195,7 @@ func Test_RunMethod(t *testing.T) { } func Test_ExternalMessage(t *testing.T) { // need to deploy contract on test-net - > than change config to test-net. - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() ctx = apiTestNet.client.StickyContext(ctx) @@ -236,7 +236,7 @@ func Test_ExternalMessage(t *testing.T) { // need to deploy contract on test-net } func Test_Account(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() ctx = api.client.StickyContext(ctx) @@ -297,6 +297,68 @@ func Test_Account(t *testing.T) { } } +func Test_AccountMaster(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + ctx = api.client.StickyContext(ctx) + + b, err := api.GetMasterchainInfo(ctx) + if err != nil { + t.Fatal("get block err:", err.Error()) + return + } + + addr := address.MustParseAddr("Ef9VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVbxn") + res, err := api.WaitForBlock(b.SeqNo).GetAccount(ctx, b, addr) + if err != nil { + t.Fatal("get account err:", err.Error()) + return + } + + if !res.HasGetMethod("list_proposals") { + t.Fatal("has no list_proposals as get method") + } + + fmt.Printf("Is active: %v\n", res.IsActive) + if res.IsActive { + fmt.Printf("Status: %s\n", res.State.Status) + fmt.Printf("Balance: %s TON\n", res.State.Balance.TON()) + if res.Data == nil { + t.Fatal("data null") + } + } else { + t.Fatal("account not active") + } + + // take last tx info from account info + lastHash := res.LastTxHash + lastLt := res.LastTxLT + + fmt.Printf("\nTransactions:\n") + for i := 0; i < 2; i++ { + // last transaction has 0 prev lt + if lastLt == 0 { + break + } + + // load transactions in batches with size 5 + list, err := api.ListTransactions(ctx, addr, 5, lastLt, lastHash) + if err != nil { + t.Fatal("send err:", err.Error()) + return + } + + // oldest = first in list + for _, t := range list { + fmt.Println(t.String()) + } + + // set previous info from the oldest transaction in list + lastHash = list[0].PrevTxHash + lastLt = list[0].PrevTxLT + } +} + func Test_AccountHasMethod(t *testing.T) { connectionPool := liteclient.NewConnectionPool() @@ -450,7 +512,7 @@ func TestAPIClient_WaitNextBlock(t *testing.T) { } func Test_GetTime(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() utime, err := api.GetTime(ctx) @@ -535,7 +597,7 @@ func Test_LSErrorCase(t *testing.T) { func TestAccountStorage_LoadFromCell_ExtraCurrencies(t *testing.T) { client := liteclient.NewConnectionPool() - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() err := client.AddConnection(context.Background(), "135.181.177.59:53312", "aF91CuUHuuOv9rm2W5+O/4h38M3sRm40DtSdRxQhmtQ=") @@ -552,12 +614,22 @@ func TestAccountStorage_LoadFromCell_ExtraCurrencies(t *testing.T) { t.Fatal(err) } - a, err := mainnetAPI.GetAccount(ctx, b, address.MustParseAddr("EQCYv992KVNNCKZHSLLJgM2GGzsgL0UgWP24BCQBaAdqSE2I")) - if err != nil { - t.Fatal(err) - } + t.Run("with proof", func(t *testing.T) { + _, err := mainnetAPI.GetAccount(ctx, b, address.MustParseAddr("EQCYv992KVNNCKZHSLLJgM2GGzsgL0UgWP24BCQBaAdqSE2I")) + if err != ErrNoProof { + t.Fatal(err) + } + }) - if a.State.ExtraCurrencies == nil { - t.Fatal("expected extra currencies dict") - } + t.Run("without proof", func(t *testing.T) { + mainnetAPI.SetSkipProofCheck(true) + a, err := mainnetAPI.GetAccount(ctx, b, address.MustParseAddr("EQCYv992KVNNCKZHSLLJgM2GGzsgL0UgWP24BCQBaAdqSE2I")) + if err != nil { + t.Fatal(err) + } + + if a.State.ExtraCurrencies == nil { + t.Fatal("expected extra currencies dict") + } + }) } diff --git a/ton/jetton/integration_test.go b/ton/jetton/integration_test.go index 4e37b278..a1750843 100644 --- a/ton/jetton/integration_test.go +++ b/ton/jetton/integration_test.go @@ -18,7 +18,7 @@ import ( var api = func() *ton.APIClient { client := liteclient.NewConnectionPool() - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() err := client.AddConnectionsFromConfigUrl(ctx, "https://ton-blockchain.github.io/testnet-global.config.json") diff --git a/ton/proof.go b/ton/proof.go new file mode 100644 index 00000000..106d4fbb --- /dev/null +++ b/ton/proof.go @@ -0,0 +1,186 @@ +package ton + +import ( + "bytes" + "errors" + "fmt" + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/tlb" + "github.com/xssnick/tonutils-go/tvm/cell" +) + +var ErrNoProof = fmt.Errorf("liteserver has no proof for this account in a given block, request newer block or disable proof checks") + +func CheckShardInMasterProof(master *BlockIDExt, shardProof []*cell.Cell, workchain int32, shardRootHash []byte) error { + shardState, err := CheckBlockShardStateProof(shardProof, master.RootHash) + if err != nil { + return fmt.Errorf("check block proof failed: %w", err) + } + + if shardState.McStateExtra == nil { + return fmt.Errorf("not a masterchain block") + } + + var stateExtra tlb.McStateExtra + err = tlb.LoadFromCell(&stateExtra, shardState.McStateExtra.BeginParse()) + if err != nil { + return fmt.Errorf("failed to load masterchain state extra: %w", err) + } + + shards, err := LoadShardsFromHashes(stateExtra.ShardHashes) + if err != nil { + return fmt.Errorf("failed to load shard hashes: %w", err) + } + + for _, shard := range shards { + if shard.Workchain == workchain && bytes.Equal(shard.RootHash, shardRootHash) { + return nil + } + } + return fmt.Errorf("required shard hash not found in proof") +} + +func CheckBlockShardStateProof(proof []*cell.Cell, blockRootHash []byte) (*tlb.ShardStateUnsplit, error) { + if len(proof) != 2 { + return nil, fmt.Errorf("should have 2 roots") + } + + block, err := CheckBlockProof(proof[1], blockRootHash) + if err != nil { + return nil, fmt.Errorf("incorrect block proof: %w", err) + } + + upd, err := block.StateUpdate.PeekRef(1) + if err != nil { + return nil, fmt.Errorf("failed to load state update ref: %w", err) + } + + shardStateProofData, err := cell.UnwrapProof(proof[0], upd.Hash(0)) + if err != nil { + return nil, fmt.Errorf("incorrect shard state proof: %w", err) + } + + var shardState tlb.ShardStateUnsplit + if err = tlb.LoadFromCellAsProof(&shardState, shardStateProofData.BeginParse(), false); err != nil { + return nil, fmt.Errorf("failed to parse ShardStateUnsplit: %w", err) + } + + return &shardState, nil +} + +func CheckBlockProof(proof *cell.Cell, blockRootHash []byte) (*tlb.Block, error) { + blockProof, err := cell.UnwrapProof(proof, blockRootHash) + if err != nil { + return nil, fmt.Errorf("block proof check failed: %w", err) + } + + var block tlb.Block + if err := tlb.LoadFromCellAsProof(&block, blockProof.BeginParse(), false); err != nil { + return nil, fmt.Errorf("failed to parse Block: %w", err) + } + + return &block, nil +} + +func CheckAccountStateProof(addr *address.Address, block *BlockIDExt, stateProof []*cell.Cell, shardProof []*cell.Cell, shardHash []byte, skipBlockCheck bool) (*tlb.ShardAccount, *tlb.DepthBalanceInfo, error) { + if len(stateProof) != 2 { + return nil, nil, fmt.Errorf("proof should have 2 roots") + } + + var shardState *tlb.ShardStateUnsplit + + if !skipBlockCheck { + blockHash := block.RootHash + // we need shard proof only for not masterchain + if len(shardHash) > 0 { + if err := CheckShardInMasterProof(block, shardProof, addr.Workchain(), shardHash); err != nil { + return nil, nil, fmt.Errorf("shard proof is incorrect: %w", err) + } + blockHash = shardHash + } + + var err error + shardState, err = CheckBlockShardStateProof(stateProof, blockHash) + if err != nil { + return nil, nil, fmt.Errorf("incorrect block proof: %w", err) + } + } else { + shardStateProofData, err := stateProof[0].BeginParse().LoadRef() + if err != nil { + return nil, nil, fmt.Errorf("shard state proof should have ref: %w", err) + } + + var state tlb.ShardStateUnsplit + if err = tlb.LoadFromCellAsProof(&state, shardStateProofData, false); err != nil { + return nil, nil, fmt.Errorf("failed to parse ShardStateUnsplit: %w", err) + } + shardState = &state + } + + if shardState.Accounts.ShardAccounts == nil { + return nil, nil, errors.New("no shard accounts in proof") + } + + addrKey := cell.BeginCell().MustStoreSlice(addr.Data(), 256).EndCell() + val := shardState.Accounts.ShardAccounts.Get(addrKey) + if val == nil { + return nil, nil, errors.New("no addr info in proof hashmap") + } + + loadVal := val.BeginParse() + + var balanceInfo tlb.DepthBalanceInfo + err := tlb.LoadFromCell(&balanceInfo, loadVal) + if err != nil { + return nil, nil, fmt.Errorf("failed to load DepthBalanceInfo: %w", err) + } + + var accInfo tlb.ShardAccount + err = tlb.LoadFromCell(&accInfo, loadVal) + if err != nil { + return nil, nil, fmt.Errorf("failed to load ShardAccount: %w", err) + } + + return &accInfo, &balanceInfo, nil +} + +func CheckTransactionProof(txHash []byte, txLT uint64, txAccount []byte, shardAccounts *tlb.ShardAccountBlocks) error { + accProof := shardAccounts.Accounts.Get(cell.BeginCell().MustStoreSlice(txAccount, 256).EndCell()) + if accProof == nil { + return fmt.Errorf("no tx account in proof") + } + + accProofSlice := accProof.BeginParse() + err := tlb.LoadFromCellAsProof(new(tlb.CurrencyCollection), accProofSlice) + if err != nil { + return fmt.Errorf("failed to load account CurrencyCollection proof cell: %w", err) + } + + var accBlock tlb.AccountBlock + err = tlb.LoadFromCellAsProof(&accBlock, accProofSlice) + if err != nil { + return fmt.Errorf("failed to load account from proof cell: %w", err) + } + + accTx := accBlock.Transactions.Get(cell.BeginCell().MustStoreUInt(txLT, 64).EndCell()) + if accTx == nil { + return fmt.Errorf("no tx in account block proof") + } + + accTxSlice := accTx.BeginParse() + err = tlb.LoadFromCellAsProof(new(tlb.CurrencyCollection), accTxSlice) + if err != nil { + return fmt.Errorf("failed to load tx CurrencyCollection proof cell: %w", err) + } + + txAccProof, err := accTxSlice.LoadRef() + if err != nil { + return fmt.Errorf("failed to load ref of acc tx proof cell: %w", err) + } + + if !bytes.Equal(txHash, txAccProof.MustToCell().Hash(0)) { + return fmt.Errorf("incorrect tx hash in proof") + } + + return nil +} diff --git a/ton/runmethod.go b/ton/runmethod.go index aa4682f9..daf2f07b 100644 --- a/ton/runmethod.go +++ b/ton/runmethod.go @@ -60,9 +60,14 @@ func (c *APIClient) RunGetMethod(ctx context.Context, blockInfo *BlockIDExt, add return nil, fmt.Errorf("build stack err: %w", err) } + mode := uint32(1 << 2) + if !c.skipProofCheck { + mode |= (1 << 1) | (1 << 0) + } + var resp tl.Serializable err = c.client.QueryLiteserver(ctx, &RunSmcMethod{ - Mode: 1 << 2, // with result + Mode: mode, ID: blockInfo, Account: AccountID{ Workchain: addr.Workchain(), @@ -83,13 +88,56 @@ func (c *APIClient) RunGetMethod(ctx context.Context, blockInfo *BlockIDExt, add } } - cl, err := cell.FromBOC(t.Result) + resCell, err := cell.FromBOC(t.Result) if err != nil { return nil, err } + if !c.skipProofCheck { + proof, err := cell.FromBOCMultiRoot(t.Proof) + if err != nil { + return nil, err + } + + var shardProof []*cell.Cell + var shardHash []byte + if !c.skipProofCheck && addr.Workchain() != address.MasterchainID { + if len(t.ShardProof) == 0 { + return nil, fmt.Errorf("liteserver has no proof for this account in a given block, request newer block or disable proof checks") + } + + shardProof, err = cell.FromBOCMultiRoot(t.ShardProof) + if err != nil { + return nil, fmt.Errorf("failed to parse shard proof boc: %w", err) + } + + if t.ShardBlock == nil || len(t.ShardBlock.RootHash) != 32 { + return nil, fmt.Errorf("shard block not passed") + } + + shardHash = t.ShardBlock.RootHash + } + + shardAcc, balanceInfo, err := CheckAccountStateProof(addr, blockInfo, proof, shardProof, shardHash, c.skipProofCheck) + if err != nil { + return nil, fmt.Errorf("failed to check acc state proof: %w", err) + } + _, _ = shardAcc, balanceInfo + + stateProofCell, err := cell.FromBOC(t.StateProof) + if err != nil { + return nil, err + } + + _, err = cell.UnwrapProof(stateProofCell, shardAcc.Account.Hash(0)) + if err != nil { + return nil, fmt.Errorf("failed to match state proof to state hash: %w", err) + } + // TODO: when tvm implementation ready - execute code and compare result + } + var resStack tlb.Stack - err = resStack.LoadFromCell(cl.BeginParse()) + err = resStack.LoadFromCell(resCell.BeginParse()) if err != nil { return nil, err } diff --git a/ton/transactions.go b/ton/transactions.go index 407051bd..c8a26920 100644 --- a/ton/transactions.go +++ b/ton/transactions.go @@ -41,7 +41,8 @@ type GetTransactions struct { TxHash []byte `tl:"int256"` } -// ListTransactions - returns list of transactions before (including) passed lt and hash, the oldest one is first in result slice +// ListTransactions - [THIS METHOD HAS NO PROOF CHECK, you can use GetTransaction to verify] +// returns list of transactions before (including) passed lt and hash, the oldest one is first in result slice func (c *APIClient) ListTransactions(ctx context.Context, addr *address.Address, limit uint32, lt uint64, txHash []byte) ([]*tlb.Transaction, error) { var resp tl.Serializable err := c.client.QueryLiteserver(ctx, GetTransactions{ @@ -106,7 +107,7 @@ func (c *APIClient) GetTransaction(ctx context.Context, block *BlockIDExt, addr case TransactionInfo: txCell, err := cell.FromBOC(t.Transaction) if err != nil { - return nil, fmt.Errorf("failed to parrse cell from transaction bytes: %w", err) + return nil, fmt.Errorf("failed to parse cell from transaction bytes: %w", err) } var tx tlb.Transaction @@ -114,8 +115,30 @@ func (c *APIClient) GetTransaction(ctx context.Context, block *BlockIDExt, addr if err != nil { return nil, fmt.Errorf("failed to load transaction from cell: %w", err) } - tx.Hash = txCell.Hash() + + if !c.skipProofCheck { + txProof, err := cell.FromBOC(t.Proof) + if err != nil { + return nil, fmt.Errorf("failed to parse proof: %w", err) + } + + blockProof, err := CheckBlockProof(txProof, block.RootHash) + if err != nil { + return nil, fmt.Errorf("failed to check proof: %w", err) + } + + var shardAccounts tlb.ShardAccountBlocks + err = tlb.LoadFromCellAsProof(&shardAccounts, blockProof.Extra.ShardAccountBlocks.BeginParse()) + if err != nil { + return nil, fmt.Errorf("failed to load shard accounts from proof: %w", err) + } + + if err = CheckTransactionProof(tx.Hash, tx.LT, tx.AccountAddr, &shardAccounts); err != nil { + return nil, fmt.Errorf("incorrect tx proof: %w", err) + } + } + return &tx, nil case LSError: if t.Code == 0 { diff --git a/ton/wallet/address.go b/ton/wallet/address.go index 1065103b..cb261406 100644 --- a/ton/wallet/address.go +++ b/ton/wallet/address.go @@ -54,20 +54,20 @@ func GetStateInit(pubKey ed25519.PublicKey, ver Version, subWallet uint32) (*tlb var data *cell.Cell switch ver { - case V3: + case V3R1, V3R2: data = cell.BeginCell(). MustStoreUInt(0, 32). // seqno MustStoreUInt(uint64(subWallet), 32). // sub wallet MustStoreSlice(pubKey, 256). EndCell() - case V4R2: + case V4R1, V4R2: data = cell.BeginCell(). MustStoreUInt(0, 32). // seqno MustStoreUInt(uint64(subWallet), 32). MustStoreSlice(pubKey, 256). MustStoreDict(nil). // empty dict of plugins EndCell() - case HighloadV2R2: + case HighloadV2R2, HighloadV2Verified: data = cell.BeginCell(). MustStoreUInt(uint64(subWallet), 32). MustStoreUInt(0, 64). // last cleaned diff --git a/ton/wallet/highloadv2r2.go b/ton/wallet/highloadv2r2.go index dd5b639b..ffd25ff3 100644 --- a/ton/wallet/highloadv2r2.go +++ b/ton/wallet/highloadv2r2.go @@ -13,6 +13,9 @@ import ( // converted to hex from https://github.com/toncenter/tonweb/blob/0a5effd36a3f342f4aacabab728b1f9747085ad1/src/contract/wallet/WalletSourcesFromCPP.txt#L18 const _HighloadV2R2CodeHex = "B5EE9C720101090100E9000114FF00F4A413F4BCF2C80B010201200203020148040501EEF28308D71820D31FD33FF823AA1F5320B9F263ED44D0D31FD33FD3FFF404D153608040F40E6FA131F2605173BAF2A207F901541087F910F2A302F404D1F8007F8E18218010F4786FA16FA1209802D307D43001FB009132E201B3E65B8325A1C840348040F4438AE631C812CB1F13CB3FCBFFF400C9ED54080004D03002012006070017BD9CE76A26869AF98EB85FFC0041BE5F976A268698F98E99FE9FF98FA0268A91040207A0737D098C92DBFC95DD1F140038208040F4966FA16FA132511094305303B9DE2093333601923230E2B3" +// technically 99% same with _HighloadV2R2CodeHex, but verified, used verified source of https://verifier.ton.org/EQB9oQAr5-OZVQAQePzWOrBwwhnIOh1OuVrYUDqfzXebXz5h +const _HighloadV2VerifiedCodeHex = "b5ee9c724101090100e5000114ff00f4a413f4bcf2c80b010201200203020148040501eaf28308d71820d31fd33ff823aa1f5320b9f263ed44d0d31fd33fd3fff404d153608040f40e6fa131f2605173baf2a207f901541087f910f2a302f404d1f8007f8e16218010f4786fa5209802d307d43001fb009132e201b3e65b8325a1c840348040f4438ae63101c8cb1f13cb3fcbfff400c9ed54080004d03002012006070017bd9ce76a26869af98eb85ffc0041be5f976a268698f98e99fe9ff98fa0268a91040207a0737d098c92dbfc95dd1f140034208040f4966fa56c122094305303b9de2093333601926c21e2b39f9e545a" + type SpecHighloadV2R2 struct { SpecRegular } diff --git a/ton/wallet/integration_test.go b/ton/wallet/integration_test.go index 75b49531..c1640f4a 100644 --- a/ton/wallet/integration_test.go +++ b/ton/wallet/integration_test.go @@ -21,7 +21,7 @@ import ( var api = func() *ton.APIClient { client := liteclient.NewConnectionPool() - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() err := client.AddConnectionsFromConfigUrl(ctx, "https://ton-blockchain.github.io/testnet-global.config.json") @@ -35,7 +35,7 @@ var api = func() *ton.APIClient { var apiMain = func() *ton.APIClient { client := liteclient.NewConnectionPool() - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() err := client.AddConnectionsFromConfigUrl(ctx, "https://ton-blockchain.github.io/global.config.json") @@ -51,8 +51,11 @@ var _seed = os.Getenv("WALLET_SEED") func Test_WalletTransfer(t *testing.T) { seed := strings.Split(_seed, " ") - for _, ver := range []Version{V3, V4R2, HighloadV2R2} { + for _, v := range []Version{V3R2, V4R2, HighloadV2R2, V3R1, V4R1, HighloadV2Verified} { + ver := v t.Run("send for wallet ver "+fmt.Sprint(ver), func(t *testing.T) { + t.Parallel() + ctx := api.Client().StickyContext(context.Background()) ctx, cancel := context.WithTimeout(ctx, 120*time.Second) defer cancel() diff --git a/ton/wallet/wallet.go b/ton/wallet/wallet.go index 6e300036..5ccb5cd2 100644 --- a/ton/wallet/wallet.go +++ b/ton/wallet/wallet.go @@ -21,34 +21,40 @@ import ( type Version int const ( - V1R1 Version = 11 - V1R2 Version = 12 - V1R3 Version = 13 - V2R1 Version = 21 - V2R2 Version = 22 - V3R1 Version = 31 - V3R2 Version = 32 - V3 = V3R2 - V4R1 Version = 41 - V4R2 Version = 42 - HighloadV2R2 Version = 122 - Lockup Version = 200 - Unknown Version = 0 + V1R1 Version = 11 + V1R2 Version = 12 + V1R3 Version = 13 + V2R1 Version = 21 + V2R2 Version = 22 + V3R1 Version = 31 + V3R2 Version = 32 + V3 = V3R2 + V4R1 Version = 41 + V4R2 Version = 42 + HighloadV2R2 Version = 122 + HighloadV2Verified Version = 123 + Lockup Version = 200 + Unknown Version = 0 ) func (v Version) String() string { if v == Unknown { return "unknown" } - if v/10 > 0 && v/10 < 10 { - return fmt.Sprintf("V%dR%d", v/10, v%10) - } - if v/100 == 1 { - return fmt.Sprintf("highload V%dR%d", v/100/10, v%10) + + switch v { + case HighloadV2R2: + return fmt.Sprintf("highload V2R2") + case HighloadV2Verified: + return fmt.Sprintf("highload V2R2 verified") } + if v/100 == 2 { return fmt.Sprintf("lockup") } + if v/10 > 0 && v/10 < 10 { + return fmt.Sprintf("V%dR%d", v/10, v%10) + } return fmt.Sprintf("%d", v) } @@ -58,8 +64,8 @@ var ( V2R1: _V2R1CodeHex, V2R2: _V2R2CodeHex, V3R1: _V3R1CodeHex, V3R2: _V3R2CodeHex, V4R1: _V4R1CodeHex, V4R2: _V4R2CodeHex, - HighloadV2R2: _HighloadV2R2CodeHex, - Lockup: _LockupCodeHex, + HighloadV2R2: _HighloadV2R2CodeHex, HighloadV2Verified: _HighloadV2VerifiedCodeHex, + Lockup: _LockupCodeHex, } walletCodeBOC = map[Version][]byte{} walletCode = map[Version]*cell.Cell{} @@ -149,11 +155,11 @@ func getSpec(w *Wallet) (any, error) { } switch w.ver { - case V3: + case V3R1, V3R2: return &SpecV3{regular}, nil - case V4R2: + case V4R1, V4R2: return &SpecV4R2{regular}, nil - case HighloadV2R2: + case HighloadV2R2, HighloadV2Verified: return &SpecHighloadV2R2{regular}, nil } @@ -207,7 +213,16 @@ func (w *Wallet) GetSpec() any { return w.spec } +func (w *Wallet) BuildExternalMessage(ctx context.Context, message *Message) (*tlb.ExternalMessage, error) { + return w.BuildExternalMessageForMany(ctx, []*Message{message}) +} + +// Deprecated: use BuildExternalMessageForMany func (w *Wallet) BuildMessageForMany(ctx context.Context, messages []*Message) (*tlb.ExternalMessage, error) { + return w.BuildExternalMessageForMany(ctx, messages) +} + +func (w *Wallet) BuildExternalMessageForMany(ctx context.Context, messages []*Message) (*tlb.ExternalMessage, error) { var stateInit *tlb.StateInit block, err := w.api.CurrentMasterchainInfo(ctx) @@ -232,12 +247,12 @@ func (w *Wallet) BuildMessageForMany(ctx context.Context, messages []*Message) ( var msg *cell.Cell switch w.ver { - case V3, V4R2: + case V3R2, V3R1, V4R2, V4R1: msg, err = w.spec.(RegularBuilder).BuildMessage(ctx, initialized, block, messages) if err != nil { return nil, fmt.Errorf("build message err: %w", err) } - case HighloadV2R2: + case HighloadV2R2, HighloadV2Verified: msg, err = w.spec.(*SpecHighloadV2R2).BuildMessage(ctx, randUint32(), messages) if err != nil { return nil, fmt.Errorf("build message err: %w", err) @@ -253,6 +268,27 @@ func (w *Wallet) BuildMessageForMany(ctx context.Context, messages []*Message) ( }, nil } +func (w *Wallet) BuildTransfer(to *address.Address, amount tlb.Coins, bounce bool, comment string) (_ *Message, err error) { + var body *cell.Cell + if comment != "" { + body, err = CreateCommentCell(comment) + if err != nil { + return nil, err + } + } + + return &Message{ + Mode: 1 + 2, + InternalMessage: &tlb.InternalMessage{ + IHRDisabled: true, + Bounce: bounce, + DstAddr: to, + Amount: amount, + Body: body, + }, + }, nil +} + func (w *Wallet) Send(ctx context.Context, message *Message, waitConfirmation ...bool) error { return w.SendMany(ctx, []*Message{message}, waitConfirmation...) } @@ -296,7 +332,7 @@ func (w *Wallet) sendMany(ctx context.Context, messages []*Message, waitConfirma return nil, nil, nil, fmt.Errorf("failed to get account state: %w", err) } - ext, err := w.BuildMessageForMany(ctx, messages) + ext, err := w.BuildExternalMessageForMany(ctx, messages) if err != nil { return nil, nil, nil, err } @@ -428,24 +464,11 @@ func CreateCommentCell(text string) (*cell.Cell, error) { } func (w *Wallet) transfer(ctx context.Context, to *address.Address, amount tlb.Coins, comment string, bounce bool, waitConfirmation ...bool) (err error) { - var body *cell.Cell - if comment != "" { - body, err = CreateCommentCell(comment) - if err != nil { - return err - } + transfer, err := w.BuildTransfer(to, amount, bounce, comment) + if err != nil { + return err } - - return w.Send(ctx, &Message{ - Mode: 1, - InternalMessage: &tlb.InternalMessage{ - IHRDisabled: true, - Bounce: bounce, - DstAddr: to, - Amount: amount, - Body: body, - }, - }, waitConfirmation...) + return w.Send(ctx, transfer, waitConfirmation...) } func (w *Wallet) DeployContract(ctx context.Context, amount tlb.Coins, msgBody, contractCode, contractData *cell.Cell, waitConfirmation ...bool) (*address.Address, error) { diff --git a/tvm/cell/cell.go b/tvm/cell/cell.go index ad90a2c7..23dd6142 100644 --- a/tvm/cell/cell.go +++ b/tvm/cell/cell.go @@ -9,13 +9,15 @@ import ( "strings" ) +type Type uint8 + const ( - _OrdinaryType = 0x00 - _PrunedType = 0x01 - _LibraryType = 0x02 - _MerkleProofType = 0x03 - _MerkleUpdateType = 0x04 - _UnknownType = 0xFF + OrdinaryCellType Type = 0x00 + PrunedCellType Type = 0x01 + LibraryCellType Type = 0x02 + MerkleProofCellType Type = 0x03 + MerkleUpdateCellType Type = 0x04 + UnknownCellType Type = 0xFF ) const maxDepth = 1024 @@ -171,7 +173,10 @@ func (c *Cell) dump(deep int, bin bool, limitLength int) string { const _DataCellMaxLevel = 3 -func (c *Cell) Hash() []byte { +func (c *Cell) Hash(level ...int) []byte { + if len(level) > 0 { + return c.getHash(level[0]) + } return c.getHash(_DataCellMaxLevel) } @@ -179,36 +184,37 @@ func (c *Cell) Sign(key ed25519.PrivateKey) []byte { return ed25519.Sign(key, c.Hash()) } -func (c *Cell) getType() int { +func (c *Cell) GetType() Type { if !c.special { - return _OrdinaryType + return OrdinaryCellType } if c.BitsSize() < 8 { - return _UnknownType + return UnknownCellType } - switch c.data[0] { - case _PrunedType: + switch Type(c.data[0]) { + case PrunedCellType: if c.BitsSize() >= 288 { - lvl := uint(c.data[1]) - if lvl > 0 && lvl <= 3 && c.BitsSize() >= 16+(256+16)*lvl { - return _PrunedType + msk := LevelMask{c.data[1]} + lvl := msk.GetLevel() + if lvl > 0 && lvl <= 3 && c.BitsSize() >= 16+(256+16)*(uint(msk.Apply(lvl-1).getHashIndex()+1)) { + return PrunedCellType } } - case _MerkleProofType: + case MerkleProofCellType: if c.RefsNum() == 1 && c.BitsSize() == 280 { - return _MerkleProofType + return MerkleProofCellType } - case _MerkleUpdateType: + case MerkleUpdateCellType: if c.RefsNum() == 2 && c.BitsSize() == 552 { - return _MerkleUpdateType + return MerkleUpdateCellType } - case _LibraryType: + case LibraryCellType: if c.BitsSize() == 8+256 { - return _LibraryType + return LibraryCellType } } - return _UnknownType + return UnknownCellType } func (c *Cell) UnmarshalJSON(bytes []byte) error { diff --git a/tvm/cell/cell_test.go b/tvm/cell/cell_test.go index 0ada4afa..d82d8748 100644 --- a/tvm/cell/cell_test.go +++ b/tvm/cell/cell_test.go @@ -223,3 +223,16 @@ func TestCell_TxWithMerkleBody(t *testing.T) { t.Fatal("incorrect hash", hex.EncodeToString(c.Hash()), hex.EncodeToString(hash)) } } + +func TestCell_ShardStateProof(t *testing.T) { + boc, _ := hex.DecodeString("b5ee9c72410208010001d400241011ef55aaffffff110103050401a09bc7a98700000000040101dfbf480000000100ffffffff000000000000000064c2108900002408eeb249c000002408eeb249c445c88f2e00070e2a01dfbf4401df8515c400000003000000000000002e02009800002408eea3078401dfbf476558f058d895ff9428b62402b459f62752a8a30b646a36f3d708f8f86a881abca5bffca86eda9bfa2efff8b6a1a0d7106945a08693e3350aaaa48bf44f1a61cd28480101f8bb09213adec01589e2b45268648023e8ef1b21af359433e7f4753fc9944f36000328480101858c4166713e4641a997b9df8fa10894a1f9d4b8966366121c6bc932b5e6afcd00072a8a0449f53a9adbf987c1552e753b6779e52e177db12b23502c568c4329f69ae9d86661874499484e58f0a538220fdc12154b0505bfc51888e6636648dd2a22bdbc2d016f016f0706688c010361874499484e58f0a538220fdc12154b0505bfc51888e6636648dd2a22bdbc2d74b93d76a6a8986dfffbe82438fac84f045b49fb868cbcdc5a0ec39c746f35f1016f0016688c010349f53a9adbf987c1552e753b6779e52e177db12b23502c568c4329f69ae9d86646af4ba188c5bba8e8ecbeac5ef9fb0d641a8776206bc4ad17a725dcf876e2c0016f0015e5b85bf3") + c, err := FromBOC(boc) + if err != nil { + t.Fatal(err) + } + + trueHash, _ := hex.DecodeString("BFAA5FC9B4588A4FD58B497E809570C75A01A369A1233817FF16C7360C1755BE") + if !bytes.Equal(c.Hash(0), trueHash) { + t.Fatal("wrong hash:", hex.EncodeToString(c.Hash(0)), "should be", hex.EncodeToString(trueHash)) + } +} diff --git a/tvm/cell/dict.go b/tvm/cell/dict.go index 8d2e3c2b..996f9e92 100644 --- a/tvm/cell/dict.go +++ b/tvm/cell/dict.go @@ -12,6 +12,7 @@ import ( type Dictionary struct { storage map[string]*HashmapKV keySz uint + hash []byte } type HashmapKV struct { @@ -103,6 +104,13 @@ func (d *Dictionary) Get(key *Cell) *Cell { return v.Value } +func (d *Dictionary) Size() int { + if d == nil { + return 0 + } + return len(d.storage) +} + func (d *Dictionary) All() []*HashmapKV { all := make([]*HashmapKV, 0, len(d.storage)) for _, v := range d.storage { diff --git a/tvm/cell/proof.go b/tvm/cell/proof.go index be67e08b..2c296080 100644 --- a/tvm/cell/proof.go +++ b/tvm/cell/proof.go @@ -22,7 +22,7 @@ func (c *Cell) CreateProof(forHashes [][]byte) (*Cell, error) { // build root Merkle Proof cell data := make([]byte, 1+32+2) - data[0] = _MerkleProofType + data[0] = byte(MerkleProofCellType) copy(data[1:], c.getHash(c.levelMask.GetLevel())) binary.BigEndian.PutUint16(data[1+32:], c.getDepth(c.levelMask.GetLevel())) @@ -84,7 +84,7 @@ func (c *Cell) toProof(parts []cellHash) ([]cellHash, error) { ourLvl := ref.levelMask.GetLevel() prunedData := make([]byte, 2+(ourLvl+1)*(32+2)) - prunedData[0] = _PrunedType + prunedData[0] = byte(PrunedCellType) prunedData[1] = byte(ref.levelMask.GetLevel()) + 1 for lvl := 0; lvl <= ourLvl; lvl++ { @@ -101,10 +101,10 @@ func (c *Cell) toProof(parts []cellHash) ([]cellHash, error) { } } - typ := c.getType() + typ := c.GetType() for _, ref := range c.refs { if ref.levelMask.GetLevel() > c.levelMask.GetLevel() { - if typ == _MerkleProofType { + if typ == MerkleProofCellType { // proof should be 1 level less than child c.levelMask = LevelMask{ref.levelMask.Mask - 1} } else { @@ -117,9 +117,14 @@ func (c *Cell) toProof(parts []cellHash) ([]cellHash, error) { } func CheckProof(proof *Cell, hash []byte) error { + _, err := UnwrapProof(proof, hash) + return err +} + +func UnwrapProof(proof *Cell, hash []byte) (*Cell, error) { if !proof.special || proof.RefsNum() != 1 || proof.BitsSize() != 280 || - proof.data[0] != _MerkleProofType { - return fmt.Errorf("not a merkle proof cell") + Type(proof.data[0]) != MerkleProofCellType { + return nil, fmt.Errorf("not a merkle proof cell") } needLvl := proof.refs[0].levelMask.GetLevel() @@ -128,18 +133,18 @@ func CheckProof(proof *Cell, hash []byte) error { } if needLvl != proof.levelMask.GetLevel() { - return fmt.Errorf("incorrect level of child") + return nil, fmt.Errorf("incorrect level of proof") } if !bytes.Equal(hash, proof.data[1:33]) { - return fmt.Errorf("incorrect proof hash") + return nil, fmt.Errorf("incorrect proof hash") } // we unwrap level by 1 to correctly check proofs on pruned cells calcHash := proof.refs[0].getHash(needLvl) if !bytes.Equal(hash, calcHash) { - return fmt.Errorf("incorrect proof") + return nil, fmt.Errorf("incorrect proof") } - return nil + return proof.refs[0], nil } func (c *Cell) getLevelMask() LevelMask { @@ -149,7 +154,7 @@ func (c *Cell) getLevelMask() LevelMask { func (c *Cell) getHash(level int) []byte { hashIndex := c.getLevelMask().Apply(level).getHashIndex() - if c.getType() == _PrunedType { + if c.GetType() == PrunedCellType { prunedHashIndex := c.getLevelMask().getHashIndex() if hashIndex != prunedHashIndex { // return hash from data @@ -172,8 +177,8 @@ func (c *Cell) calculateHashes() { c.depthLevels = make([]uint16, totalHashCount) hashCount := totalHashCount - typ := c.getType() - if typ == _PrunedType { + typ := c.GetType() + if typ == PrunedCellType { hashCount = 1 } @@ -201,7 +206,7 @@ func (c *Cell) calculateHashes() { hash.Write(dsc) if hashIndex == hashIndexOffset { - if levelIndex != 0 && typ != _PrunedType { + if levelIndex != 0 && typ != PrunedCellType { // should never happen panic("not pruned or 0") } @@ -214,7 +219,7 @@ func (c *Cell) calculateHashes() { } hash.Write(data) } else { - if levelIndex == 0 || typ == _PrunedType { + if levelIndex == 0 || typ == PrunedCellType { // should never happen panic("pruned or 0") } @@ -225,7 +230,7 @@ func (c *Cell) calculateHashes() { var depth uint16 for i := 0; i < len(c.refs); i++ { var childDepth uint16 - if typ == _MerkleProofType || typ == _MerkleUpdateType { + if typ == MerkleProofCellType || typ == MerkleUpdateCellType { childDepth = c.refs[i].getDepth(levelIndex + 1) } else { childDepth = c.refs[i].getDepth(levelIndex) @@ -247,7 +252,7 @@ func (c *Cell) calculateHashes() { } for i := 0; i < len(c.refs); i++ { - if typ == _MerkleProofType || typ == _MerkleUpdateType { + if typ == MerkleProofCellType || typ == MerkleUpdateCellType { hash.Write(c.refs[i].getHash(levelIndex + 1)) } else { hash.Write(c.refs[i].getHash(levelIndex)) @@ -262,7 +267,7 @@ func (c *Cell) calculateHashes() { func (c *Cell) getDepth(level int) uint16 { hashIndex := c.getLevelMask().Apply(level).getHashIndex() - if c.getType() == _PrunedType { + if c.GetType() == PrunedCellType { prunedHashIndex := c.getLevelMask().getHashIndex() if hashIndex != prunedHashIndex { // return depth from data diff --git a/tvm/cell/slice.go b/tvm/cell/slice.go index a5f71225..9ac257a3 100644 --- a/tvm/cell/slice.go +++ b/tvm/cell/slice.go @@ -30,13 +30,21 @@ func (c *Slice) MustLoadRef() *Slice { } func (c *Slice) LoadRef() (*Slice, error) { + ref, err := c.LoadRefCell() + if err != nil { + return nil, err + } + return ref.BeginParse(), nil +} + +func (c *Slice) LoadRefCell() (*Cell, error) { if len(c.refs) == 0 { return nil, ErrNoMoreRefs } ref := c.refs[0] c.refs = c.refs[1:] - return ref.BeginParse(), nil + return ref, nil } func (c *Slice) MustLoadMaybeRef() *Slice { From 41e9d42875c90edfb07b404dc5c42acd78060127 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Fri, 4 Aug 2023 11:55:53 +0400 Subject: [PATCH 26/38] More wallet versions support for wallets + payment channels wip --- tl/loader.go | 8 +- ton/payments/channel.go | 172 +++++++++++++++++++++++++++++++ ton/payments/integration_test.go | 91 ++++++++++++++++ ton/payments/types.go | 160 ++++++++++++++++++++++++++++ ton/wallet/highloadv2r2.go | 13 ++- ton/wallet/integration_test.go | 85 ++++++++------- ton/wallet/regular.go | 30 ++++++ ton/wallet/v3.go | 23 +++-- ton/wallet/v4r2.go | 23 +++-- ton/wallet/wallet.go | 12 +-- 10 files changed, 550 insertions(+), 67 deletions(-) create mode 100644 ton/payments/channel.go create mode 100644 ton/payments/integration_test.go create mode 100644 ton/payments/types.go diff --git a/tl/loader.go b/tl/loader.go index c7b06a52..3868daa4 100644 --- a/tl/loader.go +++ b/tl/loader.go @@ -23,8 +23,8 @@ var _SchemaIDByTypeName = map[string]uint32{} var _SchemaIDByName = map[string]uint32{} var _SchemaByID = map[uint32]reflect.Type{} -var BoolTrue = tlCRC("boolTrue = Bool") -var BoolFalse = tlCRC("boolFalse = Bool") +var BoolTrue = CRC("boolTrue = Bool") +var BoolFalse = CRC("boolFalse = Bool") var Logger = func(a ...any) {} @@ -653,7 +653,7 @@ func Register(typ any, tl string) uint32 { } id = binary.BigEndian.Uint32(b) } else { - id = tlCRC(tl) + id = CRC(tl) } _SchemaByID[id] = t _SchemaIDByTypeName[t.String()] = id @@ -668,7 +668,7 @@ func Register(typ any, tl string) uint32 { var ieeeTable = crc32.MakeTable(crc32.IEEE) -func tlCRC(schema string) uint32 { +func CRC(schema string) uint32 { schema = strings.ReplaceAll(schema, "(", "") schema = strings.ReplaceAll(schema, ")", "") data := []byte(schema) diff --git a/ton/payments/channel.go b/ton/payments/channel.go new file mode 100644 index 00000000..0469b4e5 --- /dev/null +++ b/ton/payments/channel.go @@ -0,0 +1,172 @@ +package payments + +import ( + "bytes" + "context" + "crypto/ed25519" + "encoding/hex" + "fmt" + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/tlb" + "github.com/xssnick/tonutils-go/ton" + "github.com/xssnick/tonutils-go/tvm/cell" + "math/rand" + "time" +) + +type TonApi interface { + WaitForBlock(seqno uint32) ton.APIClientWaiter + CurrentMasterchainInfo(ctx context.Context) (_ *ton.BlockIDExt, err error) + RunGetMethod(ctx context.Context, blockInfo *ton.BlockIDExt, addr *address.Address, method string, params ...any) (*ton.ExecutionResult, error) + SendExternalMessage(ctx context.Context, msg *tlb.ExternalMessage) error + GetAccount(ctx context.Context, block *ton.BlockIDExt, addr *address.Address) (*tlb.Account, error) +} + +type Client struct { + api TonApi +} + +type ChannelStatus int8 + +const ( + ChannelStatusUninitialized ChannelStatus = iota + ChannelStatusOpen + ChannelStatusClosureStarted + ChannelStatusSettlingConditionals + ChannelStatusAwaitingFinalization +) + +type AsyncChannel struct { + Status ChannelStatus + Storage AsyncChannelStorageData + addr *address.Address + client *Client +} + +type ChannelID []byte + +func NewPaymentChannelClient(api TonApi) *Client { + return &Client{ + api: api, + } +} + +func (c *Client) GetAsyncChannel(ctx context.Context, addr *address.Address, verify bool) (*AsyncChannel, error) { + block, err := c.api.CurrentMasterchainInfo(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get block: %w", err) + } + + acc, err := c.api.GetAccount(ctx, block, addr) + if err != nil { + return nil, fmt.Errorf("failed to get account: %w", err) + } + + if verify { + codeBoC, _ := hex.DecodeString(AsyncPaymentChannelCodeBoC) + code, _ := cell.FromBOC(codeBoC) + + if !bytes.Equal(acc.Code.Hash(), code.Hash()) { + return nil, fmt.Errorf("incorrect code hash") + } + } + + ch := &AsyncChannel{ + addr: addr, + client: c, + Status: ChannelStatusUninitialized, + } + + err = tlb.LoadFromCell(&ch.Storage, acc.Data.BeginParse()) + if err != nil { + return nil, fmt.Errorf("failed to load storage: %w", err) + } + + ch.Status = ch.Storage.calcState() + + return ch, nil +} + +func (c *Client) GetDeployAsyncChannelParams(channelId ChannelID, isA bool, initialBalance tlb.Coins, ourKey ed25519.PrivateKey, theirKey ed25519.PublicKey, closingConfig ClosingConfig, paymentConfig PaymentConfig) (body, code, data *cell.Cell, err error) { + codeBoC, _ := hex.DecodeString(AsyncPaymentChannelCodeBoC) + code, _ = cell.FromBOC(codeBoC) + + if len(channelId) != 16 { + return nil, nil, nil, fmt.Errorf("channelId len should be 16 bytes") + } + + storageData := AsyncChannelStorageData{ + KeyA: ourKey.Public().(ed25519.PublicKey), + KeyB: theirKey, + ChannelID: channelId, + ClosingConfig: closingConfig, + Payments: paymentConfig, + } + + if !isA { + storageData.KeyA, storageData.KeyB = storageData.KeyB, storageData.KeyA + } + + data, err = tlb.ToCell(storageData) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to serialize storage data: %w", err) + } + + initCh := InitChannel{} + initCh.IsA = isA + + if isA { + initCh.Signed.BalanceA = initialBalance + } else { + initCh.Signed.BalanceB = initialBalance + } + initCh.Signed.ChannelID = channelId + initCh.Signature, err = toSignature(initCh.Signed, ourKey) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to sign data: %w", err) + } + + body, err = tlb.ToCell(initCh) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to serialize message: %w", err) + } + return body, code, data, nil +} + +// calcState - it repeats get_channel_state method of contract, +// we do this because we cannot prove method execution for now, +// but can proof contract data and code, so this approach is safe +func (s *AsyncChannelStorageData) calcState() ChannelStatus { + if !s.Initialized { + return ChannelStatusUninitialized + } + if s.Quarantine == nil { + return ChannelStatusOpen + } + now := time.Now().Unix() + quarantineEnds := int64(s.Quarantine.QuarantineStarts) + int64(s.ClosingConfig.QuarantineDuration) + if quarantineEnds > now { + return ChannelStatusClosureStarted + } + if quarantineEnds+int64(s.ClosingConfig.ConditionalCloseDuration) > now { + return ChannelStatusSettlingConditionals + } + return ChannelStatusAwaitingFinalization +} + +func toSignature(obj any, key ed25519.PrivateKey) (Signature, error) { + toSign, err := tlb.ToCell(obj) + if err != nil { + return Signature{}, fmt.Errorf("failed to serialize body to sign: %w", err) + } + return Signature{Value: toSign.Sign(key)}, nil +} + +func RandomChannelID() (ChannelID, error) { + id := make(ChannelID, 16) + _, err := rand.Read(id) + if err != nil { + return nil, err + } + return id, nil +} diff --git a/ton/payments/integration_test.go b/ton/payments/integration_test.go new file mode 100644 index 00000000..bced9ad7 --- /dev/null +++ b/ton/payments/integration_test.go @@ -0,0 +1,91 @@ +package payments + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "fmt" + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/liteclient" + "github.com/xssnick/tonutils-go/tlb" + "github.com/xssnick/tonutils-go/ton" + "github.com/xssnick/tonutils-go/ton/wallet" + "log" + "os" + "strings" + "testing" + "time" +) + +var api = func() *ton.APIClient { + client := liteclient.NewConnectionPool() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := client.AddConnectionsFromConfigUrl(ctx, "https://ton-blockchain.github.io/testnet-global.config.json") + if err != nil { + panic(err) + } + + return ton.NewAPIClient(client) +}() + +var _seed = strings.Split(os.Getenv("WALLET_SEED"), " ") + +func TestClient_DeployAsyncChannel(t *testing.T) { + client := NewPaymentChannelClient(api) + + chID, err := RandomChannelID() + if err != nil { + t.Fatal(err) + } + + _, ourKey, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + theirKey, _, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + + w, err := wallet.FromSeed(api, _seed, wallet.HighloadV2R2) + if err != nil { + t.Fatal(fmt.Errorf("failed to init wallet: %w", err)) + } + + body, code, data, err := client.GetDeployAsyncChannelParams(chID, true, tlb.MustFromTON("0.005"), ourKey, theirKey, ClosingConfig{ + QuarantineDuration: 180, + MisbehaviorFine: tlb.MustFromTON("0.1"), + ConditionalCloseDuration: 200, + }, PaymentConfig{ + ExcessFee: tlb.MustFromTON("0.001"), + DestA: w.Address(), + DestB: address.MustParseAddr("EQBletedrsSdih8H_-bR0cDZhdbLRy73ol6psGCrRKDahFju"), + }) + if err != nil { + t.Fatal(fmt.Errorf("failed to build deploy channel params: %w", err)) + } + + channelAddr, err := w.DeployContract(context.Background(), tlb.MustFromTON("0.02"), body, code, data, true) + if err != nil { + t.Fatal(fmt.Errorf("failed to deploy channel: %w", err)) + } + + ch, err := client.GetAsyncChannel(context.Background(), channelAddr, true) + if err != nil { + t.Fatal(fmt.Errorf("failed to get channel: %w", err)) + } + + if ch.Status != ChannelStatusOpen { + t.Fatal("channel status incorrect") + } + + if ch.Storage.BalanceA.NanoTON().Cmp(tlb.MustFromTON("0.005").NanoTON()) != 0 { + t.Fatal("balance incorrect") + } + + json.NewEncoder(os.Stdout).Encode(ch.Storage) + log.Println("channel deployed:", channelAddr.String()) +} diff --git a/ton/payments/types.go b/ton/payments/types.go new file mode 100644 index 00000000..db482635 --- /dev/null +++ b/ton/payments/types.go @@ -0,0 +1,160 @@ +package payments + +import ( + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/tlb" + "github.com/xssnick/tonutils-go/tvm/cell" +) + +// AsyncPaymentChannelCodeBoC Taken from https://github.com/ton-blockchain/payment-channels/tree/master#compiled-code +const AsyncPaymentChannelCodeBoC = "B5EE9C72410230010007FB000114FF00F4A413F4BCF2C80B0102012002030201480405000AF26C21F0190202CB06070201202E2F020120080902012016170201200A0B0201200C0D0009D3610F80CC001D6B5007434C7FE8034C7CC1BC0FE19E0201580E0F0201201011002D3E11DBC4BE11DBC43232C7FE11DBC47E80B2C7F2407320008B083E1B7B51343480007E187E80007E18BE80007E18F4FFC07E1934FFC07E1974DFC07E19BC01887080A7F4C7C07E1A34C7C07E1A7D01007E1AB7807080E535007E1AF7BE1B2002012012130201201415008D3E13723E11BE117E113E10540132803E10BE80BE10FE8084F2FFC4B2FFF2DFFC02887080A7FE12BE127E121400F2C7C4B2C7FD0037807080E53E12C073253E1333C5B8B27B5520004D1C3C02FE106CFCB8193E803E800C3E1096283E18BE10C0683E18FE10BE10E8006EFCB819BC032000CF1D3C02FE106CFCB819348020C235C6083E4040E4BE1124BE117890CC3E443CB81974C7C060841A5B9A5D2EBCB81A3E118074DFD66EBCB81CBE803E800C3E1094882FBE10D4882FAC3CB819807E18BE18FE12F43E800C3E10BE10E80068006E7CB8199FFE187C0320004120843777222E9C20043232C15401B3C594013E808532DA84B2C7F2DFF2407EC02002012018190201D42B2C0201201A1B0201201E1F0201201C1D00E5473F00BD401D001D401D021F90102D31F01821043436D74BAF2E068F84601D37F59BAF2E072F844544355F910F8454330F910B0F2E065D33FD33F30F84822B9F84922B9B0F2E06C21F86820F869F84A6E915B8E19F84AD0D33FFA003171D721D33F305033BC02BCB1936DF86ADEE2F800F00C8006F3E12F43E800C7E903E900C3E09DBC41CBE10D62F24CC20C1B7BE10FE11963C03FE10BE11A04020BC03DC3E185C3E189C3E18DB7E1ABC032000B51D3C02F5007400750074087E4040B4C7C0608410DB1BDCEEBCB81A3E118074DFD66EBCB81CBE111510D57E443E1150CC3E442C3CB8197E80007E18BE80007E18F4CFF4CFCC3E1208AE7E1248AE6C3CB81B007E1A3E1A7E003C042001C1573F00BF84A6EF2E06AD2008308D71820F9012392F84492F845E24130F910F2E065D31F018210556E436CBAF2E068F84601D37F59BAF2E072D401D08308D71820F901F8444130F910F2E06501D430D08308D71820F901F8454130F910F2E06501820020120222301FED31F01821043685374BAF2E068F84601D37F59BAF2E072D33FFA00F404552003D200019AD401D0D33FFA00F40430937F206DE2303205D31F01821043685374BAF2E068F84601D37F59BAF2E072D33FFA00F404552003D200019AD401D0D33FFA00F40430937F206DE23032F8485280BEF8495250BEB0524BBE1AB0527ABE19210064B05215BE14B05248BE17B0F2E06970F82305C8CB3F5004FA0215F40015CB3F5004FA0212F400CB1F12CA00CA00C9F86AF00C01C31CFC02FE129BACFCB81AF48020C235C6083E4048E4BE1124BE1178904C3E443CB81974C7C0608410DA19D46EBCB81A3E118074DFD66EBCB81CB5007420C235C6083E407E11104C3E443CB81940750C3420C235C6083E407E11504C3E443CB81940602403F71CFC02FE129BACFCB81AF48020C235C6083E4048E4BE1124BE1178904C3E443CB81974C7C0608410DB10DBAEBCB81A3E118074DFD66EBCB81CBD010C3E12B434CFFE803D0134CFFE803D0134C7FE11DBC4148828083E08EE7CB81BBE11DBC4A83E08EF3CB81C34800C151D5A64D6D4C8F7A2B98E82A49B08B8C3816028292A01FCD31F01821043685374BAF2E068F84601D37F59BAF2E072D33FFA00F404552003D200019AD401D0D33FFA00F40430937F206DE2303205D31F01821043685374BAF2E068F84601D37F59BAF2E072D33FFA00F404552003D200019AD401D0D33FFA00F40430937F206DE230325339BE5381BEB0F8495250BEB0F8485290BEB02502FE5237BE16B05262BEB0F2E06927C20097F84918BEF2E0699137E222C20097F84813BEF2E0699132E2F84AD0D33FFA00F404D33FFA00F404D31FF8476F105220A0F823BCF2E06FD200D20030B3F2E073209C3537373A5274BC5263BC12B18E11323939395250BC5299BC18B14650134440E25319BAB3F2E06D9130E30D7F05C82627002496F8476F1114A098F8476F1117A00603E203003ECB3F5004FA0215F40012CB3F5004FA0213F400CB1F12CA00CA00C9F86AF00C00620A8020F4966FA5208E213050038020F4666FA1208E1001FA00ED1E15DA119450C3A00B9133E2923430E202926C21E2B31B000C3535075063140038C8CB3F5004FA0212F400CB3F5003FA0213F400CB1FCA00C9F86AF00C00D51D3C02FE129BACFCB81AFE12B434CFFE803D010C74CFFE803D010C74C7CC3E11DBC4283E11DBC4A83E08EE7CB81C7E003E10886808E87E18BE10D400E816287E18FE10F04026BE10BE10E83E189C3E18F7BE10B04026BE10FE10A83E18DC3E18F780693E1A293E1A7C042001F53B7EF4C7C8608419F1F4A06EA4CC7C037808608403818830AEA54C7C03B6CC780C882084155DD61FAEA54C3C0476CC780820841E6849BBEEA54C3C04B6CC7808208407C546B3EEA54C3C0576CC780820840223AA8CAEA54C3C05B6CC7808208419BDBC1A6EA54C3C05F6CC780C60840950CAA46EA53C0636CC78202D0008840FF2F00075BC7FE3A7805FC25E87D007D207D20184100D0CAF6A1EC7C217C21B7817C227C22B7817C237C23FC247C24B7817C2524C3B7818823881B22A021984008DBD0CABA7805FC20C8B870FC253748B8F07C256840206B90FD0018C020EB90FD0018B8EB90E98F987C23B7882908507C11DE491839707C23B788507C23B789507C11DE48B9F03A4331C4966" + +// Data types + +type Signature struct { + Value []byte `tlb:"bits 512"` +} + +type ClosingConfig struct { + QuarantineDuration uint32 `tlb:"## 32"` + MisbehaviorFine tlb.Coins `tlb:"."` + ConditionalCloseDuration uint32 `tlb:"## 32"` +} + +type ConditionalPayment struct { + Amount tlb.Coins `tlb:"."` + Condition *cell.Cell `tlb:"."` +} + +type SemiChannelBody struct { + Seqno uint64 `tlb:"## 64"` + Sent tlb.Coins `tlb:"."` + Conditionals *cell.Dictionary `tlb:"dict 32"` +} + +type SemiChannel struct { + _ tlb.Magic `tlb:"#43685374"` + Data SemiChannelBody `tlb:"."` + CounterpartyData *SemiChannelBody `tlb:"maybe ^"` +} + +type SignedSemiChannel struct { + Signature Signature `tlb:"."` + State SemiChannel `tlb:"."` +} + +type QuarantinedState struct { + StateA SemiChannelBody `tlb:"."` + StateB SemiChannelBody `tlb:"."` + QuarantineStarts uint32 `tlb:"## 32"` + StateCommittedByA bool `tlb:"bool"` + StateChallenged bool `tlb:"bool"` +} + +type PaymentConfig struct { + ExcessFee tlb.Coins `tlb:"."` + DestA *address.Address `tlb:"addr"` + DestB *address.Address `tlb:"addr"` +} + +type AsyncChannelStorageData struct { + Initialized bool `tlb:"bool"` + BalanceA tlb.Coins `tlb:"."` + BalanceB tlb.Coins `tlb:"."` + KeyA []byte `tlb:"bits 256"` + KeyB []byte `tlb:"bits 256"` + ChannelID ChannelID `tlb:"bits 128"` + ClosingConfig ClosingConfig `tlb:"^"` + CommittedSeqnoA uint32 `tlb:"## 32"` + CommittedSeqnoB uint32 `tlb:"## 32"` + Quarantine *QuarantinedState `tlb:"maybe ^"` + Payments PaymentConfig `tlb:"^"` +} + +/// Messages + +type InitChannel struct { + _ tlb.Magic `tlb:"#0e0620c2"` + IsA bool `tlb:"bool"` + Signature Signature `tlb:"."` + Signed struct { + _ tlb.Magic `tlb:"#696e6974"` + ChannelID ChannelID `tlb:"bits 128"` + BalanceA tlb.Coins `tlb:"."` + BalanceB tlb.Coins `tlb:"."` + } `tlb:"."` +} + +type TopupBalance struct { + _ tlb.Magic `tlb:"#67c7d281"` + AddA tlb.Coins `tlb:"."` + AddB tlb.Coins `tlb:"."` +} + +type CooperativeClose struct { + _ tlb.Magic `tlb:"#5577587e"` + IsA bool `tlb:"bool"` + SignatureA Signature `tlb:"^"` + SignatureB Signature `tlb:"^"` + Signed struct { + _ tlb.Magic `tlb:"#436c6f73"` + ChannelID ChannelID `tlb:"bits 128"` + BalanceA tlb.Coins `tlb:"."` + BalanceB tlb.Coins `tlb:"."` + SeqnoA uint64 `tlb:"## 64"` + SeqnoB uint64 `tlb:"## 64"` + } `tlb:"."` +} + +type CooperativeCommit struct { + _ tlb.Magic `tlb:"#79a126ef"` + IsA bool `tlb:"bool"` + SignatureA Signature `tlb:"^"` + SignatureB Signature `tlb:"^"` + Signed struct { + _ tlb.Magic `tlb:"#43436d74"` + ChannelID ChannelID `tlb:"bits 128"` + SeqnoA uint64 `tlb:"## 64"` + SeqnoB uint64 `tlb:"## 64"` + } `tlb:"."` +} + +type StartUncooperativeClose struct { + _ tlb.Magic `tlb:"#1f151acf"` + IsSignedByA bool `tlb:"bool"` + Signature Signature `tlb:"."` + Signed struct { + _ tlb.Magic `tlb:"#556e436c"` + ChannelID ChannelID `tlb:"bits 128"` + A SignedSemiChannel `tlb:"^"` + B SignedSemiChannel `tlb:"^"` + } `tlb:"."` +} + +type ChallengeQuarantinedState struct { + _ tlb.Magic `tlb:"#088eaa32"` + IsChallengedByA bool `tlb:"bool"` + Signature Signature `tlb:"."` + Signed struct { + _ tlb.Magic `tlb:"#43686751"` + ChannelID ChannelID `tlb:"bits 128"` + A SignedSemiChannel `tlb:"^"` + B SignedSemiChannel `tlb:"^"` + } `tlb:"."` +} + +type SettleConditionals struct { + _ tlb.Magic `tlb:"#66f6f069"` + IsFromA bool `tlb:"bool"` + Signature Signature `tlb:"."` + Signed struct { + _ tlb.Magic `tlb:"#436c436e"` + ChannelID ChannelID `tlb:"bits 128"` + ConditionalsToSettle *cell.Dictionary `tlb:"dict 32"` + B SignedSemiChannel `tlb:"^"` + } `tlb:"."` +} + +type FinishUncooperativeClose struct { + _ tlb.Magic `tlb:"#25432a91"` +} diff --git a/ton/wallet/highloadv2r2.go b/ton/wallet/highloadv2r2.go index ffd25ff3..c1e5b9b3 100644 --- a/ton/wallet/highloadv2r2.go +++ b/ton/wallet/highloadv2r2.go @@ -18,9 +18,10 @@ const _HighloadV2VerifiedCodeHex = "b5ee9c724101090100e5000114ff00f4a413f4bcf2c8 type SpecHighloadV2R2 struct { SpecRegular + SpecQuery } -func (s *SpecHighloadV2R2) BuildMessage(_ context.Context, queryID uint32, messages []*Message) (*cell.Cell, error) { +func (s *SpecHighloadV2R2) BuildMessage(_ context.Context, messages []*Message) (*cell.Cell, error) { if len(messages) > 254 { return nil, errors.New("for this type of wallet max 254 messages can be sent in the same time") } @@ -43,7 +44,15 @@ func (s *SpecHighloadV2R2) BuildMessage(_ context.Context, queryID uint32, messa } } - boundedID := uint64(timeNow().Add(time.Duration(s.messagesTTL)*time.Second).UTC().Unix()<<32) + uint64(queryID) + var ttl, queryID uint32 + if s.customQueryIDFetcher != nil { + ttl, queryID = s.customQueryIDFetcher() + } else { + queryID = randUint32() + ttl = uint32(timeNow().Add(time.Duration(s.messagesTTL) * time.Second).UTC().Unix()) + } + + boundedID := (uint64(ttl) << 32) + uint64(queryID) payload := cell.BeginCell().MustStoreUInt(uint64(s.wallet.subwallet), 32). MustStoreUInt(boundedID, 64). MustStoreDict(dict) diff --git a/ton/wallet/integration_test.go b/ton/wallet/integration_test.go index c1640f4a..a4f2a1d4 100644 --- a/ton/wallet/integration_test.go +++ b/ton/wallet/integration_test.go @@ -53,46 +53,57 @@ func Test_WalletTransfer(t *testing.T) { for _, v := range []Version{V3R2, V4R2, HighloadV2R2, V3R1, V4R1, HighloadV2Verified} { ver := v - t.Run("send for wallet ver "+fmt.Sprint(ver), func(t *testing.T) { - t.Parallel() - - ctx := api.Client().StickyContext(context.Background()) - ctx, cancel := context.WithTimeout(ctx, 120*time.Second) - defer cancel() - - w, err := FromSeed(api, seed, ver) - if err != nil { - t.Fatal("FromSeed err:", err.Error()) - return - } - - log.Println(ver, "-> test wallet address:", w.Address()) - - block, err := api.CurrentMasterchainInfo(ctx) - if err != nil { - t.Fatal("CurrentMasterchainInfo err:", err.Error()) - return - } - - balance, err := w.GetBalance(ctx, block) - if err != nil { - t.Fatal("GetBalance err:", err.Error()) - return - } - - comment := randString(150) - addr := address.MustParseAddr("EQA8aJTl0jfFnUZBJjTeUxu9OcbsoPBp9UcHE9upyY_X35kE") - if balance.NanoTON().Uint64() >= 3000000 { - err = w.Transfer(ctx, addr, tlb.MustFromTON("0.003"), comment, true) + for _, isSubwallet := range []bool{false, true} { + isSubwallet := isSubwallet + t.Run("send for wallet ver "+fmt.Sprint(ver)+" subwallet "+fmt.Sprint(isSubwallet), func(t *testing.T) { + t.Parallel() + + ctx := api.Client().StickyContext(context.Background()) + ctx, cancel := context.WithTimeout(ctx, 120*time.Second) + defer cancel() + + w, err := FromSeed(api, seed, ver) + if err != nil { + t.Fatal("FromSeed err:", err.Error()) + return + } + + if isSubwallet { + w, err = w.GetSubwallet(1) + if err != nil { + t.Fatal("GetSubwallet err:", err.Error()) + return + } + } + + log.Println(ver, "-> test wallet address:", w.Address(), isSubwallet) + + block, err := api.CurrentMasterchainInfo(ctx) if err != nil { - t.Fatal("Transfer err:", err.Error()) + t.Fatal("CurrentMasterchainInfo err:", err.Error()) return } - } else { - t.Fatal("not enough balance") - return - } - }) + + balance, err := w.GetBalance(ctx, block) + if err != nil { + t.Fatal("GetBalance err:", err.Error()) + return + } + + comment := randString(150) + addr := address.MustParseAddr("EQA8aJTl0jfFnUZBJjTeUxu9OcbsoPBp9UcHE9upyY_X35kE") + if balance.NanoTON().Uint64() >= 3000000 { + err = w.Transfer(ctx, addr, tlb.MustFromTON("0.003"), comment, true) + if err != nil { + t.Fatal("Transfer err:", err.Error()) + return + } + } else { + t.Fatal("not enough balance") + return + } + }) + } } } diff --git a/ton/wallet/regular.go b/ton/wallet/regular.go index b6759d60..079d176e 100644 --- a/ton/wallet/regular.go +++ b/ton/wallet/regular.go @@ -23,3 +23,33 @@ type SpecRegular struct { func (s *SpecRegular) SetMessagesTTL(ttl uint32) { s.messagesTTL = ttl } + +type SpecSeqno struct { + // Instead of calling contract 'seqno' method, + // this function wil be used (if not nil) to get seqno for new transaction. + // You may use it to set seqno according to your own logic, + // for example for additional idempotency, + // if build message is not enough, or for additional security + customSeqnoFetcher func() uint32 +} + +func (s *SpecSeqno) SetCustomSeqnoFetcher(fetcher func() uint32) { + s.customSeqnoFetcher = fetcher +} + +type SpecQuery struct { + // Instead of generating random query id with message ttl, + // this function wil be used (if not nil) to get query id for new transaction. + // You may use it to set query id according to your own logic, + // for example for additional idempotency, + // if build message is not enough, or for additional security + // + // Do not set ttl to high if you are sending many messages, + // unexpired executed messages will be cached in contract, + // and it may become too expensive to make transactions. + customQueryIDFetcher func() (ttl uint32, randPart uint32) +} + +func (s *SpecQuery) SetCustomQueryIDFetcher(fetcher func() (ttl uint32, randPart uint32)) { + s.customQueryIDFetcher = fetcher +} diff --git a/ton/wallet/v3.go b/ton/wallet/v3.go index 34854b31..b1b7f0d6 100644 --- a/ton/wallet/v3.go +++ b/ton/wallet/v3.go @@ -18,6 +18,7 @@ const _V3R2CodeHex = "B5EE9C724101010100710000DEFF0020DD2082014C97BA218201339CBA type SpecV3 struct { SpecRegular + SpecSeqno } func (s *SpecV3) BuildMessage(ctx context.Context, isInitialized bool, block *ton.BlockIDExt, messages []*Message) (*cell.Cell, error) { @@ -27,17 +28,21 @@ func (s *SpecV3) BuildMessage(ctx context.Context, isInitialized bool, block *to var seq uint64 - if isInitialized { - resp, err := s.wallet.api.WaitForBlock(block.SeqNo).RunGetMethod(ctx, block, s.wallet.addr, "seqno") - if err != nil { - return nil, fmt.Errorf("get seqno err: %w", err) - } + if s.customSeqnoFetcher != nil { + seq = uint64(s.customSeqnoFetcher()) + } else { + if isInitialized { + resp, err := s.wallet.api.WaitForBlock(block.SeqNo).RunGetMethod(ctx, block, s.wallet.addr, "seqno") + if err != nil { + return nil, fmt.Errorf("get seqno err: %w", err) + } - iSeq, err := resp.Int(0) - if err != nil { - return nil, fmt.Errorf("failed to parse seqno: %w", err) + iSeq, err := resp.Int(0) + if err != nil { + return nil, fmt.Errorf("failed to parse seqno: %w", err) + } + seq = iSeq.Uint64() } - seq = iSeq.Uint64() } payload := cell.BeginCell().MustStoreUInt(uint64(s.wallet.subwallet), 32). diff --git a/ton/wallet/v4r2.go b/ton/wallet/v4r2.go index 6046071a..edecfcec 100644 --- a/ton/wallet/v4r2.go +++ b/ton/wallet/v4r2.go @@ -18,6 +18,7 @@ const _V4R2CodeHex = "B5EE9C72410214010002D4000114FF00F4A413F4BCF2C80B0102012002 type SpecV4R2 struct { SpecRegular + SpecSeqno } func (s *SpecV4R2) BuildMessage(ctx context.Context, isInitialized bool, block *ton.BlockIDExt, messages []*Message) (*cell.Cell, error) { @@ -27,17 +28,21 @@ func (s *SpecV4R2) BuildMessage(ctx context.Context, isInitialized bool, block * var seq uint64 - if isInitialized { - resp, err := s.wallet.api.WaitForBlock(block.SeqNo).RunGetMethod(ctx, block, s.wallet.addr, "seqno") - if err != nil { - return nil, fmt.Errorf("get seqno err: %w", err) - } + if s.customSeqnoFetcher != nil { + seq = uint64(s.customSeqnoFetcher()) + } else { + if isInitialized { + resp, err := s.wallet.api.WaitForBlock(block.SeqNo).RunGetMethod(ctx, block, s.wallet.addr, "seqno") + if err != nil { + return nil, fmt.Errorf("get seqno err: %w", err) + } - iSeq, err := resp.Int(0) - if err != nil { - return nil, fmt.Errorf("failed to parse seqno: %w", err) + iSeq, err := resp.Int(0) + if err != nil { + return nil, fmt.Errorf("failed to parse seqno: %w", err) + } + seq = iSeq.Uint64() } - seq = iSeq.Uint64() } payload := cell.BeginCell().MustStoreUInt(uint64(s.wallet.subwallet), 32). diff --git a/ton/wallet/wallet.go b/ton/wallet/wallet.go index 5ccb5cd2..37f8b087 100644 --- a/ton/wallet/wallet.go +++ b/ton/wallet/wallet.go @@ -156,11 +156,11 @@ func getSpec(w *Wallet) (any, error) { switch w.ver { case V3R1, V3R2: - return &SpecV3{regular}, nil + return &SpecV3{regular, SpecSeqno{}}, nil case V4R1, V4R2: - return &SpecV4R2{regular}, nil + return &SpecV4R2{regular, SpecSeqno{}}, nil case HighloadV2R2, HighloadV2Verified: - return &SpecHighloadV2R2{regular}, nil + return &SpecHighloadV2R2{regular, SpecQuery{}}, nil } return nil, fmt.Errorf("cannot init spec: %w", ErrUnsupportedWalletVersion) @@ -253,7 +253,7 @@ func (w *Wallet) BuildExternalMessageForMany(ctx context.Context, messages []*Me return nil, fmt.Errorf("build message err: %w", err) } case HighloadV2R2, HighloadV2Verified: - msg, err = w.spec.(*SpecHighloadV2R2).BuildMessage(ctx, randUint32(), messages) + msg, err = w.spec.(*SpecHighloadV2R2).BuildMessage(ctx, messages) if err != nil { return nil, fmt.Errorf("build message err: %w", err) } @@ -485,7 +485,7 @@ func (w *Wallet) DeployContract(ctx context.Context, amount tlb.Coins, msgBody, addr := address.NewAddress(0, 0, stateCell.Hash()) if err = w.Send(ctx, &Message{ - Mode: 1, + Mode: 1 + 2, InternalMessage: &tlb.InternalMessage{ IHRDisabled: true, Bounce: false, @@ -561,7 +561,7 @@ func (w *Wallet) FindTransactionByInMsgHash(ctx context.Context, msgHash []byte, func SimpleMessage(to *address.Address, amount tlb.Coins, payload *cell.Cell) *Message { return &Message{ - Mode: 1, + Mode: 1 + 2, InternalMessage: &tlb.InternalMessage{ IHRDisabled: true, Bounce: true, From 6b93b443f1dc06ce68a5c113322da2b6a2328b7b Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Sun, 13 Aug 2023 23:17:22 +0400 Subject: [PATCH 27/38] Liteserver proof chains validation & TL+TLB+Cell enhancements & improvements --- README.md | 4 +- adnl/crypto.go | 16 +- adnl/dht/client.go | 12 +- adnl/dht/client_test.go | 18 +- adnl/dht/node.go | 4 +- adnl/dht/node_test.go | 20 +- adnl/dht/priority_test.go | 17 +- adnl/overlay/overlay-adnl.go | 6 +- adnl/overlay/types.go | 12 +- adnl/rldp/http/client_test.go | 4 +- adnl/rldp/http/server.go | 3 +- example/accept-payments/main.go | 73 +++++ example/account-state/main.go | 2 +- example/block-scan/main.go | 25 +- example/dns/main.go | 2 +- example/get-votes/main.go | 2 +- example/highload-wallet/main.go | 2 +- example/jetton-transfer/main.go | 70 +++++ example/jettons/main.go | 15 +- example/send-to-contract/main.go | 4 +- example/site-request/main.go | 4 +- example/wallet-cold-alike/main.go | 2 +- example/wallet/main.go | 4 +- liteclient/config_types.go | 34 +-- liteclient/connection.go | 2 +- liteclient/integration_test.go | 39 ++- liteclient/pool.go | 25 ++ tl/loader.go | 114 +++++++- tl/loader_test.go | 54 +++- tlb/account.go | 71 +---- tlb/block.go | 25 +- tlb/coins.go | 95 +++++-- tlb/coins_test.go | 85 ++++-- tlb/config.go | 100 +++++++ tlb/loader.go | 335 ++++++++++++---------- tlb/loader_test.go | 74 ++--- tlb/message.go | 17 +- tlb/register.go | 25 ++ tlb/shard.go | 214 ++++++-------- tlb/shard_test.go | 13 +- tlb/transaction.go | 189 ++----------- tlb/transaction_test.go | 15 + ton/api.go | 41 ++- ton/block.go | 153 ++++++---- ton/dns/integration_test.go | 2 +- ton/getconfig.go | 32 +-- ton/getstate.go | 40 +-- ton/integration_test.go | 97 ++++++- ton/jetton/integration_test.go | 6 +- ton/jetton/jetton.go | 1 + ton/jetton/wallet.go | 15 +- ton/nft/integration_test.go | 2 +- ton/prng.go | 83 ++++++ ton/proof.go | 444 +++++++++++++++++++++++++++++- ton/runmethod.go | 8 +- ton/transactions.go | 130 ++++++++- ton/waiter.go | 4 + ton/wallet/address.go | 24 ++ ton/wallet/integration_test.go | 33 ++- ton/wallet/wallet.go | 155 +++++++++++ ton/wallet/wallet_test.go | 39 ++- tvm/cell/builder.go | 2 +- tvm/cell/cell.go | 2 +- tvm/cell/cell_test.go | 5 +- tvm/cell/flattenIndex.go | 84 ++---- tvm/cell/serialize.go | 20 +- tvm/cell/serialize_test.go | 37 +++ 67 files changed, 2356 insertions(+), 950 deletions(-) create mode 100644 example/accept-payments/main.go create mode 100644 example/jetton-transfer/main.go create mode 100644 tlb/register.go create mode 100644 ton/prng.go create mode 100644 tvm/cell/serialize_test.go diff --git a/README.md b/README.md index d4600b13..c46ddeb1 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ You can also join our **[Telegram group](https://t.me/tonutils)** and ask any qu ### Connection You can get list of public lite servers from official TON configs: -* Mainnet - `https://ton-blockchain.github.io/global.config.json` +* Mainnet - `https://ton.org/global.config.json` * Testnet - `https://ton-blockchain.github.io/testnet-global.config.json` from liteservers section, you need to convert int to ip and take port and key. @@ -203,7 +203,7 @@ if err != nil { // Balance: ACTIVE fmt.Printf("Status: %s\n", account.State.Status) // Balance: 66559946.09 TON -fmt.Printf("Balance: %s TON\n", account.State.Balance.TON()) +fmt.Printf("Balance: %s TON\n", account.State.Balance.String()) if account.Data != nil { // Can be nil if account is not active // Data: [0000003829a9a31772c9ed6b62a6e2eba14a93b90462e7a367777beb8a38fb15b9f33844d22ce2ff] fmt.Printf("Data: %s\n", account.Data.Dump()) diff --git a/adnl/crypto.go b/adnl/crypto.go index 3fe4c5cc..35fc4271 100644 --- a/adnl/crypto.go +++ b/adnl/crypto.go @@ -4,8 +4,6 @@ import ( "crypto/aes" "crypto/cipher" "crypto/ed25519" - "crypto/sha256" - "fmt" "github.com/oasisprotocol/curve25519-voi/curve" ed25519crv "github.com/oasisprotocol/curve25519-voi/primitives/ed25519" "github.com/oasisprotocol/curve25519-voi/primitives/x25519" @@ -65,15 +63,5 @@ func NewCipherCtr(key, iv []byte) (cipher.Stream, error) { return cipher.NewCTR(c, iv), nil } -func ToKeyID(key any) ([]byte, error) { - data, err := tl.Serialize(key, true) - if err != nil { - return nil, fmt.Errorf("key serialize err: %w", err) - } - - hash := sha256.New() - hash.Write(data) - s := hash.Sum(nil) - - return s, nil -} +// Deprecated: use tl.Hash +var ToKeyID = tl.Hash diff --git a/adnl/dht/client.go b/adnl/dht/client.go index 79b165f6..4532717d 100644 --- a/adnl/dht/client.go +++ b/adnl/dht/client.go @@ -147,7 +147,7 @@ func (c *Client) addNode(node *Node) (_ *dhtNode, err error) { return nil, fmt.Errorf("unsupported id type %s", reflect.TypeOf(node.ID).String()) } - kid, err := adnl.ToKeyID(pub) + kid, err := tl.Hash(pub) if err != nil { return nil, err } @@ -178,7 +178,7 @@ func (c *Client) addNode(node *Node) (_ *dhtNode, err error) { } func (c *Client) FindOverlayNodes(ctx context.Context, overlayKey []byte, continuation ...*Continuation) (*overlay.NodesList, *Continuation, error) { - keyHash, err := adnl.ToKeyID(adnl.PublicKeyOverlay{ + keyHash, err := tl.Hash(adnl.PublicKeyOverlay{ Key: overlayKey, }) @@ -265,7 +265,7 @@ func (c *Client) StoreOverlayNodes(ctx context.Context, overlayKey []byte, nodes } func (c *Client) Store(ctx context.Context, id any, name []byte, index int32, value []byte, rule any, ttl time.Duration, ownerKey ed25519.PrivateKey, atLeastCopies int) (copiesMade int, idKey []byte, err error) { - idKey, err = adnl.ToKeyID(id) + idKey, err = tl.Hash(id) if err != nil { return 0, nil, err } @@ -296,7 +296,7 @@ func (c *Client) Store(ctx context.Context, id any, name []byte, index int32, va } } - kid, err := adnl.ToKeyID(val.KeyDescription.Key) + kid, err := tl.Hash(val.KeyDescription.Key) if err != nil { return 0, nil, err } @@ -334,7 +334,7 @@ func (c *Client) Store(ctx context.Context, id any, name []byte, index int32, va currentPriority := leadingZeroBits(xor(kid, node.adnlId)) for _, n := range nodes { var nid []byte - nid, err = adnl.ToKeyID(n.ID) + nid, err = tl.Hash(n.ID) if err != nil { continue } @@ -410,7 +410,7 @@ type foundResult struct { } func (c *Client) FindValue(ctx context.Context, key *Key, continuation ...*Continuation) (*Value, *Continuation, error) { - id, keyErr := adnl.ToKeyID(key) + id, keyErr := tl.Hash(key) if keyErr != nil { return nil, nil, keyErr } diff --git a/adnl/dht/client_test.go b/adnl/dht/client_test.go index f909f9a6..c33df296 100644 --- a/adnl/dht/client_test.go +++ b/adnl/dht/client_test.go @@ -250,7 +250,7 @@ func TestClient_FindValue(t *testing.T) { t.Fatal(err) } - k, err := adnl.ToKeyID(&Key{ + k, err := tl.Hash(&Key{ ID: siteAddr, Name: []byte("address"), Index: 0, @@ -309,7 +309,7 @@ func TestClient_NewClientFromConfig(t *testing.T) { pubKey1 := ed25519.PublicKey(byteKey1) adnlPubKey1 := adnl.PublicKeyED25519{Key: pubKey1} - tKeyId1, err := adnl.ToKeyID(adnlPubKey1) + tKeyId1, err := tl.Hash(adnlPubKey1) if err != nil { t.Fatal("failed to prepare test key id, err: ", err) } @@ -332,7 +332,7 @@ func TestClient_NewClientFromConfig(t *testing.T) { pubKey2 := ed25519.PublicKey(byteKey2) adnlPubKey2 := adnl.PublicKeyED25519{Key: pubKey2} - tKeyId2, err := adnl.ToKeyID(adnlPubKey2) + tKeyId2, err := tl.Hash(adnlPubKey2) if err != nil { t.Fatal("failed to prepare test key id, err: ", err) } @@ -476,7 +476,7 @@ func TestClient_FindAddressesUnit(t *testing.T) { t.Fatal("failed to prepare test data, err", err) } - k, err := adnl.ToKeyID(&Key{ + k, err := tl.Hash(&Key{ ID: adnlAddr, Name: []byte("address"), Index: 0, @@ -535,7 +535,7 @@ func TestClient_FindAddressesIntegration(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() - dhtClient, err := NewClientFromConfigUrl(ctx, gateway, "https://ton-blockchain.github.io/global.config.json") + dhtClient, err := NewClientFromConfigUrl(ctx, gateway, "https://ton.org/global.config.json") if err != nil { t.Fatalf("failed to init DHT client: %s", err.Error()) } @@ -718,7 +718,7 @@ func TestClient_StoreOverlayNodesIntegration(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) defer cancel() - dhtClient, err := NewClientFromConfigUrl(ctx, gateway, "https://ton-blockchain.github.io/global.config.json") + dhtClient, err := NewClientFromConfigUrl(ctx, gateway, "https://ton.org/global.config.json") if err != nil { t.Fatalf("failed to init DHT client: %s", err.Error()) } @@ -766,10 +766,10 @@ func TestClient_StoreAddressIntegration(t *testing.T) { t.Fatal(err) } - ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - dhtClient, err := NewClientFromConfigUrl(ctx, gateway, "https://ton-blockchain.github.io/global.config.json") + dhtClient, err := NewClientFromConfigUrl(ctx, gateway, "https://ton.org/global.config.json") if err != nil { t.Fatalf("failed to init DHT client: %s", err.Error()) } @@ -807,7 +807,7 @@ func TestClient_StoreAddressIntegration(t *testing.T) { t.Fatal(err) } - kid, err := adnl.ToKeyID(adnl.PublicKeyED25519{ + kid, err := tl.Hash(adnl.PublicKeyED25519{ Key: pub, }) if err != nil { diff --git a/adnl/dht/node.go b/adnl/dht/node.go index c93d3df9..d69c34be 100644 --- a/adnl/dht/node.go +++ b/adnl/dht/node.go @@ -137,7 +137,7 @@ func checkValue(id []byte, value *Value) error { return fmt.Errorf("invalid dht key") } - idKey, err := adnl.ToKeyID(k) + idKey, err := tl.Hash(k) if err != nil { return err } @@ -145,7 +145,7 @@ func checkValue(id []byte, value *Value) error { return fmt.Errorf("unwanted key received") } - idPub, err := adnl.ToKeyID(value.KeyDescription.ID) + idPub, err := tl.Hash(value.KeyDescription.ID) if err != nil { return err } diff --git a/adnl/dht/node_test.go b/adnl/dht/node_test.go index a04cf4c4..0c0a9570 100644 --- a/adnl/dht/node_test.go +++ b/adnl/dht/node_test.go @@ -22,7 +22,7 @@ func newCorrectDhtNode(a byte, b byte, c byte, d byte, port string) (*dhtNode, e return nil, err } - kId, err := adnl.ToKeyID(adnl.PublicKeyED25519{Key: tPubKey}) + kId, err := tl.Hash(adnl.PublicKeyED25519{Key: tPubKey}) if err != nil { return nil, err } @@ -46,7 +46,7 @@ func TestNode_findNodes(t *testing.T) { } pubKeyAdnl := adnl.PublicKeyED25519{Key: pubKey} - idKey, err := adnl.ToKeyID(pubKeyAdnl) + idKey, err := tl.Hash(pubKeyAdnl) if err != nil { t.Fatal("failed to prepare test key id, err: ", err) } @@ -55,7 +55,7 @@ func TestNode_findNodes(t *testing.T) { []byte("lol"), 0, } - kId, err := adnl.ToKeyID(tKey) + kId, err := tl.Hash(tKey) if err != nil { t.Fatal("failed to prepare test key id, err: ", err) } @@ -162,7 +162,7 @@ func TestNode_storeValue(t *testing.T) { val := valFound.Value - kId, err := adnl.ToKeyID(val.KeyDescription.Key) + kId, err := tl.Hash(val.KeyDescription.Key) if err != nil { t.Fatal("failed to prepare test key id, err: ", err) } @@ -290,7 +290,7 @@ func TestNode_findValue(t *testing.T) { t.Fatal("failed to prepare test data, err", err) } - k, err := adnl.ToKeyID(&Key{ + k, err := tl.Hash(&Key{ ID: siteAddr, Name: []byte("address"), Index: 0, @@ -333,7 +333,7 @@ func TestNode_findValue(t *testing.T) { Name: []byte("address"), Index: 0, } - testId, keyErr := adnl.ToKeyID(k) + testId, keyErr := tl.Hash(k) if keyErr != nil { t.Fatal("failed to prepare test id, err: ", keyErr) } @@ -377,7 +377,7 @@ func TestNode_checkValue(t *testing.T) { val := valFound.Value - kId, err := adnl.ToKeyID(val.KeyDescription.Key) + kId, err := tl.Hash(val.KeyDescription.Key) if err != nil { t.Fatal("failed to prepare test key id, err: ", err) } @@ -418,7 +418,7 @@ func TestNode_weight(t *testing.T) { t.Fatal("failed to prepare test public key, err: ", err) } - kId, err := adnl.ToKeyID(adnl.PublicKeyED25519{Key: tPubKey}) + kId, err := tl.Hash(adnl.PublicKeyED25519{Key: tPubKey}) if err != nil { t.Fatal("failed to prepare test key id, err: ", err) } @@ -436,7 +436,7 @@ func TestNode_weight(t *testing.T) { t.Fatal("failed to prepare test public key, err: ", err) } - kId, err = adnl.ToKeyID(adnl.PublicKeyED25519{Key: tPubKey}) + kId, err = tl.Hash(adnl.PublicKeyED25519{Key: tPubKey}) if err != nil { t.Fatal("failed to prepare test key id, err: ", err) } @@ -454,7 +454,7 @@ func TestNode_weight(t *testing.T) { t.Fatal("failed to prepare test public key, err: ", err) } - kId, err = adnl.ToKeyID(adnl.PublicKeyED25519{Key: tPubKey}) + kId, err = tl.Hash(adnl.PublicKeyED25519{Key: tPubKey}) if err != nil { t.Fatal("failed to prepare test key id, err: ", err) } diff --git a/adnl/dht/priority_test.go b/adnl/dht/priority_test.go index 92d7a6b2..b7dcc54d 100644 --- a/adnl/dht/priority_test.go +++ b/adnl/dht/priority_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "github.com/xssnick/tonutils-go/adnl" + "github.com/xssnick/tonutils-go/tl" "testing" ) @@ -18,7 +19,7 @@ func TestPriorityList_addNode(t *testing.T) { Name: []byte("address"), Index: 0, } - keyId, err := adnl.ToKeyID(k) + keyId, err := tl.Hash(k) if err != nil { t.Fatal("failed to prepare test key id") } @@ -29,7 +30,7 @@ func TestPriorityList_addNode(t *testing.T) { t.Fatal("failed to prepare test public key") } - kId1, err := adnl.ToKeyID(adnl.PublicKeyED25519{tPubKey1}) + kId1, err := tl.Hash(adnl.PublicKeyED25519{tPubKey1}) if err != nil { t.Fatal("failed to prepare test key ID") } @@ -40,7 +41,7 @@ func TestPriorityList_addNode(t *testing.T) { t.Fatal("failed to prepare test public key") } - kId2, err := adnl.ToKeyID(adnl.PublicKeyED25519{tPubKey2}) + kId2, err := tl.Hash(adnl.PublicKeyED25519{tPubKey2}) if err != nil { t.Fatal("failed to prepare test key ID") } @@ -51,7 +52,7 @@ func TestPriorityList_addNode(t *testing.T) { t.Fatal("failed to prepare test public key") } - kId3, err := adnl.ToKeyID(adnl.PublicKeyED25519{tPubKey3}) + kId3, err := tl.Hash(adnl.PublicKeyED25519{tPubKey3}) if err != nil { t.Fatal("failed to prepare test key ID") } @@ -110,7 +111,7 @@ func TestPriorityList_markNotUsed(t *testing.T) { Name: []byte("address"), Index: 0, } - keyId, err := adnl.ToKeyID(k) + keyId, err := tl.Hash(k) if err != nil { t.Fatal("failed to prepare test key id") } @@ -121,7 +122,7 @@ func TestPriorityList_markNotUsed(t *testing.T) { t.Fatal("failed to prepare test public key") } - kId1, err := adnl.ToKeyID(adnl.PublicKeyED25519{tPubKey1}) + kId1, err := tl.Hash(adnl.PublicKeyED25519{tPubKey1}) if err != nil { t.Fatal("failed to prepare test key ID") } @@ -132,7 +133,7 @@ func TestPriorityList_markNotUsed(t *testing.T) { t.Fatal("failed to prepare test public key") } - kId2, err := adnl.ToKeyID(adnl.PublicKeyED25519{tPubKey2}) + kId2, err := tl.Hash(adnl.PublicKeyED25519{tPubKey2}) if err != nil { t.Fatal("failed to prepare test key ID") } @@ -143,7 +144,7 @@ func TestPriorityList_markNotUsed(t *testing.T) { t.Fatal("failed to prepare test public key") } - kId3, err := adnl.ToKeyID(adnl.PublicKeyED25519{tPubKey3}) + kId3, err := tl.Hash(adnl.PublicKeyED25519{tPubKey3}) if err != nil { t.Fatal("failed to prepare test key ID") } diff --git a/adnl/overlay/overlay-adnl.go b/adnl/overlay/overlay-adnl.go index 883e2331..ba53f408 100644 --- a/adnl/overlay/overlay-adnl.go +++ b/adnl/overlay/overlay-adnl.go @@ -178,7 +178,7 @@ func (a *ADNLOverlayWrapper) processFECBroadcast(t *BroadcastFEC) error { partDataHasher.Write(t.Data) partDataHash := partDataHasher.Sum(nil) - partHash, err := adnl.ToKeyID(&BroadcastFECPartID{ + partHash, err := tl.Hash(&BroadcastFECPartID{ BroadcastHash: broadcastHash, DataHash: partDataHash, Seqno: t.Seqno, @@ -214,7 +214,7 @@ func (a *ADNLOverlayWrapper) processFECBroadcast(t *BroadcastFEC) error { return fmt.Errorf("incorrect data size") } - srcId, err := adnl.ToKeyID(t.Source) + srcId, err := tl.Hash(t.Source) if err != nil { return fmt.Errorf("source key id serialize failed: %w", err) } @@ -242,7 +242,7 @@ func (a *ADNLOverlayWrapper) processFECBroadcast(t *BroadcastFEC) error { issuedBy = cert.IssuedBy } - issuerId, err = adnl.ToKeyID(issuedBy) + issuerId, err = tl.Hash(issuedBy) if err != nil { return fmt.Errorf("issuer key id serialize failed: %w", err) } diff --git a/adnl/overlay/types.go b/adnl/overlay/types.go index 9087ad10..0f345762 100644 --- a/adnl/overlay/types.go +++ b/adnl/overlay/types.go @@ -156,20 +156,20 @@ type BroadcastFEC struct { } func (t *BroadcastFEC) CalcID() ([]byte, error) { - typeId, err := adnl.ToKeyID(t.FEC) + typeId, err := tl.Hash(t.FEC) if err != nil { return nil, fmt.Errorf("failed to compute fec type id: %w", err) } var src = make([]byte, 32) if t.Flags&_BroadcastFlagAnySender == 0 { - src, err = adnl.ToKeyID(t.Source) + src, err = tl.Hash(t.Source) if err != nil { return nil, fmt.Errorf("failed to compute source key id: %w", err) } } - broadcastHash, err := adnl.ToKeyID(&BroadcastFECID{ + broadcastHash, err := tl.Hash(&BroadcastFECID{ Source: src, Type: typeId, DataHash: t.DataHash, @@ -253,7 +253,7 @@ func (n *Node) CheckSignature() error { return fmt.Errorf("unsupported id type %s", reflect.TypeOf(n.ID).String()) } - id, err := adnl.ToKeyID(n.ID) + id, err := tl.Hash(n.ID) if err != nil { return fmt.Errorf("failed to calc id: %w", err) } @@ -282,7 +282,7 @@ func (n *Node) Sign(key ed25519.PrivateKey) error { return fmt.Errorf("incorrect private key") } - id, err := adnl.ToKeyID(n.ID) + id, err := tl.Hash(n.ID) if err != nil { return fmt.Errorf("failed to calc id: %w", err) } @@ -301,7 +301,7 @@ func (n *Node) Sign(key ed25519.PrivateKey) error { } func NewNode(overlay []byte, key ed25519.PrivateKey) (*Node, error) { - keyHash, err := adnl.ToKeyID(adnl.PublicKeyOverlay{ + keyHash, err := tl.Hash(adnl.PublicKeyOverlay{ Key: overlay, }) if err != nil { diff --git a/adnl/rldp/http/client_test.go b/adnl/rldp/http/client_test.go index 8ced9e30..24f4af93 100644 --- a/adnl/rldp/http/client_test.go +++ b/adnl/rldp/http/client_test.go @@ -409,7 +409,7 @@ func TestTransport_RoundTripIntegration(t *testing.T) { t.Fatal(err) } - dhtClient, err := dht.NewClientFromConfigUrl(context.Background(), gateway, "https://ton-blockchain.github.io/global.config.json") + dhtClient, err := dht.NewClientFromConfigUrl(context.Background(), gateway, "https://ton.org/global.config.json") if err != nil { t.Fatal(err) } @@ -434,7 +434,7 @@ func getDNSResolver() *dns.Client { client := liteclient.NewConnectionPool() // connect to testnet lite server - err := client.AddConnectionsFromConfigUrl(context.Background(), "https://ton-blockchain.github.io/global.config.json") + err := client.AddConnectionsFromConfigUrl(context.Background(), "https://ton.org/global.config.json") if err != nil { panic(err) } diff --git a/adnl/rldp/http/server.go b/adnl/rldp/http/server.go index ffd24668..0ddaa080 100644 --- a/adnl/rldp/http/server.go +++ b/adnl/rldp/http/server.go @@ -9,6 +9,7 @@ import ( "github.com/xssnick/tonutils-go/adnl" "github.com/xssnick/tonutils-go/adnl/address" "github.com/xssnick/tonutils-go/adnl/rldp" + "github.com/xssnick/tonutils-go/tl" "io" "log" "net" @@ -90,7 +91,7 @@ func NewServer(key ed25519.PrivateKey, dht DHT, handler http.Handler) *Server { Timeout: 30 * time.Second, adnlServer: newServer(key), } - s.id, _ = adnl.ToKeyID(adnl.PublicKeyED25519{Key: s.key.Public().(ed25519.PublicKey)}) + s.id, _ = tl.Hash(adnl.PublicKeyED25519{Key: s.key.Public().(ed25519.PublicKey)}) return s } diff --git a/example/accept-payments/main.go b/example/accept-payments/main.go new file mode 100644 index 00000000..ce06a2aa --- /dev/null +++ b/example/accept-payments/main.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/liteclient" + "github.com/xssnick/tonutils-go/tlb" + "github.com/xssnick/tonutils-go/ton" + "log" +) + +func main() { + client := liteclient.NewConnectionPool() + + cfg, err := liteclient.GetConfigFromUrl(context.Background(), "https://ton.org/global.config.json") + if err != nil { + log.Fatalln("get config err: ", err.Error()) + return + } + + // connect to mainnet lite servers + err = client.AddConnectionsFromConfig(context.Background(), cfg) + if err != nil { + log.Fatalln("connection err: ", err.Error()) + return + } + + // initialize ton api lite connection wrapper with full proof checks + api := ton.NewAPIClient(client, ton.ProofCheckPolicySecure) + api.SetTrustedBlockFromConfig(cfg) + + log.Println("checking proofs since config init block, it may take near a minute...") + master, err := api.CurrentMasterchainInfo(context.Background()) // we fetch block just to trigger chain proof check + if err != nil { + log.Fatalln("get masterchain info err: ", err.Error()) + return + } + log.Println("master proof checks are completed successfully, now communication is 100% safe!") + + // address on which we are accepting payments + treasuryAddress := address.MustParseAddr("EQCD39VS5jcptHL8vMjEXrzGaRcCVYto7HUn4bpAOg8xqB2N") + + acc, err := api.GetAccount(context.Background(), master, treasuryAddress) + if err != nil { + log.Fatalln("get masterchain info err: ", err.Error()) + return + } + + // Cursor of processed transaction, save it to your db + // We start from last transaction, will not process transactions older than we started from. + // After each processed transaction, save lt to your db, to continue after restart + lastProcessedLT := acc.LastTxLT + // channel with new transactions + transactions := make(chan *tlb.Transaction) + + // it is a blocking call, so we start it asynchronously + go api.SubscribeOnTransactions(context.Background(), treasuryAddress, lastProcessedLT, transactions) + + log.Println("waiting for transfers...") + + // listen for new transactions from channel + for tx := range transactions { + // process transaction here + log.Println(tx.String()) + + // update last processed lt and save it in db + lastProcessedLT = tx.LT + } + + // it can happen due to none of available liteservers know old enough state for our address + // (when our unprocessed transactions are too old) + log.Println("something went wrong, transaction listening unexpectedly finished") +} diff --git a/example/account-state/main.go b/example/account-state/main.go index 58efad59..26e62281 100644 --- a/example/account-state/main.go +++ b/example/account-state/main.go @@ -46,7 +46,7 @@ func main() { fmt.Printf("Is active: %v\n", res.IsActive) if res.IsActive { fmt.Printf("Status: %s\n", res.State.Status) - fmt.Printf("Balance: %s TON\n", res.State.Balance.TON()) + fmt.Printf("Balance: %s TON\n", res.State.Balance.String()) if res.Data != nil { fmt.Printf("Data: %s\n", res.Data.Dump()) } diff --git a/example/block-scan/main.go b/example/block-scan/main.go index 770ced7c..a2ce8433 100644 --- a/example/block-scan/main.go +++ b/example/block-scan/main.go @@ -45,15 +45,24 @@ func getNotSeenShards(ctx context.Context, api *ton.APIClient, shard *ton.BlockI func main() { client := liteclient.NewConnectionPool() + cfg, err := liteclient.GetConfigFromUrl(context.Background(), "https://ton.org/global.config.json") + if err != nil { + log.Fatalln("get config err: ", err.Error()) + return + } + // connect to mainnet lite servers - err := client.AddConnectionsFromConfigUrl(context.Background(), "https://ton-blockchain.github.io/global.config.json") + err = client.AddConnectionsFromConfig(context.Background(), cfg) if err != nil { log.Fatalln("connection err: ", err.Error()) return } - // initialize ton api lite connection wrapper - api := ton.NewAPIClient(client) + // initialize ton api lite connection wrapper with full proof checks + api := ton.NewAPIClient(client, ton.ProofCheckPolicySecure) + api.SetTrustedBlockFromConfig(cfg) + + log.Println("checking proofs since config init block, it may take near a minute...") master, err := api.GetMasterchainInfo(context.Background()) if err != nil { @@ -61,6 +70,11 @@ func main() { return } + // TIP: you could save and store last trusted master block (master variable data) + // for faster initialization later using api.SetTrustedBlock + + log.Println("master proofs chain successfully verified, all data is now safe and trusted!") + // bound all requests to single lite server for consistency, // if it will go down, another lite server will be used ctx := api.Client().StickyContext(context.Background()) @@ -101,12 +115,13 @@ func main() { shardLastSeqno[getShardID(shard)] = shard.SeqNo newShards = append(newShards, notSeen...) } + newShards = append(newShards, master) var txList []*tlb.Transaction // for each shard block getting transactions for _, shard := range newShards { - log.Printf("scanning block %d of shard %x...", shard.SeqNo, uint64(shard.Shard)) + log.Printf("scanning block %d of shard %x in workchain %d...", shard.SeqNo, uint64(shard.Shard), shard.Workchain) var fetchedIDs []ton.TransactionShortInfo var after *ton.TransactionID3 @@ -127,7 +142,7 @@ func main() { for _, id := range fetchedIDs { // get full transaction by id - tx, err := api.GetTransaction(ctx, shard, address.NewAddress(0, 0, id.Account), id.LT) + tx, err := api.GetTransaction(ctx, shard, address.NewAddress(0, byte(shard.Workchain), id.Account), id.LT) if err != nil { log.Fatalln("get tx data err:", err.Error()) return diff --git a/example/dns/main.go b/example/dns/main.go index 251334d3..a1ca90f2 100644 --- a/example/dns/main.go +++ b/example/dns/main.go @@ -14,7 +14,7 @@ func main() { client := liteclient.NewConnectionPool() // connect to testnet lite server - err := client.AddConnectionsFromConfigUrl(context.Background(), "https://ton-blockchain.github.io/global.config.json") + err := client.AddConnectionsFromConfigUrl(context.Background(), "https://ton.org/global.config.json") if err != nil { panic(err) } diff --git a/example/get-votes/main.go b/example/get-votes/main.go index a00a308b..c26089c7 100644 --- a/example/get-votes/main.go +++ b/example/get-votes/main.go @@ -14,7 +14,7 @@ func main() { client := liteclient.NewConnectionPool() // connect to testnet lite server - err := client.AddConnectionsFromConfigUrl(context.Background(), "https://ton-blockchain.github.io/global.config.json") + err := client.AddConnectionsFromConfigUrl(context.Background(), "https://ton.org/global.config.json") if err != nil { panic(err) } diff --git a/example/highload-wallet/main.go b/example/highload-wallet/main.go index 5afdd045..ada606bd 100644 --- a/example/highload-wallet/main.go +++ b/example/highload-wallet/main.go @@ -92,5 +92,5 @@ func main() { return } - log.Println("not enough balance:", balance.TON()) + log.Println("not enough balance:", balance.String()) } diff --git a/example/jetton-transfer/main.go b/example/jetton-transfer/main.go new file mode 100644 index 00000000..32c6b90e --- /dev/null +++ b/example/jetton-transfer/main.go @@ -0,0 +1,70 @@ +package main + +import ( + "context" + "encoding/base64" + "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/liteclient" + "github.com/xssnick/tonutils-go/tlb" + "github.com/xssnick/tonutils-go/ton" + "github.com/xssnick/tonutils-go/ton/jetton" + "github.com/xssnick/tonutils-go/ton/wallet" + "log" + "strings" +) + +func main() { + client := liteclient.NewConnectionPool() + + // connect to testnet lite server + err := client.AddConnectionsFromConfigUrl(context.Background(), "https://ton.org/global.config.json") + if err != nil { + panic(err) + } + + ctx := client.StickyContext(context.Background()) + + // initialize ton api lite connection wrapper + api := ton.NewAPIClient(client) + + // seed words of account, you can generate them with any wallet or using wallet.NewSeed() method + words := strings.Split("birth pattern then forest walnut then phrase walnut fan pumpkin pattern then cluster blossom verify then forest velvet pond fiction pattern collect then then", " ") + + w, err := wallet.FromSeed(api, words, wallet.V3R2) + if err != nil { + log.Fatalln("FromSeed err:", err.Error()) + return + } + + token := jetton.NewJettonMasterClient(api, address.MustParseAddr("EQD0vdSA_NedR9uvbgN9EikRX-suesDxGeFg69XQMavfLqIw")) + + // find our jetton wallet + tokenWallet, err := token.GetJettonWallet(ctx, w.Address()) + if err != nil { + log.Fatal(err) + } + + tokenBalance, err := tokenWallet.GetBalance(ctx) + if err != nil { + log.Fatal(err) + } + log.Println("our jetton balance:", tokenBalance.String()) + + amountTokens := tlb.MustFromDecimal("0.9", 9) + + // address of receiver's wallet (not token wallet, just usual) + to := address.MustParseAddr("EQAvyxX5g_GvynfNl_XVQReZ3rstK5bM2OYu9nvren1SRnuN") + transferPayload, err := tokenWallet.BuildTransferPayload(to, amountTokens, tlb.ZeroCoins, nil) + if err != nil { + log.Fatal(err) + } + + msg := wallet.SimpleMessage(tokenWallet.Address(), tlb.MustFromTON("0.06"), transferPayload) + + log.Println("sending transaction...") + tx, _, err := w.SendWaitTransaction(ctx, msg) + if err != nil { + panic(err) + } + log.Println("transaction confirmed:", base64.StdEncoding.EncodeToString(tx.Hash)) +} diff --git a/example/jettons/main.go b/example/jettons/main.go index c0f73a2c..72a6ecb7 100644 --- a/example/jettons/main.go +++ b/example/jettons/main.go @@ -4,17 +4,19 @@ import ( "context" "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/liteclient" + "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/ton" "github.com/xssnick/tonutils-go/ton/jetton" "github.com/xssnick/tonutils-go/ton/nft" "log" + "strconv" ) func main() { client := liteclient.NewConnectionPool() // connect to testnet lite server - err := client.AddConnectionsFromConfigUrl(context.Background(), "https://ton-blockchain.github.io/global.config.json") + err := client.AddConnectionsFromConfigUrl(context.Background(), "https://ton.org/global.config.json") if err != nil { panic(err) } @@ -32,6 +34,7 @@ func main() { log.Fatal(err) } + decimals := 9 content := data.Content.(*nft.ContentOnchain) log.Println("total supply:", data.TotalSupply.Uint64()) log.Println("mintable:", data.Mintable) @@ -40,10 +43,12 @@ func main() { log.Println(" name:", content.Name) log.Println(" symbol:", content.GetAttribute("symbol")) if content.GetAttribute("decimals") != "" { - log.Println(" decimals:", content.GetAttribute("decimals")) - } else { - log.Println(" decimals:", 9) + decimals, err = strconv.Atoi(content.GetAttribute("decimals")) + if err != nil { + log.Fatal("invalid decimals") + } } + log.Println(" decimals:", decimals) log.Println(" description:", content.Description) log.Println() @@ -57,5 +62,5 @@ func main() { log.Fatal(err) } - log.Println("token balance:", tokenBalance.String()) + log.Println("jetton balance:", tlb.MustFromNano(tokenBalance, decimals)) } diff --git a/example/send-to-contract/main.go b/example/send-to-contract/main.go index 9868422f..f3f64e2c 100644 --- a/example/send-to-contract/main.go +++ b/example/send-to-contract/main.go @@ -103,10 +103,10 @@ func main() { return } - log.Println("balance left:", balance.TON()) + log.Println("balance left:", balance.String()) return } - log.Println("not enough balance:", balance.TON()) + log.Println("not enough balance:", balance.String()) } diff --git a/example/site-request/main.go b/example/site-request/main.go index 33f2abac..65a58cf1 100644 --- a/example/site-request/main.go +++ b/example/site-request/main.go @@ -26,7 +26,7 @@ func main() { panic(err) } - dhtClient, err := dht.NewClientFromConfigUrl(context.Background(), gateway, "https://ton-blockchain.github.io/global.config.json") + dhtClient, err := dht.NewClientFromConfigUrl(context.Background(), gateway, "https://ton.org/global.config.json") if err != nil { panic(err) } @@ -54,7 +54,7 @@ func getDNSResolver() *dns.Client { client := liteclient.NewConnectionPool() // connect to testnet lite server - err := client.AddConnectionsFromConfigUrl(context.Background(), "https://ton-blockchain.github.io/global.config.json") + err := client.AddConnectionsFromConfigUrl(context.Background(), "https://ton.org/global.config.json") if err != nil { panic(err) } diff --git a/example/wallet-cold-alike/main.go b/example/wallet-cold-alike/main.go index 9d1cbac1..81f2e295 100644 --- a/example/wallet-cold-alike/main.go +++ b/example/wallet-cold-alike/main.go @@ -87,5 +87,5 @@ func main() { return } - log.Println("not enough balance:", balance.TON()) + log.Println("not enough balance:", balance.String()) } diff --git a/example/wallet/main.go b/example/wallet/main.go index 56ad4888..1edf7ab8 100644 --- a/example/wallet/main.go +++ b/example/wallet/main.go @@ -78,10 +78,10 @@ func main() { return } - log.Println("transaction sent, balance left:", balance.TON()) + log.Println("transaction sent, balance left:", balance.String()) return } - log.Println("not enough balance:", balance.TON()) + log.Println("not enough balance:", balance.String()) } diff --git a/liteclient/config_types.go b/liteclient/config_types.go index c4f24842..2ca2c4c3 100644 --- a/liteclient/config_types.go +++ b/liteclient/config_types.go @@ -54,32 +54,16 @@ type ServerID struct { } type ValidatorConfig struct { - Type string `json:"@type"` - ZeroState ValidatorZeroState `json:"zero_state"` - InitBlock ValidatorInitBlock `json:"init_block"` - Hardforks []ValidatorHardfork `json:"hardforks"` + Type string `json:"@type"` + ZeroState ConfigBlock `json:"zero_state"` + InitBlock ConfigBlock `json:"init_block"` + Hardforks []ConfigBlock `json:"hardforks"` } -type ValidatorZeroState struct { - Workchain int `json:"workchain"` - Shard int64 `json:"shard"` - Seqno int `json:"seqno"` - RootHash string `json:"root_hash"` - FileHash string `json:"file_hash"` -} - -type ValidatorInitBlock struct { - RootHash string `json:"root_hash"` - Seqno int `json:"seqno"` - FileHash string `json:"file_hash"` - Workchain int `json:"workchain"` - Shard int64 `json:"shard"` -} - -type ValidatorHardfork struct { - FileHash string `json:"file_hash"` - Seqno int `json:"seqno"` - RootHash string `json:"root_hash"` - Workchain int `json:"workchain"` +type ConfigBlock struct { + Workchain int32 `json:"workchain"` Shard int64 `json:"shard"` + SeqNo uint32 `json:"seqno"` + RootHash []byte `json:"root_hash"` + FileHash []byte `json:"file_hash"` } diff --git a/liteclient/connection.go b/liteclient/connection.go index f8276338..9c52a577 100644 --- a/liteclient/connection.go +++ b/liteclient/connection.go @@ -455,7 +455,7 @@ func (n *connection) handshake(data []byte, ourKey ed25519.PrivateKey, serverKey pub := ourKey.Public().(ed25519.PublicKey) - kid, err := adnl.ToKeyID(adnl.PublicKeyED25519{Key: serverKey}) + kid, err := tl.Hash(adnl.PublicKeyED25519{Key: serverKey}) if err != nil { return err } diff --git a/liteclient/integration_test.go b/liteclient/integration_test.go index 36abb5c1..00958fcd 100644 --- a/liteclient/integration_test.go +++ b/liteclient/integration_test.go @@ -4,34 +4,50 @@ import ( "context" "fmt" "github.com/xssnick/tonutils-go/tl" - "github.com/xssnick/tonutils-go/ton" + "github.com/xssnick/tonutils-go/tlb" "testing" "time" ) +func init() { + tl.Register(MasterchainInfo{}, "liteServer.masterchainInfo last:tonNode.blockIdExt state_root_hash:int256 init:tonNode.zeroStateIdExt = liteServer.MasterchainInfo") + tl.Register(GetMasterchainInf{}, "liteServer.getMasterchainInfo = liteServer.MasterchainInfo") +} + +type GetMasterchainInf struct{} + +type BlockIDExt = tlb.BlockInfo +type MasterchainInfo struct { + Last *BlockIDExt `tl:"struct"` + StateRootHash []byte `tl:"int256"` + Init *ZeroStateIDExt `tl:"struct"` +} +type ZeroStateIDExt struct { + Workchain int32 `tl:"int"` + RootHash []byte `tl:"int256"` + FileHash []byte `tl:"int256"` +} + func Test_Conn(t *testing.T) { client := NewConnectionPool() ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - err := client.AddConnectionsFromConfigUrl(ctx, "https://ton-blockchain.github.io/global.config.json") + err := client.AddConnectionsFromConfigUrl(ctx, "https://ton.org/global.config.json") if err != nil { t.Fatal("add connections err", err) } doReq := func(expErr error) { var resp tl.Serializable - err := client.QueryLiteserver(ctx, ton.GetMasterchainInf{}, &resp) + err := client.QueryLiteserver(ctx, GetMasterchainInf{}, &resp) if err != nil { t.Fatal("do err", err) } switch tb := resp.(type) { - case ton.MasterchainInfo: - if tb.Last.Workchain != -1 || tb.Last.Shard != -9223372036854775808 { - t.Fatal("data err", *tb.Last) - } + case MasterchainInfo: default: t.Fatal("bad response", fmt.Sprint(tb)) } @@ -56,23 +72,20 @@ func Test_ConnSticky(t *testing.T) { defer cancel() ctx = client.StickyContext(ctx) - err := client.AddConnectionsFromConfigUrl(ctx, "https://ton-blockchain.github.io/global.config.json") + err := client.AddConnectionsFromConfigUrl(ctx, "https://ton.org/global.config.json") if err != nil { t.Fatal("add connections err", err) } doReq := func(expErr error) { var resp tl.Serializable - err := client.QueryLiteserver(ctx, ton.GetMasterchainInf{}, &resp) + err := client.QueryLiteserver(ctx, GetMasterchainInf{}, &resp) if err != nil { t.Fatal("do err", err) } switch tb := resp.(type) { - case ton.MasterchainInfo: - if tb.Last.Workchain != -1 || tb.Last.Shard != -9223372036854775808 { - t.Fatal("data err", *tb.Last) - } + case MasterchainInfo: default: t.Fatal("bad response", fmt.Sprint(tb)) } diff --git a/liteclient/pool.go b/liteclient/pool.go index cd74d364..5d4a0f7f 100644 --- a/liteclient/pool.go +++ b/liteclient/pool.go @@ -16,6 +16,7 @@ import ( ) const _StickyCtxKey = "_ton_node_sticky" +const _StickyCtxUsedNodesKey = "_ton_used_nodes_sticky" type OnDisconnectCallback func(addr, key string) @@ -93,6 +94,30 @@ func (c *ConnectionPool) StickyContext(ctx context.Context) context.Context { return context.WithValue(ctx, _StickyCtxKey, id) } +func (c *ConnectionPool) StickyContextNextNode(ctx context.Context) (context.Context, error) { + nodeID, _ := ctx.Value(_StickyCtxKey).(uint32) + usedNodes, _ := ctx.Value(_StickyCtxUsedNodesKey).([]uint32) + if nodeID > 0 { + usedNodes = append(usedNodes, nodeID) + } + + c.nodesMx.RLock() + defer c.nodesMx.RUnlock() + +iter: + for _, node := range c.activeNodes { + for _, usedNode := range usedNodes { + if usedNode == node.id { + continue iter + } + } + + return context.WithValue(context.WithValue(ctx, _StickyCtxKey, node.id), _StickyCtxUsedNodesKey, usedNodes), nil + } + + return ctx, fmt.Errorf("no more active nodes left") +} + func (c *ConnectionPool) StickyContextWithNodeID(ctx context.Context, nodeId uint32) context.Context { return context.WithValue(ctx, _StickyCtxKey, nodeId) } diff --git a/tl/loader.go b/tl/loader.go index 3868daa4..26771f49 100644 --- a/tl/loader.go +++ b/tl/loader.go @@ -1,9 +1,11 @@ package tl import ( + "crypto/sha256" "encoding/binary" "encoding/hex" "fmt" + "github.com/xssnick/tonutils-go/tvm/cell" "hash/crc32" "net" "reflect" @@ -288,6 +290,40 @@ func serializeField(tags []string, value reflect.Value) (buf []byte, err error) case reflect.String: return ToBytes([]byte(value.String())), nil } + case "cell": + optional := len(tags) > 1 && tags[1] == "optional" + if optional { + tags = tags[1:] + } + + var num int + if len(tags) > 1 { + num, err = strconv.Atoi(tags[1]) + if err != nil || num <= 0 { + panic("cells num tag should be positive integer") + } + } + + if value.IsNil() || (value.Kind() == reflect.Slice && value.Len() == 0) { + if optional { + return ToBytes(nil), nil + } + return nil, fmt.Errorf("nil cell is not allowed in field %s", value.Type().String()) + } + + if value.Type() == cellType { + if num > 0 { + panic("field type should be cell slice to use cells num tag") + } + return ToBytes(value.Interface().(*cell.Cell).ToBOCWithFlags(false)), nil + } else if value.Type() == cellArrType { + cells := value.Interface().([]*cell.Cell) + if num > 0 && num != len(cells) { + return nil, fmt.Errorf("incorrect cells len in field %s", value.Type().String()) + } + return ToBytes(cell.ToBOCWithFlags(cells, false)), nil + } + panic("for cell tag only *cell.Cell is supported") case "int256", "bytes": if tags[0] == "bytes" && len(tags) > 1 && tags[1] == "struct" { res, err := Serialize(value.Interface(), len(tags) > 2 && tags[2] == "boxed") @@ -433,6 +469,9 @@ func splitAllowed(leftTags []string) []string { return list } +var cellType = reflect.TypeOf(&cell.Cell{}) +var cellArrType = reflect.TypeOf([]*cell.Cell{}) + func parseField(data []byte, tags []string, value *reflect.Value) (_ []byte, err error) { switch tags[0] { case "string": @@ -446,6 +485,55 @@ func parseField(data []byte, tags []string, value *reflect.Value) (_ []byte, err value.SetString(string(val)) return data, nil } + case "cell": + optional := len(tags) > 1 && tags[1] == "optional" + if optional { + tags = tags[1:] + } + + var num int + if len(tags) > 1 { + num, err = strconv.Atoi(tags[1]) + if err != nil || num <= 0 { + panic("cells num tag should be positive integer") + } + } + + var val []byte + val, data, err = FromBytes(data) + if err != nil { + return nil, fmt.Errorf("failed to parse bytes for %s, err: %w", value.Type().String(), err) + } + + var cells []*cell.Cell + if len(val) > 0 { + cells, err = cell.FromBOCMultiRoot(val) + if err != nil { + return nil, fmt.Errorf("failed to parse boc from bytes for %s, err: %w", value.Type().String(), err) + } + } + + if len(cells) == 0 { + if optional { + return data, nil + } + return nil, fmt.Errorf("nil cell is not allowed in field %s", value.Type().String()) + } + + if value.Type() == cellType { + if num > 0 { + panic("field type should be cell slice to use cells num tag") + } + value.Set(reflect.ValueOf(cells[0])) + return data, nil + } else if value.Type() == cellArrType { + if num > 0 && num != len(cells) { + return nil, fmt.Errorf("incorrect cells len in field %s", value.Type().String()) + } + value.Set(reflect.ValueOf(cells)) + return data, nil + } + panic("for cell tag only *cell.Cell and []*cell.Cell are supported") case "int256", "bytes": var val []byte @@ -565,15 +653,19 @@ func parseField(data []byte, tags []string, value *reflect.Value) (_ []byte, err return nil, fmt.Errorf("failed to parse %s for %s, err: too short data", tags[0], value.Type().String()) } - var val []byte - val, data = data[:sz], data[sz:] + val := data[:sz] + data = data[sz:] if value.Type() == reflect.TypeOf(net.IP{}) { - val[0], val[1], val[2], val[3] = val[3], val[2], val[1], val[0] + ip := make([]byte, 4) + copy(ip, val) + + ip[0], ip[1], ip[2], ip[3] = ip[3], ip[2], ip[1], ip[0] + value.SetBytes(ip) + return data, nil } value.SetBytes(val) - return data, nil } } @@ -671,8 +763,16 @@ var ieeeTable = crc32.MakeTable(crc32.IEEE) func CRC(schema string) uint32 { schema = strings.ReplaceAll(schema, "(", "") schema = strings.ReplaceAll(schema, ")", "") - data := []byte(schema) - crc := crc32.Checksum(data, ieeeTable) + return crc32.Checksum([]byte(schema), ieeeTable) +} + +func Hash(key any) ([]byte, error) { + data, err := Serialize(key, true) + if err != nil { + return nil, fmt.Errorf("key serialize err: %w", err) + } - return crc + hash := sha256.New() + hash.Write(data) + return hash.Sum(nil), nil } diff --git a/tl/loader_test.go b/tl/loader_test.go index 3a1b2421..214e16a5 100644 --- a/tl/loader_test.go +++ b/tl/loader_test.go @@ -4,6 +4,8 @@ import ( "bytes" "crypto/ed25519" "encoding/hex" + "github.com/xssnick/tonutils-go/tvm/cell" + "net" "testing" ) @@ -13,17 +15,25 @@ type TestInner struct { } type TestTL struct { - Simple int64 `tl:"int"` - Flags uint32 `tl:"flags"` - SimpleOptional int64 `tl:"?0 long"` - SimpleOptional2 int64 `tl:"?1 long"` - SimpleUint uint `tl:"int"` - SimpleUintBig uint64 `tl:"long"` - In *TestInner `tl:"struct boxed"` - InX any `tl:"struct boxed [in]"` - In2 []any `tl:"vector struct boxed [in]"` - KeyEmpty []byte `tl:"int256"` - Data [][]byte `tl:"vector bytes"` + Simple int64 `tl:"int"` + Flags uint32 `tl:"flags"` + SimpleOptional int64 `tl:"?0 long"` + SimpleOptional2 int64 `tl:"?1 long"` + SimpleUint uint `tl:"int"` + SimpleUintBig uint64 `tl:"long"` + In *TestInner `tl:"struct boxed"` + InX any `tl:"struct boxed [in]"` + In2 []any `tl:"vector struct boxed [in]"` + KeyEmpty []byte `tl:"int256"` + Data [][]byte `tl:"vector bytes"` + CellArr []*cell.Cell `tl:"cell 1"` + Cell *cell.Cell `tl:"cell"` + CellOptional *cell.Cell `tl:"cell optional"` + InBytes TestInner `tl:"bytes struct boxed"` + IP net.IP `tl:"int"` + Str string `tl:"string"` + BoolTrue bool `tl:"bool"` + BoolFalse bool `tl:"bool"` } func TestParse(t *testing.T) { @@ -35,7 +45,12 @@ func TestParse(t *testing.T) { "e323006f" + "0200000000000000" + "7777777777777777777777777777777777777777777777777777777777777777" + "e323006f" + "0800000000000000" + "7177777777777777777777777777777777777777777777777777777777777777" + "02000000" + "e323006f" + "0700000000000000" + "7777777777777777777777777777777777777777777777777777777777777777" + "e323006f" + "0800000000000000" + "7777777777777777777777777777777777777777777777777777777777777777" + - "0000000000000000000000000000000000000000000000000000000000000000" + "03000000" + "00000000" + "03112233" + "0411223344" + "000000") + "0000000000000000000000000000000000000000000000000000000000000000" + "03000000" + "00000000" + "03112233" + "0411223344" + "000000" + + "4d" + "b5ee9c72010104010042000114ff00f4a413f4bcf2c80b010201620203000ed05f04840ff2f00049a1c56105e1fab9f6e0bf71cfc409201390cedfb9e3496fe2f806bd6762483cab4506717e09" + "0000" + + "4d" + "b5ee9c72010104010042000114ff00f4a413f4bcf2c80b010201620203000ed05f04840ff2f00049a1c56105e1fab9f6e0bf71cfc409201390cedfb9e3496fe2f806bd6762483cab4506717e09" + "0000" + + "00000000" + + "2c" + "e323006f" + "0200000000000000" + "7777777777777777777777777777777777777777777777777777777777777777" + "000000" + + "04030201" + "073A3A3A3A3A3A3A" + "b5757299" + "379779bc") var tst TestTL _, err := Parse(&tst, data, true) if err != nil { @@ -43,6 +58,8 @@ func TestParse(t *testing.T) { } tst.KeyEmpty = nil + println(hex.EncodeToString(tst.IP)) + data2, err := Serialize(tst, true) if err != nil { panic(err) @@ -55,3 +72,16 @@ func TestParse(t *testing.T) { t.Fatal("data not eq after serialize") } } + +func TestHash(t *testing.T) { + Register(TestInner{}, "root 777") + + hash, err := Hash(TestInner{Double: 777}) + if err != nil { + t.Fatal(err.Error()) + } + + if hex.EncodeToString(hash) != "dc6da5a30847889eaa194ef0851c99eb721a3750450a6b353949172ea40456c2" { + t.Fatal("incorrect hash " + hex.EncodeToString(hash)) + } +} diff --git a/tlb/account.go b/tlb/account.go index 8b03a001..d9c88e6e 100644 --- a/tlb/account.go +++ b/tlb/account.go @@ -58,15 +58,15 @@ type AccountStorage struct { } type StorageUsed struct { - BitsUsed uint64 - CellsUsed uint64 - PublicCellsUsed uint64 + BitsUsed *big.Int `tlb:"var uint 7"` + CellsUsed *big.Int `tlb:"var uint 7"` + PublicCellsUsed *big.Int `tlb:"var uint 7"` } type StorageInfo struct { - StorageUsed StorageUsed - LastPaid uint32 - DuePayment *big.Int + StorageUsed StorageUsed `tlb:"."` + LastPaid uint32 `tlb:"## 32"` + DuePayment *Coins `tlb:"maybe ."` } type AccountState struct { @@ -141,13 +141,13 @@ func (a *AccountState) LoadFromCell(loader *cell.Slice) error { } var info StorageInfo - err = info.LoadFromCell(loader) + err = LoadFromCell(&info, loader) if err != nil { return err } var store AccountStorage - err = store.LoadFromCell(loader) + err = LoadFromCell(&store, loader) if err != nil { return err } @@ -162,61 +162,6 @@ func (a *AccountState) LoadFromCell(loader *cell.Slice) error { return nil } -func (s *StorageUsed) LoadFromCell(loader *cell.Slice) error { - cells, err := loader.LoadVarUInt(7) - if err != nil { - return err - } - - bits, err := loader.LoadVarUInt(7) - if err != nil { - return err - } - - pubCells, err := loader.LoadVarUInt(7) - if err != nil { - return err - } - - s.CellsUsed = cells.Uint64() - s.BitsUsed = bits.Uint64() - s.PublicCellsUsed = pubCells.Uint64() - - return nil -} - -func (s *StorageInfo) LoadFromCell(loader *cell.Slice) error { - var used StorageUsed - err := used.LoadFromCell(loader) - if err != nil { - return err - } - - lastPaid, err := loader.LoadUInt(32) - if err != nil { - return err - } - - isDuePayment, err := loader.LoadUInt(1) - if err != nil { - return err - } - - var duePayment *big.Int - if isDuePayment == 1 { - duePayment, err = loader.LoadBigCoins() - if err != nil { - return err - } - } - - s.StorageUsed = used - s.DuePayment = duePayment - s.LastPaid = uint32(lastPaid) - - return nil -} - func (s *AccountStorage) LoadFromCell(loader *cell.Slice) error { lastTransaction, err := loader.LoadUInt(64) if err != nil { diff --git a/tlb/block.go b/tlb/block.go index 5c142e23..401d6919 100644 --- a/tlb/block.go +++ b/tlb/block.go @@ -17,15 +17,21 @@ type BlockInfo struct { } type StateUpdate struct { - Old ShardState `tlb:"^"` + Old any `tlb:"^ [ShardStateUnsplit,ShardStateSplit]"` New *cell.Cell `tlb:"^"` } type McBlockExtra struct { _ Magic `tlb:"#cca5"` - KeyBlock uint8 `tlb:"## 1"` + KeyBlock bool `tlb:"bool"` ShardHashes *cell.Dictionary `tlb:"dict 32"` ShardFees *cell.Dictionary `tlb:"dict 96"` + Details struct { + PrevBlockSignatures *cell.Dictionary `tlb:"dict 16"` + RecoverCreateMsg *cell.Cell `tlb:"maybe ^"` + MintMsg *cell.Cell `tlb:"maybe ^"` + } `tlb:"^"` + ConfigParams *ConfigParams `tlb:"?KeyBlock ."` } type BlockExtra struct { @@ -117,6 +123,21 @@ func (h *BlockInfo) Equals(h2 *BlockInfo) bool { bytes.Equal(h.FileHash, h2.FileHash) && bytes.Equal(h.RootHash, h2.RootHash) } +func (h *BlockInfo) Copy() *BlockInfo { + root := make([]byte, len(h.RootHash)) + file := make([]byte, len(h.FileHash)) + copy(root, h.RootHash) + copy(file, h.FileHash) + + return &BlockInfo{ + Workchain: h.Workchain, + Shard: h.Shard, + SeqNo: h.SeqNo, + RootHash: root, + FileHash: file, + } +} + func (h *BlockHeader) LoadFromCell(loader *cell.Slice) error { var infoPart blockInfoPart err := LoadFromCell(&infoPart, loader) diff --git a/tlb/coins.go b/tlb/coins.go index 71c46d4f..55a48ebf 100644 --- a/tlb/coins.go +++ b/tlb/coins.go @@ -3,19 +3,25 @@ package tlb import ( "errors" "fmt" - "math" "math/big" - "strconv" "strings" "github.com/xssnick/tonutils-go/tvm/cell" ) type Coins struct { - val *big.Int + decimals int + val *big.Int } +var ZeroCoins = MustFromTON("0") + +// Deprecated: use String func (g Coins) TON() string { + return g.String() +} + +func (g Coins) String() string { if g.val == nil { return "0" } @@ -26,9 +32,9 @@ func (g Coins) TON() string { return a } - splitter := len(a) - 9 + splitter := len(a) - g.decimals if splitter <= 0 { - a = "0." + strings.Repeat("0", 9-len(a)) + a + a = "0." + strings.Repeat("0", g.decimals-len(a)) + a } else { // set . between lo and hi a = a[:splitter] + "." + a[splitter:] @@ -49,13 +55,26 @@ func (g Coins) TON() string { return a } +// Deprecated: use Nano func (g Coins) NanoTON() *big.Int { + return g.Nano() +} + +func (g Coins) Nano() *big.Int { if g.val == nil { return big.NewInt(0) } return g.val } +func MustFromDecimal(val string, decimals int) Coins { + v, err := FromDecimal(val, decimals) + if err != nil { + panic(err) + } + return v +} + func MustFromTON(val string) Coins { v, err := FromTON(val) if err != nil { @@ -64,19 +83,47 @@ func MustFromTON(val string) Coins { return v } +func MustFromNano(val *big.Int, decimals int) Coins { + v, err := FromNano(val, decimals) + if err != nil { + panic(err) + } + return v +} + +func FromNano(val *big.Int, decimals int) (Coins, error) { + if uint((val.BitLen()+7)>>3) >= 16 { + return Coins{}, fmt.Errorf("too big number for coins") + } + + return Coins{ + decimals: decimals, + val: new(big.Int).Set(val), + }, nil +} + func FromNanoTON(val *big.Int) Coins { return Coins{ - val: new(big.Int).Set(val), + decimals: 9, + val: new(big.Int).Set(val), } } func FromNanoTONU(val uint64) Coins { return Coins{ - val: new(big.Int).SetUint64(val), + decimals: 9, + val: new(big.Int).SetUint64(val), } } func FromTON(val string) (Coins, error) { + return FromDecimal(val, 9) +} + +func FromDecimal(val string, decimals int) (Coins, error) { + if decimals < 0 || decimals >= 128 { + return Coins{}, fmt.Errorf("invalid decmals") + } errInvalid := errors.New("invalid string") s := strings.SplitN(val, ".", 2) @@ -90,13 +137,13 @@ func FromTON(val string) (Coins, error) { return Coins{}, errInvalid } - hi = hi.Mul(hi, new(big.Int).SetUint64(1000000000)) + hi = hi.Mul(hi, new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(decimals)), nil)) if len(s) == 2 { loStr := s[1] - // lo can have max 9 digits for ton - if len(loStr) > 9 { - loStr = loStr[:9] + // lo can have max {decimals} digits + if len(loStr) > decimals { + loStr = loStr[:decimals] } leadZeroes := 0 @@ -107,20 +154,24 @@ func FromTON(val string) (Coins, error) { leadZeroes++ } - lo, err := strconv.ParseUint(loStr, 10, 64) - if err != nil { + lo, ok := new(big.Int).SetString(loStr, 10) + if !ok { return Coins{}, errInvalid } - // log10 of 1 == 0, log10 of 10 = 1, so we need offset - digits := int(math.Ceil(math.Log10(float64(lo + 1)))) - lo *= uint64(math.Pow10((9 - leadZeroes) - digits)) + digits := len(lo.String()) // =_= + lo = lo.Mul(lo, new(big.Int).Exp(big.NewInt(10), big.NewInt(int64((decimals-leadZeroes)-digits)), nil)) + + hi = hi.Add(hi, lo) + } - hi = hi.Add(hi, new(big.Int).SetUint64(lo)) + if uint((hi.BitLen()+7)>>3) >= 16 { + return Coins{}, fmt.Errorf("too big number for coins") } return Coins{ - val: hi, + decimals: decimals, + val: hi, }, nil } @@ -134,13 +185,9 @@ func (g *Coins) LoadFromCell(loader *cell.Slice) error { } func (g Coins) ToCell() (*cell.Cell, error) { - return cell.BeginCell().MustStoreBigCoins(g.NanoTON()).EndCell(), nil + return cell.BeginCell().MustStoreBigCoins(g.Nano()).EndCell(), nil } func (g Coins) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf("%q", g.NanoTON().String())), nil -} - -func (g Coins) String() string { - return g.TON() + return []byte(fmt.Sprintf("%q", g.Nano().String())), nil } diff --git a/tlb/coins_test.go b/tlb/coins_test.go index 6d0f573d..8494a7d6 100644 --- a/tlb/coins_test.go +++ b/tlb/coins_test.go @@ -1,6 +1,10 @@ package tlb import ( + "fmt" + "math/big" + "math/rand" + "strings" "testing" ) @@ -58,47 +62,92 @@ func TestCoins_FromTON(t *testing.T) { func TestCoins_TON(t *testing.T) { g := MustFromTON("0.090000001") - if g.TON() != "0.090000001" { - t.Fatalf("0.090000001 wrong: %s", g.TON()) + if g.String() != "0.090000001" { + t.Fatalf("0.090000001 wrong: %s", g.String()) } g = MustFromTON("0.19") - if g.TON() != "0.19" { - t.Fatalf("0.19 wrong: %s", g.TON()) + if g.String() != "0.19" { + t.Fatalf("0.19 wrong: %s", g.String()) } g = MustFromTON("7123.190000") - if g.TON() != "7123.19" { - t.Fatalf("7123.19 wrong: %s", g.TON()) + if g.String() != "7123.19" { + t.Fatalf("7123.19 wrong: %s", g.String()) } g = MustFromTON("5") - if g.TON() != "5" { - t.Fatalf("5 wrong: %s", g.TON()) + if g.String() != "5" { + t.Fatalf("5 wrong: %s", g.String()) } g = MustFromTON("0") - if g.TON() != "0" { - t.Fatalf("0 wrong: %s", g.TON()) + if g.String() != "0" { + t.Fatalf("0 wrong: %s", g.String()) } g = MustFromTON("0.2") - if g.TON() != "0.2" { - t.Fatalf("0.2 wrong: %s", g.TON()) + if g.String() != "0.2" { + t.Fatalf("0.2 wrong: %s", g.String()) } g = MustFromTON("300") - if g.TON() != "300" { - t.Fatalf("300 wrong: %s", g.TON()) + if g.String() != "300" { + t.Fatalf("300 wrong: %s", g.String()) } g = MustFromTON("50") - if g.TON() != "50" { - t.Fatalf("50 wrong: %s", g.TON()) + if g.String() != "50" { + t.Fatalf("50 wrong: %s", g.String()) } g = MustFromTON("350") - if g.TON() != "350" { - t.Fatalf("350 wrong: %s", g.TON()) + if g.String() != "350" { + t.Fatalf("350 wrong: %s", g.String()) + } +} + +func TestCoins_Decimals(t *testing.T) { + for i := 0; i < 16; i++ { + i := i + t.Run("decimals "+fmt.Sprint(i), func(t *testing.T) { + for x := 0; x < 5000; x++ { + rnd := make([]byte, 64) + rand.Read(rnd) + + lo := new(big.Int).Mod(new(big.Int).SetBytes(rnd), new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(i)), nil)) + if i > 0 && strings.HasSuffix(lo.String(), "0") { + lo = lo.Add(lo, big.NewInt(1)) + } + hi := big.NewInt(rand.Int63()) + + amt := new(big.Int).Mul(hi, new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(i)), nil)) + amt = amt.Add(amt, lo) + + var str string + if i > 0 { + loStr := lo.String() + str = fmt.Sprintf("%d.%s", hi, strings.Repeat("0", i-len(loStr))+loStr) + } else { + str = fmt.Sprint(hi) + } + + g, err := FromDecimal(str, i) + if err != nil { + t.Fatalf("%d %s err: %s", i, str, err.Error()) + return + } + + if g.String() != str { + t.Fatalf("%d %s wrong: %s", i, str, g.String()) + return + } + + if g.Nano().String() != amt.String() { + t.Fatalf("%d %s nano wrong: %s", i, amt.String(), g.Nano().String()) + return + } + } + }) } } diff --git a/tlb/config.go b/tlb/config.go index 8b5b5270..a74f8cd3 100644 --- a/tlb/config.go +++ b/tlb/config.go @@ -2,6 +2,23 @@ package tlb import "github.com/xssnick/tonutils-go/tvm/cell" +func init() { + Register(ValidatorSet{}) + Register(ValidatorSetExt{}) + + Register(CatchainConfigV1{}) + Register(CatchainConfigV2{}) + + Register(ConsensusConfigV1{}) + Register(ConsensusConfigV2{}) + Register(ConsensusConfigV3{}) + Register(ConsensusConfigV4{}) +} + +type ValidatorSetAny struct { + Validators any `tlb:"[ValidatorSet,ValidatorSetExt]"` +} + type ValidatorSet struct { _ Magic `tlb:"#11"` UTimeSince uint32 `tlb:"## 32"` @@ -38,3 +55,86 @@ type SigPubKeyED25519 struct { _ Magic `tlb:"#8e81278a"` Key []byte `tlb:"bits 256"` } + +type CatchainConfig struct { + Config any `tlb:"[CatchainConfigV1,CatchainConfigV2]"` +} + +type CatchainConfigV1 struct { + _ Magic `tlb:"#c1"` + McCatchainLifetime uint32 `tlb:"## 32"` + ShardCatchainLifetime uint32 `tlb:"## 32"` + ShardValidatorsLifetime uint32 `tlb:"## 32"` + ShardValidatorsNum uint32 `tlb:"## 32"` +} + +type CatchainConfigV2 struct { + _ Magic `tlb:"#c2"` + Flags uint8 `tlb:"## 7"` + ShuffleMcValidators bool `tlb:"bool"` + McCatchainLifetime uint32 `tlb:"## 32"` + ShardCatchainLifetime uint32 `tlb:"## 32"` + ShardValidatorsLifetime uint32 `tlb:"## 32"` + ShardValidatorsNum uint32 `tlb:"## 32"` +} + +type ConsensusConfig struct { + Config any `tlb:"[ConsensusConfigV1,ConsensusConfigV2,ConsensusConfigV3,ConsensusConfigV4]"` +} + +type ConsensusConfigV1 struct { + _ Magic `tlb:"#d6"` + RoundCandidates uint32 `tlb:"## 32"` + NextCandidateDelayMs uint32 `tlb:"## 32"` + ConsensusTimeoutMs uint32 `tlb:"## 32"` + FastAttempts uint32 `tlb:"## 32"` + AttemptDuration uint32 `tlb:"## 32"` + CatchainMaxDeps uint32 `tlb:"## 32"` + MaxBlockBytes uint32 `tlb:"## 32"` + MaxCollatedBytes uint32 `tlb:"## 32"` +} + +type ConsensusConfigV2 struct { + _ Magic `tlb:"#d7"` + Flags uint8 `tlb:"## 7"` + NewCatchainIds bool `tlb:"bool"` + RoundCandidates uint8 `tlb:"## 8"` + NextCandidateDelayMs uint32 `tlb:"## 32"` + ConsensusTimeoutMs uint32 `tlb:"## 32"` + FastAttempts uint32 `tlb:"## 32"` + AttemptDuration uint32 `tlb:"## 32"` + CatchainMaxDeps uint32 `tlb:"## 32"` + MaxBlockBytes uint32 `tlb:"## 32"` + MaxCollatedBytes uint32 `tlb:"## 32"` +} + +type ConsensusConfigV3 struct { + _ Magic `tlb:"#d8"` + Flags uint8 `tlb:"## 7"` + NewCatchainIds bool `tlb:"bool"` + RoundCandidates uint8 `tlb:"## 8"` + NextCandidateDelayMs uint32 `tlb:"## 32"` + ConsensusTimeoutMs uint32 `tlb:"## 32"` + FastAttempts uint32 `tlb:"## 32"` + AttemptDuration uint32 `tlb:"## 32"` + CatchainMaxDeps uint32 `tlb:"## 32"` + MaxBlockBytes uint32 `tlb:"## 32"` + MaxCollatedBytes uint32 `tlb:"## 32"` + ProtoVersion uint16 `tlb:"## 16"` +} + +type ConsensusConfigV4 struct { + _ Magic `tlb:"#d9"` + Flags uint8 `tlb:"## 7"` + NewCatchainIds bool `tlb:"bool"` + RoundCandidates uint8 `tlb:"## 8"` + NextCandidateDelayMs uint32 `tlb:"## 32"` + ConsensusTimeoutMs uint32 `tlb:"## 32"` + FastAttempts uint32 `tlb:"## 32"` + AttemptDuration uint32 `tlb:"## 32"` + CatchainMaxDeps uint32 `tlb:"## 32"` + MaxBlockBytes uint32 `tlb:"## 32"` + MaxCollatedBytes uint32 `tlb:"## 32"` + ProtoVersion uint16 `tlb:"## 16"` + CatchainMaxBlocksCoff uint32 `tlb:"## 32"` +} diff --git a/tlb/loader.go b/tlb/loader.go index e0c5cebc..5c3d5a58 100644 --- a/tlb/loader.go +++ b/tlb/loader.go @@ -25,12 +25,14 @@ type manualStore interface { // ## N - means integer with N bits, if size <= 64 it loads to uint of any size, if > 64 it loads to *big.Int // ^ - loads ref and calls recursively, if field type is *cell.Cell, it loads without parsing // . - calls recursively to continue load from current loader (inner struct) -// [^]dict N [-> array [^]] - loads dictionary with key size N, transformation '->' can be applied to convert dict to array, example: 'dict 256 -> array ^' will give you array of deserialized refs (^) of values +// dict [inline] N - loads dictionary with key size N, example: 'dict 256', inline option can be used if dict is Hashmap and not HashmapE // bits N - loads bit slice N len to []byte // bool - loads 1 bit boolean // addr - loads ton address // maybe - reads 1 bit, and loads rest if its 1, can be used in combination with others only // either X Y - reads 1 bit, if its 0 - loads X, if 1 - loads Y +// ?FieldName - Conditional field loading depending on boolean value of specified field. +// / Specified field must be declared before tag usage, or it will be always false during loading // Some tags can be combined, for example "dict 256", "maybe ^" // Magic can be used to load first bits and check struct type, in tag can be specified magic number itself, in [#]HEX or [$]BIN format // Example: @@ -44,7 +46,7 @@ func LoadFromCellAsProof(v any, loader *cell.Slice, skipMagic ...bool) error { return loadFromCell(v, loader, true, len(skipMagic) > 0 && skipMagic[0]) } -func loadFromCell(v any, loader *cell.Slice, skipProofBranches, skipMagic bool) error { +func loadFromCell(v any, slice *cell.Slice, skipProofBranches, skipMagic bool) error { rv := reflect.ValueOf(v) if rv.Kind() != reflect.Pointer || rv.IsNil() { return fmt.Errorf("v should be a pointer and not nil") @@ -52,7 +54,7 @@ func loadFromCell(v any, loader *cell.Slice, skipProofBranches, skipMagic bool) rv = rv.Elem() if ld, ok := v.(manualLoader); ok { - err := ld.LoadFromCell(loader) + err := ld.LoadFromCell(slice) if err != nil { return fmt.Errorf("failed to load from cell for %s, using manual loader, err: %w", rv.Type().Name(), err) } @@ -60,6 +62,7 @@ func loadFromCell(v any, loader *cell.Slice, skipProofBranches, skipMagic bool) } for i := 0; i < rv.NumField(); i++ { + loader := slice structField := rv.Type().Field(i) parseType := structField.Type tag := strings.TrimSpace(structField.Tag.Get("tlb")) @@ -72,6 +75,15 @@ func loadFromCell(v any, loader *cell.Slice, skipProofBranches, skipMagic bool) continue } + if settings[0][0] == '?' { + // conditional tlb parse depending on some field value of this struct + cond := rv.FieldByName(settings[0][1:]) + if !cond.Bool() { + continue + } + settings = settings[1:] + } + if settings[0] == "maybe" { if parseType.Kind() != reflect.Pointer && parseType.Kind() != reflect.Interface && parseType.Kind() != reflect.Slice { return fmt.Errorf("maybe flag can only be applied to interface or pointer, field %s", structField.Name) @@ -109,8 +121,9 @@ func loadFromCell(v any, loader *cell.Slice, skipProofBranches, skipMagic bool) } } + typeToLoad := structField.Type setVal := func(val reflect.Value) { - if structField.Type.Kind() == reflect.Pointer && val.Kind() != reflect.Pointer { + if typeToLoad.Kind() == reflect.Pointer && val.Kind() != reflect.Pointer { nw := reflect.New(val.Type()) if val.Type() != parseType { @@ -119,15 +132,69 @@ func loadFromCell(v any, loader *cell.Slice, skipProofBranches, skipMagic bool) nw.Elem().Set(val) val = nw - } else if structField.Type.Kind() != reflect.Pointer && val.Kind() == reflect.Pointer { + } else if typeToLoad.Kind() != reflect.Pointer && val.Kind() == reflect.Pointer { val = val.Elem() } - if structField.Type == val.Type() { + if typeToLoad == val.Type() { rv.Field(i).Set(val) } else { - rv.Field(i).Set(val.Convert(structField.Type)) + rv.Field(i).Set(val.Convert(typeToLoad)) + } + } + + if settings[0] == "^" { + ref, err := loader.LoadRefCell() + if err != nil { + return fmt.Errorf("failed to load ref for %s, err: %w", structField.Name, err) + } + + if skipProofBranches && ref.GetType() == cell.PrunedCellType { + continue + } + + settings = settings[1:] + loader = ref.BeginParse() + } + + if structField.Type.Kind() == reflect.Interface { + allowed := strings.Join(settings, "") + if !strings.HasPrefix(allowed, "[") || !strings.HasSuffix(allowed, "]") { + panic("corrupted allowed list tag, should be [a,b,c], got " + allowed) + } + + // cut brackets + allowed = allowed[1 : len(allowed)-1] + types := strings.Split(allowed, ",") + + for _, typ := range types { + t, ok := registered[typ] + if !ok { + panic("unregistered type " + typ) + } + + if !checkMagic(t.Field(0).Tag.Get("tlb"), loader.Copy()) { + continue + } + + typeToLoad = t + break + } + + if typeToLoad == structField.Type { + return fmt.Errorf("unexpected data to load, unknown magic") } + settings = settings[:0] + } + + if len(settings) == 0 || settings[0] == "." { + nVal, err := structLoad(typeToLoad, loader, false, skipProofBranches) + if err != nil { + return fmt.Errorf("failed to load struct for %s, err: %w", structField.Name, err) + } + + setVal(nVal) + continue } // bits @@ -226,74 +293,16 @@ func loadFromCell(v any, loader *cell.Slice, skipProofBranches, skipMagic bool) setVal(reflect.ValueOf(x)) continue - } else if settings[0] == "^" || settings[0] == "." { - next := loader - - if settings[0] == "^" { - ref, err := loader.LoadRefCell() - if err != nil { - return fmt.Errorf("failed to load ref for %s, err: %w", structField.Name, err) - } - - if skipProofBranches && ref.GetType() == cell.PrunedCellType { - continue - } - - next = ref.BeginParse() - } - - switch parseType { - case reflect.TypeOf(&cell.Cell{}): - c, err := next.ToCell() - if err != nil { - return fmt.Errorf("failed to convert ref to cell for %s, err: %w", structField.Name, err) - } - - setVal(reflect.ValueOf(c)) - continue - default: - nVal, err := structLoad(structField.Type, next) - if err != nil { - return err - } - - setVal(nVal) - continue - } } else if parseType == reflect.TypeOf(Magic{}) { if skipMagic { // it can be skipped if parsed before in parent type, to determine child type continue } - var sz, base int - if strings.HasPrefix(settings[0], "#") { - base = 16 - sz = (len(settings[0]) - 1) * 4 - } else if strings.HasPrefix(settings[0], "$") { - base = 2 - sz = len(settings[0]) - 1 - } else { - panic("unknown magic value type in tag") - } - - if sz > 64 { - panic("too big magic value type in tag") + if !checkMagic(settings[0], loader) { + return fmt.Errorf("magic is not correct for %s, want %s", rv.Type().String(), settings[0]) } - magic, err := strconv.ParseInt(settings[0][1:], base, 64) - if err != nil { - panic("corrupted magic value in tag") - } - - ldMagic, err := loader.LoadUInt(uint(sz)) - if err != nil { - return fmt.Errorf("failed to load magic: %w", err) - } - - if ldMagic != uint64(magic) { - return fmt.Errorf("magic is not correct for %s, want %x, got %x", rv.Type().String(), magic, ldMagic) - } continue } else if settings[0] == "dict" { inline := false @@ -320,42 +329,6 @@ func loadFromCell(v any, loader *cell.Slice, skipProofBranches, skipMagic bool) } } - if len(settings) >= 4 { - // transformation - if settings[2] == "->" { - isRef := false - if len(settings) >= 5 { - if settings[4] == "^" { - isRef = true - } - } - - switch settings[3] { - case "array": - arr := rv.Field(i) - for _, kv := range dict.All() { - ld := kv.Value.BeginParse() - if isRef { - ld, err = ld.LoadRef() - if err != nil { - return fmt.Errorf("failed to load ref in dict transform: %w", err) - } - } - - nVal, err := structLoad(parseType.Elem(), ld) - if err != nil { - return fmt.Errorf("failed to load struct in dict transform: %w", err) - } - - arr = reflect.Append(arr, nVal) - } - setVal(arr) - continue - default: - panic("transformation to this type is not supported") - } - } - } setVal(reflect.ValueOf(dict)) continue } else if settings[0] == "var" { @@ -382,6 +355,34 @@ func loadFromCell(v any, loader *cell.Slice, skipProofBranches, skipMagic bool) return nil } +func checkMagic(tag string, loader *cell.Slice) bool { + var sz, base int + if strings.HasPrefix(tag, "#") { + base = 16 + sz = (len(tag) - 1) * 4 + } else if strings.HasPrefix(tag, "$") { + base = 2 + sz = len(tag) - 1 + } else { + panic("unknown magic value type in tag: " + tag) + } + + if sz > 64 { + panic("too big magic value type in tag") + } + + magic, err := strconv.ParseInt(tag[1:], base, 64) + if err != nil { + panic("corrupted magic value in tag") + } + + ldMagic, err := loader.LoadUInt(uint(sz)) + if err != nil { + return false + } + return ldMagic == uint64(magic) +} + func ToCell(v any) (*cell.Cell, error) { rv := reflect.ValueOf(v) if rv.Kind() == reflect.Pointer { @@ -399,9 +400,10 @@ func ToCell(v any) (*cell.Cell, error) { return c, nil } - builder := cell.BeginCell() + root := cell.BeginCell() for i := 0; i < rv.NumField(); i++ { + builder := root structField := rv.Type().Field(i) parseType := structField.Type fieldVal := rv.Field(i) @@ -415,6 +417,15 @@ func ToCell(v any) (*cell.Cell, error) { continue } + if settings[0][0] == '?' { + // conditional tlb parse depending on some field value of this struct + cond := rv.FieldByName(settings[0][1:]) + if !cond.Bool() { + continue + } + settings = settings[1:] + } + if settings[0] == "maybe" { if structField.Type.Kind() != reflect.Pointer && structField.Type.Kind() != reflect.Interface && structField.Type.Kind() != reflect.Slice { return nil, fmt.Errorf("maybe flag can only be applied to interface or pointer, field %s", structField.Name) @@ -457,7 +468,49 @@ func ToCell(v any) (*cell.Cell, error) { fieldVal = fieldVal.Elem() } - if settings[0] == "##" { + asRef := false + if settings[0] == "^" { + asRef = true + settings = settings[1:] + builder = cell.BeginCell() + } + + if structField.Type.Kind() == reflect.Interface { + allowed := strings.Join(settings, "") + if !strings.HasPrefix(allowed, "[") || !strings.HasSuffix(allowed, "]") { + panic("corrupted allowed list tag, should be [a,b,c], got " + allowed) + } + + // cut brackets + allowed = allowed[1 : len(allowed)-1] + types := strings.Split(allowed, ",") + + t := fieldVal.Elem().Type() + found := false + for _, typ := range types { + if t.Name() == typ { + found = true + break + } + } + + if !found { + return nil, fmt.Errorf("unexpected data to serialize, not registered magic in tag") + } + settings = settings[:0] + } + + if len(settings) == 0 || settings[0] == "." { + c, err := structStore(fieldVal, structField.Type.Name()) + if err != nil { + return nil, err + } + + err = builder.StoreBuilder(c.ToBuilder()) + if err != nil { + return nil, fmt.Errorf("failed to store cell to builder for %s, err: %w", structField.Name, err) + } + } else if settings[0] == "##" { num, err := strconv.ParseUint(settings[1], 10, 64) if err != nil { // we panic, because its developer's issue, need to fix tag @@ -483,31 +536,26 @@ func ToCell(v any) (*cell.Cell, error) { if err != nil { return nil, fmt.Errorf("failed to store bigint %d, err: %w", num, err) } - continue } else { panic("unexpected field type for tag ## - " + parseType.String()) } } - continue case num <= 256: err := builder.StoreBigInt(fieldVal.Interface().(*big.Int), uint(num)) if err != nil { return nil, fmt.Errorf("failed to store bigint %d, err: %w", num, err) } - continue } } else if settings[0] == "addr" { err := builder.StoreAddr(fieldVal.Interface().(*address.Address)) if err != nil { return nil, fmt.Errorf("failed to store address, err: %w", err) } - continue } else if settings[0] == "bool" { err := builder.StoreBoolBit(fieldVal.Bool()) if err != nil { return nil, fmt.Errorf("failed to store bool, err: %w", err) } - continue } else if settings[0] == "bits" { num, err := strconv.Atoi(settings[1]) if err != nil { @@ -519,38 +567,6 @@ func ToCell(v any) (*cell.Cell, error) { if err != nil { return nil, fmt.Errorf("failed to store bits %d, err: %w", num, err) } - continue - } else if settings[0] == "^" || settings[0] == "." { - var err error - var c *cell.Cell - - switch parseType { - case reflect.TypeOf(&cell.Cell{}): - if fieldVal.IsNil() { - c = cell.BeginCell().EndCell() - } else { - c = fieldVal.Interface().(*cell.Cell) - } - default: - c, err = structStore(fieldVal, structField.Type.Name()) - if err != nil { - return nil, err - } - } - - if settings[0] == "^" { - err = builder.StoreRef(c) - if err != nil { - return nil, fmt.Errorf("failed to store cell to ref for %s, err: %w", structField.Name, err) - } - continue - } - - err = builder.StoreBuilder(c.ToBuilder()) - if err != nil { - return nil, fmt.Errorf("failed to store cell to builder for %s, err: %w", structField.Name, err) - } - continue } else if parseType == reflect.TypeOf(Magic{}) { var sz, base int if strings.HasPrefix(settings[0], "#") { @@ -576,13 +592,11 @@ func ToCell(v any) (*cell.Cell, error) { if err != nil { return nil, fmt.Errorf("failed to store magic: %w", err) } - continue } else if settings[0] == "dict" { err := builder.StoreDict(fieldVal.Interface().(*cell.Dictionary)) if err != nil { return nil, fmt.Errorf("failed to store dict for %s, err: %w", structField.Name, err) } - continue } else if settings[0] == "var" { if settings[1] == "uint" { sz, err := strconv.Atoi(settings[2]) @@ -594,19 +608,35 @@ func ToCell(v any) (*cell.Cell, error) { if err != nil { return nil, fmt.Errorf("failed to store var uint: %w", err) } - continue } else { panic("var of type " + settings[1] + " is not supported") } + } else { + panic(fmt.Sprintf("cannot serialize field '%s' as tag '%s' of struct '%s', use manual serialization", structField.Name, tag, rv.Type().String())) } - panic(fmt.Sprintf("cannot serialize field '%s' as tag '%s' of struct '%s', use manual serialization", structField.Name, tag, rv.Type().String())) + if asRef { + err := root.StoreRef(builder.EndCell()) + if err != nil { + return nil, fmt.Errorf("failed to store cell to ref for %s, err: %w", structField.Name, err) + } + } } - return builder.EndCell(), nil + return root.EndCell(), nil } -func structLoad(field reflect.Type, loader *cell.Slice) (reflect.Value, error) { +var cellType = reflect.TypeOf(&cell.Cell{}) + +func structLoad(field reflect.Type, loader *cell.Slice, skipMagic, skipProofBranches bool) (reflect.Value, error) { + if cellType == field { + c, err := loader.ToCell() + if err != nil { + return reflect.Value{}, fmt.Errorf("failed to convert slice to cell: %w", err) + } + return reflect.ValueOf(c), nil + } + newTyp := field if newTyp.Kind() == reflect.Ptr { newTyp = newTyp.Elem() @@ -614,7 +644,7 @@ func structLoad(field reflect.Type, loader *cell.Slice) (reflect.Value, error) { nVal := reflect.New(newTyp) - err := LoadFromCell(nVal.Interface(), loader) + err := loadFromCell(nVal.Interface(), loader, skipProofBranches, skipMagic) if err != nil { return reflect.Value{}, fmt.Errorf("failed to load from cell for %s, err: %w", field.Name(), err) } @@ -627,6 +657,13 @@ func structLoad(field reflect.Type, loader *cell.Slice) (reflect.Value, error) { } func structStore(field reflect.Value, name string) (*cell.Cell, error) { + if field.Type() == cellType { + if field.IsNil() { + return cell.BeginCell().EndCell(), nil + } + return field.Interface().(*cell.Cell), nil + } + inf := field.Interface() c, err := ToCell(inf) diff --git a/tlb/loader_test.go b/tlb/loader_test.go index 808171f4..98214b8d 100644 --- a/tlb/loader_test.go +++ b/tlb/loader_test.go @@ -2,7 +2,9 @@ package tlb import ( "bytes" + "encoding/json" "math/big" + "os" "testing" "github.com/xssnick/tonutils-go/address" @@ -26,8 +28,23 @@ func (m manualLoad) ToCell() (*cell.Cell, error) { return cell.BeginCell().MustStoreUInt(uint64(m.Val[0]), 8).EndCell(), nil } -type testTransform struct { - DictTransform []*manualLoad `tlb:"dict 32 -> array"` +type StructA struct { + _ Magic `tlb:"$10"` + Val int8 `tlb:"## 3"` +} + +type StructB struct { + _ Magic `tlb:"#00AACC"` + Val uint16 `tlb:"## 16"` +} + +type StructC struct { + _ Magic `tlb:"#00BBCC"` + Val bool `tlb:"bool"` +} + +type testAny struct { + StructAny any `tlb:"^ [StructA,StructB]"` } type testInner struct { @@ -57,6 +74,29 @@ type testTLB struct { EndCell *cell.Cell `tlb:"."` } +func TestLoadAnyRegistered(t *testing.T) { + Register(StructA{}) + Register(StructC{}) + + v := testAny{ + StructAny: StructA{ + Val: 2, + }, + } + + c, err := ToCell(v) + if err != nil { + t.Fatal(err) + } + + var v2 testAny + err = LoadFromCell(&v2, c.BeginParse()) + if err != nil { + t.Fatal(err) + } + json.NewEncoder(os.Stdout).Encode(v2) +} + func TestLoadFromCell(t *testing.T) { addr := address.MustParseAddr("EQCD39VS5jcptHL8vMjEXrzGaRcCVYto7HUn4bpAOg8xqB2N") dKey := cell.BeginCell().MustStoreSlice(addr.Data(), 256).EndCell() @@ -163,33 +203,3 @@ func TestLoadFromCell(t *testing.T) { t.Fatal("cell hashes not same after From to") } } - -func TestLoadFromCellTransform(t *testing.T) { - ldVal := cell.BeginCell().MustStoreUInt(uint64('M'), 8).EndCell() - - d2 := cell.NewDict(32) - for i := 0; i < 200; i++ { - err := d2.Set(cell.BeginCell().MustStoreInt(int64(i), 32).EndCell(), ldVal) - if err != nil { - t.Fatal(err) - } - } - - a := cell.BeginCell().MustStoreDict(d2).EndCell().BeginParse() - - x := testTransform{} - err := LoadFromCell(&x, a) - if err != nil { - t.Fatal(err) - } - - if len(x.DictTransform) != 200 { - t.Fatal("dict transform len not 200") - } - - for _, m := range x.DictTransform { - if m.Val != "M" { - t.Fatal("dict transform values corrupted") - } - } -} diff --git a/tlb/message.go b/tlb/message.go index 0fbfddc6..2caa6dfc 100644 --- a/tlb/message.go +++ b/tlb/message.go @@ -28,7 +28,7 @@ type Message struct { } type MessagesList struct { - List *cell.Dictionary + List *cell.Dictionary `tlb:"dict inline 15"` } type InternalMessage struct { @@ -231,7 +231,7 @@ func (m *InternalMessage) ToCell() (*cell.Cell, error) { func (m *InternalMessage) Dump() string { return fmt.Sprintf("Amount %s TON, Created at: %d, Created lt %d\nBounce: %t, Bounced %t, IHRDisabled %t\nSrcAddr: %s\nDstAddr: %s\nPayload: %s", - m.Amount.TON(), m.CreatedAt, m.CreatedLT, m.Bounce, m.Bounced, m.IHRDisabled, m.SrcAddr, m.DstAddr, m.Body.Dump()) + m.Amount.String(), m.CreatedAt, m.CreatedLT, m.Bounce, m.Bounced, m.IHRDisabled, m.SrcAddr, m.DstAddr, m.Body.Dump()) } func (m *ExternalMessage) ToCell() (*cell.Cell, error) { @@ -267,19 +267,6 @@ func (m *ExternalMessage) ToCell() (*cell.Cell, error) { return builder.EndCell(), nil } -func (m *MessagesList) LoadFromCell(loader *cell.Slice) error { - dict, err := loader.ToDict(15) - if err != nil { - return err - } - m.List = dict - return nil -} - -func (m *MessagesList) ToCell() (*cell.Cell, error) { - return m.List.ToCell() -} - func (m *MessagesList) ToSlice() ([]Message, error) { if m.List == nil { return nil, nil diff --git a/tlb/register.go b/tlb/register.go new file mode 100644 index 00000000..3c35ced3 --- /dev/null +++ b/tlb/register.go @@ -0,0 +1,25 @@ +package tlb + +import ( + "reflect" + "strings" +) + +var registered = map[string]reflect.Type{} + +var magicType = reflect.TypeOf(Magic{}) + +func Register(typ any) { + t := reflect.TypeOf(typ) + magic := t.Field(0) + if magic.Type != magicType { + panic("first field is not magic") + } + + tag := magic.Tag.Get("tlb") + if !strings.HasPrefix(tag, "#") && !strings.HasPrefix(tag, "$") { + panic("invalid magic tag") + } + + registered[t.Name()] = t +} diff --git a/tlb/shard.go b/tlb/shard.go index fac43a6a..72ec96fe 100644 --- a/tlb/shard.go +++ b/tlb/shard.go @@ -1,12 +1,18 @@ package tlb import ( - "fmt" - - "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/tvm/cell" ) +func init() { + Register(FutureSplit{}) + Register(FutureMerge{}) + Register(FutureSplitMergeNone{}) + + Register(ShardStateSplit{}) + Register(ShardStateUnsplit{}) +} + type ShardStateUnsplit struct { _ Magic `tlb:"#9023afe2"` GlobalID int32 `tlb:"## 32"` @@ -33,14 +39,64 @@ type McStateExtra struct { GlobalBalance CurrencyCollection `tlb:"."` } +/* +flags:(## 16) { flags <= 1 } + validator_info:ValidatorInfo + prev_blocks:OldMcBlocksInfo + after_key_block:Bool + last_key_block:(Maybe ExtBlkRef) + block_create_stats:(flags . 0)?BlockCreateStats + +validator_info$_ + validator_list_hash_short:uint32 + catchain_seqno:uint32 + nx_cc_updated:Bool += ValidatorInfo; + +ext_blk_ref$_ end_lt:uint64 + seq_no:uint32 root_hash:bits256 file_hash:bits256 + = ExtBlkRef; + +_ key:Bool max_end_lt:uint64 = KeyMaxLt; +_ key:Bool blk_ref:ExtBlkRef = KeyExtBlkRef; + +*/ + +type KeyExtBlkRef struct { + IsKey bool `tlb:"bool"` + BlkRef ExtBlkRef `tlb:"."` +} + +type KeyMaxLt struct { + IsKey bool `tlb:"bool"` + MaxEndLT uint64 `tlb:"## 64"` +} + +type ValidatorInfo struct { + ValidatorListHashShort uint32 `tlb:"## 32"` + CatchainSeqno uint32 `tlb:"## 32"` + NextCCUpdated bool `tlb:"bool"` +} + +type McStateExtraBlockInfo struct { + Flags uint16 `tlb:"## 16"` + ValidatorInfo ValidatorInfo `tlb:"."` + PrevBlocks *cell.Dictionary `tlb:"dict 32"` + LastKeyBlock *ExtBlkRef `tlb:"maybe ."` + BlockCreateStats *cell.Cell `tlb:"."` +} + type ConfigParams struct { - ConfigAddr *address.Address - Config *cell.Dictionary + ConfigAddr []byte `tlb:"bits 256"` + Config struct { + Params *cell.Dictionary `tlb:"dict inline 32"` + } `tlb:"^"` } -type ShardState struct { - Left ShardStateUnsplit - Right *ShardStateUnsplit +type ShardStateSplit struct { + _ Magic `tlb:"#5f327da5"` + Left ShardStateUnsplit `tlb:"^"` + Right ShardStateUnsplit `tlb:"^"` } type ShardIdent struct { @@ -51,7 +107,7 @@ type ShardIdent struct { } type FutureSplitMergeNone struct { - _ Magic `tlb:"$00"` + _ Magic `tlb:"$0"` } type FutureSplit struct { @@ -66,29 +122,25 @@ type FutureMerge struct { Interval uint32 `tlb:"## 32"` } -type FutureSplitMerge struct { - FSM any `tlb:"."` -} - type ShardDesc struct { - _ Magic `tlb:"#a"` - SeqNo uint32 `tlb:"## 32"` - RegMcSeqno uint32 `tlb:"## 32"` - StartLT uint64 `tlb:"## 64"` - EndLT uint64 `tlb:"## 64"` - RootHash []byte `tlb:"bits 256"` - FileHash []byte `tlb:"bits 256"` - BeforeSplit bool `tlb:"bool"` - BeforeMerge bool `tlb:"bool"` - WantSplit bool `tlb:"bool"` - WantMerge bool `tlb:"bool"` - NXCCUpdated bool `tlb:"bool"` - Flags uint8 `tlb:"## 3"` - NextCatchainSeqNo uint32 `tlb:"## 32"` - NextValidatorShard int64 `tlb:"## 64"` - MinRefMcSeqNo uint32 `tlb:"## 32"` - GenUTime uint32 `tlb:"## 32"` - SplitMergeAt FutureSplitMerge `tlb:"."` + _ Magic `tlb:"#a"` + SeqNo uint32 `tlb:"## 32"` + RegMcSeqno uint32 `tlb:"## 32"` + StartLT uint64 `tlb:"## 64"` + EndLT uint64 `tlb:"## 64"` + RootHash []byte `tlb:"bits 256"` + FileHash []byte `tlb:"bits 256"` + BeforeSplit bool `tlb:"bool"` + BeforeMerge bool `tlb:"bool"` + WantSplit bool `tlb:"bool"` + WantMerge bool `tlb:"bool"` + NXCCUpdated bool `tlb:"bool"` + Flags uint8 `tlb:"## 3"` + NextCatchainSeqNo uint32 `tlb:"## 32"` + NextValidatorShard int64 `tlb:"## 64"` + MinRefMcSeqNo uint32 `tlb:"## 32"` + GenUTime uint32 `tlb:"## 32"` + SplitMergeAt any `tlb:"[FutureMerge,FutureSplit,FutureSplitMergeNone]"` Currencies struct { FeesCollected CurrencyCollection `tlb:"."` FundsCreated CurrencyCollection `tlb:"."` @@ -113,105 +165,7 @@ type ShardDescB struct { NextValidatorShard int64 `tlb:"## 64"` MinRefMcSeqNo uint32 `tlb:"## 32"` GenUTime uint32 `tlb:"## 32"` - SplitMergeAt FutureSplitMerge `tlb:"."` + SplitMergeAt any `tlb:"[FutureMerge,FutureSplit,FutureSplitMergeNone]"` FeesCollected CurrencyCollection `tlb:"."` FundsCreated CurrencyCollection `tlb:"."` } - -func (s *ShardState) LoadFromCell(loader *cell.Slice) error { - preloader := loader.Copy() - tag, err := preloader.LoadUInt(32) - if err != nil { - return err - } - - switch tag { - case 0x5f327da5: - var left, right ShardStateUnsplit - leftRef, err := loader.LoadRef() - if err != nil { - return err - } - rightRef, err := loader.LoadRef() - if err != nil { - return err - } - err = LoadFromCell(&left, leftRef) - if err != nil { - return err - } - err = LoadFromCell(&right, rightRef) - if err != nil { - return err - } - s.Left = left - s.Right = &right - case 0x9023afe2: - var state ShardStateUnsplit - err = LoadFromCell(&state, preloader, true) - if err != nil { - fmt.Println("ShardStateUnsplit error", err.Error()) - return err - } - s.Left = state - } - - return nil -} - -func (p *ConfigParams) LoadFromCell(loader *cell.Slice) error { - addrBits, err := loader.LoadSlice(256) - if err != nil { - return fmt.Errorf("failed to load bits of config addr: %w", err) - } - - dictRef, err := loader.LoadRef() - if err != nil { - return fmt.Errorf("failed to load config dict ref: %w", err) - } - - dict, err := dictRef.ToDict(32) - if err != nil { - return fmt.Errorf("failed to load config dict: %w", err) - } - - p.ConfigAddr = address.NewAddress(0, 255, addrBits) - p.Config = dict - - return nil -} - -func (s *FutureSplitMerge) LoadFromCell(loader *cell.Slice) error { - isNotEmpty, err := loader.LoadBoolBit() - if err != nil { - return fmt.Errorf("load bool bit is fsm none: %w", err) - } - - if !isNotEmpty { - s.FSM = FutureSplitMergeNone{} - return nil - } - - isMerge, err := loader.LoadBoolBit() - if err != nil { - return fmt.Errorf("load bool bit is fsm merge: %w", err) - } - - if !isMerge { - var split FutureSplit - err = LoadFromCell(&split, loader, true) - if err != nil { - return fmt.Errorf("failed to parse FutureSplit: %w", err) - } - s.FSM = split - } else { - var merge FutureMerge - err = LoadFromCell(&merge, loader, true) - if err != nil { - return fmt.Errorf("failed to parse FutureMerge: %w", err) - } - s.FSM = merge - } - - return nil -} diff --git a/tlb/shard_test.go b/tlb/shard_test.go index c66c1590..a01640a5 100644 --- a/tlb/shard_test.go +++ b/tlb/shard_test.go @@ -18,9 +18,14 @@ func TestShardState_LoadFromCell(t *testing.T) { {"blockType1 (tag 0x5f327da5)", tag1, true}, //{"blockType2 (tag 0x9023afe2)", tag2, false}, } + + type testStruct struct { + State any `tlb:"[ShardStateUnsplit,ShardStateSplit]"` + } + for _, test := range tests { t.Run(test.name, func(t *testing.T) { - var testShard ShardState + var testShard testStruct var tBuilder cell.Builder err := tBuilder.StoreUInt(uint64(test.tag), 32) if err != nil { @@ -46,13 +51,13 @@ func TestShardState_LoadFromCell(t *testing.T) { } } - err = testShard.LoadFromCell(tBuilder.EndCell().BeginParse()) + err = LoadFromCell(&testShard, tBuilder.EndCell().BeginParse()) if err != nil { t.Fatal(err) } - if (testShard.Right != nil) != test.leftAndRightRef { - t.Errorf("right reference is not found") + if testShard.State.(ShardStateSplit).Left.Seqno != 24374596 { + t.Fatal("incorrect result") } }) } diff --git a/tlb/transaction.go b/tlb/transaction.go index b95334f1..6a73c9fb 100644 --- a/tlb/transaction.go +++ b/tlb/transaction.go @@ -3,15 +3,26 @@ package tlb import ( "fmt" "math/big" + "reflect" + "strings" "github.com/xssnick/tonutils-go/tvm/cell" ) -// Deprecated: use ton.TransactionShortInfo -type TransactionID struct { - LT uint64 - Hash []byte - AccountID []byte +func init() { + Register(TransactionDescriptionOrdinary{}) + Register(TransactionDescriptionTickTock{}) + Register(TransactionDescriptionStorage{}) + Register(TransactionDescriptionMergeInstall{}) + Register(TransactionDescriptionMergePrepare{}) + Register(TransactionDescriptionSplitInstall{}) + Register(TransactionDescriptionSplitPrepare{}) + + Register(ComputePhaseVM{}) + Register(ComputePhaseSkipped{}) + Register(BouncePhaseNegFunds{}) + Register(BouncePhaseOk{}) + Register(BouncePhaseNoFunds{}) } type AccStatusChangeType string @@ -75,11 +86,11 @@ type ComputePhaseVM struct { } type ComputePhase struct { - Phase any `tlb:"."` + Phase any `tlb:"[ComputePhaseVM,ComputePhaseSkipped]"` } type BouncePhase struct { - Phase any `tlb:"."` + Phase any `tlb:"[BouncePhaseOk,BouncePhaseNegFunds,BouncePhaseNoFunds]"` } type BouncePhaseNegFunds struct { @@ -192,7 +203,7 @@ type TransactionDescriptionMergeInstall struct { } type TransactionDescription struct { - Description any `tlb:"."` + Description any `tlb:"[TransactionDescriptionOrdinary,TransactionDescriptionStorage,TransactionDescriptionTickTock,TransactionDescriptionSplitPrepare,TransactionDescriptionSplitInstall,TransactionDescriptionMergePrepare,TransactionDescriptionMergeInstall]"` } type HashUpdate struct { @@ -258,6 +269,11 @@ func (t *Transaction) String() string { var build string + switch t.Description.Description.(type) { + default: + return "[" + strings.ReplaceAll(reflect.TypeOf(t.Description.Description).Name(), "TransactionDescription", "") + "]" + case TransactionDescriptionOrdinary: + } if t.IO.In != nil { if t.IO.In.MsgType == MsgTypeInternal { in = t.IO.In.AsInternal().Amount.NanoTON() @@ -265,7 +281,7 @@ func (t *Transaction) String() string { if in.Cmp(big.NewInt(0)) != 0 { intTx := t.IO.In.AsInternal() - build += fmt.Sprintf("LT: %d, In: %s TON, From %s", t.LT, FromNanoTON(in).TON(), intTx.SrcAddr) + build += fmt.Sprintf("LT: %d, In: %s TON, From %s", t.LT, FromNanoTON(in).String(), intTx.SrcAddr) comment := intTx.Comment() if comment != "" { build += ", Comment: " + comment @@ -277,7 +293,7 @@ func (t *Transaction) String() string { if len(build) > 0 { build += ", " } - build += fmt.Sprintf("Out: %s TON, To %s", FromNanoTON(out).TON(), destinations) + build += fmt.Sprintf("Out: %s TON, To %s", FromNanoTON(out).String(), destinations) } return build @@ -363,156 +379,3 @@ func (c ComputeSkipReason) ToCell() (*cell.Cell, error) { } return nil, fmt.Errorf("unknown compute skip reason %s", c.Type) } - -func (c *ComputePhase) LoadFromCell(loader *cell.Slice) error { - isNotSkipped, err := loader.LoadBoolBit() - if err != nil { - return err - } - - if isNotSkipped { - var phase ComputePhaseVM - err = LoadFromCell(&phase, loader, true) - if err != nil { - return fmt.Errorf("failed to parse ComputePhaseVM: %w", err) - } - c.Phase = phase - return nil - } - - var phase ComputePhaseSkipped - err = LoadFromCell(&phase, loader, true) - if err != nil { - return fmt.Errorf("failed to parse ComputePhaseSkipped: %w", err) - } - c.Phase = phase - return nil -} - -func (b *BouncePhase) LoadFromCell(loader *cell.Slice) error { - isOk, err := loader.LoadBoolBit() - if err != nil { - return err - } - - if isOk { - var phase BouncePhaseOk - err = LoadFromCell(&phase, loader, true) - if err != nil { - return fmt.Errorf("failed to parse BouncePhaseOk: %w", err) - } - b.Phase = phase - return nil - } - - isNoFunds, err := loader.LoadBoolBit() - if err != nil { - return err - } - - if isNoFunds { - var phase BouncePhaseNoFunds - err = LoadFromCell(&phase, loader, true) - if err != nil { - return fmt.Errorf("failed to parse BouncePhaseNoFunds: %w", err) - } - b.Phase = phase - return nil - } - - var phase BouncePhaseNegFunds - err = LoadFromCell(&phase, loader, true) - if err != nil { - return fmt.Errorf("failed to parse BouncePhaseNegFunds: %w", err) - } - b.Phase = phase - return nil -} - -func (t *TransactionDescription) LoadFromCell(loader *cell.Slice) error { - pfx, err := loader.LoadUInt(3) - if err != nil { - return err - } - - switch pfx { - case 0b000: - isStorage, err := loader.LoadBoolBit() - if err != nil { - return err - } - - if isStorage { - var desc TransactionDescriptionStorage - err = LoadFromCell(&desc, loader, true) - if err != nil { - return fmt.Errorf("failed to parse TransactionDescriptionStorage: %w", err) - } - t.Description = desc - return nil - } - - var desc TransactionDescriptionOrdinary - err = LoadFromCell(&desc, loader, true) - if err != nil { - return fmt.Errorf("failed to parse TransactionDescriptionOrdinary: %w", err) - } - t.Description = desc - return nil - case 0b001: - var desc TransactionDescriptionTickTock - err = LoadFromCell(&desc, loader, true) - if err != nil { - return fmt.Errorf("failed to parse TransactionDescriptionTickTock: %w", err) - } - t.Description = desc - return nil - case 0b010: - isInstall, err := loader.LoadBoolBit() - if err != nil { - return err - } - - if isInstall { - var desc TransactionDescriptionSplitInstall - err = LoadFromCell(&desc, loader, true) - if err != nil { - return fmt.Errorf("failed to parse TransactionDescriptionSplitInstall: %w", err) - } - t.Description = desc - return nil - } - - var desc TransactionDescriptionSplitPrepare - err = LoadFromCell(&desc, loader, true) - if err != nil { - return fmt.Errorf("failed to parse TransactionDescriptionSplitPrepare: %w", err) - } - t.Description = desc - return nil - case 0b011: - isInstall, err := loader.LoadBoolBit() - if err != nil { - return err - } - - if isInstall { - var desc TransactionDescriptionMergeInstall - err = LoadFromCell(&desc, loader, true) - if err != nil { - return fmt.Errorf("failed to parse TransactionDescriptionMergeInstall: %w", err) - } - t.Description = desc - return nil - } - - var desc TransactionDescriptionMergePrepare - err = LoadFromCell(&desc, loader, true) - if err != nil { - return fmt.Errorf("failed to parse TransactionDescriptionMergePrepare: %w", err) - } - t.Description = desc - return nil - } - return fmt.Errorf("unknown transaction description type") -} diff --git a/tlb/transaction_test.go b/tlb/transaction_test.go index 2aa7eb8d..866f0ca0 100644 --- a/tlb/transaction_test.go +++ b/tlb/transaction_test.go @@ -38,3 +38,18 @@ func TestTransaction_Dump(t *testing.T) { tx.Hash = txCell.Hash() tx.Dump() } + +func TestTransaction_NoAction(t *testing.T) { + txData, _ := hex.DecodeString("b5ee9c724102060100013e0003af719dd9de25ac93578116413f89610061cf28f52daf1581373bd8671f7abddfd640000244d94d3f309d7cfcadc8e05ebbd460c2c420020d8e3bfd336da4b1c2d0b53f79127093409090000244d94d3f30164d081fd0001408020105008272d38ee1e2b7328b24e8e3836bb288aa9c96218b9a81e7a8fd290e1a4ccf0a65da9be924ff9d7f16b238a76ae47db9d2f56769c1b0c1ccb0fa95522820318401090101a00301ab680122f3d92b6fb36afc55adb8e4e8ef8e2101e4b488d540f31b1826eb15e121b92b000677677896b24d5e045904fe258401873ca3d4b6bc5604dcef619c7deaf77f590404061ed7e60000489b29a7e610c9a103fac00400687362d09c0000244d94d3f303601062ad47c00800731f1286645e6ced11b52e9a2c07cab0d6ea42390b5b969fd204a0e031294cd0001104084049a0187a12026ec7dc45") + txCell, err := cell.FromBOC(txData) + if err != nil { + t.Fatal(err) + } + + slc := txCell.BeginParse() + var tx Transaction + err = LoadFromCell(&tx, slc) + if err != nil { + t.Fatal(err) + } +} diff --git a/ton/api.go b/ton/api.go index d0f526d1..8f52873d 100644 --- a/ton/api.go +++ b/ton/api.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/liteclient" "github.com/xssnick/tonutils-go/tl" "github.com/xssnick/tonutils-go/tlb" "reflect" @@ -15,6 +16,14 @@ func init() { tl.Register(LSError{}, "liteServer.error code:int message:string = liteServer.Error") } +type ProofCheckPolicy int + +const ( + ProofCheckPolicyUnsafe ProofCheckPolicy = iota + ProofCheckPolicyFastWithoutMasterBlockChecks + ProofCheckPolicySecure +) + const ( ErrCodeContractNotInitialized = -256 ) @@ -22,6 +31,7 @@ const ( type LiteClient interface { QueryLiteserver(ctx context.Context, payload tl.Serializable, result tl.Serializable) error StickyContext(ctx context.Context) context.Context + StickyContextNextNode(ctx context.Context) (context.Context, error) StickyNodeID(ctx context.Context) uint32 } @@ -47,15 +57,19 @@ type APIClientWaiter interface { RunGetMethod(ctx context.Context, blockInfo *BlockIDExt, addr *address.Address, method string, params ...interface{}) (*ExecutionResult, error) ListTransactions(ctx context.Context, addr *address.Address, num uint32, lt uint64, txHash []byte) ([]*tlb.Transaction, error) GetTransaction(ctx context.Context, block *BlockIDExt, addr *address.Address, lt uint64) (*tlb.Transaction, error) + GetBlockProof(ctx context.Context, known, target *BlockIDExt) (*PartialBlockProof, error) } type APIClient struct { client LiteClient parent *APIClient - curMasters map[uint32]*masterInfo - curMastersLock sync.RWMutex - skipProofCheck bool + trustedBlock *BlockIDExt + curMasters map[uint32]*masterInfo + curMastersLock sync.RWMutex + proofCheckPolicy ProofCheckPolicy + + trustedLock sync.RWMutex } type masterInfo struct { @@ -64,15 +78,26 @@ type masterInfo struct { block *BlockIDExt } -func NewAPIClient(client LiteClient) *APIClient { +func NewAPIClient(client LiteClient, proofCheckPolicy ...ProofCheckPolicy) *APIClient { + policy := ProofCheckPolicyFastWithoutMasterBlockChecks + if len(proofCheckPolicy) > 0 { + policy = proofCheckPolicy[0] + } + return &APIClient{ - curMasters: map[uint32]*masterInfo{}, - client: client, + curMasters: map[uint32]*masterInfo{}, + client: client, + proofCheckPolicy: policy, } } -func (c *APIClient) SetSkipProofCheck(skip bool) { - c.skipProofCheck = skip +func (c *APIClient) SetTrustedBlock(block *BlockIDExt) { + c.trustedBlock = block.Copy() +} + +func (c *APIClient) SetTrustedBlockFromConfig(cfg *liteclient.GlobalConfig) { + b := BlockIDExt(cfg.Validator.InitBlock) + c.trustedBlock = &b } func (c *APIClient) WaitForBlock(seqno uint32) APIClientWaiter { diff --git a/ton/block.go b/ton/block.go index c499ed1a..94f8e815 100644 --- a/ton/block.go +++ b/ton/block.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "log" "time" "github.com/xssnick/tonutils-go/tl" @@ -33,6 +34,55 @@ func init() { tl.Register(True{}, "true = True") tl.Register(TransactionID3{}, "liteServer.transactionId3 account:int256 lt:long = liteServer.TransactionId3") tl.Register(TransactionID{}, "liteServer.transactionId mode:# account:mode.0?int256 lt:mode.1?long hash:mode.2?int256 = liteServer.TransactionId") + + tl.Register(GetBlockProof{}, "liteServer.getBlockProof mode:# known_block:tonNode.blockIdExt target_block:mode.0?tonNode.blockIdExt = liteServer.PartialBlockProof") + tl.Register(PartialBlockProof{}, "liteServer.partialBlockProof complete:Bool from:tonNode.blockIdExt to:tonNode.blockIdExt steps:(vector liteServer.BlockLink) = liteServer.PartialBlockProof") + tl.Register(BlockLinkBackward{}, "liteServer.blockLinkBack to_key_block:Bool from:tonNode.blockIdExt to:tonNode.blockIdExt dest_proof:bytes proof:bytes state_proof:bytes = liteServer.BlockLink") + tl.Register(BlockLinkForward{}, "liteServer.blockLinkForward to_key_block:Bool from:tonNode.blockIdExt to:tonNode.blockIdExt dest_proof:bytes config_proof:bytes signatures:liteServer.SignatureSet = liteServer.BlockLink") + tl.Register(SignatureSet{}, "liteServer.signatureSet validator_set_hash:int catchain_seqno:int signatures:(vector liteServer.signature) = liteServer.SignatureSet") + tl.Register(Signature{}, "liteServer.signature node_id_short:int256 signature:bytes = liteServer.Signature") + tl.Register(BlockID{}, "ton.blockId root_cell_hash:int256 file_hash:int256 = ton.BlockId") +} + +type BlockID struct { + RootHash []byte `tl:"int256"` + FileHash []byte `tl:"int256"` +} + +type PartialBlockProof struct { + Complete bool `tl:"bool"` + From *BlockIDExt `tl:"struct"` + To *BlockIDExt `tl:"struct"` + Steps []any `tl:"vector struct boxed [liteServer.blockLinkForward, liteServer.blockLinkBack]"` +} + +type BlockLinkBackward struct { + ToKeyBlock bool `tl:"bool"` + From *BlockIDExt `tl:"struct"` + To *BlockIDExt `tl:"struct"` + DestProof []byte `tl:"bytes"` + Proof []byte `tl:"bytes"` + StateProof []byte `tl:"bytes"` +} + +type BlockLinkForward struct { + ToKeyBlock bool `tl:"bool"` + From *BlockIDExt `tl:"struct"` + To *BlockIDExt `tl:"struct"` + DestProof []byte `tl:"bytes"` + ConfigProof []byte `tl:"bytes"` + SignatureSet *SignatureSet `tl:"struct boxed"` +} + +type SignatureSet struct { + ValidatorSetHash int32 `tl:"int"` + CatchainSeqno int32 `tl:"int"` + Signatures []Signature `tl:"vector struct"` +} + +type Signature struct { + NodeIDShort []byte `tl:"int256"` + Signature []byte `tl:"bytes"` } type Object struct{} @@ -62,7 +112,7 @@ type ZeroStateIDExt struct { type AllShardsInfo struct { ID *BlockIDExt `tl:"struct"` Proof []byte `tl:"bytes"` - Data []byte `tl:"bytes"` + Data *cell.Cell `tl:"cell"` } type BlockTransactions struct { @@ -75,7 +125,7 @@ type BlockTransactions struct { type BlockData struct { ID *BlockIDExt `tl:"struct"` - Payload []byte `tl:"bytes"` + Payload *cell.Cell `tl:"cell"` } type LookupBlock struct { @@ -117,6 +167,12 @@ type TransactionShortInfo struct { Hash []byte } +type GetBlockProof struct { + Mode uint32 `tl:"flags"` + KnownBlock *BlockIDExt `tl:"struct"` + TargetBlock *BlockIDExt `tl:"?0 struct"` +} + func (t *TransactionShortInfo) ID3() *TransactionID3 { return &TransactionID3{ Account: t.Account, @@ -199,7 +255,27 @@ func (c *APIClient) GetMasterchainInfo(ctx context.Context) (*BlockIDExt, error) switch t := resp.(type) { case MasterchainInfo: + if c.proofCheckPolicy == ProofCheckPolicySecure { + c.trustedLock.Lock() + defer c.trustedLock.Unlock() + + if c.trustedBlock == nil { + if c.trustedBlock == nil { + // we have no block to trust, so trust first block we get + c.trustedBlock = t.Last.Copy() + log.Println("[WARNING] trusted block was not set on initialization, so first block we got was considered as trusted. " + + "For better security you should use SetTrustedBlock(block) method and pass there init block from config on start") + } + } else { + if err := c.VerifyProofChain(ctx, c.trustedBlock, t.Last); err != nil { + return nil, fmt.Errorf("failed to verify proof chain: %w", err) + } + if t.Last.SeqNo > c.trustedBlock.SeqNo { + c.trustedBlock = t.Last.Copy() + } + } + } return t.Last, nil case LSError: return nil, t @@ -245,17 +321,12 @@ func (c *APIClient) GetBlockData(ctx context.Context, block *BlockIDExt) (*tlb.B switch t := resp.(type) { case BlockData: - cl, err := cell.FromBOC(t.Payload) - if err != nil { - return nil, fmt.Errorf("failed to parse block boc: %w", err) - } - - if !bytes.Equal(cl.Hash(), block.RootHash) { + if !bytes.Equal(t.Payload.Hash(), block.RootHash) { return nil, fmt.Errorf("incorrect block") } var bData tlb.Block - if err = tlb.LoadFromCell(&bData, cl.BeginParse()); err != nil { + if err = tlb.LoadFromCell(&bData, t.Payload.BeginParse()); err != nil { return nil, fmt.Errorf("failed to parse block data: %w", err) } return &bData, nil @@ -265,32 +336,6 @@ func (c *APIClient) GetBlockData(ctx context.Context, block *BlockIDExt) (*tlb.B return nil, errUnexpectedResponse(resp) } -// GetBlockTransactions - list of block transactions -// Deprecated: Will be removed in the next release, use GetBlockTransactionsV2 -func (c *APIClient) GetBlockTransactions(ctx context.Context, block *BlockIDExt, count uint32, after ...*tlb.TransactionID) ([]*tlb.TransactionID, bool, error) { - var id3 *TransactionID3 - if len(after) > 0 && after[0] != nil { - id3 = &TransactionID3{ - Account: after[0].AccountID, - LT: after[0].LT, - } - } - - list, more, err := c.GetBlockTransactionsV2(ctx, block, count, id3) - if err != nil { - return nil, false, err - } - oldList := make([]*tlb.TransactionID, 0, len(list)) - for _, item := range list { - oldList = append(oldList, &tlb.TransactionID{ - LT: item.LT, - Hash: item.Hash, - AccountID: item.Account, - }) - } - return oldList, more, nil -} - // GetBlockTransactionsV2 - list of block transactions func (c *APIClient) GetBlockTransactionsV2(ctx context.Context, block *BlockIDExt, count uint32, after ...*TransactionID3) ([]TransactionShortInfo, bool, error) { withAfter := uint32(0) @@ -301,7 +346,7 @@ func (c *APIClient) GetBlockTransactionsV2(ctx context.Context, block *BlockIDEx } mode := 0b111 | (withAfter << 7) - if !c.skipProofCheck { + if c.proofCheckPolicy != ProofCheckPolicyUnsafe { mode |= 1 << 5 } @@ -321,7 +366,7 @@ func (c *APIClient) GetBlockTransactionsV2(ctx context.Context, block *BlockIDEx case BlockTransactions: var shardAccounts tlb.ShardAccountBlocks - if !c.skipProofCheck { + if c.proofCheckPolicy != ProofCheckPolicyUnsafe { proof, err := cell.FromBOC(t.Proof) if err != nil { return nil, false, fmt.Errorf("failed to parse proof boc: %w", err) @@ -344,7 +389,7 @@ func (c *APIClient) GetBlockTransactionsV2(ctx context.Context, block *BlockIDEx return nil, false, fmt.Errorf("invalid ls response, fields are nil") } - if !c.skipProofCheck { + if c.proofCheckPolicy != ProofCheckPolicyUnsafe { if err = CheckTransactionProof(id.Hash, id.LT, id.Account, &shardAccounts); err != nil { return nil, false, fmt.Errorf("incorrect tx %s proof: %w", hex.EncodeToString(id.Hash), err) } @@ -373,18 +418,13 @@ func (c *APIClient) GetBlockShardsInfo(ctx context.Context, master *BlockIDExt) switch t := resp.(type) { case AllShardsInfo: - shardsInfo, err := cell.FromBOC(t.Data) - if err != nil { - return nil, err - } - var inf tlb.AllShardsInfo - err = tlb.LoadFromCell(&inf, shardsInfo.BeginParse()) + err = tlb.LoadFromCell(&inf, t.Data.BeginParse()) if err != nil { return nil, err } - if !c.skipProofCheck { + if c.proofCheckPolicy != ProofCheckPolicyUnsafe { proof, err := cell.FromBOCMultiRoot(t.Proof) if err != nil { return nil, fmt.Errorf("failed to parse proof boc: %w", err) @@ -410,7 +450,7 @@ func (c *APIClient) GetBlockShardsInfo(ctx context.Context, master *BlockIDExt) } if (dictProof == nil) != (inf.ShardHashes.Size() == 0) || - !bytes.Equal(dictProof.MustToCell().Hash(0), shardsInfo.MustPeekRef(0).Hash()) { + !bytes.Equal(dictProof.MustToCell().Hash(0), t.Data.MustPeekRef(0).Hash()) { return nil, fmt.Errorf("incorrect proof") } } @@ -548,3 +588,24 @@ func (c *APIClient) WaitNextMasterBlock(ctx context.Context, master *BlockIDExt) return m, nil } + +// GetBlockProof - gets proof chain for the block +func (c *APIClient) GetBlockProof(ctx context.Context, known, target *BlockIDExt) (*PartialBlockProof, error) { + var resp tl.Serializable + err := c.client.QueryLiteserver(ctx, GetBlockProof{ + Mode: 1, + KnownBlock: known, + TargetBlock: target, + }, &resp) + if err != nil { + return nil, err + } + + switch t := resp.(type) { + case PartialBlockProof: + return &t, nil + case LSError: + return nil, t + } + return nil, errUnexpectedResponse(resp) +} diff --git a/ton/dns/integration_test.go b/ton/dns/integration_test.go index df218f19..ab5b0e50 100644 --- a/ton/dns/integration_test.go +++ b/ton/dns/integration_test.go @@ -14,7 +14,7 @@ var api = func() *ton.APIClient { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - err := client.AddConnectionsFromConfigUrl(ctx, "https://ton-blockchain.github.io/global.config.json") + err := client.AddConnectionsFromConfigUrl(ctx, "https://ton.org/global.config.json") if err != nil { panic(err) } diff --git a/ton/getconfig.go b/ton/getconfig.go index 70aacec5..b5a857a9 100644 --- a/ton/getconfig.go +++ b/ton/getconfig.go @@ -2,12 +2,10 @@ package ton import ( "context" - "errors" "fmt" "math/big" "github.com/xssnick/tonutils-go/tl" - "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/tvm/cell" ) @@ -20,8 +18,8 @@ func init() { type ConfigAll struct { Mode int `tl:"int"` ID *BlockIDExt `tl:"struct"` - StateProof []byte `tl:"bytes"` - ConfigProof []byte `tl:"bytes"` + StateProof *cell.Cell `tl:"cell"` + ConfigProof *cell.Cell `tl:"cell"` } type GetConfigAll struct { @@ -63,37 +61,17 @@ func (c *APIClient) GetBlockchainConfig(ctx context.Context, block *BlockIDExt, switch t := resp.(type) { case ConfigAll: - cfgProof, err := cell.FromBOC(t.ConfigProof) - if err != nil { - return nil, fmt.Errorf("failed to parse config proof boc: %w", err) - } - - stateProof, err := cell.FromBOC(t.StateProof) - if err != nil { - return nil, fmt.Errorf("failed to parse state proof boc: %w", err) - } - - state, err := CheckBlockShardStateProof([]*cell.Cell{cfgProof, stateProof}, block.RootHash) + stateExtra, err := CheckShardMcStateExtraProof(block, []*cell.Cell{t.ConfigProof, t.StateProof}) if err != nil { return nil, fmt.Errorf("incorrect proof: %w", err) } - if state.McStateExtra == nil { - return nil, errors.New("no mc extra state found, something went wrong") - } - - var stateExtra tlb.McStateExtra - err = tlb.LoadFromCell(&stateExtra, state.McStateExtra.BeginParse()) - if err != nil { - return nil, fmt.Errorf("load masterchain state extra: %w", err) - } - result := &BlockchainConfig{data: map[int32]*cell.Cell{}} if len(onlyParams) > 0 { // we need it because lite server may add some unwanted keys for _, param := range onlyParams { - res := stateExtra.ConfigParams.Config.GetByIntKey(big.NewInt(int64(param))) + res := stateExtra.ConfigParams.Config.Params.GetByIntKey(big.NewInt(int64(param))) if res == nil { return nil, fmt.Errorf("config param %d not found", param) } @@ -106,7 +84,7 @@ func (c *APIClient) GetBlockchainConfig(ctx context.Context, block *BlockIDExt, result.data[param] = v.MustToCell() } } else { - for _, kv := range stateExtra.ConfigParams.Config.All() { + for _, kv := range stateExtra.ConfigParams.Config.Params.All() { v, err := kv.Value.BeginParse().LoadRef() if err != nil { return nil, fmt.Errorf("failed to load config param %d, err: %w", kv.Key.BeginParse().MustLoadInt(32), err) diff --git a/ton/getstate.go b/ton/getstate.go index be0518b8..c2a09cfa 100644 --- a/ton/getstate.go +++ b/ton/getstate.go @@ -16,11 +16,11 @@ func init() { } type AccountState struct { - ID *BlockIDExt `tl:"struct"` - Shard *BlockIDExt `tl:"struct"` - ShardProof []byte `tl:"bytes"` - Proof []byte `tl:"bytes"` - State []byte `tl:"bytes"` + ID *BlockIDExt `tl:"struct"` + Shard *BlockIDExt `tl:"struct"` + ShardProof []*cell.Cell `tl:"cell optional 2"` + Proof []*cell.Cell `tl:"cell optional 2"` + State *cell.Cell `tl:"cell optional"` } type GetAccountState struct { @@ -52,33 +52,26 @@ func (c *APIClient) GetAccount(ctx context.Context, block *BlockIDExt, addr *add return nil, fmt.Errorf("response with incorrect master block") } - if len(t.State) == 0 { + if t.State == nil { return &tlb.Account{ IsActive: false, }, nil } - acc := &tlb.Account{ - IsActive: true, + if t.Proof == nil { + return nil, fmt.Errorf("no proof") } - proof, err := cell.FromBOCMultiRoot(t.Proof) - if err != nil { - return nil, fmt.Errorf("failed to parse proof boc: %w", err) + acc := &tlb.Account{ + IsActive: true, } - var shardProof []*cell.Cell var shardHash []byte - if !c.skipProofCheck && addr.Workchain() != address.MasterchainID { + if c.proofCheckPolicy != ProofCheckPolicyUnsafe && addr.Workchain() != address.MasterchainID { if len(t.ShardProof) == 0 { return nil, ErrNoProof } - shardProof, err = cell.FromBOCMultiRoot(t.ShardProof) - if err != nil { - return nil, fmt.Errorf("failed to parse shard proof boc: %w", err) - } - if t.Shard == nil || len(t.Shard.RootHash) != 32 { return nil, fmt.Errorf("shard block not passed") } @@ -86,22 +79,17 @@ func (c *APIClient) GetAccount(ctx context.Context, block *BlockIDExt, addr *add shardHash = t.Shard.RootHash } - shardAcc, balanceInfo, err := CheckAccountStateProof(addr, block, proof, shardProof, shardHash, c.skipProofCheck) + shardAcc, balanceInfo, err := CheckAccountStateProof(addr, block, t.Proof, t.ShardProof, shardHash, c.proofCheckPolicy == ProofCheckPolicyUnsafe) if err != nil { return nil, fmt.Errorf("failed to check acc state proof: %w", err) } - stateCell, err := cell.FromBOC(t.State) - if err != nil { - return nil, fmt.Errorf("failed to parse state boc: %w", err) - } - - if !bytes.Equal(shardAcc.Account.Hash(0), stateCell.Hash()) { + if !bytes.Equal(shardAcc.Account.Hash(0), t.State.Hash()) { return nil, fmt.Errorf("proof hash not match state account hash") } var st tlb.AccountState - if err = st.LoadFromCell(stateCell.BeginParse()); err != nil { + if err = st.LoadFromCell(t.State.BeginParse()); err != nil { return nil, fmt.Errorf("failed to load account state: %w", err) } diff --git a/ton/integration_test.go b/ton/integration_test.go index becddd82..da8f52a0 100644 --- a/ton/integration_test.go +++ b/ton/integration_test.go @@ -35,12 +35,19 @@ var api = func() *APIClient { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - err := client.AddConnectionsFromConfigUrl(ctx, "https://ton-blockchain.github.io/global.config.json") + cfg, err := liteclient.GetConfigFromUrl(ctx, "https://ton.org/global.config.json") if err != nil { panic(err) } - return NewAPIClient(client) + err = client.AddConnectionsFromConfig(ctx, cfg) + if err != nil { + panic(err) + } + + a := NewAPIClient(client, ProofCheckPolicySecure) + // a.SetTrustedBlockFromConfig(cfg) + return a }() var testContractAddr = func() *address.Address { @@ -195,7 +202,7 @@ func Test_RunMethod(t *testing.T) { } func Test_ExternalMessage(t *testing.T) { // need to deploy contract on test-net - > than change config to test-net. - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() ctx = apiTestNet.client.StickyContext(ctx) @@ -260,7 +267,7 @@ func Test_Account(t *testing.T) { fmt.Printf("Is active: %v\n", res.IsActive) if res.IsActive { fmt.Printf("Status: %s\n", res.State.Status) - fmt.Printf("Balance: %s TON\n", res.State.Balance.TON()) + fmt.Printf("Balance: %s TON\n", res.State.Balance.String()) if res.Data != nil { fmt.Printf("Data: %s\n", res.Data.Dump()) } @@ -322,7 +329,7 @@ func Test_AccountMaster(t *testing.T) { fmt.Printf("Is active: %v\n", res.IsActive) if res.IsActive { fmt.Printf("Status: %s\n", res.State.Status) - fmt.Printf("Balance: %s TON\n", res.State.Balance.TON()) + fmt.Printf("Balance: %s TON\n", res.State.Balance.String()) if res.Data == nil { t.Fatal("data null") } @@ -622,7 +629,8 @@ func TestAccountStorage_LoadFromCell_ExtraCurrencies(t *testing.T) { }) t.Run("without proof", func(t *testing.T) { - mainnetAPI.SetSkipProofCheck(true) + mainnetAPI := NewAPIClient(client, ProofCheckPolicyUnsafe) + a, err := mainnetAPI.GetAccount(ctx, b, address.MustParseAddr("EQCYv992KVNNCKZHSLLJgM2GGzsgL0UgWP24BCQBaAdqSE2I")) if err != nil { t.Fatal(err) @@ -633,3 +641,80 @@ func TestAccountStorage_LoadFromCell_ExtraCurrencies(t *testing.T) { } }) } + +func TestAPIClient_GetBlockProofForward(t *testing.T) { + cfg, err := liteclient.GetConfigFromUrl(context.Background(), "https://ton.org/global.config.json") + if err != nil { + t.Fatal("get cfg err:", err.Error()) + return + } + + ctx := api.client.StickyContext(context.Background()) + + initBlock := BlockIDExt(cfg.Validator.InitBlock) + known := &initBlock + + stm := time.Now() + + for _, dir := range []string{"backward", "forward"} { + b, err := api.CurrentMasterchainInfo(ctx) + if err != nil { + t.Fatal("get block err:", err.Error()) + return + } + + if dir == "backward" { + known, b = b, known + } + + t.Run("Block proof "+dir, func(t *testing.T) { + if err = api.VerifyProofChain(ctx, known, b); err != nil { + t.Fatal("failed to verify chain:", err.Error()) + return + } + log.Println("DONE!", time.Since(stm)) + }) + } +} + +func TestAPIClient_SubscribeOnTransactions(t *testing.T) { + _ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + ctx := api.client.StickyContext(_ctx) + + addr := address.MustParseAddr("EQCD39VS5jcptHL8vMjEXrzGaRcCVYto7HUn4bpAOg8xqB2N") + + b, err := api.CurrentMasterchainInfo(ctx) + if err != nil { + t.Fatal("get block err:", err.Error()) + return + } + + acc, err := api.WaitForBlock(b.SeqNo).GetAccount(ctx, b, addr) + if err != nil { + t.Fatal("get acc err:", err.Error()) + return + } + initLT := acc.LastTxLT - 600000000000 + log.Println(initLT) + lastLT := initLT + + ctx, cancel = context.WithTimeout(context.Background(), 7*time.Second) + defer cancel() + + ch := make(chan *tlb.Transaction) + go api.SubscribeOnTransactions(ctx, addr, lastLT, ch) + + for tx := range ch { + if lastLT > tx.LT { + t.Fatal("incorrect tx order") + } + lastLT = tx.LT + + println(tx.Now, tx.String()) + } + + if lastLT == initLT { + t.Fatal("no transactions") + } +} diff --git a/ton/jetton/integration_test.go b/ton/jetton/integration_test.go index a1750843..75b092b1 100644 --- a/ton/jetton/integration_test.go +++ b/ton/jetton/integration_test.go @@ -136,12 +136,12 @@ func TestJettonMasterClient_Transfer(t *testing.T) { t.Fatal(err) } - if b.NanoTON().Uint64() == b2.NanoTON().Uint64() { + if b.Uint64() == b2.Uint64() { t.Fatal("balance was not changed after burn") } - want := b.NanoTON().Uint64() - amt.NanoTON().Uint64()*2 - got := b2.NanoTON().Uint64() + want := b.Uint64() - amt.NanoTON().Uint64()*2 + got := b2.Uint64() if want != got { t.Fatal("balance not expected, want ", want, "got", got) } diff --git a/ton/jetton/jetton.go b/ton/jetton/jetton.go index dce56613..09ea47eb 100644 --- a/ton/jetton/jetton.go +++ b/ton/jetton/jetton.go @@ -16,6 +16,7 @@ type TonApi interface { WaitForBlock(seqno uint32) ton.APIClientWaiter CurrentMasterchainInfo(ctx context.Context) (_ *ton.BlockIDExt, err error) RunGetMethod(ctx context.Context, blockInfo *ton.BlockIDExt, addr *address.Address, method string, params ...any) (*ton.ExecutionResult, error) + SubscribeOnTransactions(workerCtx context.Context, addr *address.Address, lastProcessedLT uint64, channel chan<- *tlb.Transaction) } type MintPayloadMasterMsg struct { diff --git a/ton/jetton/wallet.go b/ton/jetton/wallet.go index a1ca6a86..2ce83414 100644 --- a/ton/jetton/wallet.go +++ b/ton/jetton/wallet.go @@ -3,6 +3,7 @@ package jetton import ( "context" "fmt" + "math/big" "math/rand" "github.com/xssnick/tonutils-go/address" @@ -39,29 +40,29 @@ func (c *WalletClient) Address() *address.Address { return c.addr } -func (c *WalletClient) GetBalance(ctx context.Context) (tlb.Coins, error) { +func (c *WalletClient) GetBalance(ctx context.Context) (*big.Int, error) { b, err := c.master.api.CurrentMasterchainInfo(ctx) if err != nil { - return tlb.Coins{}, fmt.Errorf("failed to get masterchain info: %w", err) + return nil, fmt.Errorf("failed to get masterchain info: %w", err) } return c.GetBalanceAtBlock(ctx, b) } -func (c *WalletClient) GetBalanceAtBlock(ctx context.Context, b *ton.BlockIDExt) (tlb.Coins, error) { +func (c *WalletClient) GetBalanceAtBlock(ctx context.Context, b *ton.BlockIDExt) (*big.Int, error) { res, err := c.master.api.WaitForBlock(b.SeqNo).RunGetMethod(ctx, b, c.addr, "get_wallet_data") if err != nil { if cErr, ok := err.(ton.ContractExecError); ok && cErr.Code == ton.ErrCodeContractNotInitialized { - return tlb.Coins{}, nil + return big.NewInt(0), nil } - return tlb.Coins{}, fmt.Errorf("failed to run get_wallet_data method: %w", err) + return nil, fmt.Errorf("failed to run get_wallet_data method: %w", err) } balance, err := res.Int(0) if err != nil { - return tlb.Coins{}, fmt.Errorf("failed to parse balance: %w", err) + return nil, fmt.Errorf("failed to parse balance: %w", err) } - return tlb.FromNanoTON(balance), nil + return balance, nil } func (c *WalletClient) BuildTransferPayload(to *address.Address, amountCoins, amountForwardTON tlb.Coins, payloadForward *cell.Cell) (*cell.Cell, error) { diff --git a/ton/nft/integration_test.go b/ton/nft/integration_test.go index 7a35bbb9..af777b00 100644 --- a/ton/nft/integration_test.go +++ b/ton/nft/integration_test.go @@ -60,7 +60,7 @@ func Test_NftMintTransfer(t *testing.T) { } if balance.NanoTON().Uint64() < 3000000 { - t.Fatal("not enough balance", w.Address(), balance.TON()) + t.Fatal("not enough balance", w.Address(), balance.String()) } collectionAddr := address.MustParseAddr("EQBTObWUuWTb5ECnLI4x6a3szzstmMDOcc5Kdo-CpbUY9Y5K") // address = deployCollection(w) w.seed = (fiction ... rather) diff --git a/ton/prng.go b/ton/prng.go new file mode 100644 index 00000000..7c26331a --- /dev/null +++ b/ton/prng.go @@ -0,0 +1,83 @@ +package ton + +import ( + "crypto/sha512" + "encoding/binary" + "math/big" +) + +type validatorSetDescription struct { + seed [32]byte + shard uint64 + workchain int32 + ccSeqno uint32 + + hash []byte +} + +type ValidatorSetPRNG struct { + desc validatorSetDescription + pos int + limit int +} + +func NewValidatorSetPRNG(shard int64, workchain int32, catchainSeqno uint32, seed []byte) *ValidatorSetPRNG { + if len(seed) != 0 && len(seed) != 32 { + panic("invalid seed len") + } + + desc := validatorSetDescription{ + shard: uint64(shard), + workchain: workchain, + ccSeqno: catchainSeqno, + } + + if len(seed) != 0 { + copy(desc.seed[:], seed) + } + + return &ValidatorSetPRNG{desc: desc} +} + +func (r *ValidatorSetPRNG) NextUint64() uint64 { + if r.pos < r.limit { + pos := r.pos + r.pos += 1 + return binary.BigEndian.Uint64(r.desc.hash[pos*8:]) + } + r.desc.calcHash() + r.desc.incSeed() + r.pos = 1 + r.limit = 8 + return binary.BigEndian.Uint64(r.desc.hash[0:]) +} + +func (r *ValidatorSetPRNG) NextRanged(rg uint64) uint64 { + ff := r.NextUint64() + y := new(big.Int).SetUint64(ff) + x := new(big.Int).SetUint64(rg) + z := x.Mul(x, y) + return z.Rsh(z, 64).Uint64() +} + +func (v *validatorSetDescription) calcHash() { + h := sha512.New() + h.Write(v.seed[:]) + + buf := make([]byte, 16) + binary.BigEndian.PutUint64(buf, v.shard) + binary.BigEndian.PutUint32(buf[8:], uint32(v.workchain)) + binary.BigEndian.PutUint32(buf[12:], v.ccSeqno) + + h.Write(buf) + v.hash = h.Sum(nil) +} + +func (v *validatorSetDescription) incSeed() { + for i := 31; i >= 0; i-- { + v.seed[i]++ + if v.seed[i] != 0 { + break + } + } +} diff --git a/ton/proof.go b/ton/proof.go index 106d4fbb..21fab4c1 100644 --- a/ton/proof.go +++ b/ton/proof.go @@ -2,29 +2,50 @@ package ton import ( "bytes" + "context" + "crypto/ed25519" + "encoding/hex" "errors" "fmt" "github.com/xssnick/tonutils-go/address" + "github.com/xssnick/tonutils-go/adnl" + "github.com/xssnick/tonutils-go/tl" "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/tvm/cell" + "hash/crc32" + "math/big" + "reflect" + "sort" ) +func init() { + tl.Register(ValidatorSetHashable{}, "test0.validatorSet#901660ed") +} + var ErrNoProof = fmt.Errorf("liteserver has no proof for this account in a given block, request newer block or disable proof checks") -func CheckShardInMasterProof(master *BlockIDExt, shardProof []*cell.Cell, workchain int32, shardRootHash []byte) error { +func CheckShardMcStateExtraProof(master *BlockIDExt, shardProof []*cell.Cell) (*tlb.McStateExtra, error) { shardState, err := CheckBlockShardStateProof(shardProof, master.RootHash) if err != nil { - return fmt.Errorf("check block proof failed: %w", err) + return nil, fmt.Errorf("check block proof failed: %w", err) } if shardState.McStateExtra == nil { - return fmt.Errorf("not a masterchain block") + return nil, fmt.Errorf("not a masterchain block") } var stateExtra tlb.McStateExtra err = tlb.LoadFromCell(&stateExtra, shardState.McStateExtra.BeginParse()) if err != nil { - return fmt.Errorf("failed to load masterchain state extra: %w", err) + return nil, fmt.Errorf("failed to load masterchain state extra: %w", err) + } + return &stateExtra, nil +} + +func CheckShardInMasterProof(master *BlockIDExt, shardProof []*cell.Cell, workchain int32, shardRootHash []byte) error { + stateExtra, err := CheckShardMcStateExtraProof(master, shardProof) + if err != nil { + return fmt.Errorf("failed to check proof for mc state extra: %w", err) } shards, err := LoadShardsFromHashes(stateExtra.ShardHashes) @@ -184,3 +205,418 @@ func CheckTransactionProof(txHash []byte, txLT uint64, txAccount []byte, shardAc return nil } + +func CheckBackwardBlockProof(from, to *BlockIDExt, toKey bool, stateProof, destProof, proof *cell.Cell) error { + if from.Workchain != address.MasterchainID || to.Workchain != address.MasterchainID { + return fmt.Errorf("both blocks should be from masterchain") + } + + if from.SeqNo <= to.SeqNo { + return fmt.Errorf("to seqno should be < from seqno") + } + + toBlock, err := CheckBlockProof(destProof, to.RootHash) + if err != nil { + return fmt.Errorf("failed to check traget block proof: %w", err) + } + + if toBlock.BlockInfo.KeyBlock != toKey { + return fmt.Errorf("target block type not matches requested") + } + + stateExtra, err := CheckShardMcStateExtraProof(from, []*cell.Cell{stateProof, proof}) + if err != nil { + return fmt.Errorf("failed to check proof for mc state extra: %w", err) + } + + var info tlb.McStateExtraBlockInfo + err = tlb.LoadFromCellAsProof(&info, stateExtra.Info.BeginParse()) + if err != nil { + return fmt.Errorf("failed to load tx CurrencyCollection proof cell: %w", err) + } + + toInfo := info.PrevBlocks.GetByIntKey(big.NewInt(int64(to.SeqNo))) + if toInfo == nil { + return fmt.Errorf("target block not found in state proof") + } + + slc := toInfo.BeginParse() + err = tlb.LoadFromCellAsProof(new(tlb.KeyMaxLt), slc) + if err != nil { + return fmt.Errorf("failed to load block KeyMaxLt proof cell: %w", err) + } + + var blk tlb.KeyExtBlkRef + err = tlb.LoadFromCellAsProof(&blk, slc) + if err != nil { + return fmt.Errorf("failed to load block KeyExtBlkRef proof cell: %w", err) + } + + if blk.IsKey != toKey { + return fmt.Errorf("target block type in proof not matches requested") + } + + if !bytes.Equal(blk.BlkRef.RootHash, to.RootHash) { + return fmt.Errorf("incorret target block hash in proof") + } + return nil +} + +func CheckForwardBlockProof(from, to *BlockIDExt, toKey bool, configProof, destProof *cell.Cell, signatures *SignatureSet) error { + if from.Workchain != address.MasterchainID || to.Workchain != address.MasterchainID { + return fmt.Errorf("both blocks should be from masterchain") + } + + if from.SeqNo >= to.SeqNo { + return fmt.Errorf("to seqno should be > from seqno") + } + + toBlock, err := CheckBlockProof(destProof, to.RootHash) + if err != nil { + return fmt.Errorf("failed to check traget block proof: %w", err) + } + + if toBlock.BlockInfo.KeyBlock != toKey { + return fmt.Errorf("target block type not matches requested") + } + + if toBlock.BlockInfo.GenValidatorListHashShort != uint32(signatures.ValidatorSetHash) { + return fmt.Errorf("incorrect validator set hash") + } + + if toBlock.BlockInfo.GenCatchainSeqno != uint32(signatures.CatchainSeqno) { + return fmt.Errorf("incorrect catchain seqno") + } + + if toBlock.BlockInfo.SeqNo <= from.SeqNo { + return fmt.Errorf("invalid target block seqno") + } + + fromBlock, err := CheckBlockProof(configProof, from.RootHash) + if err != nil { + return fmt.Errorf("failed to check source block proof: %w", err) + } + + if fromBlock.Extra == nil || fromBlock.Extra.Custom == nil { + return fmt.Errorf("source block proof is lack of info") + } + + catchainCfgCell := fromBlock.Extra.Custom.ConfigParams.Config.Params.GetByIntKey(big.NewInt(28)) + blockValidatorsCell := fromBlock.Extra.Custom.ConfigParams.Config.Params.GetByIntKey(big.NewInt(34)) + if catchainCfgCell == nil || blockValidatorsCell == nil { + return fmt.Errorf("not all required configs are in proof") + } + if catchainCfgCell, err = catchainCfgCell.PeekRef(0); err != nil { + return fmt.Errorf("no ref in catchain cell") + } + if blockValidatorsCell, err = blockValidatorsCell.PeekRef(0); err != nil { + return fmt.Errorf("no ref in validators cell") + } + + var catchainCfg tlb.CatchainConfig + if err = tlb.LoadFromCell(&catchainCfg, catchainCfgCell.BeginParse()); err != nil { + return fmt.Errorf("failed to parse catchain config: %w", err) + } + + var blockValidators tlb.ValidatorSetAny + if err = tlb.LoadFromCell(&blockValidators, blockValidatorsCell.BeginParse()); err != nil { + return fmt.Errorf("failed to parse validators config: %w", err) + } + + validators, err := getMainValidators(to, catchainCfg, blockValidators, toBlock.BlockInfo.GenCatchainSeqno) + if err != nil { + return fmt.Errorf("failed to verify and get main block validators: %w", err) + } + + if err = checkBlockSignatures(to, signatures, validators); err != nil { + return fmt.Errorf("failed to check validators signatures: %w", err) + } + + return nil +} + +func getMainValidators(block *BlockIDExt, catConfig tlb.CatchainConfig, validatorConfig tlb.ValidatorSetAny, ccSeqno uint32) ([]*tlb.ValidatorAddr, error) { + if block.Workchain != address.MasterchainID { + return nil, fmt.Errorf("only masterchain blocks currently supported") + } + + var shuffle = false + var validatorsNum int + var validatorsListDict *cell.Dictionary + + switch t := catConfig.Config.(type) { + case tlb.CatchainConfigV1: + case tlb.CatchainConfigV2: + shuffle = t.ShuffleMcValidators + default: + return nil, fmt.Errorf("unknown validator set type") + } + + var definedWeight *uint64 + switch t := validatorConfig.Validators.(type) { + case tlb.ValidatorSet: + validatorsNum = int(t.Main) + validatorsListDict = t.List + case tlb.ValidatorSetExt: + definedWeight = &t.TotalWeight + validatorsNum = int(t.Main) + validatorsListDict = t.List + default: + return nil, fmt.Errorf("unknown validator set type") + } + + type validatorWithKey struct { + addr *tlb.ValidatorAddr + key uint16 + } + + var totalWeight uint64 + var validatorsKeys = make([]validatorWithKey, validatorsListDict.Size()) + for i, kv := range validatorsListDict.All() { + var val tlb.ValidatorAddr + if err := tlb.LoadFromCell(&val, kv.Value.BeginParse()); err != nil { + return nil, fmt.Errorf("failed to parse validator addr: %w", err) + } + + key, err := kv.Key.BeginParse().LoadUInt(16) + if err != nil { + return nil, fmt.Errorf("failed to parse validator key: %w", err) + } + + totalWeight += val.Weight + validatorsKeys[i].addr = &val + validatorsKeys[i].key = uint16(key) + } + + if definedWeight != nil && totalWeight != *definedWeight { + return nil, fmt.Errorf("incorrect sum of weights") + } + + if len(validatorsKeys) == 0 { + return nil, fmt.Errorf("zero validators") + } + + sort.Slice(validatorsKeys, func(i, j int) bool { + return validatorsKeys[i].key < validatorsKeys[j].key + }) + + if validatorsNum > len(validatorsKeys) { + validatorsNum = len(validatorsKeys) + } + + var validators = make([]*tlb.ValidatorAddr, validatorsNum) + if shuffle { + prng := NewValidatorSetPRNG(block.Shard, block.Workchain, ccSeqno, nil) + + idx := make([]uint32, validatorsNum) + for i := 0; i < validatorsNum; i++ { + j := prng.NextRanged(uint64(i) + 1) + idx[i] = idx[j] + idx[j] = uint32(i) + } + + for i := 0; i < validatorsNum; i++ { + validators[i] = validatorsKeys[idx[i]].addr + } + + return validators, nil + } + + for i := 0; i < validatorsNum; i++ { + validators[i] = validatorsKeys[i].addr + } + + return validators, nil +} + +func checkBlockSignatures(block *BlockIDExt, sigs *SignatureSet, validators []*tlb.ValidatorAddr) error { + if len(sigs.Signatures) == 0 || len(validators) == 0 { + return fmt.Errorf("zero signatures or validators") + } + + setHash, err := calcValidatorSetHash(uint32(sigs.CatchainSeqno), validators) + if err != nil { + return fmt.Errorf("failed to calc validator set hash: %w", err) + } + + if setHash != uint32(sigs.ValidatorSetHash) { + return fmt.Errorf("incorrect validator set hash") + } + + var totalWeight, signedWeight uint64 + validatorsMap := map[string]*tlb.ValidatorAddr{} + for _, v := range validators { + kid, err := tl.Hash(adnl.PublicKeyED25519{Key: v.PublicKey.Key}) + if err != nil { + return fmt.Errorf("failed to calc validator key id: %w", err) + } + + totalWeight += v.Weight + validatorsMap[string(kid)] = v + } + + blockIDBytes, err := tl.Serialize(BlockID{RootHash: block.RootHash, FileHash: block.FileHash}, true) + if err != nil { + return fmt.Errorf("failed to serialize block id: %w", err) + } + + sort.Slice(sigs.Signatures, func(i, j int) bool { + return string(sigs.Signatures[i].NodeIDShort) < string(sigs.Signatures[j].NodeIDShort) + }) + + for i, sig := range sigs.Signatures { + if i > 0 && string(sigs.Signatures[i-1].NodeIDShort) == string(sig.NodeIDShort) { + return fmt.Errorf("duplicated node signature") + } + + v, ok := validatorsMap[string(sig.NodeIDShort)] + if !ok { + return fmt.Errorf("signature of unknown validator %s", hex.EncodeToString(sig.NodeIDShort)) + } + + if !ed25519.Verify(v.PublicKey.Key, blockIDBytes, sig.Signature) { + return fmt.Errorf("incorrect signature of validator %s", hex.EncodeToString(sig.NodeIDShort)) + } + signedWeight += v.Weight + + if signedWeight > totalWeight { + break + } + } + + if 3*signedWeight <= 2*totalWeight { + return fmt.Errorf("insufficient signed weight (%d/%d)", 3*signedWeight, 2*totalWeight) + } + + return nil +} + +type ValidatorItemHashable struct { + Key []byte `tl:"int256"` + Weight uint64 `tl:"long"` + Addr []byte `tl:"int256"` +} + +type ValidatorSetHashable struct { + CCSeqno uint32 `tl:"int"` + Validators []ValidatorItemHashable `tl:"vector struct"` +} + +var castTable = crc32.MakeTable(crc32.Castagnoli) + +func calcValidatorSetHash(ccSeqno uint32, validators []*tlb.ValidatorAddr) (uint32, error) { + var vls = make([]ValidatorItemHashable, len(validators)) + for i, validator := range validators { + vls[i].Key = validator.PublicKey.Key + vls[i].Weight = validator.Weight + vls[i].Addr = validator.ADNLAddr + } + + h := ValidatorSetHashable{ + CCSeqno: ccSeqno, + Validators: vls, + } + + b, err := tl.Serialize(h, true) + if err != nil { + return 0, fmt.Errorf("failed to serialize: %w", err) + } + return crc32.Checksum(b, castTable), nil +} + +func (c *APIClient) VerifyProofChain(ctx context.Context, from, to *BlockIDExt) error { + isForward := to.SeqNo > from.SeqNo + + for from.SeqNo != to.SeqNo { + part, err := c.GetBlockProof(ctx, from, to) + if err != nil { + if lsErr, ok := err.(LSError); ok && (lsErr.Code == 651 || lsErr.Code == -400) { // block not applied error + // try next node + if ctx, err = c.client.StickyContextNextNode(ctx); err != nil { + return fmt.Errorf("failed to pick next node: %w", err) + } + continue + } + return fmt.Errorf("failed to get master block proof from %d to %d: %w", from.SeqNo, to.SeqNo, err) + } + + if !part.From.Equals(from) { + return fmt.Errorf("unexpected from block: %d, want %d", part.From.SeqNo, from.SeqNo) + } + + checkBackProof := func(bwd *BlockLinkBackward) error { + destProof, err := cell.FromBOC(bwd.DestProof) + if err != nil { + return fmt.Errorf("dest proof boc parse err: %w", err) + } + + stateProof, err := cell.FromBOC(bwd.StateProof) + if err != nil { + return fmt.Errorf("state proof boc parse err: %w", err) + } + + proof, err := cell.FromBOC(bwd.Proof) + if err != nil { + return fmt.Errorf("proof boc parse err: %w", err) + } + + err = CheckBackwardBlockProof(bwd.From, bwd.To, bwd.ToKeyBlock, stateProof, destProof, proof) + if err != nil { + return fmt.Errorf("invalid backward block from %d to %d proof: %w", bwd.From.SeqNo, bwd.To.SeqNo, err) + } + return nil + } + + if isForward { + for _, step := range part.Steps { + fwd, ok := step.(BlockLinkForward) + if !ok { + // proof back to key block + bwd, ok := step.(BlockLinkBackward) + if !ok { + return fmt.Errorf("wrong proof step type %v", reflect.TypeOf(step).String()) + } + + if err = checkBackProof(&bwd); err != nil { + return err + } + + from = bwd.To + continue + } + + destProof, err := cell.FromBOC(fwd.DestProof) + if err != nil { + return fmt.Errorf("dest proof boc parse err: %w", err) + } + + configProof, err := cell.FromBOC(fwd.ConfigProof) + if err != nil { + return fmt.Errorf("config proof boc parse err: %w", err) + } + + err = CheckForwardBlockProof(fwd.From, fwd.To, fwd.ToKeyBlock, configProof, destProof, fwd.SignatureSet) + if err != nil { + return fmt.Errorf("invalid forward block from %d to %d proof: %w", fwd.From.SeqNo, fwd.To.SeqNo, err) + } + } + } else { + for _, step := range part.Steps { + bwd, ok := step.(BlockLinkBackward) + if !ok { + return fmt.Errorf("wrong proof direction in response bw %v", reflect.TypeOf(step).String()) + } + + if err = checkBackProof(&bwd); err != nil { + return err + } + } + } + from = part.To + } + + if !from.Equals(to) { + return fmt.Errorf("target block not equals expected") + } + return nil +} diff --git a/ton/runmethod.go b/ton/runmethod.go index daf2f07b..36e621bb 100644 --- a/ton/runmethod.go +++ b/ton/runmethod.go @@ -61,7 +61,7 @@ func (c *APIClient) RunGetMethod(ctx context.Context, blockInfo *BlockIDExt, add } mode := uint32(1 << 2) - if !c.skipProofCheck { + if c.proofCheckPolicy != ProofCheckPolicyUnsafe { mode |= (1 << 1) | (1 << 0) } @@ -93,7 +93,7 @@ func (c *APIClient) RunGetMethod(ctx context.Context, blockInfo *BlockIDExt, add return nil, err } - if !c.skipProofCheck { + if c.proofCheckPolicy != ProofCheckPolicyUnsafe { proof, err := cell.FromBOCMultiRoot(t.Proof) if err != nil { return nil, err @@ -101,7 +101,7 @@ func (c *APIClient) RunGetMethod(ctx context.Context, blockInfo *BlockIDExt, add var shardProof []*cell.Cell var shardHash []byte - if !c.skipProofCheck && addr.Workchain() != address.MasterchainID { + if c.proofCheckPolicy != ProofCheckPolicyUnsafe && addr.Workchain() != address.MasterchainID { if len(t.ShardProof) == 0 { return nil, fmt.Errorf("liteserver has no proof for this account in a given block, request newer block or disable proof checks") } @@ -118,7 +118,7 @@ func (c *APIClient) RunGetMethod(ctx context.Context, blockInfo *BlockIDExt, add shardHash = t.ShardBlock.RootHash } - shardAcc, balanceInfo, err := CheckAccountStateProof(addr, blockInfo, proof, shardProof, shardHash, c.skipProofCheck) + shardAcc, balanceInfo, err := CheckAccountStateProof(addr, blockInfo, proof, shardProof, shardHash, c.proofCheckPolicy == ProofCheckPolicyUnsafe) if err != nil { return nil, fmt.Errorf("failed to check acc state proof: %w", err) } diff --git a/ton/transactions.go b/ton/transactions.go index c8a26920..284c82cd 100644 --- a/ton/transactions.go +++ b/ton/transactions.go @@ -1,6 +1,7 @@ package ton import ( + "bytes" "context" "errors" "fmt" @@ -8,6 +9,7 @@ import ( "github.com/xssnick/tonutils-go/tl" "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/tvm/cell" + "time" ) func init() { @@ -41,8 +43,8 @@ type GetTransactions struct { TxHash []byte `tl:"int256"` } -// ListTransactions - [THIS METHOD HAS NO PROOF CHECK, you can use GetTransaction to verify] -// returns list of transactions before (including) passed lt and hash, the oldest one is first in result slice +// ListTransactions - returns list of transactions before (including) passed lt and hash, the oldest one is first in result slice +// Transactions will be verified to match final tx hash, which should be taken from proved account state, then it is safe. func (c *APIClient) ListTransactions(ctx context.Context, addr *address.Address, limit uint32, lt uint64, txHash []byte) ([]*tlb.Transaction, error) { var resp tl.Serializable err := c.client.QueryLiteserver(ctx, GetTransactions{ @@ -65,18 +67,23 @@ func (c *APIClient) ListTransactions(ctx context.Context, addr *address.Address, return nil, fmt.Errorf("failed to parse cell from transaction bytes: %w", err) } - res := make([]*tlb.Transaction, 0, len(txList)) - for _, txCell := range txList { - loader := txCell.BeginParse() + res := make([]*tlb.Transaction, len(txList)) + + for i := len(txList) - 1; i >= 0; i-- { + loader := txList[i].BeginParse() var tx tlb.Transaction err = tlb.LoadFromCell(&tx, loader) if err != nil { return nil, fmt.Errorf("failed to load transaction from cell: %w", err) } - tx.Hash = txCell.Hash() + tx.Hash = txList[i].Hash() - res = append(res, &tx) + if !bytes.Equal(txHash, tx.Hash) { + return nil, fmt.Errorf("incorrect transaction hash, not matches prev tx hash") + } + txHash = tx.PrevTxHash + res[i] = &tx } return res, nil case LSError: @@ -105,6 +112,10 @@ func (c *APIClient) GetTransaction(ctx context.Context, block *BlockIDExt, addr switch t := resp.(type) { case TransactionInfo: + if !t.ID.Equals(block) { + return nil, fmt.Errorf("incorrect block in response") + } + txCell, err := cell.FromBOC(t.Transaction) if err != nil { return nil, fmt.Errorf("failed to parse cell from transaction bytes: %w", err) @@ -117,7 +128,7 @@ func (c *APIClient) GetTransaction(ctx context.Context, block *BlockIDExt, addr } tx.Hash = txCell.Hash() - if !c.skipProofCheck { + if c.proofCheckPolicy != ProofCheckPolicyUnsafe { txProof, err := cell.FromBOC(t.Proof) if err != nil { return nil, fmt.Errorf("failed to parse proof: %w", err) @@ -128,6 +139,10 @@ func (c *APIClient) GetTransaction(ctx context.Context, block *BlockIDExt, addr return nil, fmt.Errorf("failed to check proof: %w", err) } + if blockProof.Extra == nil || blockProof.Extra.ShardAccountBlocks == nil { + return nil, fmt.Errorf("block proof without shard accounts") + } + var shardAccounts tlb.ShardAccountBlocks err = tlb.LoadFromCellAsProof(&shardAccounts, blockProof.Extra.ShardAccountBlocks.BeginParse()) if err != nil { @@ -148,3 +163,102 @@ func (c *APIClient) GetTransaction(ctx context.Context, block *BlockIDExt, addr } return nil, errUnexpectedResponse(resp) } + +func (c *APIClient) SubscribeOnTransactions(workerCtx context.Context, addr *address.Address, lastProcessedLT uint64, channel chan<- *tlb.Transaction) { + defer func() { + close(channel) + }() + + wait := 0 * time.Second + for { + select { + case <-workerCtx.Done(): + return + case <-time.After(wait): + } + wait = 3 * time.Second + + ctx, cancel := context.WithTimeout(workerCtx, 10*time.Second) + master, err := c.CurrentMasterchainInfo(ctx) + cancel() + if err != nil { + continue + } + + ctx, cancel = context.WithTimeout(workerCtx, 10*time.Second) + acc, err := c.GetAccount(ctx, master, addr) + cancel() + if err != nil { + continue + } + if !acc.IsActive || acc.LastTxLT == 0 { + // no transactions + continue + } + + if lastProcessedLT == acc.LastTxLT { + // already processed all + continue + } + + var transactions []*tlb.Transaction + lastHash, lastLT := acc.LastTxHash, acc.LastTxLT + + waitList := 0 * time.Second + list: + for { + select { + case <-workerCtx.Done(): + return + case <-time.After(waitList): + } + + // ctx = workerCtx + ctx, cancel = context.WithTimeout(workerCtx, 10*time.Second) + res, err := c.ListTransactions(ctx, addr, 10, lastLT, lastHash) + cancel() + if err != nil { + if lsErr, ok := err.(LSError); ok && lsErr.Code == -400 { + // lt not in db error + return + } + waitList = 3 * time.Second + continue + } + + if len(res) == 0 { + break + } + + // reverse slice + for i, j := 0, len(res)-1; i < j; i, j = i+1, j-1 { + res[i], res[j] = res[j], res[i] + } + + for i, tx := range res { + if tx.LT <= lastProcessedLT { + transactions = append(transactions, res[:i]...) + break list + } + } + + lastLT, lastHash = res[len(res)-1].PrevTxLT, res[len(res)-1].PrevTxHash + transactions = append(transactions, res...) + waitList = 0 * time.Second + } + lastProcessedLT = transactions[0].LT // mark last transaction as known to not trigger twice + + // reverse slice to send in correct time order (from old to new) + for i, j := 0, len(transactions)-1; i < j; i, j = i+1, j-1 { + transactions[i], transactions[j] = transactions[j], transactions[i] + } + + for _, tx := range transactions { + channel <- tx + } + + if len(transactions) > 0 { + wait = 0 * time.Second + } + } +} diff --git a/ton/waiter.go b/ton/waiter.go index d491c2ac..ae36d584 100644 --- a/ton/waiter.go +++ b/ton/waiter.go @@ -45,3 +45,7 @@ func (w *waiterClient) StickyContext(ctx context.Context) context.Context { func (w *waiterClient) StickyNodeID(ctx context.Context) uint32 { return w.original.StickyNodeID(ctx) } + +func (w *waiterClient) StickyContextNextNode(ctx context.Context) (context.Context, error) { + return w.original.StickyContextNextNode(ctx) +} diff --git a/ton/wallet/address.go b/ton/wallet/address.go index cb261406..a44c45ee 100644 --- a/ton/wallet/address.go +++ b/ton/wallet/address.go @@ -2,6 +2,7 @@ package wallet import ( "bytes" + "context" "crypto/ed25519" "fmt" @@ -83,3 +84,26 @@ func GetStateInit(pubKey ed25519.PublicKey, ver Version, subWallet uint32) (*tlb Code: code, }, nil } + +func GetPublicKey(ctx context.Context, api TonAPI, addr *address.Address) (ed25519.PublicKey, error) { + master, err := api.CurrentMasterchainInfo(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get current master block: %w", err) + } + + res, err := api.WaitForBlock(master.SeqNo).RunGetMethod(ctx, master, addr, "get_public_key") + if err != nil { + return nil, fmt.Errorf("failed to execute get_public_key contract method: %w", err) + } + + key, err := res.Int(0) + if err != nil { + return nil, fmt.Errorf("failed to parse get_public_key execution result: %w", err) + } + + b := key.Bytes() + pubKey := make([]byte, 32) + copy(pubKey[32-len(b):], b) + + return pubKey, nil +} diff --git a/ton/wallet/integration_test.go b/ton/wallet/integration_test.go index a4f2a1d4..4fa1e4c3 100644 --- a/ton/wallet/integration_test.go +++ b/ton/wallet/integration_test.go @@ -1,6 +1,7 @@ package wallet import ( + "bytes" "context" "encoding/hex" "fmt" @@ -38,7 +39,7 @@ var apiMain = func() *ton.APIClient { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - err := client.AddConnectionsFromConfigUrl(ctx, "https://ton-blockchain.github.io/global.config.json") + err := client.AddConnectionsFromConfigUrl(ctx, "https://ton.org/global.config.json") if err != nil { panic(err) } @@ -180,6 +181,23 @@ func TestWallet_DeployContract(t *testing.T) { } } +func TestWallet_TransferEncrypted(t *testing.T) { + seed := strings.Split(_seed, " ") + ctx := api.Client().StickyContext(context.Background()) + + // init wallet + w, err := FromSeed(api, seed, HighloadV2R2) + if err != nil { + t.Fatal("FromSeed err:", err.Error()) + } + t.Logf("wallet address: %s", w.Address().String()) + + err = w.TransferWithEncryptedComment(ctx, address.MustParseAddr("EQC9bWZd29foipyPOGWlVNVCQzpGAjvi1rGWF7EbNcSVClpA"), tlb.MustFromTON("0.005"), "привет:"+randString(30), true) + if err != nil { + t.Fatal("transfer err:", err) + } +} + func TestGetWalletVersion(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -226,6 +244,19 @@ func TestGetWalletVersion(t *testing.T) { } } +func TestWallet_GetPublicKey(t *testing.T) { + pub, err := GetPublicKey(context.Background(), apiMain, address.MustParseAddr("EQCD39VS5jcptHL8vMjEXrzGaRcCVYto7HUn4bpAOg8xqB2N")) + if err != nil { + t.Fatal(err.Error()) + return + } + + key, _ := hex.DecodeString("72c9ed6b62a6e2eba14a93b90462e7a367777beb8a38fb15b9f33844d22ce2ff") + if !bytes.Equal(pub, key) { + t.Fatal("wrong key: " + hex.EncodeToString(pub)) + } +} + func randString(n int) string { var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + "абвгдежзиклмнопрстиквфыйцэюяАБВГДЕЖЗИЙКЛМНОПРСТИЮЯЗФЫУю!№%:,.!;(!)_+" + diff --git a/ton/wallet/wallet.go b/ton/wallet/wallet.go index 37f8b087..db00ecce 100644 --- a/ton/wallet/wallet.go +++ b/ton/wallet/wallet.go @@ -3,10 +3,15 @@ package wallet import ( "bytes" "context" + "crypto/aes" + "crypto/cipher" "crypto/ed25519" + "crypto/hmac" + "crypto/sha512" "encoding/hex" "errors" "fmt" + "github.com/xssnick/tonutils-go/adnl" "math/rand" "strings" "time" @@ -289,6 +294,32 @@ func (w *Wallet) BuildTransfer(to *address.Address, amount tlb.Coins, bounce boo }, nil } +func (w *Wallet) BuildTransferEncrypted(ctx context.Context, to *address.Address, amount tlb.Coins, bounce bool, comment string) (_ *Message, err error) { + var body *cell.Cell + if comment != "" { + key, err := GetPublicKey(ctx, w.api, to) + if err != nil { + return nil, fmt.Errorf("failed to get destination contract (wallet) public key") + } + + body, err = CreateEncryptedCommentCell(comment, w.Address(), w.key, key) + if err != nil { + return nil, err + } + } + + return &Message{ + Mode: 1 + 2, + InternalMessage: &tlb.InternalMessage{ + IHRDisabled: true, + Bounce: bounce, + DstAddr: to, + Amount: amount, + Body: body, + }, + }, nil +} + func (w *Wallet) Send(ctx context.Context, message *Message, waitConfirmation ...bool) error { return w.SendMany(ctx, []*Message{message}, waitConfirmation...) } @@ -452,6 +483,15 @@ func (w *Wallet) Transfer(ctx context.Context, to *address.Address, amount tlb.C return w.transfer(ctx, to, amount, comment, true, waitConfirmation...) } +// TransferWithEncryptedComment - same as Transfer but encrypts comment, throws error if target contract (address) has no get_public_key method. +func (w *Wallet) TransferWithEncryptedComment(ctx context.Context, to *address.Address, amount tlb.Coins, comment string, waitConfirmation ...bool) error { + transfer, err := w.BuildTransferEncrypted(ctx, to, amount, true, comment) + if err != nil { + return err + } + return w.Send(ctx, transfer, waitConfirmation...) +} + func CreateCommentCell(text string) (*cell.Cell, error) { // comment ident root := cell.BeginCell().MustStoreUInt(0, 32) @@ -463,6 +503,121 @@ func CreateCommentCell(text string) (*cell.Cell, error) { return root.EndCell(), nil } +const EncryptedCommentOpcode = 0x2167da4b + +func DecryptCommentCell(commentCell *cell.Cell, ourKey ed25519.PrivateKey, theirKey ed25519.PublicKey) ([]byte, error) { + slc := commentCell.BeginParse() + op, err := slc.LoadUInt(32) + if err != nil { + return nil, fmt.Errorf("failed to load op code: %w", err) + } + + if op != EncryptedCommentOpcode { + return nil, fmt.Errorf("opcode not match encrypted comment") + } + + xorKey, err := slc.LoadSlice(256) + if err != nil { + return nil, fmt.Errorf("failed to load xor key: %w", err) + } + for i := 0; i < 32; i++ { + xorKey[i] ^= theirKey[i] + } + + if !bytes.Equal(xorKey, ourKey.Public().(ed25519.PublicKey)) { + return nil, fmt.Errorf("message was encrypted not for the given keys") + } + + msgKey, err := slc.LoadSlice(128) + if err != nil { + return nil, fmt.Errorf("failed to load xor key: %w", err) + } + + sharedKey, err := adnl.SharedKey(ourKey, theirKey) + if err != nil { + return nil, fmt.Errorf("failed to compute shared key: %w", err) + } + + h := hmac.New(sha512.New, sharedKey) + h.Write(msgKey) + x := h.Sum(nil) + + data, err := slc.LoadBinarySnake() + if err != nil { + return nil, fmt.Errorf("failed to load snake encrypted data: %w", err) + } + + if len(data) < 32 || len(data)%16 != 0 { + return nil, fmt.Errorf("invalid data") + } + + c, err := aes.NewCipher(x[:32]) + if err != nil { + return nil, err + } + enc := cipher.NewCBCDecrypter(c, x[32:48]) + enc.CryptBlocks(data, data) + + if data[0] > 31 { + return nil, fmt.Errorf("invalid prefix size") + } + return data[data[0]:], nil +} + +func CreateEncryptedCommentCell(text string, senderAddr *address.Address, ourKey ed25519.PrivateKey, theirKey ed25519.PublicKey) (*cell.Cell, error) { + // encrypted comment op code + root := cell.BeginCell().MustStoreUInt(EncryptedCommentOpcode, 32) + + sharedKey, err := adnl.SharedKey(ourKey, theirKey) + if err != nil { + return nil, fmt.Errorf("failed to compute shared key: %w", err) + } + + data := []byte(text) + + pfx := make([]byte, 16+(16-(len(data)%16))) + pfx[0] = byte(len(pfx)) + if _, err = rand.Read(pfx[1:]); err != nil { + return nil, fmt.Errorf("rand gen err: %w", err) + } + data = append(pfx, data...) + + h := hmac.New(sha512.New, []byte(senderAddr.String())) + h.Write(data) + msgKey := h.Sum(nil)[:16] + + /* msgKey := make([]byte, 16) + if _, err = rand.Read(msgKey); err != nil { + return nil, fmt.Errorf("rand gen err: %w", err) + }*/ + + h = hmac.New(sha512.New, sharedKey) + h.Write(msgKey) + x := h.Sum(nil) + + c, err := aes.NewCipher(x[:32]) + if err != nil { + return nil, err + } + + enc := cipher.NewCBCEncrypter(c, x[32:48]) + enc.CryptBlocks(data, data) + + xorKey := ourKey.Public().(ed25519.PublicKey) + for i := 0; i < 32; i++ { + xorKey[i] ^= theirKey[i] + } + + root.MustStoreSlice(xorKey, 256) + root.MustStoreSlice(msgKey, 128) + + if err := root.StoreBinarySnake(data); err != nil { + return nil, fmt.Errorf("failed to build comment: %w", err) + } + + return root.EndCell(), nil +} + func (w *Wallet) transfer(ctx context.Context, to *address.Address, amount tlb.Coins, comment string, bounce bool, waitConfirmation ...bool) (err error) { transfer, err := w.BuildTransfer(to, amount, bounce, comment) if err != nil { diff --git a/ton/wallet/wallet_test.go b/ton/wallet/wallet_test.go index 72a5fc93..27f0db0f 100644 --- a/ton/wallet/wallet_test.go +++ b/ton/wallet/wallet_test.go @@ -3,6 +3,7 @@ package wallet import ( "bytes" "context" + "crypto/ed25519" "errors" "fmt" "github.com/xssnick/tonutils-go/ton" @@ -11,8 +12,6 @@ import ( "testing" "time" - "golang.org/x/crypto/ed25519" - "github.com/xssnick/tonutils-go/address" "github.com/xssnick/tonutils-go/tlb" "github.com/xssnick/tonutils-go/tvm/cell" @@ -461,6 +460,11 @@ type WaiterMock struct { MGetTransaction func(ctx context.Context, block *ton.BlockIDExt, addr *address.Address, lt uint64) (*tlb.Transaction, error) } +func (w WaiterMock) GetBlockProof(ctx context.Context, known, target *ton.BlockIDExt) (*ton.PartialBlockProof, error) { + //TODO implement me + panic("implement me") +} + func (w WaiterMock) GetTime(ctx context.Context) (uint32, error) { return w.MGetTime(ctx) } @@ -508,3 +512,34 @@ func (w WaiterMock) ListTransactions(ctx context.Context, addr *address.Address, func (w WaiterMock) GetTransaction(ctx context.Context, block *ton.BlockIDExt, addr *address.Address, lt uint64) (*tlb.Transaction, error) { return w.MGetTransaction(ctx, block, addr, lt) } + +func TestCreateEncryptedCommentCell(t *testing.T) { + pub1, priv1, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + return + } + pub2, priv2, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + return + } + + msg := randString(200) + + c, err := CreateEncryptedCommentCell(msg, address.MustParseAddr("EQC9bWZd29foipyPOGWlVNVCQzpGAjvi1rGWF7EbNcSVClpA"), priv1, pub2) + if err != nil { + t.Fatal(err) + return + } + + data, err := DecryptCommentCell(c, priv2, pub1) + if err != nil { + t.Fatal(err) + return + } + + if string(data) != msg { + t.Fatal("incorrect result") + } +} diff --git a/tvm/cell/builder.go b/tvm/cell/builder.go index 2fb9eceb..92030dee 100644 --- a/tvm/cell/builder.go +++ b/tvm/cell/builder.go @@ -372,7 +372,7 @@ func (b *Builder) StoreBinarySnake(data []byte) error { return c, nil } - snake, err := f(127 - 4) + snake, err := f(int(b.BitsLeft() / 8)) if err != nil { return err } diff --git a/tvm/cell/cell.go b/tvm/cell/cell.go index 23dd6142..d6763b8b 100644 --- a/tvm/cell/cell.go +++ b/tvm/cell/cell.go @@ -97,7 +97,7 @@ func (c *Cell) UnsafeModify(levelMask LevelMask, special bool) { } func (c *Cell) PeekRef(i int) (*Cell, error) { - if i > len(c.refs) { + if i >= len(c.refs) { return nil, ErrNoMoreRefs } return c.refs[i], nil diff --git a/tvm/cell/cell_test.go b/tvm/cell/cell_test.go index d82d8748..efaf4617 100644 --- a/tvm/cell/cell_test.go +++ b/tvm/cell/cell_test.go @@ -63,7 +63,10 @@ func TestCell_ToBOCWithFlags(t *testing.T) { } boc := c.ToBOCWithFlags(false) - newParsed, _ := FromBOC(boc) + newParsed, err := FromBOC(boc) + if err != nil { + t.Fatal(err) + } if !bytes.Equal(c.Hash(), newParsed.Hash()) { t.Log(tt.cellHex) diff --git a/tvm/cell/flattenIndex.go b/tvm/cell/flattenIndex.go index f6ddd396..99d69743 100644 --- a/tvm/cell/flattenIndex.go +++ b/tvm/cell/flattenIndex.go @@ -1,75 +1,47 @@ package cell -import "log" +import ( + "sort" +) func flattenIndex(src []*Cell) []*Cell { pending := src allCells := map[string]*Cell{} - notPermCells := map[string]struct{}{} - var sorted []string + idx := 0 + var cells []*Cell for len(pending) > 0 { - cells := append([]*Cell{}, pending...) - pending = []*Cell{} - - for _, cell := range cells { - hash := string(cell.Hash()) - if _, ok := allCells[hash]; ok { + var next []*Cell + for _, p := range pending { + hash := string(p.Hash()) + if ps, ok := allCells[hash]; ok { + // move cell forward in boc, because behind reference is not allowed + ps.index, p.index = idx, idx + idx++ + + // we also need to move refs + next = append(next, p.refs...) continue } - notPermCells[hash] = struct{}{} - allCells[hash] = cell - - pending = append(pending, cell.refs...) - } - } - - tempMark := map[string]bool{} - var visit func(hash string) - visit = func(hash string) { - if _, ok := notPermCells[hash]; !ok { - return - } - - if tempMark[hash] { - log.Println("Unknown branch, hash exists") - return - } - - tempMark[hash] = true - - for _, c := range allCells[hash].refs { - visit(string(c.Hash())) - } + p.index = idx + idx++ - sorted = append([]string{hash}, sorted...) - delete(tempMark, hash) - delete(notPermCells, hash) - } + allCells[hash] = p + cells = append(cells, p) - for len(notPermCells) > 0 { - for k := range notPermCells { - visit(k) - break + next = append(next, p.refs...) } + pending = next } - indexes := map[string]int{} - for i := 0; i < len(sorted); i++ { - indexes[sorted[i]] = i - } - - var result []*Cell - for _, ent := range sorted { - rrr := allCells[ent] - rrr.index = indexes[string(rrr.Hash())] + sort.Slice(cells, func(i, j int) bool { + return cells[i].index < cells[j].index + }) - for _, ref := range rrr.refs { - ref.index = indexes[string(ref.Hash())] - } - result = append(result, rrr) + for i, cell := range cells { + // remove possible gaps + cell.index = i } - - return result + return cells } diff --git a/tvm/cell/serialize.go b/tvm/cell/serialize.go index eb0de1be..6cdd2c18 100644 --- a/tvm/cell/serialize.go +++ b/tvm/cell/serialize.go @@ -22,8 +22,16 @@ func (c *Cell) ToBOC() []byte { } func (c *Cell) ToBOCWithFlags(withCRC bool) []byte { + return ToBOCWithFlags([]*Cell{c}, withCRC) +} + +func ToBOCWithFlags(roots []*Cell, withCRC bool) []byte { + if len(roots) == 0 { + return nil + } + // recursively go through cells, build hash index and store unique in slice - orderCells := flattenIndex([]*Cell{c}) + orderCells := flattenIndex(roots) // bytes needed to store num of cells cellSizeBits := math.Log2(float64(len(orderCells)) + 1) @@ -58,8 +66,8 @@ func (c *Cell) ToBOCWithFlags(withCRC bool) []byte { // cells num data = append(data, dynamicIntBytes(uint64(len(orderCells)), uint(cellSizeBytes))...) - // roots num (only 1 supported for now) - data = append(data, dynamicIntBytes(1, uint(cellSizeBytes))...) + // roots num + data = append(data, dynamicIntBytes(uint64(len(roots)), uint(cellSizeBytes))...) // complete BOCs = 0 data = append(data, dynamicIntBytes(0, uint(cellSizeBytes))...) @@ -67,8 +75,10 @@ func (c *Cell) ToBOCWithFlags(withCRC bool) []byte { // len of data data = append(data, dynamicIntBytes(uint64(len(payload)), uint(sizeBytes))...) - // root should have index 0 - data = append(data, dynamicIntBytes(0, uint(cellSizeBytes))...) + // root index + for _, r := range roots { + data = append(data, dynamicIntBytes(uint64(r.index), uint(cellSizeBytes))...) + } data = append(data, payload...) if withCRC { diff --git a/tvm/cell/serialize_test.go b/tvm/cell/serialize_test.go new file mode 100644 index 00000000..68303cf2 --- /dev/null +++ b/tvm/cell/serialize_test.go @@ -0,0 +1,37 @@ +package cell + +import ( + "bytes" + "testing" +) + +func TestToBOCWithFlags(t *testing.T) { + cc1 := BeginCell().MustStoreUInt(111, 22).EndCell() + cc2 := BeginCell().MustStoreUInt(777, 256).EndCell() + cc3 := BeginCell().MustStoreBinarySnake(make([]byte, 700)).EndCell() + + boc := ToBOCWithFlags([]*Cell{cc1, cc2, cc3}, true) + cells, err := FromBOCMultiRoot(boc) + if err != nil { + t.Fatal(err.Error()) + return + } + + if len(cells) != 3 { + t.Fatal("not 3 roots") + return + } + + if !bytes.Equal(cells[0].Hash(), cc1.Hash()) { + t.Fatal("incorrect 0 cell") + return + } + if !bytes.Equal(cells[1].Hash(), cc2.Hash()) { + t.Fatal("incorrect 1 cell") + return + } + if !bytes.Equal(cells[2].Hash(), cc3.Hash()) { + t.Fatal("incorrect 2 cell") + return + } +} From 457547a0ae20d852db8e7c8df467903e7472fdbe Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 14 Aug 2023 08:46:33 +0400 Subject: [PATCH 28/38] Change from deprecated NanoTON to Nano --- README.md | 2 +- example/highload-wallet/main.go | 2 +- example/send-to-contract/main.go | 6 +++--- example/transfer-url-for-qr/main.go | 2 +- example/wallet-cold-alike/main.go | 2 +- example/wallet/main.go | 2 +- tlb/account_test.go | 4 ++-- tlb/coins_test.go | 14 +++++++------- tlb/loader_test.go | 2 +- tlb/message.go | 8 ++++---- tlb/message_test.go | 4 ++-- tlb/transaction.go | 4 ++-- ton/getstate.go | 2 +- ton/jetton/integration_test.go | 2 +- ton/nft/integration_test.go | 2 +- ton/payments/integration_test.go | 2 +- ton/wallet/integration_test.go | 2 +- 17 files changed, 31 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index c46ddeb1..e13c9db1 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ if err != nil { panic(err) } -if balance.NanoTON().Uint64() >= 3000000 { +if balance.Nano().Uint64() >= 3000000 { addr := address.MustParseAddr("EQCD39VS5jcptHL8vMjEXrzGaRcCVYto7HUn4bpAOg8xqB2N") err = w.Transfer(context.Background(), addr, tlb.MustFromTON("0.003"), "Hey bro, happy birthday!") if err != nil { diff --git a/example/highload-wallet/main.go b/example/highload-wallet/main.go index ada606bd..7756ef25 100644 --- a/example/highload-wallet/main.go +++ b/example/highload-wallet/main.go @@ -56,7 +56,7 @@ func main() { "EQBLS8WneoKVGrwq2MO786J6ruQNiv62NXr8Ko_l5Ttondoc": "0.003", } - if balance.NanoTON().Uint64() >= 3000000 { + if balance.Nano().Uint64() >= 3000000 { // create comment cell to send in body of each message comment, err := wallet.CreateCommentCell("Hello from tonutils-go!") if err != nil { diff --git a/example/send-to-contract/main.go b/example/send-to-contract/main.go index f3f64e2c..494ff0e3 100644 --- a/example/send-to-contract/main.go +++ b/example/send-to-contract/main.go @@ -50,16 +50,16 @@ func main() { return } - if balance.NanoTON().Uint64() >= 3000000 { + if balance.Nano().Uint64() >= 3000000 { // create transaction body cell, depends on what contract needs, just random example here body := cell.BeginCell(). - MustStoreUInt(0x123abc55, 32). // op code + MustStoreUInt(0x123abc55, 32). // op code MustStoreUInt(rand.Uint64(), 64). // query id // payload: MustStoreAddr(address.MustParseAddr("EQAbMQzuuGiCne0R7QEj9nrXsjM7gNjeVmrlBZouyC-SCLlO")). MustStoreRef( cell.BeginCell(). - MustStoreBigCoins(tlb.MustFromTON("1.521").NanoTON()). + MustStoreBigCoins(tlb.MustFromTON("1.521").Nano()). EndCell(), ).EndCell() diff --git a/example/transfer-url-for-qr/main.go b/example/transfer-url-for-qr/main.go index f87d4e1e..3728d1fd 100644 --- a/example/transfer-url-for-qr/main.go +++ b/example/transfer-url-for-qr/main.go @@ -18,5 +18,5 @@ func main() { // for example you can make QR code from it and scan using TonKeeper, // and this transaction will be executed by the wallet fmt.Printf("ton://transfer/%s?bin=%s&amount=%s", addr.String(), - base64.URLEncoding.EncodeToString(body.ToBOC()), tlb.MustFromTON("0.55").NanoTON().String()) + base64.URLEncoding.EncodeToString(body.ToBOC()), tlb.MustFromTON("0.55").Nano().String()) } diff --git a/example/wallet-cold-alike/main.go b/example/wallet-cold-alike/main.go index 81f2e295..583a1a52 100644 --- a/example/wallet-cold-alike/main.go +++ b/example/wallet-cold-alike/main.go @@ -48,7 +48,7 @@ func main() { return } - if balance.NanoTON().Uint64() >= 3000000 { + if balance.Nano().Uint64() >= 3000000 { addr := address.MustParseAddr("EQCD39VS5jcptHL8vMjEXrzGaRcCVYto7HUn4bpAOg8xqB2N") log.Println("sending transaction and waiting for confirmation...") diff --git a/example/wallet/main.go b/example/wallet/main.go index 1edf7ab8..db29944a 100644 --- a/example/wallet/main.go +++ b/example/wallet/main.go @@ -49,7 +49,7 @@ func main() { return } - if balance.NanoTON().Uint64() >= 3000000 { + if balance.Nano().Uint64() >= 3000000 { addr := address.MustParseAddr("EQCD39VS5jcptHL8vMjEXrzGaRcCVYto7HUn4bpAOg8xqB2N") log.Println("sending transaction and waiting for confirmation...") diff --git a/tlb/account_test.go b/tlb/account_test.go index b420d565..4f0d1c19 100644 --- a/tlb/account_test.go +++ b/tlb/account_test.go @@ -40,8 +40,8 @@ func TestAccountState_LoadFromCell(t *testing.T) { return } - if as.Balance.NanoTON().Uint64() != 31011747 { - t.Fatal("balance not eq", as.Balance.NanoTON().String()) + if as.Balance.Nano().Uint64() != 31011747 { + t.Fatal("balance not eq", as.Balance.Nano().String()) return } diff --git a/tlb/coins_test.go b/tlb/coins_test.go index 8494a7d6..0081c707 100644 --- a/tlb/coins_test.go +++ b/tlb/coins_test.go @@ -9,37 +9,37 @@ import ( ) func TestCoins_FromTON(t *testing.T) { - g := MustFromTON("0").NanoTON().Uint64() + g := MustFromTON("0").Nano().Uint64() if g != 0 { t.Fatalf("0 wrong: %d", g) } - g = MustFromTON("0.0000").NanoTON().Uint64() + g = MustFromTON("0.0000").Nano().Uint64() if g != 0 { t.Fatalf("0 wrong: %d", g) } - g = MustFromTON("7").NanoTON().Uint64() + g = MustFromTON("7").Nano().Uint64() if g != 7000000000 { t.Fatalf("7 wrong: %d", g) } - g = MustFromTON("7.518").NanoTON().Uint64() + g = MustFromTON("7.518").Nano().Uint64() if g != 7518000000 { t.Fatalf("7.518 wrong: %d", g) } - g = MustFromTON("17.98765432111").NanoTON().Uint64() + g = MustFromTON("17.98765432111").Nano().Uint64() if g != 17987654321 { t.Fatalf("17.98765432111 wrong: %d", g) } - g = MustFromTON("0.000000001").NanoTON().Uint64() + g = MustFromTON("0.000000001").Nano().Uint64() if g != 1 { t.Fatalf("0.000000001 wrong: %d", g) } - g = MustFromTON("0.090000001").NanoTON().Uint64() + g = MustFromTON("0.090000001").Nano().Uint64() if g != 90000001 { t.Fatalf("0.090000001 wrong: %d", g) } diff --git a/tlb/loader_test.go b/tlb/loader_test.go index 98214b8d..2c75e65b 100644 --- a/tlb/loader_test.go +++ b/tlb/loader_test.go @@ -149,7 +149,7 @@ func TestLoadFromCell(t *testing.T) { t.Fatal("uint 7126382921832 not eq") } - if x.Inside.ValCoins.NanoTON().Uint64() != 700000 { + if x.Inside.ValCoins.Nano().Uint64() != 700000 { t.Fatal("coins 700000 not eq") } diff --git a/tlb/message.go b/tlb/message.go index 2caa6dfc..8000240e 100644 --- a/tlb/message.go +++ b/tlb/message.go @@ -189,12 +189,12 @@ func (m *InternalMessage) ToCell() (*cell.Cell, error) { b.MustStoreBoolBit(m.Bounced) b.MustStoreAddr(m.SrcAddr) b.MustStoreAddr(m.DstAddr) - b.MustStoreBigCoins(m.Amount.NanoTON()) + b.MustStoreBigCoins(m.Amount.Nano()) b.MustStoreDict(m.ExtraCurrencies) - b.MustStoreBigCoins(m.IHRFee.NanoTON()) - b.MustStoreBigCoins(m.FwdFee.NanoTON()) + b.MustStoreBigCoins(m.IHRFee.Nano()) + b.MustStoreBigCoins(m.FwdFee.Nano()) b.MustStoreUInt(m.CreatedLT, 64) b.MustStoreUInt(uint64(m.CreatedAt), 32) @@ -238,7 +238,7 @@ func (m *ExternalMessage) ToCell() (*cell.Cell, error) { builder := cell.BeginCell().MustStoreUInt(0b10, 2). MustStoreAddr(m.SrcAddr). MustStoreAddr(m.DstAddr). - MustStoreBigCoins(m.ImportFee.NanoTON()) + MustStoreBigCoins(m.ImportFee.Nano()) builder.MustStoreBoolBit(m.StateInit != nil) // has state init if m.StateInit != nil { diff --git a/tlb/message_test.go b/tlb/message_test.go index 583be6bc..ed98f099 100644 --- a/tlb/message_test.go +++ b/tlb/message_test.go @@ -47,8 +47,8 @@ func TestInternalMessage_ToCell(t *testing.T) { // need to deploy contract on te t.Fatal("not eq dst") } - if intMsg.Amount.NanoTON().Uint64() != intMsg2.Amount.NanoTON().Uint64() { - t.Fatal("not eq ton", intMsg.Amount.NanoTON(), intMsg2.Amount.NanoTON()) + if intMsg.Amount.Nano().Uint64() != intMsg2.Amount.Nano().Uint64() { + t.Fatal("not eq ton", intMsg.Amount.Nano(), intMsg2.Amount.Nano()) } } diff --git a/tlb/transaction.go b/tlb/transaction.go index 6a73c9fb..b2d4b151 100644 --- a/tlb/transaction.go +++ b/tlb/transaction.go @@ -262,7 +262,7 @@ func (t *Transaction) String() string { for _, m := range listOut { destinations = append(destinations, m.Msg.DestAddr().String()) if m.MsgType == MsgTypeInternal { - out.Add(out, m.AsInternal().Amount.NanoTON()) + out.Add(out, m.AsInternal().Amount.Nano()) } } } @@ -276,7 +276,7 @@ func (t *Transaction) String() string { } if t.IO.In != nil { if t.IO.In.MsgType == MsgTypeInternal { - in = t.IO.In.AsInternal().Amount.NanoTON() + in = t.IO.In.AsInternal().Amount.Nano() } if in.Cmp(big.NewInt(0)) != 0 { diff --git a/ton/getstate.go b/ton/getstate.go index c2a09cfa..a4650098 100644 --- a/ton/getstate.go +++ b/ton/getstate.go @@ -93,7 +93,7 @@ func (c *APIClient) GetAccount(ctx context.Context, block *BlockIDExt, addr *add return nil, fmt.Errorf("failed to load account state: %w", err) } - if st.Balance.NanoTON().Cmp(balanceInfo.Currencies.Coins.NanoTON()) != 0 { + if st.Balance.Nano().Cmp(balanceInfo.Currencies.Coins.Nano()) != 0 { return nil, fmt.Errorf("proof balance not match state balance") } diff --git a/ton/jetton/integration_test.go b/ton/jetton/integration_test.go index 75b092b1..f2b65f4d 100644 --- a/ton/jetton/integration_test.go +++ b/ton/jetton/integration_test.go @@ -140,7 +140,7 @@ func TestJettonMasterClient_Transfer(t *testing.T) { t.Fatal("balance was not changed after burn") } - want := b.Uint64() - amt.NanoTON().Uint64()*2 + want := b.Uint64() - amt.Nano().Uint64()*2 got := b2.Uint64() if want != got { t.Fatal("balance not expected, want ", want, "got", got) diff --git a/ton/nft/integration_test.go b/ton/nft/integration_test.go index af777b00..f6da7c37 100644 --- a/ton/nft/integration_test.go +++ b/ton/nft/integration_test.go @@ -59,7 +59,7 @@ func Test_NftMintTransfer(t *testing.T) { t.Fatal("GetBalance err:", err.Error()) } - if balance.NanoTON().Uint64() < 3000000 { + if balance.Nano().Uint64() < 3000000 { t.Fatal("not enough balance", w.Address(), balance.String()) } diff --git a/ton/payments/integration_test.go b/ton/payments/integration_test.go index bced9ad7..3ecc365a 100644 --- a/ton/payments/integration_test.go +++ b/ton/payments/integration_test.go @@ -82,7 +82,7 @@ func TestClient_DeployAsyncChannel(t *testing.T) { t.Fatal("channel status incorrect") } - if ch.Storage.BalanceA.NanoTON().Cmp(tlb.MustFromTON("0.005").NanoTON()) != 0 { + if ch.Storage.BalanceA.Nano().Cmp(tlb.MustFromTON("0.005").Nano()) != 0 { t.Fatal("balance incorrect") } diff --git a/ton/wallet/integration_test.go b/ton/wallet/integration_test.go index 4fa1e4c3..2906bec0 100644 --- a/ton/wallet/integration_test.go +++ b/ton/wallet/integration_test.go @@ -93,7 +93,7 @@ func Test_WalletTransfer(t *testing.T) { comment := randString(150) addr := address.MustParseAddr("EQA8aJTl0jfFnUZBJjTeUxu9OcbsoPBp9UcHE9upyY_X35kE") - if balance.NanoTON().Uint64() >= 3000000 { + if balance.Nano().Uint64() >= 3000000 { err = w.Transfer(ctx, addr, tlb.MustFromTON("0.003"), comment, true) if err != nil { t.Fatal("Transfer err:", err.Error()) From e1fb352f1c59481f73676e3987a669dad0b45129 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 14 Aug 2023 08:55:13 +0400 Subject: [PATCH 29/38] Fixed GetWalletAddress test --- ton/jetton/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ton/jetton/integration_test.go b/ton/jetton/integration_test.go index f2b65f4d..04f248e7 100644 --- a/ton/jetton/integration_test.go +++ b/ton/jetton/integration_test.go @@ -68,7 +68,7 @@ func TestJettonMasterClient_GetWalletAddress(t *testing.T) { t.Fatal(err) } - if b.String() != "22686.666348532" { + if tlb.MustFromNano(b, 9).String() != "22686.666348532" { t.Fatal("balance diff:", b.String()) } } From fd7ba073853766b14c838b91f0c81e81a066b5c2 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 14 Aug 2023 09:45:12 +0400 Subject: [PATCH 30/38] Fixed GetAsyncChannel on inactive address --- README.md | 5 +++-- ton/payments/channel.go | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e13c9db1..917a9752 100644 --- a/README.md +++ b/README.md @@ -478,9 +478,10 @@ client.SetOnDisconnect(func(addr, serverKey string) { * ✅ Overlays * ✅ TL Parser/Serializer * ✅ TL-B Parser/Serializer +* ✅ Payment channels +* ✅ Liteserver proofs automatic validation * DHT Server -* Payment channels -* Merkle proofs automatic validation +* TVM [ton-svg]: https://img.shields.io/badge/Based%20on-TON-blue diff --git a/ton/payments/channel.go b/ton/payments/channel.go index 0469b4e5..0b609601 100644 --- a/ton/payments/channel.go +++ b/ton/payments/channel.go @@ -62,6 +62,10 @@ func (c *Client) GetAsyncChannel(ctx context.Context, addr *address.Address, ver return nil, fmt.Errorf("failed to get account: %w", err) } + if !acc.IsActive { + return nil, fmt.Errorf("channel account is not active") + } + if verify { codeBoC, _ := hex.DecodeString(AsyncPaymentChannelCodeBoC) code, _ := cell.FromBOC(codeBoC) From 2fbb5f760a8ba68ae189ef69fc0a36519fc70293 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 14 Aug 2023 22:29:45 +0400 Subject: [PATCH 31/38] Added liteclient retrier wrapper & tests --- adnl/dht/client_test.go | 4 +- ton/api.go | 34 +++++++++++-- ton/block.go | 85 +++----------------------------- ton/dns/integration_test.go | 4 +- ton/dns/resolve.go | 2 +- ton/integration_test.go | 46 +++++------------ ton/jetton/integration_test.go | 6 +-- ton/jetton/jetton.go | 2 +- ton/nft/collection.go | 2 +- ton/nft/integration_test.go | 4 +- ton/payments/channel.go | 2 +- ton/payments/integration_test.go | 4 +- ton/retrier.go | 43 ++++++++++++++++ ton/wallet/integration_test.go | 8 +-- ton/wallet/wallet.go | 5 +- ton/wallet/wallet_test.go | 35 ++++++++++++- 16 files changed, 146 insertions(+), 140 deletions(-) create mode 100644 ton/retrier.go diff --git a/adnl/dht/client_test.go b/adnl/dht/client_test.go index c33df296..44f53aa2 100644 --- a/adnl/dht/client_test.go +++ b/adnl/dht/client_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/ed25519" + "crypto/rand" "encoding/base64" "encoding/binary" "encoding/hex" @@ -14,7 +15,6 @@ import ( "github.com/xssnick/tonutils-go/adnl/overlay" "github.com/xssnick/tonutils-go/liteclient" "github.com/xssnick/tonutils-go/tl" - "math/rand" "net" "reflect" "strconv" @@ -735,7 +735,7 @@ func TestClient_StoreOverlayNodesIntegration(t *testing.T) { _, _, err = dhtClient.StoreOverlayNodes(ctx, id, &overlay.NodesList{ List: []overlay.Node{*node}, - }, 5*time.Minute, 1) + }, 5*time.Minute, 2) if err != nil { t.Fatal(err) } diff --git a/ton/api.go b/ton/api.go index 8f52873d..fbc06345 100644 --- a/ton/api.go +++ b/ton/api.go @@ -44,7 +44,11 @@ type LSError struct { Text string `tl:"string"` } -type APIClientWaiter interface { +// Deprecated: use APIClientWrapped +type APIClientWaiter = APIClientWrapped + +type APIClientWrapped interface { + Client() LiteClient GetTime(ctx context.Context) (uint32, error) LookupBlock(ctx context.Context, workchain int32, shard int64, seqno uint32) (*BlockIDExt, error) GetBlockData(ctx context.Context, block *BlockIDExt) (*tlb.Block, error) @@ -58,6 +62,11 @@ type APIClientWaiter interface { ListTransactions(ctx context.Context, addr *address.Address, num uint32, lt uint64, txHash []byte) ([]*tlb.Transaction, error) GetTransaction(ctx context.Context, block *BlockIDExt, addr *address.Address, lt uint64) (*tlb.Transaction, error) GetBlockProof(ctx context.Context, known, target *BlockIDExt) (*PartialBlockProof, error) + CurrentMasterchainInfo(ctx context.Context) (_ *BlockIDExt, err error) + SubscribeOnTransactions(workerCtx context.Context, addr *address.Address, lastProcessedLT uint64, channel chan<- *tlb.Transaction) + VerifyProofChain(ctx context.Context, from, to *BlockIDExt) error + WaitForBlock(seqno uint32) APIClientWrapped + WithRetry() APIClientWrapped } type APIClient struct { @@ -91,22 +100,41 @@ func NewAPIClient(client LiteClient, proofCheckPolicy ...ProofCheckPolicy) *APIC } } +// SetTrustedBlock - set starting point to verify master block proofs chain func (c *APIClient) SetTrustedBlock(block *BlockIDExt) { c.trustedBlock = block.Copy() } +// SetTrustedBlockFromConfig - same as SetTrustedBlock but takes init block from config func (c *APIClient) SetTrustedBlockFromConfig(cfg *liteclient.GlobalConfig) { b := BlockIDExt(cfg.Validator.InitBlock) - c.trustedBlock = &b + c.SetTrustedBlock(&b) } -func (c *APIClient) WaitForBlock(seqno uint32) APIClientWaiter { +// WaitForBlock - waits for the given master block seqno will be available on the requested node +func (c *APIClient) WaitForBlock(seqno uint32) APIClientWrapped { return &APIClient{ parent: c, client: &waiterClient{original: c.client, seqno: seqno}, } } +// WithRetry - automatically retires request to another available liteserver +// when error code 651 or -400 is received +func (c *APIClient) WithRetry() APIClientWrapped { + return &APIClient{ + parent: c, + client: &retryClient{original: c.client}, + } +} + +func (c *APIClient) root() *APIClient { + if c.parent != nil { + return c.parent.root() + } + return c +} + func (e LSError) Error() string { return fmt.Sprintf("lite server error, code %d: %s", e.Code, e.Text) } diff --git a/ton/block.go b/ton/block.go index 94f8e815..aba71e2d 100644 --- a/ton/block.go +++ b/ton/block.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "log" + "reflect" "time" "github.com/xssnick/tonutils-go/tl" @@ -205,21 +206,18 @@ func (c *APIClient) Client() LiteClient { // CurrentMasterchainInfo - cached version of GetMasterchainInfo to not do it in parallel many times func (c *APIClient) CurrentMasterchainInfo(ctx context.Context) (_ *BlockIDExt, err error) { - if c.parent != nil { - // this method should be called at top level, to share curMasters and lock. - return c.parent.CurrentMasterchainInfo(ctx) - } + root := c.root() // this method should use root level props, to share curMasters and lock. // if not sticky - id will be 0 nodeID := c.client.StickyNodeID(ctx) - c.curMastersLock.RLock() - master := c.curMasters[nodeID] + root.curMastersLock.RLock() + master := root.curMasters[nodeID] if master == nil { master = &masterInfo{} - c.curMasters[nodeID] = master + root.curMasters[nodeID] = master } - c.curMastersLock.RUnlock() + root.curMastersLock.RUnlock() master.mx.Lock() defer master.mx.Unlock() @@ -230,12 +228,7 @@ func (c *APIClient) CurrentMasterchainInfo(ctx context.Context) (_ *BlockIDExt, var block *BlockIDExt block, err = c.GetMasterchainInfo(ctx) if err != nil { - return nil, err - } - - block, err = c.waitMasterBlock(ctx, block.SeqNo) - if err != nil { - return nil, err + return nil, fmt.Errorf("get masterchain info error (%s): %w", reflect.TypeOf(c.client).String(), err) } master.updatedAt = time.Now() @@ -525,70 +518,6 @@ func LoadShardsFromHashes(shardHashes *cell.Dictionary) (shards []*BlockIDExt, e return } -// WaitNextMasterBlock - wait for the next block of master chain -func (c *APIClient) waitMasterBlock(ctx context.Context, seqno uint32) (*BlockIDExt, error) { - var timeout = 10 * time.Second - - deadline, ok := ctx.Deadline() - if ok { - t := deadline.Sub(time.Now()) - if t < timeout { - timeout = t - } - } - - prefix, err := tl.Serialize(WaitMasterchainSeqno{ - Seqno: int32(seqno), - Timeout: int32(timeout / time.Millisecond), - }, true) - if err != nil { - return nil, err - } - - suffix, err := tl.Serialize(GetMasterchainInf{}, true) - if err != nil { - return nil, err - } - - var resp tl.Serializable - err = c.client.QueryLiteserver(ctx, tl.Raw(append(prefix, suffix...)), &resp) - if err != nil { - return nil, err - } - - switch t := resp.(type) { - case MasterchainInfo: - return t.Last, nil - case LSError: - if t.Code == 652 { - return nil, ErrNoNewBlocks - } - return nil, t - } - return nil, errUnexpectedResponse(resp) -} - -// Deprecated: use APIClient.WaitForBlock as method prefix -func (c *APIClient) WaitNextMasterBlock(ctx context.Context, master *BlockIDExt) (*BlockIDExt, error) { - if c.parent != nil { - // this method should be called at top level, because wrapper already have this logic. - return c.parent.WaitNextMasterBlock(ctx, master) - } - - if master.Workchain != -1 { - return nil, errors.New("not a master block passed") - } - - ctx = c.client.StickyContext(ctx) - - m, err := c.waitMasterBlock(ctx, master.SeqNo+1) - if err != nil { - return nil, err - } - - return m, nil -} - // GetBlockProof - gets proof chain for the block func (c *APIClient) GetBlockProof(ctx context.Context, known, target *BlockIDExt) (*PartialBlockProof, error) { var resp tl.Serializable diff --git a/ton/dns/integration_test.go b/ton/dns/integration_test.go index ab5b0e50..136a2993 100644 --- a/ton/dns/integration_test.go +++ b/ton/dns/integration_test.go @@ -10,7 +10,7 @@ import ( var client = liteclient.NewConnectionPool() -var api = func() *ton.APIClient { +var api = func() ton.APIClientWrapped { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -19,7 +19,7 @@ var api = func() *ton.APIClient { panic(err) } - return ton.NewAPIClient(client) + return ton.NewAPIClient(client).WithRetry() }() func TestDNSClient_Resolve(t *testing.T) { diff --git a/ton/dns/resolve.go b/ton/dns/resolve.go index 05c85cc2..73848813 100644 --- a/ton/dns/resolve.go +++ b/ton/dns/resolve.go @@ -21,7 +21,7 @@ const _CategoryADNLSite = 0xad01 const _CategoryStorageSite = 0x7473 type TonApi interface { - WaitForBlock(seqno uint32) ton.APIClientWaiter + WaitForBlock(seqno uint32) ton.APIClientWrapped CurrentMasterchainInfo(ctx context.Context) (_ *ton.BlockIDExt, err error) RunGetMethod(ctx context.Context, blockInfo *ton.BlockIDExt, addr *address.Address, method string, params ...any) (*ton.ExecutionResult, error) GetBlockchainConfig(ctx context.Context, block *ton.BlockIDExt, onlyParams ...int32) (*ton.BlockchainConfig, error) diff --git a/ton/integration_test.go b/ton/integration_test.go index da8f52a0..3d62600c 100644 --- a/ton/integration_test.go +++ b/ton/integration_test.go @@ -29,7 +29,7 @@ var apiTestNet = func() *APIClient { return NewAPIClient(client) }() -var api = func() *APIClient { +var api = func() APIClientWrapped { client := liteclient.NewConnectionPool() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -45,7 +45,7 @@ var api = func() *APIClient { panic(err) } - a := NewAPIClient(client, ProofCheckPolicySecure) + a := NewAPIClient(client, ProofCheckPolicySecure).WithRetry() // a.SetTrustedBlockFromConfig(cfg) return a }() @@ -59,7 +59,7 @@ var testContractAddrTestNet = func() *address.Address { }() func Test_CurrentChainInfo(t *testing.T) { - ctx := api.client.StickyContext(context.Background()) + ctx := api.Client().StickyContext(context.Background()) b, err := api.CurrentMasterchainInfo(ctx) if err != nil { @@ -82,7 +82,7 @@ func Test_CurrentChainInfo(t *testing.T) { } func TestAPIClient_GetBlockData(t *testing.T) { - ctx := api.client.StickyContext(context.Background()) + ctx := api.Client().StickyContext(context.Background()) b, err := api.CurrentMasterchainInfo(ctx) if err != nil { @@ -245,7 +245,7 @@ func Test_ExternalMessage(t *testing.T) { // need to deploy contract on test-net func Test_Account(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - ctx = api.client.StickyContext(ctx) + ctx = api.Client().StickyContext(ctx) b, err := api.GetMasterchainInfo(ctx) if err != nil { @@ -307,7 +307,7 @@ func Test_Account(t *testing.T) { func Test_AccountMaster(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - ctx = api.client.StickyContext(ctx) + ctx = api.Client().StickyContext(ctx) b, err := api.GetMasterchainInfo(ctx) if err != nil { @@ -404,7 +404,7 @@ func Test_AccountHasMethod(t *testing.T) { } func Test_BlockScan(t *testing.T) { - ctx := api.client.StickyContext(context.Background()) + ctx := api.Client().StickyContext(context.Background()) var shards []*BlockIDExt for { // we need fresh block info to run get methods @@ -494,30 +494,6 @@ func Test_BlockScan(t *testing.T) { } } -func TestAPIClient_WaitNextBlock(t *testing.T) { - ctx := api.client.StickyContext(context.Background()) - - c, err := api.CurrentMasterchainInfo(ctx) - if err != nil { - t.Fatal("get curr block err:", err.Error()) - } - - n, err := api.WaitNextMasterBlock(ctx, c) - if err != nil { - t.Fatal("wait block err:", err.Error()) - } - - if n.SeqNo != c.SeqNo+1 { - t.Fatal("seqno incorrect") - } - - c.Workchain = 7 - n, err = api.WaitNextMasterBlock(ctx, c) - if err == nil { - t.Fatal("it works with not master") - } -} - func Test_GetTime(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -530,7 +506,7 @@ func Test_GetTime(t *testing.T) { } func Test_GetConfigParamsAll(t *testing.T) { - ctx := api.client.StickyContext(context.Background()) + ctx := api.Client().StickyContext(context.Background()) b, err := api.GetMasterchainInfo(ctx) if err != nil { @@ -554,7 +530,7 @@ func Test_GetConfigParamsAll(t *testing.T) { } func Test_GetConfigParams8(t *testing.T) { - ctx := api.client.StickyContext(context.Background()) + ctx := api.Client().StickyContext(context.Background()) b, err := api.GetMasterchainInfo(ctx) if err != nil { @@ -649,7 +625,7 @@ func TestAPIClient_GetBlockProofForward(t *testing.T) { return } - ctx := api.client.StickyContext(context.Background()) + ctx := api.Client().StickyContext(context.Background()) initBlock := BlockIDExt(cfg.Validator.InitBlock) known := &initBlock @@ -680,7 +656,7 @@ func TestAPIClient_GetBlockProofForward(t *testing.T) { func TestAPIClient_SubscribeOnTransactions(t *testing.T) { _ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - ctx := api.client.StickyContext(_ctx) + ctx := api.Client().StickyContext(_ctx) addr := address.MustParseAddr("EQCD39VS5jcptHL8vMjEXrzGaRcCVYto7HUn4bpAOg8xqB2N") diff --git a/ton/jetton/integration_test.go b/ton/jetton/integration_test.go index 04f248e7..cae66635 100644 --- a/ton/jetton/integration_test.go +++ b/ton/jetton/integration_test.go @@ -15,7 +15,7 @@ import ( "time" ) -var api = func() *ton.APIClient { +var api = func() ton.APIClientWrapped { client := liteclient.NewConnectionPool() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -26,7 +26,7 @@ var api = func() *ton.APIClient { panic(err) } - return ton.NewAPIClient(client) + return ton.NewAPIClient(client).WithRetry() }() func TestJettonMasterClient_GetJettonData(t *testing.T) { @@ -147,7 +147,7 @@ func TestJettonMasterClient_Transfer(t *testing.T) { } } -func getWallet(api *ton.APIClient) *wallet.Wallet { +func getWallet(api ton.APIClientWrapped) *wallet.Wallet { words := strings.Split("cement secret mad fatal tip credit thank year toddler arrange good version melt truth embark debris execute answer please narrow fiber school achieve client", " ") w, err := wallet.FromSeed(api, words, wallet.V3) if err != nil { diff --git a/ton/jetton/jetton.go b/ton/jetton/jetton.go index 09ea47eb..fa51471a 100644 --- a/ton/jetton/jetton.go +++ b/ton/jetton/jetton.go @@ -13,7 +13,7 @@ import ( ) type TonApi interface { - WaitForBlock(seqno uint32) ton.APIClientWaiter + WaitForBlock(seqno uint32) ton.APIClientWrapped CurrentMasterchainInfo(ctx context.Context) (_ *ton.BlockIDExt, err error) RunGetMethod(ctx context.Context, blockInfo *ton.BlockIDExt, addr *address.Address, method string, params ...any) (*ton.ExecutionResult, error) SubscribeOnTransactions(workerCtx context.Context, addr *address.Address, lastProcessedLT uint64, channel chan<- *tlb.Transaction) diff --git a/ton/nft/collection.go b/ton/nft/collection.go index 8f075f36..e3c5cb9f 100644 --- a/ton/nft/collection.go +++ b/ton/nft/collection.go @@ -14,7 +14,7 @@ import ( ) type TonApi interface { - WaitForBlock(seqno uint32) ton.APIClientWaiter + WaitForBlock(seqno uint32) ton.APIClientWrapped CurrentMasterchainInfo(ctx context.Context) (_ *ton.BlockIDExt, err error) RunGetMethod(ctx context.Context, blockInfo *ton.BlockIDExt, addr *address.Address, method string, params ...any) (*ton.ExecutionResult, error) } diff --git a/ton/nft/integration_test.go b/ton/nft/integration_test.go index f6da7c37..b5dc262b 100644 --- a/ton/nft/integration_test.go +++ b/ton/nft/integration_test.go @@ -19,7 +19,7 @@ import ( "github.com/xssnick/tonutils-go/tvm/cell" ) -var api = func() *ton.APIClient { +var api = func() ton.APIClientWrapped { client := liteclient.NewConnectionPool() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) @@ -30,7 +30,7 @@ var api = func() *ton.APIClient { panic(err) } - return ton.NewAPIClient(client) + return ton.NewAPIClient(client).WithRetry() }() var _seed = os.Getenv("WALLET_SEED") diff --git a/ton/payments/channel.go b/ton/payments/channel.go index 0b609601..a82e72f1 100644 --- a/ton/payments/channel.go +++ b/ton/payments/channel.go @@ -15,7 +15,7 @@ import ( ) type TonApi interface { - WaitForBlock(seqno uint32) ton.APIClientWaiter + WaitForBlock(seqno uint32) ton.APIClientWrapped CurrentMasterchainInfo(ctx context.Context) (_ *ton.BlockIDExt, err error) RunGetMethod(ctx context.Context, blockInfo *ton.BlockIDExt, addr *address.Address, method string, params ...any) (*ton.ExecutionResult, error) SendExternalMessage(ctx context.Context, msg *tlb.ExternalMessage) error diff --git a/ton/payments/integration_test.go b/ton/payments/integration_test.go index 3ecc365a..a289acfb 100644 --- a/ton/payments/integration_test.go +++ b/ton/payments/integration_test.go @@ -17,7 +17,7 @@ import ( "time" ) -var api = func() *ton.APIClient { +var api = func() ton.APIClientWrapped { client := liteclient.NewConnectionPool() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -28,7 +28,7 @@ var api = func() *ton.APIClient { panic(err) } - return ton.NewAPIClient(client) + return ton.NewAPIClient(client).WithRetry() }() var _seed = strings.Split(os.Getenv("WALLET_SEED"), " ") diff --git a/ton/retrier.go b/ton/retrier.go new file mode 100644 index 00000000..1d29b71c --- /dev/null +++ b/ton/retrier.go @@ -0,0 +1,43 @@ +package ton + +import ( + "context" + "fmt" + "github.com/xssnick/tonutils-go/tl" + "strings" +) + +type retryClient struct { + original LiteClient +} + +func (w *retryClient) QueryLiteserver(ctx context.Context, payload tl.Serializable, result tl.Serializable) error { + for { + err := w.original.QueryLiteserver(ctx, payload, result) + if err != nil { + if lsErr, ok := err.(LSError); ok && (lsErr.Code == 651 || lsErr.Code == -400) || + strings.HasPrefix(err.Error(), "adnl request timeout, node") { // block not applied error + // try next node + origErr := err + if ctx, err = w.original.StickyContextNextNode(ctx); err != nil { + return fmt.Errorf("retryable error received, but failed to try with next node, "+ + "looks like all active nodes was already tried, original error: %w", origErr) + } + continue + } + } + return err + } +} + +func (w *retryClient) StickyContext(ctx context.Context) context.Context { + return w.original.StickyContext(ctx) +} + +func (w *retryClient) StickyNodeID(ctx context.Context) uint32 { + return w.original.StickyNodeID(ctx) +} + +func (w *retryClient) StickyContextNextNode(ctx context.Context) (context.Context, error) { + return w.original.StickyContextNextNode(ctx) +} diff --git a/ton/wallet/integration_test.go b/ton/wallet/integration_test.go index 2906bec0..c6bde468 100644 --- a/ton/wallet/integration_test.go +++ b/ton/wallet/integration_test.go @@ -19,7 +19,7 @@ import ( "github.com/xssnick/tonutils-go/tvm/cell" ) -var api = func() *ton.APIClient { +var api = func() ton.APIClientWrapped { client := liteclient.NewConnectionPool() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -30,10 +30,10 @@ var api = func() *ton.APIClient { panic(err) } - return ton.NewAPIClient(client) + return ton.NewAPIClient(client).WithRetry() }() -var apiMain = func() *ton.APIClient { +var apiMain = func() ton.APIClientWrapped { client := liteclient.NewConnectionPool() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -44,7 +44,7 @@ var apiMain = func() *ton.APIClient { panic(err) } - return ton.NewAPIClient(client) + return ton.NewAPIClient(client).WithRetry() }() var _seed = os.Getenv("WALLET_SEED") diff --git a/ton/wallet/wallet.go b/ton/wallet/wallet.go index db00ecce..4f8f8077 100644 --- a/ton/wallet/wallet.go +++ b/ton/wallet/wallet.go @@ -102,14 +102,13 @@ var ( ) type TonAPI interface { - WaitForBlock(seqno uint32) ton.APIClientWaiter + WaitForBlock(seqno uint32) ton.APIClientWrapped Client() ton.LiteClient CurrentMasterchainInfo(ctx context.Context) (*ton.BlockIDExt, error) GetAccount(ctx context.Context, block *ton.BlockIDExt, addr *address.Address) (*tlb.Account, error) SendExternalMessage(ctx context.Context, msg *tlb.ExternalMessage) error RunGetMethod(ctx context.Context, blockInfo *ton.BlockIDExt, addr *address.Address, method string, params ...interface{}) (*ton.ExecutionResult, error) ListTransactions(ctx context.Context, addr *address.Address, num uint32, lt uint64, txHash []byte) ([]*tlb.Transaction, error) - WaitNextMasterBlock(ctx context.Context, master *ton.BlockIDExt) (*ton.BlockIDExt, error) } type Message struct { @@ -395,7 +394,7 @@ func (w *Wallet) waitConfirmation(ctx context.Context, block *ton.BlockIDExt, ac ctx = w.api.Client().StickyContext(ctx) for time.Now().Before(till) { - blockNew, err := w.api.WaitNextMasterBlock(ctx, block) + blockNew, err := w.api.CurrentMasterchainInfo(ctx) if err != nil { continue } diff --git a/ton/wallet/wallet_test.go b/ton/wallet/wallet_test.go index 27f0db0f..0bd62aea 100644 --- a/ton/wallet/wallet_test.go +++ b/ton/wallet/wallet_test.go @@ -27,7 +27,7 @@ type MockAPI struct { extMsgSent *tlb.ExternalMessage } -func (m MockAPI) WaitForBlock(seqno uint32) ton.APIClientWaiter { +func (m MockAPI) WaitForBlock(seqno uint32) ton.APIClientWrapped { return &WaiterMock{ MGetMasterchainInfo: m.getBlockInfo, MGetAccount: m.getAccount, @@ -458,13 +458,44 @@ type WaiterMock struct { MRunGetMethod func(ctx context.Context, blockInfo *ton.BlockIDExt, addr *address.Address, method string, params ...interface{}) (*ton.ExecutionResult, error) MListTransactions func(ctx context.Context, addr *address.Address, num uint32, lt uint64, txHash []byte) ([]*tlb.Transaction, error) MGetTransaction func(ctx context.Context, block *ton.BlockIDExt, addr *address.Address, lt uint64) (*tlb.Transaction, error) + MWaitForBlock func(seqno uint32) ton.APIClientWrapped + MWithRetry func() ton.APIClientWrapped + MCurrentMasterchainInfo func(ctx context.Context) (_ *ton.BlockIDExt, err error) + MGetBlockProof func(ctx context.Context, known, target *ton.BlockIDExt) (*ton.PartialBlockProof, error) } -func (w WaiterMock) GetBlockProof(ctx context.Context, known, target *ton.BlockIDExt) (*ton.PartialBlockProof, error) { +func (w WaiterMock) SubscribeOnTransactions(workerCtx context.Context, addr *address.Address, lastProcessedLT uint64, channel chan<- *tlb.Transaction) { + //TODO implement me + panic("implement me") +} + +func (w WaiterMock) VerifyProofChain(ctx context.Context, from, to *ton.BlockIDExt) error { //TODO implement me panic("implement me") } +func (w WaiterMock) Client() ton.LiteClient { + //TODO implement me + panic("implement me") +} + +func (w WaiterMock) CurrentMasterchainInfo(ctx context.Context) (_ *ton.BlockIDExt, err error) { + return w.MCurrentMasterchainInfo(ctx) +} + +func (w WaiterMock) WaitForBlock(seqno uint32) ton.APIClientWrapped { + return w.MWaitForBlock(seqno) +} + +func (w WaiterMock) WithRetry() ton.APIClientWrapped { + return w.MWithRetry() +} + +func (w WaiterMock) GetBlockProof(ctx context.Context, known, target *ton.BlockIDExt) (*ton.PartialBlockProof, error) { + return w.MGetBlockProof(ctx, known, target) + +} + func (w WaiterMock) GetTime(ctx context.Context) (uint32, error) { return w.MGetTime(ctx) } From 13af789dcc42b6fa7b313b4c05637deaca4340c9 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 14 Aug 2023 22:48:20 +0400 Subject: [PATCH 32/38] LS Wrapper improvements & dbg --- example/block-scan/main.go | 4 ++-- ton/api.go | 14 +++++++++----- ton/block.go | 17 +++++++++-------- ton/retrier.go | 7 +++++++ ton/wallet/wallet_test.go | 11 +++++++++++ 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/example/block-scan/main.go b/example/block-scan/main.go index a2ce8433..f0bb2ca0 100644 --- a/example/block-scan/main.go +++ b/example/block-scan/main.go @@ -15,7 +15,7 @@ func getShardID(shard *ton.BlockIDExt) string { return fmt.Sprintf("%d|%d", shard.Workchain, shard.Shard) } -func getNotSeenShards(ctx context.Context, api *ton.APIClient, shard *ton.BlockIDExt, shardLastSeqno map[string]uint32) (ret []*ton.BlockIDExt, err error) { +func getNotSeenShards(ctx context.Context, api ton.APIClientWrapped, shard *ton.BlockIDExt, shardLastSeqno map[string]uint32) (ret []*ton.BlockIDExt, err error) { if no, ok := shardLastSeqno[getShardID(shard)]; ok && no == shard.SeqNo { return nil, nil } @@ -59,7 +59,7 @@ func main() { } // initialize ton api lite connection wrapper with full proof checks - api := ton.NewAPIClient(client, ton.ProofCheckPolicySecure) + api := ton.NewAPIClient(client, ton.ProofCheckPolicySecure).WithRetry() api.SetTrustedBlockFromConfig(cfg) log.Println("checking proofs since config init block, it may take near a minute...") diff --git a/ton/api.go b/ton/api.go index fbc06345..a9717216 100644 --- a/ton/api.go +++ b/ton/api.go @@ -67,6 +67,8 @@ type APIClientWrapped interface { VerifyProofChain(ctx context.Context, from, to *BlockIDExt) error WaitForBlock(seqno uint32) APIClientWrapped WithRetry() APIClientWrapped + SetTrustedBlock(block *BlockIDExt) + SetTrustedBlockFromConfig(cfg *liteclient.GlobalConfig) } type APIClient struct { @@ -102,7 +104,7 @@ func NewAPIClient(client LiteClient, proofCheckPolicy ...ProofCheckPolicy) *APIC // SetTrustedBlock - set starting point to verify master block proofs chain func (c *APIClient) SetTrustedBlock(block *BlockIDExt) { - c.trustedBlock = block.Copy() + c.root().trustedBlock = block.Copy() } // SetTrustedBlockFromConfig - same as SetTrustedBlock but takes init block from config @@ -114,8 +116,9 @@ func (c *APIClient) SetTrustedBlockFromConfig(cfg *liteclient.GlobalConfig) { // WaitForBlock - waits for the given master block seqno will be available on the requested node func (c *APIClient) WaitForBlock(seqno uint32) APIClientWrapped { return &APIClient{ - parent: c, - client: &waiterClient{original: c.client, seqno: seqno}, + parent: c, + client: &waiterClient{original: c.client, seqno: seqno}, + proofCheckPolicy: c.proofCheckPolicy, } } @@ -123,8 +126,9 @@ func (c *APIClient) WaitForBlock(seqno uint32) APIClientWrapped { // when error code 651 or -400 is received func (c *APIClient) WithRetry() APIClientWrapped { return &APIClient{ - parent: c, - client: &retryClient{original: c.client}, + parent: c, + client: &retryClient{original: c.client}, + proofCheckPolicy: c.proofCheckPolicy, } } diff --git a/ton/block.go b/ton/block.go index aba71e2d..d808d6ed 100644 --- a/ton/block.go +++ b/ton/block.go @@ -249,23 +249,24 @@ func (c *APIClient) GetMasterchainInfo(ctx context.Context) (*BlockIDExt, error) switch t := resp.(type) { case MasterchainInfo: if c.proofCheckPolicy == ProofCheckPolicySecure { - c.trustedLock.Lock() - defer c.trustedLock.Unlock() + root := c.root() + root.trustedLock.Lock() + defer root.trustedLock.Unlock() - if c.trustedBlock == nil { - if c.trustedBlock == nil { + if root.trustedBlock == nil { + if root.trustedBlock == nil { // we have no block to trust, so trust first block we get - c.trustedBlock = t.Last.Copy() + root.trustedBlock = t.Last.Copy() log.Println("[WARNING] trusted block was not set on initialization, so first block we got was considered as trusted. " + "For better security you should use SetTrustedBlock(block) method and pass there init block from config on start") } } else { - if err := c.VerifyProofChain(ctx, c.trustedBlock, t.Last); err != nil { + if err := c.VerifyProofChain(ctx, root.trustedBlock, t.Last); err != nil { return nil, fmt.Errorf("failed to verify proof chain: %w", err) } - if t.Last.SeqNo > c.trustedBlock.SeqNo { - c.trustedBlock = t.Last.Copy() + if t.Last.SeqNo > root.trustedBlock.SeqNo { + root.trustedBlock = t.Last.Copy() } } } diff --git a/ton/retrier.go b/ton/retrier.go index 1d29b71c..53444c44 100644 --- a/ton/retrier.go +++ b/ton/retrier.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/xssnick/tonutils-go/tl" + "reflect" "strings" ) @@ -13,12 +14,18 @@ type retryClient struct { func (w *retryClient) QueryLiteserver(ctx context.Context, payload tl.Serializable, result tl.Serializable) error { for { + println("TRY", reflect.ValueOf(payload).Type().String()) err := w.original.QueryLiteserver(ctx, payload, result) + if err != nil { + println("ERR", err) + if lsErr, ok := err.(LSError); ok && (lsErr.Code == 651 || lsErr.Code == -400) || strings.HasPrefix(err.Error(), "adnl request timeout, node") { // block not applied error // try next node origErr := err + println("RETRY", err) + if ctx, err = w.original.StickyContextNextNode(ctx); err != nil { return fmt.Errorf("retryable error received, but failed to try with next node, "+ "looks like all active nodes was already tried, original error: %w", origErr) diff --git a/ton/wallet/wallet_test.go b/ton/wallet/wallet_test.go index 0bd62aea..3d2d563d 100644 --- a/ton/wallet/wallet_test.go +++ b/ton/wallet/wallet_test.go @@ -6,6 +6,7 @@ import ( "crypto/ed25519" "errors" "fmt" + "github.com/xssnick/tonutils-go/liteclient" "github.com/xssnick/tonutils-go/ton" "math/big" "strings" @@ -464,6 +465,16 @@ type WaiterMock struct { MGetBlockProof func(ctx context.Context, known, target *ton.BlockIDExt) (*ton.PartialBlockProof, error) } +func (w WaiterMock) SetTrustedBlock(block *ton.BlockIDExt) { + //TODO implement me + panic("implement me") +} + +func (w WaiterMock) SetTrustedBlockFromConfig(cfg *liteclient.GlobalConfig) { + //TODO implement me + panic("implement me") +} + func (w WaiterMock) SubscribeOnTransactions(workerCtx context.Context, addr *address.Address, lastProcessedLT uint64, channel chan<- *tlb.Transaction) { //TODO implement me panic("implement me") From e3f1732f59696a6d9878208c5278dc0192fd8c94 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Mon, 14 Aug 2023 22:50:32 +0400 Subject: [PATCH 33/38] Fixed wait for block in tx confirmation wait --- ton/wallet/wallet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ton/wallet/wallet.go b/ton/wallet/wallet.go index 4f8f8077..fcf03bf2 100644 --- a/ton/wallet/wallet.go +++ b/ton/wallet/wallet.go @@ -394,7 +394,7 @@ func (w *Wallet) waitConfirmation(ctx context.Context, block *ton.BlockIDExt, ac ctx = w.api.Client().StickyContext(ctx) for time.Now().Before(till) { - blockNew, err := w.api.CurrentMasterchainInfo(ctx) + blockNew, err := w.api.WaitForBlock(block.SeqNo + 1).CurrentMasterchainInfo(ctx) if err != nil { continue } From c743a98be0273a9de7390f5e7bb2fbc22018244e Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Tue, 15 Aug 2023 10:01:29 +0400 Subject: [PATCH 34/38] Added max attempts to retrier and fixed --- ton/api.go | 13 +++++++++---- ton/retrier.go | 35 ++++++++++++++++++++++------------- ton/wallet/wallet.go | 2 +- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/ton/api.go b/ton/api.go index a9717216..d6f41724 100644 --- a/ton/api.go +++ b/ton/api.go @@ -66,7 +66,7 @@ type APIClientWrapped interface { SubscribeOnTransactions(workerCtx context.Context, addr *address.Address, lastProcessedLT uint64, channel chan<- *tlb.Transaction) VerifyProofChain(ctx context.Context, from, to *BlockIDExt) error WaitForBlock(seqno uint32) APIClientWrapped - WithRetry() APIClientWrapped + WithRetry(maxRetries ...int) APIClientWrapped SetTrustedBlock(block *BlockIDExt) SetTrustedBlockFromConfig(cfg *liteclient.GlobalConfig) } @@ -123,11 +123,16 @@ func (c *APIClient) WaitForBlock(seqno uint32) APIClientWrapped { } // WithRetry - automatically retires request to another available liteserver -// when error code 651 or -400 is received -func (c *APIClient) WithRetry() APIClientWrapped { +// when adnl timeout, or error code 651 or -400 is received. +// If maxTries > 0, limits additional attempts to this number. +func (c *APIClient) WithRetry(maxTries ...int) APIClientWrapped { + tries := 0 + if len(maxTries) > 0 { + tries = maxTries[0] + } return &APIClient{ parent: c, - client: &retryClient{original: c.client}, + client: &retryClient{original: c.client, maxRetries: tries}, proofCheckPolicy: c.proofCheckPolicy, } } diff --git a/ton/retrier.go b/ton/retrier.go index 53444c44..98f507d2 100644 --- a/ton/retrier.go +++ b/ton/retrier.go @@ -4,36 +4,45 @@ import ( "context" "fmt" "github.com/xssnick/tonutils-go/tl" - "reflect" "strings" ) type retryClient struct { - original LiteClient + maxRetries int + original LiteClient } func (w *retryClient) QueryLiteserver(ctx context.Context, payload tl.Serializable, result tl.Serializable) error { + tries := w.maxRetries for { - println("TRY", reflect.ValueOf(payload).Type().String()) err := w.original.QueryLiteserver(ctx, payload, result) + if w.maxRetries > 0 && tries == w.maxRetries { + return err + } + tries++ if err != nil { - println("ERR", err) - - if lsErr, ok := err.(LSError); ok && (lsErr.Code == 651 || lsErr.Code == -400) || - strings.HasPrefix(err.Error(), "adnl request timeout, node") { // block not applied error + if strings.HasPrefix(err.Error(), "adnl request timeout, node") { // try next node - origErr := err - println("RETRY", err) - if ctx, err = w.original.StickyContextNextNode(ctx); err != nil { - return fmt.Errorf("retryable error received, but failed to try with next node, "+ - "looks like all active nodes was already tried, original error: %w", origErr) + return fmt.Errorf("timeout error received, but failed to try with next node, "+ + "looks like all active nodes was already tried, original error: %w", err) + } + continue + } + return err + } + + if tmp, ok := result.(*tl.Serializable); ok && tmp != nil { + if lsErr, ok := (*tmp).(LSError); ok && (lsErr.Code == 651 || lsErr.Code == 652 || lsErr.Code == -400) { + if ctx, err = w.original.StickyContextNextNode(ctx); err != nil { // try next node + // no more nodes left, return as it is + return nil } continue } } - return err + return nil } } diff --git a/ton/wallet/wallet.go b/ton/wallet/wallet.go index fcf03bf2..9852bc69 100644 --- a/ton/wallet/wallet.go +++ b/ton/wallet/wallet.go @@ -394,7 +394,7 @@ func (w *Wallet) waitConfirmation(ctx context.Context, block *ton.BlockIDExt, ac ctx = w.api.Client().StickyContext(ctx) for time.Now().Before(till) { - blockNew, err := w.api.WaitForBlock(block.SeqNo + 1).CurrentMasterchainInfo(ctx) + blockNew, err := w.api.WaitForBlock(block.SeqNo + 1).GetMasterchainInfo(ctx) if err != nil { continue } From 9aa1c93008926784ebd8e72606fce20030577dd1 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Tue, 15 Aug 2023 10:10:09 +0400 Subject: [PATCH 35/38] Fixed tests --- adnl/dht/client_test.go | 2 +- liteclient/pool.go | 1 + ton/retrier.go | 5 +++++ ton/wallet/wallet_test.go | 6 +++--- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/adnl/dht/client_test.go b/adnl/dht/client_test.go index 44f53aa2..8f6d15b3 100644 --- a/adnl/dht/client_test.go +++ b/adnl/dht/client_test.go @@ -802,7 +802,7 @@ func TestClient_StoreAddressIntegration(t *testing.T) { ExpireAt: 0, } - _, _, err = dhtClient.StoreAddress(ctx, addrList, 5*time.Minute, key, 2) + _, _, err = dhtClient.StoreAddress(ctx, addrList, 12*time.Minute, key, 2) if err != nil { t.Fatal(err) } diff --git a/liteclient/pool.go b/liteclient/pool.go index 5d4a0f7f..0c186e88 100644 --- a/liteclient/pool.go +++ b/liteclient/pool.go @@ -183,6 +183,7 @@ func (c *ConnectionPool) QueryADNL(ctx context.Context, request tl.Serializable, return err } } + println("QWITH", host) _, hasDeadline := ctx.Deadline() if !hasDeadline { diff --git a/ton/retrier.go b/ton/retrier.go index 98f507d2..b9474f41 100644 --- a/ton/retrier.go +++ b/ton/retrier.go @@ -2,8 +2,10 @@ package ton import ( "context" + "encoding/json" "fmt" "github.com/xssnick/tonutils-go/tl" + "os" "strings" ) @@ -35,6 +37,9 @@ func (w *retryClient) QueryLiteserver(ctx context.Context, payload tl.Serializab if tmp, ok := result.(*tl.Serializable); ok && tmp != nil { if lsErr, ok := (*tmp).(LSError); ok && (lsErr.Code == 651 || lsErr.Code == 652 || lsErr.Code == -400) { + println("RETRY", tries, lsErr.Code) + json.NewEncoder(os.Stdout).Encode(payload) + if ctx, err = w.original.StickyContextNextNode(ctx); err != nil { // try next node // no more nodes left, return as it is return nil diff --git a/ton/wallet/wallet_test.go b/ton/wallet/wallet_test.go index 3d2d563d..4c320ae7 100644 --- a/ton/wallet/wallet_test.go +++ b/ton/wallet/wallet_test.go @@ -460,7 +460,7 @@ type WaiterMock struct { MListTransactions func(ctx context.Context, addr *address.Address, num uint32, lt uint64, txHash []byte) ([]*tlb.Transaction, error) MGetTransaction func(ctx context.Context, block *ton.BlockIDExt, addr *address.Address, lt uint64) (*tlb.Transaction, error) MWaitForBlock func(seqno uint32) ton.APIClientWrapped - MWithRetry func() ton.APIClientWrapped + MWithRetry func(x ...int) ton.APIClientWrapped MCurrentMasterchainInfo func(ctx context.Context) (_ *ton.BlockIDExt, err error) MGetBlockProof func(ctx context.Context, known, target *ton.BlockIDExt) (*ton.PartialBlockProof, error) } @@ -498,8 +498,8 @@ func (w WaiterMock) WaitForBlock(seqno uint32) ton.APIClientWrapped { return w.MWaitForBlock(seqno) } -func (w WaiterMock) WithRetry() ton.APIClientWrapped { - return w.MWithRetry() +func (w WaiterMock) WithRetry(x ...int) ton.APIClientWrapped { + return w.MWithRetry(x...) } func (w WaiterMock) GetBlockProof(ctx context.Context, known, target *ton.BlockIDExt) (*ton.PartialBlockProof, error) { From 7554a496fe9ee6594416089a5438b2274e72314b Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Tue, 15 Aug 2023 11:20:30 +0400 Subject: [PATCH 36/38] New deploy contract method --- liteclient/pool.go | 1 - ton/payments/channel.go | 7 +------ ton/payments/integration_test.go | 4 ++-- ton/retrier.go | 10 ++++------ ton/wallet/integration_test.go | 8 +------- ton/wallet/wallet.go | 31 +++++++++++++++++++++++++++++++ 6 files changed, 39 insertions(+), 22 deletions(-) diff --git a/liteclient/pool.go b/liteclient/pool.go index 0c186e88..5d4a0f7f 100644 --- a/liteclient/pool.go +++ b/liteclient/pool.go @@ -183,7 +183,6 @@ func (c *ConnectionPool) QueryADNL(ctx context.Context, request tl.Serializable, return err } } - println("QWITH", host) _, hasDeadline := ctx.Deadline() if !hasDeadline { diff --git a/ton/payments/channel.go b/ton/payments/channel.go index a82e72f1..26c84e78 100644 --- a/ton/payments/channel.go +++ b/ton/payments/channel.go @@ -51,12 +51,7 @@ func NewPaymentChannelClient(api TonApi) *Client { } } -func (c *Client) GetAsyncChannel(ctx context.Context, addr *address.Address, verify bool) (*AsyncChannel, error) { - block, err := c.api.CurrentMasterchainInfo(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get block: %w", err) - } - +func (c *Client) GetAsyncChannel(ctx context.Context, block *ton.BlockIDExt, addr *address.Address, verify bool) (*AsyncChannel, error) { acc, err := c.api.GetAccount(ctx, block, addr) if err != nil { return nil, fmt.Errorf("failed to get account: %w", err) diff --git a/ton/payments/integration_test.go b/ton/payments/integration_test.go index a289acfb..d8f2ef20 100644 --- a/ton/payments/integration_test.go +++ b/ton/payments/integration_test.go @@ -68,12 +68,12 @@ func TestClient_DeployAsyncChannel(t *testing.T) { t.Fatal(fmt.Errorf("failed to build deploy channel params: %w", err)) } - channelAddr, err := w.DeployContract(context.Background(), tlb.MustFromTON("0.02"), body, code, data, true) + channelAddr, _, block, err := w.DeployContractWaitTransaction(context.Background(), tlb.MustFromTON("0.02"), body, code, data) if err != nil { t.Fatal(fmt.Errorf("failed to deploy channel: %w", err)) } - ch, err := client.GetAsyncChannel(context.Background(), channelAddr, true) + ch, err := client.GetAsyncChannel(context.Background(), block, channelAddr, true) if err != nil { t.Fatal(fmt.Errorf("failed to get channel: %w", err)) } diff --git a/ton/retrier.go b/ton/retrier.go index b9474f41..97fcea7e 100644 --- a/ton/retrier.go +++ b/ton/retrier.go @@ -2,10 +2,8 @@ package ton import ( "context" - "encoding/json" "fmt" "github.com/xssnick/tonutils-go/tl" - "os" "strings" ) @@ -36,10 +34,10 @@ func (w *retryClient) QueryLiteserver(ctx context.Context, payload tl.Serializab } if tmp, ok := result.(*tl.Serializable); ok && tmp != nil { - if lsErr, ok := (*tmp).(LSError); ok && (lsErr.Code == 651 || lsErr.Code == 652 || lsErr.Code == -400) { - println("RETRY", tries, lsErr.Code) - json.NewEncoder(os.Stdout).Encode(payload) - + if lsErr, ok := (*tmp).(LSError); ok && (lsErr.Code == 651 || + lsErr.Code == 652 || + lsErr.Code == -400 || + (lsErr.Code == 0 && strings.Contains(lsErr.Text, "Failed to get account state"))) { if ctx, err = w.original.StickyContextNextNode(ctx); err != nil { // try next node // no more nodes left, return as it is return nil diff --git a/ton/wallet/integration_test.go b/ton/wallet/integration_test.go index c6bde468..3bb5b5ef 100644 --- a/ton/wallet/integration_test.go +++ b/ton/wallet/integration_test.go @@ -159,18 +159,12 @@ func TestWallet_DeployContract(t *testing.T) { codeBytes, _ := hex.DecodeString("b5ee9c72410104010020000114ff00f4a413f4bcf2c80b010203844003020009a1b63c43510007a0000061d2421bb1") code, _ := cell.FromBOC(codeBytes) - addr, err := w.DeployContract(ctx, tlb.MustFromTON("0.005"), cell.BeginCell().EndCell(), code, cell.BeginCell().MustStoreUInt(rand.Uint64(), 64).EndCell(), true) + addr, _, block, err := w.DeployContractWaitTransaction(ctx, tlb.MustFromTON("0.005"), cell.BeginCell().EndCell(), code, cell.BeginCell().MustStoreUInt(rand.Uint64(), 64).EndCell()) if err != nil { t.Fatal("deploy err:", err) } t.Logf("contract address: %s", addr.String()) - block, err := api.CurrentMasterchainInfo(ctx) - if err != nil { - t.Fatal("CurrentMasterchainInfo err:", err.Error()) - return - } - res, err := api.RunGetMethod(ctx, block, addr, "dappka", 5, 10) if err != nil { t.Fatal("run err:", err) diff --git a/ton/wallet/wallet.go b/ton/wallet/wallet.go index 9852bc69..d1bb69f5 100644 --- a/ton/wallet/wallet.go +++ b/ton/wallet/wallet.go @@ -625,6 +625,37 @@ func (w *Wallet) transfer(ctx context.Context, to *address.Address, amount tlb.C return w.Send(ctx, transfer, waitConfirmation...) } +func (w *Wallet) DeployContractWaitTransaction(ctx context.Context, amount tlb.Coins, msgBody, contractCode, contractData *cell.Cell) (*address.Address, *tlb.Transaction, *ton.BlockIDExt, error) { + state := &tlb.StateInit{ + Data: contractData, + Code: contractCode, + } + + stateCell, err := tlb.ToCell(state) + if err != nil { + return nil, nil, nil, err + } + + addr := address.NewAddress(0, 0, stateCell.Hash()) + + tx, block, err := w.SendWaitTransaction(ctx, &Message{ + Mode: 1 + 2, + InternalMessage: &tlb.InternalMessage{ + IHRDisabled: true, + Bounce: false, + DstAddr: addr, + Amount: amount, + Body: msgBody, + StateInit: state, + }, + }) + if err != nil { + return nil, nil, nil, err + } + return addr, tx, block, nil +} + +// Deprecated: use DeployContractWaitTransaction func (w *Wallet) DeployContract(ctx context.Context, amount tlb.Coins, msgBody, contractCode, contractData *cell.Cell, waitConfirmation ...bool) (*address.Address, error) { state := &tlb.StateInit{ Data: contractData, From 06f040eddbc1d68afa1b39ffd4e69bfe7b3391e5 Mon Sep 17 00:00:00 2001 From: Oleg Baranov Date: Wed, 16 Aug 2023 11:28:04 +0400 Subject: [PATCH 37/38] Updated examples and DecryptCommentCell accept sender --- example/accept-payments/main.go | 4 +-- example/account-state/main.go | 2 +- example/deploy-nft-collection/main.go | 4 +-- example/detailed/main.go | 2 +- example/dns/main.go | 2 +- example/external-message/main.go | 4 --- example/highload-wallet/main.go | 2 +- example/jettons/main.go | 2 +- example/send-to-contract/main.go | 4 +-- example/wallet/main.go | 40 +++++++++++++++++---------- tlb/shard.go | 23 --------------- ton/api.go | 4 +-- ton/wallet/wallet.go | 14 ++++++---- ton/wallet/wallet_test.go | 11 ++++++-- 14 files changed, 56 insertions(+), 62 deletions(-) diff --git a/example/accept-payments/main.go b/example/accept-payments/main.go index ce06a2aa..97765f25 100644 --- a/example/accept-payments/main.go +++ b/example/accept-payments/main.go @@ -26,10 +26,10 @@ func main() { } // initialize ton api lite connection wrapper with full proof checks - api := ton.NewAPIClient(client, ton.ProofCheckPolicySecure) + api := ton.NewAPIClient(client, ton.ProofCheckPolicySecure).WithRetry() api.SetTrustedBlockFromConfig(cfg) - log.Println("checking proofs since config init block, it may take near a minute...") + log.Println("fetching and checking proofs since config init block, it may take near a minute...") master, err := api.CurrentMasterchainInfo(context.Background()) // we fetch block just to trigger chain proof check if err != nil { log.Fatalln("get masterchain info err: ", err.Error()) diff --git a/example/account-state/main.go b/example/account-state/main.go index 26e62281..f60deccd 100644 --- a/example/account-state/main.go +++ b/example/account-state/main.go @@ -21,7 +21,7 @@ func main() { } // initialize ton api lite connection wrapper - api := ton.NewAPIClient(client) + api := ton.NewAPIClient(client, ton.ProofCheckPolicyFast).WithRetry() // if we want to route all requests to the same node, we can use it ctx := client.StickyContext(context.Background()) diff --git a/example/deploy-nft-collection/main.go b/example/deploy-nft-collection/main.go index f6a75a0c..32eb2ff9 100644 --- a/example/deploy-nft-collection/main.go +++ b/example/deploy-nft-collection/main.go @@ -26,7 +26,7 @@ func main() { } // initialize ton api lite connection wrapper - api := ton.NewAPIClient(client) + api := ton.NewAPIClient(client).WithRetry() w := getWallet(api) log.Println("Deploy wallet:", w.Address().String()) @@ -43,7 +43,7 @@ func main() { fmt.Println("Deployed contract addr:", addr.String()) } -func getWallet(api *ton.APIClient) *wallet.Wallet { +func getWallet(api ton.APIClientWrapped) *wallet.Wallet { words := strings.Split("birth pattern then forest walnut then phrase walnut fan pumpkin pattern then cluster blossom verify then forest velvet pond fiction pattern collect then then", " ") w, err := wallet.FromSeed(api, words, wallet.V3) if err != nil { diff --git a/example/detailed/main.go b/example/detailed/main.go index 7f4692a8..76077b1f 100644 --- a/example/detailed/main.go +++ b/example/detailed/main.go @@ -32,7 +32,7 @@ func main() { } // initialize ton api lite connection wrapper - api := ton.NewAPIClient(client) + api := ton.NewAPIClient(client).WithRetry() // we need fresh block info to run get methods b, err := api.CurrentMasterchainInfo(context.Background()) diff --git a/example/dns/main.go b/example/dns/main.go index a1ca90f2..c81986ad 100644 --- a/example/dns/main.go +++ b/example/dns/main.go @@ -21,7 +21,7 @@ func main() { ctx := client.StickyContext(context.Background()) // initialize ton api lite connection wrapper - api := ton.NewAPIClient(client) + api := ton.NewAPIClient(client).WithRetry() // get root dns address from network config root, err := dns.RootContractAddr(api) diff --git a/example/external-message/main.go b/example/external-message/main.go index b197d8c3..0675fcaf 100644 --- a/example/external-message/main.go +++ b/example/external-message/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "encoding/hex" "log" "github.com/xssnick/tonutils-go/address" @@ -85,9 +84,6 @@ func main() { Body: data, } - msgCell, _ := tlb.ToCell(msg) - println("HASH", hex.EncodeToString(msgCell.Hash())) - err = api.SendExternalMessage(ctx, msg) if err != nil { // FYI: it can fail if not enough balance on contract diff --git a/example/highload-wallet/main.go b/example/highload-wallet/main.go index 7756ef25..dfac3632 100644 --- a/example/highload-wallet/main.go +++ b/example/highload-wallet/main.go @@ -23,7 +23,7 @@ func main() { return } - api := ton.NewAPIClient(client) + api := ton.NewAPIClient(client, ton.ProofCheckPolicyFast).WithRetry() // seed words of account, you can generate them with any wallet or using wallet.NewSeed() method words := strings.Split("birth pattern then forest walnut then phrase walnut fan pumpkin pattern then cluster blossom verify then forest velvet pond fiction pattern collect then then", " ") diff --git a/example/jettons/main.go b/example/jettons/main.go index 72a6ecb7..92951a55 100644 --- a/example/jettons/main.go +++ b/example/jettons/main.go @@ -24,7 +24,7 @@ func main() { ctx := client.StickyContext(context.Background()) // initialize ton api lite connection wrapper - api := ton.NewAPIClient(client) + api := ton.NewAPIClient(client).WithRetry() tokenContract := address.MustParseAddr("EQBCFwW8uFUh-amdRmNY9NyeDEaeDYXd9ggJGsicpqVcHq7B") master := jetton.NewJettonMasterClient(api, tokenContract) diff --git a/example/send-to-contract/main.go b/example/send-to-contract/main.go index 494ff0e3..fc76a8c9 100644 --- a/example/send-to-contract/main.go +++ b/example/send-to-contract/main.go @@ -25,7 +25,7 @@ func main() { return } - api := ton.NewAPIClient(client) + api := ton.NewAPIClient(client).WithRetry() // seed words of account, you can generate them with any wallet or using wallet.NewSeed() method words := strings.Split("birth pattern then forest walnut then phrase walnut fan pumpkin pattern then cluster blossom verify then forest velvet pond fiction pattern collect then then", " ") @@ -53,7 +53,7 @@ func main() { if balance.Nano().Uint64() >= 3000000 { // create transaction body cell, depends on what contract needs, just random example here body := cell.BeginCell(). - MustStoreUInt(0x123abc55, 32). // op code + MustStoreUInt(0x123abc55, 32). // op code MustStoreUInt(rand.Uint64(), 64). // query id // payload: MustStoreAddr(address.MustParseAddr("EQAbMQzuuGiCne0R7QEj9nrXsjM7gNjeVmrlBZouyC-SCLlO")). diff --git a/example/wallet/main.go b/example/wallet/main.go index db29944a..286f9f60 100644 --- a/example/wallet/main.go +++ b/example/wallet/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/base64" "log" "strings" @@ -15,14 +16,24 @@ import ( func main() { client := liteclient.NewConnectionPool() - // connect to mainnet lite server - err := client.AddConnection(context.Background(), "135.181.140.212:13206", "K0t3+IWLOXHYMvMcrGZDPs+pn58a17LFbnXoQkKc2xw=") + // get config + cfg, err := liteclient.GetConfigFromUrl(context.Background(), "https://ton.org/global.config.json") + if err != nil { + log.Fatalln("get config err: ", err.Error()) + return + } + + // connect to mainnet lite servers + err = client.AddConnectionsFromConfig(context.Background(), cfg) if err != nil { log.Fatalln("connection err: ", err.Error()) return } - api := ton.NewAPIClient(client) + // api client with full proof checks + api := ton.NewAPIClient(client, ton.ProofCheckPolicySecure).WithRetry() + api.SetTrustedBlockFromConfig(cfg) + // bound all requests to single ton node ctx := client.StickyContext(context.Background()) @@ -37,11 +48,13 @@ func main() { log.Println("wallet address:", w.Address()) - block, err := api.CurrentMasterchainInfo(ctx) + log.Println("fetching and checking proofs since config init block, it may take near a minute...") + block, err := api.CurrentMasterchainInfo(context.Background()) if err != nil { - log.Fatalln("CurrentMasterchainInfo err:", err.Error()) + log.Fatalln("get masterchain info err: ", err.Error()) return } + log.Println("master proof checks are completed successfully, now communication is 100% safe!") balance, err := w.GetBalance(ctx, block) if err != nil { @@ -54,21 +67,19 @@ func main() { log.Println("sending transaction and waiting for confirmation...") - // if destination wallet is not initialized you should use TransferNoBounce - // regular Transfer has bounce flag, and TONs may be returned. + // if destination wallet is not initialized (or you don't care) + // you should set bounce to true to not get money back + bounce := false - // err = w.TransferNoBounce(ctx, addr, tlb.MustFromTON("0.003"), - err = w.Transfer(ctx, addr, tlb.MustFromTON("0.003"), - "Hello from tonutils-go!", true) + transfer, err := w.BuildTransfer(addr, tlb.MustFromTON("0.003"), bounce, "Hello from tonutils-go!") if err != nil { log.Fatalln("Transfer err:", err.Error()) return } - // update chain info - block, err = api.CurrentMasterchainInfo(ctx) + tx, block, err := w.SendWaitTransaction(ctx, transfer) if err != nil { - log.Fatalln("CurrentMasterchainInfo err:", err.Error()) + log.Fatalln("SendWaitTransaction err:", err.Error()) return } @@ -78,7 +89,8 @@ func main() { return } - log.Println("transaction sent, balance left:", balance.String()) + log.Printf("transaction confirmed at block %d, hash: %s balance left: %s", block.SeqNo, + base64.StdEncoding.EncodeToString(tx.Hash), balance.String()) return } diff --git a/tlb/shard.go b/tlb/shard.go index 72ec96fe..972b791d 100644 --- a/tlb/shard.go +++ b/tlb/shard.go @@ -39,29 +39,6 @@ type McStateExtra struct { GlobalBalance CurrencyCollection `tlb:"."` } -/* -flags:(## 16) { flags <= 1 } - validator_info:ValidatorInfo - prev_blocks:OldMcBlocksInfo - after_key_block:Bool - last_key_block:(Maybe ExtBlkRef) - block_create_stats:(flags . 0)?BlockCreateStats - -validator_info$_ - validator_list_hash_short:uint32 - catchain_seqno:uint32 - nx_cc_updated:Bool -= ValidatorInfo; - -ext_blk_ref$_ end_lt:uint64 - seq_no:uint32 root_hash:bits256 file_hash:bits256 - = ExtBlkRef; - -_ key:Bool max_end_lt:uint64 = KeyMaxLt; -_ key:Bool blk_ref:ExtBlkRef = KeyExtBlkRef; - -*/ - type KeyExtBlkRef struct { IsKey bool `tlb:"bool"` BlkRef ExtBlkRef `tlb:"."` diff --git a/ton/api.go b/ton/api.go index d6f41724..c480bfe3 100644 --- a/ton/api.go +++ b/ton/api.go @@ -20,7 +20,7 @@ type ProofCheckPolicy int const ( ProofCheckPolicyUnsafe ProofCheckPolicy = iota - ProofCheckPolicyFastWithoutMasterBlockChecks + ProofCheckPolicyFast // Without master block checks ProofCheckPolicySecure ) @@ -90,7 +90,7 @@ type masterInfo struct { } func NewAPIClient(client LiteClient, proofCheckPolicy ...ProofCheckPolicy) *APIClient { - policy := ProofCheckPolicyFastWithoutMasterBlockChecks + policy := ProofCheckPolicyFast if len(proofCheckPolicy) > 0 { policy = proofCheckPolicy[0] } diff --git a/ton/wallet/wallet.go b/ton/wallet/wallet.go index d1bb69f5..8e4022cb 100644 --- a/ton/wallet/wallet.go +++ b/ton/wallet/wallet.go @@ -504,7 +504,7 @@ func CreateCommentCell(text string) (*cell.Cell, error) { const EncryptedCommentOpcode = 0x2167da4b -func DecryptCommentCell(commentCell *cell.Cell, ourKey ed25519.PrivateKey, theirKey ed25519.PublicKey) ([]byte, error) { +func DecryptCommentCell(commentCell *cell.Cell, sender *address.Address, ourKey ed25519.PrivateKey, theirKey ed25519.PublicKey) ([]byte, error) { slc := commentCell.BeginParse() op, err := slc.LoadUInt(32) if err != nil { @@ -560,6 +560,13 @@ func DecryptCommentCell(commentCell *cell.Cell, ourKey ed25519.PrivateKey, their if data[0] > 31 { return nil, fmt.Errorf("invalid prefix size") } + + h = hmac.New(sha512.New, []byte(sender.String())) + h.Write(data) + if !bytes.Equal(msgKey, h.Sum(nil)[:16]) { + return nil, fmt.Errorf("incorrect msg key") + } + return data[data[0]:], nil } @@ -585,11 +592,6 @@ func CreateEncryptedCommentCell(text string, senderAddr *address.Address, ourKey h.Write(data) msgKey := h.Sum(nil)[:16] - /* msgKey := make([]byte, 16) - if _, err = rand.Read(msgKey); err != nil { - return nil, fmt.Errorf("rand gen err: %w", err) - }*/ - h = hmac.New(sha512.New, sharedKey) h.Write(msgKey) x := h.Sum(nil) diff --git a/ton/wallet/wallet_test.go b/ton/wallet/wallet_test.go index 4c320ae7..659f700c 100644 --- a/ton/wallet/wallet_test.go +++ b/ton/wallet/wallet_test.go @@ -568,14 +568,21 @@ func TestCreateEncryptedCommentCell(t *testing.T) { } msg := randString(200) + sender := address.MustParseAddr("EQC9bWZd29foipyPOGWlVNVCQzpGAjvi1rGWF7EbNcSVClpA") - c, err := CreateEncryptedCommentCell(msg, address.MustParseAddr("EQC9bWZd29foipyPOGWlVNVCQzpGAjvi1rGWF7EbNcSVClpA"), priv1, pub2) + c, err := CreateEncryptedCommentCell(msg, sender, priv1, pub2) if err != nil { t.Fatal(err) return } - data, err := DecryptCommentCell(c, priv2, pub1) + data, err := DecryptCommentCell(c, address.MustParseAddr("EQDnYZIpTwo9RN_84KZX3qIkLVIUJSo8d1yz1vMlKAp2uRtK"), priv2, pub1) + if err == nil || err.Error() != "incorrect msg key" { + t.Fatal("should be error incorrect msg key, but it is:", err) + return + } + + data, err = DecryptCommentCell(c, sender, priv2, pub1) if err != nil { t.Fatal(err) return From 4110def728f9671380deaaad617f614ed6be3f8c Mon Sep 17 00:00:00 2001 From: Coverage Date: Wed, 16 Aug 2023 07:32:01 +0000 Subject: [PATCH 38/38] Updated coverage badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 917a9752..57805b58 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Based on TON][ton-svg]][ton] -![Coverage](https://img.shields.io/badge/Coverage-72.3%25-brightgreen) +![Coverage](https://img.shields.io/badge/Coverage-73.4%25-brightgreen) Golang library for interacting with TON blockchain.