Merge pull request #1379 from onetechnical/onetechnical/relbeta2.1.2

Onetechnical/relbeta2.1.2

.travis.yml

@@ -1,6 +1,4 @@
 dist: bionic
-go:
-  - "1.12"
 go_import_path: github.com/algorand/go-algorand
 language: go
 

Makefile

@@ -133,7 +133,7 @@ KMD_API_SWAGGER_INJECT := daemon/kmd/lib/kmdapi/bundledSpecInject.go
 
 $(KMD_API_SWAGGER_SPEC): $(KMD_API_FILES) crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a
 	cd daemon/kmd/lib/kmdapi && \
-		python genSwaggerWrappers.py $(KMD_API_SWAGGER_WRAPPER)
+		python3 genSwaggerWrappers.py $(KMD_API_SWAGGER_WRAPPER)
 	cd daemon/kmd && \
 		PATH=$(GOPATH1)/bin:$$PATH \
 		go generate ./...

README.md

@@ -181,7 +181,7 @@ A number of packages provide utilities for the various components:
   - `util` contains a variety of utilities, including a codec, a sqlite wrapper,
     a goroutine pool, a timer interface, node metrics, and more.
 
-`test` contains end-to-end tests for the above components.
+`test` ([README](test/README.md)) contains end-to-end tests and utilities for the above components.
 
 
 ## License

agreement/gossip/network_test.go

@@ -18,6 +18,7 @@ package gossip
 
 import (
 	"context"
+	"net"
 	"net/http"
 	"sync"
 	"sync/atomic"
@@ -153,6 +154,9 @@ func (w *whiteholeNetwork) GetPeers(options ...network.PeerOption) []network.Pee
 }
 func (w *whiteholeNetwork) RegisterHTTPHandler(path string, handler http.Handler) {
 }
+func (w *whiteholeNetwork) GetHTTPRequestConnection(request *http.Request) (conn net.Conn) {
+	return nil
+}
 
 func (w *whiteholeNetwork) Start() {
 	w.quit = make(chan struct{})

buildnumber.dat

@@ -1 +1 @@
-1
+2

cmd/tealdbg/README.md

@@ -152,6 +152,15 @@ If default/empty local and global state are OK for the application, the use `--p
 to automatically create necessary balance records for the application(s) so that `app_` opcodes
 do not fail due to absent data in ledger.
 
+### Indexer Support
+
+You can also supply balance records through an indexer https://github.com/algorand/indexer.
+Specify the indexer api endpoint, round number at which to fetch balance records, and an api token if necessary.
+
+```
+$ tealdbg debug myprog.teal --round roundnumber -i apiendpoint --indexer-token token
+```
+
 ### Execution mode
 
 Execution mode, either **signature** or **application** matches to **Algod**'s evaluation mode

cmd/tealdbg/local.go

@@ -349,7 +349,7 @@ func (r *LocalRunner) Setup(dp *DebugParams) (err error) {
 				ledger, states, err = makeAppLedger(
 					balances, r.txnGroup, dp.GroupIndex,
 					r.proto, dp.Round, dp.LatestTimestamp, appIdx,
-					dp.Painless,
+					dp.Painless, dp.IndexerURL, dp.IndexerToken,
 				)
 				if err != nil {
 					return
@@ -387,7 +387,7 @@ func (r *LocalRunner) Setup(dp *DebugParams) (err error) {
 					ledger, states, err = makeAppLedger(
 						balances, r.txnGroup, gi,
 						r.proto, dp.Round, dp.LatestTimestamp,
-						appIdx, dp.Painless,
+						appIdx, dp.Painless, dp.IndexerURL, dp.IndexerToken,
 					)
 					if err != nil {
 						return
@@ -421,7 +421,7 @@ func (r *LocalRunner) Setup(dp *DebugParams) (err error) {
 							ledger, states, err = makeAppLedger(
 								balances, r.txnGroup, gi,
 								r.proto, dp.Round, dp.LatestTimestamp,
-								appIdx, dp.Painless,
+								appIdx, dp.Painless, dp.IndexerURL, dp.IndexerToken,
 							)
 							if err != nil {
 								return

cmd/tealdbg/localLedger.go

@@ -17,17 +17,44 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
+	"net/http"
 
 	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/crypto"
+	"github.com/algorand/go-algorand/daemon/algod/api/server/v2"
+	"github.com/algorand/go-algorand/daemon/algod/api/server/v2/generated"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/transactions"
 	"github.com/algorand/go-algorand/data/transactions/logic"
 	"github.com/algorand/go-algorand/ledger"
 )
 
+// AccountIndexerResponse represents the Account Response object from querying indexer
+type AccountIndexerResponse struct {
+	// Account information at a given round.
+	//
+	// Definition:
+	// data/basics/userBalance.go : AccountData
+	Account generated.Account `json:"account"`
+
+	// Round at which the results were computed.
+	CurrentRound uint64 `json:"current-round"`
+}
+
+// ApplicationIndexerResponse represents the Application Response object from querying indexer
+type ApplicationIndexerResponse struct {
+
+	// Application index and its parameters
+	Application generated.Application `json:"application,omitempty"`
+
+	// Round at which the results were computed.
+	CurrentRound uint64 `json:"current-round"`
+}
+
 type balancesAdapter struct {
 	balances   map[basics.Address]basics.AccountData
 	txnGroup   []transactions.SignedTxn
@@ -39,7 +66,7 @@ type balancesAdapter struct {
 func makeAppLedger(
 	balances map[basics.Address]basics.AccountData, txnGroup []transactions.SignedTxn,
 	groupIndex int, proto config.ConsensusParams, round uint64, latestTimestamp int64,
-	appIdx basics.AppIndex, painless bool,
+	appIdx basics.AppIndex, painless bool, indexerURL string, indexerToken string,
 ) (logic.LedgerForLogic, appState, error) {
 
 	if groupIndex >= len(txnGroup) {
@@ -53,6 +80,30 @@ func makeAppLedger(
 	apps := []basics.AppIndex{appIdx}
 	apps = append(apps, txn.Txn.ForeignApps...)
 
+	// populate balances from the indexer if not already
+	if indexerURL != "" {
+		for _, acc := range accounts {
+			// only populate from indexer if balance record not specified
+			if _, ok := balances[acc]; !ok {
+				var err error
+				balances[acc], err = getBalanceFromIndexer(indexerURL, indexerToken, acc, round)
+				if err != nil {
+					return nil, appState{}, err
+				}
+			}
+		}
+		for _, app := range apps {
+			creator, err := getAppCreatorFromIndexer(indexerURL, indexerToken, app)
+			if err != nil {
+				return nil, appState{}, err
+			}
+			balances[creator], err = getBalanceFromIndexer(indexerURL, indexerToken, creator, round)
+			if err != nil {
+				return nil, appState{}, err
+			}
+		}
+	}
+
 	ba := &balancesAdapter{
 		balances:   balances,
 		txnGroup:   txnGroup,
@@ -130,6 +181,60 @@ func makeAppLedger(
 	return ledger, states, err
 }
 
+func getAppCreatorFromIndexer(indexerURL string, indexerToken string, app basics.AppIndex) (basics.Address, error) {
+	queryString := fmt.Sprintf("%s/v2/applications/%d", indexerURL, app)
+	client := &http.Client{}
+	request, err := http.NewRequest("GET", queryString, nil)
+	request.Header.Set("X-Indexer-API-Token", indexerToken)
+	resp, err := client.Do(request)
+	if err != nil {
+		return basics.Address{}, fmt.Errorf("application request error: %s", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		msg, _ := ioutil.ReadAll(resp.Body)
+		return basics.Address{}, fmt.Errorf("application response error: %s, status code: %d, request: %s", string(msg), resp.StatusCode, queryString)
+	}
+	var appResp ApplicationIndexerResponse
+	err = json.NewDecoder(resp.Body).Decode(&appResp)
+	if err != nil {
+		return basics.Address{}, fmt.Errorf("application response decode error: %s", err)
+	}
+
+	creator, err := basics.UnmarshalChecksumAddress(appResp.Application.Params.Creator)
+
+	if err != nil {
+		return basics.Address{}, fmt.Errorf("UnmarshalChecksumAddress error: %s", err)
+	}
+	return creator, nil
+}
+
+func getBalanceFromIndexer(indexerURL string, indexerToken string, account basics.Address, round uint64) (basics.AccountData, error) {
+	queryString := fmt.Sprintf("%s/v2/accounts/%s?round=%d", indexerURL, account, round)
+	client := &http.Client{}
+	request, err := http.NewRequest("GET", queryString, nil)
+	request.Header.Set("X-Indexer-API-Token", indexerToken)
+	resp, err := client.Do(request)
+	if err != nil {
+		return basics.AccountData{}, fmt.Errorf("account request error: %s", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		msg, _ := ioutil.ReadAll(resp.Body)
+		return basics.AccountData{}, fmt.Errorf("account response error: %s, status code: %d, request: %s", string(msg), resp.StatusCode, queryString)
+	}
+	var accountResp AccountIndexerResponse
+	err = json.NewDecoder(resp.Body).Decode(&accountResp)
+	if err != nil {
+		return basics.AccountData{}, fmt.Errorf("account response decode error: %s", err)
+	}
+	balance, err := v2.AccountToAccountData(&accountResp.Account)
+	if err != nil {
+		return basics.AccountData{}, fmt.Errorf("AccountToAccountData error: %s", err)
+	}
+	return balance, nil
+}
+
 func makeSchemas() basics.StateSchemas {
 	return basics.StateSchemas{
 		LocalStateSchema:  makeLocalSchema(),

cmd/tealdbg/local_test.go

@@ -17,10 +17,15 @@
 package main
 
 import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 
+	"github.com/algorand/go-algorand/daemon/algod/api/server/v2"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/transactions"
 	"github.com/algorand/go-algorand/data/transactions/logic"
@@ -158,6 +163,11 @@ func makeSampleBalanceRecord(addr basics.Address, assetIdx basics.AssetIndex, ap
 			Total:     100,
 			UnitName:  "tok",
 			AssetName: "asset",
+			Manager:   addr,
+			Reserve:   addr,
+			Freeze:    addr,
+			Clawback:  addr,
+			URL:       "http://127.0.0.1/8000",
 		},
 	}
 	br.Assets = map[basics.AssetIndex]basics.AssetHolding{
@@ -923,3 +933,127 @@ func TestLocalLedger(t *testing.T) {
 	tkv, err = ledger.AppLocalState(payTxn.Txn.Receiver, appIdx)
 	a.Error(err)
 }
+
+func TestLocalLedgerIndexer(t *testing.T) {
+	a := require.New(t)
+
+	sender, err := basics.UnmarshalChecksumAddress("47YPQTIGQEO7T4Y4RWDYWEKV6RTR2UNBQXBABEEGM72ESWDQNCQ52OPASU")
+	a.NoError(err)
+	// make balance records
+	appIdx := basics.AppIndex(100)
+	assetIdx := basics.AssetIndex(50)
+	brs := makeSampleBalanceRecord(sender, assetIdx, appIdx)
+	//balanceBlob := protocol.EncodeMsgp(&brs)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		accountPath := "/v2/accounts/"
+		applicationPath := "/v2/applications/"
+		switch {
+		case strings.HasPrefix(r.URL.Path, accountPath):
+			w.WriteHeader(200)
+			if r.URL.Path[len(accountPath):] == brs.Addr.String() {
+				account, err := v2.AccountDataToAccount(brs.Addr.String(), &brs.AccountData, map[basics.AssetIndex]string{}, 100, basics.MicroAlgos{Raw: 0})
+				a.NoError(err)
+				accountResponse := AccountIndexerResponse{Account: account, CurrentRound: 100}
+				response, err := json.Marshal(accountResponse)
+				a.NoError(err)
+				w.Write(response)
+			}
+		case strings.HasPrefix(r.URL.Path, applicationPath):
+			w.WriteHeader(200)
+			if r.URL.Path[len(applicationPath):] == strconv.FormatUint(uint64(appIdx), 10) {
+				appParams := brs.AppParams[appIdx]
+				app := v2.AppParamsToApplication(sender.String(), appIdx, &appParams)
+				a.NoError(err)
+				applicationResponse := ApplicationIndexerResponse{Application: app, CurrentRound: 100}
+				response, err := json.Marshal(applicationResponse)
+				a.NoError(err)
+				w.Write(response)
+			}
+		default:
+			w.WriteHeader(404)
+		}
+	}))
+	defer srv.Close()
+
+	// make transaction group: app call + sample payment
+	appTxn := transactions.SignedTxn{
+		Txn: transactions.Transaction{
+			Header: transactions.Header{
+				Sender: sender,
+			},
+			ApplicationCallTxnFields: transactions.ApplicationCallTxnFields{
+				ApplicationID: appIdx,
+			},
+		},
+	}
+
+	var payTxn transactions.SignedTxn
+	err = protocol.DecodeJSON([]byte(txnSample), &payTxn)
+	a.NoError(err)
+
+	txnBlob := protocol.EncodeMsgp(&appTxn)
+	txnBlob = append(txnBlob, protocol.EncodeMsgp(&payTxn)...)
+
+	l := LocalRunner{}
+	dp := DebugParams{
+		ProgramNames:    []string{"test"},
+		ProgramBlobs:    [][]byte{{1}},
+		TxnBlob:         txnBlob,
+		IndexerURL:      srv.URL,
+		RunMode:         "application",
+		GroupIndex:      0,
+		Round:           100,
+		LatestTimestamp: 333,
+	}
+
+	err = l.Setup(&dp)
+	a.NoError(err)
+	a.Equal(2, len(l.txnGroup))
+	a.Equal(1, len(l.runs))
+	a.Equal(0, l.runs[0].groupIndex)
+	a.NotNil(l.runs[0].eval)
+	a.Equal([]byte{1}, l.runs[0].program)
+	a.NotNil(l.runs[0].ledger)
+	a.NotEqual(
+		reflect.ValueOf(logic.Eval).Pointer(),
+		reflect.ValueOf(l.runs[0].eval).Pointer(),
+	)
+	ledger := l.runs[0].ledger
+	a.Equal(basics.Round(100), ledger.Round())
+	a.Equal(int64(333), ledger.LatestTimestamp())
+
+	balance, err := ledger.Balance(sender)
+	a.NoError(err)
+	a.Equal(basics.MicroAlgos{Raw: 500000000}, balance)
+
+	holdings, err := ledger.AssetHolding(sender, assetIdx)
+	a.NoError(err)
+	a.Equal(basics.AssetHolding{Amount: 10, Frozen: false}, holdings)
+	holdings, err = ledger.AssetHolding(sender, assetIdx+1)
+	a.Error(err)
+
+	params, err := ledger.AssetParams(assetIdx)
+	a.NoError(err)
+	a.Equal(uint64(100), params.Total)
+	a.Equal("tok", params.UnitName)
+
+	tkv, err := ledger.AppGlobalState(0)
+	a.NoError(err)
+	a.Equal(uint64(2), tkv["gkeyint"].Uint)
+	tkv, err = ledger.AppGlobalState(appIdx)
+	a.NoError(err)
+	a.Equal("global", tkv["gkeybyte"].Bytes)
+	tkv, err = ledger.AppGlobalState(appIdx + 1)
+	a.Error(err)
+
+	tkv, err = ledger.AppLocalState(sender, 0)
+	a.NoError(err)
+	a.Equal(uint64(1), tkv["lkeyint"].Uint)
+	tkv, err = ledger.AppLocalState(sender, appIdx)
+	a.NoError(err)
+	a.Equal("local", tkv["lkeybyte"].Bytes)
+	tkv, err = ledger.AppLocalState(sender, appIdx+1)
+	a.Error(err)
+	tkv, err = ledger.AppLocalState(payTxn.Txn.Receiver, appIdx)
+	a.Error(err)
+}