diff --git a/agreement/fuzzer/networkFacade_test.go b/agreement/fuzzer/networkFacade_test.go index 131361e3e7..804fb1e7ff 100644 --- a/agreement/fuzzer/networkFacade_test.go +++ b/agreement/fuzzer/networkFacade_test.go @@ -70,9 +70,16 @@ type NetworkFacade struct { rand *rand.Rand timeoutAtInitOnce sync.Once timeoutAtInitWait sync.WaitGroup - peerToNode map[network.Peer]int + peerToNode map[*facadePeer]int } +type facadePeer struct { + id int + net network.GossipNode +} + +func (p *facadePeer) GetNetwork() network.GossipNode { return p.net } + // MakeNetworkFacade creates a facade with a given nodeID. func MakeNetworkFacade(fuzzer *Fuzzer, nodeID int) *NetworkFacade { n := &NetworkFacade{ @@ -83,12 +90,12 @@ func MakeNetworkFacade(fuzzer *Fuzzer, nodeID int) *NetworkFacade { eventsQueues: make(map[string]int), eventsQueuesCh: make(chan int, 1000), rand: rand.New(rand.NewSource(int64(nodeID))), - peerToNode: make(map[network.Peer]int, fuzzer.nodesCount), + peerToNode: make(map[*facadePeer]int, fuzzer.nodesCount), debugMessages: false, } n.timeoutAtInitWait.Add(1) for i := 0; i < fuzzer.nodesCount; i++ { - n.peerToNode[network.Peer(new(int))] = i + n.peerToNode[&facadePeer{id: i, net: n}] = i } return n } @@ -179,7 +186,7 @@ func (n *NetworkFacade) WaitForEventsQueue(cleared bool) { func (n *NetworkFacade) Broadcast(ctx context.Context, tag protocol.Tag, data []byte, wait bool, exclude network.Peer) error { excludeNode := -1 if exclude != nil { - excludeNode = n.peerToNode[exclude] + excludeNode = n.peerToNode[exclude.(*facadePeer)] } return n.broadcast(tag, data, excludeNode, "NetworkFacade service-%v Broadcast %v %v\n") } @@ -240,7 +247,7 @@ func (n *NetworkFacade) PushDownstreamMessage(newMsg context.CancelFunc) bool { func (n *NetworkFacade) Address() (string, bool) { return "mock network", true } // Start - unused function -func (n *NetworkFacade) Start() {} +func (n *NetworkFacade) Start() error { return nil } // Stop - unused function func (n *NetworkFacade) Stop() {} @@ -341,8 +348,8 @@ func (n *NetworkFacade) ReceiveMessage(sourceNode int, tag protocol.Tag, data [] n.pushPendingReceivedMessage() } -func (n *NetworkFacade) Disconnect(sender network.Peer) { - sourceNode := n.peerToNode[sender] +func (n *NetworkFacade) Disconnect(sender network.DisconnectablePeer) { + sourceNode := n.peerToNode[sender.(*facadePeer)] n.fuzzer.Disconnect(n.nodeID, sourceNode) } diff --git a/agreement/gossip/network_test.go b/agreement/gossip/network_test.go index c168b77cce..a3c5328716 100644 --- a/agreement/gossip/network_test.go +++ b/agreement/gossip/network_test.go @@ -18,7 +18,6 @@ package gossip import ( "context" - "net" "net/http" "sync" "sync/atomic" @@ -136,7 +135,7 @@ func (w *whiteholeNetwork) Relay(ctx context.Context, tag protocol.Tag, data []b func (w *whiteholeNetwork) BroadcastSimple(tag protocol.Tag, data []byte) error { return w.Broadcast(context.Background(), tag, data, true, nil) } -func (w *whiteholeNetwork) Disconnect(badnode network.Peer) { +func (w *whiteholeNetwork) Disconnect(badnode network.DisconnectablePeer) { return } func (w *whiteholeNetwork) DisconnectPeers() { @@ -156,11 +155,11 @@ func (w *whiteholeNetwork) GetPeers(options ...network.PeerOption) []network.Pee } func (w *whiteholeNetwork) RegisterHTTPHandler(path string, handler http.Handler) { } -func (w *whiteholeNetwork) GetHTTPRequestConnection(request *http.Request) (conn net.Conn) { +func (w *whiteholeNetwork) GetHTTPRequestConnection(request *http.Request) (conn network.DeadlineSettableConn) { return nil } -func (w *whiteholeNetwork) Start() { +func (w *whiteholeNetwork) Start() error { w.quit = make(chan struct{}) go func(w *whiteholeNetwork) { w.domain.messagesMu.Lock() @@ -216,7 +215,7 @@ func (w *whiteholeNetwork) Start() { atomic.AddUint32(&w.lastMsgRead, 1) } }(w) - return + return nil } func (w *whiteholeNetwork) getMux() *network.Multiplexer { return w.mux diff --git a/catchup/fetcher_test.go b/catchup/fetcher_test.go index e219852bf4..52b0b32a8f 100644 --- a/catchup/fetcher_test.go +++ b/catchup/fetcher_test.go @@ -21,7 +21,6 @@ import ( "net" "net/http" "net/url" - "strings" "testing" "github.com/gorilla/mux" @@ -173,8 +172,8 @@ func (b *basicRPCNode) GetPeers(options ...network.PeerOption) []network.Peer { return b.peers } -func (b *basicRPCNode) SubstituteGenesisID(rawURL string) string { - return strings.Replace(rawURL, "{genesisID}", "test genesisID", -1) +func (b *basicRPCNode) GetGenesisID() string { + return "test genesisID" } type httpTestPeerSource struct { @@ -191,8 +190,8 @@ func (s *httpTestPeerSource) RegisterHandlers(dispatch []network.TaggedMessageHa s.dispatchHandlers = append(s.dispatchHandlers, dispatch...) } -func (s *httpTestPeerSource) SubstituteGenesisID(rawURL string) string { - return strings.Replace(rawURL, "{genesisID}", "test genesisID", -1) +func (s *httpTestPeerSource) GetGenesisID() string { + return "test genesisID" } // implement network.HTTPPeer @@ -201,8 +200,13 @@ type testHTTPPeer string func (p *testHTTPPeer) GetAddress() string { return string(*p) } + func (p *testHTTPPeer) GetHTTPClient() *http.Client { - return &http.Client{} + return &http.Client{ + Transport: &network.HTTPPAddressBoundTransport{ + Addr: p.GetAddress(), + InnerTransport: http.DefaultTransport}, + } } func (p *testHTTPPeer) GetHTTPPeer() network.HTTPPeer { return p @@ -238,6 +242,8 @@ func (p *testUnicastPeer) GetAddress() string { return "test" } +func (p *testUnicastPeer) GetNetwork() network.GossipNode { return p.gn } + func (p *testUnicastPeer) Request(ctx context.Context, tag protocol.Tag, topics network.Topics) (resp *network.Response, e error) { responseChannel := make(chan *network.Response, 1) diff --git a/catchup/ledgerFetcher.go b/catchup/ledgerFetcher.go index 2f87b5d576..916627db8f 100644 --- a/catchup/ledgerFetcher.go +++ b/catchup/ledgerFetcher.go @@ -23,7 +23,6 @@ import ( "fmt" "io" "net/http" - "path" "strconv" "time" @@ -74,13 +73,7 @@ func makeLedgerFetcher(net network.GossipNode, accessor ledger.CatchpointCatchup } func (lf *ledgerFetcher) requestLedger(ctx context.Context, peer network.HTTPPeer, round basics.Round, method string) (*http.Response, error) { - parsedURL, err := network.ParseHostOrURL(peer.GetAddress()) - if err != nil { - return nil, err - } - - parsedURL.Path = lf.net.SubstituteGenesisID(path.Join(parsedURL.Path, "/v1/{genesisID}/ledger/"+strconv.FormatUint(uint64(round), 36))) - ledgerURL := parsedURL.String() + ledgerURL := network.SubstituteGenesisID(lf.net, "/v1/{genesisID}/ledger/"+strconv.FormatUint(uint64(round), 36)) lf.log.Debugf("ledger %s %#v peer %#v %T", method, ledgerURL, peer, peer) request, err := http.NewRequestWithContext(ctx, method, ledgerURL, nil) if err != nil { diff --git a/catchup/ledgerFetcher_test.go b/catchup/ledgerFetcher_test.go index 6bbde32120..a080aca31e 100644 --- a/catchup/ledgerFetcher_test.go +++ b/catchup/ledgerFetcher_test.go @@ -17,6 +17,7 @@ package catchup import ( + "archive/tar" "context" "fmt" "net" @@ -30,6 +31,8 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/logging" + p2ptesting "github.com/algorand/go-algorand/network/p2p/testing" + "github.com/algorand/go-algorand/rpcs" "github.com/algorand/go-algorand/test/partitiontest" ) @@ -125,7 +128,7 @@ func TestLedgerFetcherErrorResponseHandling(t *testing.T) { } } -func TestLedgerFetcherHeadLedger(t *testing.T) { +func TestLedgerFetcher(t *testing.T) { partitiontest.PartitionTest(t) // create a dummy server. @@ -136,16 +139,19 @@ func TestLedgerFetcherHeadLedger(t *testing.T) { listener, err := net.Listen("tcp", "localhost:") var httpServerResponse = 0 - var contentTypes = make([]string, 0) require.NoError(t, err) go s.Serve(listener) defer s.Close() defer listener.Close() mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - for _, contentType := range contentTypes { - w.Header().Add("Content-Type", contentType) + if req.Method == http.MethodHead { + w.WriteHeader(httpServerResponse) + } else { + w.Header().Add("Content-Type", rpcs.LedgerResponseContentType) + w.WriteHeader(httpServerResponse) + wtar := tar.NewWriter(w) + wtar.Close() } - w.WriteHeader(httpServerResponse) }) successPeer := testHTTPPeer(listener.Addr().String()) lf := makeLedgerFetcher(&mocks.MockNetwork{}, &mocks.MockCatchpointCatchupAccessor{}, logging.TestingLog(t), &dummyLedgerFetcherReporter{}, config.GetDefaultLocal()) @@ -157,7 +163,7 @@ func TestLedgerFetcherHeadLedger(t *testing.T) { // headLedger parseURL failure parseFailurePeer := testHTTPPeer("foobar") err = lf.headLedger(context.Background(), &parseFailurePeer, basics.Round(0)) - require.Equal(t, fmt.Errorf("could not parse a host from url"), err) + require.ErrorContains(t, err, "could not parse a host from url") // headLedger 404 response httpServerResponse = http.StatusNotFound @@ -169,8 +175,46 @@ func TestLedgerFetcherHeadLedger(t *testing.T) { err = lf.headLedger(context.Background(), &successPeer, basics.Round(0)) require.NoError(t, err) + httpServerResponse = http.StatusOK + err = lf.downloadLedger(context.Background(), &successPeer, basics.Round(0)) + require.NoError(t, err) + // headLedger 500 response httpServerResponse = http.StatusInternalServerError err = lf.headLedger(context.Background(), &successPeer, basics.Round(0)) require.Equal(t, fmt.Errorf("headLedger error response status code %d", http.StatusInternalServerError), err) } + +func TestLedgerFetcherP2P(t *testing.T) { + partitiontest.PartitionTest(t) + + mux := http.NewServeMux() + nodeA := p2ptesting.MakeHTTPNode(t) + nodeA.RegisterHTTPHandler("/v1/ledger/0", mux) + var httpServerResponse = 0 + mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { + if req.Method == http.MethodHead { + w.WriteHeader(httpServerResponse) + } else { + w.Header().Add("Content-Type", rpcs.LedgerResponseContentType) + w.WriteHeader(httpServerResponse) + wtar := tar.NewWriter(w) + wtar.Close() + } + }) + + nodeA.Start() + defer nodeA.Stop() + + successPeer := nodeA.GetHTTPPeer() + lf := makeLedgerFetcher(nodeA, &mocks.MockCatchpointCatchupAccessor{}, logging.TestingLog(t), &dummyLedgerFetcherReporter{}, config.GetDefaultLocal()) + + // headLedger 200 response + httpServerResponse = http.StatusOK + err := lf.headLedger(context.Background(), successPeer, basics.Round(0)) + require.NoError(t, err) + + httpServerResponse = http.StatusOK + err = lf.downloadLedger(context.Background(), successPeer, basics.Round(0)) + require.NoError(t, err) +} diff --git a/catchup/universalFetcher.go b/catchup/universalFetcher.go index 31d82adc98..27b970fc26 100644 --- a/catchup/universalFetcher.go +++ b/catchup/universalFetcher.go @@ -219,13 +219,8 @@ type HTTPFetcher struct { // getBlockBytes gets a block. // Core piece of FetcherClient interface func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round) (data []byte, err error) { - parsedURL, err := network.ParseHostOrURL(hf.rootURL) - if err != nil { - return nil, err - } + blockURL := rpcs.FormatBlockQuery(uint64(r), "", hf.net) - parsedURL.Path = rpcs.FormatBlockQuery(uint64(r), parsedURL.Path, hf.net) - blockURL := parsedURL.String() hf.log.Debugf("block GET %#v peer %#v %T", blockURL, hf.peer, hf.peer) request, err := http.NewRequest("GET", blockURL, nil) if err != nil { diff --git a/cmd/algod/main.go b/cmd/algod/main.go index 0f93ed447f..09770cb6e0 100644 --- a/cmd/algod/main.go +++ b/cmd/algod/main.go @@ -33,7 +33,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/telemetryspec" - "github.com/algorand/go-algorand/network" + "github.com/algorand/go-algorand/network/addr" "github.com/algorand/go-algorand/protocol" toolsnet "github.com/algorand/go-algorand/tools/network" "github.com/algorand/go-algorand/util" @@ -282,7 +282,7 @@ func run() int { // make sure that the format of each entry is valid: for idx, peer := range peerOverrideArray { - addr, addrErr := network.ParseHostOrURLOrMultiaddr(peer) + addr, addrErr := addr.ParseHostOrURLOrMultiaddr(peer) if addrErr != nil { fmt.Fprintf(os.Stderr, "Provided command line parameter '%s' is not a valid host:port pair\n", peer) return 1 diff --git a/cmd/goal/node.go b/cmd/goal/node.go index 17de96a81a..2db08fd4e5 100644 --- a/cmd/goal/node.go +++ b/cmd/goal/node.go @@ -39,7 +39,7 @@ import ( "github.com/algorand/go-algorand/daemon/algod/api/server/v2/generated/model" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/libgoal" - "github.com/algorand/go-algorand/network" + naddr "github.com/algorand/go-algorand/network/addr" "github.com/algorand/go-algorand/nodecontrol" "github.com/algorand/go-algorand/util" "github.com/algorand/go-algorand/util/tokens" @@ -751,7 +751,7 @@ func verifyPeerDialArg() bool { // make sure that the format of each entry is valid: for _, peer := range strings.Split(peerDial, ";") { - _, err := network.ParseHostOrURLOrMultiaddr(peer) + _, err := naddr.ParseHostOrURLOrMultiaddr(peer) if err != nil { reportErrorf("Provided peer '%s' is not a valid peer address : %v", peer, err) return false diff --git a/components/mocks/mockNetwork.go b/components/mocks/mockNetwork.go index 25486cb050..f933a553a9 100644 --- a/components/mocks/mockNetwork.go +++ b/components/mocks/mockNetwork.go @@ -18,7 +18,7 @@ package mocks import ( "context" - "net" + "errors" "net/http" "github.com/algorand/go-algorand/network" @@ -28,6 +28,7 @@ import ( // MockNetwork is a dummy network that doesn't do anything type MockNetwork struct { network.GossipNode + GenesisID string } // Broadcast - unused function @@ -46,7 +47,8 @@ func (network *MockNetwork) Address() (string, bool) { } // Start - unused function -func (network *MockNetwork) Start() { +func (network *MockNetwork) Start() error { + return nil } // Stop - unused function @@ -58,7 +60,7 @@ func (network *MockNetwork) RequestConnectOutgoing(replace bool, quit <-chan str } // Disconnect - unused function -func (network *MockNetwork) Disconnect(badpeer network.Peer) { +func (network *MockNetwork) Disconnect(badpeer network.DisconnectablePeer) { } // DisconnectPeers - unused function @@ -74,11 +76,6 @@ func (network *MockNetwork) GetPeers(options ...network.PeerOption) []network.Pe return nil } -// GetRoundTripper -- returns the network round tripper -func (network *MockNetwork) GetRoundTripper() http.RoundTripper { - return http.DefaultTransport -} - // Ready - always ready func (network *MockNetwork) Ready() chan struct{} { c := make(chan struct{}) @@ -94,6 +91,14 @@ func (network *MockNetwork) RegisterHandlers(dispatch []network.TaggedMessageHan func (network *MockNetwork) ClearHandlers() { } +// RegisterProcessors - empty implementation. +func (network *MockNetwork) RegisterProcessors(dispatch []network.TaggedMessageProcessor) { +} + +// ClearProcessors - empty implementation +func (network *MockNetwork) ClearProcessors() { +} + // RegisterHTTPHandler - empty implementation func (network *MockNetwork) RegisterHTTPHandler(path string, handler http.Handler) { } @@ -102,11 +107,19 @@ func (network *MockNetwork) RegisterHTTPHandler(path string, handler http.Handle func (network *MockNetwork) OnNetworkAdvance() {} // GetHTTPRequestConnection - empty implementation -func (network *MockNetwork) GetHTTPRequestConnection(request *http.Request) (conn net.Conn) { +func (network *MockNetwork) GetHTTPRequestConnection(request *http.Request) (conn network.DeadlineSettableConn) { return nil } -// SubstituteGenesisID - empty implementation -func (network *MockNetwork) SubstituteGenesisID(rawURL string) string { - return rawURL +// GetGenesisID - empty implementation +func (network *MockNetwork) GetGenesisID() string { + if network.GenesisID == "" { + return "mocknet" + } + return network.GenesisID +} + +// GetHTTPClient returns a http.Client with a suitable for the network +func (network *MockNetwork) GetHTTPClient(address string) (*http.Client, error) { + return nil, errors.New("not implemented") } diff --git a/config/config.go b/config/config.go index a8beb05800..2d5d0bdbfe 100644 --- a/config/config.go +++ b/config/config.go @@ -268,6 +268,7 @@ const ( dnssecSRV = 1 << iota dnssecRelayAddr dnssecTelemetryAddr + dnssecTXT ) const ( diff --git a/config/localTemplate.go b/config/localTemplate.go index 309ffcb798..314b83a78b 100644 --- a/config/localTemplate.go +++ b/config/localTemplate.go @@ -362,8 +362,9 @@ type Local struct { // 0x01 (dnssecSRV) - validate SRV response // 0x02 (dnssecRelayAddr) - validate relays' names to addresses resolution // 0x04 (dnssecTelemetryAddr) - validate telemetry and metrics names to addresses resolution + // 0x08 (dnssecTXT) - validate TXT response // ... - DNSSecurityFlags uint32 `version[6]:"1"` + DNSSecurityFlags uint32 `version[6]:"1" version[34]:"9"` // EnablePingHandler controls whether the gossip node would respond to ping messages with a pong message. EnablePingHandler bool `version[6]:"true"` @@ -596,9 +597,19 @@ type Local struct { // When it exceeds this capacity, it redirects the block requests to a different node BlockServiceMemCap uint64 `version[28]:"500000000"` - // EnableP2P turns on the peer to peer network + // EnableP2P turns on the peer to peer network. + // When both EnableP2P and EnableP2PHybridMode (below) are set, EnableP2PHybridMode takes precedence. EnableP2P bool `version[31]:"false"` + // EnableP2PHybridMode turns on both websockets and P2P networking. + EnableP2PHybridMode bool `version[34]:"false"` + + // P2PNetAddress sets the listen address used for P2P networking, if hybrid mode is set. + P2PNetAddress string `version[34]:""` + + // EnableDHT will turn on the hash table for use with capabilities advertisement + EnableDHTProviders bool `version[34]:"false"` + // P2PPersistPeerID will write the private key used for the node's PeerID to the P2PPrivateKeyLocation. // This is only used when P2PEnable is true. If P2PPrivateKey is not specified, it uses the default location. P2PPersistPeerID bool `version[29]:"false"` @@ -683,11 +694,16 @@ func (cfg Local) DNSSecurityRelayAddrEnforced() bool { return cfg.DNSSecurityFlags&dnssecRelayAddr != 0 } -// DNSSecurityTelemeryAddrEnforced returns true if relay name to ip addr resolution enforced -func (cfg Local) DNSSecurityTelemeryAddrEnforced() bool { +// DNSSecurityTelemetryAddrEnforced returns true if relay name to ip addr resolution enforced +func (cfg Local) DNSSecurityTelemetryAddrEnforced() bool { return cfg.DNSSecurityFlags&dnssecTelemetryAddr != 0 } +// DNSSecurityTXTEnforced returns true if TXT response verification enforced +func (cfg Local) DNSSecurityTXTEnforced() bool { + return cfg.DNSSecurityFlags&dnssecTXT != 0 +} + // CatchupVerifyCertificate returns true if certificate verification is needed func (cfg Local) CatchupVerifyCertificate() bool { return cfg.CatchupBlockValidateMode&catchupValidationModeCertificate == 0 diff --git a/config/local_defaults.go b/config/local_defaults.go index f5f02082aa..ae2ed22ebf 100644 --- a/config/local_defaults.go +++ b/config/local_defaults.go @@ -51,7 +51,7 @@ var defaultLocal = Local{ ConnectionsRateLimitingWindowSeconds: 1, CrashDBDir: "", DNSBootstrapID: ".algorand.network?backup=.algorand.net&dedup=.algorand-.(network|net)", - DNSSecurityFlags: 1, + DNSSecurityFlags: 9, DeadlockDetection: 0, DeadlockDetectionThreshold: 30, DisableAPIAuth: false, @@ -64,6 +64,7 @@ var defaultLocal = Local{ EnableAgreementTimeMetrics: false, EnableAssembleStats: false, EnableBlockService: false, + EnableDHTProviders: false, EnableDeveloperAPI: false, EnableExperimentalAPI: false, EnableFollowMode: false, @@ -74,6 +75,7 @@ var defaultLocal = Local{ EnableMetricReporting: false, EnableOutgoingNetworkMessageFiltering: true, EnableP2P: false, + EnableP2PHybridMode: false, EnablePingHandler: true, EnableProcessBlockStats: false, EnableProfiler: false, @@ -117,6 +119,7 @@ var defaultLocal = Local{ OptimizeAccountsDatabaseOnStartup: false, OutgoingMessageFilterBucketCount: 3, OutgoingMessageFilterBucketSize: 128, + P2PNetAddress: "", P2PPersistPeerID: false, P2PPrivateKeyLocation: "", ParticipationKeysRefreshInterval: 60000000000, diff --git a/daemon/algod/server.go b/daemon/algod/server.go index 13432957c3..c43b0b0693 100644 --- a/daemon/algod/server.go +++ b/daemon/algod/server.go @@ -59,7 +59,7 @@ const maxHeaderBytes = 4096 type ServerNode interface { apiServer.APINodeInterface ListeningAddress() (string, bool) - Start() + Start() error Stop() } @@ -298,7 +298,13 @@ func getPortFromAddress(addr string) (string, error) { func (s *Server) Start() { s.log.Info("Trying to start an Algorand node") fmt.Print("Initializing the Algorand node... ") - s.node.Start() + err := s.node.Start() + if err != nil { + msg := fmt.Sprintf("Failed to start an Algorand node: %v", err) + s.log.Error(msg) + fmt.Println(msg) + os.Exit(1) + } s.log.Info("Successfully started an Algorand node.") fmt.Println("Success!") @@ -317,7 +323,6 @@ func (s *Server) Start() { } var apiToken string - var err error fmt.Printf("API authentication disabled: %v\n", cfg.DisableAPIAuth) if !cfg.DisableAPIAuth { apiToken, err = tokens.GetAndValidateAPIToken(s.RootPath, tokens.AlgodTokenFilename) diff --git a/data/txHandler.go b/data/txHandler.go index 871e71cbc3..eae9586c47 100644 --- a/data/txHandler.go +++ b/data/txHandler.go @@ -240,9 +240,26 @@ func (handler *TxHandler) Start() { if handler.msgCache != nil { handler.msgCache.Start(handler.ctx, 60*time.Second) } + // wsNetwork handler handler.net.RegisterHandlers([]network.TaggedMessageHandler{ {Tag: protocol.TxnTag, MessageHandler: network.HandlerFunc(handler.processIncomingTxn)}, }) + + // libp2p pubsub validator and handler abstracted as TaggedMessageProcessor + handler.net.RegisterProcessors([]network.TaggedMessageProcessor{ + { + Tag: protocol.TxnTag, + // create anonymous struct to hold the two functions and satisfy the network.MessageProcessor interface + MessageHandler: struct { + network.ProcessorValidateFunc + network.ProcessorHandleFunc + }{ + network.ProcessorValidateFunc(handler.validateIncomingTxMessage), + network.ProcessorHandleFunc(handler.processIncomingTxMessage), + }, + }, + }) + handler.backlogWg.Add(2) go handler.backlogWorker() go handler.backlogGaugeThread() @@ -533,7 +550,7 @@ func (handler *TxHandler) deleteFromCaches(msgKey *crypto.Digest, canonicalKey * // dedupCanonical checks if the transaction group has been seen before after reencoding to canonical representation. // returns a key used for insertion if the group was not found. -func (handler *TxHandler) dedupCanonical(ntx int, unverifiedTxGroup []transactions.SignedTxn, consumed int) (key *crypto.Digest, isDup bool) { +func (handler *TxHandler) dedupCanonical(unverifiedTxGroup []transactions.SignedTxn, consumed int) (key *crypto.Digest, isDup bool) { // consider situations where someone want to censor transactions A // 1. Txn A is not part of a group => txn A with a valid signature is OK // Censorship attempts are: @@ -550,6 +567,7 @@ func (handler *TxHandler) dedupCanonical(ntx int, unverifiedTxGroup []transactio // - using individual txn from a group: {A, Z} could be poisoned by {A, B}, where B is invalid var d crypto.Digest + ntx := len(unverifiedTxGroup) if ntx == 1 { // a single transaction => cache/dedup canonical txn with its signature enc := unverifiedTxGroup[0].MarshalMsg(nil) @@ -577,61 +595,58 @@ func (handler *TxHandler) dedupCanonical(ntx int, unverifiedTxGroup []transactio return &d, false } -// processIncomingTxn decodes a transaction group from incoming message and enqueues into the back log for processing. -// The function also performs some input data pre-validation; -// - txn groups are cut to MaxTxGroupSize size -// - message are checked for duplicates -// - transactions are checked for duplicates - -func (handler *TxHandler) processIncomingTxn(rawmsg network.IncomingMessage) network.OutgoingMessage { +// incomingMsgDupCheck runs the duplicate check on a raw incoming message. +// Returns: +// - the key used for insertion if the message was not found in the cache +// - a boolean indicating if the message was a duplicate +func (handler *TxHandler) incomingMsgDupCheck(data []byte) (*crypto.Digest, bool) { var msgKey *crypto.Digest var isDup bool if handler.msgCache != nil { // check for duplicate messages // this helps against relaying duplicates - if msgKey, isDup = handler.msgCache.CheckAndPut(rawmsg.Data); isDup { + if msgKey, isDup = handler.msgCache.CheckAndPut(data); isDup { transactionMessagesDupRawMsg.Inc(nil) - return network.OutgoingMessage{Action: network.Ignore} + return msgKey, true } } + return msgKey, false +} - unverifiedTxGroup := make([]transactions.SignedTxn, 1) - dec := protocol.NewMsgpDecoderBytes(rawmsg.Data) - ntx := 0 - consumed := 0 - - var err error +// incomingMsgErlCheck runs the rate limiting check on a sender. +// Returns: +// - the capacity guard returned by the elastic rate limiter +// - a boolean indicating if the sender is rate limited +func (handler *TxHandler) incomingMsgErlCheck(sender network.DisconnectablePeer) (*util.ErlCapacityGuard, bool) { var capguard *util.ErlCapacityGuard - accepted := false - defer func() { - // if we failed to put the item onto the backlog, we should release the capacity if any - if !accepted { - if capguard != nil { - if capErr := capguard.Release(); capErr != nil { - logging.Base().Warnf("Failed to release capacity to ElasticRateLimiter: %v", capErr) - } - } - } - }() - + var err error if handler.erl != nil { congestedERL := float64(cap(handler.backlogQueue))*handler.backlogCongestionThreshold < float64(len(handler.backlogQueue)) // consume a capacity unit // if the elastic rate limiter cannot vend a capacity, the error it returns // is sufficient to indicate that we should enable Congestion Control, because // an issue in vending capacity indicates the underlying resource (TXBacklog) is full - capguard, err = handler.erl.ConsumeCapacity(rawmsg.Sender.(util.ErlClient)) + capguard, err = handler.erl.ConsumeCapacity(sender.(util.ErlClient)) if err != nil { handler.erl.EnableCongestionControl() // if there is no capacity, it is the same as if we failed to put the item onto the backlog, so report such transactionMessagesDroppedFromBacklog.Inc(nil) - return network.OutgoingMessage{Action: network.Ignore} + return capguard, true } // if the backlog Queue has 50% of its buffer back, turn congestion control off if !congestedERL { handler.erl.DisableCongestionControl() } } + return capguard, false +} + +// decodeMsg decodes TX message buffer into transactions.SignedTxn, +// and returns number of bytes consumed from the buffer and a boolean indicating if the message was invalid. +func decodeMsg(data []byte) (unverifiedTxGroup []transactions.SignedTxn, consumed int, invalid bool) { + unverifiedTxGroup = make([]transactions.SignedTxn, 1) + dec := protocol.NewMsgpDecoderBytes(data) + ntx := 0 for { if len(unverifiedTxGroup) == ntx { @@ -645,7 +660,7 @@ func (handler *TxHandler) processIncomingTxn(rawmsg network.IncomingMessage) net break } logging.Base().Warnf("Received a non-decodable txn: %v", err) - return network.OutgoingMessage{Action: network.Disconnect} + return nil, 0, true } consumed = dec.Consumed() ntx++ @@ -654,13 +669,13 @@ func (handler *TxHandler) processIncomingTxn(rawmsg network.IncomingMessage) net if dec.Remaining() > 0 { // if something else left in the buffer - this is an error, drop transactionMessageTxGroupExcessive.Inc(nil) - return network.OutgoingMessage{Action: network.Disconnect} + return nil, 0, true } } } if ntx == 0 { logging.Base().Warnf("Received empty tx group") - return network.OutgoingMessage{Action: network.Disconnect} + return nil, 0, true } unverifiedTxGroup = unverifiedTxGroup[:ntx] @@ -669,22 +684,72 @@ func (handler *TxHandler) processIncomingTxn(rawmsg network.IncomingMessage) net transactionMessageTxGroupFull.Inc(nil) } + return unverifiedTxGroup, consumed, false +} + +// incomingTxGroupDupRateLimit checks +// - if the incoming transaction group has been seen before after reencoding to canonical representation, and +// - if the sender is rate limited by the per-application rate limiter. +func (handler *TxHandler) incomingTxGroupDupRateLimit(unverifiedTxGroup []transactions.SignedTxn, encodedExpectedSize int, sender network.DisconnectablePeer) (*crypto.Digest, bool) { var canonicalKey *crypto.Digest if handler.txCanonicalCache != nil { - if canonicalKey, isDup = handler.dedupCanonical(ntx, unverifiedTxGroup, consumed); isDup { + var isDup bool + if canonicalKey, isDup = handler.dedupCanonical(unverifiedTxGroup, encodedExpectedSize); isDup { transactionMessagesDupCanonical.Inc(nil) - return network.OutgoingMessage{Action: network.Ignore} + return canonicalKey, true } } // rate limit per application in a group. Limiting any app in a group drops the entire message. if handler.appLimiter != nil { congestedARL := len(handler.backlogQueue) > handler.appLimiterBacklogThreshold - if congestedARL && handler.appLimiter.shouldDrop(unverifiedTxGroup, rawmsg.Sender.(network.IPAddressable).RoutingAddr()) { + if congestedARL && handler.appLimiter.shouldDrop(unverifiedTxGroup, sender.(network.IPAddressable).RoutingAddr()) { transactionMessagesAppLimiterDrop.Inc(nil) - return network.OutgoingMessage{Action: network.Ignore} + return canonicalKey, true } } + return canonicalKey, false +} + +// processIncomingTxn decodes a transaction group from incoming message and enqueues into the back log for processing. +// The function also performs some input data pre-validation; +// - txn groups are cut to MaxTxGroupSize size +// - message are checked for duplicates +// - transactions are checked for duplicates +func (handler *TxHandler) processIncomingTxn(rawmsg network.IncomingMessage) network.OutgoingMessage { + msgKey, shouldDrop := handler.incomingMsgDupCheck(rawmsg.Data) + if shouldDrop { + return network.OutgoingMessage{Action: network.Ignore} + } + + capguard, shouldDrop := handler.incomingMsgErlCheck(rawmsg.Sender) + accepted := false + defer func() { + // if we failed to put the item onto the backlog, we should release the capacity if any + if !accepted && capguard != nil { + if capErr := capguard.Release(); capErr != nil { + logging.Base().Warnf("processIncomingTxn: failed to release capacity to ElasticRateLimiter: %v", capErr) + } + } + }() + + if shouldDrop { + // this TX message was rate-limited by ERL + return network.OutgoingMessage{Action: network.Ignore} + } + + unverifiedTxGroup, consumed, invalid := decodeMsg(rawmsg.Data) + if invalid { + // invalid encoding or exceeding txgroup, disconnect from this peer + return network.OutgoingMessage{Action: network.Disconnect} + } + + canonicalKey, drop := handler.incomingTxGroupDupRateLimit(unverifiedTxGroup, consumed, rawmsg.Sender) + if drop { + // this re-serialized txgroup was detected as a duplicate by the canonical message cache, + // or it was rate-limited by the per-app rate limiter + return network.OutgoingMessage{Action: network.Ignore} + } select { case handler.backlogQueue <- &txBacklogMsg{ @@ -701,14 +766,70 @@ func (handler *TxHandler) processIncomingTxn(rawmsg network.IncomingMessage) net transactionMessagesDroppedFromBacklog.Inc(nil) // additionally, remove the txn from duplicate caches to ensure it can be re-submitted - if handler.txCanonicalCache != nil && canonicalKey != nil { - handler.txCanonicalCache.Delete(canonicalKey) - } - if handler.msgCache != nil && msgKey != nil { - handler.msgCache.DeleteByKey(msgKey) - } + handler.deleteFromCaches(msgKey, canonicalKey) + } + + return network.OutgoingMessage{Action: network.Ignore} +} + +type validatedIncomingTxMessage struct { + rawmsg network.IncomingMessage + unverifiedTxGroup []transactions.SignedTxn + msgKey *crypto.Digest + canonicalKey *crypto.Digest +} + +// validateIncomingTxMessage is the validator for the MessageProcessor implementation used by P2PNetwork. +func (handler *TxHandler) validateIncomingTxMessage(rawmsg network.IncomingMessage) network.ValidatedMessage { + msgKey, isDup := handler.incomingMsgDupCheck(rawmsg.Data) + if isDup { + return network.ValidatedMessage{Action: network.Ignore, ValidatedMessage: nil} } + unverifiedTxGroup, consumed, invalid := decodeMsg(rawmsg.Data) + if invalid { + // invalid encoding or exceeding txgroup, disconnect from this peer + return network.ValidatedMessage{Action: network.Disconnect, ValidatedMessage: nil} + } + + canonicalKey, drop := handler.incomingTxGroupDupRateLimit(unverifiedTxGroup, consumed, rawmsg.Sender) + if drop { + // this re-serialized txgroup was detected as a duplicate by the canonical message cache, + // or it was rate-limited by the per-app rate limiter + return network.ValidatedMessage{Action: network.Ignore, ValidatedMessage: nil} + } + + return network.ValidatedMessage{ + Action: network.Accept, + Tag: rawmsg.Tag, + ValidatedMessage: &validatedIncomingTxMessage{ + rawmsg: rawmsg, + unverifiedTxGroup: unverifiedTxGroup, + msgKey: msgKey, + canonicalKey: canonicalKey, + }, + } +} + +// processIncomingTxMessage is the handler for the MessageProcessor implementation used by P2PNetwork. +func (handler *TxHandler) processIncomingTxMessage(validatedMessage network.ValidatedMessage) network.OutgoingMessage { + msg := validatedMessage.ValidatedMessage.(*validatedIncomingTxMessage) + select { + case handler.backlogQueue <- &txBacklogMsg{ + rawmsg: &msg.rawmsg, + unverifiedTxGroup: msg.unverifiedTxGroup, + rawmsgDataHash: msg.msgKey, + unverifiedTxGroupHash: msg.canonicalKey, + capguard: nil, + }: + default: + // if we failed here we want to increase the corresponding metric. It might suggest that we + // want to increase the queue size. + transactionMessagesDroppedFromBacklog.Inc(nil) + + // additionally, remove the txn from duplicate caches to ensure it can be re-submitted + handler.deleteFromCaches(msg.msgKey, msg.canonicalKey) + } return network.OutgoingMessage{Action: network.Ignore} } diff --git a/data/txHandler_test.go b/data/txHandler_test.go index d395779f33..9237865037 100644 --- a/data/txHandler_test.go +++ b/data/txHandler_test.go @@ -61,7 +61,8 @@ var txBacklogSize = config.GetDefaultLocal().TxBacklogSize // mock sender is used to implement OnClose, since TXHandlers expect to use Senders and ERL Clients type mockSender struct{} -func (m mockSender) OnClose(func()) {} +func (m mockSender) OnClose(func()) {} +func (m mockSender) GetNetwork() network.GossipNode { panic("not implemented") } func (m mockSender) IPAddr() []byte { return nil } func (m mockSender) RoutingAddr() []byte { return nil } diff --git a/go.mod b/go.mod index a32da9da8c..e4d8c5b8d5 100644 --- a/go.mod +++ b/go.mod @@ -28,10 +28,13 @@ require ( github.com/google/go-cmp v0.6.0 github.com/google/go-querystring v1.0.0 github.com/gorilla/mux v1.8.0 + github.com/ipfs/go-log v1.0.5 + github.com/ipfs/go-log/v2 v2.5.1 github.com/jmoiron/sqlx v1.2.0 github.com/karalabe/usb v0.0.2 github.com/labstack/echo/v4 v4.9.1 github.com/libp2p/go-libp2p v0.33.2 + github.com/libp2p/go-libp2p-kad-dht v0.24.3 github.com/libp2p/go-libp2p-pubsub v0.10.0 github.com/libp2p/go-yamux/v4 v4.0.1 github.com/mattn/go-sqlite3 v1.14.16 @@ -39,9 +42,13 @@ require ( github.com/multiformats/go-multiaddr v0.12.3 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/olivere/elastic v6.2.14+incompatible + github.com/prometheus/client_golang v1.18.0 + github.com/prometheus/client_model v0.6.0 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.5.0 github.com/stretchr/testify v1.8.4 + go.opencensus.io v0.24.0 + go.uber.org/zap v1.27.0 golang.org/x/crypto v0.21.0 golang.org/x/exp v0.0.0-20240213143201-ec583247a57a golang.org/x/sync v0.6.0 @@ -73,24 +80,33 @@ require ( github.com/flynn/noise v1.1.0 // indirect github.com/fortytw2/leaktest v1.3.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/swag v0.19.5 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect github.com/google/uuid v1.4.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/invopop/yaml v0.1.0 // indirect + github.com/ipfs/boxo v0.10.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect - github.com/ipfs/go-log/v2 v2.5.1 // indirect + github.com/ipfs/go-datastore v0.6.0 // indirect + github.com/ipld/go-ipld-prime v0.20.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect + github.com/jbenet/goprocess v0.1.4 // indirect github.com/jmespath/go-jmespath v0.3.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.17.6 // indirect @@ -100,8 +116,11 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/labstack/gommon v0.4.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect + github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect + github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect + github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect @@ -126,12 +145,12 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/onsi/ginkgo/v2 v2.15.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.18.0 // indirect - github.com/prometheus/client_model v0.6.0 // indirect + github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/common v0.47.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect @@ -145,16 +164,20 @@ require ( github.com/stretchr/objx v0.5.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect + github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect + go.opentelemetry.io/otel v1.16.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect + go.opentelemetry.io/otel/trace v1.16.0 // indirect go.uber.org/dig v1.17.1 // indirect go.uber.org/fx v1.20.1 // indirect go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect golang.org/x/mod v0.15.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.18.0 // indirect + gonum.org/v1/gonum v0.13.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index cdae303d5b..1a52e04711 100644 --- a/go.sum +++ b/go.sum @@ -125,6 +125,7 @@ github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0 github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= @@ -140,6 +141,8 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= @@ -152,8 +155,11 @@ github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aev github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -164,6 +170,7 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= @@ -183,6 +190,9 @@ github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -195,7 +205,9 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -206,7 +218,9 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= @@ -218,12 +232,16 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 h1:E/LAvt58di64hlYjx7AsNS6C/ysHWYo+2qPCZKTQhRo= github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -231,7 +249,14 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/ github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -244,18 +269,34 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc= github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= +github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= +github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= +github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= +github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= +github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= +github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= +github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= +github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= +github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= +github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= @@ -265,6 +306,7 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= @@ -310,14 +352,22 @@ github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= +github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= +github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= github.com/libp2p/go-libp2p v0.33.2 h1:vCdwnFxoGOXMKmaGHlDSnL4bM3fQeW8pgIa9DECnb40= github.com/libp2p/go-libp2p v0.33.2/go.mod h1:zTeppLuCvUIkT118pFVzA8xzP/p2dJYOMApCkFh0Yww= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= +github.com/libp2p/go-libp2p-kad-dht v0.24.3 h1:VjxtDVWaaf4UFjGBf+yl2JCiGaHx7+ctAUa9oJCR3QE= +github.com/libp2p/go-libp2p-kad-dht v0.24.3/go.mod h1:BShPzRbK6+fN3hk8a0WGAYKpb8m4k+DtchkqouGTrSg= +github.com/libp2p/go-libp2p-kbucket v0.6.3 h1:p507271wWzpy2f1XxPzCQG9NiN6R6lHL9GiSErbQQo0= +github.com/libp2p/go-libp2p-kbucket v0.6.3/go.mod h1:RCseT7AH6eJWxxk2ol03xtP9pEHetYSPXOaJnOiD8i0= github.com/libp2p/go-libp2p-pubsub v0.10.0 h1:wS0S5FlISavMaAbxyQn3dxMOe2eegMfswM471RuHJwA= github.com/libp2p/go-libp2p-pubsub v0.10.0/go.mod h1:1OxbaT/pFRO5h+Dpze8hdHQ63R0ke55XTs6b6NwLLkw= +github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= +github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= @@ -430,6 +480,8 @@ github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8P github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= @@ -444,6 +496,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= +github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= @@ -465,6 +519,7 @@ github.com/quic-go/webtransport-go v0.6.0 h1:CvNsKqc4W2HljHJnoT+rMmbRJybShZ0YPFD github.com/quic-go/webtransport-go v0.6.0/go.mod h1:9KjU4AEBqEQidGHNDkZrb8CAa1abRaosM2yGOyiikEc= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= @@ -503,7 +558,11 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -531,12 +590,14 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= @@ -547,6 +608,10 @@ github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= +github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -560,6 +625,15 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -572,9 +646,12 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= @@ -584,6 +661,7 @@ golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -602,6 +680,7 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -628,6 +707,7 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -711,6 +791,9 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -723,6 +806,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM= +gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= @@ -737,6 +822,7 @@ google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= @@ -744,13 +830,18 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -758,6 +849,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= @@ -783,6 +875,7 @@ honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= pgregory.net/rapid v0.6.2 h1:ErW5sL+UKtfBfUTsWHDCoeB+eZKLKMxrSd1VJY6W4bw= diff --git a/installer/config.json.example b/installer/config.json.example index 4a9714115f..7f16155303 100644 --- a/installer/config.json.example +++ b/installer/config.json.example @@ -30,7 +30,7 @@ "ConnectionsRateLimitingWindowSeconds": 1, "CrashDBDir": "", "DNSBootstrapID": ".algorand.network?backup=.algorand.net&dedup=.algorand-.(network|net)", - "DNSSecurityFlags": 1, + "DNSSecurityFlags": 9, "DeadlockDetection": 0, "DeadlockDetectionThreshold": 30, "DisableAPIAuth": false, @@ -43,6 +43,7 @@ "EnableAgreementTimeMetrics": false, "EnableAssembleStats": false, "EnableBlockService": false, + "EnableDHTProviders": false, "EnableDeveloperAPI": false, "EnableExperimentalAPI": false, "EnableFollowMode": false, @@ -53,6 +54,7 @@ "EnableMetricReporting": false, "EnableOutgoingNetworkMessageFiltering": true, "EnableP2P": false, + "EnableP2PHybridMode": false, "EnablePingHandler": true, "EnableProcessBlockStats": false, "EnableProfiler": false, @@ -96,6 +98,7 @@ "OptimizeAccountsDatabaseOnStartup": false, "OutgoingMessageFilterBucketCount": 3, "OutgoingMessageFilterBucketSize": 128, + "P2PNetAddress": "", "P2PPersistPeerID": false, "P2PPrivateKeyLocation": "", "ParticipationKeysRefreshInterval": 60000000000, diff --git a/logging/log.go b/logging/log.go index 48f83c1b9e..770bf08bb9 100644 --- a/logging/log.go +++ b/logging/log.go @@ -148,6 +148,9 @@ type Logger interface { // source adds file, line and function fields to the event source() *logrus.Entry + // Entry returns the logrus raw entry + Entry() *logrus.Entry + // Adds a hook to the logger AddHook(hook logrus.Hook) @@ -316,7 +319,11 @@ func (l logger) getOutput() io.Writer { } func (l logger) SetJSONFormatter() { - l.entry.Logger.Formatter = &logrus.JSONFormatter{TimestampFormat: "2006-01-02T15:04:05.000000Z07:00"} + l.entry.Logger.SetFormatter(&logrus.JSONFormatter{TimestampFormat: "2006-01-02T15:04:05.000000Z07:00"}) +} + +func (l logger) Entry() *logrus.Entry { + return l.entry } func (l logger) source() *logrus.Entry { diff --git a/netdeploy/remote/nodeConfig.go b/netdeploy/remote/nodeConfig.go index 2c6e0e423f..4880d76eb9 100644 --- a/netdeploy/remote/nodeConfig.go +++ b/netdeploy/remote/nodeConfig.go @@ -34,6 +34,7 @@ type NodeConfig struct { DashboardEndpoint string `json:",omitempty"` DeadlockOverride int `json:",omitempty"` // -1 = Disable deadlock detection, 0 = Use Default for build, 1 = Enable ConfigJSONOverride string `json:",omitempty"` // Raw json to merge into config.json after other modifications are complete + P2PBootstrap bool // True if this node should be a p2p bootstrap node and registered in DNS // NodeNameMatchRegex is tested against Name in generated configs and if matched the rest of the configs in this record are applied as a template NodeNameMatchRegex string `json:",omitempty"` diff --git a/netdeploy/remote/nodecfg/nodeConfigurator.go b/netdeploy/remote/nodecfg/nodeConfigurator.go index 5ab43d5ff7..842570bfc8 100644 --- a/netdeploy/remote/nodecfg/nodeConfigurator.go +++ b/netdeploy/remote/nodecfg/nodeConfigurator.go @@ -42,6 +42,7 @@ type nodeConfigurator struct { bootstrappedTrackerDir string relayEndpoints []srvEntry metricsEndpoints []srvEntry + p2pBootstrapEndpoints []txtEntry } type srvEntry struct { @@ -49,6 +50,11 @@ type srvEntry struct { port string } +type txtEntry struct { + netAddress string + peerID string +} + // ApplyConfigurationToHost attempts to apply the provided configuration to the local host, // based on the configuration specified for the provided hostName, with node // directories being created / updated under the specified rootNodeDir @@ -248,6 +254,31 @@ func (nc *nodeConfigurator) registerDNSRecords() (err error) { return } } + + dnsaddrsFrom := fmt.Sprintf("_dnsaddr.%s.algodev.network", nc.genesisData.Network) + for _, entry := range nc.p2pBootstrapEndpoints { + port, parseErr := strconv.ParseInt(strings.Split(entry.netAddress, ":")[1], 10, 64) + if parseErr != nil { + return parseErr + } + var addrType string + if isIP { + addrType = "ip4" + } else { + addrType = "dnsaddr" + } + addrInfoString := fmt.Sprintf("/%s/%s/tcp/%d/p2p/%s", addrType, nc.dnsName, port, entry.peerID) + to := fmt.Sprintf("dnsaddr=%s", addrInfoString) + + fmt.Fprintf(os.Stdout, "...... Adding P2P TXT Record '%s' -> '%s' .\n", dnsaddrsFrom, to) + const priority = 1 + const proxied = false + dnsErr := cloudflareDNS.CreateDNSRecord(context.Background(), "TXT", dnsaddrsFrom, to, cloudflare.AutomaticTTL, priority, proxied) + if dnsErr != nil { + return dnsErr + } + } + return } @@ -281,3 +312,7 @@ func (nc *nodeConfigurator) addRelaySrv(srvRecord string, port string) { func (nc *nodeConfigurator) registerMetricsSrv(srvRecord string, port string) { nc.metricsEndpoints = append(nc.metricsEndpoints, srvEntry{srvRecord, port}) } + +func (nc *nodeConfigurator) addP2PBootstrap(netAddress string, peerID string) { + nc.p2pBootstrapEndpoints = append(nc.p2pBootstrapEndpoints, txtEntry{netAddress, peerID}) +} diff --git a/netdeploy/remote/nodecfg/nodeDir.go b/netdeploy/remote/nodecfg/nodeDir.go index 9bd13343c1..bdfc037438 100644 --- a/netdeploy/remote/nodecfg/nodeDir.go +++ b/netdeploy/remote/nodecfg/nodeDir.go @@ -18,6 +18,7 @@ package nodecfg import ( "encoding/json" + "errors" "fmt" "net/url" "os" @@ -27,6 +28,7 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/netdeploy/remote" + "github.com/algorand/go-algorand/network/p2p" "github.com/algorand/go-algorand/shared/algoh" "github.com/algorand/go-algorand/util/tokens" ) @@ -101,6 +103,12 @@ func (nd *nodeDir) configure() (err error) { fmt.Fprintf(os.Stdout, "Error during configureNetAddress: %s\n", err) return } + + if err = nd.configureP2PDNSBootstrap(nd.P2PBootstrap); err != nil { + fmt.Fprintf(os.Stdout, "Error during configureP2PDNSBootstrap: %s\n", err) + return + } + fmt.Println("Done configuring node directory.") return } @@ -156,6 +164,45 @@ func (nd *nodeDir) configureNetAddress() (err error) { return } +func (nd *nodeDir) configureP2PDNSBootstrap(p2pBootstrap bool) error { + if !p2pBootstrap { + return nil + } + fmt.Fprintf(os.Stdout, " - Configuring P2P DNS Bootstrap: %s\n", nd.Name) + if err := nd.ensureConfig(); err != nil { + return err + } + // ensure p2p config params set are what is expected: + // - EnableP2P or EnableP2PHybridMode + // - NetAddress or P2PNetAddress is set + // - EnableGossipService + if !nd.config.EnableP2P && !nd.config.EnableP2PHybridMode { + return errors.New("p2p bootstrap requires EnableP2P or EnableP2PHybridMode to be set") + } + if nd.NetAddress == "" && nd.config.P2PNetAddress == "" { + return errors.New("p2p bootstrap requires NetAddress or P2PNetAddress to be set") + } + if !nd.config.EnableGossipService { + return errors.New("p2p bootstrap requires EnableGossipService to be set") + } + + netAddress := nd.NetAddress + if nd.config.P2PNetAddress != "" { + netAddress = nd.config.P2PNetAddress + } + + key, err := p2p.GetPrivKey(config.Local{P2PPersistPeerID: true}, nd.dataDir) + if err != nil { + return err + } + peerID, err := p2p.PeerIDFromPublicKey(key.GetPublic()) + if err != nil { + return err + } + nd.configurator.addP2PBootstrap(netAddress, peerID.String()) + return nil +} + func (nd *nodeDir) configureAPIEndpoint(address string) (err error) { if err = nd.ensureConfig(); err != nil { return diff --git a/network/addr.go b/network/addr.go index 1e2b04a447..00eb368881 100644 --- a/network/addr.go +++ b/network/addr.go @@ -17,73 +17,17 @@ package network import ( - "errors" - "net/url" "path" - "regexp" "strings" - "github.com/multiformats/go-multiaddr" + "github.com/algorand/go-algorand/network/addr" ) -var errURLNoHost = errors.New("could not parse a host from url") - -var errURLColonHost = errors.New("host name starts with a colon") - -// HostColonPortPattern matches "^[-a-zA-Z0-9.]+:\\d+$" e.g. "foo.com.:1234" -var HostColonPortPattern = regexp.MustCompile(`^[-a-zA-Z0-9.]+:\d+$`) - -// ParseHostOrURL handles "host:port" or a full URL. -// Standard library net/url.Parse chokes on "host:port". -func ParseHostOrURL(addr string) (*url.URL, error) { - // If the entire addr is "host:port" grab that right away. - // Don't try url.Parse() because that will grab "host:" as if it were "scheme:" - if HostColonPortPattern.MatchString(addr) { - return &url.URL{Scheme: "http", Host: addr}, nil - } - parsed, err := url.Parse(addr) - if err == nil { - if parsed.Host == "" { - return nil, errURLNoHost - } - return parsed, nil - } - if strings.HasPrefix(addr, "http:") || strings.HasPrefix(addr, "https:") || strings.HasPrefix(addr, "ws:") || strings.HasPrefix(addr, "wss:") || strings.HasPrefix(addr, "://") || strings.HasPrefix(addr, "//") { - return parsed, err - } - // This turns "[::]:4601" into "http://[::]:4601" which url.Parse can do - parsed, e2 := url.Parse("http://" + addr) - if e2 == nil { - // https://datatracker.ietf.org/doc/html/rfc1123#section-2 - // first character is relaxed to allow either a letter or a digit - if parsed.Host[0] == ':' && (len(parsed.Host) < 2 || parsed.Host[1] != ':') { - return nil, errURLColonHost - } - return parsed, nil - } - return parsed, err /* return original err, not our prefix altered try */ -} - -// ParseHostOrURLOrMultiaddr returns an error if it could not parse the provided -// string as a valid "host:port", full URL, or multiaddr. If no error, it returns -// a host:port address, or a multiaddr. -func ParseHostOrURLOrMultiaddr(addr string) (string, error) { - if strings.HasPrefix(addr, "/") && !strings.HasPrefix(addr, "//") { // multiaddr starts with '/' but not '//' which is possible for scheme relative URLS - _, err := multiaddr.NewMultiaddr(addr) - return addr, err - } - url, err := ParseHostOrURL(addr) - if err != nil { - return "", err - } - return url.Host, nil -} - // addrToGossipAddr parses host:port or a URL and returns the URL to the websocket interface at that address. -func (wn *WebsocketNetwork) addrToGossipAddr(addr string) (string, error) { - parsedURL, err := ParseHostOrURL(addr) +func (wn *WebsocketNetwork) addrToGossipAddr(a string) (string, error) { + parsedURL, err := addr.ParseHostOrURL(a) if err != nil { - wn.log.Warnf("could not parse addr %#v: %s", addr, err) + wn.log.Warnf("could not parse addr %#v: %s", a, err) return "", errBadAddr } parsedURL.Scheme = websocketsScheme[parsedURL.Scheme] diff --git a/network/addr/addr.go b/network/addr/addr.go new file mode 100644 index 0000000000..c8c0c0b6ab --- /dev/null +++ b/network/addr/addr.go @@ -0,0 +1,88 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package addr + +import ( + "errors" + "net/url" + "regexp" + "strings" + + "github.com/multiformats/go-multiaddr" +) + +var errURLNoHost = errors.New("could not parse a host from url") + +var errURLColonHost = errors.New("host name starts with a colon") + +// HostColonPortPattern matches "^[-a-zA-Z0-9.]+:\\d+$" e.g. "foo.com.:1234" +var HostColonPortPattern = regexp.MustCompile(`^[-a-zA-Z0-9.]+:\d+$`) + +// ParseHostOrURL handles "host:port" or a full URL. +// Standard library net/url.Parse chokes on "host:port". +func ParseHostOrURL(addr string) (*url.URL, error) { + // If the entire addr is "host:port" grab that right away. + // Don't try url.Parse() because that will grab "host:" as if it were "scheme:" + if HostColonPortPattern.MatchString(addr) { + return &url.URL{Scheme: "http", Host: addr}, nil + } + parsed, err := url.Parse(addr) + if err == nil { + if parsed.Host == "" { + return nil, errURLNoHost + } + return parsed, nil + } + if strings.HasPrefix(addr, "http:") || strings.HasPrefix(addr, "https:") || strings.HasPrefix(addr, "ws:") || strings.HasPrefix(addr, "wss:") || strings.HasPrefix(addr, "://") || strings.HasPrefix(addr, "//") { + return parsed, err + } + // This turns "[::]:4601" into "http://[::]:4601" which url.Parse can do + parsed, e2 := url.Parse("http://" + addr) + if e2 == nil { + // https://datatracker.ietf.org/doc/html/rfc1123#section-2 + // first character is relaxed to allow either a letter or a digit + if parsed.Host[0] == ':' && (len(parsed.Host) < 2 || parsed.Host[1] != ':') { + return nil, errURLColonHost + } + return parsed, nil + } + return parsed, err /* return original err, not our prefix altered try */ +} + +// IsMultiaddr returns true if the provided string is a valid multiaddr. +func IsMultiaddr(addr string) bool { + if strings.HasPrefix(addr, "/") && !strings.HasPrefix(addr, "//") { // multiaddr starts with '/' but not '//' which is possible for scheme relative URLS + _, err := multiaddr.NewMultiaddr(addr) + return err == nil + } + return false +} + +// ParseHostOrURLOrMultiaddr returns an error if it could not parse the provided +// string as a valid "host:port", full URL, or multiaddr. If no error, it returns +// a host:port address, or a multiaddr. +func ParseHostOrURLOrMultiaddr(addr string) (string, error) { + if strings.HasPrefix(addr, "/") && !strings.HasPrefix(addr, "//") { // multiaddr starts with '/' but not '//' which is possible for scheme relative URLS + _, err := multiaddr.NewMultiaddr(addr) + return addr, err + } + url, err := ParseHostOrURL(addr) + if err != nil { + return "", err + } + return url.Host, nil +} diff --git a/network/addr_test.go b/network/addr/addr_test.go similarity index 95% rename from network/addr_test.go rename to network/addr/addr_test.go index 377fe72a91..bceeb079f2 100644 --- a/network/addr_test.go +++ b/network/addr/addr_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with go-algorand. If not, see . -package network +package addr import ( "net/url" @@ -31,6 +31,8 @@ type urlCase struct { func TestParseHostOrURL(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() + urlTestCases := []urlCase{ {"localhost:123", url.URL{Scheme: "http", Host: "localhost:123"}}, {"http://localhost:123", url.URL{Scheme: "http", Host: "localhost:123"}}, @@ -89,10 +91,12 @@ func TestParseHostOrURL(t *testing.T) { t.Run(addr, func(t *testing.T) { _, err := ParseHostOrURL(addr) require.Error(t, err, "url should fail", addr) + require.False(t, IsMultiaddr(addr)) }) t.Run(addr+"-multiaddr", func(t *testing.T) { _, err := ParseHostOrURLOrMultiaddr(addr) require.Error(t, err, "url should fail", addr) + require.False(t, IsMultiaddr(addr)) }) } @@ -100,6 +104,7 @@ func TestParseHostOrURL(t *testing.T) { func TestParseHostURLOrMultiaddr(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() validMultiAddrs := []string{ "/ip4/127.0.0.1/tcp/8080", @@ -124,6 +129,7 @@ func TestParseHostURLOrMultiaddr(t *testing.T) { v, err := ParseHostOrURLOrMultiaddr(addr) require.NoError(t, err) require.Equal(t, addr, v) + require.True(t, IsMultiaddr(addr)) }) } @@ -131,6 +137,7 @@ func TestParseHostURLOrMultiaddr(t *testing.T) { t.Run(addr, func(t *testing.T) { _, err := ParseHostOrURLOrMultiaddr(addr) require.Error(t, err) + require.False(t, IsMultiaddr(addr)) }) } diff --git a/network/connPerfMon_test.go b/network/connPerfMon_test.go index b2f24f1214..560be72a96 100644 --- a/network/connPerfMon_test.go +++ b/network/connPerfMon_test.go @@ -48,7 +48,7 @@ func makeMsgPool(N int, peers []Peer) (out []IncomingMessage) { addMsg := func(msgCount int) { for i := 0; i < msgCount; i++ { - msg.Sender = peers[(int(msgIndex)+i)%len(peers)] + msg.Sender = peers[(int(msgIndex)+i)%len(peers)].(DisconnectablePeer) timer += int64(7 * time.Nanosecond) msg.Received = timer out = append(out, msg) diff --git a/network/gossipNode.go b/network/gossipNode.go index 3ac5cc7df0..6a028ff193 100644 --- a/network/gossipNode.go +++ b/network/gossipNode.go @@ -18,8 +18,9 @@ package network import ( "context" - "net" "net/http" + "strings" + "time" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/protocol" @@ -28,6 +29,11 @@ import ( // Peer opaque interface for referring to a neighbor in the network type Peer interface{} +// DisconnectablePeer is a Peer with a long-living connection to a network that can be disconnected +type DisconnectablePeer interface { + GetNetwork() GossipNode +} + // PeerOption allows users to specify a subset of peers to query // //msgp:ignore PeerOption @@ -44,12 +50,19 @@ const ( PeersPhonebookArchivalNodes PeerOption = iota ) +// DeadlineSettableConn abstracts net.Conn and related types as deadline-settable +type DeadlineSettableConn interface { + SetDeadline(time.Time) error + SetReadDeadline(time.Time) error + SetWriteDeadline(time.Time) error +} + // GossipNode represents a node in the gossip network type GossipNode interface { Address() (string, bool) Broadcast(ctx context.Context, tag protocol.Tag, data []byte, wait bool, except Peer) error Relay(ctx context.Context, tag protocol.Tag, data []byte, wait bool, except Peer) error - Disconnect(badnode Peer) + Disconnect(badnode DisconnectablePeer) DisconnectPeers() // only used by testing // RegisterHTTPHandler path accepts gorilla/mux path annotations @@ -64,7 +77,7 @@ type GossipNode interface { GetPeers(options ...PeerOption) []Peer // Start threads, listen on sockets. - Start() + Start() error // Close sockets. Stop threads. Stop() @@ -75,8 +88,15 @@ type GossipNode interface { // ClearHandlers deregisters all the existing message handlers. ClearHandlers() - // GetRoundTripper returns a Transport that would limit the number of outgoing connections. - GetRoundTripper() http.RoundTripper + // RegisterProcessors adds to the set of given message processors. + RegisterProcessors(dispatch []TaggedMessageProcessor) + + // ClearProcessors deregisters all the existing message processors. + ClearProcessors() + + // GetHTTPClient returns a http.Client with a suitable for the network Transport + // that would also limit the number of outgoing connections. + GetHTTPClient(address string) (*http.Client, error) // OnNetworkAdvance notifies the network library that the agreement protocol was able to make a notable progress. // this is the only indication that we have that we haven't formed a clique, where all incoming messages @@ -86,10 +106,10 @@ type GossipNode interface { // GetHTTPRequestConnection returns the underlying connection for the given request. Note that the request must be the same // request that was provided to the http handler ( or provide a fallback Context() to that ) - GetHTTPRequestConnection(request *http.Request) (conn net.Conn) + GetHTTPRequestConnection(request *http.Request) (conn DeadlineSettableConn) - // SubstituteGenesisID substitutes the "{genesisID}" with their network-specific genesisID. - SubstituteGenesisID(rawURL string) string + // GetGenesisID returns the network-specific genesisID. + GetGenesisID() string // called from wsPeer to report that it has closed peerRemoteClose(peer *wsPeer, reason disconnectReason) @@ -107,7 +127,7 @@ var outgoingMessagesBufferSize = int( // IncomingMessage represents a message arriving from some peer in our p2p network type IncomingMessage struct { - Sender Peer + Sender DisconnectablePeer Tag Tag Data []byte Err error @@ -148,6 +168,14 @@ type OutgoingMessage struct { OnRelease func() } +// ValidatedMessage is a message that has been validated and is ready to be processed. +// Think as an intermediate one between IncomingMessage and OutgoingMessage +type ValidatedMessage struct { + Action ForwardingPolicy + Tag Tag + ValidatedMessage interface{} +} + // ForwardingPolicy is an enum indicating to whom we should send a message // //msgp:ignore ForwardingPolicy @@ -165,6 +193,9 @@ const ( // Respond - reply to the sender Respond + + // Accept - accept for further processing after successful validation + Accept ) // MessageHandler takes a IncomingMessage (e.g., vote, transaction), processes it, and returns what (if anything) @@ -175,20 +206,51 @@ type MessageHandler interface { Handle(message IncomingMessage) OutgoingMessage } -// HandlerFunc represents an implemenation of the MessageHandler interface +// HandlerFunc represents an implementation of the MessageHandler interface type HandlerFunc func(message IncomingMessage) OutgoingMessage -// Handle implements MessageHandler.Handle, calling the handler with the IncomingKessage and returning the OutgoingMessage +// Handle implements MessageHandler.Handle, calling the handler with the IncomingMessage and returning the OutgoingMessage func (f HandlerFunc) Handle(message IncomingMessage) OutgoingMessage { return f(message) } -// TaggedMessageHandler receives one type of broadcast messages -type TaggedMessageHandler struct { +// MessageProcessor takes a IncomingMessage (e.g., vote, transaction), processes it, and returns what (if anything) +// to send to the network in response. +// This is an extension of the MessageHandler that works in two stages: validate ->[result]-> handle. +type MessageProcessor interface { + Validate(message IncomingMessage) ValidatedMessage + Handle(message ValidatedMessage) OutgoingMessage +} + +// ProcessorValidateFunc represents an implementation of the MessageProcessor interface +type ProcessorValidateFunc func(message IncomingMessage) ValidatedMessage + +// ProcessorHandleFunc represents an implementation of the MessageProcessor interface +type ProcessorHandleFunc func(message ValidatedMessage) OutgoingMessage + +// Validate implements MessageProcessor.Validate, calling the validator with the IncomingMessage and returning the action +// and validation extra data that can be use as the handler input. +func (f ProcessorValidateFunc) Validate(message IncomingMessage) ValidatedMessage { + return f(message) +} + +// Handle implements MessageProcessor.Handle calling the handler with the ValidatedMessage and returning the OutgoingMessage +func (f ProcessorHandleFunc) Handle(message ValidatedMessage) OutgoingMessage { + return f(message) +} + +type taggedMessageDispatcher[T any] struct { Tag - MessageHandler + MessageHandler T } +// TaggedMessageHandler receives one type of broadcast messages +type TaggedMessageHandler = taggedMessageDispatcher[MessageHandler] + +// TaggedMessageProcessor receives one type of broadcast messages +// and performs two stage processing: validating and handling +type TaggedMessageProcessor = taggedMessageDispatcher[MessageProcessor] + // Propagate is a convenience function to save typing in the common case of a message handler telling us to propagate an incoming message // "return network.Propagate(msg)" instead of "return network.OutgoingMsg{network.Broadcast, msg.Tag, msg.Data}" func Propagate(msg IncomingMessage) OutgoingMessage { @@ -205,3 +267,8 @@ func max(numbers ...uint64) (maxNum uint64) { } return } + +// SubstituteGenesisID substitutes the "{genesisID}" with their network-specific genesisID. +func SubstituteGenesisID(net GossipNode, rawURL string) string { + return strings.Replace(rawURL, "{genesisID}", net.GetGenesisID(), -1) +} diff --git a/network/hybridNetwork.go b/network/hybridNetwork.go new file mode 100644 index 0000000000..f324deb73f --- /dev/null +++ b/network/hybridNetwork.go @@ -0,0 +1,233 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package network + +import ( + "context" + "fmt" + "net/http" + "sync" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/network/addr" + "github.com/algorand/go-algorand/protocol" +) + +// HybridP2PNetwork runs both P2PNetwork and WebsocketNetwork to implement the GossipNode interface +type HybridP2PNetwork struct { + p2pNetwork *P2PNetwork + wsNetwork *WebsocketNetwork + genesisID string + + useP2PAddress bool +} + +// NewHybridP2PNetwork constructs a GossipNode that combines P2PNetwork and WebsocketNetwork +func NewHybridP2PNetwork(log logging.Logger, cfg config.Local, datadir string, phonebookAddresses []string, genesisID string, networkID protocol.NetworkID, nodeInfo NodeInfo) (*HybridP2PNetwork, error) { + // supply alternate NetAddress for P2P network + p2pcfg := cfg + p2pcfg.NetAddress = cfg.P2PNetAddress + p2pnet, err := NewP2PNetwork(log, p2pcfg, datadir, phonebookAddresses, genesisID, networkID, nodeInfo) + if err != nil { + return nil, err + } + wsnet, err := NewWebsocketNetwork(log, cfg, phonebookAddresses, genesisID, networkID, nodeInfo, p2pnet.PeerID(), p2pnet.PeerIDSigner()) + if err != nil { + return nil, err + } + return &HybridP2PNetwork{ + p2pNetwork: p2pnet, + wsNetwork: wsnet, + genesisID: genesisID, + }, nil +} + +// Address implements GossipNode +func (n *HybridP2PNetwork) Address() (string, bool) { + // TODO map from configuration? used for REST API, goal status, algod.net, etc + if n.useP2PAddress { + return n.p2pNetwork.Address() + } + return n.wsNetwork.Address() +} + +type hybridNetworkError struct{ p2pErr, wsErr error } + +func (e *hybridNetworkError) Error() string { + return fmt.Sprintf("p2pErr: %s, wsErr: %s", e.p2pErr, e.wsErr) +} +func (e *hybridNetworkError) Unwrap() []error { return []error{e.p2pErr, e.wsErr} } + +func (n *HybridP2PNetwork) runParallel(fn func(net GossipNode) error) error { + var wg sync.WaitGroup + var p2pErr, wsErr error + + wg.Add(2) + go func() { + defer wg.Done() + p2pErr = fn(n.p2pNetwork) + }() + go func() { + defer wg.Done() + wsErr = fn(n.wsNetwork) + }() + wg.Wait() + + if p2pErr != nil && wsErr != nil { + return &hybridNetworkError{p2pErr, wsErr} + } + if p2pErr != nil { + return p2pErr + } + if wsErr != nil { + return wsErr + } + return nil +} + +// Broadcast implements GossipNode +func (n *HybridP2PNetwork) Broadcast(ctx context.Context, tag protocol.Tag, data []byte, wait bool, except Peer) error { + return n.runParallel(func(net GossipNode) error { + return net.Broadcast(ctx, tag, data, wait, except) + }) +} + +// Relay implements GossipNode +func (n *HybridP2PNetwork) Relay(ctx context.Context, tag protocol.Tag, data []byte, wait bool, except Peer) error { + return n.runParallel(func(net GossipNode) error { + return net.Relay(ctx, tag, data, wait, except) + }) +} + +// Disconnect implements GossipNode +func (n *HybridP2PNetwork) Disconnect(badnode DisconnectablePeer) { + net := badnode.GetNetwork() + if net == n.p2pNetwork { + n.p2pNetwork.Disconnect(badnode) + } else if net == n.wsNetwork { + n.wsNetwork.Disconnect(badnode) + } else { + panic("badnode.GetNetwork() returned a network that is not part of this HybridP2PNetwork") + } +} + +// DisconnectPeers implements GossipNode +func (n *HybridP2PNetwork) DisconnectPeers() { + _ = n.runParallel(func(net GossipNode) error { + net.DisconnectPeers() + return nil + }) +} + +// RegisterHTTPHandler implements GossipNode +func (n *HybridP2PNetwork) RegisterHTTPHandler(path string, handler http.Handler) { + n.p2pNetwork.RegisterHTTPHandler(path, handler) + n.wsNetwork.RegisterHTTPHandler(path, handler) +} + +// RequestConnectOutgoing implements GossipNode +func (n *HybridP2PNetwork) RequestConnectOutgoing(replace bool, quit <-chan struct{}) {} + +// GetPeers implements GossipNode +func (n *HybridP2PNetwork) GetPeers(options ...PeerOption) []Peer { + // TODO better way of combining data from peerstore and returning in GetPeers + var peers []Peer + peers = append(peers, n.p2pNetwork.GetPeers(options...)...) + peers = append(peers, n.wsNetwork.GetPeers(options...)...) + return peers +} + +// Start implements GossipNode +func (n *HybridP2PNetwork) Start() error { + err := n.runParallel(func(net GossipNode) error { + return net.Start() + }) + return err +} + +// Stop implements GossipNode +func (n *HybridP2PNetwork) Stop() { + _ = n.runParallel(func(net GossipNode) error { + net.Stop() + return nil + }) +} + +// RegisterHandlers adds to the set of given message handlers. +func (n *HybridP2PNetwork) RegisterHandlers(dispatch []TaggedMessageHandler) { + n.p2pNetwork.RegisterHandlers(dispatch) + n.wsNetwork.RegisterHandlers(dispatch) +} + +// ClearHandlers deregisters all the existing message handlers. +func (n *HybridP2PNetwork) ClearHandlers() { + n.p2pNetwork.ClearHandlers() + n.wsNetwork.ClearHandlers() +} + +// RegisterProcessors adds to the set of given message processors. +func (n *HybridP2PNetwork) RegisterProcessors(dispatch []TaggedMessageProcessor) { + n.p2pNetwork.RegisterProcessors(dispatch) + n.wsNetwork.RegisterProcessors(dispatch) +} + +// ClearProcessors deregisters all the existing message processors. +func (n *HybridP2PNetwork) ClearProcessors() { + n.p2pNetwork.ClearProcessors() + n.wsNetwork.ClearProcessors() +} + +// GetHTTPClient returns a http.Client with a suitable for the network Transport +// that would also limit the number of outgoing connections. +func (n *HybridP2PNetwork) GetHTTPClient(address string) (*http.Client, error) { + if addr.IsMultiaddr(address) { + return n.p2pNetwork.GetHTTPClient(address) + } + return n.wsNetwork.GetHTTPClient(address) +} + +// OnNetworkAdvance notifies the network library that the agreement protocol was able to make a notable progress. +// this is the only indication that we have that we haven't formed a clique, where all incoming messages +// arrive very quickly, but might be missing some votes. The usage of this call is expected to have similar +// characteristics as with a watchdog timer. +func (n *HybridP2PNetwork) OnNetworkAdvance() { + _ = n.runParallel(func(net GossipNode) error { + net.OnNetworkAdvance() + return nil + }) +} + +// GetHTTPRequestConnection returns the underlying connection for the given request. Note that the request must be the same +// request that was provided to the http handler ( or provide a fallback Context() to that ) +func (n *HybridP2PNetwork) GetHTTPRequestConnection(request *http.Request) (conn DeadlineSettableConn) { + conn = n.wsNetwork.GetHTTPRequestConnection(request) + if conn != nil { + return conn + } + return n.p2pNetwork.GetHTTPRequestConnection(request) +} + +// GetGenesisID returns the network-specific genesisID. +func (n *HybridP2PNetwork) GetGenesisID() string { + return n.genesisID +} + +// called from wsPeer to report that it has closed +func (n *HybridP2PNetwork) peerRemoteClose(peer *wsPeer, reason disconnectReason) { + panic("wsPeer should only call WebsocketNetwork.peerRemoteClose or P2PNetwork.peerRemoteClose") +} diff --git a/network/dialer.go b/network/limitcaller/dialer.go similarity index 91% rename from network/dialer.go rename to network/limitcaller/dialer.go index 3aa59f493d..ee9b2e364a 100644 --- a/network/dialer.go +++ b/network/limitcaller/dialer.go @@ -14,13 +14,14 @@ // You should have received a copy of the GNU Affero General Public License // along with go-algorand. If not, see . -package network +package limitcaller import ( "context" "net" "time" + "github.com/algorand/go-algorand/network/phonebook" "github.com/algorand/go-algorand/tools/network/dnssec" "github.com/algorand/go-algorand/util" ) @@ -31,14 +32,13 @@ type netDialer interface { // Dialer establish tcp-level connection with the destination type Dialer struct { - phonebook Phonebook + phonebook phonebook.Phonebook innerDialer netDialer - resolver *net.Resolver } -// makeRateLimitingDialer creates a rate limiting dialer that would limit the connections +// MakeRateLimitingDialer creates a rate limiting dialer that would limit the connections // according to the entries in the phonebook. -func makeRateLimitingDialer(phonebook Phonebook, resolver dnssec.ResolverIf) Dialer { +func MakeRateLimitingDialer(phonebook phonebook.Phonebook, resolver dnssec.ResolverIf) Dialer { var innerDialer netDialer = &net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, diff --git a/network/rateLimitingTransport.go b/network/limitcaller/rateLimitingTransport.go similarity index 55% rename from network/rateLimitingTransport.go rename to network/limitcaller/rateLimitingTransport.go index 461a468da5..45bc0725ed 100644 --- a/network/rateLimitingTransport.go +++ b/network/limitcaller/rateLimitingTransport.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with go-algorand. If not, see . -package network +package limitcaller import ( "errors" @@ -22,24 +22,35 @@ import ( "time" "github.com/algorand/go-algorand/util" + "github.com/libp2p/go-libp2p/core/peer" ) -// rateLimitingTransport is the transport for execute a single HTTP transaction, obtaining the Response for a given Request. -type rateLimitingTransport struct { - phonebook Phonebook - innerTransport *http.Transport +// ConnectionTimeStore is a subset of the phonebook that is used to store the connection times. +type ConnectionTimeStore interface { + GetConnectionWaitTime(addrOrPeerID string) (bool, time.Duration, time.Time) + UpdateConnectionTime(addrOrPeerID string, provisionalTime time.Time) bool +} + +// RateLimitingTransport is the transport for execute a single HTTP transaction, obtaining the Response for a given Request. +type RateLimitingTransport struct { + phonebook ConnectionTimeStore + innerTransport http.RoundTripper queueingTimeout time.Duration + targetAddr interface{} // target address for the p2p http request } +// DefaultQueueingTimeout is the default timeout for queueing the request. +const DefaultQueueingTimeout = 10 * time.Second + // ErrConnectionQueueingTimeout indicates that we've exceeded the time allocated for // queueing the current request before the request attempt could be made. var ErrConnectionQueueingTimeout = errors.New("rateLimitingTransport: queueing timeout") -// makeRateLimitingTransport creates a rate limiting http transport that would limit the requests rate +// MakeRateLimitingTransport creates a rate limiting http transport that would limit the requests rate // according to the entries in the phonebook. -func makeRateLimitingTransport(phonebook Phonebook, queueingTimeout time.Duration, dialer *Dialer, maxIdleConnsPerHost int) rateLimitingTransport { +func MakeRateLimitingTransport(phonebook ConnectionTimeStore, queueingTimeout time.Duration, dialer *Dialer, maxIdleConnsPerHost int) RateLimitingTransport { defaultTransport := http.DefaultTransport.(*http.Transport) - return rateLimitingTransport{ + return RateLimitingTransport{ phonebook: phonebook, innerTransport: &http.Transport{ Proxy: defaultTransport.Proxy, @@ -54,14 +65,34 @@ func makeRateLimitingTransport(phonebook Phonebook, queueingTimeout time.Duratio } } +// MakeRateLimitingTransportWithRoundTripper creates a rate limiting http transport that would limit the requests rate +// according to the entries in the phonebook. +func MakeRateLimitingTransportWithRoundTripper(phonebook ConnectionTimeStore, queueingTimeout time.Duration, rt http.RoundTripper, target interface{}, maxIdleConnsPerHost int) RateLimitingTransport { + return RateLimitingTransport{ + phonebook: phonebook, + innerTransport: rt, + queueingTimeout: queueingTimeout, + targetAddr: target, + } +} + // RoundTrip connects to the address on the named network using the provided context. // It waits if needed not to exceed connectionsRateLimitingCount. -func (r *rateLimitingTransport) RoundTrip(req *http.Request) (res *http.Response, err error) { +func (r *RateLimitingTransport) RoundTrip(req *http.Request) (res *http.Response, err error) { var waitTime time.Duration var provisionalTime time.Time queueingDeadline := time.Now().Add(r.queueingTimeout) + addrOrPeerID := req.Host + // p2p/http clients have per-connection transport and address info so use that + if len(req.Host) == 0 && req.URL != nil && len(req.URL.Host) == 0 { + addrInfo, ok := r.targetAddr.(*peer.AddrInfo) + if !ok { + return nil, errors.New("rateLimitingTransport: request without Host/URL and targetAddr is not a peer.AddrInfo") + } + addrOrPeerID = string(addrInfo.ID) + } for { - _, waitTime, provisionalTime = r.phonebook.GetConnectionWaitTime(req.Host) + _, waitTime, provisionalTime = r.phonebook.GetConnectionWaitTime(addrOrPeerID) if waitTime == 0 { break // break out of the loop and proceed to the connection } @@ -73,6 +104,6 @@ func (r *rateLimitingTransport) RoundTrip(req *http.Request) (res *http.Response return nil, ErrConnectionQueueingTimeout } res, err = r.innerTransport.RoundTrip(req) - r.phonebook.UpdateConnectionTime(req.Host, provisionalTime) + r.phonebook.UpdateConnectionTime(addrOrPeerID, provisionalTime) return } diff --git a/network/multiplexer.go b/network/multiplexer.go index 0e97d63f28..dc38fba277 100644 --- a/network/multiplexer.go +++ b/network/multiplexer.go @@ -24,49 +24,74 @@ import ( // Multiplexer is a message handler that sorts incoming messages by Tag and passes // them along to the relevant message handler for that type of message. type Multiplexer struct { - msgHandlers atomic.Value // stores map[Tag]MessageHandler, an immutable map. + msgHandlers atomic.Value // stores map[Tag]MessageHandler, an immutable map. + msgProcessors atomic.Value // stores map[Tag]MessageProcessor, an immutable map. } // MakeMultiplexer creates an empty Multiplexer func MakeMultiplexer() *Multiplexer { m := &Multiplexer{} - m.ClearHandlers([]Tag{}) // allocate the map + m.ClearHandlers(nil) // allocate the map + m.ClearProcessors(nil) // allocate the map return m } -// getHandlersMap retrieves the handlers map. -func (m *Multiplexer) getHandlersMap() map[Tag]MessageHandler { - handlersVal := m.msgHandlers.Load() - if handlers, valid := handlersVal.(map[Tag]MessageHandler); valid { +// getMap retrieves a typed map from an atomic.Value. +func getMap[T any](source *atomic.Value) map[Tag]T { + mp := source.Load() + if handlers, valid := mp.(map[Tag]T); valid { return handlers } return nil } -// Retrives the handler for the given message Tag from the handlers array while taking a read lock. -func (m *Multiplexer) getHandler(tag Tag) (MessageHandler, bool) { - if handlers := m.getHandlersMap(); handlers != nil { +// Retrieves the handler for the given message Tag from the given value while. +func getHandler[T any](source *atomic.Value, tag Tag) (T, bool) { + if handlers := getMap[T](source); handlers != nil { handler, ok := handlers[tag] return handler, ok } - return nil, false + var empty T + return empty, false +} + +// Retrieves the handler for the given message Tag from the handlers array. +func (m *Multiplexer) getHandler(tag Tag) (MessageHandler, bool) { + return getHandler[MessageHandler](&m.msgHandlers, tag) +} + +// Retrieves the processor for the given message Tag from the processors array. +func (m *Multiplexer) getProcessor(tag Tag) (MessageProcessor, bool) { + return getHandler[MessageProcessor](&m.msgProcessors, tag) } // Handle is the "input" side of the multiplexer. It dispatches the message to the previously defined handler. func (m *Multiplexer) Handle(msg IncomingMessage) OutgoingMessage { - handler, ok := m.getHandler(msg.Tag) + if handler, ok := m.getHandler(msg.Tag); ok { + return handler.Handle(msg) + } + return OutgoingMessage{} +} - if ok { - outmsg := handler.Handle(msg) - return outmsg +// Validate is an alternative "input" side of the multiplexer. It dispatches the message to the previously defined validator. +func (m *Multiplexer) Validate(msg IncomingMessage) ValidatedMessage { + if handler, ok := m.getProcessor(msg.Tag); ok { + return handler.Validate(msg) + } + return ValidatedMessage{} +} + +// Process is the second step of message handling after validation. It dispatches the message to the previously defined processor. +func (m *Multiplexer) Process(msg ValidatedMessage) OutgoingMessage { + if handler, ok := m.getProcessor(msg.Tag); ok { + return handler.Handle(msg) } return OutgoingMessage{} } -// RegisterHandlers registers the set of given message handlers. -func (m *Multiplexer) RegisterHandlers(dispatch []TaggedMessageHandler) { - mp := make(map[Tag]MessageHandler) - if existingMap := m.getHandlersMap(); existingMap != nil { +func registerMultiplexer[T any](target *atomic.Value, dispatch []taggedMessageDispatcher[T]) { + mp := make(map[Tag]T) + if existingMap := getMap[T](target); existingMap != nil { for k, v := range existingMap { mp[k] = v } @@ -77,13 +102,22 @@ func (m *Multiplexer) RegisterHandlers(dispatch []TaggedMessageHandler) { } mp[v.Tag] = v.MessageHandler } - m.msgHandlers.Store(mp) + target.Store(mp) } -// ClearHandlers deregisters all the existing message handlers other than the one provided in the excludeTags list -func (m *Multiplexer) ClearHandlers(excludeTags []Tag) { +// RegisterHandlers registers the set of given message handlers. +func (m *Multiplexer) RegisterHandlers(dispatch []TaggedMessageHandler) { + registerMultiplexer(&m.msgHandlers, dispatch) +} + +// RegisterProcessors registers the set of given message handlers. +func (m *Multiplexer) RegisterProcessors(dispatch []TaggedMessageProcessor) { + registerMultiplexer(&m.msgProcessors, dispatch) +} + +func clearMultiplexer[T any](target *atomic.Value, excludeTags []Tag) { if len(excludeTags) == 0 { - m.msgHandlers.Store(make(map[Tag]MessageHandler)) + target.Store(make(map[Tag]T)) return } @@ -93,13 +127,23 @@ func (m *Multiplexer) ClearHandlers(excludeTags []Tag) { excludeTagsMap[tag] = true } - currentHandlersMap := m.getHandlersMap() - newMap := make(map[Tag]MessageHandler, len(excludeTagsMap)) - for tag, handler := range currentHandlersMap { + currentMap := getMap[T](target) + newMap := make(map[Tag]T, len(excludeTagsMap)) + for tag, handler := range currentMap { if excludeTagsMap[tag] { newMap[tag] = handler } } - m.msgHandlers.Store(newMap) + target.Store(newMap) +} + +// ClearHandlers deregisters all the existing message handlers other than the one provided in the excludeTags list +func (m *Multiplexer) ClearHandlers(excludeTags []Tag) { + clearMultiplexer[MessageHandler](&m.msgHandlers, excludeTags) +} + +// ClearProcessors deregisters all the existing message handlers other than the one provided in the excludeTags list +func (m *Multiplexer) ClearProcessors(excludeTags []Tag) { + clearMultiplexer[MessageProcessor](&m.msgProcessors, excludeTags) } diff --git a/network/netidentity.go b/network/netidentity.go index 1474739bd1..4d797a1a5b 100644 --- a/network/netidentity.go +++ b/network/netidentity.go @@ -94,12 +94,34 @@ type identityChallengeScheme interface { VerifyResponse(h http.Header, c identityChallengeValue) (crypto.PublicKey, []byte, error) } +type identityChallengeSigner interface { + Sign(message crypto.Hashable) crypto.Signature + SignBytes(message []byte) crypto.Signature + PublicKey() crypto.PublicKey +} + +type identityChallengeLegacySigner struct { + keys *crypto.SignatureSecrets +} + +func (s *identityChallengeLegacySigner) Sign(message crypto.Hashable) crypto.Signature { + return s.keys.Sign(message) +} + +func (s *identityChallengeLegacySigner) SignBytes(message []byte) crypto.Signature { + return s.keys.SignBytes(message) +} + +func (s *identityChallengeLegacySigner) PublicKey() crypto.PublicKey { + return s.keys.SignatureVerifier +} + // identityChallengePublicKeyScheme implements IdentityChallengeScheme by // exchanging and verifying public key challenges and attaching them to headers, // or returning the message payload to be sent type identityChallengePublicKeyScheme struct { dedupName string - identityKeys *crypto.SignatureSecrets + identityKeys identityChallengeSigner } // NewIdentityChallengeScheme will create a default Identification Scheme @@ -108,15 +130,21 @@ func NewIdentityChallengeScheme(dn string) *identityChallengePublicKeyScheme { if dn == "" { return &identityChallengePublicKeyScheme{} } + var seed crypto.Seed crypto.RandBytes(seed[:]) return &identityChallengePublicKeyScheme{ dedupName: dn, - identityKeys: crypto.GenerateSignatureSecrets(seed), + identityKeys: &identityChallengeLegacySigner{keys: crypto.GenerateSignatureSecrets(seed)}, } } +// NewIdentityChallengeSchemeWithSigner will create an identification Scheme with a given signer +func NewIdentityChallengeSchemeWithSigner(dn string, signer identityChallengeSigner) *identityChallengePublicKeyScheme { + return &identityChallengePublicKeyScheme{dedupName: dn, identityKeys: signer} +} + // AttachChallenge will generate a new identity challenge and will encode and attach the challenge // as a header. It returns the identityChallengeValue used for this challenge, so the network can // confirm it later (by passing it to VerifyResponse), or returns an empty challenge if dedupName is @@ -126,7 +154,7 @@ func (i identityChallengePublicKeyScheme) AttachChallenge(attachTo http.Header, return identityChallengeValue{} } c := identityChallenge{ - Key: i.identityKeys.SignatureVerifier, + Key: i.identityKeys.PublicKey(), Challenge: newIdentityChallengeValue(), PublicAddress: []byte(addr), } @@ -173,7 +201,7 @@ func (i identityChallengePublicKeyScheme) VerifyRequestAndAttachResponse(attachT } // make the response object, encode it and attach it to the header r := identityChallengeResponse{ - Key: i.identityKeys.SignatureVerifier, + Key: i.identityKeys.PublicKey(), Challenge: idChal.Msg.Challenge, ResponseChallenge: newIdentityChallengeValue(), } @@ -271,12 +299,12 @@ type identityVerificationMessageSigned struct { Signature crypto.Signature `codec:"sig"` } -func (i identityChallenge) signAndEncodeB64(s *crypto.SignatureSecrets) string { +func (i identityChallenge) signAndEncodeB64(s identityChallengeSigner) string { signedChal := i.Sign(s) return base64.StdEncoding.EncodeToString(protocol.Encode(&signedChal)) } -func (i identityChallenge) Sign(secrets *crypto.SignatureSecrets) identityChallengeSigned { +func (i identityChallenge) Sign(secrets identityChallengeSigner) identityChallengeSigned { return identityChallengeSigned{Msg: i, Signature: secrets.Sign(i)} } @@ -289,12 +317,12 @@ func (i identityChallengeSigned) Verify() bool { return i.Msg.Key.Verify(i.Msg, i.Signature) } -func (i identityChallengeResponse) signAndEncodeB64(s *crypto.SignatureSecrets) string { +func (i identityChallengeResponse) signAndEncodeB64(s identityChallengeSigner) string { signedChalResp := i.Sign(s) return base64.StdEncoding.EncodeToString(protocol.Encode(&signedChalResp)) } -func (i identityChallengeResponse) Sign(secrets *crypto.SignatureSecrets) identityChallengeResponseSigned { +func (i identityChallengeResponse) Sign(secrets identityChallengeSigner) identityChallengeResponseSigned { return identityChallengeResponseSigned{Msg: i, Signature: secrets.Sign(i)} } @@ -307,7 +335,7 @@ func (i identityChallengeResponseSigned) Verify() bool { return i.Msg.Key.Verify(i.Msg, i.Signature) } -func (i identityVerificationMessage) Sign(secrets *crypto.SignatureSecrets) identityVerificationMessageSigned { +func (i identityVerificationMessage) Sign(secrets identityChallengeSigner) identityVerificationMessageSigned { return identityVerificationMessageSigned{Msg: i, Signature: secrets.Sign(i)} } diff --git a/network/netidentity_test.go b/network/netidentity_test.go index 9650069224..f87480c1b1 100644 --- a/network/netidentity_test.go +++ b/network/netidentity_test.go @@ -180,7 +180,7 @@ func TestIdentityChallengeSchemeBadSignature(t *testing.T) { // Copy the logic of attaching the header and signing so we can sign it wrong c := identityChallengeSigned{ Msg: identityChallenge{ - Key: i.identityKeys.SignatureVerifier, + Key: i.identityKeys.PublicKey(), Challenge: newIdentityChallengeValue(), PublicAddress: []byte("i1"), }} @@ -232,7 +232,7 @@ func TestIdentityChallengeSchemeBadResponseSignature(t *testing.T) { r := http.Header{} resp := identityChallengeResponseSigned{ Msg: identityChallengeResponse{ - Key: i.identityKeys.SignatureVerifier, + Key: i.identityKeys.PublicKey(), Challenge: origChal, ResponseChallenge: newIdentityChallengeValue(), }} diff --git a/network/netprio.go b/network/netprio.go index 5d91dad8d1..9c6c510608 100644 --- a/network/netprio.go +++ b/network/netprio.go @@ -46,7 +46,7 @@ func prioResponseHandler(message IncomingMessage) OutgoingMessage { addr, err := wn.prioScheme.VerifyPrioResponse(challenge, message.Data) if err != nil { - wn.log.Warnf("prioScheme.VerifyPrioResponse from %s: %v", peer.rootURL, err) + wn.log.Warnf("prioScheme.VerifyPrioResponse from %s: %v", peer.GetAddress(), err) } else { weight := wn.prioScheme.GetPrioWeight(addr) diff --git a/network/p2p/capabilities.go b/network/p2p/capabilities.go new file mode 100644 index 0000000000..e5781aa389 --- /dev/null +++ b/network/p2p/capabilities.go @@ -0,0 +1,179 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package p2p + +import ( + "context" + "sync" + "time" + + dht "github.com/libp2p/go-libp2p-kad-dht" + "github.com/libp2p/go-libp2p/core/discovery" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + libpeerstore "github.com/libp2p/go-libp2p/core/peerstore" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/logging" + algoDht "github.com/algorand/go-algorand/network/p2p/dht" + "github.com/algorand/go-algorand/protocol" +) + +// Capability represents functions that some nodes may provide and other nodes would want to know about +type Capability string + +const ( + // Archival nodes + Archival Capability = "archival" + // Catchpoints storing nodes + Catchpoints = "catchpointStoring" + // Gossip nodes are non permissioned relays + Gossip = "gossip" +) + +const operationTimeout = time.Second * 5 +const maxAdvertisementInterval = time.Hour * 22 + +// CapabilitiesDiscovery exposes Discovery interfaces and wraps underlying DHT methods to provide capabilities advertisement for the node +type CapabilitiesDiscovery struct { + disc discovery.Discovery + dht *dht.IpfsDHT + log logging.Logger + wg sync.WaitGroup +} + +// Advertise implements the discovery.Discovery/discovery.Advertiser interface +func (c *CapabilitiesDiscovery) Advertise(ctx context.Context, ns string, opts ...discovery.Option) (time.Duration, error) { + return c.disc.Advertise(ctx, ns, opts...) +} + +// FindPeers implements the discovery.Discovery/discovery.Discoverer interface +func (c *CapabilitiesDiscovery) FindPeers(ctx context.Context, ns string, opts ...discovery.Option) (<-chan peer.AddrInfo, error) { + return c.disc.FindPeers(ctx, ns, opts...) +} + +// Close should be called when fully shutting down the node +func (c *CapabilitiesDiscovery) Close() error { + err := c.dht.Close() + c.wg.Wait() + return err +} + +// Host exposes the underlying libp2p host.Host object +func (c *CapabilitiesDiscovery) Host() host.Host { + return c.dht.Host() +} + +// AddPeer adds a given peer.AddrInfo to the Host's Peerstore, and the DHT's routing table +func (c *CapabilitiesDiscovery) AddPeer(p peer.AddrInfo) (bool, error) { + c.Host().Peerstore().AddAddrs(p.ID, p.Addrs, libpeerstore.AddressTTL) + return c.dht.RoutingTable().TryAddPeer(p.ID, true, true) +} + +// PeersForCapability returns a slice of peer.AddrInfo for a Capability +// Since CapabilitiesDiscovery uses a backoffcache, it will attempt to hit cache, then disk, then network +// in order to fetch n peers which are advertising the required capability. +func (c *CapabilitiesDiscovery) PeersForCapability(capability Capability, n int) ([]peer.AddrInfo, error) { + ctx, cancel := context.WithTimeout(context.Background(), operationTimeout) + defer cancel() + var peers []peer.AddrInfo + // +1 because it can include self but we exclude self from the returned list + // that might confuse the caller (and tests assertions) + peersChan, err := c.FindPeers(ctx, string(capability), discovery.Limit(n+1)) + if err != nil { + return nil, err + } + for p := range peersChan { + if p.ID.Size() > 0 && p.ID != c.Host().ID() { + peers = append(peers, p) + } + if len(peers) >= n { + break + } + } + return peers, nil +} + +// AdvertiseCapabilities periodically runs the Advertiser interface on the DHT +// If a capability fails to advertise we will retry every 10 seconds until full success +// This gets rerun every at the minimum ttl or the maxAdvertisementInterval. +func (c *CapabilitiesDiscovery) AdvertiseCapabilities(capabilities ...Capability) { + c.wg.Add(1) + go func() { + // Run the initial Advertisement immediately + nextExecution := time.After(time.Second / 10000) + defer func() { + c.wg.Done() + }() + + for { + select { + case <-c.dht.Context().Done(): + return + case <-nextExecution: + var err error + advertisementInterval := maxAdvertisementInterval + for _, capa := range capabilities { + ttl, err0 := c.Advertise(c.dht.Context(), string(capa)) + if err0 != nil { + err = err0 + c.log.Errorf("failed to advertise for capability %s: %v", capa, err0) + break + } + if ttl < advertisementInterval { + advertisementInterval = ttl + } + c.log.Infof("advertised capability %s", capa) + } + // If we failed to advertise, retry every 10 seconds until successful + if err != nil { + nextExecution = time.After(time.Second * 10) + } else { + // Otherwise, ensure we're at the correct interval + nextExecution = time.After(advertisementInterval) + } + } + } + }() +} + +// Sizer exposes the Size method +type Sizer interface { + Size() int +} + +// RoutingTable exposes some knowledge about the DHT routing table +func (c *CapabilitiesDiscovery) RoutingTable() Sizer { + return c.dht.RoutingTable() +} + +// MakeCapabilitiesDiscovery creates a new CapabilitiesDiscovery object which exposes peer discovery and capabilities advertisement +func MakeCapabilitiesDiscovery(ctx context.Context, cfg config.Local, h host.Host, networkID protocol.NetworkID, log logging.Logger, bootstrapFunc func() []peer.AddrInfo) (*CapabilitiesDiscovery, error) { + discDht, err := algoDht.MakeDHT(ctx, h, networkID, cfg, bootstrapFunc) + if err != nil { + return nil, err + } + discImpl, err := algoDht.MakeDiscovery(discDht) + if err != nil { + return nil, err + } + return &CapabilitiesDiscovery{ + disc: discImpl, + dht: discDht, + log: log, + }, nil +} diff --git a/network/p2p/capabilities_test.go b/network/p2p/capabilities_test.go new file mode 100644 index 0000000000..881860f647 --- /dev/null +++ b/network/p2p/capabilities_test.go @@ -0,0 +1,354 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package p2p + +import ( + "context" + "math/rand" + "sync" + "testing" + "time" + + golog "github.com/ipfs/go-log" + "github.com/libp2p/go-libp2p" + dht "github.com/libp2p/go-libp2p-kad-dht" + "github.com/libp2p/go-libp2p/core/discovery" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/logging" + algodht "github.com/algorand/go-algorand/network/p2p/dht" + "github.com/algorand/go-algorand/network/p2p/peerstore" + "github.com/algorand/go-algorand/test/partitiontest" +) + +func TestCapabilities_Discovery(t *testing.T) { + partitiontest.PartitionTest(t) + + golog.SetDebugLogging() + var caps []*CapabilitiesDiscovery + var addrs []peer.AddrInfo + testSize := 3 + for i := 0; i < testSize; i++ { + tempdir := t.TempDir() + ps, err := peerstore.NewPeerStore(nil, "") + require.NoError(t, err) + h, _, err := MakeHost(config.GetDefaultLocal(), tempdir, ps) + require.NoError(t, err) + capD, err := MakeCapabilitiesDiscovery(context.Background(), config.GetDefaultLocal(), h, "devtestnet", logging.Base(), func() []peer.AddrInfo { return nil }) + require.NoError(t, err) + caps = append(caps, capD) + addrs = append(addrs, peer.AddrInfo{ + ID: capD.Host().ID(), + Addrs: capD.Host().Addrs(), + }) + } + for _, capD := range caps { + peersAdded := 0 + for _, addr := range addrs { + added, err := capD.AddPeer(addr) + require.NoError(t, err) + require.True(t, added) + peersAdded++ + } + err := capD.dht.Bootstrap(context.Background()) + require.NoError(t, err) + capD.dht.ForceRefresh() + require.Equal(t, peersAdded, capD.dht.RoutingTable().Size()) + } +} + +func setupDHTHosts(t *testing.T, numHosts int) []*dht.IpfsDHT { + var hosts []host.Host + var bootstrapPeers []peer.AddrInfo + var dhts []*dht.IpfsDHT + cfg := config.GetDefaultLocal() + for i := 0; i < numHosts; i++ { + tmpdir := t.TempDir() + pk, err := GetPrivKey(cfg, tmpdir) + require.NoError(t, err) + ps, err := peerstore.NewPeerStore([]*peer.AddrInfo{}, "") + require.NoError(t, err) + h, err := libp2p.New( + libp2p.ListenAddrStrings("/dns4/localhost/tcp/0"), + libp2p.Identity(pk), + libp2p.Peerstore(ps)) + require.NoError(t, err) + hosts = append(hosts, h) + bootstrapPeers = append(bootstrapPeers, peer.AddrInfo{ID: h.ID(), Addrs: h.Addrs()}) + } + for _, h := range hosts { + ht, err := algodht.MakeDHT(context.Background(), h, "devtestnet", cfg, func() []peer.AddrInfo { return bootstrapPeers }) + require.NoError(t, err) + // this is a workaround for the following issue + // "failed to negotiate security protocol: error reading handshake message: noise: message is too short" + // it appears simultaneous connection attempts (dht.New() attempts to connect) causes this handshake error. + // https://github.com/libp2p/go-libp2p-noise/issues/70 + time.Sleep(200 * time.Millisecond) + + err = ht.Bootstrap(context.Background()) + require.NoError(t, err) + dhts = append(dhts, ht) + } + return dhts +} + +func waitForRouting(t *testing.T, disc *CapabilitiesDiscovery) { + refreshCtx, refCancel := context.WithTimeout(context.Background(), time.Second*5) + for { + select { + case <-refreshCtx.Done(): + refCancel() + require.Fail(t, "failed to populate routing table before timeout") + default: + if disc.dht.RoutingTable().Size() > 0 { + refCancel() + return + } + } + } +} + +func setupCapDiscovery(t *testing.T, numHosts int, numBootstrapPeers int) []*CapabilitiesDiscovery { + var hosts []host.Host + var bootstrapPeers []peer.AddrInfo + var capsDisc []*CapabilitiesDiscovery + cfg := config.GetDefaultLocal() + for i := 0; i < numHosts; i++ { + tmpdir := t.TempDir() + pk, err := GetPrivKey(cfg, tmpdir) + require.NoError(t, err) + ps, err := peerstore.NewPeerStore([]*peer.AddrInfo{}, "") + require.NoError(t, err) + h, err := libp2p.New( + libp2p.ListenAddrStrings("/dns4/localhost/tcp/0"), + libp2p.Identity(pk), + libp2p.Peerstore(ps)) + require.NoError(t, err) + hosts = append(hosts, h) + bootstrapPeers = append(bootstrapPeers, peer.AddrInfo{ID: h.ID(), Addrs: h.Addrs()}) + } + for _, h := range hosts { + bp := bootstrapPeers + if numBootstrapPeers != 0 && numBootstrapPeers != numHosts { + bp = make([]peer.AddrInfo, len(bootstrapPeers)) + copy(bp, bootstrapPeers) + rand.Shuffle(len(bootstrapPeers), func(i, j int) { + bp[i], bp[j] = bp[j], bp[i] + }) + bp = bp[:numBootstrapPeers] + } + ht, err := algodht.MakeDHT(context.Background(), h, "devtestnet", cfg, func() []peer.AddrInfo { return bp }) + require.NoError(t, err) + // this is a workaround for the following issue + // "failed to negotiate security protocol: error reading handshake message: noise: message is too short" + // it appears simultaneous connection attempts (dht.New() attempts to connect) causes this handshake error. + // https://github.com/libp2p/go-libp2p-noise/issues/70 + time.Sleep(200 * time.Millisecond) + + disc, err := algodht.MakeDiscovery(ht) + require.NoError(t, err) + cd := &CapabilitiesDiscovery{ + disc: disc, + dht: ht, + log: logging.Base(), + } + capsDisc = append(capsDisc, cd) + } + return capsDisc +} + +func TestCapabilities_DHTTwoPeers(t *testing.T) { + partitiontest.PartitionTest(t) + + numAdvertisers := 2 + dhts := setupDHTHosts(t, numAdvertisers) + topic := "foobar" + for i, ht := range dhts { + disc, err := algodht.MakeDiscovery(ht) + require.NoError(t, err) + refreshCtx, refCancel := context.WithTimeout(context.Background(), time.Second*5) + peersPopulated: + for { + select { + case <-refreshCtx.Done(): + refCancel() + require.Fail(t, "failed to populate routing table before timeout") + default: + if ht.RoutingTable().Size() > 0 { + refCancel() + break peersPopulated + } + } + } + _, err = disc.Advertise(context.Background(), topic) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + var advertisers []peer.AddrInfo + peersChan, err := disc.FindPeers(ctx, topic, discovery.Limit(numAdvertisers)) + require.NoError(t, err) + pollingForPeers: + for { + select { + case p, open := <-peersChan: + if p.ID.Size() > 0 { + advertisers = append(advertisers, p) + } + if !open { + break pollingForPeers + } + } + } + cancel() + // Returned peers will include the querying node's ID since it advertises for the topic as well + require.Equal(t, i+1, len(advertisers)) + } +} + +func TestCapabilities_Varying(t *testing.T) { + partitiontest.PartitionTest(t) + + const numAdvertisers = 10 + + var tests = []struct { + name string + numBootstrap int + }{ + {"bootstrap=all", numAdvertisers}, + {"bootstrap=2", 2}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + capsDisc := setupCapDiscovery(t, numAdvertisers, test.numBootstrap) + noCap := capsDisc[:3] + archOnly := capsDisc[3:5] + catchOnly := capsDisc[5:7] + archCatch := capsDisc[7:] + + var wg sync.WaitGroup + wg.Add(len(archOnly) + len(catchOnly) + len(archCatch)) + for _, disc := range archOnly { + go func(disc *CapabilitiesDiscovery) { + defer wg.Done() + waitForRouting(t, disc) + disc.AdvertiseCapabilities(Archival) + }(disc) + } + for _, disc := range catchOnly { + go func(disc *CapabilitiesDiscovery) { + defer wg.Done() + waitForRouting(t, disc) + disc.AdvertiseCapabilities(Catchpoints) + }(disc) + } + for _, disc := range archCatch { + go func(disc *CapabilitiesDiscovery) { + defer wg.Done() + waitForRouting(t, disc) + disc.AdvertiseCapabilities(Archival, Catchpoints) + }(disc) + } + + wg.Wait() + + wg.Add(len(noCap) * 2) + for _, disc := range noCap { + go func(disc *CapabilitiesDiscovery) { + defer wg.Done() + require.Eventuallyf(t, + func() bool { + numArchPeers := len(archOnly) + len(archCatch) + peers, err := disc.PeersForCapability(Archival, numArchPeers) + if err == nil && len(peers) == numArchPeers { + return true + } + return false + }, + time.Minute, + time.Second, + "Not all expected archival peers were found", + ) + }(disc) + + go func(disc *CapabilitiesDiscovery) { + defer wg.Done() + require.Eventuallyf(t, + func() bool { + numCatchPeers := len(catchOnly) + len(archCatch) + peers, err := disc.PeersForCapability(Catchpoints, numCatchPeers) + if err == nil && len(peers) == numCatchPeers { + return true + } + return false + }, + time.Minute, + time.Second, + "Not all expected catchpoint peers were found", + ) + }(disc) + } + + wg.Wait() + + for _, disc := range capsDisc[3:] { + err := disc.Close() + require.NoError(t, err) + // Make sure it actually closes + disc.wg.Wait() + } + }) + } +} + +func TestCapabilities_ExcludesSelf(t *testing.T) { + partitiontest.PartitionTest(t) + disc := setupCapDiscovery(t, 2, 2) + + testPeersFound := func(disc *CapabilitiesDiscovery, n int, cap Capability) bool { + peers, err := disc.PeersForCapability(cap, n) + if err == nil && len(peers) == n { + return true + } + return false + } + + waitForRouting(t, disc[0]) + disc[0].AdvertiseCapabilities(Archival) + // disc[1] finds Archival + require.Eventuallyf(t, + func() bool { return testPeersFound(disc[1], 1, Archival) }, + time.Minute, + time.Second, + "Could not find archival peer", + ) + + // disc[0] doesn't find itself + require.Neverf(t, + func() bool { return testPeersFound(disc[0], 1, Archival) }, + time.Second*5, + time.Second, + "Found self when searching for capability", + ) + + err := disc[0].Close() + require.NoError(t, err) + disc[0].wg.Wait() +} diff --git a/network/p2p/dht/dht.go b/network/p2p/dht/dht.go new file mode 100644 index 0000000000..1ea38bdd78 --- /dev/null +++ b/network/p2p/dht/dht.go @@ -0,0 +1,75 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package dht + +import ( + "context" + "fmt" + "math/rand" + "time" + + dht "github.com/libp2p/go-libp2p-kad-dht" + dhtmetrics "github.com/libp2p/go-libp2p-kad-dht/metrics" + "github.com/libp2p/go-libp2p/core/discovery" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" + crouting "github.com/libp2p/go-libp2p/core/routing" + "github.com/libp2p/go-libp2p/p2p/discovery/backoff" + "github.com/libp2p/go-libp2p/p2p/discovery/routing" + "go.opencensus.io/stats/view" + + "github.com/algorand/go-algorand/config" + algoproto "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/metrics" +) + +const minBackoff = time.Second * 5 +const maxBackoff = time.Second * 20 +const baseBackoff = float64(1.1) + +func dhtProtocolPrefix(networkID algoproto.NetworkID) protocol.ID { + return protocol.ID(fmt.Sprintf("/algorand/kad/%s", networkID)) +} + +// MakeDHT creates the dht.IpfsDHT object +func MakeDHT(ctx context.Context, h host.Host, networkID algoproto.NetworkID, cfg config.Local, bootstrapFunc func() []peer.AddrInfo) (*dht.IpfsDHT, error) { + dhtCfg := []dht.Option{ + // Automatically determine server or client mode + dht.Mode(dht.ModeAutoServer), + // We don't need the value store right now + dht.DisableValues(), + dht.ProtocolPrefix(dhtProtocolPrefix(networkID)), + dht.BootstrapPeersFunc(bootstrapFunc), + } + + if err := view.Register(dhtmetrics.DefaultViews...); err != nil { + return nil, err + } + metrics.DefaultRegistry().Register(&metrics.OpencensusDefaultMetrics) + + return dht.New(ctx, h, dhtCfg...) +} + +func backoffFactory() backoff.BackoffFactory { + return backoff.NewExponentialDecorrelatedJitter(minBackoff, maxBackoff, baseBackoff, rand.NewSource(rand.Int63())) +} + +// MakeDiscovery creates a discovery.Discovery object using backoff and cacching +func MakeDiscovery(r crouting.ContentRouting) (discovery.Discovery, error) { + return backoff.NewBackoffDiscovery(routing.NewRoutingDiscovery(r), backoffFactory(), backoff.WithBackoffDiscoveryReturnedChannelSize(0), backoff.WithBackoffDiscoverySimultaneousQueryBufferSize(0)) +} diff --git a/network/p2p/dht/dht_test.go b/network/p2p/dht/dht_test.go new file mode 100644 index 0000000000..51cb8978f7 --- /dev/null +++ b/network/p2p/dht/dht_test.go @@ -0,0 +1,64 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package dht + +import ( + "context" + "testing" + + logging "github.com/ipfs/go-log" + "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/test/partitiontest" +) + +func TestDHTBasic(t *testing.T) { + partitiontest.PartitionTest(t) + + h, err := libp2p.New() + require.NoError(t, err) + dht, err := MakeDHT( + context.Background(), + h, + "devtestnet", + config.GetDefaultLocal(), + func() []peer.AddrInfo { return nil }) + require.NoError(t, err) + _, err = MakeDiscovery(dht) + require.NoError(t, err) + err = dht.Bootstrap(context.Background()) + require.NoError(t, err) +} + +func TestDHTBasicAlgodev(t *testing.T) { + partitiontest.PartitionTest(t) + + logging.SetDebugLogging() + h, err := libp2p.New() + require.NoError(t, err) + cfg := config.GetDefaultLocal() + cfg.DNSBootstrapID = ".algodev.network" + dht, err := MakeDHT(context.Background(), h, "betanet", cfg, func() []peer.AddrInfo { return nil }) + require.NoError(t, err) + _, err = MakeDiscovery(dht) + require.NoError(t, err) + err = dht.Bootstrap(context.Background()) + require.NoError(t, err) +} diff --git a/network/p2p/dnsaddr/resolve.go b/network/p2p/dnsaddr/resolve.go index 176d62f946..5e0e8007fc 100644 --- a/network/p2p/dnsaddr/resolve.go +++ b/network/p2p/dnsaddr/resolve.go @@ -30,13 +30,19 @@ func isDnsaddr(maddr multiaddr.Multiaddr) bool { } // Iterate runs through the resolvable dnsaddrs in the tree using the resolveController and invokes f for each dnsaddr node lookup -func Iterate(initial multiaddr.Multiaddr, controller *MultiaddrDNSResolveController, f func(dnsaddr multiaddr.Multiaddr, entries []multiaddr.Multiaddr) error) error { +func Iterate(initial multiaddr.Multiaddr, controller ResolveController, f func(dnsaddr multiaddr.Multiaddr, entries []multiaddr.Multiaddr) error) error { resolver := controller.Resolver() if resolver == nil { return errors.New("passed controller has no resolvers Iterate") } + const maxHops = 25 // any reasonable number to prevent infinite loop in case of circular dnsaddr + hops := 0 var toResolve = []multiaddr.Multiaddr{initial} for resolver != nil && len(toResolve) > 0 { + hops++ + if hops > maxHops { + return errors.New("max hops reached while resolving dnsaddr " + initial.String()) + } curr := toResolve[0] maddrs, resolveErr := resolver.Resolve(context.Background(), curr) if resolveErr != nil { @@ -64,7 +70,7 @@ func Iterate(initial multiaddr.Multiaddr, controller *MultiaddrDNSResolveControl // Any further dnsaddrs will be looked up until all TXT records have been fetched, // and the full list of resulting Multiaddrs is returned. // It uses the MultiaddrDNSResolveController to cycle through DNS resolvers on failure. -func MultiaddrsFromResolver(domain string, controller *MultiaddrDNSResolveController) ([]multiaddr.Multiaddr, error) { +func MultiaddrsFromResolver(domain string, controller ResolveController) ([]multiaddr.Multiaddr, error) { dnsaddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/dnsaddr/%s", domain)) if err != nil { return nil, fmt.Errorf("unable to construct multiaddr for %s : %v", domain, err) diff --git a/network/p2p/dnsaddr/resolveController.go b/network/p2p/dnsaddr/resolveController.go index 73a46243d0..ff606b39a1 100644 --- a/network/p2p/dnsaddr/resolveController.go +++ b/network/p2p/dnsaddr/resolveController.go @@ -17,15 +17,29 @@ package dnsaddr import ( + "context" + + "github.com/multiformats/go-multiaddr" madns "github.com/multiformats/go-multiaddr-dns" log "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/tools/network" ) +// Resolver is an interface for resolving dnsaddrs +type Resolver interface { + Resolve(ctx context.Context, maddr multiaddr.Multiaddr) ([]multiaddr.Multiaddr, error) +} + +// ResolveController is an interface for cycling through resolvers +type ResolveController interface { + Resolver() Resolver + NextResolver() Resolver +} + // MultiaddrDNSResolveController returns a madns.Resolver, cycling through underlying net.Resolvers type MultiaddrDNSResolveController struct { - resolver *madns.Resolver + resolver Resolver nextResolvers []func() *madns.Resolver controller network.ResolveController } @@ -45,7 +59,7 @@ func NewMultiaddrDNSResolveController(secure bool, fallbackDNSResolverAddress st } // NextResolver applies the nextResolvers functions in order and returns the most recent result -func (c *MultiaddrDNSResolveController) NextResolver() *madns.Resolver { +func (c *MultiaddrDNSResolveController) NextResolver() Resolver { if len(c.nextResolvers) == 0 { c.resolver = nil } else { @@ -56,7 +70,7 @@ func (c *MultiaddrDNSResolveController) NextResolver() *madns.Resolver { } // Resolver returns the current resolver, invokes NextResolver if the resolver is nil -func (c *MultiaddrDNSResolveController) Resolver() *madns.Resolver { +func (c *MultiaddrDNSResolveController) Resolver() Resolver { if c.resolver == nil { c.resolver = c.NextResolver() } diff --git a/network/p2p/dnsaddr/resolve_test.go b/network/p2p/dnsaddr/resolve_test.go index 937e4db183..30acbd3e5f 100644 --- a/network/p2p/dnsaddr/resolve_test.go +++ b/network/p2p/dnsaddr/resolve_test.go @@ -21,6 +21,7 @@ import ( "fmt" "net" "testing" + "time" "github.com/multiformats/go-multiaddr" madns "github.com/multiformats/go-multiaddr-dns" @@ -36,22 +37,22 @@ func TestIsDnsaddr(t *testing.T) { t.Parallel() testcases := []struct { - name string - addr string - expected bool + name string + addr string + isDnsaddr bool }{ - {name: "DnsAddr", addr: "/dnsaddr/foobar.com", expected: true}, - {name: "DnsAddrWithPeerId", addr: "/dnsaddr/foobar.com/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", expected: true}, - {name: "DnsAddrWithIPPeerId", addr: "/dnsaddr/foobar.com/ip4/127.0.0.1/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", expected: true}, - {name: "Dns4Addr", addr: "/dns4/foobar.com/", expected: false}, - {name: "Dns6Addr", addr: "/dns6/foobar.com/", expected: false}, - {name: "Dns4AddrWithPeerId", addr: "/dns4/foobar.com/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", expected: false}, + {name: "DnsAddr", addr: "/dnsaddr/foobar.com", isDnsaddr: true}, + {name: "DnsAddrWithPeerId", addr: "/dnsaddr/foobar.com/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", isDnsaddr: true}, + {name: "DnsAddrWithIPPeerId", addr: "/dnsaddr/foobar.com/ip4/127.0.0.1/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", isDnsaddr: true}, + {name: "Dns4Addr", addr: "/dns4/foobar.com/", isDnsaddr: false}, + {name: "Dns6Addr", addr: "/dns6/foobar.com/", isDnsaddr: false}, + {name: "Dns4AddrWithPeerId", addr: "/dns4/foobar.com/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", isDnsaddr: false}, } for _, testcase := range testcases { t.Run(testcase.name, func(t *testing.T) { maddr, err := multiaddr.NewMultiaddr(testcase.addr) require.NoError(t, err) - require.Equal(t, testcase.expected, isDnsaddr(maddr)) + require.Equal(t, testcase.isDnsaddr, isDnsaddr(maddr)) }) } } @@ -109,3 +110,42 @@ func TestMultiaddrsFromResolverDnsFailure(t *testing.T) { assert.Empty(t, maddrs) assert.ErrorContains(t, err, "always errors") } + +type mockController struct { +} + +func (c mockController) Resolver() Resolver { + return selfResolver{} +} + +func (c mockController) NextResolver() Resolver { + return nil +} + +type selfResolver struct { +} + +func (r selfResolver) Resolve(ctx context.Context, maddr multiaddr.Multiaddr) ([]multiaddr.Multiaddr, error) { + return []multiaddr.Multiaddr{maddr}, nil +} + +// TestIterate ensures the Iterate() does not hang in infinite loop +// when resolver returns the same dnsaddr +func TestIterate(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + dnsAddr := "/dnsaddr/foobar.com" + require.True(t, isDnsaddr(multiaddr.StringCast(dnsAddr))) + ma, err := multiaddr.NewMultiaddr(dnsAddr) + require.NoError(t, err) + + require.Eventually(t, func() bool { + Iterate( + ma, + mockController{}, + func(dnsaddr multiaddr.Multiaddr, entries []multiaddr.Multiaddr) error { return nil }, + ) + return true + }, 100*time.Millisecond, 50*time.Millisecond) +} diff --git a/network/p2p/http.go b/network/p2p/http.go new file mode 100644 index 0000000000..9f2622d015 --- /dev/null +++ b/network/p2p/http.go @@ -0,0 +1,92 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package p2p + +import ( + "net/http" + "sync" + "time" + + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/network/limitcaller" + "github.com/gorilla/mux" + "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + libp2phttp "github.com/libp2p/go-libp2p/p2p/http" +) + +// algorandP2pHTTPProtocol defines a libp2p protocol name for algorand's http over p2p messages +const algorandP2pHTTPProtocol = "/algorand-http/1.0.0" + +// HTTPServer is a wrapper around libp2phttp.Host that allows registering http handlers with path parameters. +type HTTPServer struct { + libp2phttp.Host + p2phttpMux *mux.Router + p2phttpMuxRegistrarOnce sync.Once +} + +// MakeHTTPServer creates a new HTTPServer +func MakeHTTPServer(streamHost host.Host) *HTTPServer { + httpServer := HTTPServer{ + Host: libp2phttp.Host{StreamHost: streamHost}, + p2phttpMux: mux.NewRouter(), + } + return &httpServer +} + +// RegisterHTTPHandler registers a http handler with a given path. +func (s *HTTPServer) RegisterHTTPHandler(path string, handler http.Handler) { + s.p2phttpMux.Handle(path, handler) + s.p2phttpMuxRegistrarOnce.Do(func() { + s.Host.SetHTTPHandlerAtPath(algorandP2pHTTPProtocol, "/", s.p2phttpMux) + }) +} + +// MakeHTTPClient creates a http.Client that uses libp2p transport for a given protocol and peer address. +func MakeHTTPClient(addrInfo *peer.AddrInfo) (*http.Client, error) { + clientStreamHost, err := libp2p.New(libp2p.NoListenAddrs) + if err != nil { + return nil, err + } + logging.Base().Debugf("MakeHTTPClient made a new P2P host %s for %s", clientStreamHost.ID(), addrInfo.String()) + + client := libp2phttp.Host{StreamHost: clientStreamHost} + + // Do not use client.NamespacedClient to prevent it making connection to a well-known handler + // to make a NamespaceRoundTripper that limits to specific URL paths. + // First, we do not want make requests when listing peers (the main MakeHTTPClient invoker). + // Secondly, this makes unit testing easier - no need to register fake handlers. + rt, err := client.NewConstrainedRoundTripper(*addrInfo) + if err != nil { + return nil, err + } + + return &http.Client{Transport: rt}, nil +} + +// MakeHTTPClientWithRateLimit creates a http.Client that uses libp2p transport for a given protocol and peer address. +func MakeHTTPClientWithRateLimit(addrInfo *peer.AddrInfo, pstore limitcaller.ConnectionTimeStore, queueingTimeout time.Duration, maxIdleConnsPerHost int) (*http.Client, error) { + cl, err := MakeHTTPClient(addrInfo) + if err != nil { + return nil, err + } + rlrt := limitcaller.MakeRateLimitingTransportWithRoundTripper(pstore, queueingTimeout, cl.Transport, addrInfo, maxIdleConnsPerHost) + cl.Transport = &rlrt + return cl, nil + +} diff --git a/network/p2p/logger.go b/network/p2p/logger.go new file mode 100644 index 0000000000..26c738e1e1 --- /dev/null +++ b/network/p2p/logger.go @@ -0,0 +1,123 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +// This package implement a zap.Core in order to wrap lip2p logger into algod's logger. + +package p2p + +import ( + "runtime" + "strings" + + p2plogging "github.com/ipfs/go-log/v2" + "github.com/sirupsen/logrus" + "go.uber.org/zap/zapcore" + + "github.com/algorand/go-algorand/logging" +) + +// var levelsMap = map[logging.Level]zapcore.Level{ +// logging.Debug: zapcore.DebugLevel, +// logging.Info: zapcore.InfoLevel, +// logging.Warn: zapcore.WarnLevel, +// logging.Error: zapcore.ErrorLevel, +// logging.Fatal: zapcore.FatalLevel, +// logging.Panic: zapcore.PanicLevel, +// } + +var levelsMap = map[zapcore.Level]logging.Level{ + zapcore.DebugLevel: logging.Debug, + zapcore.InfoLevel: logging.Info, + zapcore.WarnLevel: logging.Warn, + zapcore.ErrorLevel: logging.Error, + zapcore.FatalLevel: logging.Fatal, + zapcore.PanicLevel: logging.Panic, +} + +// loggingCore implements zapcore.Core +type loggingCore struct { + log logging.Logger + level logging.Level + fields []zapcore.Field + zapcore.Core +} + +// EnableP2PLogging enables libp2p logging into the provided logger with the provided level. +func EnableP2PLogging(log logging.Logger, l logging.Level) { + core := loggingCore{ + log: log, + level: l, + } + for p2pLevel, logLevel := range levelsMap { + if logLevel == l { + p2plogging.SetAllLoggers(p2plogging.LogLevel(p2pLevel)) + break + } + } + p2plogging.SetPrimaryCore(&core) +} + +func (c *loggingCore) Enabled(l zapcore.Level) bool { + level := levelsMap[l] + return c.log.IsLevelEnabled(level) +} + +func (c *loggingCore) With(fields []zapcore.Field) zapcore.Core { + return &loggingCore{ + log: c.log, + level: c.level, + fields: append(c.fields, fields...), + } +} + +func (c *loggingCore) Check(e zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if c.Enabled(e.Level) { + return ce.AddCore(e, c) + } + return ce +} + +func (c *loggingCore) Write(e zapcore.Entry, fields []zapcore.Field) error { + allFields := append(c.fields, fields...) + loggingFields := make(logging.Fields, len(allFields)) + + for _, f := range allFields { + if len(f.String) > 0 { + loggingFields[f.Key] = f.String + } else if f.Interface != nil { + loggingFields[f.Key] = f.Interface + } else { + loggingFields[f.Key] = f.Integer + } + } + event := c.log.WithFields(loggingFields).With("libp2p", e.LoggerName) + file := e.Caller.File + slash := strings.LastIndex(file, "/") + file = file[slash+1:] + event = event.WithFields(logrus.Fields{ + "file": file, + "line": e.Caller.Line, + }) + if function := runtime.FuncForPC(e.Caller.PC); function != nil { + event = event.With("function", function.Name()) + } + event.Entry().Log(logrus.Level(levelsMap[e.Level]), e.Message) + return nil +} + +func (c *loggingCore) Sync() error { + return nil +} diff --git a/network/p2p/p2p.go b/network/p2p/p2p.go index cd84f1c69c..f4ed670f3e 100644 --- a/network/p2p/p2p.go +++ b/network/p2p/p2p.go @@ -18,6 +18,7 @@ package p2p import ( "context" + "encoding/base32" "fmt" "runtime" "strings" @@ -25,22 +26,37 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/logging" + pstore "github.com/algorand/go-algorand/network/p2p/peerstore" + "github.com/algorand/go-algorand/network/phonebook" + "github.com/algorand/go-algorand/util/metrics" "github.com/algorand/go-deadlock" "github.com/libp2p/go-libp2p" pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/peerstore" + "github.com/libp2p/go-libp2p/core/protocol" + libp2phttp "github.com/libp2p/go-libp2p/p2p/http" "github.com/libp2p/go-libp2p/p2p/muxer/yamux" + "github.com/libp2p/go-libp2p/p2p/security/noise" "github.com/libp2p/go-libp2p/p2p/transport/tcp" + "github.com/multiformats/go-multiaddr" ) +// SubNextCancellable is an abstraction for pubsub.Subscription +type SubNextCancellable interface { + Next(ctx context.Context) (*pubsub.Message, error) + Cancel() +} + // Service defines the interface used by the network integrating with underlying p2p implementation type Service interface { + Start() error Close() error - ID() peer.ID // return peer.ID for self + ID() peer.ID // return peer.ID for self + IDSigner() *PeerIDChallengeSigner AddrInfo() peer.AddrInfo // return addrInfo for self DialNode(context.Context, *peer.AddrInfo) error @@ -49,17 +65,21 @@ type Service interface { Conns() []network.Conn ListPeersForTopic(topic string) []peer.ID - Subscribe(topic string, val pubsub.ValidatorEx) (*pubsub.Subscription, error) + Subscribe(topic string, val pubsub.ValidatorEx) (SubNextCancellable, error) Publish(ctx context.Context, topic string, data []byte) error + + GetStream(peer.ID) (network.Stream, bool) } // serviceImpl manages integration with libp2p and implements the Service interface type serviceImpl struct { - log logging.Logger - host host.Host - streams *streamManager - pubsub *pubsub.PubSub - pubsubCtx context.Context + log logging.Logger + listenAddr string + host host.Host + streams *streamManager + pubsub *pubsub.PubSub + pubsubCtx context.Context + privKey crypto.PrivKey topics map[string]*pubsub.Topic topicsMu deadlock.RWMutex @@ -68,14 +88,19 @@ type serviceImpl struct { // AlgorandWsProtocol defines a libp2p protocol name for algorand's websockets messages const AlgorandWsProtocol = "/algorand-ws/1.0.0" +// algorandGUIDProtocolPrefix defines a libp2p protocol name for algorand node telemetry GUID exchange +const algorandGUIDProtocolPrefix = "/algorand-telemetry/1.0.0/" +const algorandGUIDProtocolTemplate = algorandGUIDProtocolPrefix + "%s/%s" + const dialTimeout = 30 * time.Second -// MakeService creates a P2P service instance -func MakeService(ctx context.Context, log logging.Logger, cfg config.Local, datadir string, pstore peerstore.Peerstore, wsStreamHandler StreamHandler) (*serviceImpl, error) { +// MakeHost creates a libp2p host but does not start listening. +// Use host.Network().Listen() on the returned address to start listening. +func MakeHost(cfg config.Local, datadir string, pstore *pstore.PeerStore) (host.Host, string, error) { // load stored peer ID, or make ephemeral peer ID privKey, err := GetPrivKey(cfg, datadir) if err != nil { - return nil, err + return nil, "", err } // muxer supports tweaking fields from yamux.Config @@ -93,38 +118,110 @@ func MakeService(ctx context.Context, log logging.Logger, cfg config.Local, data listenAddr = "/ip4/0.0.0.0/tcp/0" } - h, err := libp2p.New( + // the libp2p.NoListenAddrs builtin disables relays but this one does not + var noListenAddrs = func(cfg *libp2p.Config) error { + cfg.ListenAddrs = []multiaddr.Multiaddr{} + return nil + } + + var disableMetrics = func(cfg *libp2p.Config) error { return nil } + metrics.DefaultRegistry().Register(&metrics.PrometheusDefaultMetrics) + + host, err := libp2p.New( libp2p.Identity(privKey), libp2p.UserAgent(ua), libp2p.Transport(tcp.NewTCPTransport), libp2p.Muxer("/yamux/1.0.0", &ymx), libp2p.Peerstore(pstore), - libp2p.ListenAddrStrings(listenAddr), + noListenAddrs, + libp2p.Security(noise.ID, noise.New), + disableMetrics, ) - if err != nil { - return nil, err + return &StreamChainingHost{ + Host: host, + handlers: map[protocol.ID][]network.StreamHandler{}, + }, listenAddr, err +} + +// StreamChainingHost is a wrapper around host.Host that overrides SetStreamHandler +// to allow chaining multiple handlers for the same protocol. +// Note, there should be probably only single handler that writes/reads streams. +type StreamChainingHost struct { + host.Host + handlers map[protocol.ID][]network.StreamHandler + mutex deadlock.Mutex +} + +// SetStreamHandler overrides the host.Host.SetStreamHandler method for chaining multiple handlers. +// Function objects are not comparable so theoretically it could have duplicates. +// The main use case is to track HTTP streams for ProtocolIDForMultistreamSelect = "/http/1.1" +// so it could just filter for such protocol if there any issues with other protocols like kad or mesh. +func (h *StreamChainingHost) SetStreamHandler(pid protocol.ID, handler network.StreamHandler) { + h.mutex.Lock() + defer h.mutex.Unlock() + + handlers := h.handlers[pid] + if len(handlers) == 0 { + // no other handlers, do not set a proxy handler + h.Host.SetStreamHandler(pid, handler) + h.handlers[pid] = append(handlers, handler) + return } - log.Infof("P2P service started: peer ID %s addrs %s", h.ID(), h.Addrs()) + // otherwise chain the handlers with a copy of the existing handlers + handlers = append(handlers, handler) + // copy to save it in the closure and call lock free + currentHandlers := make([]network.StreamHandler, len(handlers)) + copy(currentHandlers, handlers) + h.Host.SetStreamHandler(pid, func(s network.Stream) { + for _, h := range currentHandlers { + h(s) + } + }) + h.handlers[pid] = handlers +} + +// MakeService creates a P2P service instance +func MakeService(ctx context.Context, log logging.Logger, cfg config.Local, h host.Host, listenAddr string, wsStreamHandler StreamHandler, bootstrapPeers []*peer.AddrInfo) (*serviceImpl, error) { sm := makeStreamManager(ctx, log, h, wsStreamHandler) h.Network().Notify(sm) h.SetStreamHandler(AlgorandWsProtocol, sm.streamHandler) + h.SetStreamHandler(libp2phttp.ProtocolIDForMultistreamSelect, sm.streamHandlerHTTP) + + // set an empty handler for telemetryID/telemetryInstance protocol in order to allow other peers to know our telemetryID + telemetryID := log.GetTelemetryGUID() + telemetryInstance := log.GetInstanceName() + telemetryProtoInfo := formatPeerTelemetryInfoProtocolName(telemetryID, telemetryInstance) + h.SetStreamHandler(protocol.ID(telemetryProtoInfo), func(s network.Stream) { s.Close() }) ps, err := makePubSub(ctx, cfg, h) if err != nil { return nil, err } - return &serviceImpl{ - log: log, - host: h, - streams: sm, - pubsub: ps, - pubsubCtx: ctx, - topics: make(map[string]*pubsub.Topic), + + log: log, + listenAddr: listenAddr, + host: h, + streams: sm, + pubsub: ps, + pubsubCtx: ctx, + privKey: h.Peerstore().PrivKey(h.ID()), + topics: make(map[string]*pubsub.Topic), }, nil } +// Start starts the P2P service +func (s *serviceImpl) Start() error { + listenAddr, err := multiaddr.NewMultiaddr(s.listenAddr) + if err != nil { + s.log.Errorf("failed to create multiaddress: %s", err) + return err + } + + return s.host.Network().Listen(listenAddr) +} + // Close shuts down the P2P service func (s *serviceImpl) Close() error { return s.host.Close() @@ -135,22 +232,35 @@ func (s *serviceImpl) ID() peer.ID { return s.host.ID() } +// IDSigner returns a PeerIDChallengeSigner that implements the network identityChallengeSigner interface +func (s *serviceImpl) IDSigner() *PeerIDChallengeSigner { + return &PeerIDChallengeSigner{key: s.privKey} +} + // DialPeersUntilTargetCount attempts to establish connections to the provided phonebook addresses func (s *serviceImpl) DialPeersUntilTargetCount(targetConnCount int) { - peerIDs := s.host.Peerstore().Peers() - for _, peerID := range peerIDs { + ps := s.host.Peerstore().(*pstore.PeerStore) + peerIDs := ps.GetAddresses(targetConnCount, phonebook.PhoneBookEntryRelayRole) + conns := s.host.Network().Conns() + var numOutgoingConns int + for _, conn := range conns { + if conn.Stat().Direction == network.DirOutbound { + numOutgoingConns++ + } + } + for _, peerInfo := range peerIDs { + peerInfo := peerInfo.(*peer.AddrInfo) // if we are at our target count stop trying to connect - if len(s.host.Network().Conns()) == targetConnCount { + if numOutgoingConns >= targetConnCount { return } // if we are already connected to this peer, skip it - if len(s.host.Network().ConnsToPeer(peerID)) > 0 { + if len(s.host.Network().ConnsToPeer(peerInfo.ID)) > 0 { continue } - peerInfo := s.host.Peerstore().PeerInfo(peerID) - err := s.DialNode(context.Background(), &peerInfo) // leaving the calls as blocking for now, to not over-connect beyond fanout + err := s.DialNode(context.Background(), peerInfo) // leaving the calls as blocking for now, to not over-connect beyond fanout if err != nil { - s.log.Warnf("failed to connect to peer %s: %v", peerID, err) + s.log.Warnf("failed to connect to peer %s: %v", peerInfo.ID, err) } } } @@ -184,6 +294,10 @@ func (s *serviceImpl) ClosePeer(peer peer.ID) error { return s.host.Network().ClosePeer(peer) } +func (s *serviceImpl) GetStream(peerID peer.ID) (network.Stream, bool) { + return s.streams.getStream(peerID) +} + // netAddressToListenAddress converts a netAddress in "ip:port" format to a listen address // that can be passed in to libp2p.ListenAddrStrings func netAddressToListenAddress(netAddress string) (string, error) { @@ -203,3 +317,32 @@ func netAddressToListenAddress(netAddress string) (string, error) { return fmt.Sprintf("/ip4/%s/tcp/%s", ip, parts[1]), nil } + +// GetPeerTelemetryInfo returns the telemetry ID of a peer by looking at its protocols +func GetPeerTelemetryInfo(peerProtocols []protocol.ID) (telemetryID string, telemetryInstance string) { + for _, protocol := range peerProtocols { + if strings.HasPrefix(string(protocol), algorandGUIDProtocolPrefix) { + telemetryInfo := string(protocol[len(algorandGUIDProtocolPrefix):]) + telemetryInfoParts := strings.Split(telemetryInfo, "/") + if len(telemetryInfoParts) == 2 { + telemetryIDBytes, err := base32.StdEncoding.DecodeString(telemetryInfoParts[0]) + if err == nil { + telemetryID = string(telemetryIDBytes) + } + telemetryInstanceBytes, err := base32.StdEncoding.DecodeString(telemetryInfoParts[1]) + if err == nil { + telemetryInstance = string(telemetryInstanceBytes) + } + return telemetryID, telemetryInstance + } + } + } + return "", "" +} + +func formatPeerTelemetryInfoProtocolName(telemetryID string, telemetryInstance string) string { + return fmt.Sprintf(algorandGUIDProtocolTemplate, + base32.StdEncoding.EncodeToString([]byte(telemetryID)), + base32.StdEncoding.EncodeToString([]byte(telemetryInstance)), + ) +} diff --git a/network/p2p/p2p_test.go b/network/p2p/p2p_test.go index 558131fe48..fb14193a55 100644 --- a/network/p2p/p2p_test.go +++ b/network/p2p/p2p_test.go @@ -17,11 +17,22 @@ package p2p import ( + "context" "fmt" + "sync/atomic" "testing" + "time" - "github.com/algorand/go-algorand/test/partitiontest" + "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" + "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/network/p2p/peerstore" + "github.com/algorand/go-algorand/test/partitiontest" ) // Tests the helper function netAddressToListenAddress which converts @@ -74,3 +85,153 @@ func TestNetAddressToListenAddress(t *testing.T) { }) } } + +func TestP2PStreamingHost(t *testing.T) { + partitiontest.PartitionTest(t) + + cfg := config.GetDefaultLocal() + dir := t.TempDir() + pstore, err := peerstore.NewPeerStore(nil, "") + require.NoError(t, err) + h, la, err := MakeHost(cfg, dir, pstore) + require.NoError(t, err) + + var h1calls atomic.Int64 + h1 := func(network.Stream) { + h1calls.Add(1) + } + var h2calls atomic.Int64 + h2 := func(network.Stream) { + h2calls.Add(1) + } + + ma, err := multiaddr.NewMultiaddr(la) + require.NoError(t, err) + h.Network().Listen(ma) + defer h.Close() + + h.SetStreamHandler(AlgorandWsProtocol, h1) + h.SetStreamHandler(AlgorandWsProtocol, h2) + + addrInfo := peer.AddrInfo{ + ID: h.ID(), + Addrs: h.Addrs(), + } + cpstore, err := peerstore.NewPeerStore([]*peer.AddrInfo{&addrInfo}, "") + require.NoError(t, err) + c, _, err := MakeHost(cfg, dir, cpstore) + require.NoError(t, err) + defer c.Close() + + s1, err := c.NewStream(context.Background(), h.ID(), AlgorandWsProtocol) + require.NoError(t, err) + s1.Write([]byte("hello")) + defer s1.Close() + + require.Eventually(t, func() bool { + return h1calls.Load() == 1 && h2calls.Load() == 1 + }, 5*time.Second, 100*time.Millisecond) + + // ensure a single handler also works as expected + h1calls.Store(0) + h.SetStreamHandler(algorandP2pHTTPProtocol, h1) + + s2, err := c.NewStream(context.Background(), h.ID(), algorandP2pHTTPProtocol) + require.NoError(t, err) + s2.Write([]byte("hello")) + defer s2.Close() + + require.Eventually(t, func() bool { + return h1calls.Load() == 1 + }, 5*time.Second, 100*time.Millisecond) + +} + +// TestP2PGetPeerTelemetryInfo tests the GetPeerTelemetryInfo function +func TestP2PGetPeerTelemetryInfo(t *testing.T) { + partitiontest.PartitionTest(t) + + testCases := []struct { + name string + peerProtocols []protocol.ID + expectedTelemetryID string + expectedTelemetryInstance string + }{ + { + name: "Valid Telemetry Info", + peerProtocols: []protocol.ID{protocol.ID(formatPeerTelemetryInfoProtocolName("telemetryID", "telemetryInstance"))}, + expectedTelemetryID: "telemetryID", + expectedTelemetryInstance: "telemetryInstance", + }, + { + name: "Partial Telemetry Info 1", + peerProtocols: []protocol.ID{protocol.ID(formatPeerTelemetryInfoProtocolName("telemetryID", ""))}, + expectedTelemetryID: "telemetryID", + expectedTelemetryInstance: "", + }, + { + name: "Partial Telemetry Info 2", + peerProtocols: []protocol.ID{protocol.ID(formatPeerTelemetryInfoProtocolName("", "telemetryInstance"))}, + expectedTelemetryID: "", + expectedTelemetryInstance: "telemetryInstance", + }, + { + name: "No Telemetry Info", + peerProtocols: []protocol.ID{protocol.ID("/some-other-protocol/1.0.0/otherID/otherInstance")}, + expectedTelemetryID: "", + expectedTelemetryInstance: "", + }, + { + name: "Invalid Telemetry Info Format", + peerProtocols: []protocol.ID{protocol.ID("/algorand-telemetry/1.0.0/invalidFormat")}, + expectedTelemetryID: "", + expectedTelemetryInstance: "", + }, + { + name: "Special Characters Telemetry Info Format", + peerProtocols: []protocol.ID{protocol.ID(formatPeerTelemetryInfoProtocolName("telemetry/ID", "123-//11-33"))}, + expectedTelemetryID: "telemetry/ID", + expectedTelemetryInstance: "123-//11-33", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + telemetryID, telemetryInstance := GetPeerTelemetryInfo(tc.peerProtocols) + if telemetryID != tc.expectedTelemetryID || telemetryInstance != tc.expectedTelemetryInstance { + t.Errorf("Expected telemetry ID: %s, telemetry instance: %s, but got telemetry ID: %s, telemetry instance: %s", + tc.expectedTelemetryID, tc.expectedTelemetryInstance, telemetryID, telemetryInstance) + } + }) + } +} + +func TestP2PProtocolAsMeta(t *testing.T) { + partitiontest.PartitionTest(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + h1, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0")) + require.NoError(t, err) + defer h1.Close() + + h1TID := "telemetryID1" + h1Inst := "telemetryInstance2" + telemetryProtoInfo := formatPeerTelemetryInfoProtocolName(h1TID, h1Inst) + h1.SetStreamHandler(protocol.ID(telemetryProtoInfo), func(s network.Stream) { s.Close() }) + + h2, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0")) + require.NoError(t, err) + defer h2.Close() + + err = h2.Connect(ctx, peer.AddrInfo{ID: h1.ID(), Addrs: h1.Addrs()}) + require.NoError(t, err) + + protos, err := h2.Peerstore().GetProtocols(h1.ID()) + require.NoError(t, err) + + tid, inst := GetPeerTelemetryInfo(protos) + require.Equal(t, h1TID, tid) + require.Equal(t, h1Inst, inst) +} diff --git a/network/p2p/peerID.go b/network/p2p/peerID.go index f31d29ffb7..ca7526977b 100644 --- a/network/p2p/peerID.go +++ b/network/p2p/peerID.go @@ -25,6 +25,7 @@ import ( "path" "github.com/algorand/go-algorand/config" + algocrypto "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/util" "github.com/libp2p/go-libp2p/core/crypto" @@ -38,6 +39,10 @@ const DefaultPrivKeyPath = "peerIDPrivKey.pem" // PeerID is a string representation of a peer's public key, primarily used to avoid importing libp2p into packages that shouldn't need it type PeerID string +func (id PeerID) String() string { + return peer.ID(id).String() +} + // GetPrivKey manages loading and creation of private keys for network PeerIDs // It prioritizes, in this order: // 1. user supplied path to privKey @@ -104,3 +109,39 @@ func generatePrivKey() (crypto.PrivKey, error) { priv, _, err := crypto.GenerateEd25519Key(rand.Reader) return priv, err } + +// PeerIDChallengeSigner implements the identityChallengeSigner interface in the network package. +type PeerIDChallengeSigner struct { + key crypto.PrivKey +} + +// Sign implements the identityChallengeSigner interface. +func (p *PeerIDChallengeSigner) Sign(message algocrypto.Hashable) algocrypto.Signature { + return p.SignBytes(algocrypto.HashRep(message)) +} + +// SignBytes implements the identityChallengeSigner interface. +func (p *PeerIDChallengeSigner) SignBytes(message []byte) algocrypto.Signature { + // libp2p Ed25519PrivateKey.Sign() returns a signature with a length of 64 bytes and no error + sig, err := p.key.Sign(message) + if len(sig) != len(algocrypto.Signature{}) { + panic(fmt.Sprintf("invalid signature length: %d", len(sig))) + } + if err != nil { + panic(err) + } + return algocrypto.Signature(sig) +} + +// PublicKey implements the identityChallengeSigner interface. +func (p *PeerIDChallengeSigner) PublicKey() algocrypto.PublicKey { + // libp2p Ed25519PublicKey.Raw() returns a 32-byte public key and no error + pub, err := p.key.GetPublic().Raw() + if len(pub) != len(algocrypto.PublicKey{}) { + panic(fmt.Sprintf("invalid public key length: %d", len(pub))) + } + if err != nil { + panic(err) + } + return algocrypto.PublicKey(pub) +} diff --git a/network/p2p/peerstore/peerstore.go b/network/p2p/peerstore/peerstore.go index fa572c5912..3eda0d3686 100644 --- a/network/p2p/peerstore/peerstore.go +++ b/network/p2p/peerstore/peerstore.go @@ -22,6 +22,8 @@ import ( "math/rand" "time" + "github.com/algorand/go-algorand/network/phonebook" + "github.com/algorand/go-deadlock" "github.com/libp2p/go-libp2p/core/peer" libp2p "github.com/libp2p/go-libp2p/core/peerstore" mempstore "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" @@ -32,12 +34,6 @@ import ( // of how many addresses the phonebook actually has. ( with the retry-after logic applied ) const getAllAddresses = math.MaxInt32 -// PhoneBookEntryRoles defines the roles that a single entry on the phonebook can take. -// currently, we have two roles : relay role and archiver role, which are mutually exclusive. -// -//msgp:ignore PhoneBookEntryRoles -type PhoneBookEntryRoles int - const addressDataKey string = "addressData" // PeerStore implements Peerstore and CertifiedAddrBook. @@ -58,9 +54,10 @@ type addressData struct { // networkNames: lists the networks to which the given address belongs. networkNames map[string]bool + mu *deadlock.RWMutex // role is the role that this address serves. - role PhoneBookEntryRoles + role phonebook.PhoneBookEntryRoles // persistent is set true for peers whose record should not be removed for the peer list persistent bool @@ -73,18 +70,20 @@ type peerStoreCAB interface { } // NewPeerStore creates a new peerstore backed by a datastore. -func NewPeerStore(addrInfo []*peer.AddrInfo) (*PeerStore, error) { +func NewPeerStore(addrInfo []*peer.AddrInfo, network string) (*PeerStore, error) { ps, err := mempstore.NewPeerstore() if err != nil { return nil, fmt.Errorf("cannot initialize a peerstore: %w", err) } // initialize peerstore with addresses + peers := make([]interface{}, len(addrInfo)) for i := 0; i < len(addrInfo); i++ { - info := addrInfo[i] - ps.AddAddrs(info.ID, info.Addrs, libp2p.AddressTTL) + peers[i] = addrInfo[i] } + pstore := &PeerStore{peerStoreCAB: ps} + pstore.AddPersistentPeers(peers, network, phonebook.PhoneBookEntryRelayRole) return pstore, nil } @@ -103,13 +102,13 @@ func MakePhonebook(connectionsRateLimitingCount uint, } // GetAddresses returns up to N addresses, but may return fewer -func (ps *PeerStore) GetAddresses(n int, role PhoneBookEntryRoles) []string { +func (ps *PeerStore) GetAddresses(n int, role phonebook.PhoneBookEntryRoles) []interface{} { return shuffleSelect(ps.filterRetryTime(time.Now(), role), n) } // UpdateRetryAfter updates the retryAfter time for the given address. func (ps *PeerStore) UpdateRetryAfter(addr string, retryAfter time.Time) { - info, err := PeerInfoFromDomainPort(addr) + info, err := peerInfoFromDomainPort(addr) if err != nil { return } @@ -130,15 +129,12 @@ func (ps *PeerStore) UpdateRetryAfter(addr string, retryAfter time.Time) { // The connection should be established when the waitTime is 0. // It will register a provisional next connection time when the waitTime is 0. // The provisional time should be updated after the connection with UpdateConnectionTime -func (ps *PeerStore) GetConnectionWaitTime(addr string) (bool, time.Duration, time.Time) { +func (ps *PeerStore) GetConnectionWaitTime(addrOrPeerID string) (bool, time.Duration, time.Time) { curTime := time.Now() - info, err := PeerInfoFromDomainPort(addr) - if err != nil { - return false, 0 /* not used */, curTime /* not used */ - } var timeSince time.Duration var numElmtsToRemove int - metadata, err := ps.Get(info.ID, addressDataKey) + peerID := peer.ID(addrOrPeerID) + metadata, err := ps.Get(peerID, addressDataKey) if err != nil { return false, 0 /* not used */, curTime /* not used */ } @@ -157,9 +153,9 @@ func (ps *PeerStore) GetConnectionWaitTime(addr string) (bool, time.Duration, ti } // Remove the expired elements from e.data[addr].recentConnectionTimes - ps.popNElements(numElmtsToRemove, peer.ID(addr)) + ps.popNElements(numElmtsToRemove, peerID) // If there are max number of connections within the time window, wait - metadata, _ = ps.Get(info.ID, addressDataKey) + metadata, _ = ps.Get(peerID, addressDataKey) ad, ok = metadata.(addressData) if !ok { return false, 0 /* not used */, curTime /* not used */ @@ -175,17 +171,14 @@ func (ps *PeerStore) GetConnectionWaitTime(addr string) (bool, time.Duration, ti // Update curTime, since it may have significantly changed if waited provisionalTime := time.Now() // Append the provisional time for the next connection request - ps.appendTime(info.ID, provisionalTime) + ps.appendTime(peerID, provisionalTime) return true, 0 /* no wait. proceed */, provisionalTime } // UpdateConnectionTime updates the connection time for the given address. -func (ps *PeerStore) UpdateConnectionTime(addr string, provisionalTime time.Time) bool { - info, err := PeerInfoFromDomainPort(addr) - if err != nil { - return false - } - metadata, err := ps.Get(info.ID, addressDataKey) +func (ps *PeerStore) UpdateConnectionTime(addrOrPeerID string, provisionalTime time.Time) bool { + peerID := peer.ID(addrOrPeerID) + metadata, err := ps.Get(peerID, addressDataKey) if err != nil { return false } @@ -194,7 +187,7 @@ func (ps *PeerStore) UpdateConnectionTime(addr string, provisionalTime time.Time return false } defer func() { - _ = ps.Put(info.ID, addressDataKey, ad) + _ = ps.Put(peerID, addressDataKey, ad) }() @@ -217,7 +210,7 @@ func (ps *PeerStore) UpdateConnectionTime(addr string, provisionalTime time.Time } // ReplacePeerList replaces the peer list for the given networkName and role. -func (ps *PeerStore) ReplacePeerList(addressesThey []string, networkName string, role PhoneBookEntryRoles) { +func (ps *PeerStore) ReplacePeerList(addressesThey []interface{}, networkName string, role phonebook.PhoneBookEntryRoles) { // prepare a map of items we'd like to remove. removeItems := make(map[peer.ID]bool, 0) peerIDs := ps.Peers() @@ -225,23 +218,24 @@ func (ps *PeerStore) ReplacePeerList(addressesThey []string, networkName string, data, _ := ps.Get(pid, addressDataKey) if data != nil { ad := data.(addressData) + ad.mu.RLock() if ad.networkNames[networkName] && ad.role == role && !ad.persistent { removeItems[pid] = true } + ad.mu.RUnlock() } } for _, addr := range addressesThey { - info, err := PeerInfoFromDomainPort(addr) - if err != nil { - return - } + info := addr.(*peer.AddrInfo) data, _ := ps.Get(info.ID, addressDataKey) if data != nil { // we already have this. // Update the networkName ad := data.(addressData) + ad.mu.Lock() ad.networkNames[networkName] = true + ad.mu.Unlock() // do not remove this entry delete(removeItems, info.ID) @@ -261,13 +255,9 @@ func (ps *PeerStore) ReplacePeerList(addressesThey []string, networkName string, // AddPersistentPeers stores addresses of peers which are persistent. // i.e. they won't be replaced by ReplacePeerList calls -func (ps *PeerStore) AddPersistentPeers(dnsAddresses []string, networkName string, role PhoneBookEntryRoles) { - +func (ps *PeerStore) AddPersistentPeers(dnsAddresses []interface{}, networkName string, role phonebook.PhoneBookEntryRoles) { for _, addr := range dnsAddresses { - info, err := PeerInfoFromDomainPort(addr) - if err != nil { - return - } + info := addr.(*peer.AddrInfo) data, _ := ps.Get(info.ID, addressDataKey) if data != nil { // we already have this. @@ -291,9 +281,10 @@ func (ps *PeerStore) Length() int { } // makePhonebookEntryData creates a new address entry for provided network name and role. -func makePhonebookEntryData(networkName string, role PhoneBookEntryRoles, persistent bool) addressData { +func makePhonebookEntryData(networkName string, role phonebook.PhoneBookEntryRoles, persistent bool) addressData { pbData := addressData{ networkNames: make(map[string]bool), + mu: &deadlock.RWMutex{}, recentConnectionTimes: make([]time.Time, 0), role: role, persistent: persistent, @@ -308,8 +299,11 @@ func (ps *PeerStore) deletePhonebookEntry(peerID peer.ID, networkName string) { return } ad := data.(addressData) + ad.mu.Lock() delete(ad.networkNames, networkName) - if 0 == len(ad.networkNames) { + isEmpty := len(ad.networkNames) == 0 + ad.mu.Unlock() + if isEmpty { ps.ClearAddrs(peerID) _ = ps.Put(peerID, addressDataKey, nil) } @@ -334,21 +328,23 @@ func (ps *PeerStore) popNElements(n int, peerID peer.ID) { _ = ps.Put(peerID, addressDataKey, ad) } -func (ps *PeerStore) filterRetryTime(t time.Time, role PhoneBookEntryRoles) []string { - o := make([]string, 0, len(ps.Peers())) +func (ps *PeerStore) filterRetryTime(t time.Time, role phonebook.PhoneBookEntryRoles) []interface{} { + o := make([]interface{}, 0, len(ps.Peers())) for _, peerID := range ps.Peers() { data, _ := ps.Get(peerID, addressDataKey) if data != nil { ad := data.(addressData) if t.After(ad.retryAfter) && role == ad.role { - o = append(o, string(peerID)) + mas := ps.Addrs(peerID) + info := peer.AddrInfo{ID: peerID, Addrs: mas} + o = append(o, &info) } } } return o } -func shuffleSelect(set []string, n int) []string { +func shuffleSelect(set []interface{}, n int) []interface{} { if n >= len(set) || n == getAllAddresses { // return shuffled copy of everything out := slices.Clone(set) @@ -365,13 +361,13 @@ func shuffleSelect(set []string, n int) []string { } } } - out := make([]string, n) + out := make([]interface{}, n) for i, index := range indexSample { out[i] = set[index] } return out } -func shuffleStrings(set []string) { +func shuffleStrings(set []interface{}) { rand.Shuffle(len(set), func(i, j int) { set[i], set[j] = set[j], set[i] }) } diff --git a/network/p2p/peerstore/peerstore_test.go b/network/p2p/peerstore/peerstore_test.go index 9bbf2b87c6..e855013d76 100644 --- a/network/p2p/peerstore/peerstore_test.go +++ b/network/p2p/peerstore/peerstore_test.go @@ -23,11 +23,13 @@ import ( "testing" "time" - "github.com/algorand/go-algorand/test/partitiontest" libp2p_crypto "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" libp2p "github.com/libp2p/go-libp2p/core/peerstore" "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/network/phonebook" + "github.com/algorand/go-algorand/test/partitiontest" ) // PhoneBookEntryRelayRole used for all the relays that are provided either via the algobootstrap SRV record @@ -49,7 +51,7 @@ func TestPeerstore(t *testing.T) { } addrInfo, _ := PeerInfoFromAddrs(peerAddrs) - ps, err := NewPeerStore(addrInfo) + ps, err := NewPeerStore(addrInfo, "net-id") require.NoError(t, err) defer ps.Close() @@ -87,12 +89,13 @@ func TestPeerstore(t *testing.T) { } -func testPhonebookAll(t *testing.T, set []string, ph *PeerStore) { +func testPhonebookAll(t *testing.T, set []*peer.AddrInfo, ph *PeerStore) { actual := ph.GetAddresses(len(set), PhoneBookEntryRelayRole) for _, got := range actual { + info := got.(*peer.AddrInfo) ok := false for _, known := range set { - if got == known { + if info.ID == known.ID { ok = true break } @@ -104,7 +107,8 @@ func testPhonebookAll(t *testing.T, set []string, ph *PeerStore) { for _, known := range set { ok := false for _, got := range actual { - if got == known { + info := got.(*peer.AddrInfo) + if info.ID == known.ID { ok = true break } @@ -115,18 +119,19 @@ func testPhonebookAll(t *testing.T, set []string, ph *PeerStore) { } } -func testPhonebookUniform(t *testing.T, set []string, ph *PeerStore, getsize int) { +func testPhonebookUniform(t *testing.T, set []*peer.AddrInfo, ph *PeerStore, getsize int) { uniformityTestLength := 250000 / len(set) expected := (uniformityTestLength * getsize) / len(set) counts := make(map[string]int) for i := 0; i < len(set); i++ { - counts[set[i]] = 0 + counts[set[i].ID.String()] = 0 } for i := 0; i < uniformityTestLength; i++ { actual := ph.GetAddresses(getsize, PhoneBookEntryRelayRole) for _, xa := range actual { - if _, ok := counts[xa]; ok { - counts[xa]++ + info := xa.(*peer.AddrInfo) + if _, ok := counts[info.ID.String()]; ok { + counts[info.ID.String()]++ } } } @@ -149,57 +154,84 @@ func TestArrayPhonebookAll(t *testing.T) { partitiontest.PartitionTest(t) set := []string{"a:4041", "b:4042", "c:4043", "d:4044", "e:4045", "f:4046", "g:4047", "h:4048", "i:4049", "j:4010"} + infoSet := make([]*peer.AddrInfo, 0) + for _, addr := range set { + info, err := peerInfoFromDomainPort(addr) + require.NoError(t, err) + infoSet = append(infoSet, info) + } + ph, err := MakePhonebook(1, 1*time.Millisecond) require.NoError(t, err) for _, addr := range set { entry := makePhonebookEntryData("", PhoneBookEntryRelayRole, false) - info, _ := PeerInfoFromDomainPort(addr) + info, _ := peerInfoFromDomainPort(addr) ph.AddAddrs(info.ID, info.Addrs, libp2p.AddressTTL) ph.Put(info.ID, addressDataKey, entry) } - testPhonebookAll(t, set, ph) + testPhonebookAll(t, infoSet, ph) } func TestArrayPhonebookUniform1(t *testing.T) { partitiontest.PartitionTest(t) set := []string{"a:4041", "b:4042", "c:4043", "d:4044", "e:4045", "f:4046", "g:4047", "h:4048", "i:4049", "j:4010"} + infoSet := make([]*peer.AddrInfo, 0) + for _, addr := range set { + info, err := peerInfoFromDomainPort(addr) + require.NoError(t, err) + infoSet = append(infoSet, info) + } + ph, err := MakePhonebook(1, 1*time.Millisecond) require.NoError(t, err) for _, addr := range set { entry := makePhonebookEntryData("", PhoneBookEntryRelayRole, false) - info, _ := PeerInfoFromDomainPort(addr) + info, _ := peerInfoFromDomainPort(addr) ph.AddAddrs(info.ID, info.Addrs, libp2p.AddressTTL) ph.Put(info.ID, addressDataKey, entry) } - testPhonebookUniform(t, set, ph, 1) + testPhonebookUniform(t, infoSet, ph, 1) } func TestArrayPhonebookUniform3(t *testing.T) { partitiontest.PartitionTest(t) set := []string{"a:4041", "b:4042", "c:4043", "d:4044", "e:4045", "f:4046", "g:4047", "h:4048", "i:4049", "j:4010"} + infoSet := make([]*peer.AddrInfo, 0) + for _, addr := range set { + info, err := peerInfoFromDomainPort(addr) + require.NoError(t, err) + infoSet = append(infoSet, info) + } + ph, err := MakePhonebook(1, 1*time.Millisecond) require.NoError(t, err) for _, addr := range set { entry := makePhonebookEntryData("", PhoneBookEntryRelayRole, false) - info, _ := PeerInfoFromDomainPort(addr) + info, _ := peerInfoFromDomainPort(addr) ph.AddAddrs(info.ID, info.Addrs, libp2p.AddressTTL) ph.Put(info.ID, addressDataKey, entry) } - testPhonebookUniform(t, set, ph, 3) + testPhonebookUniform(t, infoSet, ph, 3) } func TestMultiPhonebook(t *testing.T) { partitiontest.PartitionTest(t) set := []string{"a:4041", "b:4042", "c:4043", "d:4044", "e:4045", "f:4046", "g:4047", "h:4048", "i:4049", "j:4010"} - pha := make([]string, 0) - for _, e := range set[:5] { + infoSet := make([]*peer.AddrInfo, 0) + for _, addr := range set { + info, err := peerInfoFromDomainPort(addr) + require.NoError(t, err) + infoSet = append(infoSet, info) + } + pha := make([]interface{}, 0) + for _, e := range infoSet[:5] { pha = append(pha, e) } - phb := make([]string, 0) - for _, e := range set[5:] { + phb := make([]interface{}, 0) + for _, e := range infoSet[5:] { phb = append(phb, e) } @@ -208,9 +240,9 @@ func TestMultiPhonebook(t *testing.T) { ph.ReplacePeerList(pha, "pha", PhoneBookEntryRelayRole) ph.ReplacePeerList(phb, "phb", PhoneBookEntryRelayRole) - testPhonebookAll(t, set, ph) - testPhonebookUniform(t, set, ph, 1) - testPhonebookUniform(t, set, ph, 3) + testPhonebookAll(t, infoSet, ph) + testPhonebookUniform(t, infoSet, ph, 1) + testPhonebookUniform(t, infoSet, ph, 3) } // TestMultiPhonebookPersistentPeers validates that the peers added via Phonebook.AddPersistentPeers @@ -218,14 +250,23 @@ func TestMultiPhonebook(t *testing.T) { func TestMultiPhonebookPersistentPeers(t *testing.T) { partitiontest.PartitionTest(t) - persistentPeers := []string{"a:4041"} + info, err := peerInfoFromDomainPort("a:4041") + require.NoError(t, err) + persistentPeers := []interface{}{info} set := []string{"b:4042", "c:4043", "d:4044", "e:4045", "f:4046", "g:4047", "h:4048", "i:4049", "j:4010"} - pha := make([]string, 0) - for _, e := range set[:5] { + infoSet := make([]*peer.AddrInfo, 0) + for _, addr := range set { + info, err := peerInfoFromDomainPort(addr) + require.NoError(t, err) + infoSet = append(infoSet, info) + } + + pha := make([]interface{}, 0) + for _, e := range infoSet[:5] { pha = append(pha, e) } - phb := make([]string, 0) - for _, e := range set[5:] { + phb := make([]interface{}, 0) + for _, e := range infoSet[5:] { phb = append(phb, e) } ph, err := MakePhonebook(1, 1*time.Millisecond) @@ -235,10 +276,19 @@ func TestMultiPhonebookPersistentPeers(t *testing.T) { ph.ReplacePeerList(pha, "pha", PhoneBookEntryRelayRole) ph.ReplacePeerList(phb, "phb", PhoneBookEntryRelayRole) - testPhonebookAll(t, append(set, persistentPeers...), ph) + testPhonebookAll(t, append(infoSet, info), ph) allAddresses := ph.GetAddresses(len(set)+len(persistentPeers), PhoneBookEntryRelayRole) for _, pp := range persistentPeers { - require.Contains(t, allAddresses, pp) + pp := pp.(*peer.AddrInfo) + found := false + for _, addr := range allAddresses { + addr := addr.(*peer.AddrInfo) + if addr.ID == pp.ID { + found = true + break + } + } + require.True(t, found, fmt.Sprintf("%s not found in %v", string(pp.ID), allAddresses)) } } @@ -246,12 +296,19 @@ func TestMultiPhonebookDuplicateFiltering(t *testing.T) { partitiontest.PartitionTest(t) set := []string{"b:4042", "c:4043", "d:4044", "e:4045", "f:4046", "g:4047", "h:4048", "i:4049", "j:4010"} - pha := make([]string, 0) - for _, e := range set[:7] { + infoSet := make([]*peer.AddrInfo, 0) + for _, addr := range set { + info, err := peerInfoFromDomainPort(addr) + require.NoError(t, err) + infoSet = append(infoSet, info) + } + + pha := make([]interface{}, 0) + for _, e := range infoSet[:7] { pha = append(pha, e) } - phb := make([]string, 0) - for _, e := range set[3:] { + phb := make([]interface{}, 0) + for _, e := range infoSet[3:] { phb = append(phb, e) } ph, err := MakePhonebook(1, 1*time.Millisecond) @@ -259,9 +316,9 @@ func TestMultiPhonebookDuplicateFiltering(t *testing.T) { ph.ReplacePeerList(pha, "pha", PhoneBookEntryRelayRole) ph.ReplacePeerList(phb, "phb", PhoneBookEntryRelayRole) - testPhonebookAll(t, set, ph) - testPhonebookUniform(t, set, ph, 1) - testPhonebookUniform(t, set, ph, 3) + testPhonebookAll(t, infoSet, ph) + testPhonebookUniform(t, infoSet, ph, 1) + testPhonebookUniform(t, infoSet, ph, 3) } func TestWaitAndAddConnectionTimeLongtWindow(t *testing.T) { @@ -276,21 +333,21 @@ func TestWaitAndAddConnectionTimeLongtWindow(t *testing.T) { require.NoError(t, err) addr1 := "addrABC:4040" addr2 := "addrXYZ:4041" - info1, _ := PeerInfoFromDomainPort(addr1) - info2, _ := PeerInfoFromDomainPort(addr2) + info1, _ := peerInfoFromDomainPort(addr1) + info2, _ := peerInfoFromDomainPort(addr2) // Address not in. Should return false - addrInPhonebook, _, provisionalTime := entries.GetConnectionWaitTime(addr1) + addrInPhonebook, _, provisionalTime := entries.GetConnectionWaitTime(string(info1.ID)) require.Equal(t, false, addrInPhonebook) - require.Equal(t, false, entries.UpdateConnectionTime(addr1, provisionalTime)) + require.Equal(t, false, entries.UpdateConnectionTime(string(info1.ID), provisionalTime)) // Test the addresses are populated in the phonebook and a // time can be added to one of them - entries.ReplacePeerList([]string{addr1, addr2}, "default", PhoneBookEntryRelayRole) - addrInPhonebook, waitTime, provisionalTime := entries.GetConnectionWaitTime(addr1) + entries.ReplacePeerList([]interface{}{info1, info2}, "default", PhoneBookEntryRelayRole) + addrInPhonebook, waitTime, provisionalTime := entries.GetConnectionWaitTime(string(info1.ID)) require.Equal(t, true, addrInPhonebook) require.Equal(t, time.Duration(0), waitTime) - require.Equal(t, true, entries.UpdateConnectionTime(addr1, provisionalTime)) + require.Equal(t, true, entries.UpdateConnectionTime(string(info1.ID), provisionalTime)) data, _ := entries.Get(info1.ID, addressDataKey) require.NotNil(t, data) ad := data.(addressData) @@ -303,9 +360,9 @@ func TestWaitAndAddConnectionTimeLongtWindow(t *testing.T) { } // add another value to addr - addrInPhonebook, waitTime, provisionalTime = entries.GetConnectionWaitTime(addr1) + addrInPhonebook, waitTime, provisionalTime = entries.GetConnectionWaitTime(string(info1.ID)) require.Equal(t, time.Duration(0), waitTime) - require.Equal(t, true, entries.UpdateConnectionTime(addr1, provisionalTime)) + require.Equal(t, true, entries.UpdateConnectionTime(string(info1.ID), provisionalTime)) data, _ = entries.Get(info1.ID, addressDataKey) ad = data.(addressData) phBookData = ad.recentConnectionTimes @@ -318,9 +375,9 @@ func TestWaitAndAddConnectionTimeLongtWindow(t *testing.T) { // the first time should be removed and a new one added // there should not be any wait - addrInPhonebook, waitTime, provisionalTime = entries.GetConnectionWaitTime(addr1) + addrInPhonebook, waitTime, provisionalTime = entries.GetConnectionWaitTime(string(info1.ID)) require.Equal(t, time.Duration(0), waitTime) - require.Equal(t, true, entries.UpdateConnectionTime(addr1, provisionalTime)) + require.Equal(t, true, entries.UpdateConnectionTime(string(info1.ID), provisionalTime)) data, _ = entries.Get(info1.ID, addressDataKey) ad = data.(addressData) phBookData2 := ad.recentConnectionTimes @@ -335,9 +392,9 @@ func TestWaitAndAddConnectionTimeLongtWindow(t *testing.T) { // add 3 values to another address. should not wait // value 1 - _, waitTime, provisionalTime = entries.GetConnectionWaitTime(addr2) + _, waitTime, provisionalTime = entries.GetConnectionWaitTime(string(info2.ID)) require.Equal(t, time.Duration(0), waitTime) - require.Equal(t, true, entries.UpdateConnectionTime(addr2, provisionalTime)) + require.Equal(t, true, entries.UpdateConnectionTime(string(info2.ID), provisionalTime)) // introduce a gap between the two requests so that only the first will be removed later when waited // simulate passing a unit of time @@ -349,13 +406,13 @@ func TestWaitAndAddConnectionTimeLongtWindow(t *testing.T) { } // value 2 - _, waitTime, provisionalTime = entries.GetConnectionWaitTime(addr2) + _, waitTime, provisionalTime = entries.GetConnectionWaitTime(string(info2.ID)) require.Equal(t, time.Duration(0), waitTime) - require.Equal(t, true, entries.UpdateConnectionTime(addr2, provisionalTime)) + require.Equal(t, true, entries.UpdateConnectionTime(string(info2.ID), provisionalTime)) // value 3 - _, waitTime, provisionalTime = entries.GetConnectionWaitTime(addr2) + _, waitTime, provisionalTime = entries.GetConnectionWaitTime(string(info2.ID)) require.Equal(t, time.Duration(0), waitTime) - require.Equal(t, true, entries.UpdateConnectionTime(addr2, provisionalTime)) + require.Equal(t, true, entries.UpdateConnectionTime(string(info2.ID), provisionalTime)) data2, _ = entries.Get(info2.ID, addressDataKey) ad2 = data2.(addressData) @@ -364,7 +421,7 @@ func TestWaitAndAddConnectionTimeLongtWindow(t *testing.T) { require.Equal(t, 3, len(phBookData)) // add another element to trigger wait - _, waitTime, provisionalTime = entries.GetConnectionWaitTime(addr2) + _, waitTime, provisionalTime = entries.GetConnectionWaitTime(string(info2.ID)) require.Greater(t, int64(waitTime), int64(0)) // no element should be removed data2, _ = entries.Get(info2.ID, addressDataKey) @@ -379,9 +436,9 @@ func TestWaitAndAddConnectionTimeLongtWindow(t *testing.T) { } // The wait should be sufficient - _, waitTime, provisionalTime = entries.GetConnectionWaitTime(addr2) + _, waitTime, provisionalTime = entries.GetConnectionWaitTime(string(info2.ID)) require.Equal(t, time.Duration(0), waitTime) - require.Equal(t, true, entries.UpdateConnectionTime(addr2, provisionalTime)) + require.Equal(t, true, entries.UpdateConnectionTime(string(info2.ID), provisionalTime)) // only one element should be removed, and one added data2, _ = entries.Get(info2.ID, addressDataKey) ad2 = data2.(addressData) @@ -401,24 +458,40 @@ func TestPhonebookRoles(t *testing.T) { relaysSet := []string{"relay1:4040", "relay2:4041", "relay3:4042"} archiverSet := []string{"archiver1:1111", "archiver2:1112", "archiver3:1113"} + infoRelaySet := make([]interface{}, 0) + for _, addr := range relaysSet { + info, err := peerInfoFromDomainPort(addr) + require.NoError(t, err) + infoRelaySet = append(infoRelaySet, info) + } + + infoArchiverSet := make([]interface{}, 0) + for _, addr := range archiverSet { + info, err := peerInfoFromDomainPort(addr) + require.NoError(t, err) + infoArchiverSet = append(infoArchiverSet, info) + } + ph, err := MakePhonebook(1, 1) require.NoError(t, err) - ph.ReplacePeerList(relaysSet, "default", PhoneBookEntryRelayRole) - ph.ReplacePeerList(archiverSet, "default", PhoneBookEntryArchiverRole) + ph.ReplacePeerList(infoRelaySet, "default", PhoneBookEntryRelayRole) + ph.ReplacePeerList(infoArchiverSet, "default", PhoneBookEntryArchiverRole) require.Equal(t, len(relaysSet)+len(archiverSet), len(ph.Peers())) require.Equal(t, len(relaysSet)+len(archiverSet), ph.Length()) - for _, role := range []PhoneBookEntryRoles{PhoneBookEntryRelayRole, PhoneBookEntryArchiverRole} { + for _, role := range []phonebook.PhoneBookEntryRoles{PhoneBookEntryRelayRole, PhoneBookEntryArchiverRole} { for k := 0; k < 100; k++ { for l := 0; l < 3; l++ { entries := ph.GetAddresses(l, role) if role == PhoneBookEntryRelayRole { for _, entry := range entries { - require.Contains(t, entry, "relay") + entry := entry.(*peer.AddrInfo) + require.Contains(t, string(entry.ID), "relay") } } else if role == PhoneBookEntryArchiverRole { for _, entry := range entries { - require.Contains(t, entry, "archiver") + entry := entry.(*peer.AddrInfo) + require.Contains(t, string(entry.ID), "archiver") } } } diff --git a/network/p2p/peerstore/utils.go b/network/p2p/peerstore/utils.go index 02c6b2d8e6..90b0af497c 100644 --- a/network/p2p/peerstore/utils.go +++ b/network/p2p/peerstore/utils.go @@ -53,8 +53,8 @@ func PeerInfoFromAddr(addr string) (*peer.AddrInfo, error) { return info, nil } -// PeerInfoFromDomainPort converts a string of the form domain:port to AddrInfo -func PeerInfoFromDomainPort(domainPort string) (*peer.AddrInfo, error) { +// peerInfoFromDomainPort converts a string of the form domain:port to AddrInfo +func peerInfoFromDomainPort(domainPort string) (*peer.AddrInfo, error) { parts := strings.Split(domainPort, ":") if len(parts) != 2 || parts[0] == "" || parts[1] == "" { return nil, fmt.Errorf("invalid domain port string %s, found %d colon-separated parts", domainPort, len(parts)) diff --git a/network/p2p/peerstore/utils_test.go b/network/p2p/peerstore/utils_test.go index 9f20b0f75b..d0d67f4fe0 100644 --- a/network/p2p/peerstore/utils_test.go +++ b/network/p2p/peerstore/utils_test.go @@ -20,8 +20,9 @@ import ( "fmt" "testing" - "github.com/algorand/go-algorand/test/partitiontest" "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/test/partitiontest" ) func TestPeerInfoFromAddr(t *testing.T) { diff --git a/network/p2p/pubsub.go b/network/p2p/pubsub.go index 372c9249c8..a968bcb6a9 100644 --- a/network/p2p/pubsub.go +++ b/network/p2p/pubsub.go @@ -53,6 +53,8 @@ const ( // TXTopicName defines a pubsub topic for TX messages const TXTopicName = "/algo/tx/0.1.0" +const incomingThreads = 20 // matches to number wsNetwork workers + func makePubSub(ctx context.Context, cfg config.Local, host host.Host) (*pubsub.PubSub, error) { //defaultParams := pubsub.DefaultGossipSubParams() @@ -93,7 +95,9 @@ func makePubSub(ctx context.Context, cfg config.Local, host host.Host) (*pubsub. pubsub.WithSubscriptionFilter(pubsub.WrapLimitSubscriptionFilter(pubsub.NewAllowlistSubscriptionFilter(TXTopicName), 100)), // pubsub.WithEventTracer(jsonTracer), pubsub.WithValidateQueueSize(256), + pubsub.WithMessageSignaturePolicy(pubsub.StrictNoSign), // pubsub.WithValidateThrottle(cfg.TxBacklogSize), + pubsub.WithValidateWorkers(incomingThreads), } return pubsub.NewGossipSub(ctx, host, options...) @@ -133,7 +137,7 @@ func (s *serviceImpl) getOrCreateTopic(topicName string) (*pubsub.Topic, error) } // Subscribe returns a subscription to the given topic -func (s *serviceImpl) Subscribe(topic string, val pubsub.ValidatorEx) (*pubsub.Subscription, error) { +func (s *serviceImpl) Subscribe(topic string, val pubsub.ValidatorEx) (SubNextCancellable, error) { if err := s.pubsub.RegisterTopicValidator(topic, val); err != nil { return nil, err } diff --git a/network/p2p/streams.go b/network/p2p/streams.go index 4a7a2d8e01..d16633adfd 100644 --- a/network/p2p/streams.go +++ b/network/p2p/streams.go @@ -73,7 +73,17 @@ func (n *streamManager) streamHandler(stream network.Stream) { n.log.Infof("Failed to check old stream with %s: %v", remotePeer, err) } n.streams[stream.Conn().RemotePeer()] = stream - n.handler(n.ctx, remotePeer, stream, true) + + // streamHandler is supposed to be called for accepted streams, so we expect incoming here + incoming := stream.Conn().Stat().Direction == network.DirInbound + if !incoming { + if stream.Stat().Direction == network.DirUnknown { + n.log.Warnf("Unknown direction for a steam %s to/from %s", stream.ID(), remotePeer) + } else { + n.log.Warnf("Unexpected outgoing stream in streamHandler for connection %s (%s): %s vs %s stream", stream.Conn().ID(), remotePeer, stream.Conn().Stat().Direction, stream.Stat().Direction.String()) + } + } + n.handler(n.ctx, remotePeer, stream, incoming) return } // otherwise, the old stream is still open, so we can close the new one @@ -82,7 +92,30 @@ func (n *streamManager) streamHandler(stream network.Stream) { } // no old stream n.streams[stream.Conn().RemotePeer()] = stream - n.handler(n.ctx, remotePeer, stream, true) + // streamHandler is supposed to be called for accepted streams, so we expect incoming here + incoming := stream.Conn().Stat().Direction == network.DirInbound + if !incoming { + if stream.Stat().Direction == network.DirUnknown { + n.log.Warnf("streamHandler: unknown direction for a steam %s to/from %s", stream.ID(), remotePeer) + } else { + n.log.Warnf("Unexpected outgoing stream in streamHandler for connection %s (%s): %s vs %s stream", stream.Conn().ID(), remotePeer, stream.Conn().Stat().Direction, stream.Stat().Direction.String()) + } + } + n.handler(n.ctx, remotePeer, stream, incoming) +} + +// streamHandlerHTTP tracks the ProtocolIDForMultistreamSelect = "/http/1.1" streams +func (n *streamManager) streamHandlerHTTP(stream network.Stream) { + n.streamsLock.Lock() + defer n.streamsLock.Unlock() + n.streams[stream.Conn().LocalPeer()] = stream +} + +func (n *streamManager) getStream(peerID peer.ID) (network.Stream, bool) { + n.streamsLock.Lock() + defer n.streamsLock.Unlock() + stream, ok := n.streams[peerID] + return stream, ok } // Connected is called when a connection is opened @@ -95,8 +128,13 @@ func (n *streamManager) Connected(net network.Network, conn network.Conn) { return } + needUnlock := true n.streamsLock.Lock() - defer n.streamsLock.Unlock() + defer func() { + if needUnlock { + n.streamsLock.Unlock() + } + }() _, ok := n.streams[remotePeer] if ok { return // there's already an active stream with this peer for our protocol @@ -104,12 +142,26 @@ func (n *streamManager) Connected(net network.Network, conn network.Conn) { stream, err := n.host.NewStream(n.ctx, remotePeer, AlgorandWsProtocol) if err != nil { - n.log.Infof("Failed to open stream to %s: %v", remotePeer, err) + n.log.Infof("Failed to open stream to %s (%s): %v", remotePeer, conn.RemoteMultiaddr().String(), err) return } - n.streams[remotePeer] = stream - n.handler(n.ctx, remotePeer, stream, false) + + // release the lock to let handler do its thing + // otherwise reading/writing to the stream will deadlock + needUnlock = false + n.streamsLock.Unlock() + + // a new stream created above, expected direction is outbound + incoming := stream.Conn().Stat().Direction == network.DirInbound + if incoming { + n.log.Warnf("Unexpected incoming stream in streamHandler for connection %s (%s): %s vs %s stream", stream.Conn().ID(), remotePeer, stream.Conn().Stat().Direction, stream.Stat().Direction.String()) + } else { + if stream.Stat().Direction == network.DirUnknown { + n.log.Warnf("Connected: unknown direction for a steam %s to/from %s", stream.ID(), remotePeer) + } + } + n.handler(n.ctx, remotePeer, stream, incoming) } // Disconnected is called when a connection is closed @@ -122,6 +174,12 @@ func (n *streamManager) Disconnected(net network.Network, conn network.Conn) { stream.Close() delete(n.streams, conn.RemotePeer()) } + + stream, ok = n.streams[conn.LocalPeer()] + if ok { + stream.Close() + delete(n.streams, conn.LocalPeer()) + } } // Listen is called when network starts listening on an addr diff --git a/network/p2p/testing/httpNode.go b/network/p2p/testing/httpNode.go new file mode 100644 index 0000000000..523cdc5d4c --- /dev/null +++ b/network/p2p/testing/httpNode.go @@ -0,0 +1,122 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +// This package wraps and re-exports the libp2p functions on order to keep +// all go-libp2p imports in one place. + +package p2p + +import ( + "net/http" + "testing" + + "github.com/algorand/go-algorand/components/mocks" + "github.com/algorand/go-algorand/network" + "github.com/algorand/go-algorand/network/p2p" + "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/require" +) + +// HTTPNode is a mock network node that uses libp2p and http. +type HTTPNode struct { + mocks.MockNetwork + host.Host + httpServer *p2p.HTTPServer + peers []network.Peer + tb testing.TB + genesisID string +} + +// MakeHTTPNode returns a new P2PHTTPNode node. +func MakeHTTPNode(tb testing.TB) *HTTPNode { + p2pHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0")) + require.NoError(tb, err) + + return &HTTPNode{ + Host: p2pHost, + httpServer: p2p.MakeHTTPServer(p2pHost), + tb: tb, + } +} + +// RegisterHTTPHandler registers a http handler with a given path. +func (p *HTTPNode) RegisterHTTPHandler(path string, handler http.Handler) { + p.httpServer.RegisterHTTPHandler(path, handler) +} + +// RegisterHandlers not implemented. +func (p *HTTPNode) RegisterHandlers(dispatch []network.TaggedMessageHandler) {} + +// Start starts http service +func (p *HTTPNode) Start() error { + go func() { + err := p.httpServer.Serve() + require.NoError(p.tb, err) + }() + return nil +} + +// Stop stops http service +func (p *HTTPNode) Stop() { + p.httpServer.Close() + p.Host.Close() +} + +// GetHTTPPeer returns the http peer for connecting to this node +func (p *HTTPNode) GetHTTPPeer() network.Peer { + addrInfo := peer.AddrInfo{ID: p.ID(), Addrs: p.Addrs()} + return httpPeer{addrInfo, p.tb} +} + +// GetGenesisID returns genesisID +func (p *HTTPNode) GetGenesisID() string { return p.genesisID } + +// SetGenesisID sets genesisID +func (p *HTTPNode) SetGenesisID(genesisID string) { p.genesisID = genesisID } + +type httpPeer struct { + addrInfo peer.AddrInfo + tb testing.TB +} + +// GetAddress implements HTTPPeer interface returns the address of the peer +func (p httpPeer) GetAddress() string { + mas, err := peer.AddrInfoToP2pAddrs(&p.addrInfo) + require.NoError(p.tb, err) + require.Len(p.tb, mas, 1) + return mas[0].String() +} + +// GetAddress implements HTTPPeer interface and returns the http client for a peer +func (p httpPeer) GetHTTPClient() *http.Client { + c, err := p2p.MakeHTTPClient(&p.addrInfo) + require.NoError(p.tb, err) + return c +} + +// SetPeers sets peers +func (p *HTTPNode) SetPeers(other *HTTPNode) { + addrInfo := peer.AddrInfo{ID: other.ID(), Addrs: other.Addrs()} + hpeer := httpPeer{addrInfo, p.tb} + p.peers = append(p.peers, hpeer) +} + +// GetPeers returns peers +func (p *HTTPNode) GetPeers(options ...network.PeerOption) []network.Peer { + return p.peers +} diff --git a/network/p2pNetwork.go b/network/p2pNetwork.go index 6301b1b521..7ebbb5a665 100644 --- a/network/p2pNetwork.go +++ b/network/p2pNetwork.go @@ -18,6 +18,7 @@ package network import ( "context" + "math/rand" "net" "net/http" "strings" @@ -27,14 +28,20 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/logging/telemetryspec" + "github.com/algorand/go-algorand/network/limitcaller" "github.com/algorand/go-algorand/network/p2p" + "github.com/algorand/go-algorand/network/p2p/dnsaddr" "github.com/algorand/go-algorand/network/p2p/peerstore" + "github.com/algorand/go-algorand/network/phonebook" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-deadlock" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/p2p/discovery/backoff" + "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" ) @@ -59,17 +66,156 @@ type P2PNetwork struct { handler msgHandler broadcaster msgBroadcaster wsPeers map[peer.ID]*wsPeer + wsPeersToIDs map[*wsPeer]peer.ID wsPeersLock deadlock.RWMutex wsPeersChangeCounter atomic.Int32 wsPeersConnectivityCheckTicker *time.Ticker + peerStater peerConnectionStater + + relayMessages bool // True if we should relay messages from other nodes (nominally true for relays, false otherwise) + wantTXGossip atomic.Bool + + capabilitiesDiscovery *p2p.CapabilitiesDiscovery + + bootstrapperStart func() + bootstrapperStop func() + nodeInfo NodeInfo + pstore *peerstore.PeerStore + httpServer *p2p.HTTPServer +} + +type bootstrapper struct { + cfg config.Local + networkID protocol.NetworkID + phonebookPeers []*peer.AddrInfo + resolveController dnsaddr.ResolveController + started atomic.Bool + log logging.Logger +} + +func (b *bootstrapper) start() { + b.started.Store(true) +} + +func (b *bootstrapper) stop() { + b.started.Store(false) +} + +func (b *bootstrapper) BootstrapFunc() []peer.AddrInfo { + // not started yet, do not give it any peers + if !b.started.Load() { + return nil + } + + // have a list of peers, use them + if len(b.phonebookPeers) > 0 { + var addrs []peer.AddrInfo + for _, bPeer := range b.phonebookPeers { + if bPeer != nil { + addrs = append(addrs, *bPeer) + } + } + return addrs + } + + return dnsLookupBootstrapPeers(b.log, b.cfg, b.networkID, b.resolveController) +} + +// dnsLookupBootstrapPeers looks up a list of Multiaddrs strings from the dnsaddr records at the primary +// SRV record domain. +func dnsLookupBootstrapPeers(log logging.Logger, cfg config.Local, network protocol.NetworkID, controller dnsaddr.ResolveController) []peer.AddrInfo { + var addrs []peer.AddrInfo + bootstraps := cfg.DNSBootstrapArray(network) + for _, dnsBootstrap := range bootstraps { + var resolvedAddrs, resolvedAddrsBackup []multiaddr.Multiaddr + var errPrim, errBackup error + resolvedAddrs, errPrim = dnsaddr.MultiaddrsFromResolver(dnsBootstrap.PrimarySRVBootstrap, controller) + if errPrim != nil { + log.Infof("Failed to resolve bootstrap peers from %s: %v", dnsBootstrap.PrimarySRVBootstrap, errPrim) + } + if dnsBootstrap.BackupSRVBootstrap != "" { + resolvedAddrsBackup, errBackup = dnsaddr.MultiaddrsFromResolver(dnsBootstrap.BackupSRVBootstrap, controller) + if errBackup != nil { + log.Infof("Failed to resolve bootstrap peers from %s: %v", dnsBootstrap.BackupSRVBootstrap, errBackup) + } + } + + if len(resolvedAddrs) > 0 || len(resolvedAddrsBackup) > 0 { + resolvedAddrInfos := mergeP2PMultiaddrResolvedAddresses(resolvedAddrs, resolvedAddrsBackup) + addrs = append(addrs, resolvedAddrInfos...) + } + } + return addrs +} + +func mergeP2PMultiaddrResolvedAddresses(primary, backup []multiaddr.Multiaddr) []peer.AddrInfo { + // deduplicate addresses by PeerID + unique := make(map[peer.ID]*peer.AddrInfo) + for _, addr := range primary { + info, err0 := peer.AddrInfoFromP2pAddr(addr) + if err0 != nil { + continue + } + unique[info.ID] = info + } + for _, addr := range backup { + info, err0 := peer.AddrInfoFromP2pAddr(addr) + if err0 != nil { + continue + } + unique[info.ID] = info + } + var result []peer.AddrInfo + for _, addr := range unique { + result = append(result, *addr) + } + return result +} + +func mergeP2PAddrInfoResolvedAddresses(primary, backup []peer.AddrInfo) []peer.AddrInfo { + // deduplicate addresses by PeerID + unique := make(map[peer.ID]peer.AddrInfo) + for _, addr := range primary { + unique[addr.ID] = addr + } + for _, addr := range backup { + unique[addr.ID] = addr + } + var result []peer.AddrInfo + for _, addr := range unique { + result = append(result, addr) + } + return result } type p2pPeerStats struct { txReceived atomic.Uint64 } +// gossipSubPeer implements the DeadlineSettableConn, IPAddressable, and ErlClient interfaces. +type gossipSubPeer struct { + peerID peer.ID + net GossipNode + routingAddr [8]byte +} + +func (p gossipSubPeer) GetNetwork() GossipNode { return p.net } + +func (p gossipSubPeer) OnClose(f func()) { + net := p.GetNetwork().(*P2PNetwork) + net.wsPeersLock.Lock() + defer net.wsPeersLock.Unlock() + if wsp, ok := net.wsPeers[p.peerID]; ok { + wsp.OnClose(f) + } +} + +func (p gossipSubPeer) RoutingAddr() []byte { + return p.routingAddr[:] +} + // NewP2PNetwork returns an instance of GossipNode that uses the p2p.Service -func NewP2PNetwork(log logging.Logger, cfg config.Local, datadir string, phonebookAddresses []string, genesisID string, networkID protocol.NetworkID) (*P2PNetwork, error) { +func NewP2PNetwork(log logging.Logger, cfg config.Local, datadir string, phonebookAddresses []string, genesisID string, networkID protocol.NetworkID, node NodeInfo) (*P2PNetwork, error) { const readBufferLen = 2048 // create Peerstore and add phonebook addresses @@ -77,20 +223,31 @@ func NewP2PNetwork(log logging.Logger, cfg config.Local, datadir string, phonebo for malAddr, malErr := range malformedAddrs { log.Infof("Ignoring malformed phonebook address %s: %s", malAddr, malErr) } - pstore, err := peerstore.NewPeerStore(addrInfo) + pstore, err := peerstore.NewPeerStore(addrInfo, string(networkID)) if err != nil { return nil, err } + relayMessages := cfg.IsGossipServer() || cfg.ForceRelayMessages net := &P2PNetwork{ - log: log, - config: cfg, - genesisID: genesisID, - networkID: networkID, - topicTags: map[protocol.Tag]string{"TX": p2p.TXTopicName}, - wsPeers: make(map[peer.ID]*wsPeer), - peerStats: make(map[peer.ID]*p2pPeerStats), + log: log, + config: cfg, + genesisID: genesisID, + networkID: networkID, + topicTags: map[protocol.Tag]string{protocol.TxnTag: p2p.TXTopicName}, + wsPeers: make(map[peer.ID]*wsPeer), + wsPeersToIDs: make(map[*wsPeer]peer.ID), + peerStats: make(map[peer.ID]*p2pPeerStats), + nodeInfo: node, + pstore: pstore, + relayMessages: relayMessages, + peerStater: peerConnectionStater{ + log: log, + peerConnectionsUpdateInterval: time.Duration(cfg.PeerConnectionsUpdateInterval) * time.Second, + lastPeerConnectionsSent: time.Now(), + }, } + net.ctx, net.ctxCancel = context.WithCancel(context.Background()) net.handler = msgHandler{ ctx: net.ctx, @@ -106,10 +263,39 @@ func NewP2PNetwork(log logging.Logger, cfg config.Local, datadir string, phonebo broadcastQueueBulk: make(chan broadcastRequest, 100), } - net.service, err = p2p.MakeService(net.ctx, log, cfg, datadir, pstore, net.wsStreamHandler) + p2p.EnableP2PLogging(log, logging.Level(cfg.BaseLoggerDebugLevel)) + + h, la, err := p2p.MakeHost(cfg, datadir, pstore) if err != nil { return nil, err } + log.Infof("P2P host created: peer ID %s addrs %s", h.ID(), h.Addrs()) + + net.service, err = p2p.MakeService(net.ctx, log, cfg, h, la, net.wsStreamHandler, addrInfo) + if err != nil { + return nil, err + } + + bootstrapper := &bootstrapper{ + cfg: cfg, + networkID: networkID, + phonebookPeers: addrInfo, + resolveController: dnsaddr.NewMultiaddrDNSResolveController(cfg.DNSSecurityTXTEnforced(), ""), + log: net.log, + } + net.bootstrapperStart = bootstrapper.start + net.bootstrapperStop = bootstrapper.stop + + if cfg.EnableDHTProviders { + disc, err0 := p2p.MakeCapabilitiesDiscovery(net.ctx, cfg, h, networkID, net.log, bootstrapper.BootstrapFunc) + if err0 != nil { + log.Errorf("Failed to create dht node capabilities discovery: %v", err) + return nil, err + } + net.capabilitiesDiscovery = disc + } + + net.httpServer = p2p.MakeHTTPServer(h) err = net.setup() if err != nil { @@ -126,10 +312,30 @@ func (n *P2PNetwork) setup() error { return nil } +// PeerID returns this node's peer ID. +func (n *P2PNetwork) PeerID() p2p.PeerID { + return p2p.PeerID(n.service.ID()) +} + +// PeerIDSigner returns an identityChallengeSigner that uses the libp2p peer ID's private key. +func (n *P2PNetwork) PeerIDSigner() identityChallengeSigner { + return n.service.IDSigner() +} + // Start threads, listen on sockets. -func (n *P2PNetwork) Start() { - n.wg.Add(1) - go n.txTopicHandleLoop() +func (n *P2PNetwork) Start() error { + n.bootstrapperStart() + err := n.service.Start() + if err != nil { + return err + } + + wantTXGossip := n.relayMessages || n.config.ForceFetchTransactions || n.nodeInfo.IsParticipating() + if wantTXGossip { + n.wantTXGossip.Store(true) + n.wg.Add(1) + go n.txTopicHandleLoop() + } if n.wsPeersConnectivityCheckTicker != nil { n.wsPeersConnectivityCheckTicker.Stop() @@ -141,16 +347,31 @@ func (n *P2PNetwork) Start() { go n.handler.messageHandlerThread(&n.wg, n.wsPeersConnectivityCheckTicker.C, n, "network", "P2PNetwork") } + n.wg.Add(1) + go n.httpdThread() + n.wg.Add(1) go n.broadcaster.broadcastThread(&n.wg, n, "network", "P2PNetwork") - n.service.DialPeersUntilTargetCount(n.config.GossipFanout) n.wg.Add(1) go n.meshThread() + + if n.capabilitiesDiscovery != nil { + n.capabilitiesDiscovery.AdvertiseCapabilities(n.nodeInfo.Capabilities()...) + } + + return nil } // Stop closes sockets and stop threads. func (n *P2PNetwork) Stop() { + if n.capabilitiesDiscovery != nil { + err := n.capabilitiesDiscovery.Close() + if err != nil { + n.log.Warnf("Error closing capabilities discovery: %v", err) + } + } + n.handler.ClearHandlers([]Tag{}) if n.wsPeersConnectivityCheckTicker != nil { n.wsPeersConnectivityCheckTicker.Stop() @@ -159,6 +380,8 @@ func (n *P2PNetwork) Stop() { n.innerStop() n.ctxCancel() n.service.Close() + n.bootstrapperStop() + n.httpServer.Close() n.wg.Wait() } @@ -176,22 +399,83 @@ func (n *P2PNetwork) innerStop() { n.log.Warnf("Error closing peer %s: %v", peerID, err) } delete(n.wsPeers, peerID) + delete(n.wsPeersToIDs, peer) } n.wsPeersLock.Unlock() closeGroup.Wait() } +// meshThreadInner fetches nodes from DHT and attempts to connect to them +func (n *P2PNetwork) meshThreadInner() int { + defer n.service.DialPeersUntilTargetCount(n.config.GossipFanout) + + // fetch peers from DNS + var dnsPeers, dhtPeers []peer.AddrInfo + dnsPeers = dnsLookupBootstrapPeers(n.log, n.config, n.networkID, dnsaddr.NewMultiaddrDNSResolveController(n.config.DNSSecurityTXTEnforced(), "")) + + // discover peers from DHT + if n.capabilitiesDiscovery != nil { + var err error + dhtPeers, err = n.capabilitiesDiscovery.PeersForCapability(p2p.Gossip, n.config.GossipFanout) + if err != nil { + n.log.Warnf("Error getting relay nodes from capabilities discovery: %v", err) + } + n.log.Debugf("Discovered %d gossip peers from DHT", len(dhtPeers)) + } + + peers := mergeP2PAddrInfoResolvedAddresses(dnsPeers, dhtPeers) + replace := make([]interface{}, 0, len(peers)) + for i := range peers { + replace = append(replace, &peers[i]) + } + if len(peers) > 0 { + n.pstore.ReplacePeerList(replace, string(n.networkID), phonebook.PhoneBookEntryRelayRole) + } + return len(peers) +} + func (n *P2PNetwork) meshThread() { defer n.wg.Done() - timer := time.NewTicker(meshThreadInterval) + + timer := time.NewTicker(1) // start immediately and reset after + + // Add exponential backoff with jitter to the mesh thread to handle new networks startup + // when no DNS or DHT peers are available. + // The parameters produce approximate the following delays (although they are random but the sequence give the idea): + // 2 2.4 4.6 9 20 19.5 28 24 14 14 35 60 60 + ebf := backoff.NewExponentialDecorrelatedJitter(2*time.Second, meshThreadInterval, 3.0, rand.NewSource(rand.Int63())) + eb := ebf() + defer timer.Stop() for { select { case <-timer.C: - n.service.DialPeersUntilTargetCount(n.config.GossipFanout) + numPeers := n.meshThreadInner() + if numPeers > 0 { + // found something, reset timer to the default value + timer.Reset(meshThreadInterval) + eb.Reset() + } else { + // no peers found, backoff + timer.Reset(eb.Delay()) + } case <-n.ctx.Done(): return } + + // send the currently connected peers information to the + // telemetry server; that would allow the telemetry server + // to construct a cross-node map of all the nodes interconnections. + n.peerStater.sendPeerConnectionsTelemetryStatus(n) + } +} + +func (n *P2PNetwork) httpdThread() { + defer n.wg.Done() + err := n.httpServer.Serve() + if err != nil { + n.log.Errorf("Error serving libp2phttp: %v", err) + return } } @@ -240,31 +524,42 @@ func (n *P2PNetwork) Broadcast(ctx context.Context, tag protocol.Tag, data []byt // Relay message func (n *P2PNetwork) Relay(ctx context.Context, tag protocol.Tag, data []byte, wait bool, except Peer) error { - return n.Broadcast(ctx, tag, data, wait, except) + if n.relayMessages { + return n.Broadcast(ctx, tag, data, wait, except) + } + return nil } // Disconnect from a peer, probably due to protocol errors. -func (n *P2PNetwork) Disconnect(badnode Peer) { - node, ok := badnode.(peer.ID) - if !ok { - n.log.Warnf("Unknown peer type %T", badnode) - return - } +func (n *P2PNetwork) Disconnect(badpeer DisconnectablePeer) { + var peerID peer.ID + var wsp *wsPeer + n.wsPeersLock.Lock() defer n.wsPeersLock.Unlock() - if wsPeer, ok := n.wsPeers[node]; ok { - wsPeer.CloseAndWait(time.Now().Add(peerDisconnectionAckDuration)) - delete(n.wsPeers, node) + switch p := badpeer.(type) { + case gossipSubPeer: // Disconnect came from a message received via GossipSub + peerID, wsp = p.peerID, n.wsPeers[p.peerID] + case *wsPeer: // Disconnect came from a message received via wsPeer + peerID, wsp = n.wsPeersToIDs[p], p + default: + n.log.Warnf("Unknown peer type %T", badpeer) + return + } + if wsp != nil { + wsp.CloseAndWait(time.Now().Add(peerDisconnectionAckDuration)) + delete(n.wsPeers, peerID) + delete(n.wsPeersToIDs, wsp) } else { - n.log.Warnf("Could not find wsPeer reference for peer %s", node) + n.log.Warnf("Could not find wsPeer reference for peer %s", peerID) } - err := n.service.ClosePeer(node) + err := n.service.ClosePeer(peerID) if err != nil { - n.log.Warnf("Error disconnecting from peer %s: %v", node, err) + n.log.Warnf("Error disconnecting from peer %s: %v", peerID, err) } } -func (n *P2PNetwork) disconnectThread(badnode Peer, reason disconnectReason) { +func (n *P2PNetwork) disconnectThread(badnode DisconnectablePeer, reason disconnectReason) { defer n.wg.Done() n.Disconnect(badnode) // ignores reason } @@ -278,23 +573,106 @@ func (n *P2PNetwork) DisconnectPeers() { // RegisterHTTPHandler path accepts gorilla/mux path annotations func (n *P2PNetwork) RegisterHTTPHandler(path string, handler http.Handler) { + n.httpServer.RegisterHTTPHandler(path, handler) } // RequestConnectOutgoing asks the system to actually connect to peers. // `replace` optionally drops existing connections before making new ones. // `quit` chan allows cancellation. func (n *P2PNetwork) RequestConnectOutgoing(replace bool, quit <-chan struct{}) { + n.meshThreadInner() +} + +func addrInfoToWsPeerCore(n *P2PNetwork, addrInfo *peer.AddrInfo) (wsPeerCore, bool) { + mas, err := peer.AddrInfoToP2pAddrs(addrInfo) + if err != nil { + n.log.Warnf("Archival AddrInfo conversion error: %v", err) + return wsPeerCore{}, false + } + if len(mas) == 0 { + n.log.Warnf("Archival AddrInfo: empty multiaddr for : %v", addrInfo) + return wsPeerCore{}, false + } + addr := mas[0].String() + + maxIdleConnsPerHost := int(n.config.ConnectionsRateLimitingCount) + client, err := p2p.MakeHTTPClientWithRateLimit(addrInfo, n.pstore, limitcaller.DefaultQueueingTimeout, maxIdleConnsPerHost) + if err != nil { + n.log.Warnf("MakeHTTPClient failed: %v", err) + return wsPeerCore{}, false + } + + peerCore := makePeerCore( + n.ctx, n, n.log, n.handler.readBuffer, + addr, client, "", /*origin address*/ + ) + return peerCore, true } // GetPeers returns a list of Peers we could potentially send a direct message to. func (n *P2PNetwork) GetPeers(options ...PeerOption) []Peer { - // currently returns same list of peers for all PeerOption filters. peers := make([]Peer, 0) - n.wsPeersLock.RLock() - for _, peer := range n.wsPeers { - peers = append(peers, Peer(peer)) + for _, option := range options { + switch option { + case PeersConnectedOut: + n.wsPeersLock.RLock() + for _, peer := range n.wsPeers { + if peer.outgoing { + peers = append(peers, Peer(peer)) + } + } + n.wsPeersLock.RUnlock() + case PeersPhonebookRelays: + const maxNodes = 100 + peerIDs := n.pstore.GetAddresses(maxNodes, phonebook.PhoneBookEntryRelayRole) + for _, peerInfo := range peerIDs { + peerInfo := peerInfo.(*peer.AddrInfo) + if peerCore, ok := addrInfoToWsPeerCore(n, peerInfo); ok { + peers = append(peers, &peerCore) + } + } + if n.log.GetLevel() >= logging.Debug && len(peers) > 0 { + addrs := make([]string, 0, len(peers)) + for _, peer := range peers { + addrs = append(addrs, peer.(*wsPeerCore).GetAddress()) + } + n.log.Debugf("Relay node(s) from peerstore: %v", addrs) + } + case PeersPhonebookArchivalNodes: + // query known archival nodes from DHT if enabled + if n.config.EnableDHTProviders { + const nodesToFind = 5 + infos, err := n.capabilitiesDiscovery.PeersForCapability(p2p.Archival, nodesToFind) + if err != nil { + n.log.Warnf("Error getting archival nodes from capabilities discovery: %v", err) + return peers + } + n.log.Debugf("Got %d archival node(s) from DHT", len(infos)) + for _, addrInfo := range infos { + // TODO: remove after go1.22 + info := addrInfo + if peerCore, ok := addrInfoToWsPeerCore(n, &info); ok { + peers = append(peers, &peerCore) + } + } + if n.log.GetLevel() >= logging.Debug && len(peers) > 0 { + addrs := make([]string, 0, len(peers)) + for _, peer := range peers { + addrs = append(addrs, peer.(*wsPeerCore).GetAddress()) + } + n.log.Debugf("Archival node(s) from DHT: %v", addrs) + } + } + case PeersConnectedIn: + n.wsPeersLock.RLock() + for _, peer := range n.wsPeers { + if !peer.outgoing { + peers = append(peers, Peer(peer)) + } + } + n.wsPeersLock.RUnlock() + } } - n.wsPeersLock.RUnlock() return peers } @@ -308,31 +686,67 @@ func (n *P2PNetwork) ClearHandlers() { n.handler.ClearHandlers([]Tag{}) } -// GetRoundTripper returns a Transport that would limit the number of outgoing connections. -func (n *P2PNetwork) GetRoundTripper() http.RoundTripper { - return http.DefaultTransport +// RegisterProcessors adds to the set of given message handlers. +func (n *P2PNetwork) RegisterProcessors(dispatch []TaggedMessageProcessor) { + n.handler.RegisterProcessors(dispatch) +} + +// ClearProcessors deregisters all the existing message handlers. +func (n *P2PNetwork) ClearProcessors() { + n.handler.ClearProcessors([]Tag{}) +} + +// GetHTTPClient returns a http.Client with a suitable for the network Transport +// that would also limit the number of outgoing connections. +func (n *P2PNetwork) GetHTTPClient(address string) (*http.Client, error) { + addrInfo, err := peer.AddrInfoFromString(address) + if err != nil { + return nil, err + } + maxIdleConnsPerHost := int(n.config.ConnectionsRateLimitingCount) + return p2p.MakeHTTPClientWithRateLimit(addrInfo, n.pstore, limitcaller.DefaultQueueingTimeout, maxIdleConnsPerHost) } // OnNetworkAdvance notifies the network library that the agreement protocol was able to make a notable progress. // this is the only indication that we have that we haven't formed a clique, where all incoming messages // arrive very quickly, but might be missing some votes. The usage of this call is expected to have similar // characteristics as with a watchdog timer. -func (n *P2PNetwork) OnNetworkAdvance() {} +func (n *P2PNetwork) OnNetworkAdvance() { + if n.nodeInfo != nil { + old := n.wantTXGossip.Load() + new := n.relayMessages || n.config.ForceFetchTransactions || n.nodeInfo.IsParticipating() + if old != new { + n.wantTXGossip.Store(new) + if new { + n.wg.Add(1) + go n.txTopicHandleLoop() + } + } + } +} // GetHTTPRequestConnection returns the underlying connection for the given request. Note that the request must be the same // request that was provided to the http handler ( or provide a fallback Context() to that ) -func (n *P2PNetwork) GetHTTPRequestConnection(request *http.Request) (conn net.Conn) { return nil } - -// SubstituteGenesisID substitutes the "{genesisID}" with their network-specific genesisID. -func (n *P2PNetwork) SubstituteGenesisID(rawURL string) string { - return strings.Replace(rawURL, "{genesisID}", n.genesisID, -1) +func (n *P2PNetwork) GetHTTPRequestConnection(request *http.Request) (conn DeadlineSettableConn) { + addr := request.Context().Value(http.LocalAddrContextKey).(net.Addr) + peerID, err := peer.Decode(addr.String()) + if err != nil { + n.log.Infof("GetHTTPRequestConnection failed to decode %s", addr.String()) + return nil + } + conn, ok := n.service.GetStream(peerID) + if !ok { + n.log.Warnf("GetHTTPRequestConnection no such stream for peer %s", peerID.String()) + return nil + } + return conn } // wsStreamHandler is a callback that the p2p package calls when a new peer connects and establishes a // stream for the websocket protocol. -func (n *P2PNetwork) wsStreamHandler(ctx context.Context, peer peer.ID, stream network.Stream, incoming bool) { +func (n *P2PNetwork) wsStreamHandler(ctx context.Context, p2pPeer peer.ID, stream network.Stream, incoming bool) { if stream.Protocol() != p2p.AlgorandWsProtocol { - n.log.Warnf("unknown protocol %s", stream.Protocol()) + n.log.Warnf("unknown protocol %s from peer%s", stream.Protocol(), p2pPeer) return } @@ -340,7 +754,7 @@ func (n *P2PNetwork) wsStreamHandler(ctx context.Context, peer peer.ID, stream n var initMsg [1]byte rn, err := stream.Read(initMsg[:]) if rn == 0 || err != nil { - n.log.Warnf("wsStreamHandler: error reading initial message: %s", err) + n.log.Warnf("wsStreamHandler: error reading initial message: %s, peer %s (%s)", err, p2pPeer, stream.Conn().RemoteMultiaddr().String()) return } } else { @@ -352,21 +766,63 @@ func (n *P2PNetwork) wsStreamHandler(ctx context.Context, peer peer.ID, stream n } // get address for peer ID - addr := stream.Conn().RemoteMultiaddr().String() + ma := stream.Conn().RemoteMultiaddr() + addr := ma.String() if addr == "" { - n.log.Warnf("Could not get address for peer %s", peer) + n.log.Warnf("Could not get address for peer %s", p2pPeer) } // create a wsPeer for this stream and added it to the peers map. + + addrInfo := &peer.AddrInfo{ID: p2pPeer, Addrs: []multiaddr.Multiaddr{ma}} + maxIdleConnsPerHost := int(n.config.ConnectionsRateLimitingCount) + client, err := p2p.MakeHTTPClientWithRateLimit(addrInfo, n.pstore, limitcaller.DefaultQueueingTimeout, maxIdleConnsPerHost) + if err != nil { + client = nil + } + peerCore := makePeerCore(ctx, n, n.log, n.handler.readBuffer, addr, client, addr) wsp := &wsPeer{ - wsPeerCore: makePeerCore(ctx, n, n.log, n.handler.readBuffer, addr, n.GetRoundTripper(), addr), + wsPeerCore: peerCore, conn: &wsPeerConnP2PImpl{stream: stream}, outgoing: !incoming, } + protos, err := n.pstore.GetProtocols(p2pPeer) + if err != nil { + n.log.Warnf("Error getting protocols for peer %s: %v", p2pPeer, err) + } + wsp.TelemetryGUID, wsp.InstanceName = p2p.GetPeerTelemetryInfo(protos) + wsp.init(n.config, outgoingMessagesBufferSize) n.wsPeersLock.Lock() - n.wsPeers[peer] = wsp + n.wsPeers[p2pPeer] = wsp + n.wsPeersToIDs[wsp] = p2pPeer n.wsPeersLock.Unlock() n.wsPeersChangeCounter.Add(1) + + event := "ConnectedOut" + msg := "Made outgoing connection to peer %s" + if incoming { + event = "ConnectedIn" + msg = "Accepted incoming connection from peer %s" + } + localAddr, has := n.Address() + if !has { + n.log.Warn("Could not get local address") + } + n.log.With("event", event).With("remote", addr).With("local", localAddr).Infof(msg, p2pPeer.String()) + + if n.log.GetLevel() >= logging.Debug { + n.log.Debugf("streams for %s conn %s ", stream.Conn().Stat().Direction.String(), stream.Conn().ID()) + for _, s := range stream.Conn().GetStreams() { + n.log.Debugf("%s stream %s protocol %s", s.Stat().Direction.String(), s.ID(), s.Protocol()) + } + } + n.log.EventWithDetails(telemetryspec.Network, telemetryspec.ConnectPeerEvent, + telemetryspec.PeerEventDetails{ + Address: addr, + TelemetryGUID: wsp.TelemetryGUID, + Incoming: incoming, + InstanceName: wsp.InstanceName, + }) } // peerRemoteClose called from wsPeer to report that it has closed @@ -374,8 +830,30 @@ func (n *P2PNetwork) peerRemoteClose(peer *wsPeer, reason disconnectReason) { remotePeerID := peer.conn.(*wsPeerConnP2PImpl).stream.Conn().RemotePeer() n.wsPeersLock.Lock() delete(n.wsPeers, remotePeerID) + delete(n.wsPeersToIDs, peer) n.wsPeersLock.Unlock() n.wsPeersChangeCounter.Add(1) + + eventDetails := telemetryspec.PeerEventDetails{ + Address: peer.GetAddress(), // p2p peers store p2p addresses + TelemetryGUID: peer.TelemetryGUID, + InstanceName: peer.InstanceName, + Incoming: !peer.outgoing, + } + if peer.outgoing { + eventDetails.Endpoint = peer.GetAddress() + eventDetails.MessageDelay = peer.peerMessageDelay + } + + n.log.EventWithDetails(telemetryspec.Network, telemetryspec.DisconnectPeerEvent, + telemetryspec.DisconnectPeerEventDetails{ + PeerEventDetails: eventDetails, + Reason: string(reason), + TXCount: peer.txMessageCount.Load(), + MICount: peer.miMessageCount.Load(), + AVCount: peer.avMessageCount.Load(), + PPCount: peer.ppMessageCount.Load(), + }) } func (n *P2PNetwork) peerSnapshot(dest []*wsPeer) ([]*wsPeer, int32) { @@ -417,6 +895,7 @@ func (n *P2PNetwork) txTopicHandleLoop() { n.log.Errorf("Failed to subscribe to topic %s: %v", p2p.TXTopicName, err) return } + n.log.Debugf("Subscribed to topic %s", p2p.TXTopicName) for { msg, err := sub.Next(n.ctx) @@ -424,21 +903,40 @@ func (n *P2PNetwork) txTopicHandleLoop() { if err != pubsub.ErrSubscriptionCancelled && err != context.Canceled { n.log.Errorf("Error reading from subscription %v, peerId %s", err, n.service.ID()) } + n.log.Debugf("Cancelling subscription to topic %s due Subscription.Next error: %v", p2p.TXTopicName, err) sub.Cancel() return } + // if there is a self-sent the message no need to process it. + if msg.ReceivedFrom == n.service.ID() { + continue + } + + _ = n.handler.Process(msg.ValidatorData.(ValidatedMessage)) - // discard TX message. - // from gossipsub's point of view, it's just waiting to hear back from the validator, - // and txHandler does all its work in the validator, so we don't need to do anything here - _ = msg + // participation or configuration change, cancel subscription and quit + if !n.wantTXGossip.Load() { + n.log.Debugf("Cancelling subscription to topic %s due participation change", p2p.TXTopicName) + sub.Cancel() + return + } } } // txTopicValidator calls txHandler to validate and process incoming transactions. func (n *P2PNetwork) txTopicValidator(ctx context.Context, peerID peer.ID, msg *pubsub.Message) pubsub.ValidationResult { + var routingAddr [8]byte + n.wsPeersLock.Lock() + if wsp, ok := n.wsPeers[peerID]; ok { + copy(routingAddr[:], wsp.RoutingAddr()) + } else { + // well, otherwise use last 8 bytes of peerID + copy(routingAddr[:], peerID[len(peerID)-8:]) + } + n.wsPeersLock.Unlock() + inmsg := IncomingMessage{ - Sender: msg.ReceivedFrom, + Sender: gossipSubPeer{peerID: msg.ReceivedFrom, net: n, routingAddr: routingAddr}, Tag: protocol.TxnTag, Data: msg.Data, Net: n, @@ -446,7 +944,7 @@ func (n *P2PNetwork) txTopicValidator(ctx context.Context, peerID peer.ID, msg * } // if we sent the message, don't validate it - if inmsg.Sender == n.service.ID() { + if msg.ReceivedFrom == n.service.ID() { return pubsub.ValidationAccept } @@ -459,14 +957,15 @@ func (n *P2PNetwork) txTopicValidator(ctx context.Context, peerID peer.ID, msg * peerStats.txReceived.Add(1) n.peerStatsMu.Unlock() - outmsg := n.handler.Handle(inmsg) + outmsg := n.handler.Validate(inmsg) // there was a decision made in the handler about this message switch outmsg.Action { case Ignore: return pubsub.ValidationIgnore case Disconnect: return pubsub.ValidationReject - case Broadcast: // TxHandler.processIncomingTxn does not currently return this Action + case Accept: + msg.ValidatorData = outmsg return pubsub.ValidationAccept default: n.log.Warnf("handler returned invalid action %d", outmsg.Action) diff --git a/network/p2pNetwork_test.go b/network/p2pNetwork_test.go index c67ef5b1fc..5bd582ead0 100644 --- a/network/p2pNetwork_test.go +++ b/network/p2pNetwork_test.go @@ -18,49 +18,64 @@ package network import ( "context" + "errors" "fmt" + "io" + "net/http" + "sync" "sync/atomic" "testing" "time" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/network/limitcaller" "github.com/algorand/go-algorand/network/p2p" + "github.com/algorand/go-algorand/network/p2p/dnsaddr" + "github.com/algorand/go-algorand/network/p2p/peerstore" + "github.com/algorand/go-algorand/network/phonebook" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" + "github.com/algorand/go-algorand/util" pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" - peerstore "github.com/libp2p/go-libp2p/core/peer" ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/require" ) +func (n *P2PNetwork) hasPeers() bool { + n.wsPeersLock.RLock() + defer n.wsPeersLock.RUnlock() + return len(n.wsPeers) > 0 +} + func TestP2PSubmitTX(t *testing.T) { partitiontest.PartitionTest(t) cfg := config.GetDefaultLocal() + cfg.ForceFetchTransactions = true log := logging.TestingLog(t) - netA, err := NewP2PNetwork(log, cfg, "", nil, genesisID, config.Devtestnet) + netA, err := NewP2PNetwork(log, cfg, "", nil, genesisID, config.Devtestnet, &nopeNodeInfo{}) require.NoError(t, err) - peerInfoA := netA.service.AddrInfo() + netA.Start() + defer netA.Stop() - addrsA, err := peerstore.AddrInfoToP2pAddrs(&peerInfoA) + peerInfoA := netA.service.AddrInfo() + addrsA, err := peer.AddrInfoToP2pAddrs(&peerInfoA) require.NoError(t, err) require.NotZero(t, addrsA[0]) - netA.Start() - defer netA.Stop() multiAddrStr := addrsA[0].String() phoneBookAddresses := []string{multiAddrStr} - netB, err := NewP2PNetwork(log, cfg, "", phoneBookAddresses, genesisID, config.Devtestnet) + netB, err := NewP2PNetwork(log, cfg, "", phoneBookAddresses, genesisID, config.Devtestnet, &nopeNodeInfo{}) require.NoError(t, err) netB.Start() defer netB.Stop() - netC, err := NewP2PNetwork(log, cfg, "", phoneBookAddresses, genesisID, config.Devtestnet) - + netC, err := NewP2PNetwork(log, cfg, "", phoneBookAddresses, genesisID, config.Devtestnet, &nopeNodeInfo{}) require.NoError(t, err) netC.Start() defer netC.Stop() @@ -75,19 +90,36 @@ func TestP2PSubmitTX(t *testing.T) { 2*time.Second, 50*time.Millisecond, ) + require.Eventually(t, func() bool { + return netA.hasPeers() && netB.hasPeers() && netC.hasPeers() + }, 2*time.Second, 50*time.Millisecond) + + // for some reason the above check is not enough in race builds on CI time.Sleep(time.Second) // give time for peers to connect. + // now we should be connected in a line: B <-> A <-> C where both B and C are connected to A but not each other // Since we aren't using the transaction handler in this test, we need to register a pass-through handler - passThroughHandler := []TaggedMessageHandler{ - {Tag: protocol.TxnTag, MessageHandler: HandlerFunc(func(msg IncomingMessage) OutgoingMessage { - return OutgoingMessage{Action: Broadcast} - })}, + passThroughHandler := []TaggedMessageProcessor{ + { + Tag: protocol.TxnTag, + MessageHandler: struct { + ProcessorValidateFunc + ProcessorHandleFunc + }{ + ProcessorValidateFunc(func(msg IncomingMessage) ValidatedMessage { + return ValidatedMessage{Action: Accept, Tag: msg.Tag, ValidatedMessage: nil} + }), + ProcessorHandleFunc(func(msg ValidatedMessage) OutgoingMessage { + return OutgoingMessage{Action: Ignore} + }), + }, + }, } - netA.RegisterHandlers(passThroughHandler) - netB.RegisterHandlers(passThroughHandler) - netC.RegisterHandlers(passThroughHandler) + netA.RegisterProcessors(passThroughHandler) + netB.RegisterProcessors(passThroughHandler) + netC.RegisterProcessors(passThroughHandler) // send messages from B and confirm that they get received by C (via A) for i := 0; i < 10; i++ { @@ -111,45 +143,139 @@ func TestP2PSubmitTX(t *testing.T) { ) } -func TestP2PSubmitWS(t *testing.T) { +// TestP2PSubmitTXNoGossip tests nodes without gossip enabled cannot receive transactions +func TestP2PSubmitTXNoGossip(t *testing.T) { partitiontest.PartitionTest(t) cfg := config.GetDefaultLocal() + cfg.ForceFetchTransactions = true log := logging.TestingLog(t) - netA, err := NewP2PNetwork(log, cfg, "", nil, genesisID, config.Devtestnet) + netA, err := NewP2PNetwork(log, cfg, "", nil, genesisID, config.Devtestnet, &nopeNodeInfo{}) require.NoError(t, err) + netA.Start() + defer netA.Stop() peerInfoA := netA.service.AddrInfo() - addrsA, err := peerstore.AddrInfoToP2pAddrs(&peerInfoA) + addrsA, err := peer.AddrInfoToP2pAddrs(&peerInfoA) require.NoError(t, err) require.NotZero(t, addrsA[0]) - netA.Start() - defer netA.Stop() multiAddrStr := addrsA[0].String() phoneBookAddresses := []string{multiAddrStr} - netB, err := NewP2PNetwork(log, cfg, "", phoneBookAddresses, genesisID, config.Devtestnet) + netB, err := NewP2PNetwork(log, cfg, "", phoneBookAddresses, genesisID, config.Devtestnet, &nopeNodeInfo{}) require.NoError(t, err) netB.Start() defer netB.Stop() - netC, err := NewP2PNetwork(log, cfg, "", phoneBookAddresses, genesisID, config.Devtestnet) + require.Eventually( + t, + func() bool { + return len(netA.service.ListPeersForTopic(p2p.TXTopicName)) == 1 && + len(netB.service.ListPeersForTopic(p2p.TXTopicName)) == 1 + }, + 2*time.Second, + 50*time.Millisecond, + ) + // run netC in NPN mode (no relay => no gossip sup => no TX receiving) + cfg.ForceFetchTransactions = false + netC, err := NewP2PNetwork(log, cfg, "", phoneBookAddresses, genesisID, config.Devtestnet, &nopeNodeInfo{}) require.NoError(t, err) netC.Start() defer netC.Stop() + require.Eventually(t, func() bool { + return netA.hasPeers() && netB.hasPeers() && netC.hasPeers() + }, 2*time.Second, 50*time.Millisecond) + + time.Sleep(time.Second) // give time for peers to connect. + + // ensure netC cannot receive messages + + passThroughHandler := []TaggedMessageProcessor{ + { + Tag: protocol.TxnTag, + MessageHandler: struct { + ProcessorValidateFunc + ProcessorHandleFunc + }{ + ProcessorValidateFunc(func(msg IncomingMessage) ValidatedMessage { + return ValidatedMessage{Action: Accept, Tag: msg.Tag, ValidatedMessage: nil} + }), + ProcessorHandleFunc(func(msg ValidatedMessage) OutgoingMessage { + return OutgoingMessage{Action: Ignore} + }), + }, + }, + } + + netB.RegisterProcessors(passThroughHandler) + netC.RegisterProcessors(passThroughHandler) + for i := 0; i < 10; i++ { + err = netA.Broadcast(context.Background(), protocol.TxnTag, []byte(fmt.Sprintf("test %d", i)), false, nil) + require.NoError(t, err) + } + + // check netB received the messages require.Eventually( t, func() bool { - return len(netA.service.ListPeersForTopic(p2p.TXTopicName)) == 2 && - len(netB.service.ListPeersForTopic(p2p.TXTopicName)) == 1 && - len(netC.service.ListPeersForTopic(p2p.TXTopicName)) == 1 + netB.peerStatsMu.Lock() + netBpeerStatsA, ok := netB.peerStats[netA.service.ID()] + netB.peerStatsMu.Unlock() + if !ok { + return false + } + return netBpeerStatsA.txReceived.Load() == 10 }, - 2*time.Second, + 1*time.Second, 50*time.Millisecond, ) - time.Sleep(time.Second) // XX give time for peers to connect. Knowing about them being subscribed to topics is clearly not enough + + // check netC did not receive the messages + netC.peerStatsMu.Lock() + _, ok := netC.peerStats[netA.service.ID()] + netC.peerStatsMu.Unlock() + require.False(t, ok) +} + +func TestP2PSubmitWS(t *testing.T) { + partitiontest.PartitionTest(t) + + cfg := config.GetDefaultLocal() + log := logging.TestingLog(t) + netA, err := NewP2PNetwork(log, cfg, "", nil, genesisID, config.Devtestnet, &nopeNodeInfo{}) + require.NoError(t, err) + + err = netA.Start() + require.NoError(t, err) + defer netA.Stop() + + peerInfoA := netA.service.AddrInfo() + addrsA, err := peer.AddrInfoToP2pAddrs(&peerInfoA) + require.NoError(t, err) + require.NotZero(t, addrsA[0]) + + multiAddrStr := addrsA[0].String() + phoneBookAddresses := []string{multiAddrStr} + netB, err := NewP2PNetwork(log, cfg, "", phoneBookAddresses, genesisID, config.Devtestnet, &nopeNodeInfo{}) + require.NoError(t, err) + err = netB.Start() + require.NoError(t, err) + defer netB.Stop() + + netC, err := NewP2PNetwork(log, cfg, "", phoneBookAddresses, genesisID, config.Devtestnet, &nopeNodeInfo{}) + require.NoError(t, err) + err = netC.Start() + require.NoError(t, err) + defer netC.Stop() + + require.Eventually(t, func() bool { + return netA.hasPeers() && netB.hasPeers() && netC.hasPeers() + }, 2*time.Second, 50*time.Millisecond) + + time.Sleep(time.Second) // give time for peers to connect. + // now we should be connected in a line: B <-> A <-> C where both B and C are connected to A but not each other testTag := protocol.AgreementVoteTag @@ -189,6 +315,10 @@ type mockService struct { peers map[peer.ID]peer.AddrInfo } +func (s *mockService) Start() error { + return nil +} + func (s *mockService) Close() error { return nil } @@ -197,6 +327,10 @@ func (s *mockService) ID() peer.ID { return s.id } +func (s *mockService) IDSigner() *p2p.PeerIDChallengeSigner { + panic("not implemented") +} + func (s *mockService) AddrInfo() peer.AddrInfo { return peer.AddrInfo{ ID: s.id, @@ -213,9 +347,7 @@ func (s *mockService) DialPeersUntilTargetCount(targetConnCount int) { } func (s *mockService) ClosePeer(peer peer.ID) error { - if _, ok := s.peers[peer]; ok { - delete(s.peers, peer) - } + delete(s.peers, peer) return nil } @@ -227,15 +359,15 @@ func (s *mockService) ListPeersForTopic(topic string) []peer.ID { return nil } -func (s *mockService) Subscribe(topic string, val pubsub.ValidatorEx) (*pubsub.Subscription, error) { +func (s *mockService) Subscribe(topic string, val pubsub.ValidatorEx) (p2p.SubNextCancellable, error) { return nil, nil } func (s *mockService) Publish(ctx context.Context, topic string, data []byte) error { return nil } -func (s *mockService) setAddrs(addrs []ma.Multiaddr) { - s.addrs = addrs +func (s *mockService) GetStream(peer.ID) (network.Stream, bool) { + return nil, false } func makeMockService(id peer.ID, addrs []ma.Multiaddr) *mockService { @@ -250,7 +382,7 @@ func TestP2PNetworkAddress(t *testing.T) { cfg := config.GetDefaultLocal() log := logging.TestingLog(t) - netA, err := NewP2PNetwork(log, cfg, "", nil, genesisID, config.Devtestnet) + netA, err := NewP2PNetwork(log, cfg, "", nil, genesisID, config.Devtestnet, &nopeNodeInfo{}) defer netA.Stop() require.NoError(t, err) addrInfo := netA.service.AddrInfo() @@ -304,3 +436,692 @@ func TestP2PNetworkAddress(t *testing.T) { require.False(t, ok) require.Empty(t, retAddr) } + +type nilResolveController struct{} + +func (c *nilResolveController) Resolver() dnsaddr.Resolver { + return nil +} + +func (c *nilResolveController) NextResolver() dnsaddr.Resolver { + return nil +} + +type mockResolveController struct { + nilResolveController +} + +func (c *mockResolveController) Resolver() dnsaddr.Resolver { + return &mockResolver{} +} + +type mockResolver struct{} + +func (r *mockResolver) Resolve(ctx context.Context, _ ma.Multiaddr) ([]ma.Multiaddr, error) { + // return random stuff each time + _, publicKey, err := crypto.GenerateKeyPair(crypto.RSA, 2048) + if err != nil { + panic(err) + } + peerID, err := peer.IDFromPublicKey(publicKey) + if err != nil { + panic(err) + } + maddr, err := ma.NewMultiaddr("/ip4/127.0.0.1/p2p/" + peerID.String()) + return []ma.Multiaddr{maddr}, err +} + +func TestP2PBootstrapFunc(t *testing.T) { + t.Parallel() + partitiontest.PartitionTest(t) + + b := bootstrapper{} + require.Nil(t, b.BootstrapFunc()) + + b.started.Store(true) + p := peer.AddrInfo{ID: "test"} + b.phonebookPeers = []*peer.AddrInfo{&p} + require.Equal(t, []peer.AddrInfo{p}, b.BootstrapFunc()) + + b.phonebookPeers = nil + + b.cfg = config.GetDefaultLocal() + b.cfg.DNSBootstrapID = ".algodev.network" + b.cfg.DNSSecurityFlags = 0 + b.networkID = "devnet" + b.resolveController = &mockResolveController{} + + addrs := b.BootstrapFunc() + + require.GreaterOrEqual(t, len(addrs), 1) + addr := addrs[0] + require.Equal(t, len(addr.Addrs), 1) + require.GreaterOrEqual(t, len(addr.Addrs), 1) +} + +func TestP2PdnsLookupBootstrapPeersFailure(t *testing.T) { + t.Parallel() + partitiontest.PartitionTest(t) + + cfg := config.GetDefaultLocal() + cfg.DNSSecurityFlags = 0 + cfg.DNSBootstrapID = "non-existent.algodev.network" + + controller := nilResolveController{} + addrs := dnsLookupBootstrapPeers(logging.TestingLog(t), cfg, "test", &controller) + + require.Equal(t, 0, len(addrs)) +} + +func TestP2PdnsLookupBootstrapPeersInvalidAddr(t *testing.T) { + t.Parallel() + partitiontest.PartitionTest(t) + + cfg := config.GetDefaultLocal() + cfg.DNSSecurityFlags = 0 + cfg.DNSBootstrapID = ".algodev.network" + + controller := nilResolveController{} + addrs := dnsLookupBootstrapPeers(logging.TestingLog(t), cfg, "testInvalidAddr", &controller) + + require.Equal(t, 0, len(addrs)) +} + +func TestP2PdnsLookupBootstrapPeersWithBackup(t *testing.T) { + t.Parallel() + partitiontest.PartitionTest(t) + + cfg := config.GetDefaultLocal() + cfg.DNSSecurityFlags = 0 + cfg.DNSBootstrapID = ".algodev.network" + + controller := &mockResolveController{} + addrs := dnsLookupBootstrapPeers(logging.TestingLog(t), cfg, "test", controller) + require.GreaterOrEqual(t, len(addrs), 1) + + cfg.DNSBootstrapID = ".algodev.network?backup=.backup.algodev.network" + addrs = dnsLookupBootstrapPeers(logging.TestingLog(t), cfg, "test", controller) + require.GreaterOrEqual(t, len(addrs), 2) + +} + +type capNodeInfo struct { + nopeNodeInfo + cap p2p.Capability +} + +func (ni *capNodeInfo) Capabilities() []p2p.Capability { + return []p2p.Capability{ni.cap} +} + +func waitForRouting(t *testing.T, disc *p2p.CapabilitiesDiscovery) { + refreshCtx, refCancel := context.WithTimeout(context.Background(), time.Second*5) + for { + select { + case <-refreshCtx.Done(): + refCancel() + require.Fail(t, "failed to populate routing table before timeout") + default: + if disc.RoutingTable().Size() > 0 { + refCancel() + return + } + } + time.Sleep(50 * time.Millisecond) + } +} + +// TestP2PNetworkDHTCapabilities runs nodes with capabilities and ensures that connected nodes +// can discover itself. The other nodes receive the first node in bootstrap list before starting. +// There is two variations of the test: only netA advertises capabilities, and all nodes advertise. +func TestP2PNetworkDHTCapabilities(t *testing.T) { + partitiontest.PartitionTest(t) + + cfg := config.GetDefaultLocal() + cfg.EnableDHTProviders = true + log := logging.TestingLog(t) + + cap := p2p.Archival + tests := []struct { + name string + nis []NodeInfo + numCapPeers int + }{ + {"cap=all", []NodeInfo{&capNodeInfo{cap: cap}, &capNodeInfo{cap: cap}, &capNodeInfo{cap: cap}}, 2}, // each has 2 peers with capabilities + {"cap=netA", []NodeInfo{&capNodeInfo{cap: cap}, &nopeNodeInfo{}, &nopeNodeInfo{}}, 1}, // each has 1 peer with capabilities + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + netA, err := NewP2PNetwork(log, cfg, "", nil, genesisID, config.Devtestnet, test.nis[0]) + require.NoError(t, err) + + err = netA.Start() + require.NoError(t, err) + defer netA.Stop() + + peerInfoA := netA.service.AddrInfo() + addrsA, err := peer.AddrInfoToP2pAddrs(&peerInfoA) + require.NoError(t, err) + require.NotZero(t, addrsA[0]) + + multiAddrStr := addrsA[0].String() + phoneBookAddresses := []string{multiAddrStr} + netB, err := NewP2PNetwork(log, cfg, "", phoneBookAddresses, genesisID, config.Devtestnet, test.nis[1]) + require.NoError(t, err) + err = netB.Start() + require.NoError(t, err) + defer netB.Stop() + + netC, err := NewP2PNetwork(log, cfg, "", phoneBookAddresses, genesisID, config.Devtestnet, test.nis[2]) + require.NoError(t, err) + err = netC.Start() + require.NoError(t, err) + defer netC.Stop() + + require.Eventually(t, func() bool { + return netA.hasPeers() && netB.hasPeers() && netC.hasPeers() + }, 2*time.Second, 50*time.Millisecond) + + t.Logf("peers connected") + + nets := []*P2PNetwork{netA, netB, netC} + discs := []*p2p.CapabilitiesDiscovery{netA.capabilitiesDiscovery, netB.capabilitiesDiscovery, netC.capabilitiesDiscovery} + + var wg sync.WaitGroup + wg.Add(len(discs)) + for _, disc := range discs { + if disc == nil { + wg.Done() + continue + } + go func(disc *p2p.CapabilitiesDiscovery) { + defer wg.Done() + waitForRouting(t, disc) + }(disc) + } + wg.Wait() + + t.Logf("DHT is ready") + + // ensure all peers are connected - wait for connectivity as needed + for _, disc := range discs { + go func(disc *p2p.CapabilitiesDiscovery) { + require.Eventuallyf(t, func() bool { + return len(disc.Host().Network().Peers()) == 2 + }, time.Minute, time.Second, "Not all peers were found") + }(disc) + } + + wg.Add(len(discs)) + for i := range discs { + go func(idx int) { + disc := discs[idx] + defer wg.Done() + // skip netA since it is special for the test cap=netA + if test.name == "cap=netA" && disc == netA.capabilitiesDiscovery { + return + } + require.Eventuallyf(t, + func() bool { + peers, err := disc.PeersForCapability(cap, test.numCapPeers) + if err == nil && len(peers) == test.numCapPeers { + return true + } + return false + }, + time.Minute, + time.Second, + fmt.Sprintf("Not all expected %s cap peers were found", cap), + ) + // ensure GetPeers gets PeersPhonebookArchivalNodes peers + // it appears there are artifical peers because of listening on localhost and on a real network interface + // so filter out and save only unique peers by their IDs + net := nets[idx] + peers := net.GetPeers(PeersPhonebookArchivalNodes) + uniquePeerIDs := make(map[peer.ID]struct{}) + for _, p := range peers { + wsPeer := p.(*wsPeerCore) + pi, err := peer.AddrInfoFromString(wsPeer.GetAddress()) + require.NoError(t, err) + uniquePeerIDs[pi.ID] = struct{}{} + } + require.Equal(t, test.numCapPeers, len(uniquePeerIDs)) + }(i) + } + wg.Wait() + }) + } +} + +// TestMultiaddrConversionToFrom ensures Multiaddr can be serialized back to an address without losing information +func TestP2PMultiaddrConversionToFrom(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + const a = "/ip4/192.168.1.1/tcp/8180/p2p/Qmewz5ZHN1AAGTarRbMupNPbZRfg3p5jUGoJ3JYEatJVVk" + ma, err := ma.NewMultiaddr(a) + require.NoError(t, err) + require.Equal(t, a, ma.String()) + + // this conversion drops the p2p proto part + pi, err := peer.AddrInfoFromP2pAddr(ma) + require.NoError(t, err) + require.NotEqual(t, a, pi.Addrs[0].String()) + require.Len(t, pi.Addrs, 1) + + mas, err := peer.AddrInfoToP2pAddrs(pi) + require.NoError(t, err) + require.Len(t, mas, 1) + require.Equal(t, a, mas[0].String()) +} + +type p2phttpHandler struct { + tb testing.TB + retData string + net GossipNode +} + +func (h *p2phttpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(h.retData)) + if r.URL.Path == "/check-conn" { + c := h.net.GetHTTPRequestConnection(r) + require.NotNil(h.tb, c) + } +} + +func TestP2PHTTPHandler(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + cfg := config.GetDefaultLocal() + cfg.EnableDHTProviders = true + cfg.GossipFanout = 1 + log := logging.TestingLog(t) + + netA, err := NewP2PNetwork(log, cfg, "", nil, genesisID, config.Devtestnet, &nopeNodeInfo{}) + require.NoError(t, err) + + h := &p2phttpHandler{t, "hello", nil} + netA.RegisterHTTPHandler("/test", h) + + h2 := &p2phttpHandler{t, "world", netA} + netA.RegisterHTTPHandler("/check-conn", h2) + + netA.Start() + defer netA.Stop() + + peerInfoA := netA.service.AddrInfo() + addrsA, err := peer.AddrInfoToP2pAddrs(&peerInfoA) + require.NoError(t, err) + require.NotZero(t, addrsA[0]) + + httpClient, err := p2p.MakeHTTPClient(&peerInfoA) + require.NoError(t, err) + resp, err := httpClient.Get("/test") + require.NoError(t, err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, "hello", string(body)) + + // check another endpoint that also access the underlying connection/stream + httpClient, err = p2p.MakeHTTPClient(&peerInfoA) + require.NoError(t, err) + resp, err = httpClient.Get("/check-conn") + require.NoError(t, err) + defer resp.Body.Close() + + body, err = io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, "world", string(body)) + + // check rate limiting client: + // zero clients allowed, rate limiting window (10s) is greater than queue deadline (1s) + pstore, err := peerstore.MakePhonebook(0, 10*time.Second) + require.NoError(t, err) + pstore.AddPersistentPeers([]interface{}{&peerInfoA}, "net", phonebook.PhoneBookEntryRelayRole) + httpClient, err = p2p.MakeHTTPClientWithRateLimit(&peerInfoA, pstore, 1*time.Second, 1) + require.NoError(t, err) + _, err = httpClient.Get("/test") + require.ErrorIs(t, err, limitcaller.ErrConnectionQueueingTimeout) +} + +func TestP2PRelay(t *testing.T) { + partitiontest.PartitionTest(t) + + cfg := config.GetDefaultLocal() + cfg.ForceFetchTransactions = true + log := logging.TestingLog(t) + netA, err := NewP2PNetwork(log, cfg, "", nil, genesisID, config.Devtestnet, &nopeNodeInfo{}) + require.NoError(t, err) + + err = netA.Start() + require.NoError(t, err) + defer netA.Stop() + + peerInfoA := netA.service.AddrInfo() + addrsA, err := peer.AddrInfoToP2pAddrs(&peerInfoA) + require.NoError(t, err) + require.NotZero(t, addrsA[0]) + + multiAddrStr := addrsA[0].String() + phoneBookAddresses := []string{multiAddrStr} + + netB, err := NewP2PNetwork(log, cfg, "", phoneBookAddresses, genesisID, config.Devtestnet, &nopeNodeInfo{}) + require.NoError(t, err) + err = netB.Start() + require.NoError(t, err) + defer netB.Stop() + + require.Eventually( + t, + func() bool { + return len(netA.service.ListPeersForTopic(p2p.TXTopicName)) > 0 && + len(netB.service.ListPeersForTopic(p2p.TXTopicName)) > 0 + }, + 2*time.Second, + 50*time.Millisecond, + ) + + require.Eventually(t, func() bool { + return netA.hasPeers() && netB.hasPeers() + }, 2*time.Second, 50*time.Millisecond) + + makeCounterHandler := func(numExpected int) ([]TaggedMessageProcessor, *atomic.Uint32, chan struct{}) { + var numActual atomic.Uint32 + counterDone := make(chan struct{}) + counterHandler := []TaggedMessageProcessor{ + { + Tag: protocol.TxnTag, + MessageHandler: struct { + ProcessorValidateFunc + ProcessorHandleFunc + }{ + ProcessorValidateFunc(func(msg IncomingMessage) ValidatedMessage { + return ValidatedMessage{Action: Accept, Tag: msg.Tag, ValidatedMessage: nil} + }), + ProcessorHandleFunc(func(msg ValidatedMessage) OutgoingMessage { + if count := numActual.Add(1); int(count) >= numExpected { + close(counterDone) + } + return OutgoingMessage{Action: Ignore} + }), + }, + }, + } + return counterHandler, &numActual, counterDone + } + counterHandler, _, counterDone := makeCounterHandler(1) + netA.RegisterProcessors(counterHandler) + + // send 5 messages from both netB to netA + // since there is no node with listening address set => no messages should be received + for i := 0; i < 5; i++ { + err := netB.Relay(context.Background(), protocol.TxnTag, []byte{1, 2, 3, byte(i)}, true, nil) + require.NoError(t, err) + } + + select { + case <-counterDone: + require.Fail(t, "No messages should have been received") + case <-time.After(1 * time.Second): + } + + // add netC with listening address set, and enable relaying on netB + // ensure all messages are received by netA + cfg.NetAddress = "127.0.0.1:0" + netC, err := NewP2PNetwork(log, cfg, "", phoneBookAddresses, genesisID, config.Devtestnet, &nopeNodeInfo{}) + require.NoError(t, err) + err = netC.Start() + require.NoError(t, err) + defer netC.Stop() + + netB.relayMessages = true + + require.Eventually( + t, + func() bool { + return len(netA.service.ListPeersForTopic(p2p.TXTopicName)) > 0 && + len(netB.service.ListPeersForTopic(p2p.TXTopicName)) > 0 && + len(netC.service.ListPeersForTopic(p2p.TXTopicName)) > 0 + }, + 2*time.Second, + 50*time.Millisecond, + ) + + require.Eventually(t, func() bool { + return netA.hasPeers() && netB.hasPeers() && netC.hasPeers() + }, 2*time.Second, 50*time.Millisecond) + + const expectedMsgs = 10 + counterHandler, count, counterDone := makeCounterHandler(expectedMsgs) + netA.ClearProcessors() + netA.RegisterProcessors(counterHandler) + + for i := 0; i < expectedMsgs/2; i++ { + err := netB.Relay(context.Background(), protocol.TxnTag, []byte{1, 2, 3, byte(i)}, true, nil) + require.NoError(t, err) + err = netC.Relay(context.Background(), protocol.TxnTag, []byte{11, 12, 10 + byte(i), 14}, true, nil) + require.NoError(t, err) + } + // send some duplicate messages, they should be dropped + for i := 0; i < expectedMsgs/2; i++ { + err := netB.Relay(context.Background(), protocol.TxnTag, []byte{1, 2, 3, byte(i)}, true, nil) + require.NoError(t, err) + } + + select { + case <-counterDone: + case <-time.After(2 * time.Second): + if c := count.Load(); c < expectedMsgs { + require.Failf(t, "One or more messages failed to reach destination network", "%d > %d", expectedMsgs, c) + } else if c > expectedMsgs { + require.Failf(t, "One or more messages that were expected to be dropped, reached destination network", "%d < %d", expectedMsgs, c) + } + } +} + +type mockSubPService struct { + mockService + count atomic.Int64 + otherPeerID peer.ID + shouldNextFail bool +} + +type mockSubscription struct { + peerID peer.ID + shouldNextFail bool +} + +func (m *mockSubscription) Next(ctx context.Context) (*pubsub.Message, error) { + if m.shouldNextFail { + return nil, errors.New("mockSubscription error") + } + return &pubsub.Message{ReceivedFrom: m.peerID}, nil +} +func (m *mockSubscription) Cancel() {} + +func (m *mockSubPService) Subscribe(topic string, val pubsub.ValidatorEx) (p2p.SubNextCancellable, error) { + m.count.Add(1) + otherPeerID := m.otherPeerID + if otherPeerID == "" { + otherPeerID = "mockSubPServicePeerID" + } + return &mockSubscription{peerID: otherPeerID, shouldNextFail: m.shouldNextFail}, nil +} + +// TestP2PWantTXGossip checks txTopicHandleLoop runs as expected on wantTXGossip changes +func TestP2PWantTXGossip(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + // cancelled context to trigger subscription.Next to return + ctx, cancel := context.WithCancel(context.Background()) + cancel() + peerID := peer.ID("myPeerID") + mockService := &mockSubPService{mockService: mockService{id: peerID}, shouldNextFail: true} + net := &P2PNetwork{ + service: mockService, + log: logging.TestingLog(t), + ctx: ctx, + nodeInfo: &nopeNodeInfo{}, + } + + // ensure wantTXGossip from false to false is noop + net.wantTXGossip.Store(false) + net.OnNetworkAdvance() + require.Eventually(t, func() bool { net.wg.Wait(); return true }, 1*time.Second, 50*time.Millisecond) + require.Equal(t, int64(0), mockService.count.Load()) + require.False(t, net.wantTXGossip.Load()) + + // ensure wantTXGossip from true (wantTXGossip) to false (nopeNodeInfo) is noop + net.wantTXGossip.Store(true) + net.OnNetworkAdvance() + require.Eventually(t, func() bool { net.wg.Wait(); return true }, 1*time.Second, 50*time.Millisecond) + require.Equal(t, int64(0), mockService.count.Load()) + require.False(t, net.wantTXGossip.Load()) + + // check false to true change triggers subscription + net.wantTXGossip.Store(false) + net.nodeInfo = &participatingNodeInfo{} + net.OnNetworkAdvance() + require.Eventually(t, func() bool { return mockService.count.Load() == 1 }, 1*time.Second, 50*time.Millisecond) + require.True(t, net.wantTXGossip.Load()) + + // check IsParticipating changes wantTXGossip + net.wantTXGossip.Store(true) + net.nodeInfo = &nopeNodeInfo{} + net.config.ForceFetchTransactions = false + net.relayMessages = false + net.OnNetworkAdvance() + require.Eventually(t, func() bool { net.wg.Wait(); return true }, 1*time.Second, 50*time.Millisecond) + require.False(t, net.wantTXGossip.Load()) + + // check ForceFetchTransactions and relayMessages also take effect + net.wantTXGossip.Store(false) + net.nodeInfo = &nopeNodeInfo{} + net.config.ForceFetchTransactions = true + net.relayMessages = false + net.OnNetworkAdvance() + require.Eventually(t, func() bool { return mockService.count.Load() == 2 }, 1*time.Second, 50*time.Millisecond) + require.True(t, net.wantTXGossip.Load()) + + net.wantTXGossip.Store(false) + net.nodeInfo = &nopeNodeInfo{} + net.config.ForceFetchTransactions = false + net.relayMessages = true + net.OnNetworkAdvance() + require.Eventually(t, func() bool { return mockService.count.Load() == 3 }, 1*time.Second, 50*time.Millisecond) + require.True(t, net.wantTXGossip.Load()) + + // ensure empty nodeInfo prevents changing the value + net.wantTXGossip.Store(false) + net.nodeInfo = nil + net.config.ForceFetchTransactions = true + net.relayMessages = true + net.OnNetworkAdvance() + require.Eventually(t, func() bool { net.wg.Wait(); return true }, 1*time.Second, 50*time.Millisecond) + require.False(t, net.wantTXGossip.Load()) + + // check true to true change is noop + net.wantTXGossip.Store(true) + net.nodeInfo = &participatingNodeInfo{} + net.OnNetworkAdvance() + require.Eventually(t, func() bool { return mockService.count.Load() == 3 }, 1*time.Second, 50*time.Millisecond) + require.True(t, net.wantTXGossip.Load()) +} + +func TestMergeP2PAddrInfoResolvedAddresses(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + m1, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN") + require.NoError(t, err) + m2, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/4001/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb") + require.NoError(t, err) + m3, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/4001/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC") + require.NoError(t, err) + m4, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/4001") + require.NoError(t, err) + + var tests = []struct { + name string + primary []ma.Multiaddr + backup []ma.Multiaddr + expected int + hasInvalid bool + }{ + {"no overlap", []ma.Multiaddr{m1}, []ma.Multiaddr{m2}, 2, false}, + {"complete overlap", []ma.Multiaddr{m1}, []ma.Multiaddr{m1}, 1, false}, + {"partial overlap", []ma.Multiaddr{m1, m2}, []ma.Multiaddr{m1, m3}, 3, false}, + {"empty slices", []ma.Multiaddr{}, []ma.Multiaddr{}, 0, false}, + {"nil slices", nil, nil, 0, false}, + {"invalid p2p", []ma.Multiaddr{m1, m4}, []ma.Multiaddr{m2, m4}, 2, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r1 := mergeP2PMultiaddrResolvedAddresses(tt.primary, tt.backup) + if len(r1) != tt.expected { + t.Errorf("Expected %d addresses, got %d", tt.expected, len(r1)) + } + + var info1 []peer.AddrInfo + var info2 []peer.AddrInfo + for _, addr := range tt.primary { + info, err0 := peer.AddrInfoFromP2pAddr(addr) + if tt.hasInvalid { + if err0 == nil { + info1 = append(info1, *info) + } + } else { + require.NoError(t, err0) + info1 = append(info1, *info) + } + } + for _, addr := range tt.backup { + info, err0 := peer.AddrInfoFromP2pAddr(addr) + if tt.hasInvalid { + if err0 == nil { + info2 = append(info2, *info) + } + } else { + require.NoError(t, err0) + info2 = append(info2, *info) + } + } + if info1 == nil && tt.primary != nil { + info1 = []peer.AddrInfo{} + } + if info2 == nil && tt.backup != nil { + info1 = []peer.AddrInfo{} + } + + r2 := mergeP2PAddrInfoResolvedAddresses(info1, info2) + if len(r2) != tt.expected { + t.Errorf("Expected %d addresses, got %d", tt.expected, len(r2)) + } + }) + } +} + +// TestP2PGossipSubPeerCasts checks that gossipSubPeer implements the ErlClient and IPAddressable interfaces +// needed by TxHandler +func TestP2PGossipSubPeerCasts(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + var g interface{} = gossipSubPeer{} + _, ok := g.(util.ErlClient) + require.True(t, ok) + + _, ok = g.(IPAddressable) + require.True(t, ok) + + // check that gossipSubPeer is hashable as ERL wants + var m map[util.ErlClient]struct{} + require.Equal(t, m[gossipSubPeer{}], struct{}{}) + require.Equal(t, m[g.(util.ErlClient)], struct{}{}) +} diff --git a/network/phonebook.go b/network/phonebook/phonebook.go similarity index 96% rename from network/phonebook.go rename to network/phonebook/phonebook.go index 0c431fd2f3..634ca9c16c 100644 --- a/network/phonebook.go +++ b/network/phonebook/phonebook.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with go-algorand. If not, see . -package network +package phonebook import ( "math" @@ -55,12 +55,12 @@ type Phonebook interface { // The connection should be established when the waitTime is 0. // It will register a provisional next connection time when the waitTime is 0. // The provisional time should be updated after the connection with UpdateConnectionTime - GetConnectionWaitTime(addr string) (addrInPhonebook bool, + GetConnectionWaitTime(addrOrPeerID string) (addrInPhonebook bool, waitTime time.Duration, provisionalTime time.Time) // UpdateConnectionTime will update the provisional connection time. // Returns true of the addr was in the phonebook - UpdateConnectionTime(addr string, provisionalTime time.Time) bool + UpdateConnectionTime(addrOrPeerID string, provisionalTime time.Time) bool // ReplacePeerList merges a set of addresses with that passed in for networkName // new entries in dnsAddresses are being added @@ -231,8 +231,10 @@ func (e *phonebookImpl) UpdateRetryAfter(addr string, retryAfter time.Time) { // The connection should be established when the waitTime is 0. // It will register a provisional next connection time when the waitTime is 0. // The provisional time should be updated after the connection with UpdateConnectionTime -func (e *phonebookImpl) GetConnectionWaitTime(addr string) (addrInPhonebook bool, +func (e *phonebookImpl) GetConnectionWaitTime(addrOrPeerID string) (addrInPhonebook bool, waitTime time.Duration, provisionalTime time.Time) { + + addr := addrOrPeerID e.lock.Lock() defer e.lock.Unlock() @@ -276,7 +278,8 @@ func (e *phonebookImpl) GetConnectionWaitTime(addr string) (addrInPhonebook bool // UpdateConnectionTime will update the provisional connection time. // Returns true of the addr was in the phonebook -func (e *phonebookImpl) UpdateConnectionTime(addr string, provisionalTime time.Time) bool { +func (e *phonebookImpl) UpdateConnectionTime(addrOrPeerID string, provisionalTime time.Time) bool { + addr := addrOrPeerID e.lock.Lock() defer e.lock.Unlock() diff --git a/network/phonebook_test.go b/network/phonebook/phonebook_test.go similarity index 99% rename from network/phonebook_test.go rename to network/phonebook/phonebook_test.go index 2643e722ea..d603a51a1a 100644 --- a/network/phonebook_test.go +++ b/network/phonebook/phonebook_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with go-algorand. If not, see . -package network +package phonebook import ( "testing" diff --git a/network/requestLogger_test.go b/network/requestLogger_test.go index cb1d7b963d..0de6a41c73 100644 --- a/network/requestLogger_test.go +++ b/network/requestLogger_test.go @@ -25,6 +25,7 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/telemetryspec" + "github.com/algorand/go-algorand/network/phonebook" "github.com/algorand/go-algorand/test/partitiontest" ) @@ -49,11 +50,12 @@ func TestRequestLogger(t *testing.T) { dl := eventsDetailsLogger{Logger: log, eventReceived: make(chan interface{}, 1), eventIdentifier: telemetryspec.HTTPRequestEvent} log.SetLevel(logging.Level(defaultConfig.BaseLoggerDebugLevel)) netA := &WebsocketNetwork{ - log: dl, - config: defaultConfig, - phonebook: MakePhonebook(1, 1*time.Millisecond), - GenesisID: "go-test-network-genesis", - NetworkID: config.Devtestnet, + log: dl, + config: defaultConfig, + phonebook: phonebook.MakePhonebook(1, 1*time.Millisecond), + GenesisID: "go-test-network-genesis", + NetworkID: config.Devtestnet, + peerStater: peerConnectionStater{log: log}, } netA.config.EnableRequestLogger = true netA.setup() @@ -67,8 +69,8 @@ func TestRequestLogger(t *testing.T) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook = MakePhonebook(1, 1*time.Millisecond) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook = phonebook.MakePhonebook(1, 1*time.Millisecond) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer func() { t.Log("stopping B"); netB.Stop(); t.Log("B done") }() diff --git a/network/requestTracker.go b/network/requestTracker.go index c38cc9d2ed..47eba90c7e 100644 --- a/network/requestTracker.go +++ b/network/requestTracker.go @@ -31,6 +31,7 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/telemetryspec" + "github.com/algorand/go-algorand/network/addr" ) const ( @@ -94,12 +95,12 @@ func makeTrackerRequest(remoteAddr, remoteHost, remotePort string, createTime ti // - remoteAddr is used otherwise. func (tr *TrackerRequest) remoteAddress() string { if len(tr.otherPublicAddr) != 0 { - url, err := ParseHostOrURL(tr.otherPublicAddr) + url, err := addr.ParseHostOrURL(tr.otherPublicAddr) if err == nil && len(tr.remoteHost) > 0 && url.Hostname() == tr.remoteHost { return tr.otherPublicAddr } } - url, err := ParseHostOrURL(tr.remoteAddr) + url, err := addr.ParseHostOrURL(tr.remoteAddr) if err != nil { // tr.remoteAddr can't be parsed so try to use tr.remoteHost // there is a chance it came from a proxy and has a meaningful value diff --git a/network/requestTracker_test.go b/network/requestTracker_test.go index 0a8c934c53..158cf45336 100644 --- a/network/requestTracker_test.go +++ b/network/requestTracker_test.go @@ -27,6 +27,7 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/network/phonebook" "github.com/algorand/go-algorand/test/partitiontest" ) @@ -86,11 +87,12 @@ func TestRateLimiting(t *testing.T) { // This test is conducted locally, so we want to treat all hosts the same for counting incoming requests. testConfig.DisableLocalhostConnectionRateLimit = false wn := &WebsocketNetwork{ - log: log, - config: testConfig, - phonebook: MakePhonebook(1, 1), - GenesisID: "go-test-network-genesis", - NetworkID: config.Devtestnet, + log: log, + config: testConfig, + phonebook: phonebook.MakePhonebook(1, 1), + GenesisID: "go-test-network-genesis", + NetworkID: config.Devtestnet, + peerStater: peerConnectionStater{log: log}, } // increase the IncomingConnectionsLimit/MaxConnectionsPerIP limits, since we don't want to test these. @@ -115,15 +117,15 @@ func TestRateLimiting(t *testing.T) { clientsCount := int(testConfig.ConnectionsRateLimitingCount + 5) networks := make([]*WebsocketNetwork, clientsCount) - phonebooks := make([]Phonebook, clientsCount) + phonebooks := make([]phonebook.Phonebook, clientsCount) for i := 0; i < clientsCount; i++ { networks[i] = makeTestWebsocketNodeWithConfig(t, noAddressConfig) networks[i].config.GossipFanout = 1 - phonebooks[i] = MakePhonebook(networks[i].config.ConnectionsRateLimitingCount, + phonebooks[i] = phonebook.MakePhonebook(networks[i].config.ConnectionsRateLimitingCount, time.Duration(networks[i].config.ConnectionsRateLimitingWindowSeconds)*time.Second) - phonebooks[i].ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) - networks[i].phonebook = MakePhonebook(1, 1*time.Millisecond) - networks[i].phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + phonebooks[i].ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) + networks[i].phonebook = phonebook.MakePhonebook(1, 1*time.Millisecond) + networks[i].phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) defer func(net *WebsocketNetwork, i int) { t.Logf("stopping network %d", i) net.Stop() @@ -153,7 +155,7 @@ func TestRateLimiting(t *testing.T) { case <-readyCh: // it's closed, so this client got connected. connectedClients++ - phonebookLen := len(phonebooks[i].GetAddresses(1, PhoneBookEntryRelayRole)) + phonebookLen := len(phonebooks[i].GetAddresses(1, phonebook.PhoneBookEntryRelayRole)) // if this channel is ready, than we should have an address, since it didn't get blocked. require.Equal(t, 1, phonebookLen) default: diff --git a/network/websocketProxy_test.go b/network/websocketProxy_test.go index 73298ccd64..96628acb69 100644 --- a/network/websocketProxy_test.go +++ b/network/websocketProxy_test.go @@ -28,6 +28,7 @@ import ( "testing" "time" + "github.com/algorand/go-algorand/network/addr" "github.com/algorand/go-algorand/test/partitiontest" "github.com/algorand/websocket" "github.com/stretchr/testify/require" @@ -71,7 +72,7 @@ func (w *websocketProxy) ServeHTTP(response http.ResponseWriter, request *http.R } // set X-Forwarded-For - url, err := ParseHostOrURL(request.RemoteAddr) + url, err := addr.ParseHostOrURL(request.RemoteAddr) if err != nil { http.Error(response, err.Error(), http.StatusInternalServerError) return @@ -254,7 +255,7 @@ func TestWebsocketProxyWsNet(t *testing.T) { gossipA, err := netA.addrToGossipAddr(addrA) require.NoError(t, err) - parsedA, err := ParseHostOrURL(gossipA) + parsedA, err := addr.ParseHostOrURL(gossipA) require.NoError(t, err) // setup the proxy @@ -316,7 +317,7 @@ func TestWebsocketProxyWsNet(t *testing.T) { peerB := netA.peers[0] require.NotEmpty(t, peerB.originAddress) require.Equal(t, fakeXForwardedFor, peerB.originAddress) - require.NotEqual(t, peerB.RoutingAddr(), peerB.IPAddr()) + require.NotEqual(t, peerB.RoutingAddr(), peerB.ipAddr()) fakeXForwardedForParsed := net.ParseIP(fakeXForwardedFor) require.NotEqual(t, fakeXForwardedForParsed, peerB.RoutingAddr()) } diff --git a/network/wsNetwork.go b/network/wsNetwork.go index 4a491f4f9f..8a43ad5234 100644 --- a/network/wsNetwork.go +++ b/network/wsNetwork.go @@ -27,6 +27,7 @@ import ( "net/http" "net/textproto" "net/url" + "path" "regexp" "runtime" "strconv" @@ -43,7 +44,11 @@ import ( "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/telemetryspec" + "github.com/algorand/go-algorand/network/addr" + "github.com/algorand/go-algorand/network/limitcaller" "github.com/algorand/go-algorand/network/limitlistener" + "github.com/algorand/go-algorand/network/p2p" + "github.com/algorand/go-algorand/network/phonebook" "github.com/algorand/go-algorand/protocol" tools_network "github.com/algorand/go-algorand/tools/network" "github.com/algorand/go-algorand/tools/network/dnssec" @@ -83,7 +88,7 @@ const httpServerMaxHeaderBytes = 4096 const connectionActivityMonitorInterval = 3 * time.Minute // maxPeerInactivityDuration is the maximum allowed duration for a -// peer to remain completly idle (i.e. no inbound or outbound communication), before +// peer to remain completely idle (i.e. no inbound or outbound communication), before // we discard the connection. const maxPeerInactivityDuration = 5 * time.Minute @@ -163,6 +168,8 @@ const HealthServiceStatusPath = "/status" type NodeInfo interface { // IsParticipating returns true if this node has stake and may vote on blocks or propose blocks. IsParticipating() bool + // Capabilities returns a list of capabilities this node has. + Capabilities() []p2p.Capability } type nopeNodeInfo struct { @@ -172,6 +179,10 @@ func (nnni *nopeNodeInfo) IsParticipating() bool { return false } +func (nnni *nopeNodeInfo) Capabilities() []p2p.Capability { + return nil +} + // WebsocketNetwork implements GossipNode type WebsocketNetwork struct { listener net.Listener @@ -197,12 +208,15 @@ type WebsocketNetwork struct { broadcaster msgBroadcaster handler msgHandler - phonebook Phonebook + phonebook phonebook.Phonebook GenesisID string NetworkID protocol.NetworkID RandomID string + peerID p2p.PeerID + peerIDSigner identityChallengeSigner + ready atomic.Int32 readyChan chan struct{} @@ -238,8 +252,8 @@ type WebsocketNetwork struct { requestsTracker *RequestTracker requestsLogger *RequestLogger - // lastPeerConnectionsSent is the last time the peer connections were sent ( or attempted to be sent ) to the telemetry server. - lastPeerConnectionsSent time.Time + // peerStater collects and report peers connectivity telemetry + peerStater peerConnectionStater // connPerfMonitor is used on outgoing connections to measure their relative message timing connPerfMonitor *connectionPerformanceMonitor @@ -256,8 +270,8 @@ type WebsocketNetwork struct { // transport and dialer are customized to limit the number of // connection in compliance with connectionsRateLimitingCount. - transport rateLimitingTransport - dialer Dialer + transport limitcaller.RateLimitingTransport + dialer limitcaller.Dialer // messagesOfInterest specifies the message types that this node // wants to receive. nil means default. non-nil causes this @@ -348,7 +362,7 @@ type networkPeerManager interface { // used by msgHandler Broadcast(ctx context.Context, tag protocol.Tag, data []byte, wait bool, except Peer) error - disconnectThread(badnode Peer, reason disconnectReason) + disconnectThread(badnode DisconnectablePeer, reason disconnectReason) checkPeersConnectivity() } @@ -463,13 +477,13 @@ func (wn *WebsocketNetwork) RelayArray(ctx context.Context, tags []protocol.Tag, return nil } -func (wn *WebsocketNetwork) disconnectThread(badnode Peer, reason disconnectReason) { +func (wn *WebsocketNetwork) disconnectThread(badnode DisconnectablePeer, reason disconnectReason) { defer wn.wg.Done() wn.disconnect(badnode, reason) } // Disconnect from a peer, probably due to protocol errors. -func (wn *WebsocketNetwork) Disconnect(node Peer) { +func (wn *WebsocketNetwork) Disconnect(node DisconnectablePeer) { wn.disconnect(node, disconnectBadData) } @@ -549,16 +563,18 @@ func (wn *WebsocketNetwork) GetPeers(options ...PeerOption) []Peer { case PeersPhonebookRelays: // return copy of phonebook, which probably also contains peers we're connected to, but if it doesn't maybe we shouldn't be making new connections to those peers (because they disappeared from the directory) var addrs []string - addrs = wn.phonebook.GetAddresses(1000, PhoneBookEntryRelayRole) + addrs = wn.phonebook.GetAddresses(1000, phonebook.PhoneBookEntryRelayRole) for _, addr := range addrs { - peerCore := makePeerCore(wn.ctx, wn, wn.log, wn.handler.readBuffer, addr, wn.GetRoundTripper(), "" /*origin address*/) + client, _ := wn.GetHTTPClient(addr) + peerCore := makePeerCore(wn.ctx, wn, wn.log, wn.handler.readBuffer, addr, client, "" /*origin address*/) outPeers = append(outPeers, &peerCore) } case PeersPhonebookArchivalNodes: var addrs []string - addrs = wn.phonebook.GetAddresses(1000, PhoneBookEntryArchivalRole) + addrs = wn.phonebook.GetAddresses(1000, phonebook.PhoneBookEntryArchivalRole) for _, addr := range addrs { - peerCore := makePeerCore(wn.ctx, wn, wn.log, wn.handler.readBuffer, addr, wn.GetRoundTripper(), "" /*origin address*/) + client, _ := wn.GetHTTPClient(addr) + peerCore := makePeerCore(wn.ctx, wn, wn.log, wn.handler.readBuffer, addr, client, "" /*origin address*/) outPeers = append(outPeers, &peerCore) } case PeersConnectedIn: @@ -583,13 +599,12 @@ func (wn *WebsocketNetwork) setup() { wn.nodeInfo = &nopeNodeInfo{} } maxIdleConnsPerHost := int(wn.config.ConnectionsRateLimitingCount) - wn.dialer = makeRateLimitingDialer(wn.phonebook, preferredResolver) - wn.transport = makeRateLimitingTransport(wn.phonebook, 10*time.Second, &wn.dialer, maxIdleConnsPerHost) + wn.dialer = limitcaller.MakeRateLimitingDialer(wn.phonebook, preferredResolver) + wn.transport = limitcaller.MakeRateLimitingTransport(wn.phonebook, limitcaller.DefaultQueueingTimeout, &wn.dialer, maxIdleConnsPerHost) wn.upgrader.ReadBufferSize = 4096 wn.upgrader.WriteBufferSize = 4096 wn.upgrader.EnableCompression = false - wn.lastPeerConnectionsSent = time.Now() wn.router = mux.NewRouter() if wn.config.EnableGossipService { wn.router.Handle(GossipNetworkPath, wn) @@ -676,7 +691,7 @@ func (wn *WebsocketNetwork) setup() { } // Start makes network connections and threads -func (wn *WebsocketNetwork) Start() { +func (wn *WebsocketNetwork) Start() error { wn.messagesOfInterestMu.Lock() defer wn.messagesOfInterestMu.Unlock() wn.messagesOfInterestEncoded = true @@ -688,7 +703,7 @@ func (wn *WebsocketNetwork) Start() { listener, err := net.Listen("tcp", wn.config.NetAddress) if err != nil { wn.log.Errorf("network could not listen %v: %s", wn.config.NetAddress, err) - return + return err } // wrap the original listener with a limited connection listener listener = limitlistener.RejectingLimitListener( @@ -720,12 +735,17 @@ func (wn *WebsocketNetwork) Start() { } } } - // if the network has a public address, use that as the name for connection deduplication - if wn.config.PublicAddress != "" { + // if the network has a public address or a libp2p peer ID, use that as the name for connection deduplication + if wn.config.PublicAddress != "" || (wn.peerID != "" && wn.peerIDSigner != nil) { wn.RegisterHandlers(identityHandlers) } - if wn.identityScheme == nil && wn.config.PublicAddress != "" { - wn.identityScheme = NewIdentityChallengeScheme(wn.config.PublicAddress) + if wn.identityScheme == nil { + if wn.peerID != "" && wn.peerIDSigner != nil { + wn.identityScheme = NewIdentityChallengeSchemeWithSigner(string(wn.peerID), wn.peerIDSigner) + } + if wn.config.PublicAddress != "" { + wn.identityScheme = NewIdentityChallengeScheme(wn.config.PublicAddress) + } } wn.meshUpdateRequests <- meshRequest{false, nil} @@ -759,6 +779,8 @@ func (wn *WebsocketNetwork) Start() { go wn.postMessagesOfInterestThread() wn.log.Infof("serving genesisID=%s on %#v with RandomID=%s", wn.GenesisID, wn.PublicAddress(), wn.RandomID) + + return nil } func (wn *WebsocketNetwork) httpdThread() { @@ -842,6 +864,14 @@ func (wn *WebsocketNetwork) ClearHandlers() { wn.handler.ClearHandlers([]Tag{protocol.PingTag, protocol.PingReplyTag, protocol.NetPrioResponseTag}) } +// RegisterProcessors registers the set of given message handlers. +func (wn *WebsocketNetwork) RegisterProcessors(dispatch []TaggedMessageProcessor) { +} + +// ClearProcessors deregisters all the existing message handlers. +func (wn *WebsocketNetwork) ClearProcessors() { +} + func (wn *WebsocketNetwork) setHeaders(header http.Header) { localTelemetryGUID := wn.log.GetTelemetryGUID() localInstanceName := wn.log.GetInstanceName() @@ -1005,7 +1035,7 @@ func (wn *WebsocketNetwork) checkIncomingConnectionVariables(response http.Respo // request that was provided to the http handler ( or provide a fallback Context() to that ) // if the provided request has no associated connection, it returns nil. ( this should not happen for any http request that was registered // by WebsocketNetwork ) -func (wn *WebsocketNetwork) GetHTTPRequestConnection(request *http.Request) (conn net.Conn) { +func (wn *WebsocketNetwork) GetHTTPRequestConnection(request *http.Request) (conn DeadlineSettableConn) { if wn.requestsTracker != nil { conn = wn.requestsTracker.GetRequestConnection(request) } @@ -1080,8 +1110,9 @@ func (wn *WebsocketNetwork) ServeHTTP(response http.ResponseWriter, request *htt wn.requestsLogger.SetStatusCode(response, http.StatusSwitchingProtocols) } + client, _ := wn.GetHTTPClient(trackedRequest.remoteAddress()) peer := &wsPeer{ - wsPeerCore: makePeerCore(wn.ctx, wn, wn.log, wn.handler.readBuffer, trackedRequest.remoteAddress(), wn.GetRoundTripper(), trackedRequest.remoteHost), + wsPeerCore: makePeerCore(wn.ctx, wn, wn.log, wn.handler.readBuffer, trackedRequest.remoteAddress(), client, trackedRequest.remoteHost), conn: wsPeerWebsocketConnImpl{conn}, outgoing: false, InstanceName: trackedRequest.otherInstanceName, @@ -1528,7 +1559,7 @@ func (wn *WebsocketNetwork) isConnectedTo(addr string) bool { wn.peersLock.RLock() defer wn.peersLock.RUnlock() for _, peer := range wn.peers { - if addr == peer.rootURL { + if addr == peer.GetAddress() { return true } } @@ -1605,7 +1636,7 @@ func (wn *WebsocketNetwork) meshThread() { // send the currently connected peers information to the // telemetry server; that would allow the telemetry server // to construct a cross-node map of all the nodes interconnections. - wn.sendPeerConnectionsTelemetryStatus() + wn.peerStater.sendPeerConnectionsTelemetryStatus(wn) } } @@ -1632,12 +1663,12 @@ func (wn *WebsocketNetwork) refreshRelayArchivePhonebookAddresses() { func (wn *WebsocketNetwork) updatePhonebookAddresses(relayAddrs []string, archiveAddrs []string) { if len(relayAddrs) > 0 { wn.log.Debugf("got %d relay dns addrs, %#v", len(relayAddrs), relayAddrs[:imin(5, len(relayAddrs))]) - wn.phonebook.ReplacePeerList(relayAddrs, string(wn.NetworkID), PhoneBookEntryRelayRole) + wn.phonebook.ReplacePeerList(relayAddrs, string(wn.NetworkID), phonebook.PhoneBookEntryRelayRole) } else { wn.log.Infof("got no relay DNS addrs for network %s", wn.NetworkID) } if len(archiveAddrs) > 0 { - wn.phonebook.ReplacePeerList(archiveAddrs, string(wn.NetworkID), PhoneBookEntryArchivalRole) + wn.phonebook.ReplacePeerList(archiveAddrs, string(wn.NetworkID), phonebook.PhoneBookEntryArchivalRole) } else { wn.log.Infof("got no archive DNS addrs for network %s", wn.NetworkID) } @@ -1656,7 +1687,7 @@ func (wn *WebsocketNetwork) checkNewConnectionsNeeded() bool { return false } // get more than we need so that we can ignore duplicates - newAddrs := wn.phonebook.GetAddresses(desired+numOutgoingTotal, PhoneBookEntryRelayRole) + newAddrs := wn.phonebook.GetAddresses(desired+numOutgoingTotal, phonebook.PhoneBookEntryRelayRole) for _, na := range newAddrs { if na == wn.config.PublicAddress { // filter out self-public address, so we won't try to connect to ourselves. @@ -1767,27 +1798,38 @@ func (wn *WebsocketNetwork) OnNetworkAdvance() { } } +type peerConnectionStater struct { + log logging.Logger + + peerConnectionsUpdateInterval time.Duration + lastPeerConnectionsSent time.Time +} + +type peerSnapshotter interface { + peerSnapshot(peers []*wsPeer) ([]*wsPeer, int32) +} + // sendPeerConnectionsTelemetryStatus sends a snapshot of the currently connected peers // to the telemetry server. Internally, it's using a timer to ensure that it would only // send the information once every hour ( configurable via PeerConnectionsUpdateInterval ) -func (wn *WebsocketNetwork) sendPeerConnectionsTelemetryStatus() { - if !wn.log.GetTelemetryEnabled() { +func (pcs *peerConnectionStater) sendPeerConnectionsTelemetryStatus(snapshotter peerSnapshotter) { + if !pcs.log.GetTelemetryEnabled() { return } now := time.Now() - if wn.lastPeerConnectionsSent.Add(time.Duration(wn.config.PeerConnectionsUpdateInterval)*time.Second).After(now) || wn.config.PeerConnectionsUpdateInterval <= 0 { + if pcs.lastPeerConnectionsSent.Add(pcs.peerConnectionsUpdateInterval).After(now) || pcs.peerConnectionsUpdateInterval <= 0 { // it's not yet time to send the update. return } - wn.lastPeerConnectionsSent = now + pcs.lastPeerConnectionsSent = now var peers []*wsPeer - peers, _ = wn.peerSnapshot(peers) - connectionDetails := wn.getPeerConnectionTelemetryDetails(now, peers) - wn.log.EventWithDetails(telemetryspec.Network, telemetryspec.PeerConnectionsEvent, connectionDetails) + peers, _ = snapshotter.peerSnapshot(peers) + connectionDetails := getPeerConnectionTelemetryDetails(now, peers) + pcs.log.EventWithDetails(telemetryspec.Network, telemetryspec.PeerConnectionsEvent, connectionDetails) } -func (wn *WebsocketNetwork) getPeerConnectionTelemetryDetails(now time.Time, peers []*wsPeer) telemetryspec.PeersConnectionDetails { +func getPeerConnectionTelemetryDetails(now time.Time, peers []*wsPeer) telemetryspec.PeersConnectionDetails { var connectionDetails telemetryspec.PeersConnectionDetails for _, peer := range peers { connDetail := telemetryspec.PeerConnectionDetails{ @@ -2021,10 +2063,31 @@ func (wn *WebsocketNetwork) numOutgoingPending() int { return len(wn.tryConnectAddrs) } -// GetRoundTripper returns an http.Transport that limits the number of connection -// to comply with connectionsRateLimitingCount. -func (wn *WebsocketNetwork) GetRoundTripper() http.RoundTripper { - return &wn.transport +// GetHTTPClient returns a http.Client with a suitable for the network Transport +// that would also limit the number of outgoing connections. +func (wn *WebsocketNetwork) GetHTTPClient(address string) (*http.Client, error) { + return &http.Client{ + Transport: &HTTPPAddressBoundTransport{address, &wn.transport}, + }, nil +} + +// HTTPPAddressBoundTransport is a http.RoundTripper that sets the scheme and host of the request URL to the given address +type HTTPPAddressBoundTransport struct { + Addr string + InnerTransport http.RoundTripper +} + +// RoundTrip implements http.RoundTripper by adding the schema, host, port, path prefix from the +// parsed address to the request URL and then calling the inner transport. +func (t *HTTPPAddressBoundTransport) RoundTrip(req *http.Request) (*http.Response, error) { + url, err := addr.ParseHostOrURL(t.Addr) + if err != nil { + return nil, err + } + req.URL.Scheme = url.Scheme + req.URL.Host = url.Host + req.URL.Path = path.Join(url.Path, req.URL.Path) + return t.InnerTransport.RoundTrip(req) } // filterASCII filter out the non-ascii printable characters out of the given input string and @@ -2045,9 +2108,9 @@ func filterASCII(unfilteredString string) (filteredString string) { } // tryConnect opens websocket connection and checks initial connection parameters. -// addr should be 'host:port' or a URL, gossipAddr is the websocket endpoint URL -func (wn *WebsocketNetwork) tryConnect(addr, gossipAddr string) { - defer wn.tryConnectReleaseAddr(addr, gossipAddr) +// netAddr should be 'host:port' or a URL, gossipAddr is the websocket endpoint URL +func (wn *WebsocketNetwork) tryConnect(netAddr, gossipAddr string) { + defer wn.tryConnectReleaseAddr(netAddr, gossipAddr) defer func() { if xpanic := recover(); xpanic != nil { wn.log.Errorf("panic in tryConnect: %v", xpanic) @@ -2063,7 +2126,7 @@ func (wn *WebsocketNetwork) tryConnect(addr, gossipAddr string) { var idChallenge identityChallengeValue if wn.identityScheme != nil { - idChallenge = wn.identityScheme.AttachChallenge(requestHeader, addr) + idChallenge = wn.identityScheme.AttachChallenge(requestHeader, netAddr) } // for backward compatibility, include the ProtocolVersion header as well. @@ -2108,7 +2171,7 @@ func (wn *WebsocketNetwork) tryConnect(addr, gossipAddr string) { // we've got a retry-after header. // convert it to a timestamp so that we could use it. retryAfterTime := time.Now().Add(time.Duration(retryAfter) * time.Second) - wn.phonebook.UpdateRetryAfter(addr, retryAfterTime) + wn.phonebook.UpdateRetryAfter(netAddr, retryAfterTime) } default: wn.log.Warnf("ws connect(%s) fail - bad handshake, Status code = %d, Headers = %#v, Body = %s", gossipAddr, response.StatusCode, response.Header, errString) @@ -2149,7 +2212,7 @@ func (wn *WebsocketNetwork) tryConnect(addr, gossipAddr string) { peerID, idVerificationMessage, err = wn.identityScheme.VerifyResponse(response.Header, idChallenge) if err != nil { networkPeerIdentityError.Inc(nil) - wn.log.With("err", err).With("remote", addr).With("local", localAddr).Warn("peer supplied an invalid identity response, abandoning peering") + wn.log.With("err", err).With("remote", netAddr).With("local", localAddr).Warn("peer supplied an invalid identity response, abandoning peering") closeEarly("Invalid identity response") return } @@ -2162,8 +2225,9 @@ func (wn *WebsocketNetwork) tryConnect(addr, gossipAddr string) { wn.throttledOutgoingConnections.Add(int32(1)) } + client, _ := wn.GetHTTPClient(netAddr) peer := &wsPeer{ - wsPeerCore: makePeerCore(wn.ctx, wn, wn.log, wn.handler.readBuffer, addr, wn.GetRoundTripper(), "" /* origin */), + wsPeerCore: makePeerCore(wn.ctx, wn, wn.log, wn.handler.readBuffer, netAddr, client, "" /* origin */), conn: wsPeerWebsocketConnImpl{conn}, outgoing: true, incomingMsgFilter: wn.incomingMsgFilter, @@ -2185,7 +2249,7 @@ func (wn *WebsocketNetwork) tryConnect(addr, gossipAddr string) { wn.peersLock.Unlock() if !ok { networkPeerIdentityDisconnect.Inc(nil) - wn.log.With("remote", addr).With("local", localAddr).Warn("peer deduplicated before adding because the identity is already known") + wn.log.With("remote", netAddr).With("local", localAddr).Warn("peer deduplicated before adding because the identity is already known") closeEarly("Duplicate connection") return } @@ -2193,7 +2257,7 @@ func (wn *WebsocketNetwork) tryConnect(addr, gossipAddr string) { peer.init(wn.config, wn.outgoingMessagesBufferSize) wn.addPeer(peer) - wn.log.With("event", "ConnectedOut").With("remote", addr).With("local", localAddr).Infof("Made outgoing connection to peer %v", addr) + wn.log.With("event", "ConnectedOut").With("remote", netAddr).With("local", localAddr).Infof("Made outgoing connection to peer %v", netAddr) wn.log.EventWithDetails(telemetryspec.Network, telemetryspec.ConnectPeerEvent, telemetryspec.PeerEventDetails{ Address: justHost(conn.RemoteAddr().String()), @@ -2209,7 +2273,7 @@ func (wn *WebsocketNetwork) tryConnect(addr, gossipAddr string) { if len(idVerificationMessage) > 0 { sent := peer.writeNonBlock(context.Background(), idVerificationMessage, true, crypto.Digest{}, time.Now()) if !sent { - wn.log.With("remote", addr).With("local", localAddr).Warn("could not send identity challenge verification") + wn.log.With("remote", netAddr).With("local", localAddr).Warn("could not send identity challenge verification") } } @@ -2224,7 +2288,7 @@ func (wn *WebsocketNetwork) tryConnect(addr, gossipAddr string) { mbytes := append([]byte(protocol.NetPrioResponseTag), resp...) sent := peer.writeNonBlock(context.Background(), mbytes, true, crypto.Digest{}, time.Now()) if !sent { - wn.log.With("remote", addr).With("local", localAddr).Warnf("could not send priority response to %v", addr) + wn.log.With("remote", netAddr).With("local", localAddr).Warnf("could not send priority response to %v", netAddr) } } } @@ -2252,18 +2316,33 @@ func (wn *WebsocketNetwork) SetPeerData(peer Peer, key string, value interface{} } // NewWebsocketNetwork constructor for websockets based gossip network -func NewWebsocketNetwork(log logging.Logger, config config.Local, phonebookAddresses []string, genesisID string, networkID protocol.NetworkID, nodeInfo NodeInfo) (wn *WebsocketNetwork, err error) { - phonebook := MakePhonebook(config.ConnectionsRateLimitingCount, +func NewWebsocketNetwork(log logging.Logger, config config.Local, phonebookAddresses []string, genesisID string, networkID protocol.NetworkID, nodeInfo NodeInfo, peerID p2p.PeerID, idSigner identityChallengeSigner) (wn *WebsocketNetwork, err error) { + pb := phonebook.MakePhonebook(config.ConnectionsRateLimitingCount, time.Duration(config.ConnectionsRateLimitingWindowSeconds)*time.Second) - phonebook.AddPersistentPeers(phonebookAddresses, string(networkID), PhoneBookEntryRelayRole) + + addresses := make([]string, 0, len(phonebookAddresses)) + for _, a := range phonebookAddresses { + _, err := addr.ParseHostOrURL(a) + if err == nil { + addresses = append(addresses, a) + } + } + pb.AddPersistentPeers(addresses, string(networkID), phonebook.PhoneBookEntryRelayRole) wn = &WebsocketNetwork{ log: log, config: config, - phonebook: phonebook, + phonebook: pb, GenesisID: genesisID, NetworkID: networkID, nodeInfo: nodeInfo, + peerID: peerID, + peerIDSigner: idSigner, resolveSRVRecords: tools_network.ReadFromSRV, + peerStater: peerConnectionStater{ + log: log, + peerConnectionsUpdateInterval: time.Duration(config.PeerConnectionsUpdateInterval) * time.Second, + lastPeerConnectionsSent: time.Now(), + }, } wn.setup() @@ -2272,7 +2351,7 @@ func NewWebsocketNetwork(log logging.Logger, config config.Local, phonebookAddre // NewWebsocketGossipNode constructs a websocket network node and returns it as a GossipNode interface implementation func NewWebsocketGossipNode(log logging.Logger, config config.Local, phonebookAddresses []string, genesisID string, networkID protocol.NetworkID) (gn GossipNode, err error) { - return NewWebsocketNetwork(log, config, phonebookAddresses, genesisID, networkID, nil) + return NewWebsocketNetwork(log, config, phonebookAddresses, genesisID, networkID, nil, "", nil) } // SetPrioScheme specifies the network priority scheme for a network node @@ -2289,11 +2368,11 @@ func (wn *WebsocketNetwork) removePeer(peer *wsPeer, reason disconnectReason) { // first logging, then take the lock and do the actual accounting. // definitely don't change this to do the logging while holding the lock. localAddr, _ := wn.Address() - logEntry := wn.log.With("event", "Disconnected").With("remote", peer.rootURL).With("local", localAddr) + logEntry := wn.log.With("event", "Disconnected").With("remote", peer.GetAddress()).With("local", localAddr) if peer.outgoing && peer.peerMessageDelay > 0 { logEntry = logEntry.With("messageDelay", peer.peerMessageDelay) } - logEntry.Infof("Peer %s disconnected: %s", peer.rootURL, reason) + logEntry.Infof("Peer %s disconnected: %s", peer.GetAddress(), reason) peerAddr := peer.OriginAddress() // we might be able to get addr out of conn, or it might be closed if peerAddr == "" && peer.conn != nil { @@ -2304,12 +2383,12 @@ func (wn *WebsocketNetwork) removePeer(peer *wsPeer, reason disconnectReason) { } if peerAddr == "" { // didn't get addr from peer, try from url - url, err := url.Parse(peer.rootURL) + url, err := url.Parse(peer.GetAddress()) if err == nil { peerAddr = justHost(url.Host) } else { // use whatever it is - peerAddr = justHost(peer.rootURL) + peerAddr = justHost(peer.GetAddress()) } } eventDetails := telemetryspec.PeerEventDetails{ @@ -2493,7 +2572,5 @@ func (wn *WebsocketNetwork) postMessagesOfInterestThread() { } } -// SubstituteGenesisID substitutes the "{genesisID}" with their network-specific genesisID. -func (wn *WebsocketNetwork) SubstituteGenesisID(rawURL string) string { - return strings.Replace(rawURL, "{genesisID}", wn.GenesisID, -1) -} +// GetGenesisID returns the network-specific genesisID. +func (wn *WebsocketNetwork) GetGenesisID() string { return wn.GenesisID } diff --git a/network/wsNetwork_test.go b/network/wsNetwork_test.go index 8daf4d196f..038a9d6e2d 100644 --- a/network/wsNetwork_test.go +++ b/network/wsNetwork_test.go @@ -40,6 +40,7 @@ import ( "time" "github.com/algorand/go-algorand/internal/rapidgen" + "github.com/algorand/go-algorand/network/phonebook" "pgregory.net/rapid" "github.com/stretchr/testify/assert" @@ -127,11 +128,12 @@ func makeTestWebsocketNodeWithConfig(t testing.TB, conf config.Local, opts ...te log := logging.TestingLog(t) log.SetLevel(logging.Warn) wn := &WebsocketNetwork{ - log: log, - config: conf, - phonebook: MakePhonebook(1, 1*time.Millisecond), - GenesisID: genesisID, - NetworkID: config.Devtestnet, + log: log, + config: conf, + phonebook: phonebook.MakePhonebook(1, 1*time.Millisecond), + GenesisID: genesisID, + NetworkID: config.Devtestnet, + peerStater: peerConnectionStater{log: log}, } // apply options to newly-created WebsocketNetwork, if provided for _, opt := range opts { @@ -322,7 +324,7 @@ func setupWebsocketNetworkABwithLogger(t *testing.T, countTarget int, log loggin addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer func() { if !success { @@ -458,7 +460,7 @@ func TestWebsocketProposalPayloadCompression(t *testing.T) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer netStop(t, netB, "B") messages := [][]byte{ @@ -637,7 +639,7 @@ func TestWebsocketNetworkNoAddress(t *testing.T) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer netStop(t, netB, "B") @@ -702,7 +704,7 @@ func lineNetwork(t *testing.T, numNodes int) (nodes []*WebsocketNetwork, counter if i > 0 { addrPrev, postListen := nodes[i-1].Address() require.True(t, postListen) - nodes[i].phonebook.ReplacePeerList([]string{addrPrev}, "default", PhoneBookEntryRelayRole) + nodes[i].phonebook.ReplacePeerList([]string{addrPrev}, "default", phonebook.PhoneBookEntryRelayRole) nodes[i].RegisterHandlers([]TaggedMessageHandler{{Tag: protocol.TxnTag, MessageHandler: &counters[i]}}) } nodes[i].Start() @@ -1053,11 +1055,12 @@ func makeTestFilterWebsocketNode(t *testing.T, nodename string) *WebsocketNetwor dc.OutgoingMessageFilterBucketCount = 3 dc.OutgoingMessageFilterBucketSize = 128 wn := &WebsocketNetwork{ - log: logging.TestingLog(t).With("node", nodename), - config: dc, - phonebook: MakePhonebook(1, 1*time.Millisecond), - GenesisID: genesisID, - NetworkID: config.Devtestnet, + log: logging.TestingLog(t).With("node", nodename), + config: dc, + phonebook: phonebook.MakePhonebook(1, 1*time.Millisecond), + GenesisID: genesisID, + NetworkID: config.Devtestnet, + peerStater: peerConnectionStater{log: logging.TestingLog(t).With("node", nodename)}, } require.True(t, wn.config.EnableIncomingMessageFilter) wn.setup() @@ -1078,7 +1081,7 @@ func TestDupFilter(t *testing.T) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer netStop(t, netB, "B") counter := &messageCounterHandler{t: t, limit: 1, done: make(chan struct{})} @@ -1091,7 +1094,7 @@ func TestDupFilter(t *testing.T) { require.True(t, postListen) netC := makeTestFilterWebsocketNode(t, "c") netC.config.GossipFanout = 1 - netC.phonebook.ReplacePeerList([]string{addrB}, "default", PhoneBookEntryRelayRole) + netC.phonebook.ReplacePeerList([]string{addrB}, "default", phonebook.PhoneBookEntryRelayRole) netC.Start() defer netC.Stop() @@ -1169,8 +1172,8 @@ func TestGetPeers(t *testing.T) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - phbMulti := MakePhonebook(1, 1*time.Millisecond) - phbMulti.ReplacePeerList([]string{addrA}, "phba", PhoneBookEntryRelayRole) + phbMulti := phonebook.MakePhonebook(1, 1*time.Millisecond) + phbMulti.ReplacePeerList([]string{addrA}, "phba", phonebook.PhoneBookEntryRelayRole) netB.phonebook = phbMulti netB.Start() defer netB.Stop() @@ -1181,10 +1184,10 @@ func TestGetPeers(t *testing.T) { waitReady(t, netB, readyTimeout.C) t.Log("b ready") - phbMulti.ReplacePeerList([]string{"a", "b", "c"}, "ph", PhoneBookEntryRelayRole) + phbMulti.ReplacePeerList([]string{"a", "b", "c"}, "ph", phonebook.PhoneBookEntryRelayRole) // A few for archival node roles - phbMulti.ReplacePeerList([]string{"d", "e", "f"}, "ph", PhoneBookEntryArchivalRole) + phbMulti.ReplacePeerList([]string{"d", "e", "f"}, "ph", phonebook.PhoneBookEntryArchivalRole) //addrB, _ := netB.Address() @@ -1767,7 +1770,7 @@ func TestPeeringWithBadIdentityChallenge(t *testing.T) { attachChallenge: func(attach http.Header, addr string) identityChallengeValue { s := NewIdentityChallengeScheme("does not matter") // make a scheme to use its keys c := identityChallenge{ - Key: s.identityKeys.SignatureVerifier, + Key: s.identityKeys.PublicKey(), Challenge: newIdentityChallengeValue(), PublicAddress: []byte("incorrect address!"), } @@ -1785,7 +1788,7 @@ func TestPeeringWithBadIdentityChallenge(t *testing.T) { attachChallenge: func(attach http.Header, addr string) identityChallengeValue { s := NewIdentityChallengeScheme("does not matter") // make a scheme to use its keys c := identityChallenge{ - Key: s.identityKeys.SignatureVerifier, + Key: s.identityKeys.PublicKey(), Challenge: newIdentityChallengeValue(), PublicAddress: []byte("incorrect address!"), }.Sign(s.identityKeys) @@ -1905,7 +1908,7 @@ func TestPeeringWithBadIdentityChallengeResponse(t *testing.T) { protocol.Decode(msg, &idChal) // make the response object, with an incorrect challenge encode it and attach it to the header r := identityChallengeResponse{ - Key: s.identityKeys.SignatureVerifier, + Key: s.identityKeys.PublicKey(), Challenge: newIdentityChallengeValue(), ResponseChallenge: newIdentityChallengeValue(), } @@ -1928,7 +1931,7 @@ func TestPeeringWithBadIdentityChallengeResponse(t *testing.T) { protocol.Decode(msg, &idChal) // make the response object, then change the signature and encode and attach r := identityChallengeResponse{ - Key: s.identityKeys.SignatureVerifier, + Key: s.identityKeys.PublicKey(), Challenge: newIdentityChallengeValue(), ResponseChallenge: newIdentityChallengeValue(), }.Sign(s.identityKeys) @@ -2180,7 +2183,7 @@ func BenchmarkWebsocketNetworkBasic(t *testing.B) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer netStop(t, netB, "B") returns := make(chan uint64, 100) @@ -2262,7 +2265,7 @@ func TestWebsocketNetworkPrio(t *testing.T) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer netStop(t, netB, "B") @@ -2309,7 +2312,7 @@ func TestWebsocketNetworkPrioLimit(t *testing.T) { netB.SetPrioScheme(&prioB) netB.config.GossipFanout = 1 netB.config.NetAddress = "" - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.RegisterHandlers([]TaggedMessageHandler{{Tag: protocol.TxnTag, MessageHandler: counterB}}) netB.Start() defer netStop(t, netB, "B") @@ -2323,7 +2326,7 @@ func TestWebsocketNetworkPrioLimit(t *testing.T) { netC.SetPrioScheme(&prioC) netC.config.GossipFanout = 1 netC.config.NetAddress = "" - netC.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netC.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netC.RegisterHandlers([]TaggedMessageHandler{{Tag: protocol.TxnTag, MessageHandler: counterC}}) netC.Start() defer func() { t.Log("stopping C"); netC.Stop(); t.Log("C done") }() @@ -2366,8 +2369,8 @@ func TestWebsocketNetworkPrioLimit(t *testing.T) { } if failed { - t.Errorf("NetA had the following two peers priorities : [0]:%s=%d [1]:%s=%d", netA.peers[0].rootURL, netA.peers[0].prioWeight, netA.peers[1].rootURL, netA.peers[1].prioWeight) - t.Errorf("first peer before broadcasting was %s", firstPeer.rootURL) + t.Errorf("NetA had the following two peers priorities : [0]:%s=%d [1]:%s=%d", netA.peers[0].GetAddress(), netA.peers[0].prioWeight, netA.peers[1].GetAddress(), netA.peers[1].prioWeight) + t.Errorf("first peer before broadcasting was %s", firstPeer.GetAddress()) } } @@ -2408,7 +2411,7 @@ func TestWebsocketNetworkManyIdle(t *testing.T) { for i := 0; i < numClients; i++ { client := makeTestWebsocketNodeWithConfig(t, clientConf) client.config.GossipFanout = 1 - client.phonebook.ReplacePeerList([]string{relayAddr}, "default", PhoneBookEntryRelayRole) + client.phonebook.ReplacePeerList([]string{relayAddr}, "default", phonebook.PhoneBookEntryRelayRole) client.Start() defer client.Stop() @@ -2474,7 +2477,7 @@ func TestWebsocketNetwork_checkServerResponseVariables(t *testing.T) { noVersionHeader := http.Header{} noVersionHeader.Set(NodeRandomHeader, wn.RandomID+"tag") noVersionHeader.Set(GenesisHeader, wn.GenesisID) - responseVariableOk, matchingVersion = wn.checkServerResponseVariables(noVersionHeader, "addressX") + responseVariableOk, _ = wn.checkServerResponseVariables(noVersionHeader, "addressX") require.Equal(t, false, responseVariableOk) noRandomHeader := http.Header{} @@ -2533,7 +2536,7 @@ func TestDelayedMessageDrop(t *testing.T) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer netStop(t, netB, "B") counter := newMessageCounter(t, 5) @@ -2563,11 +2566,12 @@ func TestSlowPeerDisconnection(t *testing.T) { log := logging.TestingLog(t) log.SetLevel(logging.Info) wn := &WebsocketNetwork{ - log: log, - config: defaultConfig, - phonebook: MakePhonebook(1, 1*time.Millisecond), - GenesisID: genesisID, - NetworkID: config.Devtestnet, + log: log, + config: defaultConfig, + phonebook: phonebook.MakePhonebook(1, 1*time.Millisecond), + GenesisID: genesisID, + NetworkID: config.Devtestnet, + peerStater: peerConnectionStater{log: log}, } wn.setup() wn.broadcaster.slowWritingPeerMonitorInterval = time.Millisecond * 50 @@ -2586,7 +2590,7 @@ func TestSlowPeerDisconnection(t *testing.T) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer netStop(t, netB, "B") @@ -2638,11 +2642,12 @@ func TestForceMessageRelaying(t *testing.T) { log := logging.TestingLog(t) log.SetLevel(logging.Level(defaultConfig.BaseLoggerDebugLevel)) wn := &WebsocketNetwork{ - log: log, - config: defaultConfig, - phonebook: MakePhonebook(1, 1*time.Millisecond), - GenesisID: genesisID, - NetworkID: config.Devtestnet, + log: log, + config: defaultConfig, + phonebook: phonebook.MakePhonebook(1, 1*time.Millisecond), + GenesisID: genesisID, + NetworkID: config.Devtestnet, + peerStater: peerConnectionStater{log: log}, } wn.setup() wn.eventualReadyDelay = time.Second @@ -2663,14 +2668,14 @@ func TestForceMessageRelaying(t *testing.T) { noAddressConfig.NetAddress = "" netB := makeTestWebsocketNodeWithConfig(t, noAddressConfig) netB.config.GossipFanout = 1 - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer netStop(t, netB, "B") noAddressConfig.ForceRelayMessages = true netC := makeTestWebsocketNodeWithConfig(t, noAddressConfig) netC.config.GossipFanout = 1 - netC.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netC.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netC.Start() defer func() { t.Log("stopping C"); netC.Stop(); t.Log("C done") }() @@ -2732,11 +2737,12 @@ func TestCheckProtocolVersionMatch(t *testing.T) { log := logging.TestingLog(t) log.SetLevel(logging.Level(defaultConfig.BaseLoggerDebugLevel)) wn := &WebsocketNetwork{ - log: log, - config: defaultConfig, - phonebook: MakePhonebook(1, 1*time.Millisecond), - GenesisID: genesisID, - NetworkID: config.Devtestnet, + log: log, + config: defaultConfig, + phonebook: phonebook.MakePhonebook(1, 1*time.Millisecond), + GenesisID: genesisID, + NetworkID: config.Devtestnet, + peerStater: peerConnectionStater{log: log}, } wn.setup() wn.supportedProtocolVersions = []string{"2", "1"} @@ -2814,7 +2820,7 @@ func TestWebsocketNetworkTopicRoundtrip(t *testing.T) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer netStop(t, netB, "B") @@ -2914,7 +2920,7 @@ func TestWebsocketNetworkMessageOfInterest(t *testing.T) { addrA, postListen := netA.Address() require.True(t, postListen) t.Logf("netA %s", addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) // have netB asking netA to send it ft2, deregister ping handler to make sure that we aren't exceeding the maximum MOI messagesize // Max MOI size is calculated by encoding all of the valid tags, since we are using a custom tag here we must deregister one in the default set. @@ -3040,7 +3046,7 @@ func TestWebsocketNetworkTXMessageOfInterestRelay(t *testing.T) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer netStop(t, netB, "B") @@ -3124,7 +3130,7 @@ func TestWebsocketNetworkTXMessageOfInterestForceTx(t *testing.T) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer netStop(t, netB, "B") @@ -3206,7 +3212,7 @@ func TestWebsocketNetworkTXMessageOfInterestNPN(t *testing.T) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer netStop(t, netB, "B") require.False(t, netB.relayMessages) @@ -3286,6 +3292,7 @@ func TestWebsocketNetworkTXMessageOfInterestNPN(t *testing.T) { } type participatingNodeInfo struct { + nopeNodeInfo } func (nnni *participatingNodeInfo) IsParticipating() bool { @@ -3311,7 +3318,7 @@ func TestWebsocketNetworkTXMessageOfInterestPN(t *testing.T) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer netStop(t, netB, "B") require.False(t, netB.relayMessages) @@ -3433,7 +3440,7 @@ func testWebsocketDisconnection(t *testing.T, disconnectFunc func(wn *WebsocketN addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer netStop(t, netB, "B") @@ -3628,7 +3635,7 @@ func BenchmarkVariableTransactionMessageBlockSizes(t *testing.B) { addrA, postListen := netA.Address() require.True(t, postListen) t.Log(addrA) - netB.phonebook.ReplacePeerList([]string{addrA}, "default", PhoneBookEntryRelayRole) + netB.phonebook.ReplacePeerList([]string{addrA}, "default", phonebook.PhoneBookEntryRelayRole) netB.Start() defer func() { netB.Stop() }() @@ -3773,9 +3780,9 @@ func TestWebsocketNetworkTelemetryTCP(t *testing.T) { // get RTT from both ends and assert nonzero var peersA, peersB []*wsPeer peersA, _ = netA.peerSnapshot(peersA) - detailsA := netA.getPeerConnectionTelemetryDetails(time.Now(), peersA) + detailsA := getPeerConnectionTelemetryDetails(time.Now(), peersA) peersB, _ = netB.peerSnapshot(peersB) - detailsB := netB.getPeerConnectionTelemetryDetails(time.Now(), peersB) + detailsB := getPeerConnectionTelemetryDetails(time.Now(), peersB) require.Len(t, detailsA.IncomingPeers, 1) assert.NotZero(t, detailsA.IncomingPeers[0].TCP.RTT) require.Len(t, detailsB.OutgoingPeers, 1) @@ -3796,8 +3803,8 @@ func TestWebsocketNetworkTelemetryTCP(t *testing.T) { defer closeFunc2() // use stale peers snapshot from closed networks to get telemetry // *net.OpError "use of closed network connection" err results in 0 rtt values - detailsA = netA.getPeerConnectionTelemetryDetails(time.Now(), peersA) - detailsB = netB.getPeerConnectionTelemetryDetails(time.Now(), peersB) + detailsA = getPeerConnectionTelemetryDetails(time.Now(), peersA) + detailsB = getPeerConnectionTelemetryDetails(time.Now(), peersB) require.Len(t, detailsA.IncomingPeers, 1) assert.Zero(t, detailsA.IncomingPeers[0].TCP.RTT) require.Len(t, detailsB.OutgoingPeers, 1) @@ -4540,3 +4547,75 @@ func TestSendMessageCallbackDrain(t *testing.T) { 50*time.Millisecond, ) } + +// TestWsNetworkPhonebookMix ensures p2p addresses are not added into wsNetwork via phonebook +func TestWsNetworkPhonebookMix(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + net, err := NewWebsocketNetwork( + logging.TestingLog(t), + config.GetDefaultLocal(), + []string{"127.0.0.1:1234", "/ip4/127.0.0.1/tcp/1234", "/ip4/127.0.0.1/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC"}, + "test", + "net", + nil, + "", + nil, + ) + require.NoError(t, err) + addrs := net.phonebook.GetAddresses(10, phonebook.PhoneBookEntryRelayRole) + require.Len(t, addrs, 1) +} + +type testRecordingTransport struct { + resultURL string +} + +func (rt *testRecordingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + rt.resultURL = req.URL.String() + return &http.Response{StatusCode: 200}, nil +} + +func TestHTTPPAddressBoundTransport(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + // first ensure url.URL.String() on path-only URLs works as expected + var url = &url.URL{} + url.Path = "/test" + require.Equal(t, "/test", url.String()) + + // now test some combinations of address and path + const path = "/test/path" + const expErr = "ERR" + tests := []struct { + addr string + expected string + }{ + {"", expErr}, + {":", expErr}, + {"host:1234/lbr", expErr}, + {"host:1234", "http://host:1234" + path}, + {"http://host:1234", "http://host:1234" + path}, + {"http://host:1234/lbr", "http://host:1234/lbr" + path}, + } + + for _, test := range tests { + recorder := testRecordingTransport{} + tr := HTTPPAddressBoundTransport{ + Addr: test.addr, + InnerTransport: &recorder, + } + req, err := http.NewRequest("GET", path, nil) + require.NoError(t, err) + resp, err := tr.RoundTrip(req) + if test.expected == expErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + require.Equal(t, test.expected, recorder.resultURL) + } + } +} diff --git a/network/wsPeer.go b/network/wsPeer.go index c4b64bdec4..2b302f071f 100644 --- a/network/wsPeer.go +++ b/network/wsPeer.go @@ -172,7 +172,7 @@ type wsPeerCore struct { readBuffer chan<- IncomingMessage rootURL string originAddress string // incoming connection remote host - client http.Client + client *http.Client } type disconnectReason string @@ -329,7 +329,6 @@ type HTTPPeer interface { // IPAddressable is addressable with either IPv4 or IPv6 address type IPAddressable interface { - IPAddr() []byte RoutingAddr() []byte } @@ -352,21 +351,20 @@ type TCPInfoUnicastPeer interface { } // Create a wsPeerCore object -func makePeerCore(ctx context.Context, net GossipNode, log logging.Logger, readBuffer chan<- IncomingMessage, rootURL string, roundTripper http.RoundTripper, originAddress string) wsPeerCore { +func makePeerCore(ctx context.Context, net GossipNode, log logging.Logger, readBuffer chan<- IncomingMessage, addr string, client *http.Client, originAddress string) wsPeerCore { return wsPeerCore{ net: net, netCtx: ctx, log: log, readBuffer: readBuffer, - rootURL: rootURL, + rootURL: addr, originAddress: originAddress, - client: http.Client{Transport: roundTripper}, + client: client, } } -// GetAddress returns the root url to use to connect to this peer. -// This implements HTTPPeer interface and used by external services to determine where to connect to. -// TODO: should GetAddress be added to Peer interface? +// GetAddress returns the root url to use to identify or connect to this peer. +// This implements HTTPPeer interface and used to distinguish between peers. func (wp *wsPeerCore) GetAddress() string { return wp.rootURL } @@ -374,7 +372,11 @@ func (wp *wsPeerCore) GetAddress() string { // GetHTTPClient returns a client for this peer. // http.Client will maintain a cache of connections with some keepalive. func (wp *wsPeerCore) GetHTTPClient() *http.Client { - return &wp.client + return wp.client +} + +func (wp *wsPeerCore) GetNetwork() GossipNode { + return wp.net } // Version returns the matching version from network.SupportedProtocolVersions @@ -382,7 +384,7 @@ func (wp *wsPeer) Version() string { return wp.version } -func (wp *wsPeer) IPAddr() []byte { +func (wp *wsPeer) ipAddr() []byte { remote := wp.conn.RemoteAddr() if remote == nil { return nil @@ -417,7 +419,7 @@ func (wp *wsPeer) RoutingAddr() []byte { if wp.wsPeerCore.originAddress != "" { ip = net.ParseIP(wp.wsPeerCore.originAddress) } else { - ip = wp.IPAddr() + ip = wp.ipAddr() } if len(ip) != net.IPv6len { @@ -511,7 +513,7 @@ func (wp *wsPeer) Respond(ctx context.Context, reqMsg IncomingMessage, outMsg Ou // setup values not trivially assigned func (wp *wsPeer) init(config config.Local, sendBufferLength int) { - wp.log.Debugf("wsPeer init outgoing=%v %#v", wp.outgoing, wp.rootURL) + wp.log.Debugf("wsPeer init outgoing=%v %#v", wp.outgoing, wp.GetAddress()) wp.closing = make(chan struct{}) wp.sendBufferHighPrio = make(chan sendMessages, sendBufferLength) wp.sendBufferBulk = make(chan sendMessages, sendBufferLength) diff --git a/network/wsPeer_test.go b/network/wsPeer_test.go index d1f32302a0..973c027b16 100644 --- a/network/wsPeer_test.go +++ b/network/wsPeer_test.go @@ -288,32 +288,32 @@ func TestWsPeerIPAddr(t *testing.T) { } // some raw IPv4 address conn.addr.IP = []byte{127, 0, 0, 1} - require.Equal(t, []byte{127, 0, 0, 1}, peer.IPAddr()) + require.Equal(t, []byte{127, 0, 0, 1}, peer.ipAddr()) require.Equal(t, []byte{127, 0, 0, 1}, peer.RoutingAddr()) // IPv4 constructed from net.IPv4 conn.addr.IP = net.IPv4(127, 0, 0, 2) - require.Equal(t, []byte{127, 0, 0, 2}, peer.IPAddr()) + require.Equal(t, []byte{127, 0, 0, 2}, peer.ipAddr()) require.Equal(t, []byte{127, 0, 0, 2}, peer.RoutingAddr()) // some IPv6 address conn.addr.IP = net.IPv6linklocalallrouters - require.Equal(t, []byte(net.IPv6linklocalallrouters), peer.IPAddr()) + require.Equal(t, []byte(net.IPv6linklocalallrouters), peer.ipAddr()) require.Equal(t, []byte(net.IPv6linklocalallrouters[0:8]), peer.RoutingAddr()) // embedded IPv4 into IPv6 conn.addr.IP = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 127, 0, 0, 3} require.Equal(t, 16, len(conn.addr.IP)) - require.Equal(t, []byte{127, 0, 0, 3}, peer.IPAddr()) + require.Equal(t, []byte{127, 0, 0, 3}, peer.ipAddr()) require.Equal(t, []byte{127, 0, 0, 3}, peer.RoutingAddr()) conn.addr.IP = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 4} require.Equal(t, 16, len(conn.addr.IP)) - require.Equal(t, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 4}, peer.IPAddr()) + require.Equal(t, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 4}, peer.ipAddr()) require.Equal(t, []byte{127, 0, 0, 4}, peer.RoutingAddr()) // check incoming peer with originAddress set conn.addr.IP = []byte{127, 0, 0, 1} peer.wsPeerCore.originAddress = "127.0.0.2" - require.Equal(t, []byte{127, 0, 0, 1}, peer.IPAddr()) + require.Equal(t, []byte{127, 0, 0, 1}, peer.ipAddr()) require.Equal(t, []byte{127, 0, 0, 2}, peer.RoutingAddr()) } diff --git a/node/follower_node.go b/node/follower_node.go index e475b25481..117cc56e86 100644 --- a/node/follower_node.go +++ b/node/follower_node.go @@ -94,7 +94,7 @@ func MakeFollower(log logging.Logger, rootDir string, cfg config.Local, phoneboo node.config = cfg // tie network, block fetcher, and agreement services together - p2pNode, err := network.NewWebsocketNetwork(node.log, node.config, phonebookAddresses, genesis.ID(), genesis.Network, nil) + p2pNode, err := network.NewWebsocketNetwork(node.log, node.config, phonebookAddresses, genesis.ID(), genesis.Network, nil, "", nil) if err != nil { log.Errorf("could not create websocket node: %v", err) return nil, err @@ -163,7 +163,7 @@ func (node *AlgorandFollowerNode) Config() config.Local { } // Start the node: connect to peers while obtaining a lock. Doesn't wait for initial sync. -func (node *AlgorandFollowerNode) Start() { +func (node *AlgorandFollowerNode) Start() error { node.mu.Lock() defer node.mu.Unlock() @@ -173,22 +173,30 @@ func (node *AlgorandFollowerNode) Start() { // The start network is being called only after the various services start up. // We want to do so in order to let the services register their callbacks with the // network package before any connections are being made. - startNetwork := func() { + startNetwork := func() error { if !node.config.DisableNetworking { // start accepting connections - node.net.Start() + err := node.net.Start() + if err != nil { + return err + } node.config.NetAddress, _ = node.net.Address() } + return nil } + var err error if node.catchpointCatchupService != nil { - startNetwork() - _ = node.catchpointCatchupService.Start(node.ctx) + err = startNetwork() + if err == nil { + err = node.catchpointCatchupService.Start(node.ctx) + } } else { node.catchupService.Start() node.blockService.Start() - startNetwork() + err = startNetwork() } + return err } // ListeningAddress retrieves the node's current listening address, if any. diff --git a/node/node.go b/node/node.go index 384bd258f6..5f1baa56be 100644 --- a/node/node.go +++ b/node/node.go @@ -28,6 +28,8 @@ import ( "sync" "time" + "github.com/algorand/go-deadlock" + "github.com/algorand/go-algorand/agreement" "github.com/algorand/go-algorand/agreement/gossip" "github.com/algorand/go-algorand/catchup" @@ -47,6 +49,7 @@ import ( "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/network/messagetracer" + "github.com/algorand/go-algorand/network/p2p" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/rpcs" "github.com/algorand/go-algorand/stateproof" @@ -54,7 +57,6 @@ import ( "github.com/algorand/go-algorand/util/execpool" "github.com/algorand/go-algorand/util/metrics" "github.com/algorand/go-algorand/util/timers" - "github.com/algorand/go-deadlock" ) const ( @@ -197,16 +199,21 @@ func MakeFull(log logging.Logger, rootDir string, cfg config.Local, phonebookAdd // tie network, block fetcher, and agreement services together var p2pNode network.GossipNode - if cfg.EnableP2P { - // TODO: pass more appropriate genesisDir (hot/cold). Presently this is just used to store a peerID key. - p2pNode, err = network.NewP2PNetwork(node.log, node.config, node.genesisDirs.RootGenesisDir, phonebookAddresses, genesis.ID(), genesis.Network) + if cfg.EnableP2PHybridMode { + p2pNode, err = network.NewHybridP2PNetwork(node.log, node.config, rootDir, phonebookAddresses, genesis.ID(), genesis.Network, node) + if err != nil { + log.Errorf("could not create hybrid p2p node: %v", err) + return nil, err + } + } else if cfg.EnableP2P { + p2pNode, err = network.NewP2PNetwork(node.log, node.config, rootDir, phonebookAddresses, genesis.ID(), genesis.Network, node) if err != nil { log.Errorf("could not create p2p node: %v", err) return nil, err } } else { var wsNode *network.WebsocketNetwork - wsNode, err = network.NewWebsocketNetwork(node.log, node.config, phonebookAddresses, genesis.ID(), genesis.Network, node) + wsNode, err = network.NewWebsocketNetwork(node.log, node.config, phonebookAddresses, genesis.ID(), genesis.Network, node, "", nil) if err != nil { log.Errorf("could not create websocket node: %v", err) return nil, err @@ -340,7 +347,7 @@ func (node *AlgorandFullNode) Config() config.Local { } // Start the node: connect to peers and run the agreement service while obtaining a lock. Doesn't wait for initial sync. -func (node *AlgorandFullNode) Start() { +func (node *AlgorandFullNode) Start() error { node.mu.Lock() defer node.mu.Unlock() @@ -350,12 +357,16 @@ func (node *AlgorandFullNode) Start() { // The start network is being called only after the various services start up. // We want to do so in order to let the services register their callbacks with the // network package before any connections are being made. - startNetwork := func() { + startNetwork := func() error { if !node.config.DisableNetworking { // start accepting connections - node.net.Start() + err := node.net.Start() + if err != nil { + return err + } node.config.NetAddress, _ = node.net.Address() } + return nil } if node.catchpointCatchupService != nil { @@ -369,11 +380,29 @@ func (node *AlgorandFullNode) Start() { node.ledgerService.Start() node.txHandler.Start() node.stateProofWorker.Start() - startNetwork() + err := startNetwork() + if err != nil { + return err + } node.startMonitoringRoutines() } + return nil +} +// Capabilities returns the node's capabilities for advertising to other nodes. +func (node *AlgorandFullNode) Capabilities() []p2p.Capability { + var caps []p2p.Capability + if node.config.Archival { + caps = append(caps, p2p.Archival) + } + if node.config.StoresCatchpoints() { + caps = append(caps, p2p.Catchpoints) + } + if node.config.EnableGossipService && node.config.IsGossipServer() { + caps = append(caps, p2p.Gossip) + } + return caps } // startMonitoringRoutines starts the internal monitoring routines used by the node. diff --git a/node/node_test.go b/node/node_test.go index 54f2e1e6cc..3ea6d4a33d 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -33,19 +33,18 @@ import ( "github.com/algorand/go-algorand/agreement" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" - "github.com/algorand/go-algorand/data" "github.com/algorand/go-algorand/data/account" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" + "github.com/algorand/go-algorand/network/p2p" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/stateproof" "github.com/algorand/go-algorand/test/partitiontest" "github.com/algorand/go-algorand/util" "github.com/algorand/go-algorand/util/db" - "github.com/algorand/go-algorand/util/execpool" ) var expectedAgreementTime = 2*config.Protocol.BigLambda + config.Protocol.SmallLambda + config.Consensus[protocol.ConsensusCurrentVersion].AgreementFilterTimeout + 2*time.Second @@ -61,31 +60,78 @@ var defaultConfig = config.Local{ IncomingConnectionsLimit: -1, } -func setupFullNodes(t *testing.T, proto protocol.ConsensusVersion, verificationPool execpool.BacklogPool, customConsensus config.ConsensusProtocols) ([]*AlgorandFullNode, []string) { +type nodeInfo struct { + idx int + host string + wsPort int + p2pPort int + p2pID p2p.PeerID + rootDir string + genesis bookkeeping.Genesis +} + +func (ni nodeInfo) wsNetAddr() string { + return fmt.Sprintf("%s:%d", ni.host, ni.wsPort) +} + +func (ni nodeInfo) p2pNetAddr() string { + return fmt.Sprintf("%s:%d", ni.host, ni.p2pPort) +} + +func (ni nodeInfo) p2pMultiAddr() string { + return fmt.Sprintf("/ip4/%s/tcp/%d/p2p/%s", ni.host, ni.p2pPort, ni.p2pID.String()) +} + +type configHook func(ni nodeInfo, cfg config.Local) (nodeInfo, config.Local) +type phonebookHook func([]nodeInfo, int) []string + +func setupFullNodes(t *testing.T, proto protocol.ConsensusVersion, customConsensus config.ConsensusProtocols) ([]*AlgorandFullNode, []string) { + minMoneyAtStart := 10000 + maxMoneyAtStart := 100000 + gen := rand.New(rand.NewSource(2)) + + const numAccounts = 10 + acctStake := make([]basics.MicroAlgos, numAccounts) + for i := range acctStake { + acctStake[i] = basics.MicroAlgos{Raw: uint64(minMoneyAtStart + (gen.Int() % (maxMoneyAtStart - minMoneyAtStart)))} + } + + configHook := func(ni nodeInfo, cfg config.Local) (nodeInfo, config.Local) { + cfg.NetAddress = ni.wsNetAddr() + return ni, cfg + } + + phonebookHook := func(nodes []nodeInfo, nodeIdx int) []string { + phonebook := make([]string, 0, len(nodes)-1) + for i := range nodes { + if i != nodeIdx { + phonebook = append(phonebook, nodes[i].wsNetAddr()) + } + } + return phonebook + } + nodes, wallets := setupFullNodesEx(t, proto, customConsensus, acctStake, configHook, phonebookHook) + require.Len(t, nodes, numAccounts) + require.Len(t, wallets, numAccounts) + return nodes, wallets +} + +func setupFullNodesEx( + t *testing.T, proto protocol.ConsensusVersion, customConsensus config.ConsensusProtocols, + acctStake []basics.MicroAlgos, configHook configHook, phonebookHook phonebookHook, +) ([]*AlgorandFullNode, []string) { + util.SetFdSoftLimit(1000) + f, _ := os.Create(t.Name() + ".log") logging.Base().SetJSONFormatter() logging.Base().SetOutput(f) logging.Base().SetLevel(logging.Debug) - - numAccounts := 10 - minMoneyAtStart := 10000 - maxMoneyAtStart := 100000 + t.Logf("Logging to %s\n", t.Name()+".log") firstRound := basics.Round(0) lastRound := basics.Round(200) - genesis := make(map[basics.Address]basics.AccountData) - gen := rand.New(rand.NewSource(2)) - neighbors := make([]string, numAccounts) - for i := range neighbors { - neighbors[i] = "127.0.0.1:" + strconv.Itoa(10000+i) - } - - wallets := make([]string, numAccounts) - nodes := make([]*AlgorandFullNode, numAccounts) - rootDirs := make([]string, 0) - // The genesis configuration is missing allocations, but that's OK // because we explicitly generated the sqlite database above (in // installFullNode). @@ -97,16 +143,27 @@ func setupFullNodes(t *testing.T, proto protocol.ConsensusVersion, verificationP RewardsPool: poolAddr.String(), } + genesis := make(map[basics.Address]basics.AccountData) + numAccounts := len(acctStake) + wallets := make([]string, numAccounts) + nodeInfos := make([]nodeInfo, numAccounts) + for i := range wallets { rootDirectory := t.TempDir() - rootDirs = append(rootDirs, rootDirectory) + nodeInfos[i] = nodeInfo{ + idx: i, + host: "127.0.0.1", + wsPort: 10000 + 100*i, + p2pPort: 10000 + 100*i + 1, + rootDir: rootDirectory, + genesis: g, + } - defaultConfig.NetAddress = neighbors[i] - defaultConfig.SaveToDisk(rootDirectory) + ni, cfg := configHook(nodeInfos[i], defaultConfig) + nodeInfos[i] = ni + cfg.SaveToDisk(rootDirectory) - // Save empty phonebook - we'll add peers after they've been assigned listening ports - err := config.SavePhonebookToDisk(make([]string, 0), rootDirectory) - require.NoError(t, err) + t.Logf("Root directory of node %d (%s): %s\n", i, ni.wsNetAddr(), rootDirectory) genesisDir := filepath.Join(rootDirectory, g.ID()) os.Mkdir(genesisDir, 0700) @@ -140,7 +197,7 @@ func setupFullNodes(t *testing.T, proto protocol.ConsensusVersion, verificationP data := basics.AccountData{ Status: basics.Online, - MicroAlgos: basics.MicroAlgos{Raw: uint64(minMoneyAtStart + (gen.Int() % (maxMoneyAtStart - minMoneyAtStart)))}, + MicroAlgos: acctStake[i], SelectionID: part.VRFSecrets().PK, VoteID: part.VotingSecrets().OneTimeSignatureVerifier, } @@ -152,34 +209,37 @@ func setupFullNodes(t *testing.T, proto protocol.ConsensusVersion, verificationP MicroAlgos: basics.MicroAlgos{Raw: uint64(100000)}, } - bootstrap := bookkeeping.MakeGenesisBalances(genesis, sinkAddr, poolAddr) + for addr, data := range genesis { + g.Allocation = append(g.Allocation, bookkeeping.GenesisAllocation{ + Address: addr.String(), + State: bookkeeping.GenesisAccountData{ + Status: data.Status, + MicroAlgos: data.MicroAlgos, + VoteID: data.VoteID, + StateProofID: data.StateProofID, + SelectionID: data.SelectionID, + VoteFirstValid: data.VoteFirstValid, + VoteLastValid: data.VoteLastValid, + VoteKeyDilution: data.VoteKeyDilution, + }, + }) + } - for i, rootDirectory := range rootDirs { + nodes := make([]*AlgorandFullNode, numAccounts) + for i := range nodes { + rootDirectory := nodeInfos[i].rootDir genesisDir := filepath.Join(rootDirectory, g.ID()) - ledgerFilenamePrefix := filepath.Join(genesisDir, config.LedgerFilenamePrefix) if customConsensus != nil { - err := config.SaveConfigurableConsensus(genesisDir, customConsensus) - require.Nil(t, err) + err0 := config.SaveConfigurableConsensus(genesisDir, customConsensus) + require.Nil(t, err0) + err0 = config.LoadConfigurableConsensusProtocols(genesisDir) + require.Nil(t, err0) } - err1 := config.LoadConfigurableConsensusProtocols(genesisDir) - require.Nil(t, err1) - nodeID := fmt.Sprintf("Node%d", i) - const inMem = false - cfg, err := config.LoadConfigFromDisk(rootDirectory) - require.NoError(t, err) - cfg.Archival = true - _, err = data.LoadLedger(logging.Base().With("name", nodeID), ledgerFilenamePrefix, inMem, g.Proto, bootstrap, g.ID(), g.Hash(), cfg) - require.NoError(t, err) - } - for i := range nodes { - var nodeNeighbors []string - nodeNeighbors = append(nodeNeighbors, neighbors[:i]...) - nodeNeighbors = append(nodeNeighbors, neighbors[i+1:]...) - rootDirectory := rootDirs[i] cfg, err := config.LoadConfigFromDisk(rootDirectory) + phonebook := phonebookHook(nodeInfos, i) require.NoError(t, err) - node, err := MakeFull(logging.Base().With("source", t.Name()+strconv.Itoa(i)), rootDirectory, cfg, nodeNeighbors, g) + node, err := MakeFull(logging.Base(), rootDirectory, cfg, phonebook, g) nodes[i] = node require.NoError(t, err) } @@ -190,12 +250,16 @@ func setupFullNodes(t *testing.T, proto protocol.ConsensusVersion, verificationP func TestSyncingFullNode(t *testing.T) { partitiontest.PartitionTest(t) - t.Skip("Flaky in nightly test environment") + if testing.Short() { + t.Skip("Test takes ~50 seconds.") + } - backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) - defer backlogPool.Shutdown() + if (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64" && runtime.GOOS != "darwin") && + strings.ToUpper(os.Getenv("CIRCLECI")) == "TRUE" { + t.Skip("Test is too heavy for amd64 builder running in parallel with other packages") + } - nodes, wallets := setupFullNodes(t, protocol.ConsensusCurrentVersion, backlogPool, nil) + nodes, wallets := setupFullNodes(t, protocol.ConsensusCurrentVersion, nil) for i := 0; i < len(nodes); i++ { defer os.Remove(wallets[i]) defer nodes[i].Stop() @@ -203,7 +267,7 @@ func TestSyncingFullNode(t *testing.T) { initialRound := nodes[0].ledger.NextRound() - startAndConnectNodes(nodes, true) + startAndConnectNodes(nodes, defaultFirstNodeStartDelay) counter := 0 for tests := uint64(0); tests < 16; tests++ { @@ -252,22 +316,19 @@ func TestInitialSync(t *testing.T) { t.Skip("Test takes ~25 seconds.") } - if (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64") && + if (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64" && runtime.GOOS != "darwin") && strings.ToUpper(os.Getenv("CIRCLECI")) == "TRUE" { t.Skip("Test is too heavy for amd64 builder running in parallel with other packages") } - backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) - defer backlogPool.Shutdown() - - nodes, wallets := setupFullNodes(t, protocol.ConsensusCurrentVersion, backlogPool, nil) + nodes, wallets := setupFullNodes(t, protocol.ConsensusCurrentVersion, nil) for i := 0; i < len(nodes); i++ { defer os.Remove(wallets[i]) defer nodes[i].Stop() } initialRound := nodes[0].ledger.NextRound() - startAndConnectNodes(nodes, true) + startAndConnectNodes(nodes, defaultFirstNodeStartDelay) select { case <-nodes[0].ledger.Wait(initialRound): @@ -289,10 +350,14 @@ func TestInitialSync(t *testing.T) { func TestSimpleUpgrade(t *testing.T) { partitiontest.PartitionTest(t) - t.Skip("Flaky in nightly test environment.") + if testing.Short() { + t.Skip("Test takes ~50 seconds.") + } - backlogPool := execpool.MakeBacklog(nil, 0, execpool.LowPriority, nil) - defer backlogPool.Shutdown() + if (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64" && runtime.GOOS != "darwin") && + strings.ToUpper(os.Getenv("CIRCLECI")) == "TRUE" { + t.Skip("Test is too heavy for amd64 builder running in parallel with other packages") + } // ConsensusTest0 is a version of ConsensusV0 used for testing // (it has different approved upgrade paths). @@ -330,7 +395,7 @@ func TestSimpleUpgrade(t *testing.T) { testParams1.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{} configurableConsensus[consensusTest1] = testParams1 - nodes, wallets := setupFullNodes(t, consensusTest0, backlogPool, configurableConsensus) + nodes, wallets := setupFullNodes(t, consensusTest0, configurableConsensus) for i := 0; i < len(nodes); i++ { defer os.Remove(wallets[i]) defer nodes[i].Stop() @@ -338,13 +403,13 @@ func TestSimpleUpgrade(t *testing.T) { initialRound := nodes[0].ledger.NextRound() - startAndConnectNodes(nodes, false) + startAndConnectNodes(nodes, nodelayFirstNodeStartDelay) maxRounds := basics.Round(16) roundsCheckedForUpgrade := 0 for tests := basics.Round(0); tests < maxRounds; tests++ { - blocks := make([]bookkeeping.Block, len(wallets), len(wallets)) + blocks := make([]bookkeeping.Block, len(wallets)) for i := range wallets { select { case <-nodes[i].ledger.Wait(initialRound + tests): @@ -387,10 +452,13 @@ func TestSimpleUpgrade(t *testing.T) { require.Equal(t, 2, roundsCheckedForUpgrade) } -func startAndConnectNodes(nodes []*AlgorandFullNode, delayStartFirstNode bool) { +const defaultFirstNodeStartDelay = 20 * time.Second +const nodelayFirstNodeStartDelay = 0 + +func startAndConnectNodes(nodes []*AlgorandFullNode, delayStartFirstNode time.Duration) { var wg sync.WaitGroup for i := range nodes { - if delayStartFirstNode && i == 0 { + if delayStartFirstNode != nodelayFirstNodeStartDelay && i == 0 { continue } wg.Add(1) @@ -401,9 +469,9 @@ func startAndConnectNodes(nodes []*AlgorandFullNode, delayStartFirstNode bool) { } wg.Wait() - if delayStartFirstNode { + if delayStartFirstNode != nodelayFirstNodeStartDelay { connectPeers(nodes[1:]) - delayStartNode(nodes[0], nodes[1:], 20*time.Second) + delayStartNode(nodes[0], nodes[1:], delayStartFirstNode) } else { connectPeers(nodes) } @@ -754,3 +822,208 @@ func TestMaxSizesCorrect(t *testing.T) { tsSize := uint64(network.MaxMessageLength) require.Equal(t, tsSize, protocol.TopicMsgRespTag.MaxMessageSize()) } + +// TestNodeHybridTopology set ups 3 nodes network with the following topology: +// N -- R -- A and ensures N can discover A and download blocks from it. +// +// N is a non-part node that joins the network later +// R is a non-archival relay node with block service disabled. It MUST NOT service blocks to force N to discover A. +// A is a archival node that can only provide blocks. +// Nodes N and A have only R in their initial phonebook, and all nodes are in hybrid mode. +func TestNodeHybridTopology(t *testing.T) { + partitiontest.PartitionTest(t) + + const consensusTest0 = protocol.ConsensusVersion("test0") + + configurableConsensus := make(config.ConsensusProtocols) + + testParams0 := config.Consensus[protocol.ConsensusCurrentVersion] + testParams0.AgreementFilterTimeoutPeriod0 = 500 * time.Millisecond + configurableConsensus[consensusTest0] = testParams0 + + minMoneyAtStart := 1_000_000 + maxMoneyAtStart := 100_000_000_000 + gen := rand.New(rand.NewSource(2)) + + const numAccounts = 3 + acctStake := make([]basics.MicroAlgos, numAccounts) + for i := range acctStake { + acctStake[i] = basics.MicroAlgos{Raw: uint64(minMoneyAtStart + (gen.Int() % (maxMoneyAtStart - minMoneyAtStart)))} + } + acctStake[0] = basics.MicroAlgos{} // no stake at node 0 + + configHook := func(ni nodeInfo, cfg config.Local) (nodeInfo, config.Local) { + cfg = config.GetDefaultLocal() + if ni.idx != 2 { + cfg.EnableBlockService = false + cfg.EnableGossipBlockService = false + cfg.EnableLedgerService = false + cfg.CatchpointInterval = 0 + cfg.Archival = false + } else { + // node 2 is archival + cfg.EnableBlockService = true + cfg.EnableGossipBlockService = true + cfg.EnableLedgerService = true + cfg.CatchpointInterval = 200 + cfg.Archival = true + } + if ni.idx == 0 { + // do not allow node 0 (N) to make any outgoing connections + cfg.GossipFanout = 0 + } + + cfg.NetAddress = ni.wsNetAddr() + cfg.EnableP2PHybridMode = true + cfg.EnableDHTProviders = true + cfg.P2PPersistPeerID = true + privKey, err := p2p.GetPrivKey(cfg, ni.rootDir) + require.NoError(t, err) + ni.p2pID, err = p2p.PeerIDFromPublicKey(privKey.GetPublic()) + require.NoError(t, err) + + cfg.P2PNetAddress = ni.p2pNetAddr() + return ni, cfg + } + + phonebookHook := func(ni []nodeInfo, i int) []string { + switch i { + case 0: + // node 0 (N) only accept connections at the beginning to learn about archival node from DHT + t.Logf("Node%d phonebook: empty", i) + return []string{} + case 1: + // node 1 (R) connects to all + t.Logf("Node%d phonebook: %s, %s, %s, %s", i, ni[0].wsNetAddr(), ni[2].wsNetAddr(), ni[0].p2pMultiAddr(), ni[2].p2pMultiAddr()) + return []string{ni[0].wsNetAddr(), ni[2].wsNetAddr(), ni[0].p2pMultiAddr(), ni[2].p2pMultiAddr()} + case 2: + // node 2 (A) connects to R + t.Logf("Node%d phonebook: %s, %s", i, ni[1].wsNetAddr(), ni[1].p2pMultiAddr()) + return []string{ni[1].wsNetAddr(), ni[1].p2pMultiAddr()} + default: + t.Errorf("not expected number of nodes: %d", i) + t.FailNow() + } + return nil + } + + nodes, wallets := setupFullNodesEx(t, consensusTest0, configurableConsensus, acctStake, configHook, phonebookHook) + require.Len(t, nodes, 3) + require.Len(t, wallets, 3) + for i := 0; i < len(nodes); i++ { + defer os.Remove(wallets[i]) + defer nodes[i].Stop() + } + + startAndConnectNodes(nodes, 10*time.Second) + + initialRound := nodes[0].ledger.NextRound() + targetRound := initialRound + 10 + + select { + case <-nodes[0].ledger.Wait(targetRound): + e0, err := nodes[0].ledger.Block(targetRound) + require.NoError(t, err) + e1, err := nodes[1].ledger.Block(targetRound) + require.NoError(t, err) + require.Equal(t, e1.Hash(), e0.Hash()) + case <-time.After(120 * time.Second): + require.Fail(t, fmt.Sprintf("no block notification for wallet: %v.", wallets[0])) + } +} + +// TestNodeP2PRelays creates a network of 3 nodes with the following topology: +// R1 (relay, DHT) -> R2 (relay, phonebook) <- N (part node) +// Expect N to discover R1 via DHT and connect to it. +func TestNodeP2PRelays(t *testing.T) { + partitiontest.PartitionTest(t) + + const consensusTest0 = protocol.ConsensusVersion("test0") + + configurableConsensus := make(config.ConsensusProtocols) + + testParams0 := config.Consensus[protocol.ConsensusCurrentVersion] + testParams0.AgreementFilterTimeoutPeriod0 = 500 * time.Millisecond + configurableConsensus[consensusTest0] = testParams0 + + minMoneyAtStart := 1_000_000 + maxMoneyAtStart := 100_000_000_000 + gen := rand.New(rand.NewSource(2)) + + const numAccounts = 3 + acctStake := make([]basics.MicroAlgos, numAccounts) + // only node N has stake + acctStake[2] = basics.MicroAlgos{Raw: uint64(minMoneyAtStart + (gen.Int() % (maxMoneyAtStart - minMoneyAtStart)))} + + configHook := func(ni nodeInfo, cfg config.Local) (nodeInfo, config.Local) { + cfg = config.GetDefaultLocal() + cfg.BaseLoggerDebugLevel = uint32(logging.Debug) + cfg.EnableP2P = true + cfg.NetAddress = "" + cfg.EnableDHTProviders = true + + cfg.P2PPersistPeerID = true + privKey, err := p2p.GetPrivKey(cfg, ni.rootDir) + require.NoError(t, err) + ni.p2pID, err = p2p.PeerIDFromPublicKey(privKey.GetPublic()) + require.NoError(t, err) + + switch ni.idx { + case 2: + // N is not a relay + default: + cfg.NetAddress = ni.p2pNetAddr() + } + return ni, cfg + } + + phonebookHook := func(ni []nodeInfo, i int) []string { + switch i { + case 0: + // node R1 connects to R2 + t.Logf("Node%d phonebook: %s", i, ni[1].p2pMultiAddr()) + return []string{ni[1].p2pMultiAddr()} + case 1: + // node R2 connects to none one + t.Logf("Node%d phonebook: empty", i) + return []string{} + case 2: + // node N only connects to R1 + t.Logf("Node%d phonebook: %s", i, ni[1].p2pMultiAddr()) + return []string{ni[1].p2pMultiAddr()} + default: + t.Errorf("not expected number of nodes: %d", i) + t.FailNow() + } + return nil + } + + nodes, wallets := setupFullNodesEx(t, consensusTest0, configurableConsensus, acctStake, configHook, phonebookHook) + require.Len(t, nodes, 3) + require.Len(t, wallets, 3) + for i := 0; i < len(nodes); i++ { + defer os.Remove(wallets[i]) + defer nodes[i].Stop() + } + + startAndConnectNodes(nodes, nodelayFirstNodeStartDelay) + + require.Eventually(t, func() bool { + connectPeers(nodes) + + // since p2p open streams based on peer ID, there is no way to judge + // connectivity based on exact In/Out so count both + return len(nodes[0].net.GetPeers(network.PeersConnectedIn, network.PeersConnectedOut)) >= 1 && + len(nodes[1].net.GetPeers(network.PeersConnectedIn, network.PeersConnectedOut)) >= 2 && + len(nodes[2].net.GetPeers(network.PeersConnectedIn, network.PeersConnectedOut)) >= 1 + }, 60*time.Second, 1*time.Second) + + t.Log("Nodes connected to R2") + + // wait until N gets R1 in its phonebook + require.Eventually(t, func() bool { + // refresh N's peers in order to learn DHT data faster + nodes[2].net.RequestConnectOutgoing(false, nil) + return len(nodes[2].net.GetPeers(network.PeersPhonebookRelays)) == 2 + }, 80*time.Second, 1*time.Second) +} diff --git a/rpcs/blockService.go b/rpcs/blockService.go index d1ef82dfd4..1a9893b70a 100644 --- a/rpcs/blockService.go +++ b/rpcs/blockService.go @@ -41,6 +41,7 @@ import ( "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" + "github.com/algorand/go-algorand/network/addr" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/util/metrics" ) @@ -381,7 +382,6 @@ func (bs *BlockService) handleCatchupReq(ctx context.Context, reqMsg network.Inc return } respTopics, n = topicBlockBytes(bs.log, bs.ledger, basics.Round(round), string(requestType)) - return } // redirectRequest redirects the request to the next round robin fallback endpoint if available @@ -392,18 +392,24 @@ func (bs *BlockService) redirectRequest(round uint64, response http.ResponseWrit return false } - parsedURL, err := network.ParseHostOrURL(peerAddress) - if err != nil { - bs.log.Debugf("redirectRequest: %s", err.Error()) - return false + var redirectURL string + if addr.IsMultiaddr(peerAddress) { + redirectURL = strings.Replace(FormatBlockQuery(round, "", bs.net), "{genesisID}", bs.genesisID, 1) + } else { + parsedURL, err := addr.ParseHostOrURL(peerAddress) + if err != nil { + bs.log.Debugf("redirectRequest: %s", err.Error()) + return false + } + parsedURL.Path = strings.Replace(FormatBlockQuery(round, parsedURL.Path, bs.net), "{genesisID}", bs.genesisID, 1) + redirectURL = parsedURL.String() } - parsedURL.Path = strings.Replace(FormatBlockQuery(round, parsedURL.Path, bs.net), "{genesisID}", bs.genesisID, 1) - http.Redirect(response, request, parsedURL.String(), http.StatusTemporaryRedirect) - bs.log.Debugf("redirectRequest: redirected block request to %s", parsedURL.String()) + http.Redirect(response, request, redirectURL, http.StatusTemporaryRedirect) + bs.log.Debugf("redirectRequest: redirected block request to %s", redirectURL) return true } -// getNextCustomFallbackEndpoint returns the next custorm fallback endpoint in RR ordering +// getNextCustomFallbackEndpoint returns the next custom fallback endpoint in RR ordering func (bs *BlockService) getNextCustomFallbackEndpoint() (endpointAddress string) { if len(bs.fallbackEndpoints.endpoints) == 0 { return @@ -487,7 +493,7 @@ func RawBlockBytes(l LedgerForBlockService, round basics.Round) ([]byte, error) // FormatBlockQuery formats a block request query for the given network and round number func FormatBlockQuery(round uint64, parsedURL string, net network.GossipNode) string { - return net.SubstituteGenesisID(path.Join(parsedURL, "/v1/{genesisID}/block/"+strconv.FormatUint(uint64(round), 36))) + return network.SubstituteGenesisID(net, path.Join(parsedURL, "/v1/{genesisID}/block/"+strconv.FormatUint(uint64(round), 36))) } func makeFallbackEndpoints(log logging.Logger, customFallbackEndpoints string) (fe fallbackEndpoints) { @@ -496,12 +502,16 @@ func makeFallbackEndpoints(log logging.Logger, customFallbackEndpoints string) ( } endpoints := strings.Split(customFallbackEndpoints, ",") for _, ep := range endpoints { - parsed, err := network.ParseHostOrURL(ep) - if err != nil { - log.Warnf("makeFallbackEndpoints: error parsing %s %s", ep, err.Error()) - continue + if addr.IsMultiaddr(ep) { + fe.endpoints = append(fe.endpoints, ep) + } else { + parsed, err := addr.ParseHostOrURL(ep) + if err != nil { + log.Warnf("makeFallbackEndpoints: error parsing %s %s", ep, err.Error()) + continue + } + fe.endpoints = append(fe.endpoints, parsed.String()) } - fe.endpoints = append(fe.endpoints, parsed.String()) } return } diff --git a/rpcs/blockService_test.go b/rpcs/blockService_test.go index 7b1f756f08..e637796adf 100644 --- a/rpcs/blockService_test.go +++ b/rpcs/blockService_test.go @@ -23,6 +23,7 @@ import ( "fmt" "io" "net/http" + "net/http/httptest" "strings" "sync" "testing" @@ -38,6 +39,7 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" + "github.com/algorand/go-algorand/network/addr" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" ) @@ -70,6 +72,10 @@ func (mup *mockUnicastPeer) Respond(ctx context.Context, reqMsg network.Incoming return nil } +func (mup *mockUnicastPeer) GetNetwork() network.GossipNode { + panic("not implemented") +} + // TestHandleCatchupReqNegative covers the error reporting in handleCatchupReq func TestHandleCatchupReqNegative(t *testing.T) { partitiontest.PartitionTest(t) @@ -142,6 +148,8 @@ func TestRedirectFallbackEndpoints(t *testing.T) { net1 := &httpTestPeerSource{} net2 := &httpTestPeerSource{} + net1.GenesisID = "test-genesis-ID" + net2.GenesisID = "test-genesis-ID" nodeA := &basicRPCNode{} nodeB := &basicRPCNode{} @@ -161,7 +169,7 @@ func TestRedirectFallbackEndpoints(t *testing.T) { nodeA.RegisterHTTPHandler(BlockServiceBlockPath, bs1) nodeB.RegisterHTTPHandler(BlockServiceBlockPath, bs2) - parsedURL, err := network.ParseHostOrURL(nodeA.rootURL()) + parsedURL, err := addr.ParseHostOrURL(nodeA.rootURL()) require.NoError(t, err) client := http.Client{} @@ -206,7 +214,7 @@ func TestBlockServiceShutdown(t *testing.T) { nodeA.start() defer nodeA.stop() - parsedURL, err := network.ParseHostOrURL(nodeA.rootURL()) + parsedURL, err := addr.ParseHostOrURL(nodeA.rootURL()) require.NoError(t, err) client := http.Client{} @@ -259,6 +267,8 @@ func TestRedirectOnFullCapacity(t *testing.T) { net1 := &httpTestPeerSource{} net2 := &httpTestPeerSource{} + net1.GenesisID = "test-genesis-ID" + net2.GenesisID = "test-genesis-ID" nodeA := &basicRPCNode{} nodeB := &basicRPCNode{} @@ -286,7 +296,7 @@ func TestRedirectOnFullCapacity(t *testing.T) { nodeB.RegisterHTTPHandler(BlockServiceBlockPath, bs2) - parsedURL, err := network.ParseHostOrURL(nodeA.rootURL()) + parsedURL, err := addr.ParseHostOrURL(nodeA.rootURL()) require.NoError(t, err) client := http.Client{} @@ -445,6 +455,8 @@ func TestRedirectExceptions(t *testing.T) { net1 := &httpTestPeerSource{} net2 := &httpTestPeerSource{} + net1.GenesisID = "{genesisID}" + net2.GenesisID = "{genesisID}" nodeA := &basicRPCNode{} nodeB := &basicRPCNode{} @@ -465,7 +477,7 @@ func TestRedirectExceptions(t *testing.T) { nodeA.RegisterHTTPHandler(BlockServiceBlockPath, bs1) nodeB.RegisterHTTPHandler(BlockServiceBlockPath, bs2) - parsedURL, err := network.ParseHostOrURL(nodeA.rootURL()) + parsedURL, err := addr.ParseHostOrURL(nodeA.rootURL()) require.NoError(t, err) client := http.Client{} @@ -484,12 +496,13 @@ func TestRedirectExceptions(t *testing.T) { require.NoError(t, err) require.Equal(t, response.StatusCode, http.StatusNotFound) - parsedURLNodeB, err := network.ParseHostOrURL(nodeB.rootURL()) + parsedURLNodeB, err := addr.ParseHostOrURL(nodeB.rootURL()) require.NoError(t, err) parsedURLNodeB.Path = FormatBlockQuery(uint64(4), parsedURLNodeB.Path, net2) blockURLNodeB := parsedURLNodeB.String() requestNodeB, err := http.NewRequest("GET", blockURLNodeB, nil) + require.NoError(t, err) _, err = client.Do(requestNodeB) require.Error(t, err) @@ -548,8 +561,45 @@ func addBlock(t *testing.T, ledger *data.Ledger) (timestamp int64) { func TestErrMemoryAtCapacity(t *testing.T) { partitiontest.PartitionTest(t) + t.Parallel() macError := errMemoryAtCapacity{capacity: uint64(100), used: uint64(110)} errStr := macError.Error() require.Equal(t, "block service memory over capacity: 110 / 100", errStr) } + +func TestBlockServiceRedirect(t *testing.T) { + partitiontest.PartitionTest(t) + t.Parallel() + + log := logging.TestingLog(t) + + ep1 := "http://localhost:1234" + ep2 := "/ip4/127.0.0.1/tcp/2345/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN" + endpoints := strings.Join([]string{ep1, ep2}, ",") + fb := makeFallbackEndpoints(log, endpoints) + require.Len(t, fb.endpoints, 2) + require.Equal(t, ep1, fb.endpoints[0]) + require.Equal(t, ep2, fb.endpoints[1]) + + bs := BlockService{ + net: &httpTestPeerSource{}, + fallbackEndpoints: fb, + log: log, + } + + r := httptest.NewRequest("GET", "/", strings.NewReader("")) + w := httptest.NewRecorder() + ok := bs.redirectRequest(10, w, r) + require.True(t, ok) + expectedPath := ep1 + FormatBlockQuery(10, "/", bs.net) + require.Equal(t, expectedPath, w.Result().Header.Get("Location")) + + r = httptest.NewRequest("GET", "/", strings.NewReader("")) + w = httptest.NewRecorder() + ok = bs.redirectRequest(11, w, r) + require.True(t, ok) + // for p2p nodes the url is actually a peer address in p2p network and not part of HTTP path + expectedPath = FormatBlockQuery(11, "", bs.net) + require.Equal(t, expectedPath, w.Result().Header.Get("Location")) +} diff --git a/rpcs/healthService_test.go b/rpcs/healthService_test.go index f2846c322d..c1153d98ba 100644 --- a/rpcs/healthService_test.go +++ b/rpcs/healthService_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/algorand/go-algorand/network" + "github.com/algorand/go-algorand/network/addr" "github.com/algorand/go-algorand/test/partitiontest" "github.com/stretchr/testify/require" ) @@ -36,7 +37,7 @@ func TestHealthService_ServeHTTP(t *testing.T) { _ = MakeHealthService(nodeA) - parsedURL, err := network.ParseHostOrURL(nodeA.rootURL()) + parsedURL, err := addr.ParseHostOrURL(nodeA.rootURL()) require.NoError(t, err) client := http.Client{} diff --git a/rpcs/httpTxSync.go b/rpcs/httpTxSync.go index 43258337cb..311a87cf7b 100644 --- a/rpcs/httpTxSync.go +++ b/rpcs/httpTxSync.go @@ -24,7 +24,6 @@ import ( "math/rand" "net/http" "net/url" - "path" "strings" "github.com/algorand/go-algorand/data/bookkeeping" @@ -103,19 +102,18 @@ func (hts *HTTPTxSync) Sync(ctx context.Context, bloom *bloom.Filter) (txgroups if !ok { return nil, fmt.Errorf("cannot HTTPTxSync non http peer %T %#v", peer, peer) } + var syncURL string hts.rootURL = hpeer.GetAddress() + client := hpeer.GetHTTPClient() if client == nil { - client = &http.Client{} - client.Transport = hts.peers.GetRoundTripper() - } - parsedURL, err := network.ParseHostOrURL(hts.rootURL) - if err != nil { - hts.log.Warnf("txSync bad url %v: %s", hts.rootURL, err) - return nil, err + client, err = hts.peers.GetHTTPClient(hts.rootURL) + if err != nil { + return nil, fmt.Errorf("HTTPTxSync cannot create a HTTP client for a peer %T %#v: %s", peer, peer, err.Error()) + } } - parsedURL.Path = hts.peers.SubstituteGenesisID(path.Join(parsedURL.Path, TxServiceHTTPPath)) - syncURL := parsedURL.String() + syncURL = network.SubstituteGenesisID(hts.peers, TxServiceHTTPPath) + hts.log.Infof("http sync from %s", syncURL) params := url.Values{} params.Set("bf", bloomParam) diff --git a/rpcs/ledgerService.go b/rpcs/ledgerService.go index 5f75a36b0c..823895a417 100644 --- a/rpcs/ledgerService.go +++ b/rpcs/ledgerService.go @@ -60,19 +60,25 @@ type LedgerForService interface { GetCatchpointStream(round basics.Round) (ledger.ReadCloseSizer, error) } +// httpGossipNode is a reduced interface for the gossipNode that only includes the methods needed by the LedgerService +type httpGossipNode interface { + RegisterHTTPHandler(path string, handler http.Handler) + GetHTTPRequestConnection(request *http.Request) (conn network.DeadlineSettableConn) +} + // LedgerService represents the Ledger RPC API type LedgerService struct { // running is non-zero once the service is running, and zero when it's not running. it needs to be at a 32-bit aligned address for RasPI support. running atomic.Int32 ledger LedgerForService genesisID string - net network.GossipNode + net httpGossipNode enableService bool stopping sync.WaitGroup } // MakeLedgerService creates a LedgerService around the provider Ledger and registers it with the HTTP router -func MakeLedgerService(config config.Local, ledger LedgerForService, net network.GossipNode, genesisID string) *LedgerService { +func MakeLedgerService(config config.Local, ledger LedgerForService, net httpGossipNode, genesisID string) *LedgerService { service := &LedgerService{ ledger: ledger, genesisID: genesisID, diff --git a/rpcs/ledgerService_test.go b/rpcs/ledgerService_test.go index 1cc52fc9c0..a100f03c2b 100644 --- a/rpcs/ledgerService_test.go +++ b/rpcs/ledgerService_test.go @@ -17,6 +17,9 @@ package rpcs import ( + "archive/tar" + "bytes" + "compress/gzip" "fmt" "io" "net/http" @@ -172,3 +175,59 @@ func TestLedgerService(t *testing.T) { ledgerService.Stop() require.Equal(t, int32(0), ledgerService.running.Load()) } + +type mockSizedStream struct { + *bytes.Buffer +} + +func (mss mockSizedStream) Size() (int64, error) { + return int64(mss.Len()), nil +} + +func (mss mockSizedStream) Close() error { + return nil +} + +type mockLedgerForService struct { +} + +func (l *mockLedgerForService) GetCatchpointStream(round basics.Round) (ledger.ReadCloseSizer, error) { + buf := bytes.NewBuffer(nil) + gz := gzip.NewWriter(buf) + wtar := tar.NewWriter(gz) + wtar.Close() + gz.Close() + + buf2 := bytes.NewBuffer(buf.Bytes()) + return mockSizedStream{buf2}, nil +} + +// TestLedgerServiceP2P creates a ledger service on a node, and a p2p client tries to download +// an empty catchpoint file from the ledger service. +func TestLedgerServiceP2P(t *testing.T) { + partitiontest.PartitionTest(t) + + nodeA, nodeB := nodePairP2p(t) + defer nodeA.Stop() + defer nodeB.Stop() + + genesisID := "test GenesisID" + cfg := config.GetDefaultLocal() + cfg.EnableLedgerService = true + l := mockLedgerForService{} + ledgerService := MakeLedgerService(cfg, &l, nodeA, genesisID) + ledgerService.Start() + defer ledgerService.Stop() + + nodeA.RegisterHTTPHandler(LedgerServiceLedgerPath, ledgerService) + + httpPeer := nodeA.GetHTTPPeer().(network.HTTPPeer) + + req, err := http.NewRequest("GET", fmt.Sprintf("/v1/%s/ledger/0", genesisID), nil) + require.NoError(t, err) + resp, err := httpPeer.GetHTTPClient().Do(req) + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + + require.Equal(t, http.StatusOK, resp.StatusCode) +} diff --git a/rpcs/txService_test.go b/rpcs/txService_test.go index 1e7a5036e4..fcfae1044d 100644 --- a/rpcs/txService_test.go +++ b/rpcs/txService_test.go @@ -22,7 +22,6 @@ import ( "net/http" "net/url" "os" - "strings" "sync" "testing" "time" @@ -34,6 +33,7 @@ import ( "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" + p2ptesting "github.com/algorand/go-algorand/network/p2p/testing" "github.com/algorand/go-algorand/test/partitiontest" "github.com/algorand/go-algorand/util/bloom" ) @@ -64,7 +64,11 @@ func (p testHTTPPeer) GetAddress() string { return string(p) } func (p *testHTTPPeer) GetHTTPClient() *http.Client { - return &http.Client{} + return &http.Client{ + Transport: &network.HTTPPAddressBoundTransport{ + Addr: p.GetAddress(), + InnerTransport: http.DefaultTransport}, + } } func (p *testHTTPPeer) GetHTTPPeer() network.HTTPPeer { return p @@ -116,9 +120,7 @@ func (b *basicRPCNode) GetPeers(options ...network.PeerOption) []network.Peer { return b.peers } -func (b *basicRPCNode) SubstituteGenesisID(rawURL string) string { - return strings.Replace(rawURL, "{genesisID}", "test genesisID", -1) -} +func (b *basicRPCNode) GetGenesisID() string { return "test genesisID" } func nodePair() (*basicRPCNode, *basicRPCNode) { nodeA := &basicRPCNode{} @@ -132,27 +134,84 @@ func nodePair() (*basicRPCNode, *basicRPCNode) { return nodeA, nodeB } +func nodePairP2p(tb testing.TB) (*p2ptesting.HTTPNode, *p2ptesting.HTTPNode) { + nodeA := p2ptesting.MakeHTTPNode(tb) + addrsA := nodeA.Addrs() + require.Greater(tb, len(addrsA), 0) + + nodeB := p2ptesting.MakeHTTPNode(tb) + addrsB := nodeA.Addrs() + require.Greater(tb, len(addrsB), 0) + + nodeA.SetPeers(nodeB) + nodeB.SetPeers(nodeA) + nodeA.SetGenesisID("test genesisID") + nodeB.SetGenesisID("test genesisID") + + nodeA.Start() + nodeB.Start() + + return nodeA, nodeB +} + +// TestTxSync checks txsync on a network with two nodes, A and B func TestTxSync(t *testing.T) { partitiontest.PartitionTest(t) - // A network with two nodes, A and B - nodeA, nodeB := nodePair() - defer nodeA.stop() - defer nodeB.stop() + type txSyncNode interface { + Registrar + network.GossipNode + } - pool := makeMockPendingTxAggregate(3) - RegisterTxService(pool, nodeA, "test genesisID", config.GetDefaultLocal().TxPoolSize, config.GetDefaultLocal().TxSyncServeResponseSize) + tests := []struct { + name string + setup func(t *testing.T) (txSyncNode, txSyncNode, func()) + }{ + { + name: "tcp", + setup: func(t *testing.T) (txSyncNode, txSyncNode, func()) { + nodeA, nodeB := nodePair() + cleanup := func() { + nodeA.stop() + nodeB.stop() + } + return nodeA, nodeB, cleanup + }, + }, + { + name: "p2p", + setup: func(t *testing.T) (txSyncNode, txSyncNode, func()) { + nodeA, nodeB := nodePairP2p(t) + cleanup := func() { + nodeA.Stop() + nodeB.Stop() + } + return nodeA, nodeB, cleanup + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // A network with two nodes, A and B + nodeA, nodeB, cleanupFn := test.setup(t) + defer cleanupFn() - // B tries to fetch block - handler := mockHandler{} - syncInterval := time.Second - syncTimeout := time.Second - syncerPool := makeMockPendingTxAggregate(0) - syncer := MakeTxSyncer(syncerPool, nodeB, &handler, syncInterval, syncTimeout, config.GetDefaultLocal().TxSyncServeResponseSize) - // Since syncer is not Started, set the context here - syncer.ctx, syncer.cancel = context.WithCancel(context.Background()) - require.NoError(t, syncer.sync()) - require.Equal(t, int32(3), handler.messageCounter.Load()) + pool := makeMockPendingTxAggregate(3) + RegisterTxService(pool, nodeA, "test genesisID", config.GetDefaultLocal().TxPoolSize, config.GetDefaultLocal().TxSyncServeResponseSize) + + // B tries to fetch block + handler := mockHandler{} + syncInterval := time.Second + syncTimeout := time.Second + syncerPool := makeMockPendingTxAggregate(0) + syncer := MakeTxSyncer(syncerPool, nodeB, &handler, syncInterval, syncTimeout, config.GetDefaultLocal().TxSyncServeResponseSize) + // Since syncer is not Started, set the context here + syncer.ctx, syncer.cancel = context.WithCancel(context.Background()) + require.NoError(t, syncer.sync()) + require.Equal(t, int32(3), handler.messageCounter.Load()) + }) + } } func BenchmarkTxSync(b *testing.B) { diff --git a/rpcs/txSyncer_test.go b/rpcs/txSyncer_test.go index 1c3f071879..eb3e4eab60 100644 --- a/rpcs/txSyncer_test.go +++ b/rpcs/txSyncer_test.go @@ -22,7 +22,6 @@ import ( "math/rand" "net/http" "net/rpc" - "strings" "sync/atomic" "testing" "time" @@ -158,6 +157,7 @@ func (client *mockRPCClient) Sync(ctx context.Context, bloom *bloom.Filter) (txg func (client *mockRPCClient) GetAddress() string { return client.rootURL } + func (client *mockRPCClient) GetHTTPClient() *http.Client { return nil } @@ -170,20 +170,13 @@ type mockClientAggregator struct { func (mca *mockClientAggregator) GetPeers(options ...network.PeerOption) []network.Peer { return mca.peers } -func (mca *mockClientAggregator) SubstituteGenesisID(rawURL string) string { - return strings.Replace(rawURL, "{genesisID}", "test genesisID", -1) -} - -const numberOfPeers = 10 -func makeMockClientAggregator(t *testing.T, failWithNil bool, failWithError bool) *mockClientAggregator { - clients := make([]network.Peer, 0) - for i := 0; i < numberOfPeers; i++ { - runner := mockRunner{failWithNil: failWithNil, failWithError: failWithError, done: make(chan *rpc.Call)} - clients = append(clients, &mockRPCClient{client: &runner, log: logging.TestingLog(t)}) - } - t.Logf("len(mca.clients) = %d", len(clients)) - return &mockClientAggregator{peers: clients} +func (mca *mockClientAggregator) GetHTTPClient(address string) (*http.Client, error) { + return &http.Client{ + Transport: &network.HTTPPAddressBoundTransport{ + Addr: address, + InnerTransport: http.DefaultTransport}, + }, nil } func TestSyncFromClient(t *testing.T) { @@ -283,7 +276,7 @@ func TestSync(t *testing.T) { runner := mockRunner{failWithNil: false, failWithError: false, txgroups: pool.PendingTxGroups()[len(pool.PendingTxGroups())-1:], done: make(chan *rpc.Call)} client := mockRPCClient{client: &runner, rootURL: nodeAURL, log: logging.TestingLog(t)} - clientAgg := mockClientAggregator{peers: []network.Peer{&client}} + clientAgg := mockClientAggregator{peers: []network.Peer{&client}, MockNetwork: mocks.MockNetwork{GenesisID: "test genesisID"}} handler := mockHandler{} syncerPool := makeMockPendingTxAggregate(3) syncer := MakeTxSyncer(syncerPool, &clientAgg, &handler, testSyncInterval, testSyncTimeout, config.GetDefaultLocal().TxSyncServeResponseSize) @@ -322,7 +315,7 @@ func TestStartAndStop(t *testing.T) { runner := mockRunner{failWithNil: false, failWithError: false, txgroups: pool.PendingTxGroups()[len(pool.PendingTxGroups())-1:], done: make(chan *rpc.Call)} client := mockRPCClient{client: &runner, rootURL: nodeAURL, log: logging.TestingLog(t)} - clientAgg := mockClientAggregator{peers: []network.Peer{&client}} + clientAgg := mockClientAggregator{peers: []network.Peer{&client}, MockNetwork: mocks.MockNetwork{GenesisID: "test genesisID"}} handler := mockHandler{} syncerPool := makeMockPendingTxAggregate(0) diff --git a/test/heapwatch/agreement-log.py b/test/heapwatch/agreement-log.py new file mode 100644 index 0000000000..4109b37a71 --- /dev/null +++ b/test/heapwatch/agreement-log.py @@ -0,0 +1,187 @@ +""" +Agreement logs parser, takes either separate node.log files from a directory and guessing names from the file names, +or parses the e2e test failure log file watching for node names as " libgoalFixture.go:376: Relay0/node.log:" strings. + +This tool similar a bit to carpenter but takes multiple log files at once. +To force colors when outputting to a file, set FORCE_COLOR=1 in the environment. +""" + +import argparse +from datetime import datetime, timedelta +import glob +import json +import logging +import os +import time + +from termcolor import COLORS, colored + +logger = logging.getLogger(__name__) + +filtered_events = frozenset(['Persisted']) + +def process_json_line(line: str, node_name: str, by_node: dict, events: list): + """Handles a single line of json log file, returns parsed event or None if it's not an agreement event. + + line is a single line of json log file. + node_name is a name of the node that produced this line. + by_node is dict with unique nodes meta information. + events is a list of all parsed events. It is appended in this function to keep the caller code clean. + """ + try: + evt = json.loads(line) + except json.JSONDecodeError: + logger.error('failed to parse json: %s', line) + return None + if evt.get('Context') == 'Agreement' and evt.get('Type'): + if evt['Type'] in filtered_events: + return None + dt = datetime.strptime(evt['time'], '%Y-%m-%dT%H:%M:%S.%f%z') + sender = evt.get('Sender') + sender = sender[:12] if sender else '' + h = evt.get('Hash') + h = h[:8] if h else '' + w = evt.get('Weight', '-') if not evt['Type'].startswith('Proposal') else ' ' + wt = evt.get('WeightTotal', '-') if not evt['Type'].startswith('Proposal') else ' ' + if evt['Type'] in ('StepTimeout', 'VoteAttest', 'BlockAssembled', 'BlockPipelined'): + w, wt = ' ', ' ' + result = { + 'time': dt, + 'type': evt.get('Type'), + 'round': evt.get('Round', '-'), + 'period': evt.get('Period', '-'), + 'step': evt.get('Step', '-'), + 'object_round': evt.get('ObjectRound', '-'), + 'object_period': evt.get('ObjectPeriod', '-'), + 'object_step': evt.get('ObjectStep', '-'), + 'hash': h, + 'sender': sender, + 'weight': w, + 'weight_total': wt, + 'node': node_name, + } + events.append(result) + metadata = by_node.get(node_name) + if not metadata: + metadata = { + 'type': evt.get('Type'), + 'time': dt + } + by_node[node_name] = metadata + else: + if evt.get('Type') == 'RoundConcluded': + rt = dt - metadata['time'] + result['round_time_ms'] = rt / timedelta(milliseconds=1) + elif evt.get('Type') == 'RoundStart': + metadata['time'] = dt + metadata['type'] = 'RoundStart' + by_node[node_name] = metadata + + return result + return None + +def main(): + os.environ['TZ'] = 'UTC' + time.tzset() + + ap = argparse.ArgumentParser() + ap.add_argument('test_log_or_dir', help='Dir with log files or a single log file from e2e tests') + ap.add_argument('-e', '--end-round', type=int, help=f'Round to end at') + args = ap.parse_args() + + by_node = {} + events = [] + if os.path.isdir(args.test_log_or_dir): + logger.info('processing directory %s', args.test_log_or_dir) + log_files = sorted(glob.glob(os.path.join(args.test_log_or_dir, '*-node.log'))) + if not log_files: + logger.error('no log files found in %s', args.test_log_or_dir) + return 1 + for filename in os.listdir(args.test_log_or_dir): + if filename.endswith("-node.log"): + with open(os.path.join(args.test_log_or_dir, filename), 'r') as file: + node_name = filename[:len(filename) - len('-node.log')] + node_name = node_name.replace('relay', 'R') + node_name = node_name.replace('nonParticipatingNode', 'NPN') + node_name = node_name.replace('node', 'N') + for line in file: + event = process_json_line(line, node_name, by_node, events) + if event and args.end_round and \ + isinstance(event['round'], int) and event['round'] >= args.end_round: + break + + else: + logger.info('processing file %s', args.test_log_or_dir) + with open(args.test_log_or_dir, 'r') as file: + line0 = None + while not line0: + line0 = file.readline() + line0 = line0.strip() + + if line0[0] == '{': + # regular json line + node_name = 'node' + process_json_line(line, node_name, by_node, events) + for line in file: + line = line.strip() + event = process_json_line(line, node_name, by_node, events) + if event and args.end_round and \ + isinstance(event['round'], int) and event['round'] >= args.end_round: + break + else: + # looks like e2e test output with lines line this: + """ + libgoalFixture.go:374: ===================... + libgoalFixture.go:376: Relay0/node.log: + libgoalFixture.go:379: {"file":"server.go"... + """ + node_name = None + if line0.endswith('node.log:'): + node_name = line0.split(' ')[1].split('/')[0] + logger.info('found node name: %s', node_name) + for line in file: + line = line.strip() + if line.endswith('node.log:'): + node_name = line.split(' ')[1].split('/')[0] + logger.info('found node name: %s', node_name) + if node_name: + for line in file: + json_start = line.find('{') + if json_start == -1: + # end of continuous json block + node_name = None + break + line = line[json_start:] + event = process_json_line(line, node_name, by_node, events) + if event and args.end_round and \ + isinstance(event['round'], int) and event['round'] >= args.end_round: + break + + log = sorted(events, key=lambda x: x['time']) + + # num_nodes = len(by_node) + colors = list(COLORS) + colors = colors[colors.index('light_grey'):] + if len(colors) < len(by_node): + colors = colors * (len(by_node) // len(colors) + 1) + node_color = {k: v for k, v in zip(by_node.keys(), colors)} + + fmt = '%15s (%s,%s,%s) (%s,%s,%s) %4s|%-4s %-8s %-18s %8s %12s %5s' + print(fmt % ('TS', 'R', 'P', 'S', 'r', 'p', 's', 'W', 'WT', 'NODE', 'EVENT TYPE', 'HASH', 'SENDER', 'RT ms')) + for e in log: + color = node_color[e['node']] + text = colored(fmt % ( + e['time'].strftime('%H:%M:%S.%f'), + e['round'], e['period'], e['step'], + e['object_round'], e['object_period'], e['object_step'], + e['weight'], e['weight_total'], + e['node'][:8], + e['type'], e['hash'], e['sender'], + int(e['round_time_ms']) if 'round_time_ms' in e else ''), + color, + ) + print(text) + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + main() diff --git a/test/heapwatch/block_history_plot.py b/test/heapwatch/block_history_plot.py index 7de45e21b0..d8c86b454f 100644 --- a/test/heapwatch/block_history_plot.py +++ b/test/heapwatch/block_history_plot.py @@ -138,19 +138,21 @@ def process(path, args): min(tpsv[start:end]), max(tpsv[start:end]), )) print('long round times: {}'.format(' '.join(list(map(str,filter(lambda x: x >= 9,dtv[start:end])))))) - fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2,2) - ax1.set_title('round time (seconds)') + fig, ((ax1, ax2, ax3), (ax4, ax5, ax6)) = plt.subplots(2,3, figsize=(10, 5)) + ax1.set_title('round time histogram (sec)') ax1.hist(list(filter(lambda x: x < 9,dtv[start:end])),bins=20) - if args.rtime: - ax2.set_title('round time') - ax2.plot(dtv) - else: - ax2.set_title('TPS') - ax2.hist(tpsv[start:end],bins=20) + ax4.set_title('round time') + ax4.plot(dtv[start:end]) + + ax2.set_title('txn/block histogram') + ax2.hist(txnv[start:end],bins=20) + + ax5.set_title('txn/block') + ax5.plot(txnv[start:end]) - ax3.set_title('txn/block') - ax3.hist(txnv[start:end],bins=20) + ax3.set_title('TPS') + ax3.hist(tpsv[start:end],bins=20) # 10 round moving average TPS tpsv10 = [] @@ -165,12 +167,12 @@ def process(path, args): dtxn = tca-tc0 tpsv10.append(dtxn/dt) if args.tps1: - ax4.set_title('TPS') - ax4.plot(tpsv[start:end]) + ax6.set_title('TPS') + ax6.plot(tpsv[start:end]) print('fullish block sizes: {}'.format(list(filter(lambda x: x > 100, txnv)))) else: - ax4.set_title('TPS(10 round window)') - ax4.plot(tpsv10) + ax6.set_title('TPS(10 round window)') + ax6.plot(tpsv10) fig.tight_layout() plt.savefig(path + '_hist.svg', format='svg') plt.savefig(path + '_hist.png', format='png') diff --git a/test/heapwatch/client_ram_report.py b/test/heapwatch/client_ram_report.py index 97a1171630..f16fbeaa3f 100644 --- a/test/heapwatch/client_ram_report.py +++ b/test/heapwatch/client_ram_report.py @@ -202,6 +202,10 @@ def main(): heap_totals = get_heap_inuse_totals(args.dir) heap_details = get_heap_metrics(args.dir) + if not heap_totals and not heap_details: + print('no data found', file=sys.stderr) + return 0 + if args.csv: if args.csv == '-': csvf = sys.stdout diff --git a/test/heapwatch/metrics_aggs.py b/test/heapwatch/metrics_aggs.py new file mode 100644 index 0000000000..0189634be5 --- /dev/null +++ b/test/heapwatch/metrics_aggs.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +# Copyright (C) 2019-2024 Algorand, Inc. +# This file is part of go-algorand +# +# go-algorand is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# go-algorand is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with go-algorand. If not, see . +# +### +# +# Process and aggregate /metrics data captured by heapWatch.py +# Useful for metrics with labels and bandwidth analysis. +# +import argparse +import glob +import logging +import os +import time +import sys + +import dash +from dash import dcc, html +import plotly.graph_objs as go +from plotly.subplots import make_subplots + + +from metrics_lib import MetricType, parse_metrics, gather_metrics_files_by_nick + +logger = logging.getLogger(__name__) + + +def main(): + os.environ['TZ'] = 'UTC' + time.tzset() + default_img_filename = 'metrics_aggs.png' + default_html_filename = 'metrics_aggs.html' + + ap = argparse.ArgumentParser() + ap.add_argument('metrics_names', nargs='+', default=None, help='metric name(s) to track') + ap.add_argument('-d', '--dir', type=str, default=None, help='dir path to find /*.metrics in') + ap.add_argument('-l', '--list-nodes', default=False, action='store_true', help='list available node names with metrics') + ap.add_argument('-t', '--tags', action='append', default=[], help='tag/label pairs in a=b format to aggregate by, may be repeated. Empty means aggregation by metric name') + ap.add_argument('--nick-re', action='append', default=[], help='regexp to filter node names, may be repeated') + ap.add_argument('--nick-lre', action='append', default=[], help='label:regexp to filter node names, may be repeated') + ap.add_argument('-s', '--save', type=str, choices=['png', 'html'], help=f'save plot to \'{default_img_filename}\' or \'{default_html_filename}\' file instead of showing it') + ap.add_argument('--verbose', default=False, action='store_true') + + args = ap.parse_args() + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + tags = {} + if args.tags: + for tag in args.tags: + if '=' not in tag: + raise (f'Invalid tag: {tag}') + k, v = tag.split('=', 1) + tags[k] = v + tag_keys = set(tags.keys()) + + metrics_files = sorted(glob.glob(os.path.join(args.dir, '*.metrics'))) + metrics_files.extend(glob.glob(os.path.join(args.dir, 'terraform-inventory.host'))) + filesByNick = gather_metrics_files_by_nick(metrics_files, args.nick_re, args.nick_lre) + + if args.list_nodes: + print('Available nodes:', ', '.join(sorted(filesByNick.keys()))) + return 0 + + app = dash.Dash(__name__) + app.layout = html.Div( + html.Div([ + html.H4('Algod Metrics'), + html.Div(id='text'), + dcc.Graph(id='graph'), + ]) + ) + metrics_names = set(args.metrics_names) + nrows = len(metrics_names) + + fig = make_subplots( + rows=nrows, cols=1, + vertical_spacing=0.03, shared_xaxes=True, + subplot_titles=[f'{name}' for name in sorted(metrics_names)], + ) + + fig['layout']['margin'] = { + 'l': 30, 'r': 10, 'b': 10, 't': 20 + } + fig['layout']['height'] = 500 * nrows + + + for nick, files_by_date in filesByNick.items(): + active_metrics = {} + data = {'time': []} + raw_series = {} + raw_times = {} + idx = 0 + for dt, metrics_file in files_by_date.items(): + data['time'].append(dt) + with open(metrics_file, 'rt') as f: + metrics = parse_metrics(f, nick, metrics_names) + for metric_name, metrics_seq in metrics.items(): + active_metric_names = [] + raw_value = 0 + for metric in metrics_seq: + if metric.type != MetricType.COUNTER: + raise RuntimeError('Only COUNT metrics are supported') + if tags is None or tags is not None and metric.has_tags(tag_keys, tags): + raw_value += metric.value + full_name = metric.string(set(tag_keys).union({'n'})) + + if full_name is None: + continue + + if full_name not in data: + # handle gaps in data, sometimes metric file might miss a value + # but the chart requires matching x and y series (time and metric value) + # data is what does into the chart, and raw_series is used to calculate + data[full_name] = [0] * len(files_by_date) + raw_series[full_name] = [] + raw_times[full_name] = [] + + metric_value = raw_value + if len(raw_series[full_name]) > 0 and len(raw_times[full_name]) > 0: + metric_value = (metric_value - raw_series[full_name][-1]) / (dt - raw_times[full_name][-1]).total_seconds() + else: + metric_value = 0 + + data[full_name][idx] = metric_value + raw_series[full_name].append(raw_value) + raw_times[full_name].append(dt) + + active_metric_names.append(full_name) + + active_metric_names.sort() + active_metrics[full_name] = active_metric_names + idx += 1 + + for i, metric_pair in enumerate(sorted(active_metrics.items())): + metric_name, metric_fullnames = metric_pair + for metric_fullname in metric_fullnames: + fig.append_trace(go.Scatter( + x=data['time'], + y=data[metric_fullname], + name=metric_fullname, + mode='lines+markers', + line=dict(width=1), + ), i+1, 1) + + if args.save: + if args.save == 'html': + target_path = os.path.join(args.dir, default_html_filename) + fig.write_html(target_path) + else: + target_path = os.path.join(args.dir, default_img_filename) + fig.write_image(target_path) + print(f'Saved plot to {target_path}') + else: + fig.show() + + return 0 + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/test/heapwatch/metrics_delta.py b/test/heapwatch/metrics_delta.py index 50b1e9e2e3..2d64ee097a 100644 --- a/test/heapwatch/metrics_delta.py +++ b/test/heapwatch/metrics_delta.py @@ -22,7 +22,6 @@ # Generate text report on bandwidth in and out of relays/PN/NPN import argparse -import configparser import contextlib import csv import glob @@ -36,42 +35,10 @@ import sys import time -logger = logging.getLogger(__name__) +from metrics_lib import num, hunum, terraform_inventory_ip_not_names, \ + metric_line_re, test_metric_line_re -def num(x): - if '.' in x: - return float(x) - return int(x) - -def hunum(x): - if x >= 10000000000: - return '{:.1f}G'.format(x / 1000000000.0) - if x >= 1000000000: - return '{:.2f}G'.format(x / 1000000000.0) - if x >= 10000000: - return '{:.1f}M'.format(x / 1000000.0) - if x >= 1000000: - return '{:.2f}M'.format(x / 1000000.0) - if x >= 10000: - return '{:.1f}k'.format(x / 1000.0) - if x >= 1000: - return '{:.2f}k'.format(x / 1000.0) - return '{:.2f}x'.format(x) - -metric_line_re = re.compile(r'(\S+\{[^}]*\})\s+(.*)') - -def test_metric_line_re(): - testlines = ( - ('algod_network_connections_dropped_total{reason="write err"} 1', 1), - #('algod_network_sent_bytes_MS 274992', 274992), # handled by split - ) - for line, n in testlines: - try: - m = metric_line_re.match(line) - assert int(m.group(2)) == n - except: - logger.error('failed on line %r', line, exc_info=True) - raise +logger = logging.getLogger(__name__) def parse_metrics(fin): out = dict() @@ -86,10 +53,15 @@ def parse_metrics(fin): continue m = metric_line_re.match(line) if m: - out[m.group(1)] = num(m.group(2)) + key = m.group(1) + val = m.group(2) else: ab = line.split() - out[ab[0]] = num(ab[1]) + key = ab[0] + val = ab[1] + if key.endswith('{}'): + key = key[:-2] + out[key] = num(val) except: print(f'An exception occurred in parse_metrics: {sys.exc_info()}') pass @@ -371,21 +343,6 @@ def process_nick_re(nre, filesByNick, nick_to_tfname, rsum, args, grsum): 'npn': (.7,.7,0), } -def terraform_inventory_ip_not_names(tf_inventory_path): - """return ip to nickname mapping""" - tf_inventory = configparser.ConfigParser(allow_no_value=True) - tf_inventory.read(tf_inventory_path) - ip_to_name = {} - for k, sub in tf_inventory.items(): - if k.startswith('name_'): - for ip in sub: - if ip in ip_to_name: - logger.warning('ip %r already named %r, also got %r', ip, ip_to_name[ip], k) - ip_to_name[ip] = k - #logger.debug('names: %r', sorted(ip_to_name.values())) - #logger.debug('ip to name %r', ip_to_name) - return ip_to_name - def main(): os.environ['TZ'] = 'UTC' time.tzset() @@ -541,7 +498,7 @@ def __init__(self): self.txPLists = {} self.txPSums = {} self.times = [] - # algod_tx_pool_count{} + # algod_tx_pool_count self.txPool = [] # objectBytes = [(curtime, algod_go_memory_classes_heap_objects_bytes), ...] self.objectBytes = [] @@ -601,13 +558,13 @@ def process_files(self, args, nick=None, metrics_files=None, bisource=None): bi = bisource.get(curtime) if bi is None: logger.warning('%s no blockinfo', path) - self.txPool.append(cur.get('algod_tx_pool_count{}')) + self.txPool.append(cur.get('algod_tx_pool_count')) objectBytes = cur.get('algod_go_memory_classes_heap_objects_bytes') if objectBytes: self.objectBytes.append((curtime, objectBytes)) #logger.debug('%s: %r', path, cur) - verifyGood = cur.get('algod_agreement_proposal_verify_good{}') - verifyMs = cur.get('algod_agreement_proposal_verify_ms{}') + verifyGood = cur.get('algod_agreement_proposal_verify_good') + verifyMs = cur.get('algod_agreement_proposal_verify_ms') if verifyGood and verifyMs: # last writer wins self.verifyMillis = verifyMs / verifyGood @@ -626,8 +583,8 @@ def process_files(self, args, nick=None, metrics_files=None, bisource=None): rounds = (bi.get('block',{}).get('rnd', 0) - prevbi.get('block',{}).get('rnd', 0)) if rounds != 0: blocktime = dt/rounds - txBytes = d.get('algod_network_sent_bytes_total{}',0) - rxBytes = d.get('algod_network_received_bytes_total{}',0) + txBytes = d.get('algod_network_sent_bytes_total',0) + rxBytes = d.get('algod_network_received_bytes_total',0) txBytesPerSec = txBytes / dt rxBytesPerSec = rxBytes / dt # TODO: gather algod_network_sent_bytes_* and algod_network_received_bytes_* diff --git a/test/heapwatch/metrics_lib.py b/test/heapwatch/metrics_lib.py new file mode 100644 index 0000000000..fbda555b90 --- /dev/null +++ b/test/heapwatch/metrics_lib.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +# Copyright (C) 2019-2024 Algorand, Inc. +# This file is part of go-algorand +# +# go-algorand is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# go-algorand is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with go-algorand. If not, see . +# +### +# +# Common functions for processing /metrics data captured by heapWatch.py +# +import configparser +from datetime import datetime +from enum import Enum +import logging +import os +import re +import sys +from typing import Dict, Iterable, List, Optional, Tuple, Union +from urllib.parse import urlparse + + +from client_ram_report import dapp + + +logger = logging.getLogger(__name__) +metric_line_re = re.compile(r'(\S+\{[^}]*\})\s+(.*)') + +def num(x): + if '.' in x: + return float(x) + return int(x) + +def hunum(x): + if x >= 10000000000: + return '{:.1f}G'.format(x / 1000000000.0) + if x >= 1000000000: + return '{:.2f}G'.format(x / 1000000000.0) + if x >= 10000000: + return '{:.1f}M'.format(x / 1000000.0) + if x >= 1000000: + return '{:.2f}M'.format(x / 1000000.0) + if x >= 10000: + return '{:.1f}k'.format(x / 1000.0) + if x >= 1000: + return '{:.2f}k'.format(x / 1000.0) + return '{:.2f}x'.format(x) + + +def test_metric_line_re(): + testlines = ( + ('algod_network_connections_dropped_total{reason="write err"} 1', 1), + #('algod_network_sent_bytes_MS 274992', 274992), # handled by split + ) + for line, n in testlines: + try: + m = metric_line_re.match(line) + assert int(m.group(2)) == n + except: + print('failed on line %r', line) + raise + +def terraform_inventory_ip_not_names(tf_inventory_path): + """return ip to nickname mapping""" + tf_inventory = configparser.ConfigParser(allow_no_value=True) + tf_inventory.read(tf_inventory_path) + ip_to_name = {} + for k, sub in tf_inventory.items(): + if k.startswith('name_'): + for ip in sub: + if ip in ip_to_name: + logger.warning('ip %r already named %r, also got %r', ip, ip_to_name[ip], k) + ip_to_name[ip] = k + #logger.debug('names: %r', sorted(ip_to_name.values())) + #logger.debug('ip to name %r', ip_to_name) + return ip_to_name + +metrics_fname_re = re.compile(r'(.*?)\.(\d+_\d+)\.metrics') + +def gather_metrics_files_by_nick( + metrics_files: Iterable[str], nick_res: List[str], nick_lres: List[str] +) -> Dict[str, Dict[datetime, str]]: + """return {"node nickname": {datetime: path, ...}, ...}} + after resolving ip addresses into nodes nick names and applying nick_re and nick_lre filters. + """ + filesByNick = {} + tf_inventory_path = None + for path in metrics_files: + fname = os.path.basename(path) + if fname == 'terraform-inventory.host': + tf_inventory_path = path + continue + m = metrics_fname_re.match(fname) + if not m: + continue + nick = m.group(1) + timestamp = m.group(2) + timestamp = datetime.strptime(timestamp, '%Y%m%d_%H%M%S') + dapp(filesByNick, nick, timestamp, path) + + if tf_inventory_path: + # remap ip addresses to node names + ip_to_name = terraform_inventory_ip_not_names(tf_inventory_path) + filesByNick2 = {} + for nick in filesByNick.keys(): + parsed = urlparse('//' + nick) + name: str = ip_to_name.get(parsed.hostname) + val = filesByNick[nick] + filesByNick2[name] = val + + filesByNick = filesByNick2 + filesByNick2 = {} + + for nick in filesByNick.keys(): + if nick_res or not nick_res and not nick_lres: + # filter by regexp or apply default renaming + for nick_re in nick_res: + if re.match(nick_re, nick): + break + else: + if nick_res: + # regex is given but not matched, continue to the next node + continue + + # apply default renaming + name = nick + idx = name.find('_') + if idx != -1: + name = name[idx+1:] + val = filesByNick[nick] + filesByNick2[name] = val + + elif nick_lres: + # filter by label:regexp + label = None + for nick_lre in nick_lres: + label, nick_re = nick_lre.split(':') + if re.match(nick_re, nick): + break + else: + if nick_lres: + # regex is given but not matched, continue to the next node + continue + + val = filesByNick[nick] + filesByNick2[label] = val + else: + raise RuntimeError('unexpected options combination') + + if filesByNick2: + filesByNick = filesByNick2 + + return filesByNick + +class MetricType(Enum): + GAUGE = 0 + COUNTER = 1 + +class Metric: + """Metric with tags""" + def __init__(self, metric_name: str, type: MetricType, value: Union[int, float]): + full_name = metric_name.strip() + self.name = full_name + self.value = value + self.type = type + self.tags: Dict[str, str] = {} + self.tag_keys: set = set() + + det_idx = self.name.find('{') + if det_idx != -1: + self.name = self.name[:det_idx] + # ensure that the last character is '}' + idx = full_name.index('}') + if idx != len(full_name) - 1: + raise ValueError(f'Invalid metric name: {full_name}') + raw_tags = full_name[full_name.find('{')+1:full_name.find('}')] + tags = raw_tags.split(',') + for tag in tags: + key, value = tag.split('=') + if value[0] == '"' and value[-1] == '"': + value = value[1:-1] + self.tags[key] = value + self.tag_keys.add(key) + + def short_name(self): + return self.name + + def __str__(self): + return self.string() + + def string(self, tags: Optional[set[str]]=None): + result = self.name + if self.tags: + if not tags: + tags = self.tags + result += '{' + ','.join([f'{k}={v}' for k, v in sorted(self.tags.items()) if k in tags]) + '}' + return result + + def add_tag(self, key: str, value: str): + self.tags[key] = value + self.tag_keys.add(key) + + def has_tags(self, tag_keys: set, tags: Dict[str, str]): + """return True if all tags are present in the metric tags + tag_keys are not strictly needed but used as an optimization + """ + if self.tag_keys.intersection(tag_keys) != tag_keys: + return False + for k, v in tags.items(): + if self.tags.get(k) != v: + return False + return True + +def parse_metrics( + fin: Iterable[str], nick: str, metrics_names: set=None, diff: bool=None +) -> Dict[str, List[Metric]]: + """Parse metrics file and return dicts of metric names (no tags) and list of Metric objects + each containing the metric name, value and tags. + """ + out = {} + try: + last_type = None + for line in fin: + if not line: + continue + line = line.strip() + if not line: + continue + if line[0] == '#': + if line.startswith('# TYPE'): + tpe = line.split()[-1] + if tpe == 'gauge': + last_type = MetricType.GAUGE + elif tpe == 'counter': + last_type = MetricType.COUNTER + continue + m = metric_line_re.match(line) + if m: + name = m.group(1) + value = num(m.group(2)) + else: + ab = line.split() + name = ab[0] + value = num(ab[1]) + + metric = Metric(name, last_type, value) + metric.add_tag('n', nick) + if not metrics_names or metric.name in metrics_names: + if metric.name not in out: + out[metric.name] = [metric] + else: + out[metric.name].append(metric) + except: + print(f'An exception occurred in parse_metrics: {sys.exc_info()}') + pass + if diff and metrics_names and len(metrics_names) == 2 and len(out) == 2: + m = list(out.keys()) + name = f'{m[0]}_-_{m[1]}' + metric = Metric(name, MetricType.GAUGE, out[m[0]].value - out[m[1]].value) + out = [{name: metric}] + + return out diff --git a/test/heapwatch/metrics_viz.py b/test/heapwatch/metrics_viz.py index 584fc0ae59..741aa2dd73 100644 --- a/test/heapwatch/metrics_viz.py +++ b/test/heapwatch/metrics_viz.py @@ -11,13 +11,11 @@ """ import argparse -from datetime import datetime import glob import logging import os import re import time -from typing import Dict, Iterable, Tuple import sys import dash @@ -25,95 +23,24 @@ import plotly.graph_objs as go from plotly.subplots import make_subplots -from metrics_delta import metric_line_re, num, terraform_inventory_ip_not_names -from client_ram_report import dapp +from metrics_lib import MetricType, parse_metrics, gather_metrics_files_by_nick logger = logging.getLogger(__name__) -metrics_fname_re = re.compile(r'(.*?)\.(\d+_\d+)\.metrics') - -def gather_metrics_files_by_nick(metrics_files: Iterable[str]) -> Dict[str, Dict[datetime, str]]: - """return {"node nickname": {datetime: path, ...}, ...}}""" - filesByNick = {} - tf_inventory_path = None - for path in metrics_files: - fname = os.path.basename(path) - if fname == 'terraform-inventory.host': - tf_inventory_path = path - continue - m = metrics_fname_re.match(fname) - if not m: - continue - nick = m.group(1) - timestamp = m.group(2) - timestamp = datetime.strptime(timestamp, '%Y%m%d_%H%M%S') - dapp(filesByNick, nick, timestamp, path) - return tf_inventory_path, filesByNick - - -TYPE_GAUGE = 0 -TYPE_COUNTER = 1 - -def parse_metrics(fin: Iterable[str], nick: str, metrics_names: set=None, diff: bool=None) -> Tuple[Dict[str, float], Dict[str, int]]: - """Parse metrics file and return dicts of values and types""" - out = {} - types = {} - try: - last_type = None - for line in fin: - if not line: - continue - line = line.strip() - if not line: - continue - if line[0] == '#': - if line.startswith('# TYPE'): - tpe = line.split()[-1] - if tpe == 'gauge': - last_type = TYPE_GAUGE - elif tpe == 'counter': - last_type = TYPE_COUNTER - continue - m = metric_line_re.match(line) - if m: - name = m.group(1) - value = num(m.group(2)) - else: - ab = line.split() - name = ab[0] - value = num(ab[1]) - - det_idx = name.find('{') - if det_idx != -1: - name = name[:det_idx] - fullname = f'{name}{{n={nick}}}' - if not metrics_names or name in metrics_names: - out[fullname] = value - types[fullname] = last_type - except: - print(f'An exception occurred in parse_metrics: {sys.exc_info()}') - pass - if diff and metrics_names and len(metrics_names) == 2 and len(out) == 2: - m = list(out.keys()) - name = f'{m[0]}_-_{m[1]}' - new_out = {name: out[m[0]] - out[m[1]]} - new_types = {name: TYPE_GAUGE} - out = new_out - types = new_types - - return out, types - def main(): os.environ['TZ'] = 'UTC' time.tzset() - default_output_file = 'metrics_viz.png' + default_img_filename = 'metrics_viz.png' + default_html_filename = 'metrics_viz.html' ap = argparse.ArgumentParser() ap.add_argument('metrics_names', nargs='+', default=None, help='metric name(s) to track') ap.add_argument('-d', '--dir', type=str, default=None, help='dir path to find /*.metrics in') ap.add_argument('-l', '--list-nodes', default=False, action='store_true', help='list available node names with metrics') - ap.add_argument('-s', '--save', action='store_true', default=None, help=f'save plot to \'{default_output_file}\' file instead of showing it') + ap.add_argument('--nick-re', action='append', default=[], help='regexp to filter node names, may be repeated') + ap.add_argument('--nick-lre', action='append', default=[], help='label:regexp to filter node names, may be repeated') + ap.add_argument('-s', '--save', type=str, choices=['png', 'html'], help=f'save plot to \'{default_img_filename}\' or \'{default_html_filename}\' file instead of showing it') ap.add_argument('--diff', action='store_true', default=None, help='diff two gauge metrics instead of plotting their values. Requires two metrics names to be set') ap.add_argument('--verbose', default=False, action='store_true') @@ -128,16 +55,8 @@ def main(): return 1 metrics_files = sorted(glob.glob(os.path.join(args.dir, '*.metrics'))) - tf_inventory_path, filesByNick = gather_metrics_files_by_nick(metrics_files) - if tf_inventory_path: - # remap ip addresses to node names - ip_to_name = terraform_inventory_ip_not_names(tf_inventory_path) - for nick in filesByNick.keys(): - name = ip_to_name.get(nick) - if name: - val = filesByNick[nick] - filesByNick[name] = val - del filesByNick[nick] + metrics_files.extend(glob.glob(os.path.join(args.dir, 'terraform-inventory.host'))) + filesByNick = gather_metrics_files_by_nick(metrics_files, args.nick_re, args.nick_lre) if args.list_nodes: print('Available nodes:', ', '.join(sorted(filesByNick.keys()))) @@ -156,50 +75,76 @@ def main(): fig = make_subplots( rows=nrows, cols=1, - vertical_spacing=0.03, shared_xaxes=True) + vertical_spacing=0.03, shared_xaxes=True, + subplot_titles=[f'{name}' for name in sorted(metrics_names)], + ) fig['layout']['margin'] = { - 'l': 30, 'r': 10, 'b': 10, 't': 10 + 'l': 30, 'r': 10, 'b': 10, 't': 20 } fig['layout']['height'] = 500 * nrows # fig.update_layout(template="plotly_dark") - data = { - 'time': [], - } - raw_series = {} - for nick, items in filesByNick.items(): - active_metrics = set() - for dt, metrics_file in items.items(): + for nick, files_by_date in filesByNick.items(): + active_metrics = {} + data = {'time': []} + raw_series = {} + raw_times = {} + idx = 0 + for dt, metrics_file in files_by_date.items(): data['time'].append(dt) with open(metrics_file, 'rt') as f: - metrics, types = parse_metrics(f, nick, metrics_names, args.diff) - for metric_name, metric_value in metrics.items(): - raw_value = metric_value - if metric_name not in data: - data[metric_name] = [] - raw_series[metric_name] = [] - if types[metric_name] == TYPE_COUNTER: - if len(raw_series[metric_name]) > 0: - metric_value = (metric_value - raw_series[metric_name][-1]) / (dt - data['time'][-2]).total_seconds() - else: - metric_value = 0 - data[metric_name].append(metric_value) - raw_series[metric_name].append(raw_value) - - active_metrics.add(metric_name) - - for i, metric in enumerate(sorted(active_metrics)): - fig.append_trace(go.Scatter( - x=data['time'], - y=data[metric], - name=metric, - mode='lines+markers', - line=dict(width=1), - ), i+1, 1) + metrics = parse_metrics(f, nick, metrics_names, args.diff) + for metric_name, metrics_seq in metrics.items(): + active_metric_names = [] + for metric in metrics_seq: + raw_value = metric.value + + full_name = metric.string() + if full_name not in data: + # handle gaps in data, sometimes metric file might miss a value + # but the chart requires matching x and y series (time and metric value) + # data is what does into the chart, and raw_series is used to calculate + data[full_name] = [0] * len(files_by_date) + raw_series[full_name] = [] + raw_times[full_name] = [] + + metric_value = metric.value + if metric.type == MetricType.COUNTER: + if len(raw_series[full_name]) > 0 and len(raw_times[full_name]) > 0: + metric_value = (metric_value - raw_series[full_name][-1]) / (dt - raw_times[full_name][-1]).total_seconds() + else: + metric_value = 0 + + data[full_name][idx] = metric_value + raw_series[full_name].append(raw_value) + raw_times[full_name].append(dt) + + active_metric_names.append(full_name) + + active_metric_names.sort() + active_metrics[metric_name] = active_metric_names + idx += 1 + + for i, metric_pair in enumerate(sorted(active_metrics.items())): + metric_name, metric_fullnames = metric_pair + for metric_fullname in metric_fullnames: + fig.append_trace(go.Scatter( + x=data['time'], + y=data[metric_fullname], + name=metric_fullname, + mode='lines+markers', + line=dict(width=1), + ), i+1, 1) if args.save: - fig.write_image(os.path.join(args.dir, default_output_file)) + if args.save == 'html': + target_path = os.path.join(args.dir, default_html_filename) + fig.write_html(target_path) + else: + target_path = os.path.join(args.dir, default_img_filename) + fig.write_image(target_path) + print(f'Saved plot to {target_path}') else: fig.show() diff --git a/test/heapwatch/requirements.txt b/test/heapwatch/requirements.txt index d4d68874dd..db92372c6d 100644 --- a/test/heapwatch/requirements.txt +++ b/test/heapwatch/requirements.txt @@ -5,3 +5,6 @@ matplotlib==3.7.2 plotly==5.16.0 py-algorand-sdk==2.3.0 kaleido==0.2.1 +networkx==3.3 +gravis=0.1.0 +termcolor=2.4.0 diff --git a/test/heapwatch/topology-extract-p2p.py b/test/heapwatch/topology-extract-p2p.py new file mode 100644 index 0000000000..41f2be9ffc --- /dev/null +++ b/test/heapwatch/topology-extract-p2p.py @@ -0,0 +1,104 @@ +""" +P2P network topology extraction script from node.log files. + +1. Run P2P scenario like scenario1s-p2p +2. Fetch logs with `algonet play fetch_node_logs` +3. Extract logs +``` +cd nodelog +find . -name 'nodelog.tar.gz' -print | xargs -I{} tar -zxf {} +``` +4. Run this script `python3 topology-extract-p2p.py -o top.json nodelog` +5. Run the visualizer `topology-viz.py top.json` +""" +import argparse +from datetime import datetime +import json +import re +import os +import sys + + +def main(): + # Regex patterns to find node IDs and connections + node_pattern = r"P2P host created: peer ID (\w{52})" + edge_pattern = r"Made outgoing connection to peer (\w{52})" + + ap = argparse.ArgumentParser() + ap.add_argument('log_dir_path', help='logs directory path') + ap.add_argument('-o', '--output', type=argparse.FileType('wt', encoding='utf-8'), help=f'save topology to the file specified instead of showing it') + ap.add_argument('-t', '--timestamp', action='store_true', help=f'store connection timestamp for each edge') + + args = ap.parse_args() + + # Directory containing log files + log_dir_path = args.log_dir_path + + nodes = [] + edges = [] + mapping = {} + + # Iterate through all files in the specified directory + for filename in os.listdir(log_dir_path): + if filename.endswith("-node.log"): + with open(os.path.join(log_dir_path, filename), 'r') as file: + mapped = filename[:len(filename) - len('-node.log')] + mapped = mapped.replace('relay', 'R') + mapped = mapped.replace('nonParticipatingNode', 'NPN') + mapped = mapped.replace('node', 'N') + node_id = None + for line in file: + # Check if line contains relevant substrings before parsing as JSON + if "P2P host created" in line or "Made outgoing connection to peer" in line: + data = json.loads(line.strip()) + + # Check for node creation + if "P2P host created" in data.get("msg", ""): + match = re.search(node_pattern, data["msg"]) + if match: + node_id = match.group(1) + nodes.append(node_id) + mapping[node_id] = mapped + + # Check for connections + elif "Made outgoing connection to peer" in data.get("msg", ""): + match = re.search(edge_pattern, data["msg"]) + if match: + target_node_id = match.group(1) + match = re.findall(r"/p2p/(\w{52})", data["local"]) + if match: + source_node_id = match[0] + else: + print('WARN: no local addr set', data, file=sys.stderr) + source_node_id = node_id + + if args.timestamp: + # datetime is not serializable, so we store it as string for now + edge = (source_node_id, target_node_id, {'dt': data["time"]}) + else: + edge = (source_node_id, target_node_id) + + edges.append(edge) + + result = { + "mapping": mapping, + "nodes": nodes, + "edges": edges + } + + if args.timestamp and not args.output: + edges = sorted(edges, key=lambda x: x[2]['dt']) + for edge in edges: + ts = datetime.strptime(edge[2]['dt'], "%Y-%m-%dT%H:%M:%S.%f%z") + print('%15s %5s -> %-5s' % (ts.strftime('%H:%M:%S.%f'), mapping[edge[0]], mapping[edge[1]])) + return + + if args.output: + json.dump(result, args.output, indent=2) + else: + json.dump(result, sys.stdout, indent=2) + print(file=sys.stdout) + + +if __name__ == '__main__': + main() diff --git a/test/heapwatch/topology-extract-ws.py b/test/heapwatch/topology-extract-ws.py new file mode 100644 index 0000000000..75f1d99f57 --- /dev/null +++ b/test/heapwatch/topology-extract-ws.py @@ -0,0 +1,115 @@ +""" +WSNet network topology extraction script from node.log files. + +1. Run cluster scenario like scenario1s +2. Fetch logs with `algonet play fetch_node_logs` +3. Extract logs +``` +cd nodelog +find . -name 'nodelog.tar.gz' -print | xargs -I{} tar -zxf {} +``` +4. Run this script `python3 topology-extract-ws.py -o top.json -i ../terraform-inventory.json nodelog` +5. Run the visualizer `topology-viz.py top.json` +""" +import argparse +from datetime import datetime +import json +import os +import sys + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('log_dir_path', help='logs directory path') + ap.add_argument('-i', '--inventory-file', type=argparse.FileType('rt', encoding='utf-8'), required=True, help='terraform inventory file path') + ap.add_argument('-o', '--output', type=argparse.FileType('wt', encoding='utf-8'), help=f'save topology to the file specified instead of showing it') + ap.add_argument('-t', '--timestamp', action='store_true', help=f'store connection timestamp for each edge') + + args = ap.parse_args() + + # Directory containing log files + log_dir_path = args.log_dir_path + inventory_file = args.inventory_file + + nodes = [] + edges = [] + mapping = {} + + inventory = json.load(inventory_file) + + ip_to_name = {} + for k, v in inventory.items(): + if k.startswith('name_'): + name = k.split('_')[1].upper() + if not isinstance(v, list) or len(v) != 1: + raise RuntimeError(f"Invalid inventory entry, expected a single item list: {k}={v}") + ip = v[0] + ip_to_name[ip] = name + # no need for mapping but keep the data compatible with the topology-viz script + mapping[name] = name + + # Iterate through all files in the specified directory + for filename in os.listdir(log_dir_path): + if filename.endswith('-node.log'): + with open(os.path.join(log_dir_path, filename), 'r') as file: + mapped = filename[:len(filename) - len('-node.log')] + mapped = mapped.replace('relay', 'R') + mapped = mapped.replace('nonParticipatingNode', 'NPN') + mapped = mapped.replace('node', 'N') + nodes.append(mapped) + for line in file: + # Check if line contains relevant substrings before parsing as JSON + if "Accepted incoming connection from peer" in line or "Made outgoing connection to peer" in line: + data = json.loads(line.strip()) + + # Check for incoming connections + if "Accepted incoming connection from peer" in data.get("msg", ""): + remote = data['remote'] + remote_ip = remote.split(':')[0] + remote_name = ip_to_name[remote_ip] + source = remote_name + target = mapped + edges.append((source, target)) + + # Check for outgoing connections + elif "Made outgoing connection to peer" in data.get('msg', ""): + remote = data['remote'] + name: str = remote.split('.')[0] + # check ip or name + if name.isdigit(): + remote_ip = remote.split(':')[0] + remote_name = ip_to_name[remote_ip] + target = remote_name + source = mapped + else: + target = name.upper() + source = mapped + + if args.timestamp: + # datetime is not serializable, so we store it as string for now + edge = (source, target, {'dt': data["time"]}) + else: + edge = (source, target) + + edges.append(edge) + + result = { + "mapping": mapping, + "nodes": nodes, + "edges": edges + } + + if args.timestamp and not args.output: + edges = sorted(edges, key=lambda x: x[2]['dt']) + for edge in edges: + ts = datetime.strptime(edge[2]['dt'], "%Y-%m-%dT%H:%M:%S.%f%z") + print('%15s %5s -> %-5s' % (ts.strftime('%H:%M:%S.%f'), edge[0], edge[1])) + return + + if args.output: + json.dump(result, args.output, indent=2) + else: + json.dump(result, sys.stdout, indent=2) + print(file=sys.stdout) + +if __name__ == '__main__': + main() diff --git a/test/heapwatch/topology-viz.py b/test/heapwatch/topology-viz.py new file mode 100644 index 0000000000..1393421696 --- /dev/null +++ b/test/heapwatch/topology-viz.py @@ -0,0 +1,75 @@ +""" +P2P network topology visualization script. +See topology-extract-p2p[-ws].py for details. +""" +import argparse +import json +import sys + +import gravis as gv +import networkx as nx + +ap = argparse.ArgumentParser() +ap.add_argument('topology_filename', help='topology json file') +ap.add_argument('-o', '--output', type=argparse.FileType('wt', encoding='utf-8'), help=f'save plot to the file specified instead of showing it') + +args = ap.parse_args() + +with open(args.topology_filename, 'rt') as f: + topology = json.load(f) + +# Create a new directed graph +G = nx.DiGraph() + +G.add_edges_from(topology['edges']) +nx.relabel_nodes(G, topology['mapping'], copy=False) + +# Set node colors +for node in G: + if node.startswith('R'): + G.nodes[node]['color'] = 'red' + elif node.startswith('NPN'): + G.nodes[node]['color'] = 'blue' + elif node.startswith('N'): + G.nodes[node]['color'] = 'green' + else: + raise RuntimeError(f"Unknown node type: {node}") + +# Calculate in-degrees +in_degrees = dict(G.in_degree()) +out_degrees = dict(G.out_degree()) +degree_centrality = nx.degree_centrality(G) +load_centrality = nx.algorithms.load_centrality(G) + +for node in G: + size = max(2, in_degrees[node]) + G.nodes[node]['size'] = size + G.nodes[node]['in_degree'] = in_degrees[node] + G.nodes[node]['out_degree'] = out_degrees[node] + hover = f'In: {in_degrees[node]}, Out: {out_degrees[node]}' + hover += f'\nDegree centrality: {degree_centrality[node]:.2f}' + hover += f'\nLoad centrality: {load_centrality[node]:.2f}' + G.nodes[node]['hover'] = hover + +print('Transitivity:', nx.transitivity(G)) +print('Clustering coefficient:', nx.average_clustering(G)) +print('Avg shortest path length:', nx.average_shortest_path_length(G.to_undirected(as_view=True))) + +res = gv.d3( + G, + node_hover_tooltip=True, + node_size_data_source='size', + node_label_size_factor=0.5, + use_node_size_normalization=True, + node_size_normalization_max=20, + use_edge_size_normalization=True, + edge_curvature=0.1 + ) + +if not args.output: + res.display() + sys.exit(0) + +# Save to file +data = res.to_html() +args.output.write(data) diff --git a/test/testdata/configs/config-v34.json b/test/testdata/configs/config-v34.json index 4a9714115f..7f16155303 100644 --- a/test/testdata/configs/config-v34.json +++ b/test/testdata/configs/config-v34.json @@ -30,7 +30,7 @@ "ConnectionsRateLimitingWindowSeconds": 1, "CrashDBDir": "", "DNSBootstrapID": ".algorand.network?backup=.algorand.net&dedup=.algorand-.(network|net)", - "DNSSecurityFlags": 1, + "DNSSecurityFlags": 9, "DeadlockDetection": 0, "DeadlockDetectionThreshold": 30, "DisableAPIAuth": false, @@ -43,6 +43,7 @@ "EnableAgreementTimeMetrics": false, "EnableAssembleStats": false, "EnableBlockService": false, + "EnableDHTProviders": false, "EnableDeveloperAPI": false, "EnableExperimentalAPI": false, "EnableFollowMode": false, @@ -53,6 +54,7 @@ "EnableMetricReporting": false, "EnableOutgoingNetworkMessageFiltering": true, "EnableP2P": false, + "EnableP2PHybridMode": false, "EnablePingHandler": true, "EnableProcessBlockStats": false, "EnableProfiler": false, @@ -96,6 +98,7 @@ "OptimizeAccountsDatabaseOnStartup": false, "OutgoingMessageFilterBucketCount": 3, "OutgoingMessageFilterBucketSize": 128, + "P2PNetAddress": "", "P2PPersistPeerID": false, "P2PPrivateKeyLocation": "", "ParticipationKeysRefreshInterval": 60000000000, diff --git a/test/testdata/deployednettemplates/recipes/hello-world-small-p2p/genesis.json b/test/testdata/deployednettemplates/recipes/hello-world-small-p2p/genesis.json new file mode 100644 index 0000000000..7ae67edf88 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/hello-world-small-p2p/genesis.json @@ -0,0 +1,30 @@ +{ + "NetworkName": "hello-p2p", + "VersionModifier": "", + "ConsensusProtocol": "future", + "FirstPartKeyRound": 0, + "LastPartKeyRound": 5000, + "PartKeyDilution": 0, + "Wallets": [ + { + "Name": "Wallet1", + "Stake": 25, + "Online": true + }, + { + "Name": "Wallet2", + "Stake": 25, + "Online": true + }, + { + "Name": "Wallet3", + "Stake": 25, + "Online": true + }, + { + "Name": "Wallet4", + "Stake": 25, + "Online": false + } + ] +} diff --git a/test/testdata/deployednettemplates/recipes/hello-world-small-p2p/net.json b/test/testdata/deployednettemplates/recipes/hello-world-small-p2p/net.json new file mode 100644 index 0000000000..423d31c1a4 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/hello-world-small-p2p/net.json @@ -0,0 +1,107 @@ +{ + "Hosts": [ + { + "Name": "R1", + "Nodes": [ + { + "Name": "relay1", + "IsRelay": true, + "Wallets": [ + { + "Name": "Wallet1", + "ParticipationOnly": false + } + ], + "NetAddress": "{{NetworkPort}}", + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "AdminAPIToken": "{{AdminAPIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "P2PBootstrap": true, + "ConfigJSONOverride": "{ \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"EnableProfiler\": true, \"EnableRuntimeMetrics\": true, \"EnableExperimentalAPI\": true, \"EnableAccountUpdatesStats\": true, \"EnableP2P\": true }" + } + ] + }, + { + "Name": "R2", + "Nodes": [ + { + "Name": "relay2", + "IsRelay": true, + "Wallets": [ + { + "Name": "Wallet2", + "ParticipationOnly": false + } + ], + "NetAddress": "{{NetworkPort}}", + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "AdminAPIToken": "{{AdminAPIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "P2PBootstrap": true, + "ConfigJSONOverride": "{ \"DNSBootstrapID\": \".algodev.network\",\"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"EnableProfiler\": true, \"EnableRuntimeMetrics\": true, \"EnableExperimentalAPI\": true, \"EnableAccountUpdatesStats\": true, \"EnableP2P\": true }" + } + ] + }, + { + "Name": "N1", + "Group": "", + "Nodes": [ + { + "Name": "node1", + "Wallets": [ + { + "Name": "Wallet3", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "AdminAPIToken": "{{AdminAPIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true, \"CadaverSizeTarget\": 0, \"EnableAccountUpdatesStats\": true, \"EnableProfiler\": true, \"EnableRuntimeMetrics\": true, \"EnableExperimentalAPI\": true, \"EnableAccountUpdatesStats\": true, \"EnableP2P\": true }" + } + ] + }, + { + "Name": "NPN1", + "Group": "", + "Nodes": [ + { + "Name": "nonParticipatingNode1", + "Wallets": [ + { + "Name": "Wallet4", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "AdminAPIToken": "{{AdminAPIToken}}", + "EnableTelemetry": false, + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4, \"CadaverSizeTarget\": 0, \"EnableProfiler\": true, \"EnableRuntimeMetrics\": true, \"EnableExperimentalAPI\": true, \"EnableAccountUpdatesStats\": true, \"EnableP2P\": true}" + } + ] + } + ] +} diff --git a/test/testdata/deployednettemplates/recipes/hello-world-small-p2p/recipe.json b/test/testdata/deployednettemplates/recipes/hello-world-small-p2p/recipe.json new file mode 100644 index 0000000000..a2f88f63b4 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/hello-world-small-p2p/recipe.json @@ -0,0 +1,7 @@ +{ + "GenesisFile":"genesis.json", + "NetworkFile":"net.json", + "ConfigFile": "../../configs/reference.json", + "HostTemplatesFile": "../../hosttemplates/hosttemplates.json", + "TopologyFile": "topology.json" +} diff --git a/test/testdata/deployednettemplates/recipes/hello-world-small-p2p/topology.json b/test/testdata/deployednettemplates/recipes/hello-world-small-p2p/topology.json new file mode 100644 index 0000000000..acc7cca9ec --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/hello-world-small-p2p/topology.json @@ -0,0 +1,20 @@ +{ + "Hosts": [ + { + "Name": "R1", + "Template": "AWS-US-EAST-1-Small" + }, + { + "Name": "R2", + "Template": "AWS-US-EAST-1-Small" + }, + { + "Name": "N1", + "Template": "AWS-US-EAST-1-Small" + }, + { + "Name": "NPN1", + "Template": "AWS-US-EAST-1-Small" + } + ] +} diff --git a/test/testdata/deployednettemplates/recipes/hello-world-tiny-p2p/genesis.json b/test/testdata/deployednettemplates/recipes/hello-world-tiny-p2p/genesis.json new file mode 100644 index 0000000000..7ae67edf88 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/hello-world-tiny-p2p/genesis.json @@ -0,0 +1,30 @@ +{ + "NetworkName": "hello-p2p", + "VersionModifier": "", + "ConsensusProtocol": "future", + "FirstPartKeyRound": 0, + "LastPartKeyRound": 5000, + "PartKeyDilution": 0, + "Wallets": [ + { + "Name": "Wallet1", + "Stake": 25, + "Online": true + }, + { + "Name": "Wallet2", + "Stake": 25, + "Online": true + }, + { + "Name": "Wallet3", + "Stake": 25, + "Online": true + }, + { + "Name": "Wallet4", + "Stake": 25, + "Online": false + } + ] +} diff --git a/test/testdata/deployednettemplates/recipes/hello-world/hosttemplates.json b/test/testdata/deployednettemplates/recipes/hello-world-tiny-p2p/hosttemplates.json similarity index 100% rename from test/testdata/deployednettemplates/recipes/hello-world/hosttemplates.json rename to test/testdata/deployednettemplates/recipes/hello-world-tiny-p2p/hosttemplates.json diff --git a/test/testdata/deployednettemplates/recipes/hello-world-tiny-p2p/net.json b/test/testdata/deployednettemplates/recipes/hello-world-tiny-p2p/net.json new file mode 100644 index 0000000000..8ea8328c62 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/hello-world-tiny-p2p/net.json @@ -0,0 +1,101 @@ +{ + "Hosts": [ + { + "Name": "R1", + "Nodes": [ + { + "Name": "relay1", + "IsRelay": true, + "Wallets": [ + { + "Name": "Wallet1", + "ParticipationOnly": false + } + ], + "NetAddress": "{{NetworkPort}}", + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "P2PBootstrap": true, + "ConfigJSONOverride": "{ \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"EnableP2P\": true }" + } + ] + }, + { + "Name": "R2", + "Nodes": [ + { + "Name": "relay2", + "IsRelay": true, + "Wallets": [ + { + "Name": "Wallet2", + "ParticipationOnly": false + } + ], + "NetAddress": "{{NetworkPort}}", + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": true, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": true, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": true, + "ConfigJSONOverride": "{ \"DNSBootstrapID\": \".algodev.network\",\"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"EnableAgreementReporting\": true, \"EnableAgreementTimeMetrics\": true, \"EnableAssembleStats\": true, \"EnableProcessBlockStats\": true, \"EnableP2P\": true }" + } + ] + }, + { + "Name": "N1", + "Group": "", + "Nodes": [ + { + "Name": "node1", + "Wallets": [ + { + "Name": "Wallet3", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "TelemetryURI": "{{TelemetryURI}}", + "EnableMetrics": false, + "MetricsURI": "{{MetricsURI}}", + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"PeerPingPeriodSeconds\": 30, \"BaseLoggerDebugLevel\": 4, \"EnableProfiler\": true, \"CadaverSizeTarget\": 0, \"EnableAccountUpdatesStats\": true, \"EnableP2P\": true }" + } + ] + }, + { + "Name": "NPN1", + "Group": "", + "Nodes": [ + { + "Name": "nonParticipatingNode1", + "Wallets": [ + { + "Name": "Wallet4", + "ParticipationOnly": false + } + ], + "APIEndpoint": "{{APIEndpoint}}", + "APIToken": "{{APIToken}}", + "EnableTelemetry": false, + "EnableMetrics": false, + "EnableService": false, + "EnableBlockStats": false, + "ConfigJSONOverride": "{ \"TxPoolExponentialIncreaseFactor\": 1, \"DNSBootstrapID\": \".algodev.network\", \"DeadlockDetection\": -1, \"BaseLoggerDebugLevel\": 4, \"CadaverSizeTarget\": 0, \"EnableP2P\": true }" + } + ] + } + ] +} diff --git a/test/testdata/deployednettemplates/recipes/hello-world-tiny-p2p/recipe.json b/test/testdata/deployednettemplates/recipes/hello-world-tiny-p2p/recipe.json new file mode 100644 index 0000000000..be6b71ec55 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/hello-world-tiny-p2p/recipe.json @@ -0,0 +1,7 @@ +{ + "GenesisFile":"genesis.json", + "NetworkFile":"net.json", + "ConfigFile": "../../configs/reference.json", + "HostTemplatesFile": "../../hosttemplates/t2micro-useast1.json", + "TopologyFile": "topology.json" +} diff --git a/test/testdata/deployednettemplates/recipes/hello-world-tiny-p2p/topology.json b/test/testdata/deployednettemplates/recipes/hello-world-tiny-p2p/topology.json new file mode 100644 index 0000000000..acc7cca9ec --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/hello-world-tiny-p2p/topology.json @@ -0,0 +1,20 @@ +{ + "Hosts": [ + { + "Name": "R1", + "Template": "AWS-US-EAST-1-Small" + }, + { + "Name": "R2", + "Template": "AWS-US-EAST-1-Small" + }, + { + "Name": "N1", + "Template": "AWS-US-EAST-1-Small" + }, + { + "Name": "NPN1", + "Template": "AWS-US-EAST-1-Small" + } + ] +} diff --git a/test/testdata/deployednettemplates/recipes/hello-world/genesis.json b/test/testdata/deployednettemplates/recipes/hello-world/genesis.json index 218b694d5f..b7fdd9502b 100644 --- a/test/testdata/deployednettemplates/recipes/hello-world/genesis.json +++ b/test/testdata/deployednettemplates/recipes/hello-world/genesis.json @@ -3,7 +3,7 @@ "VersionModifier": "", "ConsensusProtocol": "future", "FirstPartKeyRound": 0, - "LastPartKeyRound": 1000300, + "LastPartKeyRound": 5000, "PartKeyDilution": 0, "Wallets": [ { diff --git a/test/testdata/deployednettemplates/recipes/scenario1s-p2p/Makefile b/test/testdata/deployednettemplates/recipes/scenario1s-p2p/Makefile new file mode 100644 index 0000000000..f4ec4b3c1f --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/scenario1s-p2p/Makefile @@ -0,0 +1,23 @@ +# scenario1s is scenario1 but smaller, (100 nodes, 100 wallets) -> (20 nodes, 20 wallets), each algod gets single tenancy on a smaller ec2 instance +PARAMS=-w 20 -R 8 -N 20 -n 20 --npn-algod-nodes 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json + +.PHONY: clean all + +all: net.json genesis.json topology.json + +node.json nonPartNode.json relay.json: + python3 copy-node-configs.py + +net.json: node.json nonPartNode.json relay.json ${GOPATH}/bin/netgoal Makefile + netgoal generate -t net -r /tmp/wat -o net.json ${PARAMS} + +genesis.json: ${GOPATH}/bin/netgoal Makefile + netgoal generate -t genesis -r /tmp/wat -o genesis.l.json ${PARAMS} + jq '.LastPartKeyRound=5000|.NetworkName="s1s-p2p"|.ConsensusProtocol="future"' < genesis.l.json > genesis.json + rm genesis.l.json + +topology.json: ../scenario1s/gen_topology.py + python3 ../scenario1s/gen_topology.py + +clean: + rm -f net.json genesis.json node.json nonPartNode.json relay.json topology.json diff --git a/test/testdata/deployednettemplates/recipes/scenario1s-p2p/README.md b/test/testdata/deployednettemplates/recipes/scenario1s-p2p/README.md new file mode 100644 index 0000000000..1cad95bc2d --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/scenario1s-p2p/README.md @@ -0,0 +1,16 @@ +# Scenario1s for P2P testing + +This is a copy of scenario1s with the following changes in nodes configuration: +1. All nodes get `"EnableP2P": true` into their config. +1. All relays additionally get `"P2PBootstrap": true` to their netgoal config. + +## Build + +```sh +export GOPATH=~/go +make +``` + +## Run + +Run as usual cluster test scenario with algonet. diff --git a/test/testdata/deployednettemplates/recipes/scenario1s-p2p/copy-node-configs.py b/test/testdata/deployednettemplates/recipes/scenario1s-p2p/copy-node-configs.py new file mode 100644 index 0000000000..6ffbc01d8d --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/scenario1s-p2p/copy-node-configs.py @@ -0,0 +1,55 @@ +""" +Copies node.json, relay.json and nonPartNode.json from scenario1s: +1. Append \"EnableP2P\": true to all configs +2. Set P2PBootstrap: true to relay.json +3. Set DNSSecurityFlags: 0 to all configs +""" + +import json +import os + +CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) +SCENARIO1S_DIR = os.path.join(CURRENT_DIR, "..", "scenario1s") + +def main(): + """main""" + with open(os.path.join(SCENARIO1S_DIR, "node.json"), "r") as f: + node = json.load(f) + with open(os.path.join(SCENARIO1S_DIR, "relay.json"), "r") as f: + relay = json.load(f) + with open(os.path.join(SCENARIO1S_DIR, "nonPartNode.json"), "r") as f: + non_part_node = json.load(f) + + # make all relays P2PBootstrap'able + relay["P2PBootstrap"] = True + + # enable P2P for all configs + for config in (node, relay, non_part_node): + override = config.get("ConfigJSONOverride") + if override: + override_json = json.loads(override) + override_json["EnableP2P"] = True + override_json["DNSSecurityFlags"] = 0x8000 # set to some unused value otherwise 0 would be migrated to default that enables DNSSEC + config["ConfigJSONOverride"] = json.dumps(override_json) + altconfigs = config.get("AltConfigs", []) + if altconfigs: + for i, altconfig in enumerate(altconfigs): + override = altconfig.get("ConfigJSONOverride") + if override: + override_json = json.loads(override) + override_json["EnableP2P"] = True + override_json["DNSSecurityFlags"] = 0x8000 # set to some unused value otherwise 0 would be migrated to default that enables DNSSEC + altconfigs[i]["ConfigJSONOverride"] = json.dumps(override_json) + config["AltConfigs"] = altconfigs + + with open("node.json", "w") as f: + json.dump(node, f, indent=4) + with open("relay.json", "w") as f: + json.dump(relay, f, indent=4) + with open("nonPartNode.json", "w") as f: + json.dump(non_part_node, f, indent=4) + + print("Done!") + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/test/testdata/deployednettemplates/recipes/scenario1s-p2p/recipe.json b/test/testdata/deployednettemplates/recipes/scenario1s-p2p/recipe.json new file mode 100644 index 0000000000..a2f88f63b4 --- /dev/null +++ b/test/testdata/deployednettemplates/recipes/scenario1s-p2p/recipe.json @@ -0,0 +1,7 @@ +{ + "GenesisFile":"genesis.json", + "NetworkFile":"net.json", + "ConfigFile": "../../configs/reference.json", + "HostTemplatesFile": "../../hosttemplates/hosttemplates.json", + "TopologyFile": "topology.json" +} diff --git a/test/testdata/deployednettemplates/recipes/scenario1s/Makefile b/test/testdata/deployednettemplates/recipes/scenario1s/Makefile index ed8a70132e..8b83c38b6c 100644 --- a/test/testdata/deployednettemplates/recipes/scenario1s/Makefile +++ b/test/testdata/deployednettemplates/recipes/scenario1s/Makefile @@ -1,14 +1,14 @@ # scenario1s is scenario1 but smaller, (100 nodes, 100 wallets) -> (20 nodes, 20 wallets), each algod gets single tenancy on a smaller ec2 instance PARAMS=-w 20 -R 8 -N 20 -n 20 --npn-algod-nodes 10 --node-template node.json --relay-template relay.json --non-participating-node-template nonPartNode.json -all: net.json genesis.json topology.json bootstrappedFile.json +all: net.json genesis.json topology.json net.json: node.json nonPartNode.json ${GOPATH}/bin/netgoal Makefile netgoal generate -t net -r /tmp/wat -o net.json ${PARAMS} genesis.json: ${GOPATH}/bin/netgoal Makefile netgoal generate -t genesis -r /tmp/wat -o genesis.l.json ${PARAMS} - jq '.LastPartKeyRound=22000|.NetworkName="s1s"|.ConsensusProtocol="future"' < genesis.l.json > genesis.json + jq '.LastPartKeyRound=5000|.NetworkName="s1s"|.ConsensusProtocol="future"' < genesis.l.json > genesis.json rm genesis.l.json topology.json: gen_topology.py diff --git a/tools/block-generator/go.mod b/tools/block-generator/go.mod index cfeb442668..9baf9fafef 100644 --- a/tools/block-generator/go.mod +++ b/tools/block-generator/go.mod @@ -48,22 +48,33 @@ require ( github.com/elastic/gosigar v0.14.2 // indirect github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect github.com/google/uuid v1.4.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/ipfs/boxo v0.10.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect + github.com/ipfs/go-datastore v0.6.0 // indirect + github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-log/v2 v2.5.1 // indirect + github.com/ipld/go-ipld-prime v0.20.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect + github.com/jbenet/goprocess v0.1.4 // indirect github.com/jmespath/go-jmespath v0.3.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.17.6 // indirect @@ -72,10 +83,14 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect + github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p v0.33.2 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.24.3 // indirect + github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect github.com/libp2p/go-libp2p-pubsub v0.10.0 // indirect + github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect @@ -104,10 +119,12 @@ require ( github.com/olivere/elastic v6.2.14+incompatible // indirect github.com/onsi/ginkgo/v2 v2.15.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/common v0.47.0 // indirect @@ -120,6 +137,11 @@ require ( github.com/sirupsen/logrus v1.8.1 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/otel v1.16.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect + go.opentelemetry.io/otel/trace v1.16.0 // indirect go.uber.org/dig v1.17.1 // indirect go.uber.org/fx v1.20.1 // indirect go.uber.org/mock v0.4.0 // indirect @@ -133,6 +155,7 @@ require ( golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.18.0 // indirect + gonum.org/v1/gonum v0.13.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/sohlich/elogrus.v3 v3.0.0-20180410122755-1fa29e2f2009 // indirect lukechampine.com/blake3 v1.2.1 // indirect diff --git a/tools/block-generator/go.sum b/tools/block-generator/go.sum index dbd7cdeb72..5ed96a265b 100644 --- a/tools/block-generator/go.sum +++ b/tools/block-generator/go.sum @@ -118,6 +118,7 @@ github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0 github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= @@ -131,6 +132,8 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= @@ -141,12 +144,16 @@ github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aev github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= @@ -162,6 +169,9 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -174,7 +184,9 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -185,7 +197,9 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= @@ -196,12 +210,16 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 h1:E/LAvt58di64hlYjx7AsNS6C/ysHWYo+2qPCZKTQhRo= github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -209,7 +227,14 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/ github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -221,18 +246,34 @@ github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/C github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/ipfs/boxo v0.10.0 h1:tdDAxq8jrsbRkYoF+5Rcqyeb91hgWe2hp7iLu7ORZLY= +github.com/ipfs/boxo v0.10.0/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= +github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= +github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= +github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= +github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= +github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= +github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= +github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= +github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= +github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= +github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= +github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= @@ -240,6 +281,7 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= @@ -278,14 +320,22 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= +github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= +github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= github.com/libp2p/go-libp2p v0.33.2 h1:vCdwnFxoGOXMKmaGHlDSnL4bM3fQeW8pgIa9DECnb40= github.com/libp2p/go-libp2p v0.33.2/go.mod h1:zTeppLuCvUIkT118pFVzA8xzP/p2dJYOMApCkFh0Yww= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= +github.com/libp2p/go-libp2p-kad-dht v0.24.3 h1:VjxtDVWaaf4UFjGBf+yl2JCiGaHx7+ctAUa9oJCR3QE= +github.com/libp2p/go-libp2p-kad-dht v0.24.3/go.mod h1:BShPzRbK6+fN3hk8a0WGAYKpb8m4k+DtchkqouGTrSg= +github.com/libp2p/go-libp2p-kbucket v0.6.3 h1:p507271wWzpy2f1XxPzCQG9NiN6R6lHL9GiSErbQQo0= +github.com/libp2p/go-libp2p-kbucket v0.6.3/go.mod h1:RCseT7AH6eJWxxk2ol03xtP9pEHetYSPXOaJnOiD8i0= github.com/libp2p/go-libp2p-pubsub v0.10.0 h1:wS0S5FlISavMaAbxyQn3dxMOe2eegMfswM471RuHJwA= github.com/libp2p/go-libp2p-pubsub v0.10.0/go.mod h1:1OxbaT/pFRO5h+Dpze8hdHQ63R0ke55XTs6b6NwLLkw= +github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= +github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= @@ -388,6 +438,8 @@ github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8P github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= @@ -402,6 +454,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= +github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= @@ -423,6 +477,7 @@ github.com/quic-go/webtransport-go v0.6.0 h1:CvNsKqc4W2HljHJnoT+rMmbRJybShZ0YPFD github.com/quic-go/webtransport-go v0.6.0/go.mod h1:9KjU4AEBqEQidGHNDkZrb8CAa1abRaosM2yGOyiikEc= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= @@ -460,7 +515,11 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -476,6 +535,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -484,12 +544,16 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= @@ -497,6 +561,10 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= +github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= +github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -510,6 +578,15 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -522,9 +599,12 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= @@ -534,6 +614,7 @@ golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -552,6 +633,7 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -578,6 +660,7 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -655,6 +738,9 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -667,6 +753,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM= +gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= @@ -681,6 +769,7 @@ google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= @@ -688,13 +777,18 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -702,6 +796,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= @@ -726,6 +821,7 @@ honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= pgregory.net/rapid v0.6.2 h1:ErW5sL+UKtfBfUTsWHDCoeB+eZKLKMxrSd1VJY6W4bw= diff --git a/tools/debug/algodump/main.go b/tools/debug/algodump/main.go index 4899fb0c73..525ddba705 100644 --- a/tools/debug/algodump/main.go +++ b/tools/debug/algodump/main.go @@ -179,7 +179,11 @@ func main() { *genesisID, protocol.NetworkID(*networkID)) setDumpHandlers(n) - n.Start() + err := n.Start() + if err != nil { + log.Errorf("Failed to start network: %v", err) + return + } for { time.Sleep(time.Second) diff --git a/tools/debug/transplanter/main.go b/tools/debug/transplanter/main.go index 1a41504c99..e6f88d007e 100644 --- a/tools/debug/transplanter/main.go +++ b/tools/debug/transplanter/main.go @@ -393,7 +393,11 @@ func main() { os.Exit(1) } - followerNode.Start() + err = followerNode.Start() + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot start follower node: %v", err) + os.Exit(1) + } for followerNode.Ledger().Latest() < basics.Round(*roundStart) { fmt.Printf("At round %d, waiting for %d\n", followerNode.Ledger().Latest(), *roundStart) diff --git a/util/metrics/counter.go b/util/metrics/counter.go index 38852386d8..e9b437a4a2 100644 --- a/util/metrics/counter.go +++ b/util/metrics/counter.go @@ -28,6 +28,14 @@ type Counter struct { // MakeCounter create a new counter with the provided name and description. func MakeCounter(metric MetricName) *Counter { + c := makeCounter(metric) + c.Register(nil) + return c +} + +// makeCounter create a new counter with the provided name and description +// but does not register it with the default registry. +func makeCounter(metric MetricName) *Counter { c := &Counter{c: couge{ values: make([]*cougeValues, 0), description: metric.Description, @@ -35,7 +43,6 @@ func MakeCounter(metric MetricName) *Counter { labels: make(map[string]int), valuesIndices: make(map[int]int), }} - c.Register(nil) return c } diff --git a/util/metrics/gauge.go b/util/metrics/gauge.go index bbc143a14f..edf144e48f 100644 --- a/util/metrics/gauge.go +++ b/util/metrics/gauge.go @@ -27,6 +27,14 @@ type Gauge struct { // MakeGauge create a new gauge with the provided name and description. func MakeGauge(metric MetricName) *Gauge { + c := makeGauge(metric) + c.Register(nil) + return c +} + +// makeGauge create a new gauge with the provided name and description +// but does not register it with the default registry. +func makeGauge(metric MetricName) *Gauge { c := &Gauge{g: couge{ values: make([]*cougeValues, 0), description: metric.Description, @@ -34,7 +42,6 @@ func MakeGauge(metric MetricName) *Gauge { labels: make(map[string]int), valuesIndices: make(map[int]int), }} - c.Register(nil) return c } diff --git a/util/metrics/opencensus.go b/util/metrics/opencensus.go new file mode 100644 index 0000000000..fefb1d054b --- /dev/null +++ b/util/metrics/opencensus.go @@ -0,0 +1,172 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +// Functions for opencensus stats aggs conversion to our internal data type +// suitable for further reporting + +package metrics + +import ( + "context" + "strings" + + "go.opencensus.io/metric/metricdata" + "go.opencensus.io/metric/metricexport" + "golang.org/x/exp/slices" +) + +type defaultOpencensusGatherer struct { + names []string +} + +// WriteMetric return opencensus data converted to algorand format +func (og *defaultOpencensusGatherer) WriteMetric(buf *strings.Builder, parentLabels string) { + metrics := collectOpenCensusMetrics(og.names) + for _, metric := range metrics { + metric.WriteMetric(buf, parentLabels) + } +} + +// AddMetric return opencensus data converted to algorand format +func (og *defaultOpencensusGatherer) AddMetric(values map[string]float64) { + metrics := collectOpenCensusMetrics(og.names) + for _, metric := range metrics { + metric.AddMetric(values) + } +} + +type statExporter struct { + names map[string]struct{} + metrics []Metric +} + +func collectOpenCensusMetrics(names []string) []Metric { + exporter := &statExporter{} + if len(names) > 0 { + exporter.names = make(map[string]struct{}, len(names)) + for _, name := range names { + exporter.names[name] = struct{}{} + } + } + reader := metricexport.NewReader() + reader.ReadAndExport(exporter) + + return exporter.metrics +} + +// statCounter stores single int64 value per stat with labels +type statCounter struct { + name string + description string + labels []map[string]string + values []int64 +} + +// WriteMetric outputs Prometheus metrics for all labels/values in statCounter +func (st *statCounter) WriteMetric(buf *strings.Builder, parentLabels string) { + name := sanitizePrometheusName(st.name) + counter := makeCounter(MetricName{name, st.description}) + for i := 0; i < len(st.labels); i++ { + counter.AddUint64(uint64(st.values[i]), st.labels[i]) + } + counter.WriteMetric(buf, parentLabels) +} + +// AddMetric outputs all statCounter's labels/values into a map +func (st *statCounter) AddMetric(values map[string]float64) { + counter := makeCounter(MetricName{st.name, st.description}) + for i := 0; i < len(st.labels); i++ { + counter.AddUint64(uint64(st.values[i]), st.labels[i]) + } + counter.AddMetric(values) +} + +// statCounter stores single float64 sun value per stat with labels +type statDistribution struct { + name string + description string + labels []map[string]string + values []float64 +} + +// WriteMetric outputs Prometheus metrics for all labels/values in statCounter +func (st *statDistribution) WriteMetric(buf *strings.Builder, parentLabels string) { + name := sanitizePrometheusName(st.name) + gauge := makeGauge(MetricName{name, st.description}) + for i := 0; i < len(st.labels); i++ { + gauge.SetLabels(uint64(st.values[i]), st.labels[i]) + } + gauge.WriteMetric(buf, parentLabels) +} + +// AddMetric outputs all statCounter's labels/values into a map +func (st *statDistribution) AddMetric(values map[string]float64) { + gauge := makeGauge(MetricName{st.name, st.description}) + for i := 0; i < len(st.labels); i++ { + gauge.SetLabels(uint64(st.values[i]), st.labels[i]) + } + gauge.AddMetric(values) +} + +func (s *statExporter) ExportMetrics(ctx context.Context, data []*metricdata.Metric) error { + labeler := func(lk []metricdata.LabelKey, lv []metricdata.LabelValue, ignores ...string) map[string]string { + // default labeler concatenates labels + labels := make(map[string]string, len(lk)) + for i := range lk { + if lv[i].Present && (len(ignores) == 0 || len(ignores) > 0 && !slices.Contains(ignores, lk[i].Key)) { + labels[lk[i].Key] = lv[i].Value + } + } + return labels + } + + for _, m := range data { + if _, ok := s.names[m.Descriptor.Name]; len(s.names) > 0 && !ok { + continue + } + if m.Descriptor.Type == metricdata.TypeCumulativeInt64 { + counter := statCounter{ + name: m.Descriptor.Name, + description: m.Descriptor.Description, + } + for _, d := range m.TimeSeries { + // ignore a known useless instance_id label + labels := labeler(m.Descriptor.LabelKeys, d.LabelValues, "instance_id") + counter.labels = append(counter.labels, labels) + counter.values = append(counter.values, d.Points[0].Value.(int64)) + } + + s.metrics = append(s.metrics, &counter) + } else if m.Descriptor.Type == metricdata.TypeCumulativeDistribution { + // TODO: the metrics below cannot be integer gauge, and Sum statistic does not make any sense. + // libp2p.io/dht/kad/outbound_request_latency + // libp2p.io/dht/kad/inbound_request_latency + // Ignore? + dist := statDistribution{ + name: m.Descriptor.Name, + description: m.Descriptor.Description, + } + // check if we are processing a known DHT metric + for _, d := range m.TimeSeries { + label := labeler(m.Descriptor.LabelKeys, d.LabelValues, "instance_id") + dist.labels = append(dist.labels, label) + dist.values = append(dist.values, d.Points[0].Value.(*metricdata.Distribution).Sum) + } + s.metrics = append(s.metrics, &dist) + } + } + return nil +} diff --git a/util/metrics/opencensus_test.go b/util/metrics/opencensus_test.go new file mode 100644 index 0000000000..f5401af541 --- /dev/null +++ b/util/metrics/opencensus_test.go @@ -0,0 +1,147 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package metrics + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + + "github.com/algorand/go-algorand/test/partitiontest" + "github.com/stretchr/testify/require" +) + +// TestDHTOpenCensusMetrics ensures both count and distribution stats are properly converted to our metrics +func TestDHTOpenCensusMetrics(t *testing.T) { + partitiontest.PartitionTest(t) + + defaultBytesDistribution := view.Distribution(1024, 2048, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864, 268435456, 1073741824, 4294967296) + + keyMessageType := tag.MustNewKey("message_type") + keyPeerID := tag.MustNewKey("peer_id") + keyInstanceID := tag.MustNewKey("instance_id") + + sentMessages := stats.Int64("my_sent_messages", "Total number of messages sent per RPC", stats.UnitDimensionless) + receivedBytes := stats.Int64("my_received_bytes", "Total received bytes per RPC", stats.UnitBytes) + + receivedBytesView := &view.View{ + Measure: receivedBytes, + TagKeys: []tag.Key{keyMessageType, keyPeerID, keyInstanceID}, + Aggregation: defaultBytesDistribution, + } + sentMessagesView := &view.View{ + Measure: sentMessages, + TagKeys: []tag.Key{keyMessageType, keyPeerID, keyInstanceID}, + Aggregation: view.Count(), + } + + err := view.Register(receivedBytesView, sentMessagesView) + require.NoError(t, err) + defer view.Unregister(receivedBytesView, sentMessagesView) + + ctx := context.Background() + tags1 := []tag.Mutator{ + tag.Upsert(keyMessageType, "UNKNOWN"), + tag.Upsert(keyPeerID, "1234"), + tag.Upsert(keyInstanceID, fmt.Sprintf("%p", t)), + } + ctx1, _ := tag.New(ctx, tags1...) + + stats.Record(ctx1, + sentMessages.M(1), + receivedBytes.M(int64(100)), + ) + + tags2 := []tag.Mutator{ + tag.Upsert(keyMessageType, "ADD_PROVIDER"), + tag.Upsert(keyPeerID, "abcd"), + tag.Upsert(keyInstanceID, fmt.Sprintf("%p", t)), + } + ctx2, _ := tag.New(ctx, tags2...) + + stats.Record(ctx2, + sentMessages.M(1), + receivedBytes.M(int64(123)), + ) + + // first check some metrics are collected when no names provided + // cannot assert on specific values because network tests might run in parallel with this package + // and produce some metric under specific configuration + require.Eventually(t, func() bool { + // stats are written by a background goroutine, give it a chance to finish + metrics := collectOpenCensusMetrics(nil) + return len(metrics) >= 2 + }, 10*time.Second, 20*time.Millisecond) + + // now assert on specific names and values + metrics := collectOpenCensusMetrics([]string{"my_sent_messages", "my_received_bytes"}) + require.Len(t, metrics, 2) + for _, m := range metrics { + var buf strings.Builder + m.WriteMetric(&buf, "") + promValue := buf.String() + if strings.Contains(promValue, "my_sent_messages") { + require.Contains(t, promValue, "my_sent_messages counter\n") + require.Contains(t, promValue, `peer_id="abcd"`) + require.Contains(t, promValue, `peer_id="1234"`) + require.Contains(t, promValue, `message_type="ADD_PROVIDER"`) + require.Contains(t, promValue, `message_type="UNKNOWN"`) + require.Contains(t, promValue, "} 1\n") + } else if strings.Contains(promValue, "my_received_bytes") { + require.Contains(t, promValue, "my_received_bytes gauge\n") + require.Contains(t, promValue, `peer_id="1234"`) + require.Contains(t, promValue, `peer_id="abcd"`) + require.Contains(t, promValue, `message_type="ADD_PROVIDER"`) + require.Contains(t, promValue, `message_type="UNKNOWN"`) + require.Contains(t, promValue, "} 123\n") + require.Contains(t, promValue, "} 100\n") + } else { + require.Fail(t, "not expected metric", promValue) + } + + values := make(map[string]float64) + m.AddMetric(values) + for k, v := range values { + require.True(t, strings.Contains(k, "message_type__ADD_PROVIDER") || strings.Contains(k, "message_type__UNKNOWN")) + require.True(t, strings.Contains(k, "peer_id__1234") || strings.Contains(k, "peer_id__abcd")) + if strings.Contains(k, "my_sent_messages") { + require.Equal(t, v, float64(1)) + } else if strings.Contains(k, "my_received_bytes") { + require.True(t, v == 100 || v == 123) + } else { + require.Fail(t, "not expected metric key", k) + } + } + } + + // ensure the exported gatherer works + reg := MakeRegistry() + reg.Register(&OpencensusDefaultMetrics) + defer reg.Deregister(&OpencensusDefaultMetrics) + + var buf strings.Builder + reg.WriteMetrics(&buf, "") + + require.Contains(t, buf.String(), "my_sent_messages") + require.Contains(t, buf.String(), "my_received_bytes") +} diff --git a/util/metrics/prometheus.go b/util/metrics/prometheus.go new file mode 100644 index 0000000000..b55f931001 --- /dev/null +++ b/util/metrics/prometheus.go @@ -0,0 +1,106 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +// Functions for Prometheus metrics conversion to our internal data type +// suitable for further reporting + +package metrics + +import ( + "strings" + + "github.com/prometheus/client_golang/prometheus" + iopc "github.com/prometheus/client_model/go" +) + +type defaultPrometheusGatherer struct { + names []string +} + +// WriteMetric return prometheus converted to algorand format. +// Supports only counter and gauge types and ignores go_ metrics. +func (pg *defaultPrometheusGatherer) WriteMetric(buf *strings.Builder, parentLabels string) { + metrics := collectPrometheusMetrics(pg.names) + for _, metric := range metrics { + metric.WriteMetric(buf, parentLabels) + } +} + +// AddMetric return prometheus data converted to algorand format. +// Supports only counter and gauge types and ignores go_ metrics. +func (pg *defaultPrometheusGatherer) AddMetric(values map[string]float64) { + metrics := collectPrometheusMetrics(pg.names) + for _, metric := range metrics { + metric.AddMetric(values) + } +} + +func collectPrometheusMetrics(names []string) []Metric { + var result []Metric + var namesMap map[string]struct{} + if len(names) > 0 { + namesMap = make(map[string]struct{}, len(names)) + for _, name := range names { + namesMap[name] = struct{}{} + } + } + + convertLabels := func(m *iopc.Metric) map[string]string { + var labels map[string]string + if lbls := m.GetLabel(); len(lbls) > 0 { + labels = make(map[string]string, len(lbls)) + for _, lbl := range lbls { + labels[lbl.GetName()] = lbl.GetValue() + } + } + return labels + } + metrics, _ := prometheus.DefaultGatherer.Gather() + for _, metric := range metrics { + if strings.HasPrefix(metric.GetName(), "go_") { + continue + } + if _, ok := namesMap[metric.GetName()]; len(namesMap) > 0 && ok || len(namesMap) == 0 { + if metric.GetType() == iopc.MetricType_COUNTER && metric.GetMetric() != nil { + counter := makeCounter(MetricName{metric.GetName(), metric.GetHelp()}) + ma := metric.GetMetric() + for _, m := range ma { + if m.GetCounter() == nil { + continue + } + val := uint64(m.GetCounter().GetValue()) + labels := convertLabels(m) + counter.AddUint64(val, labels) + } + result = append(result, counter) + } else if metric.GetType() == iopc.MetricType_GAUGE && metric.GetMetric() != nil { + gauge := makeGauge(MetricName{metric.GetName(), metric.GetHelp()}) + + ma := metric.GetMetric() + for _, m := range ma { + if m.GetGauge() == nil { + continue + } + val := uint64(m.GetGauge().GetValue()) + labels := convertLabels(m) + gauge.SetLabels(val, labels) + } + result = append(result, gauge) + } + } + } + return result +} diff --git a/util/metrics/prometheus_test.go b/util/metrics/prometheus_test.go new file mode 100644 index 0000000000..75ef94f97e --- /dev/null +++ b/util/metrics/prometheus_test.go @@ -0,0 +1,148 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +// Functions for Prometheus metrics conversion to our internal data type +// suitable for further reporting + +package metrics + +import ( + "strings" + "testing" + "time" + + "github.com/algorand/go-algorand/test/partitiontest" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" +) + +func TestPrometheusMetrics(t *testing.T) { + partitiontest.PartitionTest(t) + + const metricNamespace = "test_metric" + + // gauge vec with labels + gaugeLabels := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "streams", + Help: "Number of Streams", + }, []string{"dir", "scope", "protocol"}) + + // gauge without labels + gauge := prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: "protocols_count", + Help: "Protocols Count", + }, + ) + + // counter with labels + counterLabels := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "identify_total", + Help: "Identify", + }, + []string{"dir"}, + ) + + // counter without labels + counter := prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Name: "counter_total", + Help: "Counter", + }, + ) + + prometheus.DefaultRegisterer.MustRegister(gaugeLabels) + prometheus.DefaultRegisterer.MustRegister(gauge) + prometheus.DefaultRegisterer.MustRegister(counterLabels) + prometheus.DefaultRegisterer.MustRegister(counter) + + defer prometheus.DefaultRegisterer.Unregister(gaugeLabels) + defer prometheus.DefaultRegisterer.Unregister(gauge) + defer prometheus.DefaultRegisterer.Unregister(counterLabels) + defer prometheus.DefaultRegisterer.Unregister(counter) + + // set some values + tags := []string{"outbound", "protocol", "/test/proto"} + gaugeLabels.WithLabelValues(tags...).Set(float64(1)) + + gauge.Set(float64(2)) + + tags = []string{"inbound"} + counterLabels.WithLabelValues(tags...).Add(float64(3)) + + counter.Add(float64(4)) + + // wait they collected and ready for gathering + require.Eventually(t, func() bool { + metrics := collectPrometheusMetrics(nil) + return len(metrics) >= 4 + }, 5*time.Second, 100*time.Millisecond) + + metrics := collectPrometheusMetrics([]string{ + metricNamespace + "_streams", + metricNamespace + "_protocols_count", + metricNamespace + "_identify_total", + metricNamespace + "_counter_total"}) + require.Len(t, metrics, 4) + + for _, m := range metrics { + buf := strings.Builder{} + m.WriteMetric(&buf, "") + promValue := buf.String() + if strings.Contains(promValue, metricNamespace+"_streams") { + require.Contains(t, promValue, metricNamespace+"_streams gauge\n") + require.Contains(t, promValue, metricNamespace+"_streams{") + // map/labels order is not guaranteed + require.Contains(t, promValue, "dir=\"outbound\"") + require.Contains(t, promValue, "protocol=\"/test/proto\"") + require.Contains(t, promValue, "scope=\"protocol\"") + require.Contains(t, promValue, "} 1\n") + } else if strings.Contains(promValue, metricNamespace+"_protocols_count") { + require.Contains(t, promValue, metricNamespace+"_protocols_count gauge\n") + require.Contains(t, promValue, metricNamespace+"_protocols_count 2\n") + } else if strings.Contains(promValue, metricNamespace+"_identify_total") { + require.Contains(t, promValue, metricNamespace+"_identify_total counter\n") + require.Contains(t, promValue, metricNamespace+"_identify_total{dir=\"inbound\"} 3\n") + } else if strings.Contains(promValue, metricNamespace+"_counter_total") { + require.Contains(t, promValue, metricNamespace+"_counter_total counter\n") + require.Contains(t, promValue, metricNamespace+"_counter_total 4\n") + } else { + require.Fail(t, "not expected metric", promValue) + } + + values := make(map[string]float64) + m.AddMetric(values) + require.Len(t, values, 1) + } + + // ensure the exported gatherer works + reg := MakeRegistry() + reg.Register(&PrometheusDefaultMetrics) + defer reg.Deregister(&PrometheusDefaultMetrics) + + var buf strings.Builder + reg.WriteMetrics(&buf, "") + + require.Contains(t, buf.String(), metricNamespace+"_streams") + require.Contains(t, buf.String(), metricNamespace+"_protocols_count") + require.Contains(t, buf.String(), metricNamespace+"_identify_total") + require.Contains(t, buf.String(), metricNamespace+"_counter_total") +} diff --git a/util/metrics/registry.go b/util/metrics/registry.go index 43078bb4c2..d525bc1833 100644 --- a/util/metrics/registry.go +++ b/util/metrics/registry.go @@ -37,6 +37,12 @@ func DefaultRegistry() *Registry { return defaultRegistry } +// PrometheusDefaultMetrics is the default prometheus gatherer implementing the Metric interface +var PrometheusDefaultMetrics = defaultPrometheusGatherer{} + +// OpencensusDefaultMetrics is the default prometheus gatherer implementing the Metric interface +var OpencensusDefaultMetrics = defaultOpencensusGatherer{} + func init() { defaultRegistry = MakeRegistry() }