diff --git a/pkg/networkservice/chains/nsmgr/dns_test.go b/pkg/networkservice/chains/nsmgr/dns_test.go index 4a97baafd..0072ed642 100644 --- a/pkg/networkservice/chains/nsmgr/dns_test.go +++ b/pkg/networkservice/chains/nsmgr/dns_test.go @@ -1,6 +1,6 @@ // Copyright (c) 2020-2022 Doc.ai and/or its affiliates. // -// Copyright (c) 2023 Cisco and/or its affiliates. +// Copyright (c) 2023-2024 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -24,6 +24,7 @@ package nsmgr_test import ( "context" "net" + "net/url" "testing" "time" @@ -31,6 +32,7 @@ import ( "github.com/networkservicemesh/api/pkg/api/networkservice" "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/cls" kernelmech "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/kernel" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "go.uber.org/goleak" @@ -55,6 +57,53 @@ func requireIPv4Lookup(ctx context.Context, t *testing.T, r *net.Resolver, host, require.Equal(t, expected, addrs[0].String()) } +func Test_LocalUsecase_DNSTarget(t *testing.T) { + t.Cleanup(func() { goleak.VerifyNone(t) }) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + domain := sandbox.NewBuilder(ctx, t). + SetNodesCount(1). + SetNSMgrProxySupplier(nil). + SetRegistryProxySupplier(nil). + SetupDefaultDNSServer(). + Build() + + nsRegistryClient := domain.NewNSRegistryClient(ctx, sandbox.GenerateTestToken) + + nsReg, err := nsRegistryClient.Register(ctx, defaultRegistryService(t.Name())) + require.NoError(t, err) + + nseReg := defaultRegistryEndpoint(nsReg.Name) + + nse := domain.Nodes[0].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken) + + var addr, _ = url.Parse("dns://" + domain.DNSServer.URL.Host + "/nsmgr-0.cluster.local:" + domain.Nodes[0].NSMgr.URL.Port()) + logrus.Error(addr) + nsc := domain.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken, client.WithClientURL(addr)) + + request := &networkservice.NetworkServiceRequest{ + MechanismPreferences: []*networkservice.Mechanism{ + {Cls: cls.LOCAL, Type: kernelmech.MECHANISM}, + }, + Connection: &networkservice.Connection{ + Id: "1", + NetworkService: nsReg.Name, + Labels: make(map[string]string), + }, + } + + conn, err := nsc.Request(ctx, request) + require.NoError(t, err) + + _, err = nsc.Close(ctx, conn) + require.NoError(t, err) + + _, err = nse.Unregister(ctx, nseReg) + require.NoError(t, err) +} + func Test_DNSUsecase(t *testing.T) { t.Cleanup(func() { goleak.VerifyNone(t) }) diff --git a/pkg/networkservice/chains/nsmgr/single_test.go b/pkg/networkservice/chains/nsmgr/single_test.go index e34e48787..14a34a857 100644 --- a/pkg/networkservice/chains/nsmgr/single_test.go +++ b/pkg/networkservice/chains/nsmgr/single_test.go @@ -1,6 +1,6 @@ // Copyright (c) 2020-2022 Doc.ai and/or its affiliates. // -// Copyright (c) 2023 Cisco and/or its affiliates. +// Copyright (c) 2023-2024 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -530,6 +530,7 @@ func Test_FailedRegistryAuthorization(t *testing.T) { ctx context.Context, tokenGenerator token.GeneratorFunc, expiryDuration time.Duration, + nsmgrProxyURL *url.URL, proxyRegistryURL *url.URL, options ...grpc.DialOption) registry.Registry { registryName := sandbox.UniqueName("registry-memory") @@ -537,6 +538,7 @@ func Test_FailedRegistryAuthorization(t *testing.T) { return memory.NewServer( ctx, tokenGeneratorFunc("spiffe://test.com/"+registryName), + memory.WithNSMgrProxyURL(nsmgrProxyURL), memory.WithProxyRegistryURL(proxyRegistryURL), memory.WithDefaultExpiration(expiryDuration), memory.WithDialOptions(options...), @@ -773,12 +775,14 @@ func Test_Expire(t *testing.T) { ctx context.Context, tokenGenerator token.GeneratorFunc, expiryDuration time.Duration, + nsmgrProxyURL *url.URL, proxyRegistryURL *url.URL, options ...grpc.DialOption) registry.Registry { return memory.NewServer( ctx, tokenGenerator, memory.WithProxyRegistryURL(proxyRegistryURL), + memory.WithNSMgrProxyURL(nsmgrProxyURL), memory.WithDefaultExpiration(expiryDuration), memory.WithDialOptions(options...), memory.WithAuthorizeNSRegistryServer( diff --git a/pkg/networkservice/chains/nsmgr/vl3_test.go b/pkg/networkservice/chains/nsmgr/vl3_test.go index 813e60e74..b676991f3 100644 --- a/pkg/networkservice/chains/nsmgr/vl3_test.go +++ b/pkg/networkservice/chains/nsmgr/vl3_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Cisco and/or its affiliates. +// Copyright (c) 2022-2024 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -325,20 +325,17 @@ func Test_Interdomain_vl3_dns(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer cancel() - var dnsServer = sandbox.NewFakeResolver() - - cluster1 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSResolver(dnsServer). - SetDNSDomainName("cluster1"). - Build() - - cluster2 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSDomainName("cluster2"). - SetDNSResolver(dnsServer). + var domains = sandbox.NewInterdomainBuilder(ctx, t). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster1") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster2") + }). Build() + cluster1 := domains[0] + cluster2 := domains[1] nsRegistryClient := cluster2.NewNSRegistryClient(ctx, sandbox.GenerateTestToken) nsReg, err := nsRegistryClient.Register(ctx, defaultRegistryService("vl3")) @@ -418,27 +415,21 @@ func Test_FloatingInterdomain_vl3_dns(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer cancel() - var dnsServer = sandbox.NewFakeResolver() - - cluster1 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSResolver(dnsServer). - SetDNSDomainName("cluster1"). + var domains = sandbox.NewInterdomainBuilder(ctx, t). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster1") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster2") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(0).SetNSMgrProxySupplier(nil).SetRegistryProxySupplier(nil).SetName("floating") + }). Build() - cluster2 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSDomainName("cluster2"). - SetDNSResolver(dnsServer). - Build() - - floating := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetDNSDomainName("floating.domain"). - SetDNSResolver(dnsServer). - SetNSMgrProxySupplier(nil). - SetRegistryProxySupplier(nil). - Build() + cluster1 := domains[0] + cluster2 := domains[1] + floating := domains[2] nsRegistryClient := cluster2.NewNSRegistryClient(ctx, sandbox.GenerateTestToken) @@ -492,7 +483,7 @@ func Test_FloatingInterdomain_vl3_dns(t *testing.T) { req.Connection = resp.Clone() require.Len(t, resp.GetContext().GetDnsContext().GetConfigs(), 1) require.Len(t, resp.GetContext().GetDnsContext().GetConfigs()[0].DnsServerIps, 1) - require.Len(t, resp.GetContext().GetDnsContext().GetConfigs()[0].SearchDomains, 3) + require.Len(t, resp.GetContext().GetDnsContext().GetConfigs()[0].SearchDomains, 2) searchDomain := resp.GetContext().GetDnsContext().GetConfigs()[0].SearchDomains[0] diff --git a/pkg/networkservice/chains/nsmgrproxy/server.go b/pkg/networkservice/chains/nsmgrproxy/server.go index 2a0795727..21dee3a7c 100644 --- a/pkg/networkservice/chains/nsmgrproxy/server.go +++ b/pkg/networkservice/chains/nsmgrproxy/server.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2022 Doc.ai and/or its affiliates. // +// Copyright (c) 2023-2024 Cisco and/or its affiliates. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,49 +33,43 @@ import ( "github.com/networkservicemesh/sdk/pkg/networkservice/chains/client" "github.com/networkservicemesh/sdk/pkg/networkservice/chains/endpoint" - "github.com/networkservicemesh/sdk/pkg/networkservice/chains/nsmgr" "github.com/networkservicemesh/sdk/pkg/networkservice/common/authorize" "github.com/networkservicemesh/sdk/pkg/networkservice/common/clusterinfo" "github.com/networkservicemesh/sdk/pkg/networkservice/common/connect" "github.com/networkservicemesh/sdk/pkg/networkservice/common/discover" "github.com/networkservicemesh/sdk/pkg/networkservice/common/interdomainbypass" "github.com/networkservicemesh/sdk/pkg/networkservice/common/swapip" - "github.com/networkservicemesh/sdk/pkg/registry" + "github.com/networkservicemesh/sdk/pkg/networkservice/common/switchcase" registryauthorize "github.com/networkservicemesh/sdk/pkg/registry/common/authorize" "github.com/networkservicemesh/sdk/pkg/registry/common/begin" "github.com/networkservicemesh/sdk/pkg/registry/common/clientconn" "github.com/networkservicemesh/sdk/pkg/registry/common/clienturl" - registryclusterinfo "github.com/networkservicemesh/sdk/pkg/registry/common/clusterinfo" registryconnect "github.com/networkservicemesh/sdk/pkg/registry/common/connect" "github.com/networkservicemesh/sdk/pkg/registry/common/dial" - "github.com/networkservicemesh/sdk/pkg/registry/common/grpcmetadata" - registryswapip "github.com/networkservicemesh/sdk/pkg/registry/common/swapip" - "github.com/networkservicemesh/sdk/pkg/registry/common/updatepath" "github.com/networkservicemesh/sdk/pkg/registry/core/chain" + "github.com/networkservicemesh/sdk/pkg/tools/clienturlctx" "github.com/networkservicemesh/sdk/pkg/tools/fs" "github.com/networkservicemesh/sdk/pkg/tools/grpcutils" + "github.com/networkservicemesh/sdk/pkg/tools/interdomain" "github.com/networkservicemesh/sdk/pkg/tools/log" authmonitor "github.com/networkservicemesh/sdk/pkg/tools/monitorconnection/authorize" "github.com/networkservicemesh/sdk/pkg/tools/token" ) func (n *nsmgrProxyServer) Register(s *grpc.Server) { - grpcutils.RegisterHealthServices(s, n, n.NetworkServiceEndpointRegistryServer(), n.NetworkServiceRegistryServer()) + grpcutils.RegisterHealthServices(s, n) networkservice.RegisterNetworkServiceServer(s, n) networkservice.RegisterMonitorConnectionServer(s, n) - registryapi.RegisterNetworkServiceRegistryServer(s, n.Registry.NetworkServiceRegistryServer()) - registryapi.RegisterNetworkServiceEndpointRegistryServer(s, n.Registry.NetworkServiceEndpointRegistryServer()) } type nsmgrProxyServer struct { endpoint.Endpoint - registry.Registry } type serverOptions struct { name string mapipFilePath string - listenOn *url.URL + listenOn, registryURL *url.URL authorizeServer networkservice.NetworkServiceServer authorizeMonitorConnectionServer networkservice.MonitorConnectionServer authorizeNSRegistryServer registryapi.NetworkServiceRegistryServer @@ -204,8 +200,15 @@ func WithDialTimeout(dialTimeout time.Duration) Option { } } +// WithRegistryURL sets URL to the registry +func WithRegistryURL(u *url.URL) Option { + return func(o *serverOptions) { + o.registryURL = u + } +} + // NewServer creates new proxy NSMgr -func NewServer(ctx context.Context, regURL, proxyURL *url.URL, tokenGenerator token.GeneratorFunc, options ...Option) nsmgr.Nsmgr { +func NewServer(ctx context.Context, regURL *url.URL, tokenGenerator token.GeneratorFunc, options ...Option) endpoint.Endpoint { rv := new(nsmgrProxyServer) opts := &serverOptions{ name: "nsmgr-proxy-" + uuid.New().String(), @@ -222,11 +225,8 @@ func NewServer(ctx context.Context, regURL, proxyURL *url.URL, tokenGenerator to opt(opts) } - var interdomainBypassNSEServer registryapi.NetworkServiceEndpointRegistryServer - nseClient := chain.NewNetworkServiceEndpointRegistryClient( begin.NewNetworkServiceEndpointRegistryClient(), - clienturl.NewNetworkServiceEndpointRegistryClient(regURL), clientconn.NewNetworkServiceEndpointRegistryClient(), dial.NewNetworkServiceEndpointRegistryClient(ctx, dial.WithDialOptions(opts.dialOptions...), @@ -235,9 +235,18 @@ func NewServer(ctx context.Context, regURL, proxyURL *url.URL, tokenGenerator to registryconnect.NewNetworkServiceEndpointRegistryClient(), ) + regNSEClient := chain.NewNetworkServiceEndpointRegistryClient( + clienturl.NewNetworkServiceEndpointRegistryClient(opts.registryURL), + nseClient, + ) + + proxyRegNSEClient := chain.NewNetworkServiceEndpointRegistryClient( + clienturl.NewNetworkServiceEndpointRegistryClient(regURL), + nseClient, + ) + nsClient := chain.NewNetworkServiceRegistryClient( begin.NewNetworkServiceRegistryClient(), - clienturl.NewNetworkServiceRegistryClient(regURL), clientconn.NewNetworkServiceRegistryClient(), dial.NewNetworkServiceRegistryClient(ctx, dial.WithDialOptions(opts.dialOptions...), @@ -245,13 +254,37 @@ func NewServer(ctx context.Context, regURL, proxyURL *url.URL, tokenGenerator to registryconnect.NewNetworkServiceRegistryClient(), ) + regNSClient := chain.NewNetworkServiceRegistryClient( + clienturl.NewNetworkServiceRegistryClient(opts.registryURL), + nsClient, + ) + + proxyRegNSClient := chain.NewNetworkServiceRegistryClient( + clienturl.NewNetworkServiceRegistryClient(regURL), + nsClient, + ) + rv.Endpoint = endpoint.NewServer(ctx, tokenGenerator, endpoint.WithName(opts.name), endpoint.WithAuthorizeServer(opts.authorizeServer), endpoint.WithAuthorizeMonitorConnectionServer(opts.authorizeMonitorConnectionServer), endpoint.WithAdditionalFunctionality( - interdomainbypass.NewServer(&interdomainBypassNSEServer, opts.listenOn), - discover.NewServer(nsClient, nseClient), + switchcase.NewServer(&switchcase.ServerCase{ + Condition: func(ctx context.Context, c *networkservice.Connection) bool { + return interdomain.Is(c.GetNetworkServiceEndpointName()) + }, + Server: discover.NewServer(proxyRegNSClient, proxyRegNSEClient), + }, &switchcase.ServerCase{ + Condition: switchcase.Default, + Server: discover.NewServer(regNSClient, regNSEClient), + }), + switchcase.NewServer(&switchcase.ServerCase{ + Condition: func(ctx context.Context, c *networkservice.Connection) bool { + var u = clienturlctx.ClientURL(ctx) + return u != nil && u.Scheme != "tcp" + }, + Server: interdomainbypass.NewServer(), + }), swapip.NewServer(opts.openMapIPChannel(ctx)), clusterinfo.NewServer(), connect.NewServer( @@ -269,50 +302,5 @@ func NewServer(ctx context.Context, regURL, proxyURL *url.URL, tokenGenerator to ), ) - var nsServerChain = registryconnect.NewNetworkServiceRegistryServer( - chain.NewNetworkServiceRegistryClient( - begin.NewNetworkServiceRegistryClient(), - clienturl.NewNetworkServiceRegistryClient(proxyURL), - clientconn.NewNetworkServiceRegistryClient(), - opts.authorizeNSRegistryClient, - grpcmetadata.NewNetworkServiceRegistryClient(), - dial.NewNetworkServiceRegistryClient(ctx, - dial.WithDialOptions(opts.dialOptions...), - ), - registryconnect.NewNetworkServiceRegistryClient(), - ), - ) - - nsServerChain = chain.NewNetworkServiceRegistryServer( - grpcmetadata.NewNetworkServiceRegistryServer(), - updatepath.NewNetworkServiceRegistryServer(tokenGenerator), - opts.authorizeNSRegistryServer, - nsServerChain, - ) - - var nseServerChain = chain.NewNetworkServiceEndpointRegistryServer( - grpcmetadata.NewNetworkServiceEndpointRegistryServer(), - updatepath.NewNetworkServiceEndpointRegistryServer(tokenGenerator), - opts.authorizeNSERegistryServer, - begin.NewNetworkServiceEndpointRegistryServer(), - clienturl.NewNetworkServiceEndpointRegistryServer(proxyURL), - interdomainBypassNSEServer, - registryswapip.NewNetworkServiceEndpointRegistryServer(opts.openMapIPChannel(ctx)), - registryclusterinfo.NewNetworkServiceEndpointRegistryServer(), - registryconnect.NewNetworkServiceEndpointRegistryServer( - chain.NewNetworkServiceEndpointRegistryClient( - clientconn.NewNetworkServiceEndpointRegistryClient(), - opts.authorizeNSERegistryClient, - grpcmetadata.NewNetworkServiceEndpointRegistryClient(), - dial.NewNetworkServiceEndpointRegistryClient(ctx, - dial.WithDialOptions(opts.dialOptions...), - dial.WithDialTimeout(opts.dialTimeout), - ), - registryconnect.NewNetworkServiceEndpointRegistryClient(), - ), - ), - ) - - rv.Registry = registry.NewServer(nsServerChain, nseServerChain) return rv } diff --git a/pkg/networkservice/chains/nsmgrproxy/server_test.go b/pkg/networkservice/chains/nsmgrproxy/server_test.go index f66d346de..6f56fd04f 100644 --- a/pkg/networkservice/chains/nsmgrproxy/server_test.go +++ b/pkg/networkservice/chains/nsmgrproxy/server_test.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2022 Doc.ai and/or its affiliates. // +// Copyright (c) 2023-2024 Cisco and/or its affiliates. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,9 +25,12 @@ import ( "time" "github.com/golang/protobuf/ptypes/empty" + "google.golang.org/grpc" + "github.com/stretchr/testify/require" "go.uber.org/goleak" - "google.golang.org/grpc" + + kernelmech "github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms/kernel" "github.com/networkservicemesh/api/pkg/api/networkservice" "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/cls" @@ -35,7 +40,6 @@ import ( "github.com/networkservicemesh/sdk/pkg/networkservice/chains/client" "github.com/networkservicemesh/sdk/pkg/networkservice/common/clienturl" "github.com/networkservicemesh/sdk/pkg/networkservice/common/connect" - kernelmech "github.com/networkservicemesh/sdk/pkg/networkservice/common/mechanisms/kernel" "github.com/networkservicemesh/sdk/pkg/networkservice/core/chain" "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" "github.com/networkservicemesh/sdk/pkg/networkservice/utils/checks/checkrequest" @@ -46,28 +50,23 @@ import ( // TestNSMGR_InterdomainUseCase covers simple interdomain scenario: // -// nsc -> nsmgr1 -> forwarder1 -> nsmgr1 -> nsmgr-proxy1 -> nsmg-proxy2 -> nsmgr2 ->forwarder2 -> nsmgr2 -> nse3 +// nsc -> nsmgr1 -> forwarder1 -> nsmgr1 -> nsmgr-proxy1 -> nsmg-proxy2 -> nsmgr2 ->forwarder2 -> nsmgr2 -> nse func TestNSMGR_InterdomainUseCase(t *testing.T) { t.Cleanup(func() { goleak.VerifyNone(t) }) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + var ctx, cancel = context.WithTimeout(context.Background(), time.Second*10) defer cancel() - var dnsServer = sandbox.NewFakeResolver() - - cluster1 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSResolver(dnsServer). - SetDNSDomainName("cluster1"). + var domains = sandbox.NewInterdomainBuilder(ctx, t). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster1") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster2") + }). Build() - cluster2 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSDomainName("cluster2"). - SetDNSResolver(dnsServer). - Build() - - nsRegistryClient := cluster2.NewNSRegistryClient(ctx, sandbox.GenerateTestToken) + nsRegistryClient := domains[1].NewNSRegistryClient(ctx, sandbox.GenerateTestToken) nsReg := ®istry.NetworkService{ Name: "my-service-interdomain", @@ -81,11 +80,11 @@ func TestNSMGR_InterdomainUseCase(t *testing.T) { NetworkServiceNames: []string{nsReg.Name}, } - cluster2.Nodes[0].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, checkrequest.NewServer(t, func(t *testing.T, nsr *networkservice.NetworkServiceRequest) { + domains[1].Nodes[0].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, checkrequest.NewServer(t, func(t *testing.T, nsr *networkservice.NetworkServiceRequest) { require.False(t, interdomain.Is(nsr.GetConnection().GetNetworkService())) })) - nsc := cluster1.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken) + nsc := domains[0].Nodes[0].NewClient(ctx, sandbox.GenerateTestToken) request := &networkservice.NetworkServiceRequest{ MechanismPreferences: []*networkservice.Mechanism{ @@ -93,7 +92,7 @@ func TestNSMGR_InterdomainUseCase(t *testing.T) { }, Connection: &networkservice.Connection{ Id: "1", - NetworkService: fmt.Sprint(nsReg.Name, "@", cluster2.Name), + NetworkService: fmt.Sprint(nsReg.Name, "@", domains[1].Name), Context: &networkservice.ConnectionContext{}, }, } @@ -121,36 +120,27 @@ func TestNSMGR_InterdomainUseCase(t *testing.T) { // TestNSMGR_InterdomainUseCase covers simple interdomain scenario: // -// request1: nsc -> nsmgr1 -> forwarder1 -> nsmgr1 -> nsmgr-proxy1 -> nsmg-proxy2 -> nsmgr2 ->forwarder2 -> nsmgr2 -> final-endpoint via cluster2 -// request2: nsc -> nsmgr1 -> forwarder1 -> nsmgr1 -> nsmgr-proxy1 -> nsmg-proxy2 -> nsmgr2 ->forwarder2 -> nsmgr2 -> final-endpoint via floating registry +// request1: nsc -> nsmgr1 -> forwarder1 -> nsmgr-proxy1 -> nsmg-proxy2 -> nsmgr2 ->forwarder2 -> final-endpoint via cluster2 +// request2: nsc -> nsmgr1 -> forwarder1 -> nsmgr-proxy1 -> nsmg-proxy2 -> nsmgr2 ->forwarder2 -> final-endpoint via floating registry func Test_NSEMovedFromInterdomainToFloatingUseCase(t *testing.T) { t.Cleanup(func() { goleak.VerifyNone(t) }) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - var dnsServer = sandbox.NewFakeResolver() - - cluster1 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSResolver(dnsServer). - SetDNSDomainName("cluster1"). + var domains = sandbox.NewInterdomainBuilder(ctx, t). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster1") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster2") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(0).SetRegistryProxySupplier(nil).SetNSMgrProxySupplier(nil).SetName("floating") + }). Build() - cluster2 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSDomainName("cluster2"). - SetDNSResolver(dnsServer). - Build() - - floating := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetDNSDomainName("floating.domain"). - SetDNSResolver(dnsServer). - SetNSMgrProxySupplier(nil). - SetRegistryProxySupplier(nil). - Build() - nsRegistryClient := cluster2.NewNSRegistryClient(ctx, sandbox.GenerateTestToken) + nsRegistryClient := domains[1].NewNSRegistryClient(ctx, sandbox.GenerateTestToken) nsReg1 := ®istry.NetworkService{ Name: "my-service-interdomain", @@ -162,7 +152,7 @@ func Test_NSEMovedFromInterdomainToFloatingUseCase(t *testing.T) { require.NoError(t, err) nsReg2 := ®istry.NetworkService{ - Name: "my-service-interdomain@" + floating.Name, + Name: "my-service-interdomain@" + domains[2].Name, } nsReg2, err = nsRegistryClient.Register(ctx, nsReg2) @@ -173,41 +163,35 @@ func Test_NSEMovedFromInterdomainToFloatingUseCase(t *testing.T) { NetworkServiceNames: []string{nsReg1.Name}, } - cluster2.Nodes[0].NewEndpoint(ctx, nseReg1, sandbox.GenerateTestToken) + domains[1].Nodes[0].NewEndpoint(ctx, nseReg1, sandbox.GenerateTestToken) nseReg2 := ®istry.NetworkServiceEndpoint{ - Name: "final-endpoint@" + floating.Name, + Name: "final-endpoint@" + domains[2].Name, NetworkServiceNames: []string{nsReg1.Name}, } - cluster2.Nodes[0].NewEndpoint(ctx, nseReg2, sandbox.GenerateTestToken) + domains[1].Nodes[0].NewEndpoint(ctx, nseReg2, sandbox.GenerateTestToken) - stream, err := adapters.NetworkServiceEndpointServerToClient(cluster2.Registry.NetworkServiceEndpointRegistryServer()).Find(context.Background(), ®istry.NetworkServiceEndpointQuery{NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ + stream, err := adapters.NetworkServiceEndpointServerToClient(domains[2].Registry.NetworkServiceEndpointRegistryServer()).Find(context.Background(), ®istry.NetworkServiceEndpointQuery{NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ Name: nseReg1.Name, }}) require.NoError(t, err) require.Len(t, registry.ReadNetworkServiceEndpointList(stream), 1) - stream, err = adapters.NetworkServiceEndpointServerToClient(cluster1.Registry.NetworkServiceEndpointRegistryServer()).Find(context.Background(), ®istry.NetworkServiceEndpointQuery{NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ - Name: nseReg1.Name, - }}) - require.NoError(t, err) - require.Len(t, registry.ReadNetworkServiceEndpointList(stream), 0) - - stream, err = adapters.NetworkServiceEndpointServerToClient(floating.Registry.NetworkServiceEndpointRegistryServer()).Find(context.Background(), ®istry.NetworkServiceEndpointQuery{NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ + stream, err = adapters.NetworkServiceEndpointServerToClient(domains[2].Registry.NetworkServiceEndpointRegistryServer()).Find(context.Background(), ®istry.NetworkServiceEndpointQuery{NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ Name: nseReg1.Name, }}) require.NoError(t, err) require.Len(t, registry.ReadNetworkServiceEndpointList(stream), 1) - nsc := cluster1.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken) + nsc := domains[0].Nodes[0].NewClient(ctx, sandbox.GenerateTestToken) var finalNSE = map[string]string{ - fmt.Sprint(nsReg1.Name, "@", cluster2.Name): nseReg1.GetName(), + fmt.Sprint(nsReg1.Name, "@", domains[2].Name): nseReg1.GetName(), nsReg2.GetName(): nseReg2.GetName(), } - for _, nsName := range []string{fmt.Sprint(nsReg1.Name, "@", cluster2.Name), nsReg2.GetName()} { + for nsName := range finalNSE { request := &networkservice.NetworkServiceRequest{ MechanismPreferences: []*networkservice.Mechanism{ {Cls: cls.LOCAL, Type: kernel.MECHANISM}, @@ -254,19 +238,17 @@ func TestNSMGR_Interdomain_TwoNodesNSEs(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - var dnsServer = sandbox.NewFakeResolver() - - cluster1 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSResolver(dnsServer). - SetDNSDomainName("cluster1"). + var domains = sandbox.NewInterdomainBuilder(ctx, t). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster1") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster2") + }). Build() - cluster2 := sandbox.NewBuilder(ctx, t). - SetNodesCount(2). - SetDNSDomainName("cluster2"). - SetDNSResolver(dnsServer). - Build() + cluster1 := domains[0] + cluster2 := domains[1] nsRegistryClient := cluster2.NewNSRegistryClient(ctx, sandbox.GenerateTestToken) @@ -355,30 +337,24 @@ func TestNSMGR_Interdomain_TwoNodesNSEs(t *testing.T) { func TestNSMGR_FloatingInterdomainUseCase(t *testing.T) { t.Cleanup(func() { goleak.VerifyNone(t) }) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + ctx, cancel := context.WithTimeout(context.Background(), time.Hour*5) defer cancel() - var dnsServer = sandbox.NewFakeResolver() - - cluster1 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSResolver(dnsServer). - SetDNSDomainName("cluster1"). + var domains = sandbox.NewInterdomainBuilder(ctx, t). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster1") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster2") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(0).SetNSMgrProxySupplier(nil).SetRegistryProxySupplier(nil).SetName("floating") + }). Build() - cluster2 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSDomainName("cluster2"). - SetDNSResolver(dnsServer). - Build() - - floating := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetDNSDomainName("floating.domain"). - SetDNSResolver(dnsServer). - SetNSMgrProxySupplier(nil). - SetRegistryProxySupplier(nil). - Build() + cluster1 := domains[0] + cluster2 := domains[1] + floating := domains[2] nsRegistryClient := cluster2.NewNSRegistryClient(ctx, sandbox.GenerateTestToken) @@ -396,10 +372,10 @@ func TestNSMGR_FloatingInterdomainUseCase(t *testing.T) { cluster2.Nodes[0].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken) - c := adapters.NetworkServiceEndpointServerToClient(cluster2.Nodes[0].NSMgr.NetworkServiceEndpointRegistryServer()) + c := adapters.NetworkServiceEndpointServerToClient(floating.Registry.NetworkServiceEndpointRegistryServer()) s, err := c.Find(ctx, ®istry.NetworkServiceEndpointQuery{NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ - Name: "final-endpoint@" + floating.Name, + Name: "final-endpoint", }}) require.NoError(t, err) @@ -453,27 +429,21 @@ func TestNSMGR_FloatingInterdomainUseCase_FloatingNetworkServiceNameRegistration ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - var dnsServer = sandbox.NewFakeResolver() - - cluster1 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSResolver(dnsServer). - SetDNSDomainName("cluster1"). + var domains = sandbox.NewInterdomainBuilder(ctx, t). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster1") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster2") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(0).SetNSMgrProxySupplier(nil).SetRegistryProxySupplier(nil).SetName("floating") + }). Build() - cluster2 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSDomainName("cluster2"). - SetDNSResolver(dnsServer). - Build() - - floating := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetDNSDomainName("floating.domain"). - SetDNSResolver(dnsServer). - SetNSMgrProxySupplier(nil). - SetRegistryProxySupplier(nil). - Build() + cluster1 := domains[0] + cluster2 := domains[1] + floating := domains[2] nsRegistryClient := cluster2.NewNSRegistryClient(ctx, sandbox.GenerateTestToken) @@ -536,35 +506,27 @@ func TestNSMGR_FloatingInterdomain_FourClusters(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - var dnsServer = sandbox.NewFakeResolver() - // setup clusters - cluster1 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSResolver(dnsServer). - SetDNSDomainName("cluster1"). - Build() - - cluster2 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSDomainName("cluster2"). - SetDNSResolver(dnsServer). - Build() - - cluster3 := sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSDomainName("cluster3"). - SetDNSResolver(dnsServer). + var domains = sandbox.NewInterdomainBuilder(ctx, t). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster1") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster2") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(1).SetName("cluster3") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(0).SetNSMgrProxySupplier(nil).SetRegistryProxySupplier(nil).SetName("floating") + }). Build() - floating := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetDNSDomainName("floating.domain"). - SetDNSResolver(dnsServer). - SetNSMgrProxySupplier(nil). - SetRegistryProxySupplier(nil). - Build() + cluster1 := domains[0] + cluster2 := domains[1] + cluster3 := domains[2] + floating := domains[3] // register first ednpoint @@ -694,20 +656,25 @@ func (c *passThroughClient) Close(ctx context.Context, conn *networkservice.Conn // nse1 -> nsmgr1 -> forwarder1 -> nsmg1 -> nsmgr-proxy1 -> nsmgr-proxy0 -> nsmgr0 -> forwarder0 -> nsmgr0 -> nse0 func Test_Interdomain_PassThroughUsecase(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer cancel() - const clusterCount = 5 + const clusterCount = 4 + + var builder = sandbox.NewInterdomainBuilder(ctx, t) - var dnsServer = sandbox.NewFakeResolver() - var clusters = make([]*sandbox.Domain, clusterCount) + for i := 0; i < clusterCount; i++ { + var index = i + builder.BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b. + SetNodesCount(1). + SetName("cluster" + fmt.Sprint(index)) + }) + } + var clusters = builder.Build() + require.Equal(t, clusterCount, len(clusters)) for i := 0; i < clusterCount; i++ { - clusters[i] = sandbox.NewBuilder(ctx, t). - SetNodesCount(1). - SetDNSResolver(dnsServer). - SetDNSDomainName("cluster" + fmt.Sprint(i)). - Build() var additionalFunctionality []networkservice.NetworkServiceServer if i != 0 { // Passtrough to the node i-1 diff --git a/pkg/networkservice/common/interdomainbypass/server.go b/pkg/networkservice/common/interdomainbypass/server.go index ccda893b4..683dcf57d 100644 --- a/pkg/networkservice/common/interdomainbypass/server.go +++ b/pkg/networkservice/common/interdomainbypass/server.go @@ -1,6 +1,6 @@ // Copyright (c) 2021-2022 Doc.ai and/or its affiliates. // -// Copyright (c) 2023 Cisco and/or its affiliates. +// Copyright (c) 2023-2024 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -21,38 +21,22 @@ package interdomainbypass import ( "context" - "net/url" - "github.com/edwarnicke/genericsync" "github.com/golang/protobuf/ptypes/empty" "github.com/networkservicemesh/api/pkg/api/networkservice" - "github.com/networkservicemesh/api/pkg/api/registry" "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" - "github.com/networkservicemesh/sdk/pkg/registry/common/interdomainbypass" - "github.com/networkservicemesh/sdk/pkg/tools/clienturlctx" "github.com/networkservicemesh/sdk/pkg/tools/interdomain" ) -type interdomainBypassServer struct { - m genericsync.Map[string, *url.URL] -} +type interdomainBypassServer struct{} // NewServer - returns a new NetworkServiceServer that injects the URL to remote side into context on requesting resolved endpoint -func NewServer(rs *registry.NetworkServiceEndpointRegistryServer, listenOn *url.URL) networkservice.NetworkServiceServer { - var rv = new(interdomainBypassServer) - *rs = interdomainbypass.NewNetworkServiceEndpointRegistryServer(&rv.m, listenOn) - return rv +func NewServer() networkservice.NetworkServiceServer { + return new(interdomainBypassServer) } func (n *interdomainBypassServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) { - u, ok := n.m.Load(request.Connection.NetworkServiceEndpointName) - // Always true when we are on local nsmgr proxy side. - // True on theremote nsmgr proxy side when it is floating interdomain usecase. - if ok { - ctx = clienturlctx.WithClientURL(ctx, u) - return next.Server(ctx).Request(ctx, request) - } originalNSEName := request.GetConnection().NetworkServiceEndpointName originalNS := request.GetConnection().NetworkService request.GetConnection().NetworkServiceEndpointName = interdomain.Target(originalNSEName) @@ -67,13 +51,6 @@ func (n *interdomainBypassServer) Request(ctx context.Context, request *networks } func (n *interdomainBypassServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) { - u, ok := n.m.Load(conn.NetworkServiceEndpointName) - // Always true when we are on local nsmgr proxy side. - // True on theremote nsmgr proxy side when it is floating interdomain usecase. - if ok { - ctx = clienturlctx.WithClientURL(ctx, u) - return next.Server(ctx).Close(ctx, conn) - } originalNSEName := conn.GetNetworkServiceEndpointName() originalNS := conn.GetNetworkService() conn.NetworkServiceEndpointName = interdomain.Target(originalNSEName) diff --git a/pkg/networkservice/common/interdomainbypass/server_test.go b/pkg/networkservice/common/interdomainbypass/server_test.go deleted file mode 100644 index a2b5f0b3f..000000000 --- a/pkg/networkservice/common/interdomainbypass/server_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) 2021-2022 Doc.ai and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package interdomainbypass_test - -import ( - "context" - "net/url" - "testing" - "time" - - "github.com/networkservicemesh/api/pkg/api/networkservice" - "github.com/networkservicemesh/api/pkg/api/registry" - "github.com/stretchr/testify/require" - "go.uber.org/goleak" - - "github.com/networkservicemesh/sdk/pkg/networkservice/common/interdomainbypass" - "github.com/networkservicemesh/sdk/pkg/networkservice/core/chain" - "github.com/networkservicemesh/sdk/pkg/networkservice/utils/checks/checkcontext" - "github.com/networkservicemesh/sdk/pkg/registry/common/memory" - "github.com/networkservicemesh/sdk/pkg/registry/core/adapters" - "github.com/networkservicemesh/sdk/pkg/registry/core/next" - "github.com/networkservicemesh/sdk/pkg/tools/clienturlctx" -) - -func Test_StoreUrlNSEServer(t *testing.T) { - defer goleak.VerifyNone(t) - - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - - defer cancel() - - var storeRegServer registry.NetworkServiceEndpointRegistryServer - - var s = chain.NewNetworkServiceServer( - interdomainbypass.NewServer(&storeRegServer, new(url.URL)), - checkcontext.NewServer(t, func(t *testing.T, c context.Context) { - v := clienturlctx.ClientURL(c) - require.NotNil(t, v) - require.Equal(t, url.URL{Scheme: "unix", Host: "file.sock"}, *v) - }), - ) - - registryServer := next.NewNetworkServiceEndpointRegistryServer( - storeRegServer, - memory.NewNetworkServiceEndpointRegistryServer(), - ) - - _, err := registryServer.Register(ctx, ®istry.NetworkServiceEndpoint{ - Name: "nse-1", - Url: "unix://file.sock", - }) - require.NoError(t, err) - - stream, err := adapters.NetworkServiceEndpointServerToClient(registryServer).Find(ctx, ®istry.NetworkServiceEndpointQuery{ - NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ - Name: "nse-1", - }, - }) - require.NoError(t, err) - - list := registry.ReadNetworkServiceEndpointList(stream) - require.Len(t, list, 1) - - _, err = s.Request(ctx, &networkservice.NetworkServiceRequest{ - Connection: &networkservice.Connection{ - NetworkServiceEndpointName: "nse-1", - }, - }) - - require.NoError(t, err) -} diff --git a/pkg/networkservice/common/mechanisms/common.go b/pkg/networkservice/common/mechanisms/common.go index b3852e142..bbe7babc5 100644 --- a/pkg/networkservice/common/mechanisms/common.go +++ b/pkg/networkservice/common/mechanisms/common.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2023 Doc.ai and/or its affiliates. +// Copyright (c) 2021-2024 Doc.ai and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // diff --git a/pkg/networkservice/common/mechanisms/server.go b/pkg/networkservice/common/mechanisms/server.go index 95ff60994..9dae5a628 100644 --- a/pkg/networkservice/common/mechanisms/server.go +++ b/pkg/networkservice/common/mechanisms/server.go @@ -1,6 +1,6 @@ // Copyright (c) 2021 Doc.ai and/or its affiliates. // -// Copyright (c) 2020-2023 Cisco and/or its affiliates. +// Copyright (c) 2020-2024 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // diff --git a/pkg/registry/chains/memory/server.go b/pkg/registry/chains/memory/server.go index c35583d3a..90e8e31cb 100644 --- a/pkg/registry/chains/memory/server.go +++ b/pkg/registry/chains/memory/server.go @@ -1,6 +1,6 @@ // Copyright (c) 2020-2022 Doc.ai and/or its affiliates. // -// Copyright (c) 2023 Cisco and/or its affiliates. +// Copyright (c) 2023-2024 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -31,6 +31,7 @@ import ( registryserver "github.com/networkservicemesh/sdk/pkg/registry" registryauthorize "github.com/networkservicemesh/sdk/pkg/registry/common/authorize" "github.com/networkservicemesh/sdk/pkg/registry/common/grpcmetadata" + "github.com/networkservicemesh/sdk/pkg/registry/common/replaceurl" "github.com/networkservicemesh/sdk/pkg/registry/common/updatepath" "github.com/networkservicemesh/sdk/pkg/registry/common/begin" @@ -50,13 +51,13 @@ import ( ) type serverOptions struct { - authorizeNSRegistryServer registry.NetworkServiceRegistryServer - authorizeNSERegistryServer registry.NetworkServiceEndpointRegistryServer - authorizeNSRegistryClient registry.NetworkServiceRegistryClient - authorizeNSERegistryClient registry.NetworkServiceEndpointRegistryClient - defaultExpiration time.Duration - proxyRegistryURL *url.URL - dialOptions []grpc.DialOption + authorizeNSRegistryServer registry.NetworkServiceRegistryServer + authorizeNSERegistryServer registry.NetworkServiceEndpointRegistryServer + authorizeNSRegistryClient registry.NetworkServiceRegistryClient + authorizeNSERegistryClient registry.NetworkServiceEndpointRegistryClient + defaultExpiration time.Duration + nsmgrProxyURL, proxyRegistryURL *url.URL + dialOptions []grpc.DialOption } // Option modifies server option value @@ -109,6 +110,13 @@ func WithDefaultExpiration(d time.Duration) Option { } } +// WithNSMgrProxyURL sets url to the NSMgr proxy +func WithNSMgrProxyURL(u *url.URL) Option { + return func(o *serverOptions) { + o.nsmgrProxyURL = u + } +} + // WithProxyRegistryURL sets URL to reach the proxy registry func WithProxyRegistryURL(proxyRegistryURL *url.URL) Option { return func(o *serverOptions) { @@ -131,8 +139,8 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, options authorizeNSRegistryClient: registryauthorize.NewNetworkServiceRegistryClient(registryauthorize.Any()), authorizeNSERegistryClient: registryauthorize.NewNetworkServiceEndpointRegistryClient(registryauthorize.Any()), defaultExpiration: time.Minute, - proxyRegistryURL: nil, } + for _, opt := range options { opt(opts) } @@ -145,6 +153,9 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, options metadata.NewNetworkServiceEndpointServer(), switchcase.NewNetworkServiceEndpointRegistryServer(switchcase.NSEServerCase{ Condition: func(c context.Context, nse *registry.NetworkServiceEndpoint) bool { + if opts.nsmgrProxyURL == nil { + return false + } if interdomain.Is(nse.GetName()) { return true } @@ -156,6 +167,7 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, options return false }, Action: chain.NewNetworkServiceEndpointRegistryServer( + replaceurl.NewNetworkServiceEndpointRegistryServer(opts.nsmgrProxyURL), connect.NewNetworkServiceEndpointRegistryServer( chain.NewNetworkServiceEndpointRegistryClient( begin.NewNetworkServiceEndpointRegistryClient(), @@ -172,7 +184,7 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, options ), }, switchcase.NSEServerCase{ - Condition: func(c context.Context, nse *registry.NetworkServiceEndpoint) bool { return true }, + Condition: switchcase.Otherwise[*registry.NetworkServiceEndpoint], Action: chain.NewNetworkServiceEndpointRegistryServer( setregistrationtime.NewNetworkServiceEndpointRegistryServer(), expire.NewNetworkServiceEndpointRegistryServer(ctx, expire.WithDefaultExpiration(opts.defaultExpiration)), diff --git a/pkg/registry/chains/proxydns/server.go b/pkg/registry/chains/proxydns/server.go index 7f61e5c9f..62ce041c3 100644 --- a/pkg/registry/chains/proxydns/server.go +++ b/pkg/registry/chains/proxydns/server.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2022 Doc.ai and/or its affiliates. // +// Copyright (c) 2024 Cisco and/or its affiliates. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +21,7 @@ package proxydns import ( "context" + "net/url" "google.golang.org/grpc" @@ -28,13 +31,20 @@ import ( registryauthorize "github.com/networkservicemesh/sdk/pkg/registry/common/authorize" "github.com/networkservicemesh/sdk/pkg/registry/common/begin" "github.com/networkservicemesh/sdk/pkg/registry/common/clientconn" + "github.com/networkservicemesh/sdk/pkg/registry/common/clienturl" "github.com/networkservicemesh/sdk/pkg/registry/common/connect" "github.com/networkservicemesh/sdk/pkg/registry/common/dial" "github.com/networkservicemesh/sdk/pkg/registry/common/dnsresolve" "github.com/networkservicemesh/sdk/pkg/registry/common/grpcmetadata" + "github.com/networkservicemesh/sdk/pkg/registry/common/interdomainbypass" + "github.com/networkservicemesh/sdk/pkg/registry/common/replaceurl" + "github.com/networkservicemesh/sdk/pkg/registry/common/seturl" + "github.com/networkservicemesh/sdk/pkg/registry/switchcase" "github.com/networkservicemesh/sdk/pkg/registry/common/updatepath" "github.com/networkservicemesh/sdk/pkg/registry/core/chain" + "github.com/networkservicemesh/sdk/pkg/registry/core/next" + "github.com/networkservicemesh/sdk/pkg/tools/interdomain" "github.com/networkservicemesh/sdk/pkg/tools/token" ) @@ -43,6 +53,7 @@ type serverOptions struct { authorizeNSERegistryServer registryapi.NetworkServiceEndpointRegistryServer authorizeNSRegistryClient registryapi.NetworkServiceRegistryClient authorizeNSERegistryClient registryapi.NetworkServiceEndpointRegistryClient + registryURL, nsmgrProxyURL *url.URL dialOptions []grpc.DialOption } @@ -89,6 +100,20 @@ func WithAuthorizeNSERegistryClient(authorizeNSERegistryClient registryapi.Netwo } } +// WithNSMgrProxyURL sets url to the nsmgr proxys +func WithNSMgrProxyURL(u *url.URL) Option { + return func(o *serverOptions) { + o.nsmgrProxyURL = u + } +} + +// WithRegistryURL sets url to the registry +func WithRegistryURL(u *url.URL) Option { + return func(o *serverOptions) { + o.registryURL = u + } +} + // WithDialOptions sets grpc.DialOptions for the server func WithDialOptions(dialOptions ...grpc.DialOption) Option { return func(o *serverOptions) { @@ -113,7 +138,30 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, dnsResol updatepath.NewNetworkServiceEndpointRegistryServer(tokenGenerator), opts.authorizeNSERegistryServer, begin.NewNetworkServiceEndpointRegistryServer(), - dnsresolve.NewNetworkServiceEndpointRegistryServer(dnsresolve.WithResolver(dnsResolver)), + interdomainbypass.NewNetworkServiceEndpointRegistryServer(), + switchcase.NewNetworkServiceEndpointRegistryServer( + switchcase.NSEServerCase{ + Condition: func(ctx context.Context, nse *registryapi.NetworkServiceEndpoint) bool { + for _, service := range nse.GetNetworkServiceNames() { + if interdomain.Is(service) { + return true + } + } + return interdomain.Is(nse.GetName()) + }, + Action: next.NewNetworkServiceEndpointRegistryServer( + seturl.NewNetworkServiceEndpointRegistryServer(opts.nsmgrProxyURL), + dnsresolve.NewNetworkServiceEndpointRegistryServer(dnsresolve.WithResolver(dnsResolver)), + ), + }, + switchcase.NSEServerCase{ + Condition: switchcase.Otherwise[*registryapi.NetworkServiceEndpoint], + Action: next.NewNetworkServiceEndpointRegistryServer( + replaceurl.NewNetworkServiceEndpointRegistryServer(opts.nsmgrProxyURL), + clienturl.NewNetworkServiceEndpointRegistryServer(opts.registryURL), + ), + }, + ), connect.NewNetworkServiceEndpointRegistryServer( chain.NewNetworkServiceEndpointRegistryClient( clientconn.NewNetworkServiceEndpointRegistryClient(), @@ -130,7 +178,18 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, dnsResol updatepath.NewNetworkServiceRegistryServer(tokenGenerator), begin.NewNetworkServiceRegistryServer(), opts.authorizeNSRegistryServer, - dnsresolve.NewNetworkServiceRegistryServer(dnsresolve.WithResolver(dnsResolver)), + switchcase.NewNetworkServiceRegistryServer( + switchcase.NSServerCase{ + Condition: func(ctx context.Context, ns *registryapi.NetworkService) bool { + return interdomain.Is(ns.GetName()) + }, + Action: dnsresolve.NewNetworkServiceRegistryServer(dnsresolve.WithResolver(dnsResolver)), + }, + switchcase.NSServerCase{ + Condition: switchcase.Otherwise[*registryapi.NetworkService], + Action: clienturl.NewNetworkServiceRegistryServer(opts.registryURL), + }, + ), connect.NewNetworkServiceRegistryServer( chain.NewNetworkServiceRegistryClient( clientconn.NewNetworkServiceRegistryClient(), diff --git a/pkg/registry/chains/proxydns/server_ns_test.go b/pkg/registry/chains/proxydns/server_ns_test.go index f7fba1294..f1a270942 100644 --- a/pkg/registry/chains/proxydns/server_ns_test.go +++ b/pkg/registry/chains/proxydns/server_ns_test.go @@ -1,6 +1,6 @@ // Copyright (c) 2020-2022 Doc.ai and/or its affiliates. // -// Copyright (c) 2022 Cisco Systems, Inc. +// Copyright (c) 2024 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -20,6 +20,7 @@ package proxydns_test import ( "context" + "net" "strings" "testing" "time" @@ -56,36 +57,36 @@ domain1: domain2: func TestInterdomainNetworkServiceRegistry(t *testing.T) { t.Cleanup(func() { goleak.VerifyNone(t) }) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - dnsServer := sandbox.NewFakeResolver() - - domain1 := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetDNSResolver(dnsServer). - Build() - - domain2 := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetDNSResolver(dnsServer). - SetDNSDomainName("cluster.remote"). + var domains = sandbox.NewInterdomainBuilder(ctx, t). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b. + SetNodesCount(0). + SetName("cluster1") + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b. + SetNodesCount(0). + SetName("cluster2") + }). Build() client1 := registryclient.NewNetworkServiceRegistryClient(ctx, registryclient.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), - registryclient.WithClientURL(domain1.Registry.URL)) + registryclient.WithClientURL(domains[0].Registry.URL)) client2 := registryclient.NewNetworkServiceRegistryClient(ctx, registryclient.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), - registryclient.WithClientURL(domain2.Registry.URL)) + registryclient.WithClientURL(domains[1].Registry.URL)) _, err := client2.Register(context.Background(), ®istryapi.NetworkService{Name: "ns-1"}) require.NoError(t, err) stream, err := client1.Find(ctx, ®istryapi.NetworkServiceQuery{ NetworkService: ®istryapi.NetworkService{ - Name: "ns-1@" + domain2.Name, + Name: "ns-1@" + domains[1].Name, }, }) @@ -94,7 +95,7 @@ func TestInterdomainNetworkServiceRegistry(t *testing.T) { list := registryapi.ReadNetworkServiceList(stream) require.Len(t, list, 1) - require.Equal(t, "ns-1@"+domain2.Name, list[0].Name) + require.Equal(t, "ns-1@"+domains[1].Name, list[0].Name) } /* @@ -117,15 +118,13 @@ domain1: domain1 func TestLocalDomain_NetworkServiceRegistry(t *testing.T) { t.Cleanup(func() { goleak.VerifyNone(t) }) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Hour*15) defer cancel() - dnsServer := sandbox.NewFakeResolver() - domain1 := sandbox.NewBuilder(ctx, t). SetNodesCount(0). - SetDNSDomainName("cluster.local"). - SetDNSResolver(dnsServer). + EnableInterdomain(). + SetName("cluster.local"). Build() client1 := registryclient.NewNetworkServiceRegistryClient(ctx, @@ -157,6 +156,31 @@ func TestLocalDomain_NetworkServiceRegistry(t *testing.T) { require.Equal(t, "ns-1@cluster.local", list[0].Name) } +func TestDNSServer(t *testing.T) { + t.Cleanup(func() { goleak.VerifyNone(t) }) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + var domain = sandbox.NewBuilder(ctx, t). + SetNodesCount(0). + SetName("cluster.floating"). + EnableInterdomain(). + Build() + + require.NotNil(t, domain.DNSServer) + + var res = net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + return net.Dial(network, domain.DNSServer.URL.Host) + }, + } + + _, srvs, err := res.LookupSRV(ctx, "", "", "registry."+domain.Name+".") + require.NoError(t, err) + require.Len(t, srvs, 1) +} + /* TestInterdomainNetworkServiceEndpointRegistry covers the next scenario: 1. local registry from domain2 has entry "ns-1" @@ -177,42 +201,36 @@ func TestLocalDomain_NetworkServiceRegistry(t *testing.T) { func TestInterdomainFloatingNetworkServiceRegistry(t *testing.T) { t.Cleanup(func() { goleak.VerifyNone(t) }) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + var ctx, cancel = context.WithTimeout(context.Background(), time.Second*10) defer cancel() - dnsServer := sandbox.NewFakeResolver() - - domain1 := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetDNSResolver(dnsServer). - Build() - - domain2 := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetDNSResolver(dnsServer). - Build() - - domain3 := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetRegistryProxySupplier(nil). - SetNSMgrProxySupplier(nil). - SetDNSResolver(dnsServer). - SetDNSDomainName("floating.domain"). + var domains = sandbox.NewInterdomainBuilder(ctx, t). + BuildDomain( + func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(0).SetNSMgrProxySupplier(nil).SetName("cluster1") + }, + ). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(0).SetName("cluster2").SetNSMgrProxySupplier(nil) + }). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(0).SetNSMgrProxySupplier(nil).SetName("floating.domain") + }). Build() registryClient := registryclient.NewNetworkServiceRegistryClient(ctx, registryclient.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), - registryclient.WithClientURL(domain2.Registry.URL)) + registryclient.WithClientURL(domains[0].Registry.URL)) _, err := registryClient.Register( ctx, ®istryapi.NetworkService{ - Name: "ns-1@" + domain3.Name, + Name: "ns-1@" + domains[2].Name, }, ) require.Nil(t, err) - cc, err := grpc.DialContext(ctx, grpcutils.URLToTarget(domain1.Registry.URL), grpc.WithBlock(), grpc.WithTransportCredentials(insecure.NewCredentials())) + cc, err := grpc.DialContext(ctx, grpcutils.URLToTarget(domains[0].Registry.URL), grpc.WithBlock(), grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) defer func() { _ = cc.Close() @@ -222,7 +240,7 @@ func TestInterdomainFloatingNetworkServiceRegistry(t *testing.T) { stream, err := client.Find(ctx, ®istryapi.NetworkServiceQuery{ NetworkService: ®istryapi.NetworkService{ - Name: "ns-1@" + domain3.Name, + Name: "ns-1@" + domains[2].Name, }, }) @@ -231,5 +249,5 @@ func TestInterdomainFloatingNetworkServiceRegistry(t *testing.T) { list := registryapi.ReadNetworkServiceList(stream) require.Len(t, list, 1) - require.Equal(t, "ns-1@"+domain3.Name, list[0].Name) + require.Equal(t, "ns-1@"+domains[2].Name, list[0].Name) } diff --git a/pkg/registry/chains/proxydns/server_nse_test.go b/pkg/registry/chains/proxydns/server_nse_test.go index a45f337b7..2d70d2abd 100644 --- a/pkg/registry/chains/proxydns/server_nse_test.go +++ b/pkg/registry/chains/proxydns/server_nse_test.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2022 Doc.ai and/or its affiliates. // +// Copyright (c) 2024 Cisco and/or its affiliates. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +20,7 @@ package proxydns_test import ( "context" + "fmt" "strings" "testing" "time" @@ -55,27 +58,26 @@ domain1: domain2: func TestInterdomainNetworkServiceEndpointRegistry(t *testing.T) { t.Cleanup(func() { goleak.VerifyNone(t) }) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + var ctx, cancel = context.WithTimeout(context.Background(), time.Second*5) defer cancel() - dnsServer := sandbox.NewFakeResolver() - - domain1 := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetDNSResolver(dnsServer). - Build() - - domain2 := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetDNSResolver(dnsServer). - SetDNSDomainName("domain2"). - Build() + var domains = sandbox. + NewInterdomainBuilder(ctx, t). + BuildDomain( + func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(0).SetName("domain1") + }, + ).BuildDomain( + func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(0).SetName("domain2") + }, + ).Build() expirationTime := timestamppb.New(time.Now().Add(time.Hour)) registryClient := registryclient.NewNetworkServiceEndpointRegistryClient(ctx, registryclient.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), - registryclient.WithClientURL(domain2.Registry.URL)) + registryclient.WithClientURL(domains[1].Registry.URL)) reg, err := registryClient.Register( context.Background(), @@ -87,18 +89,13 @@ func TestInterdomainNetworkServiceEndpointRegistry(t *testing.T) { ) require.Nil(t, err) - cc, err := grpc.DialContext(ctx, grpcutils.URLToTarget(domain1.Registry.URL), grpc.WithBlock(), grpc.WithTransportCredentials(insecure.NewCredentials())) - require.Nil(t, err) - - defer func() { - _ = cc.Close() - }() - - client := registryapi.NewNetworkServiceEndpointRegistryClient(cc) + client := registryclient.NewNetworkServiceEndpointRegistryClient(ctx, + registryclient.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), + registryclient.WithClientURL(domains[0].Registry.URL)) stream, err := client.Find(ctx, ®istryapi.NetworkServiceEndpointQuery{ NetworkServiceEndpoint: ®istryapi.NetworkServiceEndpoint{ - Name: reg.Name + "@" + domain2.Name, + Name: reg.Name + "@" + domains[1].Name, }, }) @@ -107,37 +104,49 @@ func TestInterdomainNetworkServiceEndpointRegistry(t *testing.T) { list := registryapi.ReadNetworkServiceEndpointList(stream) require.Len(t, list, 1) - require.Equal(t, reg.Name+"@"+domain2.Name, list[0].Name) + require.Equal(t, reg.Name+"@"+domains[1].Name, list[0].Name) + + client = registryclient.NewNetworkServiceEndpointRegistryClient(ctx, + registryclient.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), + registryclient.WithClientURL(domains[0].RegistryProxy.URL)) + + stream, err = client.Find(ctx, ®istryapi.NetworkServiceEndpointQuery{ + NetworkServiceEndpoint: ®istryapi.NetworkServiceEndpoint{ + Name: reg.Name + "@" + domains[1].Name, + }, + }) + + require.Nil(t, err) + + list = registryapi.ReadNetworkServiceEndpointList(stream) + require.Len(t, list, 1) } /* TestLocalDomain_NetworkServiceEndpointRegistry covers the next scenario: 1. nsmgr from domain1 calls find with query "nse-1@domain1" - 2. local registry proxies query to nsmgr proxy registry - 3. nsmgr proxy registry proxies query to proxy registry - 4. proxy registry proxies query to local registry + 2. local registry proxies query to proxy registry + 3. proxy registry proxies query to local registry Expected: nsmgr found nse domain1: ----------------------------------------------------------------------------------- | | -| local registry -> nsmgr proxy registry -> proxy registry -> local registry | +| local registry -> proxy registry -> local registry | | | ----------------------------------------------------------------------------------- */ func TestLocalDomain_NetworkServiceEndpointRegistry(t *testing.T) { t.Cleanup(func() { goleak.VerifyNone(t) }) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - dnsServer := sandbox.NewFakeResolver() - domain1 := sandbox.NewBuilder(ctx, t). SetNodesCount(0). - SetDNSDomainName("cluster.local"). - SetDNSResolver(dnsServer). + SetName("cluster.local"). + EnableInterdomain(). Build() expirationTime := timestamppb.New(time.Now().Add(time.Hour)) @@ -198,39 +207,36 @@ domain1: domain2: func TestInterdomainFloatingNetworkServiceEndpointRegistry(t *testing.T) { t.Cleanup(func() { goleak.VerifyNone(t) }) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - dnsServer := sandbox.NewFakeResolver() - - domain1 := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetDNSResolver(dnsServer). - Build() - - domain2 := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetDNSResolver(dnsServer). - Build() - - domain3 := sandbox.NewBuilder(ctx, t). - SetNodesCount(0). - SetDNSResolver(dnsServer). - SetNSMgrProxySupplier(nil). - SetRegistryProxySupplier(nil). - SetDNSDomainName("floating.domain"). + var domains = sandbox. + NewInterdomainBuilder(ctx, t). + BuildDomain( + func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(0).SetName("domain1") + }, + ). + BuildDomain( + func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(0).SetName("domain2") + }, + ). + BuildDomain(func(b *sandbox.Builder) *sandbox.Builder { + return b.SetNodesCount(0).SetName("domain3") + }). Build() expirationTime := timestamppb.New(time.Now().Add(time.Hour)) registryClient := registryclient.NewNetworkServiceEndpointRegistryClient(ctx, registryclient.WithDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials())), - registryclient.WithClientURL(domain2.Registry.URL)) + registryclient.WithClientURL(domains[1].Registry.URL)) reg, err := registryClient.Register( context.Background(), ®istryapi.NetworkServiceEndpoint{ - Name: "nse-1@" + domain3.Name, + Name: "nse-1@" + domains[2].Name, Url: "test://publicNSMGRurl", ExpirationTime: expirationTime, }, @@ -239,7 +245,7 @@ func TestInterdomainFloatingNetworkServiceEndpointRegistry(t *testing.T) { name := strings.Split(reg.Name, "@")[0] - cc, err := grpc.DialContext(ctx, grpcutils.URLToTarget(domain1.Registry.URL), grpc.WithBlock(), grpc.WithTransportCredentials(insecure.NewCredentials())) + cc, err := grpc.DialContext(ctx, grpcutils.URLToTarget(domains[0].Registry.URL), grpc.WithBlock(), grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) defer func() { _ = cc.Close() @@ -249,7 +255,7 @@ func TestInterdomainFloatingNetworkServiceEndpointRegistry(t *testing.T) { stream, err := client.Find(ctx, ®istryapi.NetworkServiceEndpointQuery{ NetworkServiceEndpoint: ®istryapi.NetworkServiceEndpoint{ - Name: name + "@" + domain3.Name, + Name: name + "@" + domains[2].Name, }, }) @@ -258,5 +264,29 @@ func TestInterdomainFloatingNetworkServiceEndpointRegistry(t *testing.T) { list := registryapi.ReadNetworkServiceEndpointList(stream) require.Len(t, list, 1) - require.Equal(t, name+"@"+domain3.Name, list[0].Name) + require.Equal(t, name+"@"+domains[2].Name, list[0].Name) +} + +func TestXX(t *testing.T) { + var list = &ListNode{5, &ListNode{Val: 10, Next: &ListNode{Val: 15}}} + list = reverseList(list) + for list != nil { + fmt.Println(list.Val) + list = list.Next + } +} + +type ListNode struct { + Val int + Next *ListNode +} + +func reverseList(head *ListNode) *ListNode { + if head != nil && head.Next != nil { + var result = reverseList(head.Next) + head.Next.Next = head + head.Next = nil + return result + } + return head } diff --git a/pkg/registry/common/dial/ns_client.go b/pkg/registry/common/dial/ns_client.go index b77e9d8ef..2fc57a772 100644 --- a/pkg/registry/common/dial/ns_client.go +++ b/pkg/registry/common/dial/ns_client.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2022 Cisco and/or its affiliates. +// Copyright (c) 2021-2024 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -29,7 +29,6 @@ import ( "github.com/networkservicemesh/sdk/pkg/registry/common/clientconn" "github.com/networkservicemesh/sdk/pkg/registry/core/next" - "github.com/networkservicemesh/sdk/pkg/tools/postpone" "github.com/networkservicemesh/sdk/pkg/tools/clienturlctx" "github.com/networkservicemesh/sdk/pkg/tools/grpcutils" @@ -43,69 +42,30 @@ type dialNSClient struct { } func (c *dialNSClient) Register(ctx context.Context, in *registry.NetworkService, opts ...grpc.CallOption) (*registry.NetworkService, error) { - closeContextFunc := postpone.ContextWithValues(ctx) - // If no clientURL, we have no work to do - // call the next in the chain - clientURL := clienturlctx.ClientURL(ctx) - if clientURL == nil { - return next.NetworkServiceRegistryClient(ctx).Register(ctx, in, opts...) - } - - cc, _ := clientconn.LoadOrStore(ctx, newDialer(c.chainCtx, c.dialTimeout, c.dialOptions...)) - - // If there's an existing grpc.ClientConnInterface and it's not ours, call the next in the chain - di, ok := cc.(*dialer) - if !ok { - return next.NetworkServiceRegistryClient(ctx).Register(ctx, in, opts...) - } - - // If our existing dialer has a different URL close down the chain - if di.clientURL != nil && di.clientURL.String() != clientURL.String() { - closeCtx, closeCancel := closeContextFunc() - defer closeCancel() - err := di.Dial(closeCtx, di.clientURL) - if err != nil { - log.FromContext(ctx).Errorf("can not redial to %v, err %v. Deleting clientconn...", grpcutils.URLToTarget(di.clientURL), err) - clientconn.Delete(ctx) - return nil, err - } - _, _ = next.NetworkServiceRegistryClient(ctx).Unregister(clienturlctx.WithClientURL(closeCtx, di.clientURL), in, opts...) - } - - err := di.Dial(ctx, clientURL) + var di = newDialer(c.chainCtx, c.dialTimeout, c.dialOptions...) + clientconn.Store(ctx, di) + err := di.Dial(ctx, clienturlctx.ClientURL(ctx)) if err != nil { - log.FromContext(ctx).Errorf("can not dial to %v, err %v. Deleting clientconn...", grpcutils.URLToTarget(clientURL), err) - clientconn.Delete(ctx) return nil, err } - conn, err := next.NetworkServiceRegistryClient(ctx).Register(ctx, in, opts...) - if err != nil { + defer func() { + clientconn.Delete(ctx) _ = di.Close() - return nil, err - } - return conn, nil + }() + return next.NetworkServiceRegistryClient(ctx).Register(ctx, in, opts...) } func (c *dialNSClient) Unregister(ctx context.Context, in *registry.NetworkService, opts ...grpc.CallOption) (*empty.Empty, error) { - // If no clientURL, we have no work to do - // call the next in the chain - clientURL := clienturlctx.ClientURL(ctx) - if clientURL == nil { - return next.NetworkServiceRegistryClient(ctx).Unregister(ctx, in, opts...) - } - - cc, _ := clientconn.Load(ctx) - - di, ok := cc.(*dialer) - if !ok { - return next.NetworkServiceRegistryClient(ctx).Unregister(ctx, in, opts...) + var di = newDialer(c.chainCtx, c.dialTimeout, c.dialOptions...) + clientconn.Store(ctx, di) + err := di.Dial(ctx, clienturlctx.ClientURL(ctx)) + if err != nil { + return nil, err } defer func() { - _ = di.Close() clientconn.Delete(ctx) + _ = di.Close() }() - _ = di.Dial(ctx, clientURL) - return next.NetworkServiceRegistryClient(ctx).Unregister(ctx, in, opts...) } diff --git a/pkg/registry/common/dial/nse_client.go b/pkg/registry/common/dial/nse_client.go index 5864b5643..802789643 100644 --- a/pkg/registry/common/dial/nse_client.go +++ b/pkg/registry/common/dial/nse_client.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2023 Cisco and/or its affiliates. +// Copyright (c) 2021-2024 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -26,11 +26,9 @@ import ( "github.com/golang/protobuf/ptypes/empty" "github.com/networkservicemesh/api/pkg/api/registry" "google.golang.org/grpc" - "google.golang.org/protobuf/types/known/emptypb" "github.com/networkservicemesh/sdk/pkg/registry/common/clientconn" "github.com/networkservicemesh/sdk/pkg/registry/core/next" - "github.com/networkservicemesh/sdk/pkg/tools/postpone" "github.com/networkservicemesh/sdk/pkg/tools/clienturlctx" "github.com/networkservicemesh/sdk/pkg/tools/grpcutils" @@ -44,74 +42,29 @@ type dialNSEClient struct { } func (c *dialNSEClient) Register(ctx context.Context, in *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (*registry.NetworkServiceEndpoint, error) { - closeContextFunc := postpone.ContextWithValues(ctx) - // If no clientURL, we have no work to do - // call the next in the chain - clientURL := clienturlctx.ClientURL(ctx) - if clientURL == nil { - return next.NetworkServiceEndpointRegistryClient(ctx).Register(ctx, in, opts...) - } - - cc, _ := clientconn.LoadOrStore(ctx, newDialer(c.chainCtx, c.dialTimeout, c.dialOptions...)) - - // If there's an existing grpc.ClientConnInterface and it's not ours, call the next in the chain - di, ok := cc.(*dialer) - if !ok { - return next.NetworkServiceEndpointRegistryClient(ctx).Register(ctx, in, opts...) - } - - // If our existing dialer has a different URL close down the chain - if di.clientURL != nil && di.clientURL.String() != clientURL.String() { - closeCtx, closeCancel := closeContextFunc() - defer closeCancel() - err := di.Dial(closeCtx, di.clientURL) - if err != nil { - log.FromContext(ctx).Errorf("can not redial to %v, err %v. Deleting clientconn...", grpcutils.URLToTarget(di.clientURL), err) - clientconn.Delete(ctx) - return nil, err - } - _, _ = next.NetworkServiceEndpointRegistryClient(ctx).Unregister(clienturlctx.WithClientURL(closeCtx, di.clientURL), in, opts...) - } - - err := di.Dial(ctx, clientURL) + var di = newDialer(c.chainCtx, c.dialTimeout, c.dialOptions...) + clientconn.Store(ctx, di) + err := di.Dial(ctx, clienturlctx.ClientURL(ctx)) if err != nil { - log.FromContext(ctx).Errorf("can not dial to %v, err %v. Deleting clientconn...", grpcutils.URLToTarget(clientURL), err) - clientconn.Delete(ctx) return nil, err } - - conn, err := next.NetworkServiceEndpointRegistryClient(ctx).Register(ctx, in, opts...) - if err != nil { + defer func() { + clientconn.Delete(ctx) _ = di.Close() - return nil, err - } - return conn, nil + }() + return next.NetworkServiceEndpointRegistryClient(ctx).Register(ctx, in, opts...) } func (c *dialNSEClient) Unregister(ctx context.Context, in *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (*empty.Empty, error) { - // If no clientURL, we have no work to do - // call the next in the chain - clientURL := clienturlctx.ClientURL(ctx) - if clientURL == nil { - return next.NetworkServiceEndpointRegistryClient(ctx).Unregister(ctx, in, opts...) - } - - cc, loaded := clientconn.LoadOrStore(ctx, newDialer(c.chainCtx, c.dialTimeout, c.dialOptions...)) - - di, ok := cc.(*dialer) - if !ok { - return next.NetworkServiceEndpointRegistryClient(ctx).Unregister(ctx, in, opts...) + var di = newDialer(c.chainCtx, c.dialTimeout, c.dialOptions...) + clientconn.Store(ctx, di) + err := di.Dial(ctx, clienturlctx.ClientURL(ctx)) + if err != nil { + return nil, err } - - err := di.Dial(ctx, clientURL) defer func() { - _ = di.Close() clientconn.Delete(ctx) + _ = di.Close() }() - - if err != nil && !loaded { - log.FromContext(ctx).Errorf("can not dial to %v, err %v", grpcutils.URLToTarget(clientURL), err) - return &emptypb.Empty{}, err - } return next.NetworkServiceEndpointRegistryClient(ctx).Unregister(ctx, in, opts...) } diff --git a/pkg/registry/common/dnsresolve/common.go b/pkg/registry/common/dnsresolve/common.go index 2f12e7b3b..b7e415033 100644 --- a/pkg/registry/common/dnsresolve/common.go +++ b/pkg/registry/common/dnsresolve/common.go @@ -1,6 +1,6 @@ // Copyright (c) 2020-2022 Doc.ai and/or its affiliates. // -// Copyright (c) 2023 Cisco and/or its affiliates. +// Copyright (c) 2023-2024 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -31,11 +31,17 @@ import ( "github.com/networkservicemesh/sdk/pkg/tools/log" ) -// DefaultNsmgrProxyService default NSM nsmgr proxy service name for SRV lookup -const DefaultNsmgrProxyService = "nsmgr-proxy.nsm-system" +// DefaultRegistryProxyService default NSM registry proxy service name for SRV lookup +const DefaultRegistryProxyService = "registry-proxy" // DefaultRegistryService default NSM registry service name for SRV lookup -const DefaultRegistryService = "registry.nsm-system" +const DefaultRegistryService = "registry" + +// DefaultNSMgrProxyService default NSM registry service name for SRV lookup +const DefaultNSMgrProxyService = "nsmgr-proxy" + +// DefaultExternalDNSService default NSM dns service name for SRV lookup +const DefaultExternalDNSService = "external-dns" // Resolver is DNS resolver type Resolver interface { diff --git a/pkg/registry/common/dnsresolve/ns_client.go b/pkg/registry/common/dnsresolve/ns_client.go index 820885c16..86e7ebeb4 100644 --- a/pkg/registry/common/dnsresolve/ns_client.go +++ b/pkg/registry/common/dnsresolve/ns_client.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Cisco and/or its affiliates. +// Copyright (c) 2022-2024 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -18,6 +18,7 @@ package dnsresolve import ( "context" + "fmt" "net" "github.com/golang/protobuf/ptypes/empty" @@ -57,19 +58,21 @@ func NewNetworkServiceRegistryClient(opts ...Option) registry.NetworkServiceRegi } func (d *dnsNSResolveClient) Register(ctx context.Context, ns *registry.NetworkService, opts ...grpc.CallOption) (*registry.NetworkService, error) { - domain := interdomain.Domain(ns.Name) + var original = ns.GetName() + domain := interdomain.Domain(original) url, err := resolveDomain(ctx, d.registryService, domain, d.resolver) if err != nil { return nil, err } ctx = clienturlctx.WithClientURL(ctx, url) - ns.Name = interdomain.Target(ns.Name) + ns.Name = interdomain.Target(original) resp, err := next.NetworkServiceRegistryClient(ctx).Register(ctx, ns, opts...) if err != nil { + fmt.Println(err.Error()) return nil, err } - resp.Name = interdomain.Join(resp.Name, domain) + resp.Name = original return resp, nil } diff --git a/pkg/registry/common/dnsresolve/ns_server.go b/pkg/registry/common/dnsresolve/ns_server.go index fe819b8dd..8d3c19c04 100644 --- a/pkg/registry/common/dnsresolve/ns_server.go +++ b/pkg/registry/common/dnsresolve/ns_server.go @@ -1,6 +1,6 @@ // Copyright (c) 2020-2022 Doc.ai and/or its affiliates. // -// Copyright (c) 2023 Cisco Systems, Inc. +// Copyright (c) 2023-2024 Cisco Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -22,8 +22,6 @@ import ( "context" "net" - "github.com/pkg/errors" - "github.com/networkservicemesh/sdk/pkg/tools/clienturlctx" "github.com/networkservicemesh/sdk/pkg/tools/interdomain" @@ -88,19 +86,20 @@ func (s *dnsFindNSServer) Send(nseResp *registry.NetworkServiceResponse) error { } func (d *dnsNSResolveServer) Find(q *registry.NetworkServiceQuery, s registry.NetworkServiceRegistry_FindServer) error { - ctx := s.Context() - domain := interdomain.Domain(q.NetworkService.Name) - if domain == "" { - return errors.New("domain cannot be empty") - } - url, err := resolveDomain(ctx, d.registryService, domain, d.resolver) + var ctx = s.Context() + var domain = interdomain.Domain(q.GetNetworkService().GetName()) + var registryProxyURL, err = resolveDomain(ctx, d.registryService, domain, d.resolver) if err != nil { return err } - ctx = clienturlctx.WithClientURL(s.Context(), url) + + ctx = clienturlctx.WithClientURL(s.Context(), registryProxyURL) + s = streamcontext.NetworkServiceRegistryFindServer(ctx, s) - q.NetworkService.Name = interdomain.Target(q.NetworkService.Name) - return next.NetworkServiceRegistryServer(s.Context()).Find(q, &dnsFindNSServer{domain: domain, NetworkServiceRegistry_FindServer: s}) + + q.GetNetworkService().Name = interdomain.Target(q.GetNetworkService().GetName()) + + return next.NetworkServiceRegistryServer(s.Context()).Find(q, &dnsFindNSServer{NetworkServiceRegistry_FindServer: s, domain: domain}) } func (d *dnsNSResolveServer) Unregister(ctx context.Context, ns *registry.NetworkService) (*empty.Empty, error) { diff --git a/pkg/registry/common/dnsresolve/nse_client.go b/pkg/registry/common/dnsresolve/nse_client.go index 16fb7b188..e8f4230e3 100644 --- a/pkg/registry/common/dnsresolve/nse_client.go +++ b/pkg/registry/common/dnsresolve/nse_client.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Cisco and/or its affiliates. +// Copyright (c) 2022-2024 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -56,7 +56,7 @@ func NewNetworkServiceEndpointRegistryClient(opts ...Option) registry.NetworkSer } func (d *dnsNSEResolveClient) Register(ctx context.Context, nse *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (*registry.NetworkServiceEndpoint, error) { - var domain = resolveNSE(nse) + var domain = findDomainFromNSE(nse) var u, err = resolveDomain(ctx, d.registryService, domain, d.resolver) if err != nil { @@ -99,7 +99,7 @@ func (c *dnsNSEResolveFindClient) Recv() (*registry.NetworkServiceEndpointRespon } func (d *dnsNSEResolveClient) Find(ctx context.Context, q *registry.NetworkServiceEndpointQuery, opts ...grpc.CallOption) (registry.NetworkServiceEndpointRegistry_FindClient, error) { - var domain = resolveNSE(q.NetworkServiceEndpoint) + var domain = findDomainFromNSE(q.NetworkServiceEndpoint) var nsmgrProxyURL, err = resolveDomain(ctx, d.registryService, domain, d.resolver) if err != nil { @@ -121,7 +121,7 @@ func (d *dnsNSEResolveClient) Find(ctx context.Context, q *registry.NetworkServi } func (d *dnsNSEResolveClient) Unregister(ctx context.Context, nse *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (*empty.Empty, error) { - var domain = resolveNSE(nse) + var domain = findDomainFromNSE(nse) var u, err = resolveDomain(ctx, d.registryService, domain, d.resolver) if err != nil { diff --git a/pkg/registry/common/dnsresolve/nse_server.go b/pkg/registry/common/dnsresolve/nse_server.go index 70de67175..b93150cee 100644 --- a/pkg/registry/common/dnsresolve/nse_server.go +++ b/pkg/registry/common/dnsresolve/nse_server.go @@ -1,6 +1,6 @@ // Copyright (c) 2020-2022 Doc.ai and/or its affiliates. // -// Copyright (c) 2022-2023 Cisco Systems, Inc. +// Copyright (c) 2022-2024 Cisco Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -20,11 +20,11 @@ package dnsresolve import ( "context" + "fmt" "net" "net/url" "github.com/networkservicemesh/sdk/pkg/tools/clienturlctx" - "github.com/networkservicemesh/sdk/pkg/tools/log" "github.com/networkservicemesh/sdk/pkg/tools/interdomain" @@ -36,17 +36,17 @@ import ( ) type dnsNSEResolveServer struct { - resolver Resolver - nsmgrProxyService string - registryService string + resolver Resolver + registryService, registryProxyService, nsmgrProxyService string } // NewNetworkServiceEndpointRegistryServer creates new NetworkServiceRegistryServer that can resolve passed domain to clienturl func NewNetworkServiceEndpointRegistryServer(opts ...Option) registry.NetworkServiceEndpointRegistryServer { var serverOptions = &options{ - resolver: net.DefaultResolver, - registryService: DefaultRegistryService, - nsmgrProxyService: DefaultNsmgrProxyService, + resolver: net.DefaultResolver, + registryService: DefaultRegistryService, + registryProxyService: DefaultRegistryProxyService, + nsmgrProxyService: DefaultNSMgrProxyService, } for _, opt := range opts { @@ -54,9 +54,10 @@ func NewNetworkServiceEndpointRegistryServer(opts ...Option) registry.NetworkSer } r := &dnsNSEResolveServer{ - resolver: serverOptions.resolver, - nsmgrProxyService: serverOptions.nsmgrProxyService, - registryService: serverOptions.registryService, + resolver: serverOptions.resolver, + registryService: serverOptions.registryService, + registryProxyService: serverOptions.registryProxyService, + nsmgrProxyService: serverOptions.nsmgrProxyService, } return r @@ -88,7 +89,7 @@ func translateNSE(nse *registry.NetworkServiceEndpoint, translator func(string) } // TODO: consider to return error if NSE is not consistent and have multi domains target. -func resolveNSE(nse *registry.NetworkServiceEndpoint) string { +func findDomainFromNSE(nse *registry.NetworkServiceEndpoint) string { var domain string for _, name := range append([]string{nse.Name}, nse.GetNetworkServiceNames()...) { @@ -102,7 +103,7 @@ func resolveNSE(nse *registry.NetworkServiceEndpoint) string { } func (d *dnsNSEResolveServer) Register(ctx context.Context, nse *registry.NetworkServiceEndpoint) (*registry.NetworkServiceEndpoint, error) { - var domain = resolveNSE(nse) + var domain = findDomainFromNSE(nse) var u, err = resolveDomain(ctx, d.registryService, domain, d.resolver) if err != nil { @@ -127,8 +128,8 @@ func (d *dnsNSEResolveServer) Register(ctx context.Context, nse *registry.Networ } type dnsFindNSEServer struct { - domain string - nseURL *url.URL + domain string + externalDNSURL, nsmgrProxyURL *url.URL registry.NetworkServiceEndpointRegistry_FindServer } @@ -137,8 +138,18 @@ func (s *dnsFindNSEServer) Send(nseResp *registry.NetworkServiceEndpointResponse return interdomain.Join(str, s.domain) }) - if s.nseURL != nil { - nseResp.NetworkServiceEndpoint.Url = s.nseURL.String() + if s.nsmgrProxyURL != nil { + var u *url.URL + var err error + if s.externalDNSURL != nil { + u, err = url.Parse(fmt.Sprintf("dns://%v/%v.%v:%v", s.externalDNSURL.Host, DefaultNSMgrProxyService, s.domain, s.nsmgrProxyURL.Port())) + } else { + u, err = url.Parse(fmt.Sprintf("dns://%v.%v:%v", DefaultNSMgrProxyService, s.domain, s.nsmgrProxyURL.Port())) + } + if err != nil { + return err + } + nseResp.GetNetworkServiceEndpoint().Url = u.String() } return s.NetworkServiceEndpointRegistry_FindServer.Send(nseResp) @@ -146,29 +157,28 @@ func (s *dnsFindNSEServer) Send(nseResp *registry.NetworkServiceEndpointResponse func (d *dnsNSEResolveServer) Find(q *registry.NetworkServiceEndpointQuery, s registry.NetworkServiceEndpointRegistry_FindServer) error { var ctx = s.Context() - var domain = resolveNSE(q.NetworkServiceEndpoint) - var nsmgrProxyURL, err = resolveDomain(ctx, d.registryService, domain, d.resolver) - + var domain = findDomainFromNSE(q.NetworkServiceEndpoint) + var registryProxyURL, err = resolveDomain(ctx, d.registryProxyService, domain, d.resolver) if err != nil { - return err + registryProxyURL, err = resolveDomain(ctx, d.registryService, domain, d.resolver) } - - ctx = clienturlctx.WithClientURL(s.Context(), nsmgrProxyURL) - nsmgrProxyURL, err = resolveDomain(ctx, d.nsmgrProxyService, domain, d.resolver) - if err != nil { - log.FromContext(ctx).Errorf("nsmgrProxyService is not reachable by domain: %v", domain) + return err } + var nsmgrProxyURL, _ = resolveDomain(ctx, d.nsmgrProxyService, domain, d.resolver) + var externalDNSURL, _ = resolveDomain(ctx, DefaultExternalDNSService, domain, d.resolver) + + ctx = clienturlctx.WithClientURL(s.Context(), registryProxyURL) s = streamcontext.NetworkServiceEndpointRegistryFindServer(ctx, s) translateNSE(q.NetworkServiceEndpoint, interdomain.Target) - return next.NetworkServiceEndpointRegistryServer(s.Context()).Find(q, &dnsFindNSEServer{NetworkServiceEndpointRegistry_FindServer: s, domain: domain, nseURL: nsmgrProxyURL}) + return next.NetworkServiceEndpointRegistryServer(s.Context()).Find(q, &dnsFindNSEServer{NetworkServiceEndpointRegistry_FindServer: s, domain: domain, nsmgrProxyURL: nsmgrProxyURL, externalDNSURL: externalDNSURL}) } func (d *dnsNSEResolveServer) Unregister(ctx context.Context, nse *registry.NetworkServiceEndpoint) (*empty.Empty, error) { - var domain = resolveNSE(nse) + var domain = findDomainFromNSE(nse) var u, err = resolveDomain(ctx, d.registryService, domain, d.resolver) if err != nil { @@ -186,10 +196,6 @@ func (d *dnsNSEResolveServer) setResolver(r Resolver) { d.resolver = r } -func (d *dnsNSEResolveServer) setNSMgrProxyService(service string) { - d.nsmgrProxyService = service -} - func (d *dnsNSEResolveServer) setRegistryService(service string) { d.registryService = service } diff --git a/pkg/registry/common/dnsresolve/nse_server_test.go b/pkg/registry/common/dnsresolve/nse_server_test.go index 79bae5e73..244261511 100644 --- a/pkg/registry/common/dnsresolve/nse_server_test.go +++ b/pkg/registry/common/dnsresolve/nse_server_test.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2022 Doc.ai and/or its affiliates. // +// Copyright (c) 2024 Cisco and/or its affiliates. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -156,7 +158,7 @@ func Test_DNSResolve_LookupNsmgrProxy(t *testing.T) { r, err := resp.Recv() require.NoError(t, err) - require.Equal(t, nsmgrProxyURL.String(), r.NetworkServiceEndpoint.Url) + require.Contains(t, r.NetworkServiceEndpoint.Url, "dns:") _, err = s.Unregister(ctx, ®istry.NetworkServiceEndpoint{Name: "nse-1@" + domain}) require.NoError(t, err) diff --git a/pkg/registry/common/dnsresolve/options.go b/pkg/registry/common/dnsresolve/options.go index 595618c0c..19e301d4b 100644 --- a/pkg/registry/common/dnsresolve/options.go +++ b/pkg/registry/common/dnsresolve/options.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2021 Doc.ai and/or its affiliates. // +// Copyright (c) 2024 Cisco and/or its affiliates. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,9 +19,10 @@ package dnsresolve type options struct { - resolver Resolver - nsmgrProxyService string - registryService string + resolver Resolver + nsmgrProxyService string + registryService string + registryProxyService string } // Option is option to configure dnsresovle chain elements diff --git a/pkg/registry/common/interdomainbypass/nse_server.go b/pkg/registry/common/interdomainbypass/nse_server.go new file mode 100644 index 000000000..ada97b3c6 --- /dev/null +++ b/pkg/registry/common/interdomainbypass/nse_server.go @@ -0,0 +1,69 @@ +// Copyright (c) 2020-2024 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package interdomainbypass provides a chain element that manages registered NSEs +package interdomainbypass + +import ( + "context" + + "github.com/edwarnicke/genericsync" + "github.com/golang/protobuf/ptypes/empty" + "github.com/networkservicemesh/api/pkg/api/registry" + + "github.com/networkservicemesh/sdk/pkg/registry/core/next" +) + +type interdomainBypassFindServer struct { + value string + registry.NetworkServiceEndpointRegistry_FindServer +} + +func (s *interdomainBypassFindServer) Send(resp *registry.NetworkServiceEndpointResponse) error { + resp.GetNetworkServiceEndpoint().Url = s.value + return s.NetworkServiceEndpointRegistry_FindServer.Send(resp) +} + +type interdomainBypassServer struct { + nseToNsmgrMap genericsync.Map[string, string] +} + +func (n *interdomainBypassServer) Register(ctx context.Context, service *registry.NetworkServiceEndpoint) (*registry.NetworkServiceEndpoint, error) { + n.nseToNsmgrMap.Store(service.GetName(), service.GetUrl()) + return next.NetworkServiceEndpointRegistryServer(ctx).Register(ctx, service) +} + +func (n *interdomainBypassServer) Find(query *registry.NetworkServiceEndpointQuery, server registry.NetworkServiceEndpointRegistry_FindServer) error { + if v, ok := n.nseToNsmgrMap.Load(query.GetNetworkServiceEndpoint().GetName()); ok { + server = &interdomainBypassFindServer{ + NetworkServiceEndpointRegistry_FindServer: server, + value: v, + } + } + + return next.NetworkServiceEndpointRegistryServer(server.Context()).Find(query, server) +} + +func (n *interdomainBypassServer) Unregister(ctx context.Context, service *registry.NetworkServiceEndpoint) (*empty.Empty, error) { + n.nseToNsmgrMap.Delete(service.GetName()) + + return next.NetworkServiceEndpointRegistryServer(ctx).Unregister(ctx, service) +} + +// NewNetworkServiceEndpointRegistryServer - returns a new null server that does nothing but call next.NetworkServiceEndpointRegistryServer(ctx). +func NewNetworkServiceEndpointRegistryServer() registry.NetworkServiceEndpointRegistryServer { + return new(interdomainBypassServer) +} diff --git a/pkg/registry/common/interdomainbypass/server.go b/pkg/registry/common/interdomainbypass/server.go deleted file mode 100644 index 38c7e7702..000000000 --- a/pkg/registry/common/interdomainbypass/server.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2022-2023 Cisco and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package interdomainbypass provides registry chain element that sets to outgoing NSE the public nsmgr-proxy and stores into the shared map the public nsmgr URL from the incoming endpoint. -package interdomainbypass - -import ( - "context" - "net/url" - - "github.com/edwarnicke/genericsync" - "github.com/golang/protobuf/ptypes/empty" - "github.com/networkservicemesh/api/pkg/api/registry" - "github.com/pkg/errors" - - "github.com/networkservicemesh/sdk/pkg/registry/core/next" -) - -type interdomainBypassNSEServer struct { - m *genericsync.Map[string, *url.URL] - u *url.URL -} - -type interdomainBypassNSEFindServer struct { - m *genericsync.Map[string, *url.URL] - u *url.URL - registry.NetworkServiceEndpointRegistry_FindServer -} - -func (n *interdomainBypassNSEServer) Register(ctx context.Context, service *registry.NetworkServiceEndpoint) (*registry.NetworkServiceEndpoint, error) { - var originalURL = service.Url - service.Url = n.u.String() - - resp, err := next.NetworkServiceEndpointRegistryServer(ctx).Register(ctx, service) - - if err != nil { - return nil, err - } - - u, _ := url.Parse(originalURL) - - n.m.Store(service.Name, u) - - resp.Url = originalURL - - return resp, nil -} - -func (n *interdomainBypassNSEServer) Find(query *registry.NetworkServiceEndpointQuery, server registry.NetworkServiceEndpointRegistry_FindServer) error { - return next.NetworkServiceEndpointRegistryServer(server.Context()).Find(query, &interdomainBypassNSEFindServer{NetworkServiceEndpointRegistry_FindServer: server, m: n.m, u: n.u}) -} - -func (n *interdomainBypassNSEServer) Unregister(ctx context.Context, service *registry.NetworkServiceEndpoint) (*empty.Empty, error) { - n.m.Delete(service.Name) - var originalURL = service.Url - service.Url = n.u.String() - defer func() { - service.Url = originalURL - }() - return next.NetworkServiceEndpointRegistryServer(ctx).Unregister(ctx, service) -} - -// NewNetworkServiceEndpointRegistryServer creates new instance of interdomainbypass NSE server. -// It simply stores into passed stringurl.Map all incoming nse.Name:nse.URL entries. -// And sets passed URL for outgoing NSEs. -func NewNetworkServiceEndpointRegistryServer(m *genericsync.Map[string, *url.URL], u *url.URL) registry.NetworkServiceEndpointRegistryServer { - if m == nil { - panic("m can not be nil") - } - if u == nil { - panic("u can not be nil") - } - return &interdomainBypassNSEServer{m: m, u: u} -} - -func (s *interdomainBypassNSEFindServer) Send(nseResp *registry.NetworkServiceEndpointResponse) error { - nseURL := nseResp.GetNetworkServiceEndpoint().GetUrl() - u, err := url.Parse(nseURL) - if err != nil { - return errors.Wrapf(err, "failed to parse url %s", nseURL) - } - s.m.LoadOrStore(nseResp.NetworkServiceEndpoint.GetName(), u) - nseResp.GetNetworkServiceEndpoint().Url = s.u.String() - return s.NetworkServiceEndpointRegistry_FindServer.Send(nseResp) -} diff --git a/pkg/registry/common/replaceurl/nse_server.go b/pkg/registry/common/replaceurl/nse_server.go new file mode 100644 index 000000000..9991d3c2b --- /dev/null +++ b/pkg/registry/common/replaceurl/nse_server.go @@ -0,0 +1,64 @@ +// Copyright (c) 2024 Cisco and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package replaceurl provides a chain element that replaces the URL of found NSEs. +package replaceurl + +import ( + "context" + "net/url" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/networkservicemesh/api/pkg/api/registry" + + "github.com/networkservicemesh/sdk/pkg/registry/core/next" +) + +type findReplaceNSEURLServer struct { + value *url.URL + registry.NetworkServiceEndpointRegistry_FindServer +} + +func (s *findReplaceNSEURLServer) Send(resp *registry.NetworkServiceEndpointResponse) error { + if s.value != nil { + resp.GetNetworkServiceEndpoint().Url = s.value.String() + } + return s.NetworkServiceEndpointRegistry_FindServer.Send(resp) +} + +type replaceNSEURLServer struct { + value *url.URL +} + +// NewNetworkServiceEndpointRegistryServer ccreates a new NetworkServiceEndpointRegistryServer that replaces urls of found NSEs +func NewNetworkServiceEndpointRegistryServer(v *url.URL) registry.NetworkServiceEndpointRegistryServer { + return &replaceNSEURLServer{value: v} +} + +func (n *replaceNSEURLServer) Register(ctx context.Context, nse *registry.NetworkServiceEndpoint) (*registry.NetworkServiceEndpoint, error) { + return next.NetworkServiceEndpointRegistryServer(ctx).Register(ctx, nse) +} + +func (n *replaceNSEURLServer) Find(query *registry.NetworkServiceEndpointQuery, server registry.NetworkServiceEndpointRegistry_FindServer) error { + return next.NetworkServiceEndpointRegistryServer(server.Context()).Find(query, &findReplaceNSEURLServer{ + NetworkServiceEndpointRegistry_FindServer: server, + value: n.value, + }) +} + +func (n *replaceNSEURLServer) Unregister(ctx context.Context, nse *registry.NetworkServiceEndpoint) (*empty.Empty, error) { + return next.NetworkServiceEndpointRegistryServer(ctx).Unregister(ctx, nse) +} diff --git a/pkg/registry/common/seturl/server.go b/pkg/registry/common/seturl/server.go new file mode 100644 index 000000000..be9d4633d --- /dev/null +++ b/pkg/registry/common/seturl/server.go @@ -0,0 +1,51 @@ +// Copyright (c) 2024 Cisco and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package seturl provides chain elements for changing URLs on registration / unregistration +package seturl + +import ( + "context" + "net/url" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/networkservicemesh/api/pkg/api/registry" + + "github.com/networkservicemesh/sdk/pkg/registry/core/next" +) + +type setNSEURLServer struct { + value *url.URL +} + +// NewNetworkServiceEndpointRegistryServer creates a new NetworkServiceEndpointRegistryServer that replaces URL on register and unregister +func NewNetworkServiceEndpointRegistryServer(v *url.URL) registry.NetworkServiceEndpointRegistryServer { + return &setNSEURLServer{value: v} +} + +func (n *setNSEURLServer) Register(ctx context.Context, nse *registry.NetworkServiceEndpoint) (*registry.NetworkServiceEndpoint, error) { + nse.Url = n.value.String() + return next.NetworkServiceEndpointRegistryServer(ctx).Register(ctx, nse) +} + +func (n *setNSEURLServer) Find(query *registry.NetworkServiceEndpointQuery, server registry.NetworkServiceEndpointRegistry_FindServer) error { + return next.NetworkServiceEndpointRegistryServer(server.Context()).Find(query, server) +} + +func (n *setNSEURLServer) Unregister(ctx context.Context, nse *registry.NetworkServiceEndpoint) (*empty.Empty, error) { + nse.Url = n.value.String() + return next.NetworkServiceEndpointRegistryServer(ctx).Unregister(ctx, nse) +} diff --git a/pkg/registry/switchcase/common.go b/pkg/registry/switchcase/common.go new file mode 100644 index 000000000..127c3eff5 --- /dev/null +++ b/pkg/registry/switchcase/common.go @@ -0,0 +1,24 @@ +// Copyright (c) 2024 Cisco and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package switchcase + +import "context" + +// Otherwise can be used as condition for behavior 'by default'. +func Otherwise[T any](context.Context, T) bool { + return true +} diff --git a/pkg/tools/dnsutils/memory/handler.go b/pkg/tools/dnsutils/memory/handler.go index fc94d0c2a..9d14e625a 100644 --- a/pkg/tools/dnsutils/memory/handler.go +++ b/pkg/tools/dnsutils/memory/handler.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022-2023 Cisco Systems, Inc. +// Copyright (c) 2022-2024 Cisco Systems, Inc. // // SPDX-License-Identifier: Apache-2.0 // @@ -45,7 +45,8 @@ func (r *responseWriter) WriteMsg(m *dns.Msg) error { } type memoryHandler struct { - records *genericsync.Map[string, []net.IP] + ipRecords *genericsync.Map[string, []net.IP] + srvRecords *genericsync.Map[string, []*net.TCPAddr] } func (f *memoryHandler) ServeDNS(ctx context.Context, rw dns.ResponseWriter, msg *dns.Msg) { @@ -66,6 +67,8 @@ func (f *memoryHandler) ServeDNS(ctx context.Context, rw dns.ResponseWriter, msg resp.Answer = append(resp.Answer, f.a(name)...) case dns.TypePTR: resp.Answer = append(resp.Answer, f.ptr(name)...) + case dns.TypeSRV: + resp.Answer = append(resp.Answer, f.srv(name)...) } if len(resp.Answer) != 0 { @@ -75,7 +78,7 @@ func (f *memoryHandler) ServeDNS(ctx context.Context, rw dns.ResponseWriter, msg return } - if _, ok := f.records.Load(name); ok { + if _, ok := f.ipRecords.Load(name); ok { m := new(dns.Msg) _ = rw.WriteMsg(m.SetRcode(msg, dns.RcodeSuccess)) } else { @@ -93,10 +96,36 @@ func NewDNSHandler(records *genericsync.Map[string, []net.IP]) dnsutils.Handler if records == nil { panic("records cannot be nil") } - return &memoryHandler{records: records} + return &memoryHandler{ipRecords: records} +} + +// NewDNSHandlerWithOptions creates a new dnsutils.Handler with specific options +func NewDNSHandlerWithOptions(opts ...Option) dnsutils.Handler { + var r = new(memoryHandler) + for _, opt := range opts { + opt(r) + } + return r +} + +func (f *memoryHandler) srv(domain string) []dns.RR { + v, ok := f.srvRecords.Load(domain) + if !ok { + return nil + } + var result []dns.RR + for _, record := range v { + result = append(result, &dns.SRV{ + Port: uint16(record.Port), + Target: domain, + Hdr: dns.RR_Header{Name: domain, Rrtype: dns.TypeSRV, Class: dns.ClassINET, Ttl: defaultTTL}, + }) + } + + return result } func (f *memoryHandler) a(domain string) []dns.RR { - var ips, _ = f.records.Load(domain) + var ips, _ = f.ipRecords.Load(domain) var answers []dns.RR for _, ip := range ips { if ip.To4() == nil { @@ -111,7 +140,7 @@ func (f *memoryHandler) a(domain string) []dns.RR { } func (f *memoryHandler) aaaa(domain string) []dns.RR { - var ips, _ = f.records.Load(domain) + var ips, _ = f.ipRecords.Load(domain) var answers []dns.RR for _, ip := range ips { if ip.To4() != nil { @@ -152,7 +181,7 @@ func (f *memoryHandler) ptr(domain string) []dns.RR { } var recordNames []string - f.records.Range(func(key string, value []net.IP) bool { + f.ipRecords.Range(func(key string, value []net.IP) bool { for _, v := range value { if v.Equal(requestedIP) { recordNames = append(recordNames, key) diff --git a/pkg/tools/dnsutils/memory/options.go b/pkg/tools/dnsutils/memory/options.go new file mode 100644 index 000000000..106f01f1f --- /dev/null +++ b/pkg/tools/dnsutils/memory/options.go @@ -0,0 +1,40 @@ +// Copyright (c) 2024 Cisco and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memory + +import ( + "net" + + "github.com/edwarnicke/genericsync" +) + +// Option replaces default behavior for memory dns chain element +type Option func(*memoryHandler) + +// WithIPRecords sets IP responses +func WithIPRecords(r *genericsync.Map[string, []net.IP]) Option { + return func(mh *memoryHandler) { + mh.ipRecords = r + } +} + +// WithSRVRecords sets SRV responses +func WithSRVRecords(r *genericsync.Map[string, []*net.TCPAddr]) Option { + return func(mh *memoryHandler) { + mh.srvRecords = r + } +} diff --git a/pkg/tools/interdomain/utils.go b/pkg/tools/interdomain/utils.go index d37608219..b0a4e6d67 100644 --- a/pkg/tools/interdomain/utils.go +++ b/pkg/tools/interdomain/utils.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2021 Doc.ai and/or its affiliates. // +// Copyright (c) 2024 Cisco and/or its affiliates. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +18,9 @@ package interdomain -import "strings" +import ( + "strings" +) const identifier = "@" diff --git a/pkg/tools/sandbox/builder.go b/pkg/tools/sandbox/builder.go index 504cf4334..0402bbe23 100644 --- a/pkg/tools/sandbox/builder.go +++ b/pkg/tools/sandbox/builder.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2022 Doc.ai and/or its affiliates. // +// Copyright (c) 2024 Cisco and/or its affiliates. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,6 +25,7 @@ import ( "net/url" "os" "runtime" + "strconv" "testing" "time" @@ -34,7 +37,9 @@ import ( "github.com/networkservicemesh/sdk/pkg/registry" "github.com/networkservicemesh/sdk/pkg/registry/chains/memory" "github.com/networkservicemesh/sdk/pkg/registry/chains/proxydns" - "github.com/networkservicemesh/sdk/pkg/registry/common/dnsresolve" + "github.com/networkservicemesh/sdk/pkg/tools/dnsutils" + dnsmemory "github.com/networkservicemesh/sdk/pkg/tools/dnsutils/memory" + "github.com/networkservicemesh/sdk/pkg/tools/dnsutils/next" "github.com/networkservicemesh/sdk/pkg/tools/grpcutils" "github.com/networkservicemesh/sdk/pkg/tools/log" "github.com/networkservicemesh/sdk/pkg/tools/token" @@ -51,23 +56,25 @@ type Builder struct { supplyNSMgrProxy SupplyNSMgrProxyFunc supplyRegistry SupplyRegistryFunc supplyRegistryProxy SupplyRegistryProxyFunc + supplyDNSServer SupplyDNSServerFunc setupNode SetupNodeFunc name string - dnsResolver dnsresolve.Resolver generateTokenFunc token.GeneratorFunc registryDefaultExpiration time.Duration + dnsServer *DNSServerEntry useUnixSockets bool domain *Domain } -func newRegistryMemoryServer(ctx context.Context, tokenGenerator token.GeneratorFunc, defaultExpiration time.Duration, proxyRegistryURL *url.URL, options ...grpc.DialOption) registry.Registry { +func newRegistryMemoryServer(ctx context.Context, tokenGenerator token.GeneratorFunc, defaultExpiration time.Duration, nsmgrProxyURL, proxyRegistryURL *url.URL, options ...grpc.DialOption) registry.Registry { return memory.NewServer( ctx, tokenGenerator, memory.WithDefaultExpiration(defaultExpiration), + memory.WithNSMgrProxyURL(nsmgrProxyURL), memory.WithProxyRegistryURL(proxyRegistryURL), memory.WithDialOptions(options...)) } @@ -79,11 +86,8 @@ func NewBuilder(ctx context.Context, t *testing.T) *Builder { ctx: ctx, nodesCount: 1, supplyNSMgr: nsmgr.NewServer, - supplyNSMgrProxy: nsmgrproxy.NewServer, supplyRegistry: newRegistryMemoryServer, - supplyRegistryProxy: proxydns.NewServer, name: "cluster.local", - dnsResolver: NewFakeResolver(), generateTokenFunc: GenerateTestToken, registryDefaultExpiration: time.Minute, } @@ -107,6 +111,26 @@ func (b *Builder) SetNSMgrSupplier(f SupplyNSMgrFunc) *Builder { return b } +// SetupDefaultDNSServer sets default dns handler for the domain +func (b *Builder) SetupDefaultDNSServer() *Builder { + return b.SetDNSServerSupplier(func(ctx context.Context) dnsutils.Handler { + return nil + }) +} + +// EnableInterdomain enables interdomain components for the NSM domain +func (b *Builder) EnableInterdomain() *Builder { + b.supplyNSMgrProxy = nsmgrproxy.NewServer + b.supplyRegistryProxy = proxydns.NewServer + return b.SetupDefaultDNSServer() +} + +// SetDNSServerSupplier sets handler for the dns server +func (b *Builder) SetDNSServerSupplier(f SupplyDNSServerFunc) *Builder { + b.supplyDNSServer = f + return b +} + // SetNSMgrProxySupplier replaces default nsmgr-proxy supplier to custom function func (b *Builder) SetNSMgrProxySupplier(f SupplyNSMgrProxyFunc) *Builder { b.supplyNSMgrProxy = f @@ -133,18 +157,12 @@ func (b *Builder) SetNodeSetup(f SetupNodeFunc) *Builder { return b } -// SetDNSDomainName sets DNS domain name for the building NSM domain -func (b *Builder) SetDNSDomainName(name string) *Builder { +// SetName sets DNS domain name for the building NSM domain +func (b *Builder) SetName(name string) *Builder { b.name = name return b } -// SetDNSResolver sets DNS resolver for proxy registries -func (b *Builder) SetDNSResolver(d dnsresolve.Resolver) *Builder { - b.dnsResolver = d - return b -} - // SetTokenGenerateFunc sets function for the token generation func (b *Builder) SetTokenGenerateFunc(f token.GeneratorFunc) *Builder { b.generateTokenFunc = f @@ -172,8 +190,7 @@ func (b *Builder) UseUnixSockets() *Builder { // Build builds Domain and Supplier func (b *Builder) Build() *Domain { b.domain = &Domain{ - Name: b.name, - DNSResolver: b.dnsResolver, + Name: b.name, } if b.useUnixSockets { @@ -196,21 +213,26 @@ func (b *Builder) Build() *Domain { b.domain.supplyURL = b.supplyTCPAddress() } - if b.supplyRegistryProxy != nil { - require.NotNil(b.t, b.supplyNSMgrProxy, "NSMgr proxy supplier should be set if registry proxy supplier is set") - b.domain.NSMgrProxy = &NSMgrEntry{ + if b.supplyRegistry != nil { + b.domain.Registry = &RegistryEntry{ + URL: b.domain.supplyURL("reg"), + } + } + if b.supplyNSMgrProxy != nil { + b.domain.NSMgrProxy = &EndpointEntry{ URL: b.domain.supplyURL("nsmgr-proxy"), } } + b.domain.DNSServer = b.newDNSServer() b.domain.RegistryProxy = b.newRegistryProxy() - b.domain.Registry = b.newRegistry() b.domain.NSMgrProxy = b.newNSMgrProxy() + b.domain.Registry = b.newRegistry() for i := 0; i < b.nodesCount; i++ { b.domain.Nodes = append(b.domain.Nodes, b.newNode(i)) } - b.buildDNSServer() + b.addDNSRecords() return b.domain } @@ -235,20 +257,71 @@ func (b *Builder) supplyTCPAddress() func(prefix string) *url.URL { } } +func (b *Builder) newDNSServer() *DNSServerEntry { + if b.supplyDNSServer == nil { + return nil + } + if b.dnsServer != nil { + return b.dnsServer + } + entry := &DNSServerEntry{ + URL: b.domain.supplyURL("dns-server"), + } + + entry.restartableServer = newRestartableServer(b.ctx, b.t, entry.URL, func(ctx context.Context) { + var handlers = []dnsutils.Handler{ + dnsmemory.NewDNSHandlerWithOptions( + dnsmemory.WithIPRecords(&entry.IPRecords), + dnsmemory.WithSRVRecords(&entry.SRVRecords), + ), + } + if additionalFunctionality := b.supplyDNSServer(ctx); additionalFunctionality != nil { + handlers = append(handlers, additionalFunctionality) + } + + entry.Handler = next.NewDNSHandler( + handlers..., + ) + + dnsutils.ListenAndServe(ctx, entry.Handler, entry.URL.Host) + + log.FromContext(ctx).Infof("%s: dns-seever serving on: %v", b.name, entry.URL) + }) + + return entry +} + func (b *Builder) newRegistryProxy() *RegistryEntry { - if b.supplyRegistryProxy == nil { + if b.supplyRegistryProxy == nil || b.domain.DNSServer == nil { return nil } - entry := &RegistryEntry{ + var entry = &RegistryEntry{ URL: b.domain.supplyURL("reg-proxy"), } + + var nsmgrProxyURL *url.URL + + if b.domain.NSMgrProxy != nil { + nsmgrProxyURL = b.domain.NSMgrProxy.URL + } + + var dialer net.Dialer + var resolver = &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + return dialer.DialContext(ctx, network, b.domain.DNSServer.URL.Host) + }, + } + entry.restartableServer = newRestartableServer(b.ctx, b.t, entry.URL, func(ctx context.Context) { entry.Registry = b.supplyRegistryProxy( ctx, b.generateTokenFunc, - b.dnsResolver, + resolver, proxydns.WithDialOptions(DialOptions(WithTokenGenerator(b.generateTokenFunc))...), + proxydns.WithRegistryURL(CloneURL(b.domain.Registry.URL)), + proxydns.WithNSMgrProxyURL(CloneURL(nsmgrProxyURL)), ) serve(ctx, b.t, entry.URL, entry.Register) @@ -263,13 +336,16 @@ func (b *Builder) newRegistry() *RegistryEntry { return nil } - var nsmgrProxyURL *url.URL + var nsmgrProxyURL, registryProxyURL *url.URL if b.domain.NSMgrProxy != nil { nsmgrProxyURL = CloneURL(b.domain.NSMgrProxy.URL) } + if b.domain.RegistryProxy != nil { + registryProxyURL = CloneURL(b.domain.RegistryProxy.URL) + } entry := &RegistryEntry{ - URL: b.domain.supplyURL("reg"), + URL: b.domain.Registry.URL, } entry.restartableServer = newRestartableServer(b.ctx, b.t, entry.URL, func(ctx context.Context) { entry.Registry = b.supplyRegistry( @@ -277,6 +353,7 @@ func (b *Builder) newRegistry() *RegistryEntry { b.generateTokenFunc, b.registryDefaultExpiration, nsmgrProxyURL, + registryProxyURL, DialOptions(WithTokenGenerator(b.generateTokenFunc))..., ) serve(ctx, b.t, entry.URL, entry.Register) @@ -287,26 +364,27 @@ func (b *Builder) newRegistry() *RegistryEntry { return entry } -func (b *Builder) newNSMgrProxy() *NSMgrEntry { - if b.supplyRegistryProxy == nil { +func (b *Builder) newNSMgrProxy() *EndpointEntry { + if b.supplyNSMgrProxy == nil { return nil } - entry := &NSMgrEntry{ + entry := &EndpointEntry{ Name: UniqueName("nsmgr-proxy"), URL: b.domain.NSMgrProxy.URL, } + entry.restartableServer = newRestartableServer(b.ctx, b.t, entry.URL, func(ctx context.Context) { dialOptions := DialOptions(WithTokenGenerator(b.generateTokenFunc)) - entry.Nsmgr = b.supplyNSMgrProxy(ctx, - CloneURL(b.domain.Registry.URL), + entry.Endpoint = b.supplyNSMgrProxy(ctx, CloneURL(b.domain.RegistryProxy.URL), b.generateTokenFunc, nsmgrproxy.WithListenOn(entry.URL), nsmgrproxy.WithName(entry.Name), + nsmgrproxy.WithRegistryURL(b.domain.Registry.URL), nsmgrproxy.WithDialOptions(dialOptions...), ) - serve(ctx, b.t, entry.URL, entry.Register) + serve(ctx, b.t, entry.URL, entry.Endpoint.Register) log.FromContext(ctx).Infof("%s: NSMgr proxy %s serve on: %v", b.name, entry.Name, entry.URL) }) @@ -328,11 +406,41 @@ func (b *Builder) newNode(nodeNum int) *Node { return node } -func (b *Builder) buildDNSServer() { - if b.domain.Registry != nil { - require.NoError(b.t, AddSRVEntry(b.dnsResolver, b.name, dnsresolve.DefaultRegistryService, CloneURL(b.domain.Registry.URL))) +func (b *Builder) addDNSRecords() { + if b.domain.DNSServer == nil { + return } - if b.domain.NSMgrProxy != nil { - require.NoError(b.t, AddSRVEntry(b.dnsResolver, b.name, dnsresolve.DefaultNsmgrProxyService, CloneURL(b.domain.NSMgrProxy.URL))) + var domain = b.domain + b.addRecord("external-dns", domain.DNSServer.URL) + if domain.Registry != nil { + b.addRecord("registry", domain.Registry.URL) + } + if domain.NSMgrProxy != nil { + b.addRecord("nsmgr-proxy", domain.NSMgrProxy.URL) } + if domain.RegistryProxy != nil { + b.addRecord("registry-proxy", domain.RegistryProxy.URL) + } + for i, node := range b.domain.Nodes { + if node.NSMgr != nil { + b.addRecord(fmt.Sprintf("nsmgr-%v", i), node.NSMgr.URL) + } + for name, fwd := range node.Forwarders { + b.addRecord(fmt.Sprintf("fwd-%v-%v", name, i), fwd.URL) + } + } +} + +func (b *Builder) addRecord(name string, u *url.URL) { + name = fmt.Sprintf("%v.%v.", name, b.domain.Name) + log.FromContext(b.ctx).Infof("Adding srv, aaaa, records for name %v, dns server: %v", name, b.domain.DNSServer.URL.String()) + var ip = net.ParseIP(u.Hostname()) + var port, _ = strconv.Atoi(u.Port()) + b.domain.DNSServer.IPRecords.Store(name, []net.IP{ip}) + b.domain.DNSServer.SRVRecords.Store(name, []*net.TCPAddr{ + { + IP: ip, + Port: port, + }, + }) } diff --git a/pkg/tools/sandbox/interdomain_builder.go b/pkg/tools/sandbox/interdomain_builder.go new file mode 100644 index 000000000..a61c4e461 --- /dev/null +++ b/pkg/tools/sandbox/interdomain_builder.go @@ -0,0 +1,68 @@ +// Copyright (c) 2023-2024 Cisco and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sandbox + +import ( + "context" + "testing" +) + +// InterdomainBuilder helps configure multi domain setups for unit testing +type InterdomainBuilder struct { + domains []*Domain + ctx context.Context + t *testing.T + buildFn func() + dnsServer *DNSServerEntry +} + +// NewInterdomainBuilder creates a new instance of InterdomainBuilder +func NewInterdomainBuilder(ctx context.Context, t *testing.T) *InterdomainBuilder { + var res = &InterdomainBuilder{ + t: t, + ctx: ctx, + } + + res.buildFn = func() { + res.dnsServer = NewBuilder(ctx, t).SetNodesCount(0).SetRegistrySupplier(nil).SetupDefaultDNSServer().Build().DNSServer + } + + return res +} + +// BuildDomain configures a new domain +func (b *InterdomainBuilder) BuildDomain(creator func(*Builder) *Builder) *InterdomainBuilder { + var builder = NewBuilder(b.ctx, b.t).EnableInterdomain() + creator(builder) + var fn = b.buildFn + b.buildFn = func() { + fn() + builder.dnsServer = b.dnsServer + b.domains = append(b.domains, builder.Build()) + } + + return b +} + +// Build starts all domains +func (b *InterdomainBuilder) Build() []*Domain { + b.buildFn() + b.buildFn = func() {} + var res = b.domains + b.domains = nil + return res +} diff --git a/pkg/tools/sandbox/types.go b/pkg/tools/sandbox/types.go index 112d52389..0669fac5f 100644 --- a/pkg/tools/sandbox/types.go +++ b/pkg/tools/sandbox/types.go @@ -1,5 +1,7 @@ // Copyright (c) 2020-2022 Doc.ai and/or its affiliates. // +// Copyright (c) 2024 Cisco and/or its affiliates. +// // SPDX-License-Identifier: Apache-2.0 // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,9 +20,11 @@ package sandbox import ( "context" + "net" "net/url" "time" + "github.com/edwarnicke/genericsync" registryapi "github.com/networkservicemesh/api/pkg/api/registry" "google.golang.org/grpc" @@ -31,17 +35,18 @@ import ( registryclient "github.com/networkservicemesh/sdk/pkg/registry/chains/client" "github.com/networkservicemesh/sdk/pkg/registry/chains/proxydns" "github.com/networkservicemesh/sdk/pkg/registry/common/dnsresolve" + "github.com/networkservicemesh/sdk/pkg/tools/dnsutils" "github.com/networkservicemesh/sdk/pkg/tools/token" ) // SupplyNSMgrProxyFunc nsmgr proxy -type SupplyNSMgrProxyFunc func(ctx context.Context, regURL, proxyURL *url.URL, tokenGenerator token.GeneratorFunc, options ...nsmgrproxy.Option) nsmgr.Nsmgr +type SupplyNSMgrProxyFunc func(ctx context.Context, regURL *url.URL, tokenGenerator token.GeneratorFunc, options ...nsmgrproxy.Option) endpoint.Endpoint // SupplyNSMgrFunc supplies NSMGR type SupplyNSMgrFunc func(ctx context.Context, tokenGenerator token.GeneratorFunc, options ...nsmgr.Option) nsmgr.Nsmgr // SupplyRegistryFunc supplies Registry -type SupplyRegistryFunc func(ctx context.Context, tokenGenerator token.GeneratorFunc, defaultExpiration time.Duration, proxyRegistryURL *url.URL, options ...grpc.DialOption) registry.Registry +type SupplyRegistryFunc func(ctx context.Context, tokenGenerator token.GeneratorFunc, defaultExpiration time.Duration, proxyRegistryURL, nsmgrProxyURL *url.URL, options ...grpc.DialOption) registry.Registry // SupplyRegistryProxyFunc supplies registry proxy type SupplyRegistryProxyFunc func(ctx context.Context, tokenGenerator token.GeneratorFunc, dnsResolver dnsresolve.Resolver, options ...proxydns.Option) registry.Registry @@ -49,6 +54,9 @@ type SupplyRegistryProxyFunc func(ctx context.Context, tokenGenerator token.Gene // SetupNodeFunc setups each node on Builder.Build() stage type SetupNodeFunc func(ctx context.Context, node *Node, nodeNum int) +// SupplyDNSServerFunc creates dns handler for dns server +type SupplyDNSServerFunc func(context.Context) dnsutils.Handler + // RegistryEntry is pair of registry.Registry and url.URL type RegistryEntry struct { URL *url.URL @@ -57,6 +65,15 @@ type RegistryEntry struct { registry.Registry } +// DNSServerEntry represents DNS server record +type DNSServerEntry struct { + URL *url.URL + IPRecords genericsync.Map[string, []net.IP] + SRVRecords genericsync.Map[string, []*net.TCPAddr] + *restartableServer + dnsutils.Handler +} + // NSMgrEntry is pair of nsmgr.Nsmgr and url.URL type NSMgrEntry struct { Name string @@ -79,9 +96,10 @@ type EndpointEntry struct { // Domain contains attached to domain nodes, registry type Domain struct { Nodes []*Node - NSMgrProxy *NSMgrEntry + NSMgrProxy *EndpointEntry Registry *RegistryEntry RegistryProxy *RegistryEntry + DNSServer *DNSServerEntry DNSResolver dnsresolve.Resolver Name string