From 7b761145087103639d7f90ee20169934f1bf6f7f Mon Sep 17 00:00:00 2001 From: Kevin Parsons Date: Mon, 12 Feb 2024 23:31:13 -0800 Subject: [PATCH] network restore working --- internal/guest/runtime/hcsv2/uvm.go | 2 +- internal/hcsoci/hcsdoc_lcow.go | 3 +- internal/uvm/network.go | 89 +++++++++++------------------ internal/uvm/saverestore.go | 58 +++++-------------- 4 files changed, 50 insertions(+), 102 deletions(-) diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index bd8cdce929..c792075f7f 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -1166,7 +1166,7 @@ func modifyNetwork(ctx context.Context, rt guestrequest.RequestType, na *guestre // container or not so it must always call `Sync`. return ns.Sync(ctx) case guestrequest.RequestTypeRemove: - ns := GetOrAddNetworkNamespace(na.ID) + ns := GetOrAddNetworkNamespace(na.NamespaceID) if err := ns.RemoveAdapter(ctx, na.ID); err != nil { return err } diff --git a/internal/hcsoci/hcsdoc_lcow.go b/internal/hcsoci/hcsdoc_lcow.go index db94d73df0..3be2a5cb1c 100644 --- a/internal/hcsoci/hcsdoc_lcow.go +++ b/internal/hcsoci/hcsdoc_lcow.go @@ -11,6 +11,7 @@ import ( "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/schemaversion" + "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -62,7 +63,7 @@ func setWindowsNetworkNamespace(coi *createOptionsInternal, spec *specs.Spec) { spec.Windows = &specs.Windows{} } spec.Windows.Network = &specs.WindowsNetwork{ - NetworkNamespace: coi.Spec.Windows.Network.NetworkNamespace, + NetworkNamespace: uvm.GuestNamespaceID.String(), } } } diff --git a/internal/uvm/network.go b/internal/uvm/network.go index 4b2790e134..940a4a4e47 100644 --- a/internal/uvm/network.go +++ b/internal/uvm/network.go @@ -36,11 +36,17 @@ var ( ErrNICNotFound = errors.New("NIC not found in network namespace") ) +var ( + // baaa389b-bfd2-4500-b972-000000000000 + // base guid, chosen arbitrarily + GuestNamespaceID = guid.GUID{Data1: 0xbaaa389b, Data2: 0xbfd2, Data3: 0x4500, Data4: [8]byte{0xb9, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}} +) + // In this function we take the namespace ID of the namespace that was created for this // UVM. We hot add the namespace. We get the endpoints associated with this namespace // and then hot add those endpoints. -func (uvm *UtilityVM) SetupNetworkNamespace(ctx context.Context, nsid string) error { - nsidInsideUVM := nsid +func (uvm *UtilityVM) SetupNetworkNamespace(ctx context.Context, nsid string, doGuestRequest bool) error { + nsidInsideUVM := GuestNamespaceID.String() // Query endpoints with actual nsid endpoints, err := GetNamespaceEndpoints(ctx, nsid) @@ -55,11 +61,11 @@ func (uvm *UtilityVM) SetupNetworkNamespace(ctx context.Context, nsid string) er return err } - if err = uvm.AddNetNS(ctx, hcnNamespace); err != nil { + if err = uvm.AddNetNS(ctx, hcnNamespace, nsidInsideUVM); err != nil { return err } - if err = uvm.AddEndpointsToNS(ctx, nsidInsideUVM, endpoints); err != nil { + if err = uvm.AddEndpointsToNS(ctx, nsidInsideUVM, endpoints, doGuestRequest); err != nil { // Best effort clean up the NS if removeErr := uvm.RemoveNetNS(ctx, nsidInsideUVM); removeErr != nil { log.G(ctx).Warn(removeErr) @@ -184,7 +190,7 @@ func NewInternalNetworkSetup(vm *UtilityVM) NetworkSetup { func (i *internalNetworkSetup) ConfigureNetworking(ctx context.Context, namespaceID string, configType NetworkConfigType) error { switch configType { case NetworkRequestSetup: - if err := i.vm.SetupNetworkNamespace(ctx, namespaceID); err != nil { + if err := i.vm.SetupNetworkNamespace(ctx, namespaceID, true); err != nil { return err } case NetworkRequestTearDown: @@ -305,10 +311,10 @@ func (endpoints *NetworkEndpoints) Release(ctx context.Context) error { // struct returned by the GetNamespaceByID. For most uses cases AddNetNSByID is more appropriate. // // If a namespace with the same id already exists this returns `ErrNetNSAlreadyAttached`. -func (uvm *UtilityVM) AddNetNS(ctx context.Context, hcnNamespace *hcn.HostComputeNamespace) error { +func (uvm *UtilityVM) AddNetNS(ctx context.Context, hcnNamespace *hcn.HostComputeNamespace, guestNSID string) error { uvm.m.Lock() defer uvm.m.Unlock() - if _, ok := uvm.namespaces[hcnNamespace.Id]; ok { + if _, ok := uvm.namespaces[guestNSID]; ok { return ErrNetNSAlreadyAttached } @@ -332,7 +338,7 @@ func (uvm *UtilityVM) AddNetNS(ctx context.Context, hcnNamespace *hcn.HostComput if uvm.namespaces == nil { uvm.namespaces = make(map[string]*namespaceInfo) } - uvm.namespaces[hcnNamespace.Id] = &namespaceInfo{ + uvm.namespaces[guestNSID] = &namespaceInfo{ nics: make(map[string]*nicInfo), } return nil @@ -348,7 +354,7 @@ func (uvm *UtilityVM) AddNetNSByID(ctx context.Context, id string) error { return err } - if err = uvm.AddNetNS(ctx, hcnNamespace); err != nil { + if err = uvm.AddNetNS(ctx, hcnNamespace, GuestNamespaceID.String()); err != nil { return err } return nil @@ -373,7 +379,7 @@ func (uvm *UtilityVM) AddEndpointToNSWithID(ctx context.Context, nsID, nicID str } nicID = id.String() } - if err := uvm.addNIC(ctx, nicID, endpoint); err != nil { + if err := uvm.addNIC(ctx, nicID, endpoint, nsID, true); err != nil { return err } ns.nics[endpoint.Id] = &nicInfo{ @@ -389,7 +395,7 @@ func (uvm *UtilityVM) AddEndpointToNSWithID(ctx context.Context, nsID, nicID str // added endpoints. // // If no network namespace matches `id` returns `ErrNetNSNotFound`. -func (uvm *UtilityVM) AddEndpointsToNS(ctx context.Context, id string, endpoints []*hns.HNSEndpoint) error { +func (uvm *UtilityVM) AddEndpointsToNS(ctx context.Context, id string, endpoints []*hns.HNSEndpoint, doGuestRequest bool) error { uvm.m.Lock() defer uvm.m.Unlock() @@ -404,7 +410,7 @@ func (uvm *UtilityVM) AddEndpointsToNS(ctx context.Context, id string, endpoints if err != nil { return err } - if err := uvm.addNIC(ctx, nicID.String(), endpoint); err != nil { + if err := uvm.addNIC(ctx, nicID.String(), endpoint, id, doGuestRequest); err != nil { return err } ns.nics[endpoint.Id] = &nicInfo{ @@ -421,11 +427,12 @@ func (uvm *UtilityVM) AddEndpointsToNS(ctx context.Context, id string, endpoints // // If a namespace matching `id` is not found this command silently succeeds. func (uvm *UtilityVM) RemoveNetNS(ctx context.Context, id string) error { + id = GuestNamespaceID.String() uvm.m.Lock() defer uvm.m.Unlock() if ns, ok := uvm.namespaces[id]; ok { for _, ninfo := range ns.nics { - if err := uvm.removeNIC(ctx, ninfo.ID, ninfo.Endpoint); err != nil { + if err := uvm.removeNIC(ctx, ninfo.ID, ninfo.Endpoint, id); err != nil { return err } ns.nics[ninfo.Endpoint.Id] = nil @@ -454,31 +461,6 @@ func (uvm *UtilityVM) RemoveNetNS(ctx context.Context, id string) error { return nil } -// RemoveEndpointsFromNS removes all matching `endpoints` in the network -// namespace matching `id`. If no endpoint matching `endpoint.Id` is found in -// the network namespace this command silently succeeds. -// -// If no network namespace matches `id` returns `ErrNetNSNotFound`. -func (uvm *UtilityVM) RemoveEndpointsFromNS(ctx context.Context, id string, endpoints []*hns.HNSEndpoint) error { - uvm.m.Lock() - defer uvm.m.Unlock() - - ns, ok := uvm.namespaces[id] - if !ok { - return ErrNetNSNotFound - } - - for _, endpoint := range endpoints { - if ninfo, ok := ns.nics[endpoint.Id]; ok && ninfo != nil { - if err := uvm.removeNIC(ctx, ninfo.ID, ninfo.Endpoint); err != nil { - return err - } - delete(ns.nics, endpoint.Id) - } - } - return nil -} - // RemoveEndpointFromNS removes `endpoint` in the network // namespace matching `id`. If no endpoint matching `endpoint.Id` is found in // the network namespace this command returns `ErrNICNotFound`. @@ -494,7 +476,7 @@ func (uvm *UtilityVM) RemoveEndpointFromNS(ctx context.Context, id string, endpo } if ninfo, ok := ns.nics[endpoint.Id]; ok && ninfo != nil { - if err := uvm.removeNIC(ctx, ninfo.ID, ninfo.Endpoint); err != nil { + if err := uvm.removeNIC(ctx, ninfo.ID, ninfo.Endpoint, id); err != nil { return err } delete(ns.nics, endpoint.Id) @@ -525,7 +507,7 @@ func getNetworkModifyRequest(adapterID string, requestType guestrequest.RequestT } // addNIC adds a nic to the Utility VM. -func (uvm *UtilityVM) addNIC(ctx context.Context, id string, endpoint *hns.HNSEndpoint) error { +func (uvm *UtilityVM) addNIC(ctx context.Context, id string, endpoint *hns.HNSEndpoint, guestNSID string, doGuestRequest bool) error { // First a pre-add. This is a guest-only request and is only done on Windows. if uvm.operatingSystem == "windows" { preAddRequest := hcsschema.ModifySettingRequest{ @@ -565,7 +547,7 @@ func (uvm *UtilityVM) addNIC(ctx context.Context, id string, endpoint *hns.HNSEn } else { // Verify this version of LCOW supports Network HotAdd s := &guestresource.LCOWNetworkAdapter{ - NamespaceID: endpoint.Namespace.ID, + NamespaceID: guestNSID, ID: id, MacAddress: endpoint.MacAddress, IPAddress: endpoint.IPAddress.String(), @@ -588,7 +570,7 @@ func (uvm *UtilityVM) addNIC(ctx context.Context, id string, endpoint *hns.HNSEn "gateway": s.IPv6GatewayAddress, }).Debug("adding IPv6 settings") } - if uvm.isNetworkNamespaceSupported() { + if uvm.isNetworkNamespaceSupported() && doGuestRequest { request.GuestRequest = guestrequest.ModificationRequest{ ResourceType: guestresource.ResourceTypeNetwork, RequestType: guestrequest.RequestTypeAdd, @@ -598,13 +580,18 @@ func (uvm *UtilityVM) addNIC(ctx context.Context, id string, endpoint *hns.HNSEn } if err := uvm.modify(ctx, &request); err != nil { + // log.G(ctx).WithError(err).Error("modify failed, retrying") + // time.Sleep(5 * time.Second) + // if err := uvm.modify(ctx, &hcsschema.ModifySettingRequest{GuestRequest: request.GuestRequest}); err != nil { + // return fmt.Errorf("retried add nic modify: %w", err) + // } return err } return nil } -func (uvm *UtilityVM) removeNIC(ctx context.Context, id string, endpoint *hns.HNSEndpoint) error { +func (uvm *UtilityVM) removeNIC(ctx context.Context, id string, endpoint *hns.HNSEndpoint, guestNSID string) error { request := hcsschema.ModifySettingRequest{ RequestType: guestrequest.RequestTypeRemove, ResourcePath: fmt.Sprintf(resourcepaths.NetworkResourceFormat, id), @@ -629,8 +616,8 @@ func (uvm *UtilityVM) removeNIC(ctx context.Context, id string, endpoint *hns.HN ResourceType: guestresource.ResourceTypeNetwork, RequestType: guestrequest.RequestTypeRemove, Settings: &guestresource.LCOWNetworkAdapter{ - NamespaceID: endpoint.Namespace.ID, - ID: endpoint.Id, + NamespaceID: guestNSID, + ID: id, }, } } @@ -642,18 +629,6 @@ func (uvm *UtilityVM) removeNIC(ctx context.Context, id string, endpoint *hns.HN return nil } -// Removes all NICs added to this uvm. -func (uvm *UtilityVM) RemoveAllNICs(ctx context.Context) error { - for _, ns := range uvm.namespaces { - for _, ninfo := range ns.nics { - if err := uvm.removeNIC(ctx, ninfo.ID, ninfo.Endpoint); err != nil { - return err - } - } - } - return nil -} - // UpdateNIC updates a UVM's network adapter. func (uvm *UtilityVM) UpdateNIC(ctx context.Context, id string, settings *hcsschema.NetworkAdapter) error { req := &hcsschema.ModifySettingRequest{ diff --git a/internal/uvm/saverestore.go b/internal/uvm/saverestore.go index c8a6ee1f97..c6bd6a3466 100644 --- a/internal/uvm/saverestore.go +++ b/internal/uvm/saverestore.go @@ -7,8 +7,8 @@ import ( "os" "path/filepath" "regexp" + "strings" - "github.com/Microsoft/hcsshim/hcn" "github.com/Microsoft/hcsshim/internal/cow" "github.com/Microsoft/hcsshim/internal/gcs" "github.com/Microsoft/hcsshim/internal/hcs" @@ -124,6 +124,9 @@ func updateConfig(config *hcsschema.ComputeSystem, changeAny any) error { "change": string(j2), }).Info("UPDATED CONFIG") case guestrequest.RequestTypeRemove: + if strings.HasPrefix(change.ResourcePath, "VirtualMachine/Devices/NetworkAdapters/") { + return nil + } return fmt.Errorf("unrecognized update path: %s with payload type %T", change.ResourcePath, change.Settings) default: return fmt.Errorf("unrecognized request type: %s", change.RequestType) @@ -148,6 +151,11 @@ func (uvm *UtilityVM) StartSave(ctx context.Context, path string) error { if err := os.MkdirAll(path, 0755); err != nil { return err } + for ns := range uvm.namespaces { + if err := uvm.RemoveNetNS(ctx, ns); err != nil { + return fmt.Errorf("remove netns %s: %w", ns, err) + } + } if err := uvm.hcsSystem.Pause(ctx); err != nil { return err } @@ -176,30 +184,6 @@ func (uvm *UtilityVM) StartSave(ctx context.Context, path string) error { return err } - // resources := make(map[string]Resource) - // for controllerID, attachments := range uvm.config.VirtualMachine.Devices.Scsi { - // for lun, att := range attachments.Attachments { - // if att.ReadOnly { - // continue - // } - // r := Resource{ - // SCSIDisk: &SCSIDisk{ - // Controller: controllerID, - // LUN: lun, - // Path: att.Path, - // }, - // } - // g, err := guid.NewV4() - // if err != nil { - // return err - // } - // resources[g.String()] = r - // } - // } - // if err := statepkg.Write(filepath.Join(path, "resources.json"), &resources); err != nil { - // return err - // } - return nil } @@ -267,13 +251,6 @@ func RestoreUVM(ctx context.Context, path string, netNS string, id string, edits } // NICs - hcnNamespace, err := hcn.GetNamespaceByID(netNS) - if err != nil { - return nil, err - } - uvm.namespaces = map[string]*namespaceInfo{ - hcnNamespace.Id: {make(map[string]*nicInfo)}, - } endpoints, err := GetNamespaceEndpoints(ctx, netNS) if err != nil { return nil, err @@ -284,17 +261,8 @@ func RestoreUVM(ctx context.Context, path string, netNS string, id string, edits if len(endpoints) != 1 { return nil, fmt.Errorf("can only support one endpoint right now") } - e := endpoints[0] - for k := range config.VirtualMachine.Devices.NetworkAdapters { - config.VirtualMachine.Devices.NetworkAdapters[k] = hcsschema.NetworkAdapter{ - EndpointId: e.Id, - MacAddress: e.MacAddress, - } - uvm.namespaces[hcnNamespace.Id].nics[k] = &nicInfo{ - ID: k, - Endpoint: e, - } - } + // e := endpoints[0] + config.VirtualMachine.Devices.NetworkAdapters = nil for _, e := range edits { att := config.VirtualMachine.Devices.Scsi[e.Controller].Attachments[e.LUN] @@ -337,6 +305,10 @@ func RestoreUVM(ctx context.Context, path string, netNS string, id string, edits return nil, err } + if err := uvm.SetupNetworkNamespace(ctx, netNS, true); err != nil { + return nil, fmt.Errorf("add netns %s as %s: %w", netNS, GuestNamespaceID, err) + } + return uvm, nil }