From 5453b703b25685ae771cb5744be82f8f1306d038 Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Wed, 27 Sep 2017 11:13:19 -0600 Subject: [PATCH] re-enable the in sandbox portmapping with exec in sandbox make it more flexible, update without modify hyperstart protocol Signed-off-by: Wang Xu --- daemon/pod/pod.go | 2 + daemon/pod/portmappings.go | 53 ++++++++- daemon/pod/provision.go | 36 +++--- examples/port-mapping-in-sandbox.pod | 22 ++++ .../portmapping/host_portmapping_linux.go | 11 +- .../portmapping/portmapping_interfaces.go | 44 +++++++ networking/portmapping/sandbox_portmapping.go | 107 ++++++++++++++++++ 7 files changed, 249 insertions(+), 26 deletions(-) create mode 100644 examples/port-mapping-in-sandbox.pod create mode 100644 networking/portmapping/portmapping_interfaces.go create mode 100644 networking/portmapping/sandbox_portmapping.go diff --git a/daemon/pod/pod.go b/daemon/pod/pod.go index b8a14708..2eb08aa8 100644 --- a/daemon/pod/pod.go +++ b/daemon/pod/pod.go @@ -63,6 +63,8 @@ type XPod struct { labels map[string]string resourceLock *sync.Mutex + prestartExecs [][]string + sandbox *hypervisor.Vm factory *PodFactory diff --git a/daemon/pod/portmappings.go b/daemon/pod/portmappings.go index 61c4c1ab..c17b9bad 100644 --- a/daemon/pod/portmappings.go +++ b/daemon/pod/portmappings.go @@ -31,11 +31,20 @@ func (p *XPod) initPortMapping() error { hlog.Log(ERROR, err) return err } - err = portmapping.SetupPortMaps(p.containerIP, pms) + var extPrefix []string + if p.globalSpec.PortmappingWhiteLists != nil && + len(p.globalSpec.PortmappingWhiteLists.InternalNetworks) > 0 && + len(p.globalSpec.PortmappingWhiteLists.ExternalNetworks) > 0 { + extPrefix = p.globalSpec.PortmappingWhiteLists.ExternalNetworks + } + preExec, err := portmapping.SetupPortMaps(p.containerIP, extPrefix, pms) if err != nil { p.Log(ERROR, "failed to setup port mappings: %v", err) return err } + if len(preExec) > 0 { + p.prestartExecs = append(p.prestartExecs, preExec...) + } } return nil } @@ -47,7 +56,7 @@ func (p *XPod) flushPortMapping() error { hlog.Log(ERROR, err) return err } - err = portmapping.ReleasePortMaps(p.containerIP, pms) + _, err = portmapping.ReleasePortMaps(p.containerIP, nil, pms) if err != nil { p.Log(ERROR, "release port mappings failed: %v", err) return err @@ -75,11 +84,29 @@ func (p *XPod) AddPortMapping(spec []*apitypes.PortMapping) error { p.Log(ERROR, "failed to generate port mapping rules: %v", err) return err } - err = portmapping.SetupPortMaps(p.containerIP, pms) + var extPrefix []string + if p.globalSpec.PortmappingWhiteLists != nil && + len(p.globalSpec.PortmappingWhiteLists.InternalNetworks) > 0 && + len(p.globalSpec.PortmappingWhiteLists.ExternalNetworks) > 0 { + extPrefix = p.globalSpec.PortmappingWhiteLists.ExternalNetworks + } + preExec, err := portmapping.SetupPortMaps(p.containerIP, extPrefix, pms) if err != nil { p.Log(ERROR, "failed to apply port mapping rules: %v", err) return err } + if len(preExec) > 0 { + p.prestartExecs = append(p.prestartExecs, preExec...) + if p.sandbox != nil { + for _, ex := range preExec { + _, stderr, err := p.sandbox.HyperstartExecSync(ex, nil) + if err != nil { + p.Log(ERROR, "failed to setup inSandbox mapping: %v [ %s", err, string(stderr)) + return err + } + } + } + } all := make([]*apitypes.PortMapping, len(p.portMappings)+len(spec)) copy(all, spec) @@ -134,11 +161,29 @@ func (p *XPod) removePortMapping(tbr []*apitypes.PortMapping, eq portMappingComp return err } - err = portmapping.ReleasePortMaps(p.containerIP, act) + var extPrefix []string + if p.globalSpec.PortmappingWhiteLists != nil && + len(p.globalSpec.PortmappingWhiteLists.InternalNetworks) > 0 && + len(p.globalSpec.PortmappingWhiteLists.ExternalNetworks) > 0 { + extPrefix = p.globalSpec.PortmappingWhiteLists.ExternalNetworks + } + postExec, err := portmapping.ReleasePortMaps(p.containerIP, extPrefix, act) if err != nil { p.Log(ERROR, "failed to clean up rules: %v", err) return err } + if len(postExec) > 0 { + // don't need to release prestartExec here, it is not persistent + if p.sandbox != nil { + for _, ex := range postExec { + _, stderr, err := p.sandbox.HyperstartExecSync(ex, nil) + if err != nil { + p.Log(ERROR, "failed to setup inSandbox mapping: %v [ %s", err, string(stderr)) + return err + } + } + } + } p.portMappings = other err = p.savePortMapping() diff --git a/daemon/pod/provision.go b/daemon/pod/provision.go index c71310a7..57867035 100644 --- a/daemon/pod/provision.go +++ b/daemon/pod/provision.go @@ -86,19 +86,20 @@ func newXPod(factory *PodFactory, spec *apitypes.UserPod) (*XPod, error) { factory.hosts = HostsCreator(spec.Id) factory.logCreator = initLogCreator(factory, spec) p := &XPod{ - name: spec.Id, - logPrefix: fmt.Sprintf("Pod[%s] ", spec.Id), - globalSpec: spec.CloneGlobalPart(), - containers: make(map[string]*Container), - volumes: make(map[string]*Volume), - interfaces: make(map[string]*Interface), - portMappings: spec.Portmappings, - labels: spec.Labels, - execs: make(map[string]*Exec), - resourceLock: &sync.Mutex{}, - statusLock: &sync.RWMutex{}, - stoppedChan: make(chan bool, 1), - factory: factory, + name: spec.Id, + logPrefix: fmt.Sprintf("Pod[%s] ", spec.Id), + globalSpec: spec.CloneGlobalPart(), + containers: make(map[string]*Container), + volumes: make(map[string]*Volume), + interfaces: make(map[string]*Interface), + portMappings: spec.Portmappings, + labels: spec.Labels, + prestartExecs: [][]string{}, + execs: make(map[string]*Exec), + resourceLock: &sync.Mutex{}, + statusLock: &sync.RWMutex{}, + stoppedChan: make(chan bool, 1), + factory: factory, } p.initCond = sync.NewCond(p.statusLock.RLocker()) return p, nil @@ -484,6 +485,15 @@ func (p *XPod) startAll() error { p.Log(INFO, "start all containers") future := utils.NewFutureSet() + for _, pre := range p.prestartExecs { + p.Log(DEBUG, "run prestart exec %v", pre) + _, stderr, err := p.sandbox.HyperstartExecSync(pre, nil) + if err != nil { + p.Log(ERROR, "failed to execute prestart command %v: %v [ %s", pre, err, string(stderr)) + return err + } + } + for ic, c := range p.containers { future.Add(ic, c.start) } diff --git a/examples/port-mapping-in-sandbox.pod b/examples/port-mapping-in-sandbox.pod new file mode 100644 index 00000000..60a1e3d2 --- /dev/null +++ b/examples/port-mapping-in-sandbox.pod @@ -0,0 +1,22 @@ +{ + "id": "port-mapping-test", + "containers" : [{ + "name": "pmtest", + "image": "busybox:latest", + "command": ["/bin/nc", "-l", "-p", "1300"] + }], + "resource": { + "vcpu": 1, + "memory": 128 + }, + "portmappingWhiteLists":{ + "internalNetworks": ["127.0.0.1/32", "192.168.123.0/24"], + "externalNetworks": ["0.0.0.0/0"] + }, + "portmappings": [{ + "containerport": "1300-1310", + "hostport": "3000-3010", + "protocol": "tcp" + }] +} + diff --git a/networking/portmapping/host_portmapping_linux.go b/networking/portmapping/host_portmapping_linux.go index 9101efb1..53394510 100644 --- a/networking/portmapping/host_portmapping_linux.go +++ b/networking/portmapping/host_portmapping_linux.go @@ -130,11 +130,7 @@ func parseRawResultOnHyper(output []byte, err error) error { return nil } -func SetupPortMaps(containerip string, maps []*PortMapping) error { - if disableIptables || len(maps) == 0 { - return nil - } - +func setupIptablesPortMaps(containerip string, maps []*PortMapping) error { var ( revert bool revertRules = [][]string{} @@ -209,10 +205,7 @@ func SetupPortMaps(containerip string, maps []*PortMapping) error { return nil } -func ReleasePortMaps(containerip string, maps []*PortMapping) error { - if disableIptables || len(maps) == 0 { - return nil - } +func releaseIptablesPortMaps(containerip string, maps []*PortMapping) error { release_loop: for _, m := range maps { diff --git a/networking/portmapping/portmapping_interfaces.go b/networking/portmapping/portmapping_interfaces.go new file mode 100644 index 00000000..ceeb24d8 --- /dev/null +++ b/networking/portmapping/portmapping_interfaces.go @@ -0,0 +1,44 @@ +package portmapping + +func SetupPortMaps(containerip string, externalPrefix []string, maps []*PortMapping) (preExec [][]string, err error) { + if len(maps) == 0 { + return [][]string{}, nil + } + if len(externalPrefix) > 0 { + preExec, err = setupInSandboxMappings(externalPrefix, maps) + if err != nil { + return [][]string{}, err + } + defer func() { + if err != nil { + preExec, _ = releaseInSandboxMappings(externalPrefix, maps) + } + }() + } + if !disableIptables { + err = setupIptablesPortMaps(containerip, maps) + if err != nil { + return [][]string{}, err + } + } + return preExec, nil +} + +func ReleasePortMaps(containerip string, externalPrefix []string, maps []*PortMapping) (postExec [][]string, err error) { + if len(maps) == 0 { + return [][]string{}, nil + } + if len(externalPrefix) > 0 { + postExec, err = releaseInSandboxMappings(externalPrefix, maps) + if err != nil { + return [][]string{}, err + } + } + if !disableIptables { + err = releaseIptablesPortMaps(containerip, maps) + if err != nil { + return [][]string{}, err + } + } + return postExec, nil +} diff --git a/networking/portmapping/sandbox_portmapping.go b/networking/portmapping/sandbox_portmapping.go new file mode 100644 index 00000000..af58d9de --- /dev/null +++ b/networking/portmapping/sandbox_portmapping.go @@ -0,0 +1,107 @@ +package portmapping + +import ( + "fmt" + "strconv" + "strings" +) + +/* Inside hyperstart, there are iptables init job should be done before configure in sandbox rules. + + // iptables -t filter -N hyperstart-INPUT + // iptables -t nat -N hyperstart-PREROUTING + // iptables -t filter -I INPUT -j hyperstart-INPUT + // iptables -t nat -I PREROUTING -j hyperstart-PREROUTING + // iptables -t filter -A hyperstart-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + // iptables -t filter -A hyperstart-INPUT -p icmp -j ACCEPT + // iptables -t filter -A hyperstart-INPUT -i lo -j ACCEPT + // iptables -t filter -A hyperstart-INPUT -j DROP + // iptables -t nat -A hyperstart-PREROUTING -j RETURN + // sh -c "echo 10485760 > /proc/sys/net/nf_conntrack_max" + // sh -c "echo 300 > /proc/sys/net/netfilter/nf_conntrack_tcp_timeout_established" + + // lan + // iptables -t filter -I hyperstart-INPUT -s %s -j ACCEPT + + These job has been done by hyperstart during initSandbox. +*/ + +func generateRedirectArgs(prefix string, m *PortMapping, insert bool) ([]string, []string, error) { + // outside + //iptables -t nat -I hyperstart-PREROUTING -s %s -p %s -m %s --dport %d -j REDIRECT --to-ports %d" + //iptables -t filter -I hyperstart-INPUT -s %s -p %s -m %s --dport %d -j ACCEPT + var ( + action = "-I" + proto string + dest string + to string + ) + + if !insert { + action = "-D" + } + + if strings.EqualFold(m.Protocol, "udp") { + proto = "udp" + } else { + proto = "tcp" + } + + if m.FromPorts.End == 0 || m.FromPorts.End == m.FromPorts.Begin { + dest = strconv.Itoa(m.FromPorts.Begin) + m.FromPorts.End = m.FromPorts.Begin + } else if m.FromPorts.End > m.FromPorts.Begin { + dest = fmt.Sprintf("%d:%d", m.FromPorts.Begin, m.FromPorts.End) + } else { + return []string{}, []string{}, fmt.Errorf("invalid from port range %d-%d", m.FromPorts.Begin, m.FromPorts.End) + } + + if m.ToPorts.End == 0 || m.ToPorts.End == m.ToPorts.Begin { + to = strconv.Itoa(m.ToPorts.Begin) + m.ToPorts.End = m.ToPorts.Begin + } else if m.ToPorts.End > m.ToPorts.Begin { + to = fmt.Sprintf("%d-%d", m.ToPorts.Begin, m.ToPorts.End) + } else { + return []string{}, []string{}, fmt.Errorf("invalid to port range %d-%d", m.ToPorts.Begin, m.ToPorts.End) + } + + //we may map ports 1:N or N:N, but not M:N (M!=1, M!=N) + hostRange := m.FromPorts.End - m.FromPorts.Begin + containerRange := m.ToPorts.End - m.ToPorts.Begin + if hostRange != 0 && hostRange != containerRange { + return []string{}, []string{}, fmt.Errorf("range mismatch, cannot map ports %s to %s", dest, to) + } + + filterArgs := []string{"iptables", "-t", "filter", action, "hyperstart-INPUT", "-s", prefix, "-p", proto, "-m", proto, "--dport", dest, "-j", "ACCEPT"} + redirectArgs := []string{"iptables", "-t", "nat", action, "hyperstart-PREROUTING", "-s", prefix, "-p", proto, "-m", proto, "--dport", dest, "-j", "REDIRECT", "--to-port", to} + + return redirectArgs, filterArgs, nil +} + +func setupInSandboxMappings(extPrefix []string, maps []*PortMapping) ([][]string, error) { + res := [][]string{} + for _, prefix := range extPrefix { + for _, m := range maps { + redirect, filter, err := generateRedirectArgs(prefix, m, true) + if err != nil { + return nil, err + } + res = append(res, redirect, filter) + } + } + return res, nil +} + +func releaseInSandboxMappings(extPrefix []string, maps []*PortMapping) ([][]string, error) { + res := [][]string{} + for _, prefix := range extPrefix { + for _, m := range maps { + redirect, filter, err := generateRedirectArgs(prefix, m, false) + if err != nil { + return nil, err + } + res = append(res, redirect, filter) + } + } + return res, nil +}