From b3ae710b6a62e78c998adac2d3a07d2761c693be Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Tue, 15 Aug 2017 00:21:47 +0800 Subject: [PATCH 01/14] move iptables related code from runv to hyperd runv does not setup port mapping, which is a hyperd daemon function. Let's move it back to make things easier. Signed-off-by: Wang Xu --- daemon/daemon.go | 12 +- .../portmapping/host_portmapping_linux.go | 162 +++++++++++++++ .../portmapping/iptables/iptables_linux.go | 194 ++++++++++++++++++ .../portmapping/portmapper/portmapper.go | 79 +++++++ networking/portmapping/setup_linux.go | 147 +++++++++++++ 5 files changed, 593 insertions(+), 1 deletion(-) create mode 100644 networking/portmapping/host_portmapping_linux.go create mode 100644 networking/portmapping/iptables/iptables_linux.go create mode 100644 networking/portmapping/portmapper/portmapper.go create mode 100644 networking/portmapping/setup_linux.go diff --git a/daemon/daemon.go b/daemon/daemon.go index a0b27167..322a3d56 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -18,10 +18,12 @@ import ( "github.com/docker/docker/registry" dockerutils "github.com/docker/docker/utils" "github.com/golang/glog" + "github.com/hyperhq/hyperd/networking/portmapping" "github.com/hyperhq/hyperd/utils" "github.com/hyperhq/runv/driverloader" "github.com/hyperhq/runv/factory" "github.com/hyperhq/runv/hypervisor" + "github.com/hyperhq/runv/hypervisor/network" ) var ( @@ -224,10 +226,18 @@ func (daemon *Daemon) initRunV(c *apitypes.HyperConfig) error { } func (daemon *Daemon) initNetworks(c *apitypes.HyperConfig) error { - if err := hypervisor.InitNetwork(c.Bridge, c.BridgeIP, c.DisableIptables); err != nil { + if err := hypervisor.InitNetwork(c.Bridge, c.BridgeIP, true); err != nil { glog.Errorf("InitNetwork failed, %s", err.Error()) return err } + addr, err := network.GetIfaceAddr(network.BridgeIface) + if err != nil { + glog.Errorf("failed to get address of the configured bridge: %v", err) + return err + } + if err := portmapping.Setup(network.BridgeIface, addr, c.DisableIptables); err != nil { + glog.Errorf("Setup portmapping failed: %v", err) + } return nil } diff --git a/networking/portmapping/host_portmapping_linux.go b/networking/portmapping/host_portmapping_linux.go new file mode 100644 index 00000000..2f54eb86 --- /dev/null +++ b/networking/portmapping/host_portmapping_linux.go @@ -0,0 +1,162 @@ +package portmapping + +import ( + "fmt" + "net" + "strconv" + "strings" + + "github.com/hyperhq/hypercontainer-utils/hlog" + "github.com/hyperhq/hyperd/networking/portmapping/iptables" + "github.com/hyperhq/hyperd/networking/portmapping/portmapper" +) + +var ( + PortMapper = portmapper.New() +) + +type PortRange struct { + Begin int + End int +} + +type PortMapping struct { + Protocol string + ToPorts PortRange + FromPorts PortRange +} + +func generateIptablesArgs(containerip string, m *PortMapping) ([]string, []string, error) { + var ( + proto string + from string + to string + ) + + if strings.EqualFold(m.Protocol, "udp") { + proto = "udp" + } else { + proto = "tcp" + } + + if m.FromPorts.End == 0 || m.FromPorts.End == m.FromPorts.Begin { + from = strconv.Itoa(m.FromPorts.Begin) + m.FromPorts.End = m.FromPorts.Begin + } else if m.FromPorts.End > m.FromPorts.Begin { + from = fmt.Sprintf("%d:%d", m.FromPorts, m.FromPorts.End) + } else { + return []string{}, []string{}, fmt.Errorf("invalid from port range %d-%d", m.FromPorts.Begin, m.FromPorts.End) + } + + if m.ToPorts.End == 0 || m.ToPorts.End == m.ToPorts.Begin { + to = net.JoinHostPort(containerip, strconv.Itoa(m.ToPorts.Begin)) + m.ToPorts.End = m.ToPorts.Begin + } else if m.ToPorts.End > m.ToPorts.Begin { + to = net.JoinHostPort(containerip, fmt.Sprintf("%d-%d", m.ToPorts, m.ToPorts.End)) + } else { + return []string{}, []string{}, fmt.Errorf("invalid to port range %d-%d", m.ToPorts.Begin, m.ToPorts.End) + } + + //we may map ports 1:N or N:N, but not M:N (M!=1, M!=N) + hostRange := m.FromPorts.End - m.FromPorts.Begin + containerRange := m.ToPorts.End - m.ToPorts.Begin + if hostRange != 0 && hostRange != containerRange { + return []string{}, []string{}, fmt.Errorf("range mismatch, cannot map ports %s to %s", from, to) + } + + natArgs := []string{"-p", proto, "-m", proto, "--dport", from, "-j", "DNAT", "--to-destination", to} + filterArgs := []string{"-d", containerip, "-p", proto, "-m", proto, "--dport", to, "-j", "ACCEPT"} + + return natArgs, filterArgs, nil +} + +func SetupPortMaps(containerip string, maps []*PortMapping) error { + if disableIptables || len(maps) == 0 { + return nil + } + + for _, m := range maps { + + natArgs, filterArgs, err := generateIptablesArgs(containerip, m) + if err != nil { + return err + } + + //check if this rule has already existed + if iptables.PortMapExists("HYPER", natArgs) { + return nil + } + + if iptables.PortMapUsed("HYPER", m.Protocol, m.FromPorts.Begin, m.FromPorts.End) { + return fmt.Errorf("Host port %v has aleady been used", m.FromPorts) + } + + err = iptables.OperatePortMap(iptables.Insert, "HYPER", natArgs) + if err != nil { + return err + } + + if output, err := iptables.Raw(append([]string{"-I", "HYPER"}, filterArgs...)...); err != nil { + return fmt.Errorf("Unable to setup forward rule in HYPER chain: %s", err) + } else if len(output) != 0 { + return &iptables.ChainError{Chain: "HYPER", Output: output} + } + + i := m.FromPorts.Begin + j := m.ToPorts.Begin + defer func() { + if err != nil { + i-- + for i >= m.FromPorts.Begin { + PortMapper.ReleaseMap(m.Protocol, i) + i-- + } + } + }() + + for i <= m.FromPorts.End { + if err = PortMapper.AllocateMap(m.Protocol, i, containerip, j); err != nil { + return err + } + i++ + j++ + } + } + /* forbid to map ports twice */ + return nil +} + +func ReleasePortMaps(containerip string, maps []*PortMapping) error { + if disableIptables || len(maps) == 0 { + return nil + } + + for _, m := range maps { + if !strings.EqualFold(m.Protocol, "udp") { + m.Protocol = "tcp" + } + + if m.FromPorts.End == 0 { + m.FromPorts.End = m.FromPorts.Begin + } + + hlog.Log(hlog.DEBUG, "release port map %d-%d/%s", m.FromPorts.Begin, m.FromPorts.End, m.Protocol) + for i := m.FromPorts.Begin; i <= m.FromPorts.End; i++ { + err := PortMapper.ReleaseMap(m.Protocol, i) + if err != nil { + continue + } + } + + natArgs, filterArgs, err := generateIptablesArgs(containerip, m) + if err != nil { + continue + } + + iptables.OperatePortMap(iptables.Delete, "HYPER", natArgs) + + iptables.Raw(append([]string{"-D", "HYPER"}, filterArgs...)...) + } + /* forbid to map ports twice */ + return nil +} diff --git a/networking/portmapping/iptables/iptables_linux.go b/networking/portmapping/iptables/iptables_linux.go new file mode 100644 index 00000000..7297e9ac --- /dev/null +++ b/networking/portmapping/iptables/iptables_linux.go @@ -0,0 +1,194 @@ +package iptables + +import ( + "bytes" + "errors" + "fmt" + "io" + "os/exec" + "regexp" + "strconv" + "strings" + + "github.com/hyperhq/hypercontainer-utils/hlog" +) + +type Action string +type Table string + +const ( + Append Action = "-A" + Delete Action = "-D" + Insert Action = "-I" + Nat Table = "nat" + Filter Table = "filter" + Mangle Table = "mangle" +) + +var ( + iptablesPath string + supportsXlock = false + ErrIptablesNotFound = errors.New("Iptables not found") +) + +type Chain struct { + Name string + Bridge string + Table Table +} + +type ChainError struct { + Chain string + Output []byte +} + +func (e *ChainError) Error() string { + return fmt.Sprintf("Error iptables %s: %s", e.Chain, string(e.Output)) +} + +func initCheck() error { + if iptablesPath == "" { + path, err := exec.LookPath("iptables") + if err != nil { + return ErrIptablesNotFound + } + iptablesPath = path + supportsXlock = exec.Command(iptablesPath, "--wait", "-L", "-n").Run() == nil + } + return nil +} + +// Check if a dnat rule exists +func OperatePortMap(action Action, chain string, rule []string) error { + if output, err := Raw(append([]string{ + "-t", string(Nat), string(action), chain}, rule...)...); err != nil { + return fmt.Errorf("Unable to setup network port map: %s", err) + } else if len(output) != 0 { + return &ChainError{Chain: chain, Output: output} + } + + return nil +} + +func PortMapExists(chain string, rule []string) bool { + // iptables -C, --check option was added in v.1.4.11 + // http://ftp.netfilter.org/pub/iptables/changes-iptables-1.4.11.txt + + // try -C + // if exit status is 0 then return true, the rule exists + if _, err := Raw(append([]string{ + "-t", "nat", "-C", chain}, rule...)...); err == nil { + return true + } + + return false +} + +var hostPortRulePattern = regexp.MustCompile(`.* -p ([cdtpu]{3}) .* --dport ([0-9]{1,5})(:([0-9]{1,5}))?`) + +func PortMapUsed(chain string, proto string, begin, end int) bool { + // parse "iptables -S" for the rule (this checks rules in a specific chain + // in a specific table) + outputs, _ := exec.Command("iptables", "-t", "nat", "-S", chain).Output() + existingRules := bytes.NewBuffer(outputs) + var fin = false + for !fin { + rule, err := existingRules.ReadString(byte('\n')) + if err == io.EOF { + fin = true + } else if err != nil { + break + } + + match := hostPortRulePattern.FindStringSubmatch(rule) + if len(match) < 5 { + hlog.Log(hlog.TRACE, "pass short rule line: %v", rule) + continue + } + + p := match[1] + if p != proto { + continue + } + + if match[2] == "" { + continue + } + + pt, err := strconv.ParseInt(match[2], 10, 32) + if err != nil { + continue + } + p1 := int(pt) + + p2 := p1 + if match[4] != "" { + pt, err = strconv.ParseInt(match[4], 10, 32) + if err != nil { + continue + } + p2 = int(pt) + } + + if (begin >= p1 && begin <= p2) || (end >= p1 && end <= p2) || (begin < p1 && end > p2) { + return true + } + } + + return false +} + +// Check if a rule exists +func Exists(table Table, chain string, rule ...string) bool { + if string(table) == "" { + table = Filter + } + + // iptables -C, --check option was added in v.1.4.11 + // http://ftp.netfilter.org/pub/iptables/changes-iptables-1.4.11.txt + + // try -C + // if exit status is 0 then return true, the rule exists + if _, err := Raw(append([]string{ + "-t", string(table), "-C", chain}, rule...)...); err == nil { + return true + } + + // parse "iptables -S" for the rule (this checks rules in a specific chain + // in a specific table) + ruleString := strings.Join(rule, " ") + existingRules, _ := exec.Command("iptables", "-t", string(table), "-S", chain).Output() + + // regex to replace ips in rule + // because MASQUERADE rule will not be exactly what was passed + re := regexp.MustCompile(`[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/[0-9]{1,2}`) + + return strings.Contains( + re.ReplaceAllString(string(existingRules), "?"), + re.ReplaceAllString(ruleString, "?"), + ) +} + +// Call 'iptables' system command, passing supplied arguments +func Raw(args ...string) ([]byte, error) { + if err := initCheck(); err != nil { + return nil, err + } + if supportsXlock { + args = append([]string{"--wait"}, args...) + } + + hlog.Log(hlog.TRACE, "%s, %v", iptablesPath, args) + + output, err := exec.Command(iptablesPath, args...).CombinedOutput() + if err != nil { + return nil, fmt.Errorf("iptables failed: iptables %v: %s (%s)", strings.Join(args, " "), output, err) + } + + // ignore iptables' message about xtables lock + if strings.Contains(string(output), "waiting for it to exit") { + output = []byte("") + } + + return output, err +} diff --git a/networking/portmapping/portmapper/portmapper.go b/networking/portmapping/portmapper/portmapper.go new file mode 100644 index 00000000..2544ebe6 --- /dev/null +++ b/networking/portmapping/portmapper/portmapper.go @@ -0,0 +1,79 @@ +package portmapper + +import ( + "fmt" + "strings" + "sync" + + "github.com/golang/glog" +) + +type PortMap struct { + containerIP string + containerPort int +} + +func newPortMap(containerip string, containerport int) *PortMap { + return &PortMap{ + containerIP: containerip, + containerPort: containerport, + } +} + +type PortSet map[int]*PortMap + +type PortMapper struct { + tcpMap PortSet + udpMap PortSet + mutex sync.Mutex +} + +func New() *PortMapper { + return &PortMapper{PortSet{}, PortSet{}, sync.Mutex{}} +} + +func (p *PortMapper) AllocateMap(protocol string, hostPort int, + containerIP string, ContainerPort int) error { + p.mutex.Lock() + defer p.mutex.Unlock() + + var pset PortSet + + if strings.EqualFold(protocol, "udp") { + pset = p.udpMap + } else { + pset = p.tcpMap + } + + e, ok := pset[hostPort] + if ok { + return fmt.Errorf("Host port %d had already been used, %s %d", + hostPort, e.containerIP, e.containerPort) + } + + allocated := newPortMap(containerIP, ContainerPort) + pset[hostPort] = allocated + + return nil +} + +func (p *PortMapper) ReleaseMap(protocol string, hostPort int) error { + p.mutex.Lock() + defer p.mutex.Unlock() + + var pset PortSet + + if strings.EqualFold(protocol, "udp") { + pset = p.udpMap + } else { + pset = p.tcpMap + } + + _, ok := pset[hostPort] + if !ok { + glog.Errorf("Host port %d has not been used", hostPort) + } + + delete(pset, hostPort) + return nil +} diff --git a/networking/portmapping/setup_linux.go b/networking/portmapping/setup_linux.go new file mode 100644 index 00000000..e15d8d33 --- /dev/null +++ b/networking/portmapping/setup_linux.go @@ -0,0 +1,147 @@ +package portmapping + +import ( + "fmt" + "net" + "os" + "os/exec" + + "github.com/hyperhq/hypercontainer-utils/hlog" + "github.com/hyperhq/hyperd/networking/portmapping/iptables" +) + +var ( + disableIptables bool + bridgeIface string +) + +//setup environment for iptables and IP forwarding +func Setup(bIface string, addr net.Addr, disable bool) error { + var err error + + disableIptables = disable + bridgeIface = bIface + + if disableIptables { + hlog.Log(hlog.DEBUG, "Iptables is disabled") + return nil + } + + hlog.Log(hlog.TRACE, "setting up iptables") + err = setupIPTables(addr) + if err != nil { + hlog.Log(hlog.ERROR, "failed to setup iptables: %v", err) + return err + } + + return nil +} + +func setupIPTables(addr net.Addr) error { + if disableIptables { + return nil + } + + // Enable NAT + natArgs := []string{"-s", addr.String(), "!", "-o", bridgeIface, "-j", "MASQUERADE"} + + if !iptables.Exists(iptables.Nat, "POSTROUTING", natArgs...) { + if output, err := iptables.Raw(append([]string{ + "-t", string(iptables.Nat), "-I", "POSTROUTING"}, natArgs...)...); err != nil { + return fmt.Errorf("Unable to enable network bridge NAT: %s", err) + } else if len(output) != 0 { + return &iptables.ChainError{Chain: "POSTROUTING", Output: output} + } + } + + // Create HYPER iptables Chain + iptables.Raw("-N", "HYPER") + + // Goto HYPER chain + gotoArgs := []string{"-o", bridgeIface, "-j", "HYPER"} + if !iptables.Exists(iptables.Filter, "FORWARD", gotoArgs...) { + if output, err := iptables.Raw(append([]string{"-I", "FORWARD"}, gotoArgs...)...); err != nil { + return fmt.Errorf("Unable to setup goto HYPER rule %s", err) + } else if len(output) != 0 { + return &iptables.ChainError{Chain: "FORWARD goto HYPER", Output: output} + } + } + + // Accept all outgoing packets + outgoingArgs := []string{"-i", bridgeIface, "-j", "ACCEPT"} + if !iptables.Exists(iptables.Filter, "FORWARD", outgoingArgs...) { + if output, err := iptables.Raw(append([]string{"-I", "FORWARD"}, outgoingArgs...)...); err != nil { + return fmt.Errorf("Unable to allow outgoing packets: %s", err) + } else if len(output) != 0 { + return &iptables.ChainError{Chain: "FORWARD outgoing", Output: output} + } + } + + // Accept incoming packets for existing connections + existingArgs := []string{"-o", bridgeIface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"} + + if !iptables.Exists(iptables.Filter, "FORWARD", existingArgs...) { + if output, err := iptables.Raw(append([]string{"-I", "FORWARD"}, existingArgs...)...); err != nil { + return fmt.Errorf("Unable to allow incoming packets: %s", err) + } else if len(output) != 0 { + return &iptables.ChainError{Chain: "FORWARD incoming", Output: output} + } + } + + err := Modprobe("br_netfilter") + if err != nil { + hlog.Log(hlog.DEBUG, "modprobe br_netfilter failed %s", err) + } + + file, err := os.OpenFile("/proc/sys/net/bridge/bridge-nf-call-iptables", + os.O_RDWR, 0) + if err != nil { + return err + } + + _, err = file.WriteString("1") + if err != nil { + return err + } + + // Create HYPER iptables Chain + iptables.Raw("-t", string(iptables.Nat), "-N", "HYPER") + // Goto HYPER chain + gotoArgs = []string{"-m", "addrtype", "--dst-type", "LOCAL", "!", + "-d", "127.0.0.1/8", "-j", "HYPER"} + if !iptables.Exists(iptables.Nat, "OUTPUT", gotoArgs...) { + if output, err := iptables.Raw(append([]string{"-t", string(iptables.Nat), + "-I", "OUTPUT"}, gotoArgs...)...); err != nil { + return fmt.Errorf("Unable to setup goto HYPER rule %s", err) + } else if len(output) != 0 { + return &iptables.ChainError{Chain: "OUTPUT goto HYPER", Output: output} + } + } + + gotoArgs = []string{"-m", "addrtype", "--dst-type", "LOCAL", + "-j", "HYPER"} + if !iptables.Exists(iptables.Nat, "PREROUTING", gotoArgs...) { + if output, err := iptables.Raw(append([]string{"-t", string(iptables.Nat), + "-I", "PREROUTING"}, gotoArgs...)...); err != nil { + return fmt.Errorf("Unable to setup goto HYPER rule %s", err) + } else if len(output) != 0 { + return &iptables.ChainError{Chain: "PREROUTING goto HYPER", Output: output} + } + } + + return nil +} + +func Modprobe(module string) error { + modprobePath, err := exec.LookPath("modprobe") + if err != nil { + return fmt.Errorf("modprobe not found") + } + + _, err = exec.Command(modprobePath, module).CombinedOutput() + if err != nil { + return fmt.Errorf("modprobe %s failed", module) + } + + return nil +} From d32c2e794922807a72318a5317eb6f4f93864a1f Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Thu, 31 Aug 2017 23:44:19 +0800 Subject: [PATCH 02/14] test cases for port mapping spec manipulation Signed-off-by: Wang Xu --- types/spec_test.go | 622 +++++++++++++++++++++++++++++++++++++++++++++ types/utils.go | 95 ++++--- 2 files changed, 687 insertions(+), 30 deletions(-) create mode 100644 types/spec_test.go diff --git a/types/spec_test.go b/types/spec_test.go new file mode 100644 index 00000000..01e3be00 --- /dev/null +++ b/types/spec_test.go @@ -0,0 +1,622 @@ +package types + +import ( + "flag" + "testing" +) + +func init() { + flag.Set("alsologtostderr", "true") + flag.Set("v", "5") +} + +//PortMapping related tests +func TestPortRange(t *testing.T) { + var ( + res *_PortRange + err error + ) + + t.Log("> testing single port") + t.Log("> > testing read empty port range") + res, err = readPortRange("") + if err != nil { + t.Fatal("readPortRange cann't handle default range") + } + if res == nil || res.start != 1025 || res.end != 65535 { + t.Fatalf("readPortRange mistake handles default range, result: %#v", res) + } + + t.Log("> > testing read none number port") + res, err = readPortRange("whatever") + if err == nil { + t.Fatal("didn't found non number port") + } + t.Logf("----successfully found non number port: %v", err) + + t.Log("> > testing out of range port") + res, err = readPortRange("-1") + if err == nil { + t.Fatal("didn't found negative port") + } + t.Logf("----successfully found negative number: %v", err) + + res, err = readPortRange("65536") + if err == nil { + t.Fatal("didn't found too large port") + } + t.Logf("----successfully found too large number: %v", err) + + t.Log("> > testing correct port") + res, err = readPortRange("22") + if err != nil { + t.Fatalf("failed with correct port: %v", err) + } + if res == nil || res.start != 22 || res.end != 22 { + t.Fatalf("readPortRange mistake handles single port range, result: %#v", res) + } + + t.Log("> testing port range") + + t.Log("> > testing correct port range") + res, err = readPortRange("0-65535") + if err != nil { + t.Fatalf("failed with correct port range: %v", err) + } + if res == nil || res.start != 0 || res.end != 65535 { + t.Fatalf("readPortRange mistake handles normal port range, result: %#v", res) + } + + t.Log("> > testing correct port range (start==end)") + res, err = readPortRange("22-22") + if err != nil { + t.Fatalf("failed with correct port range (start==end): %v", err) + } + if res == nil || res.start != 22 || res.end != 22 { + t.Fatalf("readPortRange mistake normal port range (start==end), result: %#v", res) + } + + t.Log("> > testing inverse range port") + res, err = readPortRange("22-2") + if err == nil { + t.Fatal("didn't found inverse range port") + } + t.Logf("----successfully found inverse range port: %v", err) + + t.Log("> > testing negative end port") + res, err = readPortRange("22--22") + if err == nil { + t.Fatal("didn't found negative end port") + } + t.Logf("----successfully found negative end port: %v", err) + + t.Log("> > testing too large end port") + res, err = readPortRange("22-65536") + if err == nil { + t.Fatal("didn't found too large end port") + } + t.Logf("----successfully found too large end port: %v", err) +} + +func TestReadPortMapping(t *testing.T) { + var ( + res *_PortMapping + err error + ) + + t.Log("> testing normal one to one map") + res, err = readPortMapping(&PortMapping{ + HostPort: "80", + ContainerPort: "3000", + Protocol: "tcp", + }) + if err != nil { + t.Fatalf("failed to read single port mapping: %v", err) + } + if res == nil || res.host == nil || res.container == nil || + res.host.start != 80 || res.host.end != 80 || + res.container.start != 3000 || res.container.end != 3000 || + res.protocol != "tcp" { + t.Fatalf("mistaken read port mapping as h: %#v; c: %#v, p:%#v", res.host, res.container, res.protocol) + } + + t.Log("> testing normal multi to one map (and default proto tcp)") + res, err = readPortMapping(&PortMapping{ + HostPort: "80-88", + ContainerPort: "3000", + Protocol: "", + }) + if err != nil { + t.Fatalf("failed to read N-1 port mapping: %v", err) + } + if res == nil || res.host == nil || res.container == nil || + res.host.start != 80 || res.host.end != 88 || + res.container.start != 3000 || res.container.end != 3000 || + res.protocol != "tcp" { + t.Fatalf("mistaken read N:1 port mapping as h: %#v; c: %#v, p:%#v", res.host, res.container, res.protocol) + } + + t.Log("> testing normal multi to multi map") + res, err = readPortMapping(&PortMapping{ + HostPort: "80-88", + ContainerPort: "3000-3008", + Protocol: "tcp", + }) + if err != nil { + t.Fatalf("failed to read N:N port mapping: %v", err) + } + if res == nil || res.host == nil || res.container == nil || + res.host.start != 80 || res.host.end != 88 || + res.container.start != 3000 || res.container.end != 3008 || + res.protocol != "tcp" { + t.Fatalf("mistaken read port mapping as h: %#v; c: %#v, p:%#v", res.host, res.container, res.protocol) + } + + t.Log("> testing mismatch map") + res, err = readPortMapping(&PortMapping{ + HostPort: "80-88", + ContainerPort: "3000-3010", + Protocol: "tcp", + }) + if err == nil { + t.Fatal("not found N:M port mapping") + } + t.Logf("--found N:M port mapping: %v", err) + + t.Log("> testing 1:N map") + res, err = readPortMapping(&PortMapping{ + HostPort: "80", + ContainerPort: "3000-3010", + Protocol: "tcp", + }) + if err == nil { + t.Fatal("not found 1:N port mapping") + } + t.Logf("--found 1:N port mapping: %v", err) + + t.Log("> testing map wrong range") + res, err = readPortMapping(&PortMapping{ + HostPort: "80--88", + ContainerPort: "3000-3010", + Protocol: "tcp", + }) + if err == nil { + t.Fatal("not found host port range fault") + } + t.Logf("--found host port range fault: %v", err) + + t.Log("> testing map wrong range") + res, err = readPortMapping(&PortMapping{ + HostPort: "80-88", + ContainerPort: "3020-3010", + Protocol: "tcp", + }) + if err == nil { + t.Fatal("not found container port range fault") + } + t.Logf("--found container port range fault: %v", err) + + t.Log("> testing illegal protocol") + res, err = readPortMapping(&PortMapping{ + HostPort: "80", + ContainerPort: "3000", + Protocol: "sctp", + }) + if err == nil { + t.Fatal("not found illegal protocol") + } + t.Logf("--found illegal protocol: %v", err) +} + +func TestMergePorts(t *testing.T) { + var ( + pms []*_PortMapping + res []*_PortMapping + err error + ) + + t.Log("> testing empty port mapping list") + pms = []*_PortMapping{} + res, err = mergePorts(pms) + if err != nil { + t.Fatalf("failed with emport pm list: %v", err) + } + if len(res) != 0 { + t.Fatalf("non empty output on empty input %v", res) + } + + t.Log("> testing sparse merge") + pms = []*_PortMapping{ + { + host: &_PortRange{start: 8000, end: 8000}, + container: &_PortRange{start: 8000, end: 8000}, + }, + { + host: &_PortRange{start: 8010, end: 8010}, + container: &_PortRange{start: 8010, end: 8010}, + }, + { + host: &_PortRange{start: 8020, end: 8030}, + container: &_PortRange{start: 8020, end: 8030}, + }, + } + t.Logf("---[debug] input items %v", pms) + res, err = mergePorts(pms) + if err != nil { + t.Fatalf("error with normal pm list: %v", err) + } + if len(res) != 3 { + t.Fatalf("failed with normal pm list: %v", res) + } + t.Logf("---[debug] result items %v", res) + + t.Log("> testing connected merge") + pms = []*_PortMapping{ + { + host: &_PortRange{start: 8000, end: 8000}, + container: &_PortRange{start: 8000, end: 8000}, + }, + { + host: &_PortRange{start: 8001, end: 8001}, + container: &_PortRange{start: 8001, end: 8001}, + }, + { + host: &_PortRange{start: 8002, end: 8030}, + container: &_PortRange{start: 8002, end: 8030}, + }, + } + t.Logf("---[debug] input items %v", pms) + res, err = mergePorts(pms) + if err != nil { + t.Fatalf("error with normal pm list: %v", err) + } + if len(res) != 2 { + t.Fatalf("failed with normal pm list: %v", res) + } + t.Logf("---[debug] result items %v", res) + + pms = []*_PortMapping{ + { + host: &_PortRange{start: 8000, end: 8000}, + container: &_PortRange{start: 8000, end: 8000}, + }, + { + host: &_PortRange{start: 8001, end: 8001}, + container: &_PortRange{start: 8080, end: 8080}, + }, + { + host: &_PortRange{start: 8002, end: 8030}, + container: &_PortRange{start: 8002, end: 8030}, + }, + } + t.Logf("---[debug] input items %v", pms) + res, err = mergePorts(pms) + if err != nil { + t.Fatalf("error with normal pm list: %v", err) + } + if len(res) != 3 { + t.Fatalf("failed with normal pm list: %v", res) + } + t.Logf("---[debug] result items %v", res) + + t.Log("> testing overlap merge") + pms = []*_PortMapping{ + { + host: &_PortRange{start: 8000, end: 8000}, + container: &_PortRange{start: 8000, end: 8000}, + }, + { + host: &_PortRange{start: 8001, end: 8001}, + container: &_PortRange{start: 8001, end: 8001}, + }, + { + host: &_PortRange{start: 8001, end: 8030}, + container: &_PortRange{start: 8001, end: 8030}, + }, + } + t.Logf("---[debug] input items %v", pms) + res, err = mergePorts(pms) + if err == nil { + t.Fatalf("failed with overlapped pm list: %v", res) + } + t.Logf("---[debug] found overlapped: %v", err) + + pms = []*_PortMapping{ + { + host: &_PortRange{start: 8000, end: 8000}, + container: &_PortRange{start: 8000, end: 8000}, + }, + { + host: &_PortRange{start: 8000, end: 8000}, + container: &_PortRange{start: 8080, end: 8080}, + }, + { + host: &_PortRange{start: 8001, end: 8030}, + container: &_PortRange{start: 8001, end: 8030}, + }, + } + t.Logf("---[debug] input items %v", pms) + res, err = mergePorts(pms) + if err == nil { + t.Fatalf("failed with overlapped pm list: %v", res) + } + t.Logf("---[debug] found overlapped: %v", err) + + pms = []*_PortMapping{ + { + host: &_PortRange{start: 8080, end: 8080}, + container: &_PortRange{start: 8080, end: 8080}, + }, + { + host: &_PortRange{start: 8000, end: 8000}, + container: &_PortRange{start: 8000, end: 8000}, + }, + { + host: &_PortRange{start: 8001, end: 8090}, + container: &_PortRange{start: 8001, end: 8090}, + }, + } + t.Logf("---[debug] input items %v", pms) + res, err = mergePorts(pms) + if err == nil { + t.Fatalf("failed with overlapped pm list: %v", res) + } + t.Logf("---[debug] found overlapped: %v", err) + + t.Log("> testing random port") + pms = []*_PortMapping{ + { + host: &_PortRange{start: 8000, end: 8000}, + container: &_PortRange{start: 8000, end: 8000}, + }, + { + host: &_PortRange{start: 8001, end: 8001}, + container: &_PortRange{start: 8001, end: 8001}, + }, + { + host: &_PortRange{start: 8000, end: 8003}, + container: &_PortRange{start: 8002, end: 8002}, + }, + } + t.Logf("---[debug] input items %v", pms) + res, err = mergePorts(pms) + if err != nil { + t.Fatalf("error with random pm list: %v", err) + } + if len(res) > 2 { + t.Fatalf("failed with random pm list: %v", res) + } + t.Logf("---[debug] result items %v", res) + + pms = []*_PortMapping{ + { + host: &_PortRange{start: 8000, end: 8000}, + container: &_PortRange{start: 8000, end: 8000}, + }, + { + host: &_PortRange{start: 8001, end: 8001}, + container: &_PortRange{start: 8001, end: 8001}, + }, + { + host: &_PortRange{start: 8000, end: 8001}, + container: &_PortRange{start: 8001, end: 8001}, + }, + } + t.Logf("---[debug] input items %v", pms) + res, err = mergePorts(pms) + if err == nil { + t.Fatalf("didn't found collision in random pm list: %v", res) + } + t.Logf("found error with random pm list: %v", err) + + pms = []*_PortMapping{ + { + host: &_PortRange{start: 8000, end: 8000}, + container: &_PortRange{start: 8000, end: 8000}, + }, + { + host: &_PortRange{start: 8001, end: 8001}, + container: &_PortRange{start: 8001, end: 8001}, + }, + { + host: &_PortRange{start: 8000, end: 8003}, + container: &_PortRange{start: 8003, end: 8003}, + }, + { + host: &_PortRange{start: 8001, end: 8002}, + container: &_PortRange{start: 8002, end: 8002}, + }, + } + t.Logf("---[debug] input items %v", pms) + res, err = mergePorts(pms) + if err != nil { + t.Logf("got an error with random pm list: %v", err) + t.Log("it's acceptable to fail in this case") + } + if len(res) > 3 { + t.Fatalf("failed with random pm list: %v", res) + } + t.Logf("---[debug] result items %v", res) +} + +func TestContainerPortMigrate(t *testing.T) { + var ( + tp = &UserPod{} + err error + ) + + t.Log("> testing nil containers") + err = tp.migrateContainerPorts() + if err != nil { + t.Fatal("failed to migrate nil containers") + } + + t.Log("> testing normal migrate") + tp.Containers = []*UserContainer{ + { + Ports: []*UserContainerPort{ + { + HostPort: 80, + ContainerPort: 80, + }, + { + HostPort: 81, + ContainerPort: 81, + }, + }, + }, + } + err = tp.migrateContainerPorts() + if err != nil { + t.Fatalf("failed to migrate container ports: %v", err) + } + + t.Log("> testing Illegal proto") + tp.Containers = []*UserContainer{ + { + Ports: []*UserContainerPort{ + { + HostPort: 80, + ContainerPort: 80, + Protocol: "icmp", + }, + { + HostPort: 81, + ContainerPort: 81, + }, + }, + }, + } + err = tp.migrateContainerPorts() + if err == nil { + t.Fatalf("failed to find illegal container port protocol: %v", tp.Portmappings) + } + t.Logf("found illegal proto: %v", err) +} + +func TestMergePortMappings(t *testing.T) { + var ( + tp = &UserPod{} + err error + ) + + t.Log("> testing nil port mappings") + err = tp.MergePortmappings() + if err != nil { + t.Fatalf("failed with nil port mappings rules: %v", err) + } + + t.Log("> testing empty port mappings") + tp.Portmappings = []*PortMapping{} + err = tp.MergePortmappings() + if err != nil { + t.Fatalf("failed with empty port mappings rules: %v", err) + } + + t.Log("> testing normal trans") + tp.Portmappings = []*PortMapping{ + { + HostPort: "8000", + ContainerPort: "8000", + }, + { + HostPort: "8010", + ContainerPort: "8010", + }, + { + HostPort: "8020-8030", + ContainerPort: "8020-8030", + Protocol: "udp", + }, + } + err = tp.MergePortmappings() + if err != nil { + t.Fatalf("failed with normal port mappings rules: %v", err) + } + + t.Log("> testing fail container ports migrate") + tp.Portmappings = nil + tp.Containers = []*UserContainer{ + { + Ports: []*UserContainerPort{ + { + HostPort: 80, + ContainerPort: 80, + Protocol: "icmp", + }, + { + HostPort: 81, + ContainerPort: 81, + }, + }, + }, + } + err = tp.MergePortmappings() + if err == nil { + t.Fatalf("failed with failed container ports migrate rules: %v", tp.Portmappings) + } + + t.Log("> testing illegal proto") + tp.Portmappings = []*PortMapping{ + { + HostPort: "8000", + ContainerPort: "8000", + }, + { + HostPort: "8010", + ContainerPort: "8010", + }, + { + HostPort: "8020-8030", + ContainerPort: "8020-8030", + Protocol: "icmp", + }, + } + err = tp.MergePortmappings() + if err == nil { + t.Fatalf("failed with illegal proto: %#v", tp.Portmappings) + } + + t.Log("> testing tcp merge failed") + tp.Portmappings = []*PortMapping{ + { + HostPort: "8000", + ContainerPort: "8000", + }, + { + HostPort: "8000", + ContainerPort: "8010", + }, + { + HostPort: "8020-8030", + ContainerPort: "8020-8030", + Protocol: "udp", + }, + } + err = tp.MergePortmappings() + if err == nil { + t.Fatalf("failed with tcp port overlapped rules: %v", tp.Portmappings) + } + + t.Log("> testing udp merge failed") + tp.Portmappings = []*PortMapping{ + { + HostPort: "8000", + ContainerPort: "8000", + }, + { + HostPort: "8023", + ContainerPort: "8010", + Protocol: "udp", + }, + { + HostPort: "8020-8030", + ContainerPort: "8020-8030", + Protocol: "udp", + }, + } + err = tp.MergePortmappings() + if err == nil { + t.Fatalf("failed with udp port overlapped rules: %v", tp.Portmappings) + } +} diff --git a/types/utils.go b/types/utils.go index 51ef9eb4..2c46c6c0 100644 --- a/types/utils.go +++ b/types/utils.go @@ -2,10 +2,12 @@ package types import ( "fmt" - "github.com/hyperhq/hyperd/utils" "sort" "strconv" "strings" + + "github.com/hyperhq/hypercontainer-utils/hlog" + "github.com/hyperhq/hyperd/utils" ) func (p *UserPod) LookupContainer(idOrName string) *UserContainer { @@ -135,25 +137,25 @@ func readPortRange(p string) (*_PortRange, error) { return &_PortRange{1025, 65535}, nil } else if strings.Contains(p, "-") { parts := strings.SplitN(p, "-", 2) - start, err := strconv.Atoi(parts[0]) + start, err := strconv.ParseUint(parts[0], 10, 16) if err != nil { return nil, err } - end, err := strconv.Atoi(parts[1]) + end, err := strconv.ParseUint(parts[1], 10, 16) if err != nil { return nil, err } if end < start { return nil, fmt.Errorf("max %d is smaller than min %d", end, start) } - return &_PortRange{start, end}, nil + return &_PortRange{int(start), int(end)}, nil } - start, err := strconv.Atoi(p) + start, err := strconv.ParseUint(p, 10, 16) if err != nil { return nil, err } - return &_PortRange{start, start}, nil + return &_PortRange{int(start), int(start)}, nil } func (pr *_PortRange) isRange() bool { @@ -164,7 +166,7 @@ func (pr *_PortRange) count() int { return pr.end - pr.start + 1 } -func (pr *_PortRange) toString() string { +func (pr *_PortRange) String() string { if pr.isRange() { return fmt.Sprintf("%d-%d", pr.start, pr.end) } @@ -177,7 +179,17 @@ type _PortMapping struct { protocol string } +func (pm *_PortMapping) String() string { + return strings.Join([]string{pm.protocol, pm.host.String(), pm.container.String()}, ":") +} + func readPortMapping(pm *PortMapping) (*_PortMapping, error) { + proto := "tcp" + if pm.Protocol == "udp" { + proto = "udp" + } else if pm.Protocol != "tcp" && pm.Protocol != "" { + return nil, fmt.Errorf("unrecongnized protocol %s", pm.Protocol) + } h, err := readPortRange(pm.HostPort) if err != nil { return nil, err @@ -187,19 +199,19 @@ func readPortMapping(pm *PortMapping) (*_PortMapping, error) { return nil, err } if c.isRange() && c.count() != h.count() { - return nil, fmt.Errorf("port range mismatch: %d vs %d", h.toString(), c.toString()) + return nil, fmt.Errorf("port range mismatch: %d vs %d", h.String(), c.String()) } return &_PortMapping{ host: h, container: c, - protocol: pm.Protocol, + protocol: proto, }, nil } func (pm *_PortMapping) toSpec() *PortMapping { return &PortMapping{ - ContainerPort: pm.container.toString(), - HostPort: pm.host.toString(), + ContainerPort: pm.container.String(), + HostPort: pm.host.String(), Protocol: pm.protocol, } } @@ -244,6 +256,23 @@ func mergePorts(pms []*_PortMapping) ([]*_PortMapping, error) { tbm = append(tbm, pm.host.start) } + for _, pm := range remains { + for p := pm.host.start; p <= pm.host.end; p++ { + if occupy[p] { + continue + } + pm.host.start = p + pm.host.end = p + occupy[p] = true + singles[p] = pm + tbm = append(tbm, p) + break + } + if pm.notDetermined() { + return nil, fmt.Errorf("cannot allocate port for %s", pm.host.String()) + } + } + sort.Ints(tbm) var last *_PortMapping for _, p := range tbm { @@ -264,23 +293,26 @@ func mergePorts(pms []*_PortMapping) ([]*_PortMapping, error) { results = append(results, last) } - for _, pm := range remains { - for p := pm.host.start; p <= pm.host.end; p++ { - if occupy[p] { - continue - } - pm.host.start = p - pm.host.end = p - occupy[p] = true - results = append(results, pm) - break + return results, nil +} + +func (p *UserPod) migrateContainerPorts() error { + if p.Portmappings == nil { + p.Portmappings = []*PortMapping{} + } + for _, c := range p.Containers { + if len(c.Ports) == 0 { + continue } - if pm.notDetermined() { - return nil, fmt.Errorf("cannot allocate port for %s", pm.host.toString()) + pms, err := c.ToPodPortmappings(false) + if err != nil { + hlog.Log(hlog.ERROR, "failed to convert container port to pod scope: %v", err) + return err } + p.Portmappings = append(p.Portmappings, pms...) + c.Ports = nil } - - return results, nil + return nil } func (p *UserPod) MergePortmappings() error { @@ -290,26 +322,29 @@ func (p *UserPod) MergePortmappings() error { err error ) + if err = p.migrateContainerPorts(); err != nil { + return err + } + for _, pm := range p.Portmappings { port, err := readPortMapping(pm) if err != nil { return err } - if pm.Protocol == "tcp" { + if port.protocol == "tcp" { tcpPorts = append(tcpPorts, port) - } else if pm.Protocol == "udp" { + } else if port.protocol == "udp" { udpPorts = append(udpPorts, port) - } else { - err := fmt.Errorf("unrecognized protocol %s", pm.Protocol) - return err } } + hlog.Log(hlog.TRACE, "TCP ports to be merged: %v", tcpPorts) tcpPorts, err = mergePorts(tcpPorts) if err != nil { return err } + hlog.Log(hlog.TRACE, "UDP ports to be merged: %v", udpPorts) udpPorts, err = mergePorts(udpPorts) if err != nil { return err From 85dc61b646a783492a744cd3c8019702dee9d359 Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Wed, 6 Sep 2017 15:29:03 +0800 Subject: [PATCH 03/14] new ctor for portmapping rules Signed-off-by: Wang Xu --- .../portmapping/host_portmapping_linux.go | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/networking/portmapping/host_portmapping_linux.go b/networking/portmapping/host_portmapping_linux.go index 2f54eb86..39b1801c 100644 --- a/networking/portmapping/host_portmapping_linux.go +++ b/networking/portmapping/host_portmapping_linux.go @@ -22,8 +22,55 @@ type PortRange struct { type PortMapping struct { Protocol string - ToPorts PortRange - FromPorts PortRange + ToPorts *PortRange + FromPorts *PortRange +} + +// NewPortRange generate a port range from string r. the r should be a decimal number or +// in format begin-end, where begin and end are both decimal number. And the port range should +// be 0-65535, i.e. 16-bit unsigned int +// It returns PortRange pointer for valid input, otherwise return error +func NewPortRange(r string) (*PortRange, error) { + segs := strings.SplitN(r, "-", 2) + b, err := strconv.ParseUint(segs[0], 10, 16) + if err != nil { + return nil, err + } + e := b + if len(segs) > 1 { + e, err = strconv.ParseUint(segs[1], 10, 16) + if err != nil { + return nil, err + } + } + return &PortRange{ + Begin: int(b), + End: int(e), + } +} + +// NewPortMapping generate a PortMapping from three strings: proto (tcp or udp, default is tcp), +// and from/to port (single port or a range, see NewPortRange) +func NewPortMapping(proto, from, to string) (*PortMapping, error) { + if proto == "" { + proto = "tcp" + } + if proto != "tcp" && proto != "udp" { + return nil, fmt.Errorf("unsupported protocol %s", proto) + } + f, err := NewPortRange(from) + if err != nil { + return nil, err + } + t, err := NewPortRange(to) + if err != nil { + return nil, err + } + return &PortMapping{ + Protocol: proto, + ToPorts: t, + FromPorts: f, + }, nil } func generateIptablesArgs(containerip string, m *PortMapping) ([]string, []string, error) { From 9979a0a534061a2a5c048c91a7129550d126e8ca Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Wed, 6 Sep 2017 15:29:53 +0800 Subject: [PATCH 04/14] operations for portmappings Signed-off-by: Wang Xu --- daemon/pod/decommission.go | 8 ++ daemon/pod/pod.go | 1 + daemon/pod/portmappings.go | 152 +++++++++++++++++++++++++++++++++++++ daemon/pod/provision.go | 15 +++- types/utils.go | 18 +++++ 5 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 daemon/pod/portmappings.go diff --git a/daemon/pod/decommission.go b/daemon/pod/decommission.go index c3464bc3..7fb02081 100644 --- a/daemon/pod/decommission.go +++ b/daemon/pod/decommission.go @@ -549,11 +549,13 @@ func (p *XPod) cleanup() { if err != nil { // even if error, we set the vm to be stopped p.Log(ERROR, "pod stopping failed, failed to decommit the resources: %v", err) + err = nil } err = p.removeSandboxFromDB() if err != nil { p.Log(ERROR, "pod stopping failed, failed to remove sandbox persist data: %v", err) + err = nil } p.Log(DEBUG, "tag pod as stopped") @@ -573,6 +575,12 @@ func (p *XPod) cleanup() { func (p *XPod) decommissionResources() (err error) { p.Log(DEBUG, "umount all containers and volumes, release IP addresses") + err = p.flushPortMapping() + if err != nil { + p.Log(WARNING, "(ignored) flush port mappings failed: %v", err) + err = nil + } + for _, c := range p.containers { ec := c.umountRootVol() if ec != nil { diff --git a/daemon/pod/pod.go b/daemon/pod/pod.go index d676edfa..b8a14708 100644 --- a/daemon/pod/pod.go +++ b/daemon/pod/pod.go @@ -58,6 +58,7 @@ type XPod struct { volumes map[string]*Volume interfaces map[string]*Interface services *Services + containerIP string // only for doing portMapping portMappings []*apitypes.PortMapping labels map[string]string resourceLock *sync.Mutex diff --git a/daemon/pod/portmappings.go b/daemon/pod/portmappings.go new file mode 100644 index 00000000..52cc9bff --- /dev/null +++ b/daemon/pod/portmappings.go @@ -0,0 +1,152 @@ +package pod + +import ( + "fmt" + "github.com/hyperhq/hypercontainer-utils/hlog" + "github.com/hyperhq/hyperd/networking/portmapping" + apitypes "github.com/hyperhq/hyperd/types" +) + +func translatePortMapping(spec []*apitypes.PortMapping) ([]*portmapping.PortMapping, error) { + if len(spec) == 0 { + return []*portmapping.PortMapping{}, nil + } + var pms = make([]*portmapping.PortMapping, 0, len(spec)) + for _, entry := range spec { + pm, err := portmapping.NewPortMapping(entry.Protocol, entry.HostPort, entry.ContainerPort) + if err != nil { + hlog.Log(ERROR, "failed to parsing portmappings: %v", err) + return pms, err + } + hlog.Log(TRACE, "parsed portmapping rule %#v", entry) + pms = append(pms, pm) + } + return pms, nil +} + +func (p *XPod) initPortMapping() error { + if p.containerIP != "" && len(p.portMappings) > 0 { + pms, err := translatePortMapping(p.portMappings) + if err != nil { + hlog.Log(ERROR, err) + return nil, err + } + err = portmapping.SetupPortMaps(p.containerIP, pms) + if err != nil { + p.Log(ERROR, "failed to setup port mappings: %v", err) + return err + } + } + return nil +} + +func (p *XPod) flushPortMapping() error { + if p.containerIP != "" && len(p.portMappings) > 0 { + err := portmapping.ReleasePortMaps(p.containerIP, p.portMappings) + if err != nil { + p.Log(ERROR, "release port mappings failed: %v", err) + return err + } + } + return nil +} + +func (p *XPod) AddPortMapping(spec []*apitypes.PortMapping) error { + if !p.IsAlive() { + err := fmt.Errorf("portmapping could apply to running pod only (%v)", spec) + p.Log(ERROR, "port mapping failed: %v", err) + return err + } + p.resourceLock.Lock() + defer p.resourceLock.Unlock() + + if p.containerIP == "" || len(spec) == 0 { + p.Log(INFO, "Skip port maping setup [%v], container IP: %s", spec, p.containerIP) + return nil + } + + pms, err := translatePortMapping(spec) + if err != nil { + p.Log(ERROR, "failed to generate port mapping rules: %v", err) + return err + } + err = portmapping.SetupPortMaps(p.containerIP, pms) + if err != nil { + p.Log(ERROR, "failed to apply port mapping rules: %v", err) + return err + } + + all := make([]*apitypes.PortMapping, len(p.portMappings)+len(spec)) + copy(all, spec) + copy(all[len(spec):], p.portMappings) + p.portMappings = all + + return nil +} + +type portMappingCompare func(pm1, pm2 *apitypes.PortMapping) bool + +func (p *XPod) removePortMapping(tbr []*apitypes.PortMapping, eq portMappingCompare) error { + p.resourceLock.Lock() + defer p.resourceLock.Unlock() + + if p.containerIP == "" || len(p.portMappings) == 0 || len(tbr) == 0 { + return nil + } + + rm := make([]*apitypes.PortMapping, 0, len(p.portMappings)) + other := make([]*apitypes.PortMapping, 0, len(p.portMappings)) + + for _, pm := range p.portMappings { + selected := false + for _, sel := range tbr { + if eq(pm, sel) { + rm = append(rm, pm) + selected = true + break + } + } + if !selected { + other = append(other, pm) + } + } + + if len(rm) == 0 { + p.Log(DEBUG, "no portmapping to be removed by %v", tbr) + return nil + } + + act, err := translatePortMapping(rm) + if err != nil { + p.Log(ERROR, "failed to generate removing rules: %v", err) + return err + } + + err = portmapping.ReleasePortMaps(p.containerIP, act) + if err != nil { + p.Log(ERROR, "failed to clean up rules: %v", err) + return err + } + + return err +} + +func (p *XPod) RemovePortMappingByDest(spec []*apitypes.PortMapping) error { + return p.removePortMapping(spec, func(pm1, pm2 *apitypes.PortMapping) bool { + return pm1.SameDestWith(pm2) + }) +} + +func (p *XPod) RemovePortMappingStricted(spec []*apitypes.PortMapping) error { + return p.removePortMapping(spec, func(pm1, pm2 *apitypes.PortMapping) bool { + return pm1.EqualTo(pm2) + }) +} + +func (p *XPod) ListPortMappings() []*apitypes.PortMapping { + p.resourceLock.Lock() + res := make([]*apitypes.PortMapping, len(p.portMappings)) + copy(res, p.portMappings) + p.resourceLock.Unlock() + return res +} diff --git a/daemon/pod/provision.go b/daemon/pod/provision.go index 818591d4..40221bc1 100644 --- a/daemon/pod/provision.go +++ b/daemon/pod/provision.go @@ -8,6 +8,7 @@ import ( "github.com/hyperhq/hypercontainer-utils/hlog" "github.com/hyperhq/hyperd/errors" + "github.com/hyperhq/hyperd/networking/portmapping" apitypes "github.com/hyperhq/hyperd/types" "github.com/hyperhq/hyperd/utils" runv "github.com/hyperhq/runv/api" @@ -384,7 +385,6 @@ func (p *XPod) initResources(spec *apitypes.UserPod, allowCreate bool) error { } p.services = newServices(p, spec.Services) - p.portMappings = spec.Portmappings return nil } @@ -419,12 +419,23 @@ func (p *XPod) prepareResources() error { if err = inf.prepare(); err != nil { return err } + if p.containerIP == "" { + p.containerIP = inf.descript.Ip + } + } + + err = p.initPortMapping() + if err != nil { + p.Log(ERROR, "failed to initial setup port mappings: %v", err) + return err } + // if insert any other operations here, add rollback code for the + // port mapping operation return nil } -// addResourcesToSandbox() add resources to sandbox parallelly, it issues +// addResourcesToSandbox() add resources to sandbox in parallel, it issues // runV API parallelly to send the NIC, Vols, and Containers to sandbox func (p *XPod) addResourcesToSandbox() error { p.resourceLock.Lock() diff --git a/types/utils.go b/types/utils.go index 2c46c6c0..ddb9dc05 100644 --- a/types/utils.go +++ b/types/utils.go @@ -208,6 +208,24 @@ func readPortMapping(pm *PortMapping) (*_PortMapping, error) { }, nil } +func (pm *PortMapping) EqualTo(other *PortMapping) bool { + if other == nil && pm == nil { + return true + } else if other == nil || pm == nil { + return false + } + return pm.Protocol == other.Protocol && pm.ContainerPort == other.ContainerPort && pm.HostPort == other.ContainerPort +} + +func (pm *PortMapping) SameDestWith(other *PortMapping) bool { + if other == nil && pm == nil { + return true + } else if other == nil || pm == nil { + return false + } + return pm.Protocol == other.Protocol && pm.ContainerPort == other.ContainerPort +} + func (pm *_PortMapping) toSpec() *PortMapping { return &PortMapping{ ContainerPort: pm.container.String(), From c7d092c27401e11cebd03804f981a6f4d1f253c2 Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Wed, 6 Sep 2017 16:59:45 +0800 Subject: [PATCH 05/14] persist the portmapping info Signed-off-by: Wang Xu --- daemon/pod/persist.go | 49 +++++++++++++++++-- daemon/pod/portmappings.go | 15 ++++++ types/persist.pb.go | 98 ++++++++++++++++++++++++++------------ types/persist.proto | 8 +++- types/types.pb.go | 1 + 5 files changed, 135 insertions(+), 36 deletions(-) diff --git a/daemon/pod/persist.go b/daemon/pod/persist.go index a9e72fcb..c61c89f8 100644 --- a/daemon/pod/persist.go +++ b/daemon/pod/persist.go @@ -21,6 +21,9 @@ import ( /// PM-{Pod.Id()}: Pod level metadata that could be changed /// |`- services: service list /// `- labels +/// PP-{Pod.Id()}: Port Mapping rules +/// |`- containerIp: container IP for portmapping operations +/// `- portMappings: rules /// CX-{Container.Id()} Container Persistent Info /// VX-{Pod.ID()}-{Volume.Name()} Volume Persist Info /// IF-{Pod.ID()}-{Inf.Id()} @@ -30,7 +33,8 @@ const ( LAYOUT_KEY_FMT = "PL-%s" SB_KEY_FMT = "SB-%s" PS_KEY_FMT = "PS-%s" - PM_KEY_FMT = "PM-%s" + PMETA_KEY_FMT = "PM-%s" + PMAP_KEY_FMT = "PP-%s" CX_KEY_FMT = "CX-%s" VX_KEY_FMT = "VX-%s-%s" IF_KEY_FMT = "IF-%s-%s" @@ -131,6 +135,11 @@ func LoadXPod(factory *PodFactory, layout *types.PersistPodLayout) (*XPod, error return nil, err } + err = p.loadPortMapping() + if err != nil { + return nil, err + } + //associate containers if p.status == S_POD_RUNNING { for _, c := range p.containers { @@ -163,6 +172,10 @@ func (p *XPod) savePod() error { return err } + if err := p.savePortMapping(); err != nil { + return err + } + for cid, c := range p.containers { containers = append(containers, cid) if err := c.saveContainer(); err != nil { @@ -221,6 +234,10 @@ func (p *XPod) removeFromDB() (err error) { return err } + if err = p.removePortMappingFromDB(); err != nil { + return err + } + if err = p.removeSandboxFromDB(); err != nil { return err } @@ -286,12 +303,12 @@ func (p *XPod) savePodMeta() error { if p.info != nil { meta.CreatedAt = p.info.CreatedAt } - return saveMessage(p.factory.db, fmt.Sprintf(PM_KEY_FMT, p.Id()), meta, p, "pod meta") + return saveMessage(p.factory.db, fmt.Sprintf(PMETA_KEY_FMT, p.Id()), meta, p, "pod meta") } func (p *XPod) loadPodMeta() error { var meta types.PersistPodMeta - err := loadMessage(p.factory.db, fmt.Sprintf(PM_KEY_FMT, p.Id()), &meta, p, "pod meta") + err := loadMessage(p.factory.db, fmt.Sprintf(PMETA_KEY_FMT, p.Id()), &meta, p, "pod meta") if err != nil { return err } @@ -305,7 +322,31 @@ func (p *XPod) loadPodMeta() error { } func (p *XPod) removePodMetaFromDB() error { - return removeMessage(p.factory.db, fmt.Sprintf(PM_KEY_FMT, p.Id()), p, "pod meta") + return removeMessage(p.factory.db, fmt.Sprintf(PMETA_KEY_FMT, p.Id()), p, "pod meta") +} + +func (p *XPod) savePortMapping() error { + pm := &types.PersistPortmappings { + Pod: p.Id(), + ContainerIP: p.containerIP, + PortMappings: p.portMappings, + } + return saveMessage(p.factory.db, fmt.Sprintf(PMAP_KEY_FMT, p.Id()), pm, p, "port mappings") +} + +func (p *XPod) loadPortMapping() error { + var pm types.PersistPortmappings + err := loadMessage(p.factory.db, fmt.Sprintf(PMAP_KEY_FMT, p.Id()), &pm, p, "port mappings") + if err != nil { + return err + } + p.containerIP = pm.ContainerIP + p.portMappings = pm.PortMappings + return nil +} + +func (p *XPod) removePortMappingFromDB() error { + return removeMessage(p.factory.db, fmt.Sprintf(PMAP_KEY_FMT, p.Id()), p, "port mappings") } func (c *Container) saveContainer() error { diff --git a/daemon/pod/portmappings.go b/daemon/pod/portmappings.go index 52cc9bff..187d2e18 100644 --- a/daemon/pod/portmappings.go +++ b/daemon/pod/portmappings.go @@ -81,6 +81,13 @@ func (p *XPod) AddPortMapping(spec []*apitypes.PortMapping) error { copy(all[len(spec):], p.portMappings) p.portMappings = all + err = p.savePortMapping() + if err != nil { + p.Log(WARNING, "failed to persist new portmapping rules") + // ignore the error + err = nil + } + return nil } @@ -128,6 +135,14 @@ func (p *XPod) removePortMapping(tbr []*apitypes.PortMapping, eq portMappingComp return err } + p.portMappings = other + err = p.savePortMapping() + if err != nil { + p.Log(WARNING, "failed to persist removed portmapping rules") + // ignore the error + err = nil + } + return err } diff --git a/types/persist.pb.go b/types/persist.pb.go index ae1a3786..effde733 100644 --- a/types/persist.pb.go +++ b/types/persist.pb.go @@ -246,6 +246,38 @@ func (m *PersistInterface) GetDescript() *api.InterfaceDescription { return nil } +type PersistPortmappings struct { + Pod string `protobuf:"bytes,1,opt,name=pod,proto3" json:"pod,omitempty"` + ContainerIP string `protobuf:"bytes,2,opt,name=containerIP,proto3" json:"containerIP,omitempty"` + PortMappings []*PortMapping `protobuf:"bytes,11,rep,name=portMappings" json:"portMappings,omitempty"` +} + +func (m *PersistPortmappings) Reset() { *m = PersistPortmappings{} } +func (m *PersistPortmappings) String() string { return proto.CompactTextString(m) } +func (*PersistPortmappings) ProtoMessage() {} +func (*PersistPortmappings) Descriptor() ([]byte, []int) { return fileDescriptorPersist, []int{6} } + +func (m *PersistPortmappings) GetPod() string { + if m != nil { + return m.Pod + } + return "" +} + +func (m *PersistPortmappings) GetContainerIP() string { + if m != nil { + return m.ContainerIP + } + return "" +} + +func (m *PersistPortmappings) GetPortMappings() []*PortMapping { + if m != nil { + return m.PortMappings + } + return nil +} + func init() { proto.RegisterType((*PersistPodLayout)(nil), "types.PersistPodLayout") proto.RegisterType((*PersistPodMeta)(nil), "types.PersistPodMeta") @@ -253,40 +285,44 @@ func init() { proto.RegisterType((*PersistContainer)(nil), "types.PersistContainer") proto.RegisterType((*PersistVolume)(nil), "types.PersistVolume") proto.RegisterType((*PersistInterface)(nil), "types.PersistInterface") + proto.RegisterType((*PersistPortmappings)(nil), "types.PersistPortmappings") } func init() { proto.RegisterFile("persist.proto", fileDescriptorPersist) } var fileDescriptorPersist = []byte{ - // 467 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xa4, 0x93, 0xdf, 0x6a, 0xdb, 0x30, - 0x14, 0xc6, 0x71, 0xd2, 0x76, 0xed, 0x71, 0x5a, 0x32, 0xd1, 0x76, 0x5a, 0x18, 0xc3, 0x33, 0x0c, - 0x72, 0xe5, 0x40, 0xc6, 0x60, 0xdd, 0xdd, 0xd8, 0x1f, 0x28, 0x74, 0x50, 0x1c, 0xb6, 0x7b, 0xd9, - 0x3e, 0x6d, 0xc4, 0x1c, 0x4b, 0x93, 0xe4, 0x30, 0xbf, 0xc5, 0x6e, 0x76, 0xb7, 0xc7, 0xdb, 0x83, - 0x0c, 0x4b, 0xfe, 0x57, 0x12, 0xd8, 0xc5, 0xee, 0xa4, 0xef, 0x7c, 0xe7, 0xf3, 0x4f, 0x47, 0x32, - 0x9c, 0x4a, 0x54, 0x9a, 0x6b, 0x13, 0x49, 0x25, 0x8c, 0x20, 0x87, 0xa6, 0x92, 0xa8, 0x67, 0xd1, - 0x3d, 0x37, 0xeb, 0x32, 0x89, 0x52, 0xb1, 0x59, 0xac, 0x2b, 0x89, 0x6a, 0xfd, 0x7d, 0xa1, 0xca, - 0x62, 0xbb, 0x60, 0x92, 0x2f, 0x32, 0xd4, 0xa9, 0xe2, 0xd2, 0x70, 0x51, 0x68, 0xd7, 0x36, 0xf3, - 0x6d, 0x9b, 0xdb, 0x84, 0xbf, 0x3d, 0x98, 0xde, 0xba, 0xd4, 0x5b, 0x91, 0xdd, 0xb0, 0x4a, 0x94, - 0x86, 0x9c, 0xc1, 0x88, 0x67, 0xd4, 0x0b, 0xbc, 0xf9, 0x49, 0x3c, 0xe2, 0x19, 0x79, 0x0e, 0x70, - 0x9f, 0x8b, 0x84, 0xe5, 0x2b, 0x89, 0x29, 0xf5, 0xad, 0x3e, 0x50, 0xea, 0x7a, 0x2a, 0x0a, 0xc3, - 0x78, 0x81, 0x4a, 0xd3, 0x8b, 0x60, 0x5c, 0xd7, 0x7b, 0x85, 0x50, 0x78, 0xb4, 0x15, 0x79, 0xb9, - 0x41, 0x4d, 0x2f, 0x6d, 0xb1, 0xdd, 0xd6, 0x9d, 0xbc, 0x30, 0xa8, 0xee, 0x58, 0x8a, 0x9a, 0x3e, - 0x71, 0x9d, 0xbd, 0x12, 0xfe, 0xf1, 0xe0, 0xac, 0xc7, 0xfb, 0x8c, 0x86, 0xed, 0xc0, 0x45, 0x70, - 0xac, 0x51, 0x6d, 0x79, 0x1d, 0xe0, 0x07, 0xe3, 0xb9, 0xbf, 0x24, 0x91, 0x3b, 0xe1, 0x17, 0x8d, - 0x6a, 0xe5, 0x4a, 0x71, 0xe7, 0x21, 0x57, 0x70, 0x94, 0xb3, 0x04, 0x73, 0x4d, 0x27, 0xd6, 0xfd, - 0xa2, 0x71, 0x3f, 0xfc, 0x4c, 0x74, 0x63, 0x3d, 0x1f, 0x0b, 0xa3, 0xaa, 0xb8, 0x69, 0x20, 0xcf, - 0xe0, 0x24, 0x55, 0xc8, 0x0c, 0x66, 0xef, 0x0c, 0xbd, 0x08, 0xbc, 0xf9, 0x38, 0xee, 0x85, 0xd9, - 0x15, 0xf8, 0x83, 0x26, 0x32, 0x85, 0xf1, 0x37, 0xac, 0x1a, 0xd0, 0x7a, 0x49, 0xce, 0xe1, 0x70, - 0xcb, 0xf2, 0x12, 0xe9, 0xc8, 0x6a, 0x6e, 0xf3, 0x76, 0xf4, 0xc6, 0x0b, 0x3f, 0x01, 0x59, 0xb1, - 0x22, 0x4b, 0xc4, 0x8f, 0x86, 0xe2, 0xba, 0xb8, 0x13, 0x3b, 0x27, 0x0d, 0xc0, 0x1f, 0x94, 0x6d, - 0xca, 0x24, 0x1e, 0x4a, 0xe1, 0xaf, 0xfe, 0x36, 0xdf, 0xb7, 0xe3, 0xdf, 0x89, 0x99, 0xc2, 0x58, - 0x8a, 0xac, 0x81, 0xa8, 0x97, 0x64, 0x0e, 0x07, 0xba, 0xbd, 0x59, 0x7f, 0x79, 0x3e, 0x18, 0x5f, - 0x97, 0x12, 0x5b, 0x07, 0x79, 0x0d, 0xc7, 0xed, 0x8b, 0xa2, 0x13, 0xeb, 0x7e, 0x1a, 0x31, 0xc9, - 0xa3, 0xce, 0xf7, 0xa1, 0x7f, 0x6f, 0x71, 0x67, 0x0d, 0x7f, 0x7a, 0x70, 0xda, 0x70, 0x7d, 0xb5, - 0x37, 0x4f, 0x08, 0x1c, 0x14, 0x6c, 0x83, 0x0d, 0x96, 0x5d, 0xef, 0x01, 0x7b, 0xf9, 0x00, 0xec, - 0xf1, 0x00, 0xcc, 0xc5, 0x34, 0x54, 0xcb, 0x1d, 0xaa, 0x4b, 0x4b, 0xe5, 0x4c, 0xfb, 0x91, 0x06, - 0xa3, 0xba, 0x6e, 0xdf, 0xdb, 0x7f, 0x8d, 0xaa, 0x4b, 0xf9, 0xc7, 0xa8, 0x3a, 0xdf, 0x5e, 0xae, - 0xe4, 0xc8, 0xfe, 0x97, 0xaf, 0xfe, 0x06, 0x00, 0x00, 0xff, 0xff, 0x92, 0x40, 0xb8, 0x94, 0xec, - 0x03, 0x00, 0x00, + // 514 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xa4, 0x94, 0x5d, 0x6f, 0xd3, 0x3e, + 0x14, 0xc6, 0x95, 0x76, 0xdb, 0x7f, 0x3b, 0xe9, 0xa6, 0xfe, 0xcd, 0x36, 0x4c, 0x85, 0x50, 0xa8, + 0x84, 0xd4, 0xab, 0x54, 0x2a, 0x02, 0x31, 0xee, 0x10, 0x2f, 0x52, 0xa5, 0x4d, 0xaa, 0x52, 0xc1, + 0xbd, 0x93, 0x78, 0xad, 0x45, 0x6a, 0x1b, 0xdb, 0xa9, 0xc8, 0x25, 0xdf, 0x80, 0x1b, 0xee, 0xf8, + 0x78, 0x7c, 0x10, 0x14, 0xc7, 0x79, 0x29, 0xad, 0xc4, 0x05, 0x77, 0xf6, 0x73, 0x9e, 0xf3, 0xf4, + 0x97, 0x63, 0xbb, 0x70, 0x2e, 0xa9, 0xd2, 0x4c, 0x9b, 0x50, 0x2a, 0x61, 0x04, 0x3a, 0x36, 0x85, + 0xa4, 0x7a, 0x14, 0xae, 0x98, 0x59, 0xe7, 0x71, 0x98, 0x88, 0xcd, 0x74, 0x5d, 0x48, 0xaa, 0xd6, + 0x5f, 0xa6, 0x2a, 0xe7, 0xdb, 0x29, 0x91, 0x6c, 0x9a, 0x52, 0x9d, 0x28, 0x26, 0x0d, 0x13, 0x5c, + 0x57, 0x6d, 0x23, 0xdf, 0xb6, 0x55, 0x9b, 0xf1, 0x4f, 0x0f, 0x86, 0x8b, 0x2a, 0x75, 0x21, 0xd2, + 0x5b, 0x52, 0x88, 0xdc, 0xa0, 0x0b, 0xe8, 0xb1, 0x14, 0x7b, 0x81, 0x37, 0x39, 0x8b, 0x7a, 0x2c, + 0x45, 0x4f, 0x00, 0x56, 0x99, 0x88, 0x49, 0xb6, 0x94, 0x34, 0xc1, 0xbe, 0xd5, 0x3b, 0x4a, 0x59, + 0x4f, 0x04, 0x37, 0x84, 0x71, 0xaa, 0x34, 0xbe, 0x0a, 0xfa, 0x65, 0xbd, 0x55, 0x10, 0x86, 0xff, + 0xb6, 0x22, 0xcb, 0x37, 0x54, 0xe3, 0x6b, 0x5b, 0xac, 0xb7, 0x65, 0x27, 0xe3, 0x86, 0xaa, 0x7b, + 0x92, 0x50, 0x8d, 0x1f, 0x56, 0x9d, 0xad, 0x32, 0xfe, 0xe5, 0xc1, 0x45, 0x8b, 0x77, 0x47, 0x0d, + 0xd9, 0x83, 0x0b, 0xe1, 0x54, 0x53, 0xb5, 0x65, 0x65, 0x80, 0x1f, 0xf4, 0x27, 0xfe, 0x0c, 0x85, + 0xd5, 0x17, 0x7e, 0xd4, 0x54, 0x2d, 0xab, 0x52, 0xd4, 0x78, 0xd0, 0x0d, 0x9c, 0x64, 0x24, 0xa6, + 0x99, 0xc6, 0x03, 0xeb, 0x7e, 0xea, 0xdc, 0xbb, 0x3f, 0x13, 0xde, 0x5a, 0xcf, 0x7b, 0x6e, 0x54, + 0x11, 0xb9, 0x06, 0xf4, 0x18, 0xce, 0x12, 0x45, 0x89, 0xa1, 0xe9, 0x1b, 0x83, 0xaf, 0x02, 0x6f, + 0xd2, 0x8f, 0x5a, 0x61, 0x74, 0x03, 0x7e, 0xa7, 0x09, 0x0d, 0xa1, 0xff, 0x99, 0x16, 0x0e, 0xb4, + 0x5c, 0xa2, 0x4b, 0x38, 0xde, 0x92, 0x2c, 0xa7, 0xb8, 0x67, 0xb5, 0x6a, 0xf3, 0xba, 0xf7, 0xca, + 0x1b, 0x7f, 0x00, 0xb4, 0x24, 0x3c, 0x8d, 0xc5, 0x57, 0x47, 0x31, 0xe7, 0xf7, 0x62, 0xef, 0x4b, + 0x03, 0xf0, 0x3b, 0x65, 0x9b, 0x32, 0x88, 0xba, 0xd2, 0xf8, 0x47, 0x7b, 0x9a, 0x6f, 0xeb, 0xf1, + 0xef, 0xc5, 0x0c, 0xa1, 0x2f, 0x45, 0xea, 0x20, 0xca, 0x25, 0x9a, 0xc0, 0x91, 0xae, 0x4f, 0xd6, + 0x9f, 0x5d, 0x76, 0xc6, 0xd7, 0xa4, 0x44, 0xd6, 0x81, 0x5e, 0xc0, 0x69, 0x7d, 0xa3, 0xf0, 0xc0, + 0xba, 0x1f, 0x85, 0x44, 0xb2, 0xb0, 0xf1, 0xbd, 0x6b, 0xef, 0x5b, 0xd4, 0x58, 0xc7, 0xdf, 0x3d, + 0x38, 0x77, 0x5c, 0x9f, 0xec, 0xc9, 0x23, 0x04, 0x47, 0x9c, 0x6c, 0xa8, 0xc3, 0xb2, 0xeb, 0x03, + 0x60, 0xcf, 0x76, 0xc0, 0xfe, 0xef, 0x80, 0x55, 0x31, 0x8e, 0x6a, 0xb6, 0x47, 0x75, 0x6d, 0xa9, + 0x2a, 0xd3, 0x61, 0xa4, 0xce, 0xa8, 0xe6, 0xf5, 0x7d, 0xfb, 0xa7, 0x51, 0x35, 0x29, 0x7f, 0x19, + 0x55, 0xe3, 0x3b, 0xcc, 0xf5, 0xcd, 0x83, 0x07, 0xcd, 0x55, 0x54, 0x66, 0x43, 0xa4, 0x64, 0x7c, + 0xa5, 0x6b, 0x14, 0xaf, 0x45, 0x09, 0xc0, 0x6f, 0xde, 0xd8, 0x7c, 0xe1, 0x20, 0xbb, 0x12, 0x7a, + 0x09, 0x03, 0x29, 0x94, 0xb9, 0x73, 0x19, 0x7f, 0x3c, 0x8f, 0x45, 0x5b, 0x8a, 0x76, 0x7c, 0xf1, + 0x89, 0xfd, 0x6f, 0x78, 0xfe, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x65, 0x89, 0x38, 0x67, 0x70, 0x04, + 0x00, 0x00, } diff --git a/types/persist.proto b/types/persist.proto index f7b93a7f..fbeb33cb 100644 --- a/types/persist.proto +++ b/types/persist.proto @@ -46,4 +46,10 @@ message PersistInterface { string pod = 2; UserInterface spec = 11; api.InterfaceDescription descript = 12; -} \ No newline at end of file +} + +message PersistPortmappings { + string pod =1 ; + string containerIP = 2; + repeated PortMapping portMappings = 11; +} diff --git a/types/types.pb.go b/types/types.pb.go index fa149bc4..72fbe779 100644 --- a/types/types.pb.go +++ b/types/types.pb.go @@ -147,6 +147,7 @@ It has these top-level messages: PersistContainer PersistVolume PersistInterface + PersistPortmappings */ package types From 886a916a867a321843ff97512b47dc2bcc55d5ec Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Wed, 6 Sep 2017 17:56:16 +0800 Subject: [PATCH 06/14] add gRPC API for portMappings Signed-off-by: Wang Xu --- serverrpc/portmapping.go | 61 +++ serverrpc/server.go | 17 +- types/types.pb.go | 878 +++++++++++++++++++++++---------------- types/types.proto | 22 + 4 files changed, 627 insertions(+), 351 deletions(-) create mode 100644 serverrpc/portmapping.go diff --git a/serverrpc/portmapping.go b/serverrpc/portmapping.go new file mode 100644 index 00000000..00ae883a --- /dev/null +++ b/serverrpc/portmapping.go @@ -0,0 +1,61 @@ +package serverrpc + +import ( + "errors" + + "github.com/hyperhq/hypercontainer-utils/hlog" + "github.com/hyperhq/hyperd/types" + "golang.org/x/net/context" +) + +// PortMappingList get a list of PortMappings +func (s *ServerRPC) PortMappingList(ctx context.Context, req *types.PortMappingListRequest) (*types.PortMappingListResponse, error) { + s.Log(hlog.TRACE, "PortMappingList with request %s", req.String()) + + p, ok := s.daemon.PodList.Get(req.PodID) + if !ok { + s.Log(hlog.INFO, "PortMappingList: pod %s not found", req.PodID) + return nil, errors.New("Pod not found") + } + + return &types.PortMappingListResponse{ + PortMappings: p.ListPortMappings(), + }, nil +} + +// PortMappingAdd add a list of PortMapping rules to a Pod +func (s *ServerRPC) PortMappingAdd(ctx context.Context, req *types.PortMappingModifyRequest) (*types.PortMappingModifyResponse, error) { + s.Log(hlog.TRACE, "PortMappingAdd with request %s", req.String()) + + p, ok := s.daemon.PodList.Get(req.PodID) + if !ok { + s.Log(hlog.INFO, "PortMappingAdd: pod %s not found", req.PodID) + return nil, errors.New("Pod not found") + } + + err := p.AddPortMapping(req.PortMappings) + if err != nil { + s.Log(hlog.ERROR, "failed to add port mappings: %v", err) + return nil, err + } + return &types.PortMappingModifyResponse{}, nil +} + +// PortMappingDel remove a list of PortMapping rules from a Pod +func (s *ServerRPC) PortMappingDel(ctx context.Context, req *types.PortMappingModifyRequest) (*types.PortMappingModifyResponse, error) { + s.Log(hlog.TRACE, "PortMappingDel with request %s", req.String()) + + p, ok := s.daemon.PodList.Get(req.PodID) + if !ok { + s.Log(hlog.INFO, "PortMappingDel: pod %s not found", req.PodID) + return nil, errors.New("Pod not found") + } + + err := p.RemovePortMappingByDest(req.PortMappings) + if err != nil { + s.Log(hlog.ERROR, "failed to add port mappings: %v", err) + return nil, err + } + return &types.PortMappingModifyResponse{}, nil +} + diff --git a/serverrpc/server.go b/serverrpc/server.go index 3507d877..50d2f90e 100644 --- a/serverrpc/server.go +++ b/serverrpc/server.go @@ -3,7 +3,7 @@ package serverrpc import ( "net" - "github.com/golang/glog" + "github.com/hyperhq/hypercontainer-utils/hlog" "github.com/hyperhq/hyperd/daemon" "github.com/hyperhq/hyperd/types" "google.golang.org/grpc" @@ -25,16 +25,27 @@ func NewServerRPC(d *daemon.Daemon) *ServerRPC { return s } +// LogPrefix() belongs to the interface `github.com/hyperhq/hypercontainer-utils/hlog.LogOwner`, which helps `hlog.HLog` get +// proper prefix from the owner object. +func (s *ServerRPC) LogPrefix() string { + return "[gRPC] " +} + +// Log() employ `github.com/hyperhq/hypercontainer-utils/hlog.HLog` to add pod information to the log +func (s *ServerRPC) Log(level hlog.LogLevel, args ...interface{}) { + hlog.HLog(level, s, 1, args...) +} + func (s *ServerRPC) registerServer() { types.RegisterPublicAPIServer(s.server, s) } // Serve serves gRPC request by goroutines func (s *ServerRPC) Serve(addr string) error { - glog.V(1).Infof("Start gRPC server at %s", addr) + s.Log(hlog.DEBUG, "start server at %s", addr) l, err := net.Listen("tcp", addr) if err != nil { - glog.Errorf("Failed to listen %s: %v", addr, err) + s.Log(hlog.ERROR, "Failed to listen %s: %v", addr, err) return err } diff --git a/types/types.pb.go b/types/types.pb.go index 72fbe779..082eba26 100644 --- a/types/types.pb.go +++ b/types/types.pb.go @@ -123,6 +123,10 @@ It has these top-level messages: ServiceDelResponse ServiceUpdateRequest ServiceUpdateResponse + PortMappingListRequest + PortMappingListResponse + PortMappingModifyRequest + PortMappingModifyResponse PodStopRequest PodStopResponse PodSignalRequest @@ -4088,6 +4092,70 @@ func (m *ServiceUpdateResponse) String() string { return proto.Compac func (*ServiceUpdateResponse) ProtoMessage() {} func (*ServiceUpdateResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{112} } +type PortMappingListRequest struct { + PodID string `protobuf:"bytes,1,opt,name=podID,proto3" json:"podID,omitempty"` +} + +func (m *PortMappingListRequest) Reset() { *m = PortMappingListRequest{} } +func (m *PortMappingListRequest) String() string { return proto.CompactTextString(m) } +func (*PortMappingListRequest) ProtoMessage() {} +func (*PortMappingListRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{113} } + +func (m *PortMappingListRequest) GetPodID() string { + if m != nil { + return m.PodID + } + return "" +} + +type PortMappingListResponse struct { + PortMappings []*PortMapping `protobuf:"bytes,1,rep,name=portMappings" json:"portMappings,omitempty"` +} + +func (m *PortMappingListResponse) Reset() { *m = PortMappingListResponse{} } +func (m *PortMappingListResponse) String() string { return proto.CompactTextString(m) } +func (*PortMappingListResponse) ProtoMessage() {} +func (*PortMappingListResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{114} } + +func (m *PortMappingListResponse) GetPortMappings() []*PortMapping { + if m != nil { + return m.PortMappings + } + return nil +} + +type PortMappingModifyRequest struct { + PodID string `protobuf:"bytes,1,opt,name=podID,proto3" json:"podID,omitempty"` + PortMappings []*PortMapping `protobuf:"bytes,2,rep,name=portMappings" json:"portMappings,omitempty"` +} + +func (m *PortMappingModifyRequest) Reset() { *m = PortMappingModifyRequest{} } +func (m *PortMappingModifyRequest) String() string { return proto.CompactTextString(m) } +func (*PortMappingModifyRequest) ProtoMessage() {} +func (*PortMappingModifyRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{115} } + +func (m *PortMappingModifyRequest) GetPodID() string { + if m != nil { + return m.PodID + } + return "" +} + +func (m *PortMappingModifyRequest) GetPortMappings() []*PortMapping { + if m != nil { + return m.PortMappings + } + return nil +} + +type PortMappingModifyResponse struct { +} + +func (m *PortMappingModifyResponse) Reset() { *m = PortMappingModifyResponse{} } +func (m *PortMappingModifyResponse) String() string { return proto.CompactTextString(m) } +func (*PortMappingModifyResponse) ProtoMessage() {} +func (*PortMappingModifyResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{116} } + type PodStopRequest struct { PodID string `protobuf:"bytes,1,opt,name=podID,proto3" json:"podID,omitempty"` } @@ -4095,7 +4163,7 @@ type PodStopRequest struct { func (m *PodStopRequest) Reset() { *m = PodStopRequest{} } func (m *PodStopRequest) String() string { return proto.CompactTextString(m) } func (*PodStopRequest) ProtoMessage() {} -func (*PodStopRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{113} } +func (*PodStopRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{117} } func (m *PodStopRequest) GetPodID() string { if m != nil { @@ -4112,7 +4180,7 @@ type PodStopResponse struct { func (m *PodStopResponse) Reset() { *m = PodStopResponse{} } func (m *PodStopResponse) String() string { return proto.CompactTextString(m) } func (*PodStopResponse) ProtoMessage() {} -func (*PodStopResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{114} } +func (*PodStopResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{118} } func (m *PodStopResponse) GetCode() int32 { if m != nil { @@ -4136,7 +4204,7 @@ type PodSignalRequest struct { func (m *PodSignalRequest) Reset() { *m = PodSignalRequest{} } func (m *PodSignalRequest) String() string { return proto.CompactTextString(m) } func (*PodSignalRequest) ProtoMessage() {} -func (*PodSignalRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{115} } +func (*PodSignalRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{119} } func (m *PodSignalRequest) GetPodID() string { if m != nil { @@ -4158,7 +4226,7 @@ type PodSignalResponse struct { func (m *PodSignalResponse) Reset() { *m = PodSignalResponse{} } func (m *PodSignalResponse) String() string { return proto.CompactTextString(m) } func (*PodSignalResponse) ProtoMessage() {} -func (*PodSignalResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{116} } +func (*PodSignalResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{120} } type PodPauseRequest struct { PodID string `protobuf:"bytes,1,opt,name=podID,proto3" json:"podID,omitempty"` @@ -4167,7 +4235,7 @@ type PodPauseRequest struct { func (m *PodPauseRequest) Reset() { *m = PodPauseRequest{} } func (m *PodPauseRequest) String() string { return proto.CompactTextString(m) } func (*PodPauseRequest) ProtoMessage() {} -func (*PodPauseRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{117} } +func (*PodPauseRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{121} } func (m *PodPauseRequest) GetPodID() string { if m != nil { @@ -4182,7 +4250,7 @@ type PodPauseResponse struct { func (m *PodPauseResponse) Reset() { *m = PodPauseResponse{} } func (m *PodPauseResponse) String() string { return proto.CompactTextString(m) } func (*PodPauseResponse) ProtoMessage() {} -func (*PodPauseResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{118} } +func (*PodPauseResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{122} } type PodUnpauseRequest struct { PodID string `protobuf:"bytes,1,opt,name=podID,proto3" json:"podID,omitempty"` @@ -4191,7 +4259,7 @@ type PodUnpauseRequest struct { func (m *PodUnpauseRequest) Reset() { *m = PodUnpauseRequest{} } func (m *PodUnpauseRequest) String() string { return proto.CompactTextString(m) } func (*PodUnpauseRequest) ProtoMessage() {} -func (*PodUnpauseRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{119} } +func (*PodUnpauseRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{123} } func (m *PodUnpauseRequest) GetPodID() string { if m != nil { @@ -4206,7 +4274,7 @@ type PodUnpauseResponse struct { func (m *PodUnpauseResponse) Reset() { *m = PodUnpauseResponse{} } func (m *PodUnpauseResponse) String() string { return proto.CompactTextString(m) } func (*PodUnpauseResponse) ProtoMessage() {} -func (*PodUnpauseResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{120} } +func (*PodUnpauseResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{124} } type PodLabelsRequest struct { PodID string `protobuf:"bytes,1,opt,name=podID,proto3" json:"podID,omitempty"` @@ -4217,7 +4285,7 @@ type PodLabelsRequest struct { func (m *PodLabelsRequest) Reset() { *m = PodLabelsRequest{} } func (m *PodLabelsRequest) String() string { return proto.CompactTextString(m) } func (*PodLabelsRequest) ProtoMessage() {} -func (*PodLabelsRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{121} } +func (*PodLabelsRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{125} } func (m *PodLabelsRequest) GetPodID() string { if m != nil { @@ -4246,7 +4314,7 @@ type PodLabelsResponse struct { func (m *PodLabelsResponse) Reset() { *m = PodLabelsResponse{} } func (m *PodLabelsResponse) String() string { return proto.CompactTextString(m) } func (*PodLabelsResponse) ProtoMessage() {} -func (*PodLabelsResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{122} } +func (*PodLabelsResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{126} } type PodStatsRequest struct { PodID string `protobuf:"bytes,1,opt,name=podID,proto3" json:"podID,omitempty"` @@ -4255,7 +4323,7 @@ type PodStatsRequest struct { func (m *PodStatsRequest) Reset() { *m = PodStatsRequest{} } func (m *PodStatsRequest) String() string { return proto.CompactTextString(m) } func (*PodStatsRequest) ProtoMessage() {} -func (*PodStatsRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{123} } +func (*PodStatsRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{127} } func (m *PodStatsRequest) GetPodID() string { if m != nil { @@ -4271,7 +4339,7 @@ type PodStatsResponse struct { func (m *PodStatsResponse) Reset() { *m = PodStatsResponse{} } func (m *PodStatsResponse) String() string { return proto.CompactTextString(m) } func (*PodStatsResponse) ProtoMessage() {} -func (*PodStatsResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{124} } +func (*PodStatsResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{128} } func (m *PodStatsResponse) GetPodStats() *PodStats { if m != nil { @@ -4286,7 +4354,7 @@ type PingRequest struct { func (m *PingRequest) Reset() { *m = PingRequest{} } func (m *PingRequest) String() string { return proto.CompactTextString(m) } func (*PingRequest) ProtoMessage() {} -func (*PingRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{125} } +func (*PingRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{129} } type PingResponse struct { HyperdStats string `protobuf:"bytes,1,opt,name=hyperdStats,proto3" json:"hyperdStats,omitempty"` @@ -4295,7 +4363,7 @@ type PingResponse struct { func (m *PingResponse) Reset() { *m = PingResponse{} } func (m *PingResponse) String() string { return proto.CompactTextString(m) } func (*PingResponse) ProtoMessage() {} -func (*PingResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{126} } +func (*PingResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{130} } func (m *PingResponse) GetHyperdStats() string { if m != nil { @@ -4313,7 +4381,7 @@ type ContainerSignalRequest struct { func (m *ContainerSignalRequest) Reset() { *m = ContainerSignalRequest{} } func (m *ContainerSignalRequest) String() string { return proto.CompactTextString(m) } func (*ContainerSignalRequest) ProtoMessage() {} -func (*ContainerSignalRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{127} } +func (*ContainerSignalRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{131} } func (m *ContainerSignalRequest) GetPodID() string { if m != nil { @@ -4342,7 +4410,7 @@ type ContainerSignalResponse struct { func (m *ContainerSignalResponse) Reset() { *m = ContainerSignalResponse{} } func (m *ContainerSignalResponse) String() string { return proto.CompactTextString(m) } func (*ContainerSignalResponse) ProtoMessage() {} -func (*ContainerSignalResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{128} } +func (*ContainerSignalResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{132} } type TTYResizeRequest struct { ContainerID string `protobuf:"bytes,1,opt,name=containerID,proto3" json:"containerID,omitempty"` @@ -4354,7 +4422,7 @@ type TTYResizeRequest struct { func (m *TTYResizeRequest) Reset() { *m = TTYResizeRequest{} } func (m *TTYResizeRequest) String() string { return proto.CompactTextString(m) } func (*TTYResizeRequest) ProtoMessage() {} -func (*TTYResizeRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{129} } +func (*TTYResizeRequest) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{133} } func (m *TTYResizeRequest) GetContainerID() string { if m != nil { @@ -4390,7 +4458,7 @@ type TTYResizeResponse struct { func (m *TTYResizeResponse) Reset() { *m = TTYResizeResponse{} } func (m *TTYResizeResponse) String() string { return proto.CompactTextString(m) } func (*TTYResizeResponse) ProtoMessage() {} -func (*TTYResizeResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{130} } +func (*TTYResizeResponse) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{134} } func init() { proto.RegisterType((*ContainerPort)(nil), "types.ContainerPort") @@ -4506,6 +4574,10 @@ func init() { proto.RegisterType((*ServiceDelResponse)(nil), "types.ServiceDelResponse") proto.RegisterType((*ServiceUpdateRequest)(nil), "types.ServiceUpdateRequest") proto.RegisterType((*ServiceUpdateResponse)(nil), "types.ServiceUpdateResponse") + proto.RegisterType((*PortMappingListRequest)(nil), "types.PortMappingListRequest") + proto.RegisterType((*PortMappingListResponse)(nil), "types.PortMappingListResponse") + proto.RegisterType((*PortMappingModifyRequest)(nil), "types.PortMappingModifyRequest") + proto.RegisterType((*PortMappingModifyResponse)(nil), "types.PortMappingModifyResponse") proto.RegisterType((*PodStopRequest)(nil), "types.PodStopRequest") proto.RegisterType((*PodStopResponse)(nil), "types.PodStopResponse") proto.RegisterType((*PodSignalRequest)(nil), "types.PodSignalRequest") @@ -4605,6 +4677,12 @@ type PublicAPIClient interface { ServiceDelete(ctx context.Context, in *ServiceDelRequest, opts ...grpc.CallOption) (*ServiceDelResponse, error) // ServiceUpdate updates an existing service of a pod ServiceUpdate(ctx context.Context, in *ServiceUpdateRequest, opts ...grpc.CallOption) (*ServiceUpdateResponse, error) + // PortMappingList get a list of PortMappings + PortMappingList(ctx context.Context, in *PortMappingListRequest, opts ...grpc.CallOption) (*PortMappingListResponse, error) + // PortMappingAdd add a list of PortMapping rules to a Pod + PortMappingAdd(ctx context.Context, in *PortMappingModifyRequest, opts ...grpc.CallOption) (*PortMappingModifyResponse, error) + // PortMappingDel remove a list of PortMapping rules from a Pod + PortMappingDel(ctx context.Context, in *PortMappingModifyRequest, opts ...grpc.CallOption) (*PortMappingModifyResponse, error) // ImagePull pulls a image from registry ImagePull(ctx context.Context, in *ImagePullRequest, opts ...grpc.CallOption) (PublicAPI_ImagePullClient, error) // ImagePush pushes a local image to registry @@ -5013,6 +5091,33 @@ func (c *publicAPIClient) ServiceUpdate(ctx context.Context, in *ServiceUpdateRe return out, nil } +func (c *publicAPIClient) PortMappingList(ctx context.Context, in *PortMappingListRequest, opts ...grpc.CallOption) (*PortMappingListResponse, error) { + out := new(PortMappingListResponse) + err := grpc.Invoke(ctx, "/types.PublicAPI/PortMappingList", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *publicAPIClient) PortMappingAdd(ctx context.Context, in *PortMappingModifyRequest, opts ...grpc.CallOption) (*PortMappingModifyResponse, error) { + out := new(PortMappingModifyResponse) + err := grpc.Invoke(ctx, "/types.PublicAPI/PortMappingAdd", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *publicAPIClient) PortMappingDel(ctx context.Context, in *PortMappingModifyRequest, opts ...grpc.CallOption) (*PortMappingModifyResponse, error) { + out := new(PortMappingModifyResponse) + err := grpc.Invoke(ctx, "/types.PublicAPI/PortMappingDel", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *publicAPIClient) ImagePull(ctx context.Context, in *ImagePullRequest, opts ...grpc.CallOption) (PublicAPI_ImagePullClient, error) { stream, err := grpc.NewClientStream(ctx, &_PublicAPI_serviceDesc.Streams[4], c.cc, "/types.PublicAPI/ImagePull", opts...) if err != nil { @@ -5184,6 +5289,12 @@ type PublicAPIServer interface { ServiceDelete(context.Context, *ServiceDelRequest) (*ServiceDelResponse, error) // ServiceUpdate updates an existing service of a pod ServiceUpdate(context.Context, *ServiceUpdateRequest) (*ServiceUpdateResponse, error) + // PortMappingList get a list of PortMappings + PortMappingList(context.Context, *PortMappingListRequest) (*PortMappingListResponse, error) + // PortMappingAdd add a list of PortMapping rules to a Pod + PortMappingAdd(context.Context, *PortMappingModifyRequest) (*PortMappingModifyResponse, error) + // PortMappingDel remove a list of PortMapping rules from a Pod + PortMappingDel(context.Context, *PortMappingModifyRequest) (*PortMappingModifyResponse, error) // ImagePull pulls a image from registry ImagePull(*ImagePullRequest, PublicAPI_ImagePullServer) error // ImagePush pushes a local image to registry @@ -5823,6 +5934,60 @@ func _PublicAPI_ServiceUpdate_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _PublicAPI_PortMappingList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PortMappingListRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PublicAPIServer).PortMappingList(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/types.PublicAPI/PortMappingList", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PublicAPIServer).PortMappingList(ctx, req.(*PortMappingListRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _PublicAPI_PortMappingAdd_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PortMappingModifyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PublicAPIServer).PortMappingAdd(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/types.PublicAPI/PortMappingAdd", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PublicAPIServer).PortMappingAdd(ctx, req.(*PortMappingModifyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _PublicAPI_PortMappingDel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PortMappingModifyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PublicAPIServer).PortMappingDel(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/types.PublicAPI/PortMappingDel", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PublicAPIServer).PortMappingDel(ctx, req.(*PortMappingModifyRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _PublicAPI_ImagePull_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(ImagePullRequest) if err := stream.RecvMsg(m); err != nil { @@ -6057,6 +6222,18 @@ var _PublicAPI_serviceDesc = grpc.ServiceDesc{ MethodName: "ServiceUpdate", Handler: _PublicAPI_ServiceUpdate_Handler, }, + { + MethodName: "PortMappingList", + Handler: _PublicAPI_PortMappingList_Handler, + }, + { + MethodName: "PortMappingAdd", + Handler: _PublicAPI_PortMappingAdd_Handler, + }, + { + MethodName: "PortMappingDel", + Handler: _PublicAPI_PortMappingDel_Handler, + }, { MethodName: "ImageRemove", Handler: _PublicAPI_ImageRemove_Handler, @@ -6115,334 +6292,339 @@ var _PublicAPI_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("types.proto", fileDescriptorTypes) } var fileDescriptorTypes = []byte{ - // 5250 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x3c, 0x4d, 0x6f, 0x1d, 0x47, - 0x72, 0x7a, 0x5f, 0x7c, 0x7c, 0xc5, 0x4f, 0xb5, 0x28, 0xe9, 0x89, 0xd6, 0x7a, 0xb5, 0xb3, 0xf1, - 0x4a, 0x96, 0x63, 0xda, 0xd6, 0x3a, 0xb6, 0x57, 0x8e, 0xb1, 0xa6, 0x49, 0x79, 0x4d, 0xc4, 0xb2, - 0xe9, 0xa1, 0x24, 0xc3, 0xc8, 0x02, 0x9b, 0xd1, 0x9b, 0xe6, 0xe3, 0x98, 0xf3, 0x66, 0x26, 0x33, - 0xf3, 0x28, 0xd1, 0xb7, 0xe4, 0xb4, 0x80, 0x11, 0xe4, 0xb0, 0x40, 0x90, 0xcd, 0x31, 0xc9, 0x21, - 0xc8, 0x35, 0xa7, 0x04, 0xb9, 0xe4, 0x92, 0x53, 0xfe, 0x48, 0xb2, 0x97, 0x5c, 0x72, 0x0d, 0x82, - 0xaa, 0xae, 0xee, 0xe9, 0x9e, 0x37, 0x7c, 0x94, 0x6c, 0xe5, 0x40, 0x68, 0xaa, 0xba, 0xba, 0xba, - 0xba, 0xba, 0xba, 0xaa, 0xba, 0xba, 0x9f, 0x60, 0xa9, 0x3c, 0xcd, 0x64, 0xb1, 0x95, 0xe5, 0x69, - 0x99, 0x8a, 0x1e, 0x01, 0xde, 0x6f, 0x5b, 0xb0, 0xb2, 0x93, 0x26, 0x65, 0x10, 0x25, 0x32, 0xdf, - 0x4f, 0xf3, 0x52, 0x08, 0xe8, 0x26, 0xc1, 0x44, 0x0e, 0x5b, 0x37, 0x5a, 0xb7, 0x06, 0x3e, 0x7d, - 0x8b, 0x4d, 0x58, 0x3c, 0x4a, 0x8b, 0x12, 0xdb, 0x87, 0xed, 0x1b, 0xad, 0x5b, 0x3d, 0xdf, 0xc0, - 0xe2, 0xf7, 0x60, 0x65, 0x64, 0x33, 0x18, 0x76, 0x88, 0xc0, 0x45, 0x22, 0x07, 0x1a, 0x77, 0x94, - 0xc6, 0xc3, 0x2e, 0x71, 0x36, 0xb0, 0xb8, 0x02, 0x0b, 0xc8, 0x6d, 0x6f, 0x7f, 0xd8, 0xa3, 0x16, - 0x86, 0xbc, 0xf7, 0x60, 0xf5, 0x5e, 0x72, 0x12, 0xe5, 0x69, 0x32, 0x91, 0x49, 0xf9, 0x28, 0xc8, - 0xc5, 0x3a, 0x74, 0x64, 0x72, 0xc2, 0xa2, 0xe1, 0xa7, 0xd8, 0x80, 0xde, 0x49, 0x10, 0x4f, 0x25, - 0x89, 0x35, 0xf0, 0x15, 0xe0, 0xfd, 0x31, 0x2c, 0x3d, 0x4a, 0xe3, 0xe9, 0x44, 0xde, 0x4f, 0xa7, - 0x49, 0xf3, 0x94, 0xae, 0xc3, 0x60, 0x82, 0x8d, 0xfb, 0x41, 0x79, 0xc4, 0x9d, 0x2b, 0x04, 0x8a, - 0x9b, 0xcb, 0x20, 0xfc, 0x3c, 0x89, 0x4f, 0x69, 0x3e, 0x8b, 0xbe, 0x81, 0xbd, 0x9b, 0xb0, 0xf2, - 0x65, 0x10, 0x95, 0x51, 0x32, 0x3e, 0x28, 0x83, 0x72, 0x5a, 0xa0, 0xfc, 0xb9, 0x0c, 0x8a, 0x34, - 0xe1, 0x01, 0x18, 0xf2, 0x5e, 0x87, 0x15, 0x7f, 0x9a, 0x24, 0x15, 0xe1, 0x75, 0x18, 0x14, 0x65, - 0x90, 0x97, 0x32, 0xdc, 0x2e, 0x99, 0xb6, 0x42, 0x78, 0x7f, 0xdd, 0x02, 0x78, 0x20, 0xf3, 0x09, - 0x13, 0x6f, 0xc2, 0xa2, 0x7c, 0x1a, 0x95, 0x3b, 0x69, 0xa8, 0x04, 0xef, 0xf9, 0x06, 0xb6, 0x46, - 0x6c, 0xdb, 0x23, 0x8a, 0x21, 0xf4, 0x27, 0xb2, 0x28, 0x82, 0xb1, 0x24, 0xa9, 0x07, 0xbe, 0x06, - 0xdd, 0xa1, 0xbb, 0xb5, 0xa1, 0xc5, 0xcb, 0x00, 0x87, 0x51, 0x12, 0x15, 0x47, 0xd4, 0xac, 0x56, - 0xc1, 0xc2, 0x78, 0xff, 0xdd, 0x82, 0x35, 0x63, 0x25, 0x2c, 0x5f, 0x93, 0x52, 0x6f, 0xc0, 0x92, - 0x59, 0xf6, 0xbd, 0x5d, 0x16, 0xce, 0x46, 0xe1, 0x7a, 0x65, 0x47, 0x41, 0xa1, 0xe5, 0x53, 0x80, - 0xd8, 0x82, 0xfe, 0x13, 0xa5, 0x52, 0x92, 0x6d, 0xe9, 0xce, 0xc6, 0x96, 0xb2, 0x55, 0x47, 0xd1, - 0xbe, 0x26, 0x42, 0xfa, 0x5c, 0x69, 0x96, 0x84, 0xad, 0xe8, 0x1d, 0x7d, 0xfb, 0x9a, 0x48, 0xbc, - 0x05, 0x50, 0xca, 0x7c, 0x12, 0x25, 0x41, 0x29, 0xc3, 0xe1, 0x02, 0x75, 0xb9, 0xc8, 0x5d, 0x2a, - 0x95, 0xfb, 0x16, 0x91, 0xf7, 0x77, 0xf6, 0xc6, 0xd8, 0x4b, 0x0e, 0x53, 0xb1, 0x05, 0x03, 0x33, - 0x13, 0x9a, 0xf5, 0xd2, 0x9d, 0x75, 0xe6, 0x61, 0x08, 0xfd, 0x8a, 0x04, 0x55, 0x3e, 0xca, 0x65, - 0xa0, 0x54, 0x8e, 0xaa, 0xe8, 0xf8, 0x15, 0x82, 0x14, 0x91, 0x86, 0x7b, 0xbb, 0x46, 0x11, 0x08, - 0x88, 0x2d, 0x58, 0x28, 0x48, 0x16, 0xd6, 0xc3, 0x95, 0xfa, 0x00, 0x2c, 0x29, 0x53, 0x79, 0x7f, - 0xd9, 0x85, 0x81, 0x69, 0xfb, 0xee, 0x4b, 0x12, 0x4d, 0x2a, 0x93, 0x51, 0x00, 0x9a, 0x12, 0x7d, - 0xec, 0xed, 0xb2, 0xb9, 0x68, 0x50, 0xdc, 0x82, 0x35, 0xfa, 0xdc, 0x9f, 0xc6, 0xf1, 0x7e, 0x1a, - 0x47, 0xa3, 0x53, 0xb6, 0x98, 0x3a, 0x1a, 0xcd, 0xea, 0x49, 0x9a, 0x1f, 0x47, 0xc9, 0x78, 0x37, - 0xca, 0x49, 0xed, 0x03, 0xdf, 0xc2, 0xa0, 0xbc, 0xd3, 0x42, 0xe6, 0xc3, 0xbe, 0x92, 0x17, 0xbf, - 0x71, 0x8b, 0x97, 0xe5, 0xe9, 0x70, 0x91, 0x36, 0x1d, 0x7e, 0xe2, 0x46, 0x18, 0xa5, 0x93, 0x49, - 0x90, 0x84, 0xc5, 0x70, 0x70, 0xa3, 0x83, 0xae, 0x43, 0xc3, 0xc8, 0x21, 0xc8, 0xc7, 0xc5, 0x10, - 0x08, 0x4f, 0xdf, 0xe2, 0x36, 0x6a, 0x36, 0x2f, 0x8b, 0xe1, 0xd2, 0x8d, 0x8e, 0x65, 0x1a, 0x8e, - 0x97, 0xf3, 0x15, 0x89, 0xb8, 0xa9, 0x1c, 0xca, 0x32, 0x51, 0x5e, 0x66, 0x4a, 0xd7, 0xe9, 0x28, - 0x3f, 0xf3, 0x0e, 0x2c, 0x9f, 0x54, 0x1e, 0xa5, 0x18, 0xae, 0x50, 0x0f, 0xc1, 0x3d, 0x2c, 0x67, - 0xe3, 0x3b, 0x74, 0xe2, 0x6d, 0x58, 0x88, 0x83, 0xc7, 0x32, 0x2e, 0x86, 0xab, 0xd4, 0xe3, 0x7a, - 0x5d, 0x9a, 0xad, 0x4f, 0xa9, 0xf9, 0x5e, 0x52, 0xe6, 0xa7, 0x3e, 0xd3, 0x6e, 0xfe, 0x0c, 0x96, - 0x2c, 0x34, 0xea, 0xe4, 0x58, 0x9e, 0x6a, 0xb7, 0x77, 0x2c, 0x4f, 0x9b, 0xdd, 0xde, 0xdd, 0xf6, - 0x7b, 0x2d, 0xef, 0x9f, 0x5b, 0xb0, 0xe6, 0x7f, 0xb4, 0xab, 0x24, 0x3a, 0x48, 0xa7, 0xf9, 0x88, - 0xdc, 0xf7, 0x24, 0x4d, 0xa2, 0x32, 0xcd, 0x8b, 0x61, 0x4b, 0x69, 0x50, 0xc3, 0xd5, 0xea, 0xb7, - 0xed, 0xd5, 0xbf, 0x02, 0x0b, 0x87, 0xc5, 0x83, 0xd3, 0x4c, 0x1b, 0x05, 0x43, 0xa8, 0xef, 0x2c, - 0x35, 0x2e, 0x9c, 0xbe, 0xcd, 0x2a, 0xf6, 0xac, 0x55, 0x1c, 0x42, 0xff, 0x58, 0x9e, 0xe6, 0xb8, - 0x41, 0xd5, 0xb2, 0x6b, 0xd0, 0xf1, 0xac, 0xfd, 0x9a, 0x67, 0x3d, 0x85, 0xc1, 0x7e, 0x1a, 0x2a, - 0xd1, 0x1b, 0x8d, 0xf9, 0x0a, 0x2c, 0x14, 0x34, 0x25, 0xed, 0xf7, 0x14, 0x84, 0xf8, 0x30, 0x8f, - 0x4e, 0x64, 0xae, 0xc5, 0x55, 0x90, 0xb8, 0x05, 0x9d, 0xfc, 0x71, 0x58, 0xdb, 0x4b, 0x35, 0xed, - 0xf8, 0x48, 0xe2, 0xfd, 0x79, 0x1b, 0xfa, 0xfb, 0x69, 0x78, 0x90, 0xc9, 0x91, 0xb8, 0x0d, 0x7d, - 0xb5, 0x86, 0x4a, 0x5b, 0xd5, 0x36, 0x37, 0xc2, 0xf9, 0x9a, 0x40, 0xbc, 0x09, 0x60, 0xf6, 0x52, - 0x31, 0x6c, 0x3b, 0xe4, 0x95, 0x57, 0xb0, 0x68, 0xc4, 0x1d, 0x63, 0x11, 0x1d, 0xa2, 0xde, 0xac, - 0x98, 0xe3, 0xe8, 0x4d, 0xf6, 0x80, 0xba, 0x38, 0x19, 0x65, 0x53, 0x9a, 0x48, 0xcf, 0xa7, 0x6f, - 0x9c, 0xf3, 0x44, 0x4e, 0xd2, 0x5c, 0xed, 0xbe, 0x9e, 0xcf, 0xd0, 0xf7, 0xb1, 0x9d, 0x3f, 0x6b, - 0xd3, 0x02, 0xb0, 0x83, 0x37, 0xae, 0xba, 0x65, 0xbb, 0x6a, 0x2b, 0xc4, 0xb4, 0xdd, 0x10, 0x53, - 0x05, 0xa5, 0x8e, 0x13, 0x94, 0xaa, 0xf0, 0xde, 0xb5, 0xc3, 0xbb, 0xf6, 0x80, 0x18, 0xf5, 0x3b, - 0xda, 0x03, 0xee, 0x9b, 0x40, 0xf5, 0x20, 0x9a, 0x48, 0xb6, 0x9d, 0x0a, 0x21, 0x3e, 0x84, 0xb5, - 0x91, 0xeb, 0x0a, 0x87, 0x7d, 0xd2, 0xe2, 0x59, 0x8e, 0xb2, 0x4e, 0x5e, 0x85, 0x3a, 0x1a, 0x60, - 0xd1, 0x0e, 0x75, 0x88, 0xf1, 0xfe, 0xb3, 0x45, 0x86, 0x40, 0x1e, 0xdf, 0xf8, 0xe8, 0x96, 0xed, - 0xa3, 0x05, 0x74, 0x8f, 0xa3, 0x24, 0xe4, 0xe9, 0xd3, 0x37, 0x72, 0x0d, 0xb2, 0xe8, 0x91, 0xcc, - 0x8b, 0xc8, 0xcc, 0xdf, 0xc2, 0x88, 0x55, 0x68, 0x9f, 0x4c, 0x78, 0xfe, 0xed, 0x93, 0x89, 0x1b, - 0x1b, 0x7a, 0xf5, 0xd8, 0xe0, 0x41, 0xb7, 0xc8, 0xe4, 0x88, 0x03, 0xd5, 0xaa, 0x6b, 0x20, 0x3e, - 0xb5, 0x89, 0x5b, 0x26, 0x52, 0xf4, 0x9d, 0x50, 0x64, 0xd6, 0x4f, 0xc7, 0x08, 0x5c, 0xb1, 0x2c, - 0x0d, 0x3f, 0x0b, 0xcc, 0x74, 0x35, 0xe8, 0xfd, 0x6d, 0x1b, 0x06, 0x7b, 0xe4, 0xd5, 0x71, 0xb6, - 0xab, 0xd0, 0x8e, 0x42, 0x9e, 0x6a, 0x3b, 0x0a, 0x29, 0x65, 0x0b, 0x72, 0x99, 0x94, 0x26, 0x6c, - 0x18, 0x58, 0xed, 0xe2, 0x2c, 0x7d, 0x10, 0x8c, 0x95, 0x19, 0x0f, 0x7c, 0x03, 0x63, 0xc4, 0xc1, - 0xef, 0xdd, 0x68, 0x2c, 0x8b, 0x12, 0x03, 0x19, 0x36, 0xdb, 0x28, 0x94, 0x88, 0x27, 0xcb, 0x73, - 0xd7, 0x20, 0xf6, 0x3d, 0x89, 0xf2, 0x72, 0x1a, 0xc4, 0x07, 0xd1, 0x37, 0x6a, 0xfd, 0x3b, 0xbe, - 0x8d, 0xb2, 0x1c, 0x6a, 0xdf, 0x71, 0xa8, 0x66, 0x1e, 0x2f, 0xda, 0xa1, 0xfe, 0x5b, 0x1b, 0x16, - 0x59, 0xa9, 0x85, 0xf8, 0x11, 0x74, 0x70, 0x1f, 0xaa, 0xe8, 0xbf, 0xa6, 0x6d, 0x2e, 0x9b, 0x52, - 0xab, 0x8f, 0x6d, 0xe2, 0x26, 0xf4, 0x1e, 0xc7, 0xe9, 0xe8, 0x98, 0x38, 0x55, 0x69, 0xc6, 0x47, - 0xf1, 0x71, 0x94, 0x2a, 0x32, 0xd5, 0x2e, 0x6e, 0x9b, 0x0d, 0xdc, 0x21, 0x4a, 0x1d, 0x4c, 0xee, - 0x13, 0x52, 0x91, 0x32, 0x85, 0x78, 0x1d, 0xfa, 0x89, 0x2c, 0x31, 0x74, 0xb2, 0x33, 0xbb, 0xc4, - 0xc4, 0x9f, 0x29, 0xac, 0xa2, 0xd6, 0x34, 0x62, 0x0b, 0x8d, 0x3c, 0x96, 0xc5, 0x69, 0x51, 0xca, - 0x09, 0xed, 0xaf, 0xca, 0x8c, 0x3e, 0x2e, 0x14, 0xb1, 0x45, 0x81, 0xe6, 0x58, 0x46, 0x13, 0x59, - 0x94, 0xc1, 0x24, 0x63, 0xa5, 0x57, 0x08, 0x67, 0xd3, 0xa9, 0xce, 0x67, 0x6d, 0x3a, 0x66, 0x5d, - 0x27, 0xf7, 0x0e, 0x60, 0x51, 0x2b, 0x49, 0xbc, 0x02, 0xbd, 0x29, 0xb9, 0x8f, 0x19, 0x25, 0x3e, - 0x44, 0xb4, 0xaf, 0x5a, 0xd1, 0x12, 0x3e, 0x4d, 0x83, 0x70, 0xfb, 0x44, 0xe6, 0xda, 0xd7, 0xf4, - 0x7c, 0x1b, 0xe5, 0x85, 0xc4, 0x94, 0x3a, 0xe1, 0xf2, 0x95, 0x69, 0x19, 0xc4, 0xc4, 0xb4, 0xeb, - 0x2b, 0x00, 0x3d, 0x4f, 0x26, 0xf3, 0x9d, 0x6c, 0x4a, 0x8e, 0xb9, 0xeb, 0x33, 0x64, 0x22, 0x56, - 0x87, 0x88, 0x55, 0xc4, 0xc2, 0xd0, 0xa2, 0xd4, 0xd5, 0x25, 0x2c, 0x43, 0xde, 0x7f, 0x74, 0x01, - 0xaa, 0xb5, 0x13, 0x9f, 0xc3, 0xd5, 0x28, 0x3d, 0x90, 0xf9, 0x49, 0x34, 0x92, 0x1f, 0x9d, 0x96, - 0xb2, 0xf0, 0xe5, 0x68, 0x9a, 0x17, 0xd1, 0x89, 0xe4, 0x58, 0x71, 0xb9, 0xbe, 0xde, 0xca, 0x10, - 0xcf, 0xea, 0x25, 0x7e, 0x01, 0x97, 0x4c, 0x53, 0x58, 0x31, 0x6b, 0xcf, 0x63, 0xd6, 0xd4, 0x43, - 0xec, 0xc0, 0xc5, 0x28, 0xfd, 0x62, 0x2a, 0xa7, 0x36, 0x9b, 0xce, 0x3c, 0x36, 0xb3, 0xf4, 0xe2, - 0x3e, 0x5c, 0x31, 0xbc, 0xd1, 0x1d, 0x56, 0x9c, 0xba, 0xf3, 0x38, 0x9d, 0xd1, 0x49, 0x4d, 0x0e, - 0x73, 0x78, 0x97, 0x57, 0xef, 0x9c, 0xc9, 0xcd, 0xf4, 0x50, 0x93, 0xbb, 0x2f, 0xf3, 0xb1, 0x3d, - 0xb9, 0x85, 0x73, 0x26, 0x57, 0xa3, 0x17, 0x3f, 0x87, 0xb5, 0x28, 0x75, 0x25, 0xe9, 0xcf, 0x63, - 0x51, 0xa7, 0x16, 0xdb, 0xb0, 0x5e, 0xc8, 0x11, 0xa6, 0x4d, 0x15, 0x87, 0xc5, 0x79, 0x1c, 0x66, - 0xc8, 0xbd, 0xff, 0x6a, 0xc1, 0xaa, 0x4b, 0xd4, 0x98, 0xe8, 0x08, 0xe8, 0x22, 0x43, 0x1d, 0x63, - 0xf0, 0xdb, 0x4a, 0x7e, 0x3a, 0x4e, 0xf2, 0xb3, 0x01, 0xbd, 0x49, 0xf0, 0x75, 0x9a, 0xb3, 0xe1, - 0x2a, 0x80, 0xb0, 0x51, 0x92, 0xaa, 0xb4, 0x0c, 0xb1, 0x08, 0x88, 0x9f, 0x42, 0x17, 0xa3, 0x02, - 0xab, 0xee, 0x87, 0x8d, 0x52, 0x6f, 0x55, 0xf2, 0x13, 0xf1, 0xe6, 0xbb, 0x30, 0xa8, 0xa4, 0x3d, - 0xc7, 0x75, 0x76, 0x6d, 0xd7, 0xf9, 0xbb, 0x16, 0x2c, 0x59, 0xde, 0x0c, 0x29, 0xab, 0xad, 0xdf, - 0xd5, 0x3b, 0xbd, 0x3a, 0x25, 0x1c, 0xc8, 0x92, 0x99, 0x58, 0x18, 0x8c, 0x16, 0x87, 0x41, 0x14, - 0x8f, 0x92, 0x92, 0x37, 0xac, 0x06, 0xc5, 0x47, 0x56, 0xe9, 0x61, 0x37, 0x28, 0x03, 0xf6, 0x8d, - 0xd7, 0x67, 0x1d, 0xa9, 0xfa, 0x44, 0x1a, 0xdf, 0xed, 0x22, 0x3e, 0x81, 0xf5, 0xa3, 0x48, 0xe6, - 0x41, 0x3e, 0x3a, 0x8a, 0x46, 0x41, 0x4c, 0x6c, 0x7a, 0xcf, 0xc0, 0x66, 0xa6, 0x97, 0xf7, 0x05, - 0x5c, 0x6e, 0x24, 0xa5, 0x00, 0x3c, 0x3e, 0x0c, 0xa6, 0x71, 0xc9, 0x13, 0xd7, 0x20, 0x4e, 0x3d, - 0x1b, 0x4f, 0x82, 0xaf, 0x55, 0x23, 0x4f, 0xbd, 0xc2, 0x78, 0xdf, 0xb6, 0x60, 0xd9, 0xf6, 0xf0, - 0xe2, 0x0f, 0x00, 0xa2, 0xa4, 0x94, 0xf9, 0x61, 0x30, 0x32, 0xd9, 0xa9, 0xb6, 0xbd, 0x3d, 0xdd, - 0xc0, 0xfe, 0xbd, 0x22, 0x14, 0x37, 0xa0, 0x53, 0x8e, 0x32, 0x8e, 0x48, 0x3a, 0x10, 0x3c, 0x18, - 0x65, 0x48, 0xe9, 0x63, 0x13, 0xa6, 0x1c, 0xe5, 0x28, 0x7b, 0x87, 0x43, 0x51, 0x9d, 0x84, 0xda, - 0xbc, 0x7f, 0x6a, 0x43, 0x9f, 0x31, 0xe8, 0x9e, 0x31, 0x3a, 0x3c, 0x8e, 0xa9, 0x44, 0xc0, 0xf3, - 0xb2, 0x51, 0x38, 0xeb, 0xe2, 0x34, 0x39, 0x90, 0x89, 0x9e, 0x98, 0x06, 0xb9, 0xc5, 0x97, 0xa3, - 0x13, 0xbd, 0xa0, 0x0c, 0x62, 0x5a, 0x71, 0x18, 0x25, 0xb8, 0xfd, 0xdf, 0x62, 0x6b, 0x36, 0xb0, - 0xd5, 0x76, 0x87, 0x6d, 0xda, 0xc0, 0xd8, 0x86, 0xe1, 0x0a, 0x01, 0x0a, 0x5f, 0x5d, 0xdf, 0xc0, - 0x68, 0x74, 0xa3, 0x38, 0x2d, 0x24, 0xe5, 0x49, 0x5d, 0x5f, 0x01, 0x94, 0x80, 0xe1, 0x07, 0x75, - 0x59, 0xa4, 0x96, 0x0a, 0x81, 0x12, 0xc6, 0x41, 0x51, 0x6e, 0x8f, 0x8e, 0x87, 0x03, 0x25, 0x21, - 0x83, 0xb8, 0x09, 0xe3, 0xa8, 0x28, 0x65, 0x32, 0x04, 0x15, 0x26, 0x14, 0x44, 0x29, 0x4d, 0x9c, - 0x16, 0x78, 0xe0, 0x59, 0x52, 0x3d, 0x18, 0xf4, 0x7e, 0xdd, 0x86, 0x55, 0x77, 0x69, 0x1a, 0x77, - 0xfc, 0x10, 0xfa, 0xf9, 0x53, 0x8a, 0x0d, 0x5a, 0x5d, 0x0c, 0xa2, 0xa8, 0xf9, 0xd3, 0xfd, 0x60, - 0x74, 0x2c, 0xcb, 0x82, 0x15, 0x56, 0x21, 0x28, 0x13, 0x7b, 0x7a, 0x2f, 0xcf, 0xf1, 0x6c, 0xc7, - 0x2a, 0xd3, 0xb0, 0xea, 0xb9, 0x9b, 0xa7, 0x59, 0xc6, 0x99, 0x16, 0xf5, 0x64, 0x04, 0x8e, 0x58, - 0xf2, 0x88, 0x4a, 0x67, 0x1a, 0xa4, 0x74, 0xc0, 0x8c, 0xa8, 0xd4, 0x56, 0x21, 0x48, 0xd9, 0x7a, - 0xc4, 0x45, 0x56, 0xb6, 0x35, 0x62, 0x69, 0x46, 0x1c, 0xe8, 0x9e, 0x8c, 0xf0, 0x7e, 0xd7, 0x81, - 0x3e, 0xa7, 0x1f, 0x74, 0x64, 0x93, 0x18, 0x31, 0x74, 0xd1, 0x4c, 0x41, 0xb8, 0x5c, 0x71, 0x34, - 0x89, 0xb4, 0xd1, 0x28, 0xa0, 0xf2, 0x1c, 0x1d, 0xdb, 0x73, 0x5c, 0x87, 0x41, 0x70, 0x12, 0x44, - 0x71, 0xf0, 0x38, 0x96, 0x3c, 0xf9, 0x0a, 0x21, 0x7e, 0x02, 0xab, 0x78, 0xb2, 0x2c, 0x76, 0xd2, - 0x49, 0x16, 0xcb, 0xd2, 0xa8, 0xa0, 0x86, 0x55, 0xf9, 0x6a, 0x10, 0x16, 0x2a, 0x5c, 0xb0, 0x2e, - 0x6c, 0x14, 0x52, 0x18, 0x47, 0x1e, 0x84, 0xac, 0x11, 0x1b, 0xa5, 0x4f, 0xb5, 0xe6, 0x4c, 0xd1, - 0xf5, 0x0d, 0x2c, 0x6e, 0xc1, 0xda, 0x93, 0x3c, 0x2a, 0xa5, 0x25, 0x88, 0xd2, 0x4c, 0x1d, 0x2d, - 0x3c, 0x58, 0x56, 0x28, 0x16, 0x45, 0x99, 0x98, 0x83, 0xc3, 0x59, 0xf1, 0xc0, 0x5f, 0xe6, 0x51, - 0x89, 0x86, 0xa8, 0xec, 0xad, 0x86, 0x45, 0xdd, 0x50, 0x3f, 0x12, 0x69, 0x59, 0xe9, 0xc6, 0x20, - 0x70, 0xa4, 0x28, 0xdd, 0x4b, 0xf6, 0xf3, 0x74, 0x9c, 0xcb, 0xa2, 0x18, 0xae, 0xa8, 0x91, 0x6c, - 0x1c, 0xae, 0x90, 0x0a, 0x80, 0xc3, 0x55, 0x65, 0xea, 0x0a, 0x42, 0x09, 0x9e, 0xc8, 0x68, 0x7c, - 0x54, 0xca, 0x70, 0x4f, 0xb5, 0xaf, 0x29, 0x09, 0x5c, 0xac, 0xf7, 0x0f, 0x6d, 0xab, 0x68, 0xc8, - 0xab, 0x5e, 0xab, 0x46, 0xb5, 0x66, 0xab, 0x51, 0x9c, 0x61, 0xb7, 0x9f, 0x25, 0xc3, 0xee, 0x3c, - 0x73, 0x86, 0xdd, 0x7d, 0x9e, 0x0c, 0xbb, 0xf7, 0xdc, 0x19, 0xf6, 0xc2, 0xf3, 0x65, 0xd8, 0xfd, - 0x5a, 0x86, 0xed, 0xfd, 0x04, 0x56, 0xf9, 0xcc, 0xe9, 0xcb, 0x3f, 0x9d, 0xca, 0xa2, 0x6c, 0x3e, - 0x7a, 0x7a, 0xef, 0xc3, 0x9a, 0xa1, 0x2b, 0xb2, 0x34, 0x29, 0xd0, 0xba, 0xf0, 0x38, 0x87, 0x28, - 0x4e, 0xa8, 0xad, 0xe3, 0x22, 0x11, 0xea, 0x66, 0xef, 0x2e, 0x0d, 0xf2, 0x69, 0x54, 0x94, 0x73, - 0x07, 0xa1, 0x62, 0xc3, 0xc4, 0x9c, 0xf9, 0xe8, 0xdb, 0xfb, 0xdf, 0x16, 0xac, 0x98, 0xce, 0x05, - 0x86, 0xae, 0xe6, 0xbe, 0xd6, 0x59, 0xb3, 0xed, 0x9c, 0x35, 0x0d, 0xd7, 0x4e, 0xc5, 0x95, 0x32, - 0x9a, 0xaa, 0xda, 0x39, 0x30, 0x27, 0xd6, 0xf9, 0xa7, 0xe3, 0xf7, 0xcc, 0x09, 0x50, 0xa9, 0xfd, - 0x46, 0x35, 0xe1, 0x4a, 0xbe, 0x17, 0x7d, 0x0a, 0xdc, 0x26, 0xcd, 0x33, 0x7f, 0xa5, 0xf9, 0x2d, - 0x9a, 0x2b, 0xa2, 0x38, 0x10, 0x6f, 0x34, 0x09, 0xe2, 0x6b, 0x22, 0xef, 0x43, 0xd8, 0x30, 0xdb, - 0xe1, 0xbb, 0xad, 0xc2, 0xb7, 0x2d, 0xb8, 0x54, 0x63, 0x41, 0x6b, 0x71, 0xfe, 0xae, 0xb2, 0x2f, - 0x69, 0xac, 0xd5, 0x71, 0x91, 0x67, 0xd4, 0xa4, 0xcf, 0x58, 0x25, 0xef, 0x2b, 0xb8, 0x5c, 0x17, - 0x46, 0x29, 0xe6, 0x43, 0x6b, 0x30, 0x4b, 0x3d, 0x9b, 0xf5, 0xd3, 0xa2, 0xa5, 0x24, 0xb7, 0x83, - 0xf7, 0xb6, 0xa5, 0x2a, 0x7b, 0x57, 0x5c, 0xaf, 0x97, 0xe0, 0x07, 0x56, 0xc1, 0xdd, 0x3b, 0xb0, - 0x04, 0x72, 0xf6, 0xc8, 0x5d, 0x4b, 0x20, 0x6b, 0xa7, 0xcc, 0x54, 0x86, 0xa9, 0x93, 0x4b, 0xea, - 0xed, 0xc3, 0xf2, 0xa3, 0xfb, 0x96, 0xae, 0xf5, 0xba, 0xb4, 0x2c, 0x3b, 0x36, 0x7a, 0x6b, 0x37, - 0xeb, 0xad, 0xe3, 0xe8, 0xed, 0x67, 0xb0, 0xa2, 0x39, 0x3e, 0xaf, 0x01, 0x7c, 0x00, 0xab, 0x46, - 0x18, 0x35, 0xb5, 0xd7, 0x60, 0xe1, 0x64, 0x62, 0x29, 0x59, 0x7b, 0x2d, 0x5b, 0x66, 0x9f, 0x49, - 0xbc, 0x5f, 0xc2, 0x3a, 0x95, 0x49, 0xec, 0xc1, 0xa9, 0x1e, 0x16, 0x97, 0x32, 0xdf, 0xce, 0xc7, - 0x05, 0x4b, 0x60, 0x61, 0xa8, 0x12, 0x4c, 0x90, 0x2e, 0xb9, 0x2a, 0x08, 0x37, 0x4f, 0x10, 0xc7, - 0x7c, 0x39, 0x86, 0x9f, 0xde, 0x0e, 0x5c, 0xb4, 0xb8, 0x9b, 0x4d, 0x32, 0x88, 0x34, 0xb2, 0x56, - 0x4d, 0x35, 0x15, 0x1b, 0xbf, 0x22, 0x41, 0x0f, 0xf7, 0xe8, 0xfe, 0x0e, 0xed, 0x75, 0x2d, 0xe1, - 0x7a, 0x55, 0x73, 0xe9, 0xa9, 0x00, 0x50, 0x95, 0x3e, 0xdb, 0x76, 0xe9, 0xd3, 0xfb, 0x09, 0xac, - 0x57, 0x9d, 0x59, 0x80, 0x86, 0xf5, 0xf2, 0x5e, 0xc1, 0x41, 0x7c, 0x39, 0x49, 0x4f, 0xcc, 0x20, - 0x4d, 0x64, 0x7f, 0x88, 0xec, 0x34, 0x59, 0xc5, 0x6e, 0x54, 0xdd, 0xc8, 0xd1, 0x37, 0x65, 0x98, - 0xc1, 0xb4, 0x30, 0x5e, 0x83, 0x00, 0xef, 0x37, 0x2d, 0xb8, 0xf8, 0xb0, 0x90, 0xf9, 0x4e, 0xfd, - 0x1e, 0xd4, 0xdc, 0xa4, 0xb6, 0xce, 0xbb, 0x49, 0x6d, 0x37, 0xdd, 0xa4, 0x52, 0x32, 0x42, 0x67, - 0x6d, 0xeb, 0xb6, 0xd5, 0x46, 0xcd, 0xbb, 0x6b, 0xf5, 0x7e, 0xdd, 0x82, 0x4b, 0x28, 0x15, 0xd7, - 0xb1, 0xe5, 0xa1, 0xcc, 0x65, 0x32, 0x52, 0x85, 0xfd, 0xa0, 0x3c, 0xd2, 0xf3, 0xc7, 0x6f, 0x54, - 0xb3, 0x2a, 0x73, 0xeb, 0xa5, 0x57, 0xd0, 0xbc, 0xcb, 0x51, 0xf1, 0x2a, 0xa6, 0x75, 0x65, 0x10, - 0xc5, 0x1c, 0x72, 0x75, 0x70, 0xb6, 0xc6, 0x64, 0x02, 0xef, 0x1f, 0x59, 0x41, 0x1f, 0x47, 0xf1, - 0x39, 0x82, 0x50, 0xea, 0x1f, 0xcb, 0xa4, 0x72, 0x5c, 0x06, 0x26, 0x7a, 0x99, 0x4f, 0x74, 0x5c, - 0xc1, 0x6f, 0x53, 0xdf, 0xe9, 0x5a, 0x37, 0x12, 0x1b, 0xd0, 0x1b, 0xe7, 0xe9, 0x34, 0xe3, 0x6b, - 0x0a, 0x05, 0x88, 0x9b, 0x46, 0xdc, 0x05, 0x27, 0xe1, 0x30, 0x72, 0x69, 0x61, 0xff, 0x04, 0x16, - 0x11, 0x87, 0x7f, 0x8d, 0xe9, 0xbb, 0x61, 0xdf, 0xb6, 0xd9, 0xdf, 0x86, 0xf5, 0x20, 0x0c, 0xa3, - 0x32, 0x4a, 0x93, 0x20, 0xfe, 0x05, 0xa2, 0x74, 0xb9, 0x74, 0x06, 0xef, 0xed, 0xc2, 0xc2, 0x43, - 0x95, 0xec, 0x0a, 0xe8, 0x7e, 0x66, 0xf1, 0xd7, 0xe1, 0xf3, 0x93, 0x20, 0x0f, 0x39, 0x2b, 0xa6, - 0x6f, 0xc4, 0x1d, 0xa4, 0x87, 0xfa, 0x54, 0x4c, 0xdf, 0xde, 0xdf, 0x2f, 0xc0, 0x8a, 0x63, 0x75, - 0x67, 0x49, 0xdb, 0x70, 0xe9, 0x33, 0x84, 0x3e, 0xe6, 0x36, 0x61, 0xa4, 0xaf, 0x51, 0x34, 0x88, - 0x96, 0x99, 0x4b, 0xaa, 0xc2, 0xf3, 0x85, 0x9f, 0xd2, 0xac, 0x8b, 0xd4, 0x57, 0x77, 0xbd, 0xea, - 0xea, 0xee, 0x3d, 0x2a, 0xaa, 0x8d, 0xca, 0xb8, 0x16, 0xaa, 0x1d, 0x09, 0xb7, 0x0e, 0x88, 0x84, - 0x43, 0xb5, 0xa2, 0x17, 0xaf, 0x42, 0x57, 0x26, 0x27, 0x45, 0xad, 0x40, 0x53, 0xbb, 0x99, 0x23, - 0x12, 0x3a, 0x7a, 0xa9, 0xfb, 0x40, 0x2a, 0xc6, 0x0c, 0x7c, 0x0d, 0xa2, 0x6f, 0x93, 0xc8, 0x35, - 0x4b, 0xa3, 0xa4, 0xe4, 0xbb, 0x43, 0x0b, 0x23, 0xb6, 0xf4, 0x4d, 0x21, 0xd0, 0x28, 0xc3, 0x26, - 0xe9, 0xec, 0xdb, 0xc2, 0xb7, 0xab, 0x8b, 0xa1, 0x25, 0x27, 0xa4, 0x35, 0xec, 0xa8, 0xea, 0x8a, - 0x68, 0x0b, 0x7a, 0x94, 0x08, 0xf2, 0x2d, 0xe3, 0xb0, 0x6e, 0x62, 0xa6, 0x87, 0x22, 0x13, 0x3f, - 0x66, 0xeb, 0x5d, 0x99, 0xb1, 0x48, 0xfc, 0x63, 0x73, 0x7e, 0xaf, 0x76, 0xaf, 0xd8, 0xac, 0xd9, - 0xa6, 0xbb, 0x24, 0x55, 0xe6, 0x5f, 0x33, 0x65, 0xfe, 0x97, 0x01, 0x0e, 0xca, 0x34, 0x3b, 0x88, - 0xc6, 0x49, 0x10, 0x0f, 0x2f, 0xaa, 0x00, 0x50, 0x61, 0xc4, 0x4d, 0xe8, 0x4f, 0xc9, 0x2e, 0x8b, - 0xa1, 0xa0, 0xa1, 0x56, 0xf4, 0x50, 0x84, 0xf5, 0x75, 0x2b, 0x1d, 0x9a, 0xd3, 0x31, 0xbd, 0xa7, - 0xb8, 0xa4, 0xcc, 0x87, 0x41, 0xc7, 0x61, 0x6c, 0xb8, 0x0e, 0x03, 0x73, 0x32, 0x6b, 0xfd, 0x9f, - 0x27, 0x27, 0xfb, 0x3e, 0xe9, 0xdc, 0x5d, 0x58, 0x26, 0x65, 0x4a, 0xae, 0xa1, 0xe9, 0x0b, 0xb6, - 0x56, 0xe3, 0x05, 0x9b, 0x1b, 0x65, 0x0e, 0x95, 0x2b, 0xc0, 0xb5, 0x3b, 0xeb, 0xb1, 0x8c, 0x4c, - 0x46, 0x69, 0x18, 0x25, 0x63, 0xed, 0xad, 0x34, 0x8c, 0x32, 0x4e, 0xf3, 0x88, 0xb7, 0x17, 0x7e, - 0x2a, 0xeb, 0x4d, 0x4a, 0x99, 0xe8, 0x67, 0x19, 0x1a, 0xc4, 0x68, 0x5d, 0xd9, 0xd5, 0xe7, 0x19, - 0x3a, 0x0b, 0xe3, 0xd9, 0x5a, 0xcd, 0x77, 0xad, 0xed, 0x99, 0xbb, 0x56, 0x73, 0xef, 0xdb, 0x71, - 0xef, 0x7d, 0xbd, 0xdf, 0xb6, 0x00, 0x2a, 0xf6, 0xcf, 0x7b, 0xdb, 0x7a, 0x98, 0xe6, 0x93, 0xa0, - 0x34, 0x97, 0xc3, 0x04, 0x89, 0x37, 0x60, 0x21, 0x25, 0x31, 0xd9, 0xf7, 0x5f, 0x9d, 0xd9, 0x1d, - 0x6a, 0x16, 0x3e, 0x93, 0xa9, 0x5b, 0x66, 0xaa, 0x73, 0xf6, 0xf4, 0x2d, 0x33, 0x42, 0xde, 0x5f, - 0xb4, 0x94, 0x13, 0x33, 0x65, 0x13, 0xa4, 0x7c, 0x9c, 0x47, 0xe1, 0xd8, 0x54, 0x0b, 0x14, 0x44, - 0xc6, 0xac, 0x7d, 0x6e, 0x3b, 0xca, 0xe8, 0xcc, 0x7a, 0x48, 0x13, 0x61, 0xd1, 0x14, 0x84, 0x7a, - 0x9f, 0x04, 0x23, 0xd6, 0x30, 0x7e, 0xa2, 0xd6, 0xc6, 0x41, 0x29, 0x9f, 0x04, 0xfa, 0xf5, 0x82, - 0x06, 0xc9, 0x8d, 0x05, 0x19, 0xdf, 0x3d, 0xe2, 0xa7, 0xf7, 0x09, 0x08, 0x14, 0x47, 0x17, 0xf0, - 0x83, 0xd1, 0xb1, 0x4c, 0x42, 0xeb, 0x5e, 0xb3, 0xe5, 0xdc, 0x6b, 0xce, 0x79, 0x2c, 0xe5, 0xfd, - 0x4d, 0x0b, 0x96, 0x2c, 0x56, 0x74, 0xdb, 0xa9, 0x3e, 0x0d, 0x9b, 0x0a, 0xe1, 0x04, 0xf2, 0x76, - 0xed, 0xd1, 0xd4, 0xf9, 0x69, 0xc0, 0x1b, 0xd0, 0xc3, 0x71, 0x0b, 0x2e, 0xdd, 0x5f, 0xb3, 0x56, - 0xc3, 0x9d, 0x89, 0xaf, 0xe8, 0xbc, 0xbf, 0x6a, 0xc1, 0x32, 0x9e, 0x5d, 0xd2, 0xf1, 0x4e, 0x9a, - 0x1c, 0x46, 0x63, 0x53, 0x85, 0x6e, 0x59, 0x55, 0xe8, 0x77, 0x61, 0x61, 0x44, 0xad, 0x7c, 0x45, - 0xf1, 0x43, 0xeb, 0xd0, 0xa3, 0x3b, 0x6e, 0xa9, 0x7f, 0xd8, 0xef, 0x28, 0x72, 0xdc, 0xad, 0x16, - 0xfa, 0xb9, 0x76, 0xeb, 0x31, 0x2c, 0xe1, 0x8c, 0xee, 0x07, 0x59, 0x86, 0x66, 0x3d, 0x93, 0x27, - 0xb5, 0x6a, 0x87, 0x99, 0x99, 0x4c, 0x8b, 0x95, 0x67, 0x32, 0x2d, 0x5b, 0xb1, 0x9d, 0x5a, 0x86, - 0x94, 0xc0, 0x06, 0xd2, 0x4c, 0xd4, 0x60, 0x5f, 0x1e, 0x45, 0x25, 0x65, 0xa6, 0x18, 0xcb, 0xa9, - 0xa2, 0x9a, 0x04, 0x31, 0x97, 0x04, 0xf4, 0x63, 0x8a, 0x19, 0x3c, 0xd2, 0xca, 0xa7, 0x35, 0xda, - 0xb6, 0xa2, 0xad, 0xe3, 0xbd, 0xdf, 0x2c, 0x40, 0x1f, 0xd7, 0x64, 0x3f, 0x0d, 0x9b, 0xae, 0x60, - 0x51, 0x66, 0x3b, 0xf1, 0xd1, 0xb0, 0x59, 0x9c, 0x8e, 0xb5, 0x38, 0xdf, 0x35, 0x4e, 0xdf, 0xa9, - 0x1d, 0xa9, 0xed, 0xb8, 0xb6, 0x9f, 0x86, 0x8d, 0x71, 0xe4, 0x0d, 0x74, 0xea, 0xec, 0x1f, 0xfa, - 0x4e, 0xc5, 0xc4, 0xf6, 0xac, 0xbe, 0x21, 0x12, 0xaf, 0x40, 0x27, 0x4e, 0xc7, 0x54, 0x1e, 0xab, - 0x68, 0x6d, 0xb3, 0xf1, 0xb1, 0x1d, 0xa5, 0x0b, 0x13, 0xfd, 0xd2, 0x07, 0x3f, 0xc5, 0xdb, 0xce, - 0x1b, 0x0b, 0x70, 0xce, 0xda, 0x4e, 0xbc, 0x73, 0xde, 0x59, 0xbc, 0xa2, 0xc3, 0xae, 0x0a, 0xd5, - 0x33, 0x99, 0x1d, 0x47, 0xdb, 0xd7, 0xaa, 0x98, 0xae, 0xe2, 0x73, 0x43, 0xc6, 0x6a, 0x42, 0xf9, - 0xdb, 0x4e, 0xf9, 0x7d, 0x65, 0x46, 0x12, 0xe3, 0xb0, 0x9c, 0xea, 0xfb, 0x16, 0x2c, 0xf2, 0xbe, - 0xd4, 0xd1, 0x5a, 0xcc, 0xee, 0x45, 0xdf, 0xd0, 0x88, 0x2f, 0xe0, 0x72, 0xd6, 0x60, 0x81, 0x05, - 0x05, 0xed, 0xa5, 0x3b, 0x2f, 0x19, 0xd5, 0xcd, 0xd2, 0xf8, 0xcd, 0x3d, 0xc5, 0x3b, 0xb0, 0x6c, - 0x35, 0x14, 0xc3, 0x75, 0x47, 0x0c, 0x6b, 0x73, 0xf9, 0x0e, 0x1d, 0x26, 0x07, 0x61, 0x52, 0x28, - 0xb7, 0x5d, 0x0c, 0x2f, 0xaa, 0x0c, 0xaa, 0xc2, 0xa0, 0xff, 0x0a, 0x93, 0xe2, 0x40, 0x06, 0xf9, - 0xe8, 0x88, 0xd2, 0x83, 0x81, 0x5f, 0x21, 0xbe, 0x4f, 0x80, 0xf6, 0x61, 0x7d, 0x3f, 0x0d, 0xdd, - 0x83, 0xa0, 0x2a, 0x75, 0x1d, 0x64, 0x72, 0x54, 0x2b, 0x75, 0xb1, 0x99, 0xfa, 0xba, 0xb9, 0xf9, - 0x40, 0xee, 0xbd, 0x0a, 0x17, 0x2d, 0x9e, 0x7c, 0xa0, 0x6b, 0x2e, 0xb4, 0xdd, 0xa2, 0xe1, 0xdd, - 0x23, 0x62, 0x33, 0xe5, 0x07, 0xc4, 0xf4, 0x3b, 0x9f, 0x12, 0xff, 0xbd, 0x65, 0x57, 0x85, 0xd2, - 0x71, 0xf1, 0x4c, 0xa5, 0x0e, 0x15, 0x82, 0xe3, 0x38, 0x7d, 0x42, 0xdc, 0x16, 0x7d, 0x86, 0x70, - 0xbd, 0x4c, 0x55, 0xb1, 0xe0, 0xc3, 0x99, 0x85, 0x21, 0xa7, 0xa1, 0x0f, 0x67, 0xe8, 0x34, 0x82, - 0x28, 0x46, 0xc1, 0x8a, 0x28, 0x19, 0xe9, 0x20, 0xac, 0x00, 0x55, 0xbd, 0x08, 0xd3, 0xa9, 0xba, - 0x50, 0x59, 0xf4, 0x19, 0x62, 0xbc, 0xcc, 0x73, 0x7e, 0xbd, 0xc5, 0x90, 0xf7, 0xaa, 0x5d, 0x0d, - 0xa2, 0x79, 0xb0, 0x2e, 0xd6, 0xd5, 0xb6, 0xc7, 0x29, 0x2c, 0xd3, 0x0e, 0xc7, 0xe4, 0x6b, 0x97, - 0xde, 0x67, 0xcd, 0x79, 0x49, 0x5a, 0x15, 0x4f, 0xda, 0x4e, 0xf1, 0x64, 0x05, 0x96, 0xac, 0x82, - 0x90, 0xf7, 0x6d, 0x07, 0x96, 0x9d, 0x52, 0xcf, 0x2a, 0xb4, 0xcd, 0x0a, 0xb5, 0xf7, 0x76, 0x51, - 0x21, 0xce, 0xfb, 0x2c, 0x5c, 0x0f, 0xdb, 0x4b, 0x60, 0xc2, 0x80, 0x87, 0x9f, 0x82, 0x23, 0x28, - 0x43, 0xd6, 0x8b, 0xb2, 0xae, 0xf3, 0xa2, 0xec, 0x75, 0xe8, 0x87, 0x2c, 0x58, 0xcf, 0x29, 0xb8, - 0xd8, 0x33, 0xf2, 0x35, 0x0d, 0x3a, 0xe4, 0x30, 0x1d, 0x1d, 0xcb, 0xdc, 0x4f, 0xd3, 0xb2, 0x7a, - 0x04, 0xe9, 0x22, 0xc5, 0x16, 0x88, 0x28, 0x09, 0xe5, 0x53, 0x74, 0x05, 0x32, 0xdf, 0x0e, 0x43, - 0xaa, 0xc9, 0xab, 0x57, 0x91, 0x0d, 0x2d, 0xe2, 0x16, 0xac, 0xc9, 0xa7, 0x72, 0x34, 0xc5, 0x3d, - 0xa8, 0xc6, 0xe5, 0x97, 0x3d, 0x75, 0x34, 0x65, 0x80, 0x72, 0xf2, 0x80, 0x9e, 0x46, 0x0c, 0xa8, - 0x90, 0x6a, 0x60, 0xf5, 0x96, 0x2f, 0x2c, 0xe8, 0x96, 0xa1, 0xe3, 0xd3, 0x37, 0x72, 0x4e, 0x33, - 0x99, 0x07, 0xf4, 0xe8, 0x56, 0xd5, 0xb6, 0x97, 0x14, 0xe7, 0x1a, 0xda, 0x2c, 0xda, 0x72, 0xb5, - 0x68, 0x5e, 0x00, 0x17, 0xef, 0x3d, 0x95, 0x23, 0x77, 0xd7, 0x9e, 0x5f, 0x9c, 0xb4, 0x0e, 0x70, - 0x6d, 0xf7, 0x00, 0xc7, 0x91, 0xaa, 0x63, 0x22, 0x95, 0xf7, 0xfb, 0x20, 0xec, 0x21, 0x78, 0xd5, - 0xaf, 0xc0, 0x02, 0xce, 0xdc, 0xb0, 0x67, 0xc8, 0x7b, 0x0c, 0xeb, 0x48, 0x7d, 0x80, 0xc1, 0xef, - 0xd9, 0xe5, 0xa9, 0xb8, 0xb5, 0x6d, 0x6e, 0xb4, 0x51, 0xca, 0x30, 0x52, 0xef, 0xbb, 0x96, 0x7d, - 0x05, 0x78, 0xaf, 0xa9, 0x49, 0xf3, 0x18, 0x95, 0x40, 0xbc, 0x7b, 0x94, 0xdd, 0x33, 0xe4, 0x3d, - 0x84, 0x15, 0x24, 0x7e, 0x74, 0x7f, 0x7e, 0xed, 0xef, 0x6c, 0x8d, 0x34, 0xcb, 0xb0, 0x0b, 0xab, - 0x9a, 0xed, 0x7c, 0x01, 0x9c, 0x57, 0xe5, 0x6d, 0xf7, 0x55, 0xb9, 0x27, 0x79, 0x26, 0x74, 0xee, - 0xfb, 0xfe, 0xea, 0x42, 0x11, 0xd4, 0xa1, 0xb2, 0x43, 0x16, 0xc6, 0x90, 0xb7, 0xa1, 0x96, 0x50, - 0x0f, 0xa3, 0x04, 0xf6, 0x6e, 0x52, 0x81, 0xdd, 0x59, 0xa9, 0x66, 0x87, 0x2b, 0xc8, 0x35, 0x3b, - 0xea, 0xf6, 0x02, 0x58, 0xfa, 0x32, 0x88, 0xca, 0x67, 0xf3, 0x9d, 0xd7, 0x61, 0x90, 0xe5, 0xe9, - 0x48, 0x16, 0xc5, 0x9e, 0x7e, 0xc4, 0x57, 0x21, 0x50, 0xea, 0x24, 0xfd, 0x24, 0x48, 0xc6, 0x6c, - 0x75, 0x0c, 0x79, 0xb7, 0x61, 0x59, 0x0d, 0xc1, 0x0a, 0x9e, 0xf3, 0x3c, 0xdf, 0xbb, 0x07, 0x2b, - 0xdb, 0x65, 0x19, 0x8c, 0x8e, 0xee, 0xf3, 0xd3, 0xc8, 0xf3, 0x95, 0x28, 0xa0, 0x1b, 0x06, 0x65, - 0x40, 0xf2, 0x2c, 0xfb, 0xf4, 0xed, 0x7d, 0x0d, 0x57, 0x8c, 0x4b, 0x75, 0xf7, 0x94, 0x5d, 0xd0, - 0xb6, 0xe2, 0x61, 0x73, 0x52, 0xe4, 0x92, 0x9e, 0x11, 0x1b, 0xdf, 0x87, 0xab, 0x33, 0x63, 0xf1, - 0x4c, 0xcf, 0x15, 0xde, 0xbb, 0x6b, 0xf9, 0x7e, 0x67, 0x05, 0x7f, 0x04, 0xcb, 0x86, 0xee, 0x57, - 0x26, 0xb3, 0xb5, 0xfa, 0x86, 0xde, 0xd0, 0x9a, 0xa4, 0xbb, 0xa8, 0x99, 0xd5, 0xe2, 0x53, 0xb1, - 0x4f, 0xb3, 0xbd, 0x0d, 0xeb, 0x69, 0x1c, 0xee, 0x38, 0x17, 0x1a, 0x8a, 0xf5, 0x0c, 0x1e, 0x69, - 0x13, 0xf9, 0x64, 0xa7, 0xe1, 0xf2, 0x63, 0x06, 0xef, 0x5d, 0xb3, 0x94, 0xa0, 0x47, 0x64, 0x61, - 0xde, 0x77, 0x84, 0xb1, 0xd3, 0x82, 0x67, 0x98, 0xa3, 0xcb, 0xd7, 0xce, 0x14, 0xbc, 0x7f, 0x69, - 0x01, 0x6c, 0x4f, 0xcb, 0x23, 0x3e, 0x71, 0x6d, 0xc2, 0x22, 0x9e, 0xe9, 0xad, 0x70, 0x68, 0x60, - 0xf5, 0x1e, 0xb3, 0x28, 0x9e, 0xa4, 0x79, 0x58, 0xbd, 0xc7, 0x54, 0x30, 0xbd, 0x83, 0x9f, 0x96, - 0x47, 0xfa, 0x30, 0x80, 0xdf, 0xb8, 0xd0, 0x72, 0x52, 0x05, 0x7b, 0x05, 0x60, 0x44, 0x2a, 0x28, - 0x98, 0x04, 0x1c, 0x66, 0x54, 0xd4, 0x77, 0x91, 0xea, 0x20, 0x31, 0x8e, 0x8a, 0x32, 0x3f, 0x2d, - 0xd3, 0x63, 0x99, 0xe8, 0xb8, 0xe5, 0x20, 0xbd, 0x80, 0xef, 0x13, 0xf6, 0xa7, 0x71, 0x6c, 0x6d, - 0x5a, 0x55, 0x5a, 0x6c, 0xd9, 0xa5, 0x45, 0x3a, 0x53, 0xeb, 0xfa, 0x04, 0x7e, 0x8a, 0x57, 0x2c, - 0x89, 0xab, 0xa4, 0xbb, 0x52, 0x85, 0x9a, 0x84, 0x77, 0x93, 0x2f, 0x15, 0xd4, 0x10, 0x55, 0x7a, - 0x45, 0x9b, 0xa5, 0x65, 0x6d, 0x96, 0x5f, 0x19, 0x59, 0x8a, 0x23, 0xab, 0xa8, 0x9f, 0xcb, 0x2c, - 0xd5, 0x89, 0x05, 0x7e, 0xbf, 0x08, 0x49, 0x70, 0x80, 0x39, 0x92, 0x3c, 0x02, 0x41, 0x84, 0x33, - 0xd9, 0x63, 0x83, 0x5e, 0x36, 0xa0, 0x77, 0x98, 0xea, 0x0a, 0xcb, 0xa2, 0xaf, 0x00, 0xda, 0xa2, - 0xf9, 0x34, 0x91, 0xec, 0x82, 0x14, 0xe0, 0x6d, 0xc3, 0x12, 0xf1, 0xdd, 0x95, 0xb1, 0x2c, 0xa9, - 0x5a, 0x3b, 0x4d, 0xca, 0x60, 0x2c, 0xb5, 0xc9, 0x69, 0x10, 0x5b, 0x42, 0xa9, 0x1e, 0x1a, 0x70, - 0x41, 0x88, 0x41, 0x6f, 0x1b, 0x2e, 0x39, 0xa2, 0xf1, 0x2c, 0x6e, 0x9b, 0x24, 0xa8, 0xe5, 0x9c, - 0x0b, 0xac, 0xe1, 0x74, 0x62, 0xe4, 0xf9, 0x56, 0xbe, 0x7a, 0x50, 0xa6, 0xd9, 0x73, 0x85, 0x79, - 0xcc, 0x44, 0x31, 0x26, 0xa9, 0x5f, 0xc3, 0x68, 0xd0, 0xbb, 0xea, 0xf8, 0x0f, 0xe4, 0xc9, 0xbb, - 0x63, 0x1d, 0x56, 0xf9, 0x05, 0xb5, 0x4e, 0xf8, 0xfe, 0x08, 0xd6, 0x0c, 0x86, 0xa5, 0x1f, 0x42, - 0xff, 0x84, 0x1f, 0x5e, 0xb3, 0x22, 0x18, 0xac, 0xbd, 0xca, 0x6e, 0xd7, 0x5f, 0x65, 0x7b, 0xf7, - 0xe0, 0x12, 0x9f, 0xbe, 0x6a, 0x77, 0x56, 0xd5, 0x79, 0xad, 0x75, 0xfe, 0x79, 0xcd, 0xbb, 0x0d, - 0xc2, 0x61, 0x33, 0x2f, 0x7a, 0x7d, 0x05, 0x17, 0x99, 0x76, 0x3b, 0x0c, 0xe7, 0x27, 0x01, 0xb6, - 0x18, 0xed, 0x67, 0x10, 0x63, 0xc3, 0x88, 0x41, 0xac, 0x59, 0x85, 0xd5, 0x80, 0xbb, 0x32, 0xfe, - 0xff, 0x1a, 0x90, 0x58, 0xf3, 0x80, 0xbf, 0x84, 0x0d, 0xc6, 0x3e, 0xcc, 0x42, 0x2b, 0x66, 0xbd, - 0x98, 0x31, 0xaf, 0xc2, 0xe5, 0x1a, 0x77, 0x1e, 0x56, 0x3d, 0xa1, 0xb0, 0x2d, 0x72, 0xde, 0x13, - 0x0a, 0xdb, 0xca, 0x9e, 0xe3, 0xb4, 0xf6, 0xa1, 0xca, 0x3d, 0x9c, 0x04, 0xa9, 0x79, 0x5e, 0x55, - 0xf2, 0xd3, 0x76, 0x92, 0x9f, 0x4b, 0x74, 0x5c, 0x6c, 0xcc, 0x7d, 0xf6, 0x71, 0x88, 0x67, 0xc9, - 0x7d, 0x98, 0x90, 0x3b, 0xab, 0x53, 0xed, 0xc3, 0x24, 0x3b, 0xbf, 0xfb, 0x06, 0x08, 0x9b, 0x94, - 0x19, 0xfc, 0x6b, 0x8b, 0xb8, 0xaa, 0x93, 0xfa, 0xfc, 0x59, 0x6d, 0xc2, 0x62, 0x7a, 0x22, 0xf3, - 0x3c, 0x0a, 0xb5, 0xc7, 0x32, 0xb0, 0x78, 0xbf, 0xf6, 0xbb, 0x96, 0x1f, 0x5b, 0x15, 0x1e, 0x9b, - 0xf5, 0x8b, 0x7e, 0x99, 0xa1, 0x34, 0xaa, 0x87, 0xa8, 0x67, 0x93, 0xe5, 0xfc, 0x19, 0x79, 0x3f, - 0xd7, 0xd9, 0x64, 0x59, 0x58, 0x77, 0xea, 0x8b, 0x19, 0xe3, 0x6a, 0x8f, 0xd4, 0x0d, 0xa9, 0x21, - 0xc0, 0x03, 0xe9, 0x7e, 0x94, 0x8c, 0xb5, 0x7f, 0x7a, 0x13, 0x96, 0x15, 0x58, 0x25, 0x4f, 0x47, - 0xa7, 0x99, 0xcc, 0x2d, 0x76, 0x03, 0xdf, 0x46, 0x79, 0x47, 0x76, 0x02, 0xf4, 0x0c, 0x96, 0x75, - 0xfe, 0x0f, 0xfa, 0xce, 0x4a, 0xbc, 0xed, 0x34, 0xa4, 0x66, 0x81, 0xdf, 0xc0, 0xfa, 0x83, 0x07, - 0x5f, 0xf9, 0xb2, 0x88, 0xbe, 0x91, 0x2f, 0xe4, 0xa0, 0xf4, 0x24, 0x0a, 0x39, 0xa4, 0xf6, 0x7c, - 0x05, 0x50, 0xbd, 0x9c, 0x5e, 0x88, 0xf1, 0xcf, 0x98, 0x18, 0xc2, 0x05, 0xb4, 0xc6, 0x56, 0x02, - 0xdd, 0xf9, 0x9f, 0x0d, 0x18, 0xec, 0x4f, 0x1f, 0xc7, 0xd1, 0x68, 0x7b, 0x7f, 0x4f, 0xdc, 0xa5, - 0xdf, 0xe4, 0x50, 0x19, 0xf6, 0x72, 0xfd, 0x91, 0x0d, 0x09, 0xbb, 0x79, 0x65, 0xe6, 0xed, 0x8d, - 0x9a, 0xd8, 0x05, 0xf1, 0x21, 0xfd, 0xa6, 0x49, 0xe5, 0xb4, 0xe2, 0x6a, 0x45, 0xe6, 0x64, 0xd4, - 0x9b, 0xc3, 0xd9, 0x06, 0xc3, 0xe1, 0x6e, 0xf5, 0x8b, 0xa0, 0xcb, 0xb5, 0xc7, 0x55, 0xb3, 0xa3, - 0xdb, 0xd5, 0x08, 0x33, 0xba, 0x8a, 0xb7, 0xf6, 0xe8, 0x4e, 0x72, 0x60, 0x8f, 0x5e, 0xcb, 0x0f, - 0x2f, 0x88, 0x0f, 0xf4, 0xcf, 0x4f, 0xf2, 0x52, 0x5c, 0x71, 0xec, 0xd0, 0xe4, 0xd9, 0x9b, 0x57, - 0x67, 0xf0, 0x35, 0xe1, 0xd1, 0xdf, 0xd9, 0xc2, 0x5b, 0x7e, 0x72, 0xf3, 0x4a, 0x1d, 0x5d, 0x13, - 0x9e, 0xef, 0x01, 0xed, 0x31, 0x6c, 0x33, 0xb5, 0x85, 0xaf, 0x59, 0x95, 0x16, 0x9e, 0x1c, 0x96, - 0x2d, 0xbc, 0xed, 0xea, 0x6c, 0xe1, 0x5d, 0xcf, 0x76, 0x41, 0xec, 0x00, 0x54, 0x0e, 0x4b, 0x58, - 0x03, 0xb9, 0xee, 0x6e, 0xf3, 0x5a, 0x43, 0x8b, 0x61, 0xf2, 0x3e, 0x2c, 0xa8, 0xc3, 0xb1, 0xd0, - 0xe7, 0x23, 0xe7, 0x08, 0xbe, 0x79, 0xb9, 0x86, 0xd5, 0x1d, 0x6f, 0xb5, 0xde, 0x6c, 0x89, 0x4f, - 0xad, 0x5f, 0x01, 0x93, 0xfd, 0xbd, 0xd4, 0xfc, 0x8a, 0x49, 0xb1, 0xba, 0x7e, 0xc6, 0x13, 0x27, - 0x2d, 0xca, 0xa7, 0xf5, 0xdf, 0x14, 0xbf, 0xd4, 0xf8, 0x04, 0xe9, 0x2c, 0x6e, 0xb3, 0xb6, 0x65, - 0x1e, 0xdc, 0x98, 0xe5, 0xa9, 0x3f, 0xf0, 0x31, 0xcb, 0x33, 0xf3, 0x36, 0xc7, 0xbb, 0x20, 0xde, - 0x85, 0x05, 0xf5, 0x50, 0xc8, 0xa8, 0xc6, 0x79, 0x99, 0x64, 0x54, 0xe3, 0x3e, 0x3a, 0xa2, 0x85, - 0x59, 0x3e, 0x90, 0xa5, 0xf1, 0xbb, 0xb6, 0x71, 0x38, 0xce, 0xde, 0x36, 0x8e, 0x9a, 0x8b, 0xb6, - 0x2c, 0xbb, 0x2c, 0x6a, 0x96, 0x6d, 0xbc, 0x76, 0xcd, 0xb2, 0x4b, 0xbb, 0xfb, 0x67, 0xf6, 0xd2, - 0xa4, 0xe3, 0xa2, 0x61, 0x69, 0xaa, 0x7a, 0x6a, 0xc3, 0xd2, 0x58, 0x45, 0x4a, 0xef, 0xc2, 0x9b, - 0x2d, 0xe1, 0x5b, 0xcf, 0x55, 0xd9, 0x5d, 0xfc, 0xa0, 0xde, 0xc9, 0x75, 0x1a, 0x2f, 0x9f, 0xd5, - 0x6c, 0x64, 0xfc, 0x1c, 0x56, 0xdd, 0xd3, 0xad, 0xb8, 0xde, 0xf0, 0x43, 0xc5, 0x6a, 0x23, 0xff, - 0xe0, 0x8c, 0x56, 0xc3, 0xd0, 0x16, 0x52, 0x1d, 0x51, 0x67, 0x85, 0x74, 0x0e, 0xcb, 0xb3, 0x42, - 0xd6, 0x4e, 0xb6, 0x2e, 0x4f, 0xde, 0xec, 0xb3, 0x72, 0x38, 0x5b, 0xfe, 0xe5, 0xb3, 0x9a, 0x1b, - 0x2d, 0x9d, 0x9c, 0xcf, 0x4b, 0xb3, 0x33, 0xab, 0x5c, 0xd0, 0xf5, 0xe6, 0xc6, 0x33, 0x66, 0x4d, - 0xbe, 0xb4, 0x61, 0xd6, 0xb6, 0x47, 0x7d, 0xf9, 0xac, 0x66, 0xdb, 0xb7, 0x54, 0x95, 0x44, 0xe3, - 0x5b, 0x66, 0xea, 0x97, 0xc6, 0xb7, 0xcc, 0x96, 0x1d, 0xbd, 0x0b, 0x62, 0x17, 0x06, 0xa6, 0xf8, - 0x67, 0x36, 0x41, 0xbd, 0xe4, 0xb8, 0x39, 0x9c, 0x6d, 0x70, 0x9c, 0x0c, 0x8b, 0xc2, 0xba, 0x77, - 0xa8, 0x1d, 0xb5, 0x5f, 0x6b, 0x68, 0xb1, 0x1c, 0xfd, 0x82, 0x2a, 0x3a, 0x99, 0xbd, 0xec, 0xd4, - 0xa0, 0x36, 0x1b, 0xb1, 0x2c, 0xc0, 0x5b, 0xd0, 0xa5, 0xdf, 0x3d, 0x08, 0xeb, 0xbf, 0x5d, 0xd0, - 0x83, 0x5e, 0x72, 0x70, 0xb6, 0xf3, 0x31, 0x51, 0xdb, 0xcc, 0xbc, 0x9e, 0x43, 0x98, 0x99, 0xcf, - 0x04, 0x78, 0xef, 0x82, 0xf8, 0x18, 0x96, 0xac, 0x63, 0x93, 0xd0, 0x93, 0x9b, 0x3d, 0x4a, 0x6d, - 0x6e, 0x36, 0x35, 0xd9, 0x0b, 0x59, 0x9d, 0x7b, 0x8c, 0xf6, 0x66, 0x4e, 0x59, 0x9b, 0xd7, 0x1a, - 0x5a, 0x2c, 0x61, 0x56, 0xaa, 0xb3, 0x8c, 0xb4, 0x0c, 0x62, 0xe6, 0xf0, 0x54, 0xe7, 0x63, 0x9f, - 0x7d, 0xc8, 0xee, 0x9d, 0xf3, 0x89, 0xb1, 0xfb, 0xa6, 0x33, 0x91, 0xb1, 0xfb, 0xe6, 0x23, 0xcd, - 0x05, 0xf1, 0x11, 0x7b, 0xf8, 0xfd, 0x69, 0x1c, 0xbb, 0x1e, 0xde, 0x2a, 0xb9, 0xb8, 0x1e, 0xde, - 0x2e, 0x94, 0x90, 0x5b, 0xab, 0x78, 0x14, 0x47, 0x75, 0x1e, 0xa6, 0x54, 0x52, 0xe7, 0x51, 0x95, - 0x38, 0x88, 0xc7, 0xc7, 0x5c, 0x7a, 0xe0, 0xbd, 0x77, 0xcd, 0x26, 0x76, 0xf7, 0xdd, 0x66, 0x53, - 0x93, 0x99, 0xcf, 0x5b, 0xd0, 0xc5, 0xec, 0xd8, 0xd8, 0x99, 0x95, 0x39, 0x1b, 0x3b, 0xb3, 0xd3, - 0x67, 0xd5, 0x85, 0x22, 0xa5, 0xa9, 0x49, 0x58, 0x01, 0xf2, 0x92, 0x83, 0xb3, 0x53, 0x1e, 0xfd, - 0xbb, 0x6b, 0x13, 0xc0, 0x9c, 0x2a, 0x82, 0x49, 0x79, 0x6a, 0xa5, 0x04, 0xef, 0xc2, 0xe3, 0x05, - 0x7a, 0x07, 0xf0, 0xd3, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xb1, 0x78, 0x55, 0xc4, 0x2c, 0x47, - 0x00, 0x00, + // 5342 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x3c, 0x4d, 0x6f, 0x1c, 0x47, + 0x76, 0x9a, 0x2f, 0x0e, 0xe7, 0xf1, 0x53, 0x25, 0x8a, 0x1a, 0xd1, 0x5a, 0x5b, 0xdb, 0x1b, 0x5b, + 0xb2, 0x1c, 0xd3, 0xb6, 0xd6, 0xb1, 0xbd, 0x72, 0x8c, 0x35, 0x4d, 0xca, 0x6b, 0x22, 0xa2, 0x4d, + 0x37, 0x25, 0x19, 0x46, 0x16, 0xd8, 0xb4, 0xa6, 0x8b, 0xc3, 0x36, 0x7b, 0xba, 0x27, 0xdd, 0x3d, + 0x94, 0xe8, 0x5b, 0x72, 0x5a, 0xc0, 0x08, 0x72, 0x58, 0x20, 0xc8, 0xe6, 0x98, 0xe4, 0x10, 0xe4, + 0x9a, 0x53, 0x82, 0x5c, 0x72, 0xc9, 0x29, 0x7f, 0x24, 0xd9, 0x4b, 0x6e, 0xb9, 0x05, 0xc1, 0x7b, + 0xf5, 0xaa, 0xba, 0xaa, 0xa7, 0x39, 0xa4, 0x6c, 0xe5, 0x40, 0xa8, 0xdf, 0xab, 0x57, 0xaf, 0x5e, + 0xbd, 0xaa, 0x7a, 0x5f, 0x55, 0x23, 0x58, 0x28, 0x4e, 0xc7, 0x32, 0xdf, 0x1c, 0x67, 0x69, 0x91, + 0x8a, 0x0e, 0x01, 0xde, 0x6f, 0x1b, 0xb0, 0xb4, 0x9d, 0x26, 0x45, 0x10, 0x25, 0x32, 0xdb, 0x4f, + 0xb3, 0x42, 0x08, 0x68, 0x27, 0xc1, 0x48, 0xf6, 0x1b, 0x37, 0x1b, 0xb7, 0x7b, 0x3e, 0x7d, 0x8b, + 0x0d, 0x98, 0x3f, 0x4a, 0xf3, 0x02, 0xdb, 0xfb, 0xcd, 0x9b, 0x8d, 0xdb, 0x1d, 0xdf, 0xc0, 0xe2, + 0xf7, 0x60, 0x69, 0x60, 0x33, 0xe8, 0xb7, 0x88, 0xc0, 0x45, 0x22, 0x07, 0x1a, 0x77, 0x90, 0xc6, + 0xfd, 0x36, 0x71, 0x36, 0xb0, 0x58, 0x87, 0x39, 0xe4, 0xb6, 0xbb, 0xdf, 0xef, 0x50, 0x0b, 0x43, + 0xde, 0x07, 0xb0, 0x7c, 0x3f, 0x39, 0x89, 0xb2, 0x34, 0x19, 0xc9, 0xa4, 0x78, 0x1c, 0x64, 0x62, + 0x15, 0x5a, 0x32, 0x39, 0x61, 0xd1, 0xf0, 0x53, 0xac, 0x41, 0xe7, 0x24, 0x88, 0x27, 0x92, 0xc4, + 0xea, 0xf9, 0x0a, 0xf0, 0xfe, 0x18, 0x16, 0x1e, 0xa7, 0xf1, 0x64, 0x24, 0xf7, 0xd2, 0x49, 0x52, + 0x3f, 0xa5, 0x1b, 0xd0, 0x1b, 0x61, 0xe3, 0x7e, 0x50, 0x1c, 0x71, 0xe7, 0x12, 0x81, 0xe2, 0x66, + 0x32, 0x08, 0xbf, 0x48, 0xe2, 0x53, 0x9a, 0xcf, 0xbc, 0x6f, 0x60, 0xef, 0x16, 0x2c, 0x7d, 0x15, + 0x44, 0x45, 0x94, 0x0c, 0x0f, 0x8a, 0xa0, 0x98, 0xe4, 0x28, 0x7f, 0x26, 0x83, 0x3c, 0x4d, 0x78, + 0x00, 0x86, 0xbc, 0x37, 0x61, 0xc9, 0x9f, 0x24, 0x49, 0x49, 0x78, 0x03, 0x7a, 0x79, 0x11, 0x64, + 0x85, 0x0c, 0xb7, 0x0a, 0xa6, 0x2d, 0x11, 0xde, 0x5f, 0x37, 0x00, 0x1e, 0xca, 0x6c, 0xc4, 0xc4, + 0x1b, 0x30, 0x2f, 0x9f, 0x45, 0xc5, 0x76, 0x1a, 0x2a, 0xc1, 0x3b, 0xbe, 0x81, 0xad, 0x11, 0x9b, + 0xf6, 0x88, 0xa2, 0x0f, 0xdd, 0x91, 0xcc, 0xf3, 0x60, 0x28, 0x49, 0xea, 0x9e, 0xaf, 0x41, 0x77, + 0xe8, 0x76, 0x65, 0x68, 0xf1, 0x32, 0xc0, 0x61, 0x94, 0x44, 0xf9, 0x11, 0x35, 0xab, 0x55, 0xb0, + 0x30, 0xde, 0x7f, 0x37, 0x60, 0xc5, 0xec, 0x12, 0x96, 0xaf, 0x4e, 0xa9, 0x37, 0x61, 0xc1, 0x2c, + 0xfb, 0xee, 0x0e, 0x0b, 0x67, 0xa3, 0x70, 0xbd, 0xc6, 0x47, 0x41, 0xae, 0xe5, 0x53, 0x80, 0xd8, + 0x84, 0xee, 0x53, 0xa5, 0x52, 0x92, 0x6d, 0xe1, 0xee, 0xda, 0xa6, 0xda, 0xab, 0x8e, 0xa2, 0x7d, + 0x4d, 0x84, 0xf4, 0x99, 0xd2, 0x2c, 0x09, 0x5b, 0xd2, 0x3b, 0xfa, 0xf6, 0x35, 0x91, 0x78, 0x07, + 0xa0, 0x90, 0xd9, 0x28, 0x4a, 0x82, 0x42, 0x86, 0xfd, 0x39, 0xea, 0x72, 0x99, 0xbb, 0x94, 0x2a, + 0xf7, 0x2d, 0x22, 0xef, 0xef, 0xec, 0x83, 0xb1, 0x9b, 0x1c, 0xa6, 0x62, 0x13, 0x7a, 0x66, 0x26, + 0x34, 0xeb, 0x85, 0xbb, 0xab, 0xcc, 0xc3, 0x10, 0xfa, 0x25, 0x09, 0xaa, 0x7c, 0x90, 0xc9, 0x40, + 0xa9, 0x1c, 0x55, 0xd1, 0xf2, 0x4b, 0x04, 0x29, 0x22, 0x0d, 0x77, 0x77, 0x8c, 0x22, 0x10, 0x10, + 0x9b, 0x30, 0x97, 0x93, 0x2c, 0xac, 0x87, 0xf5, 0xea, 0x00, 0x2c, 0x29, 0x53, 0x79, 0x7f, 0xd9, + 0x86, 0x9e, 0x69, 0xfb, 0xfe, 0x4b, 0x12, 0x8d, 0xca, 0x2d, 0xa3, 0x00, 0xdc, 0x4a, 0xf4, 0xb1, + 0xbb, 0xc3, 0xdb, 0x45, 0x83, 0xe2, 0x36, 0xac, 0xd0, 0xe7, 0xfe, 0x24, 0x8e, 0xf7, 0xd3, 0x38, + 0x1a, 0x9c, 0xf2, 0x8e, 0xa9, 0xa2, 0x71, 0x5b, 0x3d, 0x4d, 0xb3, 0xe3, 0x28, 0x19, 0xee, 0x44, + 0x19, 0xa9, 0xbd, 0xe7, 0x5b, 0x18, 0x94, 0x77, 0x92, 0xcb, 0xac, 0xdf, 0x55, 0xf2, 0xe2, 0x37, + 0x1e, 0xf1, 0xa2, 0x38, 0xed, 0xcf, 0xd3, 0xa1, 0xc3, 0x4f, 0x3c, 0x08, 0x83, 0x74, 0x34, 0x0a, + 0x92, 0x30, 0xef, 0xf7, 0x6e, 0xb6, 0xd0, 0x74, 0x68, 0x18, 0x39, 0x04, 0xd9, 0x30, 0xef, 0x03, + 0xe1, 0xe9, 0x5b, 0xdc, 0x41, 0xcd, 0x66, 0x45, 0xde, 0x5f, 0xb8, 0xd9, 0xb2, 0xb6, 0x86, 0x63, + 0xe5, 0x7c, 0x45, 0x22, 0x6e, 0x29, 0x83, 0xb2, 0x48, 0x94, 0x57, 0x99, 0xd2, 0x35, 0x3a, 0xca, + 0xce, 0xbc, 0x07, 0x8b, 0x27, 0xa5, 0x45, 0xc9, 0xfb, 0x4b, 0xd4, 0x43, 0x70, 0x0f, 0xcb, 0xd8, + 0xf8, 0x0e, 0x9d, 0x78, 0x17, 0xe6, 0xe2, 0xe0, 0x89, 0x8c, 0xf3, 0xfe, 0x32, 0xf5, 0xb8, 0x51, + 0x95, 0x66, 0xf3, 0x01, 0x35, 0xdf, 0x4f, 0x8a, 0xec, 0xd4, 0x67, 0xda, 0x8d, 0x9f, 0xc1, 0x82, + 0x85, 0x46, 0x9d, 0x1c, 0xcb, 0x53, 0x6d, 0xf6, 0x8e, 0xe5, 0x69, 0xbd, 0xd9, 0xbb, 0xd7, 0xfc, + 0xa0, 0xe1, 0xfd, 0x73, 0x03, 0x56, 0xfc, 0x4f, 0x76, 0x94, 0x44, 0x07, 0xe9, 0x24, 0x1b, 0x90, + 0xf9, 0x1e, 0xa5, 0x49, 0x54, 0xa4, 0x59, 0xde, 0x6f, 0x28, 0x0d, 0x6a, 0xb8, 0x5c, 0xfd, 0xa6, + 0xbd, 0xfa, 0xeb, 0x30, 0x77, 0x98, 0x3f, 0x3c, 0x1d, 0xeb, 0x4d, 0xc1, 0x10, 0xea, 0x7b, 0x9c, + 0x1a, 0x13, 0x4e, 0xdf, 0x66, 0x15, 0x3b, 0xd6, 0x2a, 0xf6, 0xa1, 0x7b, 0x2c, 0x4f, 0x33, 0x3c, + 0xa0, 0x6a, 0xd9, 0x35, 0xe8, 0x58, 0xd6, 0x6e, 0xc5, 0xb2, 0x9e, 0x42, 0x6f, 0x3f, 0x0d, 0x95, + 0xe8, 0xb5, 0x9b, 0x79, 0x1d, 0xe6, 0x72, 0x9a, 0x92, 0xb6, 0x7b, 0x0a, 0x42, 0x7c, 0x98, 0x45, + 0x27, 0x32, 0xd3, 0xe2, 0x2a, 0x48, 0xdc, 0x86, 0x56, 0xf6, 0x24, 0xac, 0x9c, 0xa5, 0x8a, 0x76, + 0x7c, 0x24, 0xf1, 0xfe, 0xbc, 0x09, 0xdd, 0xfd, 0x34, 0x3c, 0x18, 0xcb, 0x81, 0xb8, 0x03, 0x5d, + 0xb5, 0x86, 0x4a, 0x5b, 0xe5, 0x31, 0x37, 0xc2, 0xf9, 0x9a, 0x40, 0xbc, 0x0d, 0x60, 0xce, 0x52, + 0xde, 0x6f, 0x3a, 0xe4, 0xa5, 0x55, 0xb0, 0x68, 0xc4, 0x5d, 0xb3, 0x23, 0x5a, 0x44, 0xbd, 0x51, + 0x32, 0xc7, 0xd1, 0xeb, 0xf6, 0x03, 0xea, 0xe2, 0x64, 0x30, 0x9e, 0xd0, 0x44, 0x3a, 0x3e, 0x7d, + 0xe3, 0x9c, 0x47, 0x72, 0x94, 0x66, 0xea, 0xf4, 0x75, 0x7c, 0x86, 0x7e, 0xc8, 0xde, 0xf9, 0xb3, + 0x26, 0x2d, 0x00, 0x1b, 0x78, 0x63, 0xaa, 0x1b, 0xb6, 0xa9, 0xb6, 0x5c, 0x4c, 0xd3, 0x75, 0x31, + 0xa5, 0x53, 0x6a, 0x39, 0x4e, 0xa9, 0x74, 0xef, 0x6d, 0xdb, 0xbd, 0x6b, 0x0b, 0x88, 0x5e, 0xbf, + 0xa5, 0x2d, 0xe0, 0xbe, 0x71, 0x54, 0x0f, 0xa3, 0x91, 0xe4, 0xbd, 0x53, 0x22, 0xc4, 0xc7, 0xb0, + 0x32, 0x70, 0x4d, 0x61, 0xbf, 0x4b, 0x5a, 0x3c, 0xcb, 0x50, 0x56, 0xc9, 0x4b, 0x57, 0x47, 0x03, + 0xcc, 0xdb, 0xae, 0x0e, 0x31, 0xde, 0x7f, 0x36, 0x68, 0x23, 0x90, 0xc5, 0x37, 0x36, 0xba, 0x61, + 0xdb, 0x68, 0x01, 0xed, 0xe3, 0x28, 0x09, 0x79, 0xfa, 0xf4, 0x8d, 0x5c, 0x83, 0x71, 0xf4, 0x58, + 0x66, 0x79, 0x64, 0xe6, 0x6f, 0x61, 0xc4, 0x32, 0x34, 0x4f, 0x46, 0x3c, 0xff, 0xe6, 0xc9, 0xc8, + 0xf5, 0x0d, 0x9d, 0xaa, 0x6f, 0xf0, 0xa0, 0x9d, 0x8f, 0xe5, 0x80, 0x1d, 0xd5, 0xb2, 0xbb, 0x41, + 0x7c, 0x6a, 0x13, 0xb7, 0x8d, 0xa7, 0xe8, 0x3a, 0xae, 0xc8, 0xac, 0x9f, 0xf6, 0x11, 0xb8, 0x62, + 0xe3, 0x34, 0xfc, 0x3c, 0x30, 0xd3, 0xd5, 0xa0, 0xf7, 0xb7, 0x4d, 0xe8, 0xed, 0x92, 0x55, 0xc7, + 0xd9, 0x2e, 0x43, 0x33, 0x0a, 0x79, 0xaa, 0xcd, 0x28, 0xa4, 0x90, 0x2d, 0xc8, 0x64, 0x52, 0x18, + 0xb7, 0x61, 0x60, 0x75, 0x8a, 0xc7, 0xe9, 0xc3, 0x60, 0xa8, 0xb6, 0x71, 0xcf, 0x37, 0x30, 0x7a, + 0x1c, 0xfc, 0xde, 0x89, 0x86, 0x32, 0x2f, 0xd0, 0x91, 0x61, 0xb3, 0x8d, 0x42, 0x89, 0x78, 0xb2, + 0x3c, 0x77, 0x0d, 0x62, 0xdf, 0x93, 0x28, 0x2b, 0x26, 0x41, 0x7c, 0x10, 0x7d, 0xab, 0xd6, 0xbf, + 0xe5, 0xdb, 0x28, 0xcb, 0xa0, 0x76, 0x1d, 0x83, 0x6a, 0xe6, 0xf1, 0xa2, 0x0d, 0xea, 0xbf, 0x35, + 0x61, 0x9e, 0x95, 0x9a, 0x8b, 0x1f, 0x43, 0x0b, 0xcf, 0xa1, 0xf2, 0xfe, 0x2b, 0x7a, 0xcf, 0x8d, + 0x27, 0xd4, 0xea, 0x63, 0x9b, 0xb8, 0x05, 0x9d, 0x27, 0x71, 0x3a, 0x38, 0x26, 0x4e, 0x65, 0x98, + 0xf1, 0x49, 0x7c, 0x1c, 0xa5, 0x8a, 0x4c, 0xb5, 0x8b, 0x3b, 0xe6, 0x00, 0xb7, 0x88, 0x52, 0x3b, + 0x93, 0x3d, 0x42, 0x2a, 0x52, 0xa6, 0x10, 0x6f, 0x42, 0x37, 0x91, 0x05, 0xba, 0x4e, 0x36, 0x66, + 0x57, 0x98, 0xf8, 0x73, 0x85, 0x55, 0xd4, 0x9a, 0x46, 0x6c, 0xe2, 0x26, 0x8f, 0x65, 0x7e, 0x9a, + 0x17, 0x72, 0x44, 0xe7, 0xab, 0xdc, 0x46, 0x9f, 0xe6, 0x8a, 0xd8, 0xa2, 0xc0, 0xed, 0x58, 0x44, + 0x23, 0x99, 0x17, 0xc1, 0x68, 0xcc, 0x4a, 0x2f, 0x11, 0xce, 0xa1, 0x53, 0x9d, 0xcf, 0x3a, 0x74, + 0xcc, 0xba, 0x4a, 0xee, 0x1d, 0xc0, 0xbc, 0x56, 0x92, 0x78, 0x15, 0x3a, 0x13, 0x32, 0x1f, 0x53, + 0x4a, 0x7c, 0x84, 0x68, 0x5f, 0xb5, 0xe2, 0x4e, 0x78, 0x90, 0x06, 0xe1, 0xd6, 0x89, 0xcc, 0xb4, + 0xad, 0xe9, 0xf8, 0x36, 0xca, 0x0b, 0x89, 0x29, 0x75, 0xc2, 0xe5, 0x2b, 0xd2, 0x22, 0x88, 0x89, + 0x69, 0xdb, 0x57, 0x00, 0x5a, 0x9e, 0xb1, 0xcc, 0xb6, 0xc7, 0x13, 0x32, 0xcc, 0x6d, 0x9f, 0x21, + 0xe3, 0xb1, 0x5a, 0x44, 0xac, 0x3c, 0x16, 0xba, 0x16, 0xa5, 0xae, 0x36, 0x61, 0x19, 0xf2, 0xfe, + 0xa3, 0x0d, 0x50, 0xae, 0x9d, 0xf8, 0x02, 0xae, 0x45, 0xe9, 0x81, 0xcc, 0x4e, 0xa2, 0x81, 0xfc, + 0xe4, 0xb4, 0x90, 0xb9, 0x2f, 0x07, 0x93, 0x2c, 0x8f, 0x4e, 0x24, 0xfb, 0x8a, 0xab, 0xd5, 0xf5, + 0x56, 0x1b, 0xf1, 0xac, 0x5e, 0xe2, 0x17, 0x70, 0xc5, 0x34, 0x85, 0x25, 0xb3, 0xe6, 0x2c, 0x66, + 0x75, 0x3d, 0xc4, 0x36, 0x5c, 0x8e, 0xd2, 0x2f, 0x27, 0x72, 0x62, 0xb3, 0x69, 0xcd, 0x62, 0x33, + 0x4d, 0x2f, 0xf6, 0x60, 0xdd, 0xf0, 0x46, 0x73, 0x58, 0x72, 0x6a, 0xcf, 0xe2, 0x74, 0x46, 0x27, + 0x35, 0x39, 0x8c, 0xe1, 0x5d, 0x5e, 0x9d, 0x73, 0x26, 0x37, 0xd5, 0x43, 0x4d, 0x6e, 0x4f, 0x66, + 0x43, 0x7b, 0x72, 0x73, 0xe7, 0x4c, 0xae, 0x42, 0x2f, 0x7e, 0x0e, 0x2b, 0x51, 0xea, 0x4a, 0xd2, + 0x9d, 0xc5, 0xa2, 0x4a, 0x2d, 0xb6, 0x60, 0x35, 0x97, 0x03, 0x0c, 0x9b, 0x4a, 0x0e, 0xf3, 0xb3, + 0x38, 0x4c, 0x91, 0x7b, 0xff, 0xd5, 0x80, 0x65, 0x97, 0xa8, 0x36, 0xd0, 0x11, 0xd0, 0x46, 0x86, + 0xda, 0xc7, 0xe0, 0xb7, 0x15, 0xfc, 0xb4, 0x9c, 0xe0, 0x67, 0x0d, 0x3a, 0xa3, 0xe0, 0x9b, 0x34, + 0xe3, 0x8d, 0xab, 0x00, 0xc2, 0x46, 0x49, 0xaa, 0xc2, 0x32, 0xc4, 0x22, 0x20, 0x7e, 0x0a, 0x6d, + 0xf4, 0x0a, 0xac, 0xba, 0x57, 0x6a, 0xa5, 0xde, 0x2c, 0xe5, 0x27, 0xe2, 0x8d, 0xf7, 0xa1, 0x57, + 0x4a, 0x7b, 0x8e, 0xe9, 0x6c, 0xdb, 0xa6, 0xf3, 0x77, 0x0d, 0x58, 0xb0, 0xac, 0x19, 0x52, 0x96, + 0x47, 0xbf, 0xad, 0x4f, 0x7a, 0x99, 0x25, 0x1c, 0xc8, 0x82, 0x99, 0x58, 0x18, 0xf4, 0x16, 0x87, + 0x41, 0x14, 0x0f, 0x92, 0x82, 0x0f, 0xac, 0x06, 0xc5, 0x27, 0x56, 0xe9, 0x61, 0x27, 0x28, 0x02, + 0xb6, 0x8d, 0x37, 0xa6, 0x0d, 0xa9, 0xfa, 0x44, 0x1a, 0xdf, 0xed, 0x22, 0x3e, 0x83, 0xd5, 0xa3, + 0x48, 0x66, 0x41, 0x36, 0x38, 0x8a, 0x06, 0x41, 0x4c, 0x6c, 0x3a, 0x17, 0x60, 0x33, 0xd5, 0xcb, + 0xfb, 0x12, 0xae, 0xd6, 0x92, 0x92, 0x03, 0x1e, 0x1e, 0x06, 0x93, 0xb8, 0xe0, 0x89, 0x6b, 0x10, + 0xa7, 0x3e, 0x1e, 0x8e, 0x82, 0x6f, 0x54, 0x23, 0x4f, 0xbd, 0xc4, 0x78, 0xdf, 0x35, 0x60, 0xd1, + 0xb6, 0xf0, 0xe2, 0x0f, 0x00, 0xa2, 0xa4, 0x90, 0xd9, 0x61, 0x30, 0x30, 0xd1, 0xa9, 0xde, 0x7b, + 0xbb, 0xba, 0x81, 0xed, 0x7b, 0x49, 0x28, 0x6e, 0x42, 0xab, 0x18, 0x8c, 0xd9, 0x23, 0x69, 0x47, + 0xf0, 0x70, 0x30, 0x46, 0x4a, 0x1f, 0x9b, 0x30, 0xe4, 0x28, 0x06, 0xe3, 0xf7, 0xd8, 0x15, 0x55, + 0x49, 0xa8, 0xcd, 0xfb, 0xa7, 0x26, 0x74, 0x19, 0x83, 0xe6, 0x19, 0xbd, 0xc3, 0x93, 0x98, 0x4a, + 0x04, 0x3c, 0x2f, 0x1b, 0x85, 0xb3, 0xce, 0x4f, 0x93, 0x03, 0x99, 0xe8, 0x89, 0x69, 0x90, 0x5b, + 0x7c, 0x39, 0x38, 0xd1, 0x0b, 0xca, 0x20, 0x86, 0x15, 0x87, 0x51, 0x82, 0xc7, 0xff, 0x1d, 0xde, + 0xcd, 0x06, 0xb6, 0xda, 0xee, 0xf2, 0x9e, 0x36, 0x30, 0xb6, 0xa1, 0xbb, 0x42, 0x80, 0xdc, 0x57, + 0xdb, 0x37, 0x30, 0x6e, 0xba, 0x41, 0x9c, 0xe6, 0x92, 0xe2, 0xa4, 0xb6, 0xaf, 0x00, 0x0a, 0xc0, + 0xf0, 0x83, 0xba, 0xcc, 0x53, 0x4b, 0x89, 0x40, 0x09, 0xe3, 0x20, 0x2f, 0xb6, 0x06, 0xc7, 0xfd, + 0x9e, 0x92, 0x90, 0x41, 0x3c, 0x84, 0x71, 0x94, 0x17, 0x32, 0xe9, 0x83, 0x72, 0x13, 0x0a, 0xa2, + 0x90, 0x26, 0x4e, 0x73, 0x4c, 0x78, 0x16, 0x54, 0x0f, 0x06, 0xbd, 0x5f, 0x37, 0x61, 0xd9, 0x5d, + 0x9a, 0xda, 0x13, 0xdf, 0x87, 0x6e, 0xf6, 0x8c, 0x7c, 0x83, 0x56, 0x17, 0x83, 0x28, 0x6a, 0xf6, + 0x6c, 0x3f, 0x18, 0x1c, 0xcb, 0x22, 0x67, 0x85, 0x95, 0x08, 0x8a, 0xc4, 0x9e, 0xdd, 0xcf, 0x32, + 0xcc, 0xed, 0x58, 0x65, 0x1a, 0x56, 0x3d, 0x77, 0xb2, 0x74, 0x3c, 0xe6, 0x48, 0x8b, 0x7a, 0x32, + 0x02, 0x47, 0x2c, 0x78, 0x44, 0xa5, 0x33, 0x0d, 0x52, 0x38, 0x60, 0x46, 0x54, 0x6a, 0x2b, 0x11, + 0xa4, 0x6c, 0x3d, 0xe2, 0x3c, 0x2b, 0xdb, 0x1a, 0xb1, 0x30, 0x23, 0xf6, 0x74, 0x4f, 0x46, 0x78, + 0xbf, 0x6b, 0x41, 0x97, 0xc3, 0x0f, 0x4a, 0xd9, 0x24, 0x7a, 0x0c, 0x5d, 0x34, 0x53, 0x10, 0x2e, + 0x57, 0x1c, 0x8d, 0x22, 0xbd, 0x69, 0x14, 0x50, 0x5a, 0x8e, 0x96, 0x6d, 0x39, 0x6e, 0x40, 0x2f, + 0x38, 0x09, 0xa2, 0x38, 0x78, 0x12, 0x4b, 0x9e, 0x7c, 0x89, 0x10, 0xaf, 0xc1, 0x32, 0x66, 0x96, + 0xf9, 0x76, 0x3a, 0x1a, 0xc7, 0xb2, 0x30, 0x2a, 0xa8, 0x60, 0x55, 0xbc, 0x1a, 0x84, 0xb9, 0x72, + 0x17, 0xac, 0x0b, 0x1b, 0x85, 0x14, 0xc6, 0x90, 0x07, 0x21, 0x6b, 0xc4, 0x46, 0xe9, 0xac, 0xd6, + 0xe4, 0x14, 0x6d, 0xdf, 0xc0, 0xe2, 0x36, 0xac, 0x3c, 0xcd, 0xa2, 0x42, 0x5a, 0x82, 0x28, 0xcd, + 0x54, 0xd1, 0xc2, 0x83, 0x45, 0x85, 0x62, 0x51, 0xd4, 0x16, 0x73, 0x70, 0x38, 0x2b, 0x1e, 0xf8, + 0xab, 0x2c, 0x2a, 0x70, 0x23, 0xaa, 0xfd, 0x56, 0xc1, 0xa2, 0x6e, 0xa8, 0x1f, 0x89, 0xb4, 0xa8, + 0x74, 0x63, 0x10, 0x38, 0x52, 0x94, 0xee, 0x26, 0xfb, 0x59, 0x3a, 0xcc, 0x64, 0x9e, 0xf7, 0x97, + 0xd4, 0x48, 0x36, 0x0e, 0x57, 0x48, 0x39, 0xc0, 0xfe, 0xb2, 0xda, 0xea, 0x0a, 0x42, 0x09, 0x9e, + 0xca, 0x68, 0x78, 0x54, 0xc8, 0x70, 0x57, 0xb5, 0xaf, 0x28, 0x09, 0x5c, 0xac, 0xf7, 0x0f, 0x4d, + 0xab, 0x68, 0xc8, 0xab, 0x5e, 0xa9, 0x46, 0x35, 0xa6, 0xab, 0x51, 0x1c, 0x61, 0x37, 0x2f, 0x12, + 0x61, 0xb7, 0x2e, 0x1c, 0x61, 0xb7, 0x9f, 0x27, 0xc2, 0xee, 0x3c, 0x77, 0x84, 0x3d, 0xf7, 0x7c, + 0x11, 0x76, 0xb7, 0x12, 0x61, 0x7b, 0xaf, 0xc1, 0x32, 0xe7, 0x9c, 0xbe, 0xfc, 0xd3, 0x89, 0xcc, + 0x8b, 0xfa, 0xd4, 0xd3, 0xfb, 0x10, 0x56, 0x0c, 0x5d, 0x3e, 0x4e, 0x93, 0x1c, 0x77, 0x17, 0xa6, + 0x73, 0x88, 0xe2, 0x80, 0xda, 0x4a, 0x17, 0x89, 0x50, 0x37, 0x7b, 0xf7, 0x68, 0x90, 0x07, 0x51, + 0x5e, 0xcc, 0x1c, 0x84, 0x8a, 0x0d, 0x23, 0x93, 0xf3, 0xd1, 0xb7, 0xf7, 0xbf, 0x0d, 0x58, 0x32, + 0x9d, 0x73, 0x74, 0x5d, 0xf5, 0x7d, 0xad, 0x5c, 0xb3, 0xe9, 0xe4, 0x9a, 0x86, 0x6b, 0xab, 0xe4, + 0x4a, 0x11, 0x4d, 0x59, 0xed, 0xec, 0x99, 0x8c, 0x75, 0x76, 0x76, 0xfc, 0x81, 0xc9, 0x00, 0x95, + 0xda, 0x6f, 0x96, 0x13, 0x2e, 0xe5, 0x7b, 0xd1, 0x59, 0xe0, 0x16, 0x69, 0x9e, 0xf9, 0x2b, 0xcd, + 0x6f, 0xd2, 0x5c, 0x11, 0xc5, 0x8e, 0x78, 0xad, 0x4e, 0x10, 0x5f, 0x13, 0x79, 0x1f, 0xc3, 0x9a, + 0x39, 0x0e, 0xdf, 0x6f, 0x15, 0xbe, 0x6b, 0xc0, 0x95, 0x0a, 0x0b, 0x5a, 0x8b, 0xf3, 0x4f, 0x95, + 0x7d, 0x49, 0x63, 0xad, 0x8e, 0x8b, 0x3c, 0xa3, 0x26, 0x7d, 0xc6, 0x2a, 0x79, 0x5f, 0xc3, 0xd5, + 0xaa, 0x30, 0x4a, 0x31, 0x1f, 0x5b, 0x83, 0x59, 0xea, 0xd9, 0xa8, 0x66, 0x8b, 0x96, 0x92, 0xdc, + 0x0e, 0xde, 0xbb, 0x96, 0xaa, 0xec, 0x53, 0x71, 0xa3, 0x5a, 0x82, 0xef, 0x59, 0x05, 0x77, 0xef, + 0xc0, 0x12, 0xc8, 0x39, 0x23, 0xf7, 0x2c, 0x81, 0xac, 0x93, 0x32, 0x55, 0x19, 0xa6, 0x4e, 0x2e, + 0xa9, 0xb7, 0x0f, 0x8b, 0x8f, 0xf7, 0x2c, 0x5d, 0xeb, 0x75, 0x69, 0x58, 0xfb, 0xd8, 0xe8, 0xad, + 0x59, 0xaf, 0xb7, 0x96, 0xa3, 0xb7, 0x9f, 0xc1, 0x92, 0xe6, 0xf8, 0xbc, 0x1b, 0xe0, 0x23, 0x58, + 0x36, 0xc2, 0xa8, 0xa9, 0xbd, 0x01, 0x73, 0x27, 0x23, 0x4b, 0xc9, 0xda, 0x6a, 0xd9, 0x32, 0xfb, + 0x4c, 0xe2, 0xfd, 0x12, 0x56, 0xa9, 0x4c, 0x62, 0x0f, 0x4e, 0xf5, 0xb0, 0xb8, 0x90, 0xd9, 0x56, + 0x36, 0xcc, 0x59, 0x02, 0x0b, 0x43, 0x95, 0x60, 0x82, 0x74, 0xc9, 0x55, 0x41, 0x78, 0x78, 0x82, + 0x38, 0xe6, 0xcb, 0x31, 0xfc, 0xf4, 0xb6, 0xe1, 0xb2, 0xc5, 0xdd, 0x1c, 0x92, 0x5e, 0xa4, 0x91, + 0x95, 0x6a, 0xaa, 0xa9, 0xd8, 0xf8, 0x25, 0x09, 0x5a, 0xb8, 0xc7, 0x7b, 0xdb, 0x74, 0xd6, 0xb5, + 0x84, 0xab, 0x65, 0xcd, 0xa5, 0xa3, 0x1c, 0x40, 0x59, 0xfa, 0x6c, 0xda, 0xa5, 0x4f, 0xef, 0x35, + 0x58, 0x2d, 0x3b, 0xb3, 0x00, 0x35, 0xeb, 0xe5, 0xbd, 0x8a, 0x83, 0xf8, 0x72, 0x94, 0x9e, 0x98, + 0x41, 0xea, 0xc8, 0xfe, 0x10, 0xd9, 0x69, 0xb2, 0x92, 0xdd, 0xa0, 0xbc, 0x91, 0xa3, 0x6f, 0x8a, + 0x30, 0x83, 0x49, 0x6e, 0xac, 0x06, 0x01, 0xde, 0x6f, 0x1a, 0x70, 0xf9, 0x51, 0x2e, 0xb3, 0xed, + 0xea, 0x3d, 0xa8, 0xb9, 0x49, 0x6d, 0x9c, 0x77, 0x93, 0xda, 0xac, 0xbb, 0x49, 0xa5, 0x60, 0x84, + 0x72, 0x6d, 0xeb, 0xb6, 0xd5, 0x46, 0xcd, 0xba, 0x6b, 0xf5, 0x7e, 0xdd, 0x80, 0x2b, 0x28, 0x15, + 0xd7, 0xb1, 0xe5, 0xa1, 0xcc, 0x64, 0x32, 0x50, 0x85, 0xfd, 0xa0, 0x38, 0xd2, 0xf3, 0xc7, 0x6f, + 0x54, 0xb3, 0x2a, 0x73, 0xeb, 0xa5, 0x57, 0xd0, 0xac, 0xcb, 0x51, 0xf1, 0x3a, 0x86, 0x75, 0x45, + 0x10, 0xc5, 0xec, 0x72, 0xb5, 0x73, 0xb6, 0xc6, 0x64, 0x02, 0xef, 0x1f, 0x59, 0x41, 0x9f, 0x46, + 0xf1, 0x39, 0x82, 0x50, 0xe8, 0x1f, 0xcb, 0xa4, 0x34, 0x5c, 0x06, 0x26, 0x7a, 0x99, 0x8d, 0xb4, + 0x5f, 0xc1, 0x6f, 0x53, 0xdf, 0x69, 0x5b, 0x37, 0x12, 0x6b, 0xd0, 0x19, 0x66, 0xe9, 0x64, 0xcc, + 0xd7, 0x14, 0x0a, 0x10, 0xb7, 0x8c, 0xb8, 0x73, 0x4e, 0xc0, 0x61, 0xe4, 0xd2, 0xc2, 0xfe, 0x09, + 0xcc, 0x23, 0x0e, 0xff, 0x6a, 0xc3, 0x77, 0xc3, 0xbe, 0x69, 0xb3, 0xbf, 0x03, 0xab, 0x41, 0x18, + 0x46, 0x45, 0x94, 0x26, 0x41, 0xfc, 0x0b, 0x44, 0xe9, 0x72, 0xe9, 0x14, 0xde, 0xdb, 0x81, 0xb9, + 0x47, 0x2a, 0xd8, 0x15, 0xd0, 0xfe, 0xdc, 0xe2, 0xaf, 0xdd, 0xe7, 0x67, 0x41, 0x16, 0x72, 0x54, + 0x4c, 0xdf, 0x88, 0x3b, 0x48, 0x0f, 0x75, 0x56, 0x4c, 0xdf, 0xde, 0xdf, 0xcf, 0xc1, 0x92, 0xb3, + 0xeb, 0xce, 0x92, 0xb6, 0xe6, 0xd2, 0xa7, 0x0f, 0x5d, 0x8c, 0x6d, 0xc2, 0x48, 0x5f, 0xa3, 0x68, + 0x10, 0x77, 0x66, 0x26, 0xa9, 0x0a, 0xcf, 0x17, 0x7e, 0x4a, 0xb3, 0x2e, 0x52, 0x5f, 0xdd, 0x75, + 0xca, 0xab, 0xbb, 0x0f, 0xa8, 0xa8, 0x36, 0x28, 0xe2, 0x8a, 0xab, 0x76, 0x24, 0xdc, 0x3c, 0x20, + 0x12, 0x76, 0xd5, 0x8a, 0x5e, 0xbc, 0x0e, 0x6d, 0x99, 0x9c, 0xe4, 0x95, 0x02, 0x4d, 0xe5, 0x66, + 0x8e, 0x48, 0x28, 0xf5, 0x52, 0xf7, 0x81, 0x54, 0x8c, 0xe9, 0xf9, 0x1a, 0x44, 0xdb, 0x26, 0x91, + 0xeb, 0x38, 0x8d, 0x92, 0x82, 0xef, 0x0e, 0x2d, 0x8c, 0xd8, 0xd4, 0x37, 0x85, 0x40, 0xa3, 0xf4, + 0xeb, 0xa4, 0xb3, 0x6f, 0x0b, 0xdf, 0x2d, 0x2f, 0x86, 0x16, 0x1c, 0x97, 0x56, 0x73, 0xa2, 0xca, + 0x2b, 0xa2, 0x4d, 0xe8, 0x50, 0x20, 0xc8, 0xb7, 0x8c, 0xfd, 0xea, 0x16, 0x33, 0x3d, 0x14, 0x99, + 0xf8, 0x09, 0xef, 0xde, 0xa5, 0xa9, 0x1d, 0x89, 0x7f, 0xbc, 0x9d, 0x3f, 0xa8, 0xdc, 0x2b, 0xd6, + 0x6b, 0xb6, 0xee, 0x2e, 0x49, 0x95, 0xf9, 0x57, 0x4c, 0x99, 0xff, 0x65, 0x80, 0x83, 0x22, 0x1d, + 0x1f, 0x44, 0xc3, 0x24, 0x88, 0xfb, 0x97, 0x95, 0x03, 0x28, 0x31, 0xe2, 0x16, 0x74, 0x27, 0xb4, + 0x2f, 0xf3, 0xbe, 0xa0, 0xa1, 0x96, 0xf4, 0x50, 0x84, 0xf5, 0x75, 0x2b, 0x25, 0xcd, 0xe9, 0x90, + 0xde, 0x53, 0x5c, 0x51, 0xdb, 0x87, 0x41, 0xc7, 0x60, 0xac, 0xb9, 0x06, 0x03, 0x63, 0x32, 0x6b, + 0xfd, 0x9f, 0x27, 0x26, 0xfb, 0x21, 0xe1, 0xdc, 0x3d, 0x58, 0x24, 0x65, 0x4a, 0xae, 0xa1, 0xe9, + 0x0b, 0xb6, 0x46, 0xed, 0x05, 0x9b, 0xeb, 0x65, 0x0e, 0x95, 0x29, 0xc0, 0xb5, 0x3b, 0xeb, 0xb1, + 0x8c, 0x4c, 0x06, 0x69, 0x18, 0x25, 0x43, 0x6d, 0xad, 0x34, 0x8c, 0x32, 0x4e, 0xb2, 0x88, 0x8f, + 0x17, 0x7e, 0xaa, 0xdd, 0x9b, 0x14, 0x32, 0xd1, 0xcf, 0x32, 0x34, 0x88, 0xde, 0xba, 0xdc, 0x57, + 0x5f, 0x8c, 0xd1, 0x58, 0x18, 0xcb, 0xd6, 0xa8, 0xbf, 0x6b, 0x6d, 0x4e, 0xdd, 0xb5, 0x9a, 0x7b, + 0xdf, 0x96, 0x7b, 0xef, 0xeb, 0xfd, 0xb6, 0x01, 0x50, 0xb2, 0x7f, 0xde, 0xdb, 0xd6, 0xc3, 0x34, + 0x1b, 0x05, 0x85, 0xb9, 0x1c, 0x26, 0x48, 0xbc, 0x05, 0x73, 0x29, 0x89, 0xc9, 0xb6, 0xff, 0xda, + 0xd4, 0xe9, 0x50, 0xb3, 0xf0, 0x99, 0x4c, 0xdd, 0x32, 0x53, 0x9d, 0xb3, 0xa3, 0x6f, 0x99, 0x11, + 0xf2, 0xfe, 0xa2, 0xa1, 0x8c, 0x98, 0x29, 0x9b, 0x20, 0xe5, 0x93, 0x2c, 0x0a, 0x87, 0xa6, 0x5a, + 0xa0, 0x20, 0xda, 0xcc, 0xda, 0xe6, 0x36, 0xa3, 0x31, 0xe5, 0xac, 0x87, 0x34, 0x11, 0x16, 0x4d, + 0x41, 0xa8, 0xf7, 0x51, 0x30, 0x60, 0x0d, 0xe3, 0x27, 0x6a, 0x6d, 0x18, 0x14, 0xf2, 0x69, 0xa0, + 0x5f, 0x2f, 0x68, 0x90, 0xcc, 0x58, 0x30, 0xe6, 0xbb, 0x47, 0xfc, 0xf4, 0x3e, 0x03, 0x81, 0xe2, + 0xe8, 0x02, 0x7e, 0x30, 0x38, 0x96, 0x49, 0x68, 0xdd, 0x6b, 0x36, 0x9c, 0x7b, 0xcd, 0x19, 0x8f, + 0xa5, 0xbc, 0xbf, 0x69, 0xc0, 0x82, 0xc5, 0x8a, 0x6e, 0x3b, 0xd5, 0xa7, 0x61, 0x53, 0x22, 0x1c, + 0x47, 0xde, 0xac, 0x3c, 0x9a, 0x3a, 0x3f, 0x0c, 0x78, 0x0b, 0x3a, 0x38, 0x6e, 0xce, 0xa5, 0xfb, + 0xeb, 0xd6, 0x6a, 0xb8, 0x33, 0xf1, 0x15, 0x9d, 0xf7, 0x57, 0x0d, 0x58, 0xc4, 0xdc, 0x25, 0x1d, + 0x6e, 0xa7, 0xc9, 0x61, 0x34, 0x34, 0x55, 0xe8, 0x86, 0x55, 0x85, 0x7e, 0x1f, 0xe6, 0x06, 0xd4, + 0xca, 0x57, 0x14, 0xaf, 0x58, 0x49, 0x8f, 0xee, 0xb8, 0xa9, 0xfe, 0x61, 0xbb, 0xa3, 0xc8, 0xf1, + 0xb4, 0x5a, 0xe8, 0xe7, 0x3a, 0xad, 0xc7, 0xb0, 0x80, 0x33, 0xda, 0x0b, 0xc6, 0x63, 0xdc, 0xd6, + 0x53, 0x71, 0x52, 0xa3, 0x92, 0xcc, 0x4c, 0x45, 0x5a, 0xac, 0x3c, 0x13, 0x69, 0xd9, 0x8a, 0x6d, + 0x55, 0x22, 0xa4, 0x04, 0xd6, 0x90, 0x66, 0xa4, 0x06, 0xfb, 0xea, 0x28, 0x2a, 0x28, 0x32, 0x45, + 0x5f, 0x4e, 0x15, 0xd5, 0x24, 0x88, 0xb9, 0x24, 0xa0, 0x1f, 0x53, 0x4c, 0xe1, 0x91, 0x56, 0x3e, + 0xab, 0xd0, 0x36, 0x15, 0x6d, 0x15, 0xef, 0xfd, 0x66, 0x0e, 0xba, 0xb8, 0x26, 0xfb, 0x69, 0x58, + 0x77, 0x05, 0x8b, 0x32, 0xdb, 0x81, 0x8f, 0x86, 0xcd, 0xe2, 0xb4, 0xac, 0xc5, 0xf9, 0xbe, 0x7e, + 0xfa, 0x6e, 0x25, 0xa5, 0xb6, 0xfd, 0xda, 0x7e, 0x1a, 0xd6, 0xfa, 0x91, 0xb7, 0xd0, 0xa8, 0xb3, + 0x7d, 0xe8, 0x3a, 0x15, 0x13, 0xdb, 0xb2, 0xfa, 0x86, 0x48, 0xbc, 0x0a, 0xad, 0x38, 0x1d, 0x52, + 0x79, 0xac, 0xa4, 0xb5, 0xb7, 0x8d, 0x8f, 0xed, 0x28, 0x5d, 0x98, 0xe8, 0x97, 0x3e, 0xf8, 0x29, + 0xde, 0x75, 0xde, 0x58, 0x80, 0x93, 0x6b, 0x3b, 0xfe, 0xce, 0x79, 0x67, 0xf1, 0xaa, 0x76, 0xbb, + 0xca, 0x55, 0x4f, 0x45, 0x76, 0xec, 0x6d, 0xdf, 0x28, 0x7d, 0xba, 0xf2, 0xcf, 0x35, 0x11, 0xab, + 0x71, 0xe5, 0xef, 0x3a, 0xe5, 0xf7, 0xa5, 0x29, 0x49, 0x8c, 0xc1, 0x72, 0xaa, 0xef, 0x9b, 0x30, + 0xcf, 0xe7, 0x52, 0x7b, 0x6b, 0x31, 0x7d, 0x16, 0x7d, 0x43, 0x23, 0xbe, 0x84, 0xab, 0xe3, 0x9a, + 0x1d, 0x98, 0x93, 0xd3, 0x5e, 0xb8, 0xfb, 0x92, 0x51, 0xdd, 0x34, 0x8d, 0x5f, 0xdf, 0x53, 0xbc, + 0x07, 0x8b, 0x56, 0x43, 0xde, 0x5f, 0x75, 0xc4, 0xb0, 0x0e, 0x97, 0xef, 0xd0, 0x61, 0x70, 0x10, + 0x26, 0xb9, 0x32, 0xdb, 0x79, 0xff, 0xb2, 0x8a, 0xa0, 0x4a, 0x0c, 0xda, 0xaf, 0x30, 0xc9, 0x0f, + 0x64, 0x90, 0x0d, 0x8e, 0x28, 0x3c, 0xe8, 0xf9, 0x25, 0xe2, 0x87, 0x38, 0x68, 0x1f, 0x56, 0xf7, + 0xd3, 0xd0, 0x4d, 0x04, 0x55, 0xa9, 0xeb, 0x60, 0x2c, 0x07, 0x95, 0x52, 0x17, 0x6f, 0x53, 0x5f, + 0x37, 0xd7, 0x27, 0xe4, 0xde, 0xeb, 0x70, 0xd9, 0xe2, 0xc9, 0x09, 0x5d, 0x7d, 0xa1, 0xed, 0x36, + 0x0d, 0xef, 0xa6, 0x88, 0xf5, 0x94, 0x1f, 0x11, 0xd3, 0xef, 0x9d, 0x25, 0xfe, 0x7b, 0xc3, 0xae, + 0x0a, 0xa5, 0xc3, 0xfc, 0x42, 0xa5, 0x0e, 0xe5, 0x82, 0xe3, 0x38, 0x7d, 0x4a, 0xdc, 0xe6, 0x7d, + 0x86, 0x70, 0xbd, 0x4c, 0x55, 0x31, 0xe7, 0xe4, 0xcc, 0xc2, 0x90, 0xd1, 0xd0, 0xc9, 0x19, 0x1a, + 0x8d, 0x20, 0x8a, 0x51, 0xb0, 0x3c, 0x4a, 0x06, 0xda, 0x09, 0x2b, 0x40, 0x55, 0x2f, 0xc2, 0x74, + 0xa2, 0x2e, 0x54, 0xe6, 0x7d, 0x86, 0x18, 0x2f, 0xb3, 0x8c, 0x5f, 0x6f, 0x31, 0xe4, 0xbd, 0x6e, + 0x57, 0x83, 0x68, 0x1e, 0xac, 0x8b, 0x55, 0x75, 0xec, 0x71, 0x0a, 0x8b, 0x74, 0xc2, 0x31, 0xf8, + 0xda, 0xa1, 0xf7, 0x59, 0x33, 0x5e, 0x92, 0x96, 0xc5, 0x93, 0xa6, 0x53, 0x3c, 0x59, 0x82, 0x05, + 0xab, 0x20, 0xe4, 0x7d, 0xd7, 0x82, 0x45, 0xa7, 0xd4, 0xb3, 0x0c, 0x4d, 0xb3, 0x42, 0xcd, 0xdd, + 0x1d, 0x54, 0x88, 0xf3, 0x3e, 0x0b, 0xd7, 0xc3, 0xb6, 0x12, 0x18, 0x30, 0x60, 0xf2, 0x93, 0xb3, + 0x07, 0x65, 0xc8, 0x7a, 0x51, 0xd6, 0x76, 0x5e, 0x94, 0xbd, 0x09, 0xdd, 0x90, 0x05, 0xeb, 0x38, + 0x05, 0x17, 0x7b, 0x46, 0xbe, 0xa6, 0x41, 0x83, 0x1c, 0xa6, 0x83, 0x63, 0x99, 0xf9, 0x69, 0x5a, + 0x94, 0x8f, 0x20, 0x5d, 0xa4, 0xd8, 0x04, 0x11, 0x25, 0xa1, 0x7c, 0x86, 0xa6, 0x40, 0x66, 0x5b, + 0x61, 0x48, 0x35, 0x79, 0xf5, 0x2a, 0xb2, 0xa6, 0x45, 0xdc, 0x86, 0x15, 0xf9, 0x4c, 0x0e, 0x26, + 0x78, 0x06, 0xd5, 0xb8, 0xfc, 0xb2, 0xa7, 0x8a, 0xa6, 0x08, 0x50, 0x8e, 0x1e, 0xd2, 0xd3, 0x88, + 0x1e, 0x15, 0x52, 0x0d, 0xac, 0xde, 0xf2, 0x85, 0x39, 0xdd, 0x32, 0xb4, 0x7c, 0xfa, 0x46, 0xce, + 0xe9, 0x58, 0x66, 0x01, 0x3d, 0xba, 0x55, 0xb5, 0xed, 0x05, 0xc5, 0xb9, 0x82, 0x36, 0x8b, 0xb6, + 0x58, 0x2e, 0x9a, 0x17, 0xc0, 0xe5, 0xfb, 0xcf, 0xe4, 0xc0, 0x3d, 0xb5, 0xe7, 0x17, 0x27, 0xad, + 0x04, 0xae, 0xe9, 0x26, 0x70, 0xec, 0xa9, 0x5a, 0xc6, 0x53, 0x79, 0xbf, 0x0f, 0xc2, 0x1e, 0x82, + 0x57, 0x7d, 0x1d, 0xe6, 0x70, 0xe6, 0x86, 0x3d, 0x43, 0xde, 0x13, 0x58, 0x45, 0xea, 0x03, 0x74, + 0x7e, 0x17, 0x97, 0xa7, 0xe4, 0xd6, 0xb4, 0xb9, 0xd1, 0x41, 0x29, 0xc2, 0x48, 0xbd, 0xef, 0x5a, + 0xf4, 0x15, 0xe0, 0xbd, 0xa1, 0x26, 0xcd, 0x63, 0x94, 0x02, 0xf1, 0xe9, 0x51, 0xfb, 0x9e, 0x21, + 0xef, 0x11, 0x2c, 0x21, 0xf1, 0xe3, 0xbd, 0xd9, 0xb5, 0xbf, 0xb3, 0x35, 0x52, 0x2f, 0xc3, 0x0e, + 0x2c, 0x6b, 0xb6, 0xb3, 0x05, 0x70, 0x5e, 0x95, 0x37, 0xdd, 0x57, 0xe5, 0x9e, 0xe4, 0x99, 0x50, + 0xde, 0xf7, 0xc3, 0xd5, 0x85, 0x22, 0xa8, 0xa4, 0xb2, 0x45, 0x3b, 0x8c, 0x21, 0x6f, 0x4d, 0x2d, + 0xa1, 0x1e, 0x46, 0x09, 0xec, 0xdd, 0xa2, 0x02, 0xbb, 0xb3, 0x52, 0xf5, 0x06, 0x57, 0x90, 0x69, + 0x76, 0xd4, 0xed, 0x05, 0xb0, 0xf0, 0x55, 0x10, 0x15, 0x17, 0xb3, 0x9d, 0x37, 0xa0, 0x37, 0xce, + 0xd2, 0x81, 0xcc, 0xf3, 0x5d, 0xfd, 0x88, 0xaf, 0x44, 0xa0, 0xd4, 0x49, 0xfa, 0x59, 0x90, 0x0c, + 0x79, 0xd7, 0x31, 0xe4, 0xdd, 0x81, 0x45, 0x35, 0x04, 0x2b, 0x78, 0xc6, 0xf3, 0x7c, 0xef, 0x3e, + 0x2c, 0x6d, 0x15, 0x45, 0x30, 0x38, 0xda, 0xe3, 0xa7, 0x91, 0xe7, 0x2b, 0x51, 0x40, 0x3b, 0x0c, + 0x8a, 0x80, 0xe4, 0x59, 0xf4, 0xe9, 0xdb, 0xfb, 0x06, 0xd6, 0x8d, 0x49, 0x75, 0xcf, 0x94, 0x5d, + 0xd0, 0xb6, 0xfc, 0x61, 0x7d, 0x50, 0xe4, 0x92, 0x9e, 0xe1, 0x1b, 0x3f, 0x84, 0x6b, 0x53, 0x63, + 0xf1, 0x4c, 0xcf, 0x15, 0xde, 0xbb, 0x67, 0xd9, 0x7e, 0x67, 0x05, 0x7f, 0x0c, 0x8b, 0x86, 0xee, + 0x57, 0x26, 0xb2, 0xb5, 0xfa, 0x86, 0x5e, 0xdf, 0x9a, 0xa4, 0xbb, 0xa8, 0x63, 0xab, 0xc5, 0xa7, + 0x62, 0x9f, 0x66, 0x7b, 0x07, 0x56, 0xd3, 0x38, 0xdc, 0x76, 0x2e, 0x34, 0x14, 0xeb, 0x29, 0x3c, + 0xd2, 0x26, 0xf2, 0xe9, 0x76, 0xcd, 0xe5, 0xc7, 0x14, 0xde, 0xbb, 0x6e, 0x29, 0x41, 0x8f, 0xc8, + 0xc2, 0x7c, 0xe8, 0x08, 0x63, 0x87, 0x05, 0x17, 0x98, 0xa3, 0xcb, 0xd7, 0x8e, 0x14, 0xbc, 0x7f, + 0x69, 0x00, 0x6c, 0x4d, 0x8a, 0x23, 0xce, 0xb8, 0x36, 0x60, 0x1e, 0x73, 0x7a, 0xcb, 0x1d, 0x1a, + 0x58, 0xbd, 0xc7, 0xcc, 0xf3, 0xa7, 0x69, 0x16, 0x96, 0xef, 0x31, 0x15, 0x4c, 0xef, 0xe0, 0x27, + 0xc5, 0x91, 0x4e, 0x06, 0xf0, 0x1b, 0x17, 0x5a, 0x8e, 0x4a, 0x67, 0xaf, 0x00, 0xf4, 0x48, 0x39, + 0x39, 0x93, 0x80, 0xdd, 0x8c, 0xf2, 0xfa, 0x2e, 0x52, 0x25, 0x12, 0xc3, 0x28, 0x2f, 0xb2, 0xd3, + 0x22, 0x3d, 0x96, 0x89, 0xf6, 0x5b, 0x0e, 0xd2, 0x0b, 0xf8, 0x3e, 0x61, 0x7f, 0x12, 0xc7, 0xd6, + 0xa1, 0x55, 0xa5, 0xc5, 0x86, 0x5d, 0x5a, 0xa4, 0x9c, 0x5a, 0xd7, 0x27, 0xf0, 0x53, 0xbc, 0x6a, + 0x49, 0x5c, 0x06, 0xdd, 0xa5, 0x2a, 0xd4, 0x24, 0xbc, 0x5b, 0x7c, 0xa9, 0xa0, 0x86, 0x28, 0xc3, + 0x2b, 0x3a, 0x2c, 0x0d, 0xeb, 0xb0, 0xfc, 0xca, 0xc8, 0x92, 0x1f, 0x59, 0x45, 0xfd, 0x4c, 0x8e, + 0x53, 0x1d, 0x58, 0xe0, 0xf7, 0x8b, 0x90, 0x04, 0x07, 0x98, 0x21, 0xc9, 0x63, 0x10, 0x44, 0x38, + 0x15, 0x3d, 0xd6, 0xe8, 0x65, 0x0d, 0x3a, 0x87, 0xa9, 0xae, 0xb0, 0xcc, 0xfb, 0x0a, 0xa0, 0x23, + 0x9a, 0x4d, 0x12, 0xc9, 0x26, 0x48, 0x01, 0xde, 0x16, 0x2c, 0x10, 0xdf, 0x1d, 0x19, 0xcb, 0x82, + 0xaa, 0xb5, 0x93, 0xa4, 0x08, 0x86, 0x52, 0x6f, 0x39, 0x0d, 0x62, 0x4b, 0x28, 0xd5, 0x43, 0x03, + 0x2e, 0x08, 0x31, 0xe8, 0x6d, 0xc1, 0x15, 0x47, 0x34, 0x9e, 0xc5, 0x1d, 0x13, 0x04, 0x35, 0x9c, + 0xbc, 0xc0, 0x1a, 0x4e, 0x07, 0x46, 0x9e, 0x6f, 0xc5, 0xab, 0x07, 0x45, 0x3a, 0x7e, 0x2e, 0x37, + 0x8f, 0x91, 0x28, 0xfa, 0x24, 0xf5, 0x6b, 0x18, 0x0d, 0x7a, 0xd7, 0x1c, 0xfb, 0x81, 0x3c, 0xf9, + 0x74, 0xac, 0xc2, 0x32, 0xbf, 0xa0, 0xd6, 0x01, 0xdf, 0x1f, 0xc1, 0x8a, 0xc1, 0xb0, 0xf4, 0x7d, + 0xe8, 0x9e, 0xf0, 0xc3, 0x6b, 0x56, 0x04, 0x83, 0x95, 0x57, 0xd9, 0xcd, 0xea, 0xab, 0x6c, 0xef, + 0x3e, 0x5c, 0xe1, 0xec, 0xab, 0x72, 0x67, 0x55, 0xe6, 0x6b, 0x8d, 0xf3, 0xf3, 0x35, 0xef, 0x0e, + 0x08, 0x87, 0xcd, 0x2c, 0xef, 0xf5, 0x35, 0x5c, 0x66, 0xda, 0xad, 0x30, 0x9c, 0x1d, 0x04, 0xd8, + 0x62, 0x34, 0x2f, 0x20, 0xc6, 0x9a, 0x11, 0x83, 0x58, 0xb3, 0x0a, 0xcb, 0x01, 0x77, 0x64, 0xfc, + 0xff, 0x35, 0x20, 0xb1, 0xe6, 0x01, 0x7f, 0x09, 0x6b, 0x8c, 0x7d, 0x34, 0x0e, 0x2d, 0x9f, 0xf5, + 0x62, 0xc6, 0xbc, 0x06, 0x57, 0x2b, 0xdc, 0x79, 0xd8, 0x4d, 0x58, 0xb7, 0xd2, 0xd8, 0xf3, 0x17, + 0xe2, 0x4b, 0xb8, 0x36, 0x45, 0xcf, 0xeb, 0xcf, 0xc9, 0xf2, 0x9e, 0x4e, 0x96, 0x1b, 0xb3, 0x93, + 0x65, 0x4d, 0xe7, 0x1d, 0x41, 0xdf, 0x6a, 0xdc, 0x4b, 0xc3, 0xe8, 0xf0, 0x74, 0xf6, 0xec, 0xab, + 0x23, 0x35, 0x2f, 0x38, 0xd2, 0x4b, 0x70, 0xbd, 0x66, 0x24, 0xd6, 0x84, 0x7a, 0x4c, 0x62, 0x9f, + 0xcd, 0x59, 0x8f, 0x49, 0xec, 0xf3, 0xf6, 0x1c, 0x79, 0xeb, 0xc7, 0x2a, 0x0a, 0x73, 0x42, 0xc5, + 0xfa, 0x39, 0x96, 0x61, 0x60, 0xd3, 0x09, 0x03, 0xaf, 0x50, 0xe2, 0x5c, 0x1b, 0x05, 0xee, 0xe3, + 0x10, 0x17, 0x89, 0x02, 0x99, 0x90, 0x3b, 0xab, 0xfc, 0xfe, 0x51, 0x32, 0x3e, 0xbf, 0xfb, 0x1a, + 0x08, 0x9b, 0x94, 0x19, 0xfc, 0x6b, 0x83, 0xb8, 0xaa, 0x9a, 0xc5, 0xec, 0x59, 0x6d, 0xc0, 0x7c, + 0x7a, 0x22, 0xb3, 0x2c, 0x0a, 0xb5, 0xed, 0x36, 0xb0, 0xf8, 0xb0, 0xf2, 0x0b, 0x9f, 0x9f, 0x58, + 0xb5, 0x2e, 0x9b, 0xf5, 0x8b, 0x7e, 0xa3, 0xa2, 0x34, 0xaa, 0x87, 0xa8, 0xc6, 0xd5, 0xc5, 0xec, + 0x19, 0x79, 0x3f, 0xd7, 0x71, 0x75, 0x91, 0x5b, 0xaf, 0x0b, 0xe6, 0xc7, 0x8c, 0xab, 0x3c, 0xd7, + 0x37, 0xa4, 0x86, 0x00, 0x53, 0xf3, 0x7d, 0xdc, 0xaa, 0x6c, 0xa9, 0xdf, 0x86, 0x45, 0x05, 0x96, + 0x61, 0xe4, 0xd1, 0xe9, 0x58, 0x66, 0x16, 0xbb, 0x9e, 0x6f, 0xa3, 0xbc, 0x23, 0x3b, 0x14, 0xbc, + 0xc0, 0xce, 0x3a, 0xff, 0xa7, 0x8d, 0x67, 0xa5, 0x20, 0x76, 0x40, 0x56, 0xd9, 0x81, 0xdf, 0xc2, + 0xea, 0xc3, 0x87, 0x5f, 0xfb, 0x32, 0x8f, 0xbe, 0x95, 0x2f, 0x24, 0x65, 0x7c, 0x1a, 0x85, 0x1c, + 0x5c, 0x74, 0x7c, 0x05, 0xd0, 0xcd, 0x01, 0xbd, 0x95, 0xe3, 0x1f, 0x74, 0x31, 0x84, 0x0b, 0x68, + 0x8d, 0xad, 0x04, 0xba, 0xfb, 0x3f, 0xeb, 0xd0, 0xdb, 0x9f, 0x3c, 0x89, 0xa3, 0xc1, 0xd6, 0xfe, + 0xae, 0xb8, 0x47, 0xbf, 0x4e, 0xa2, 0x82, 0xf4, 0xd5, 0xea, 0x73, 0x23, 0x12, 0x76, 0x63, 0x7d, + 0xea, 0x15, 0x92, 0x9a, 0xd8, 0x25, 0xf1, 0x31, 0xfd, 0xba, 0x4b, 0x45, 0xf7, 0xe2, 0x5a, 0x49, + 0xe6, 0xe4, 0x16, 0x1b, 0xfd, 0xe9, 0x06, 0xc3, 0xe1, 0x5e, 0xf9, 0xdb, 0xa8, 0xab, 0x95, 0x67, + 0x66, 0xd3, 0xa3, 0xdb, 0x75, 0x19, 0x33, 0xba, 0x8a, 0x3c, 0xec, 0xd1, 0x9d, 0x30, 0xc9, 0x1e, + 0xbd, 0x12, 0x29, 0x5f, 0x12, 0x1f, 0xe9, 0x1f, 0xe2, 0x64, 0x85, 0x58, 0x77, 0xf6, 0xa1, 0xc9, + 0x38, 0x36, 0xae, 0x4d, 0xe1, 0x2b, 0xc2, 0xa3, 0xbd, 0xb3, 0x85, 0xb7, 0xec, 0xe4, 0xc6, 0x7a, + 0x15, 0x5d, 0x11, 0x9e, 0x6f, 0x44, 0xed, 0x31, 0xec, 0x6d, 0x6a, 0x0b, 0x5f, 0xd9, 0x55, 0x5a, + 0x78, 0x32, 0x58, 0xb6, 0xf0, 0xb6, 0xa9, 0xb3, 0x85, 0x77, 0x2d, 0xdb, 0x25, 0xb1, 0x0d, 0x50, + 0x1a, 0x2c, 0x61, 0x0d, 0xe4, 0x9a, 0xbb, 0x8d, 0xeb, 0x35, 0x2d, 0x86, 0xc9, 0x87, 0x30, 0xa7, + 0xca, 0x04, 0x42, 0x67, 0x8a, 0x4e, 0x31, 0x62, 0xe3, 0x6a, 0x05, 0xab, 0x3b, 0xde, 0x6e, 0xbc, + 0xdd, 0x10, 0x0f, 0xac, 0xdf, 0x43, 0xd3, 0xfe, 0x7b, 0xa9, 0xfe, 0x3d, 0x97, 0x62, 0x75, 0xe3, + 0x8c, 0xc7, 0x5e, 0x5a, 0x94, 0x07, 0xd5, 0x5f, 0x57, 0xbf, 0x54, 0xfb, 0x18, 0xeb, 0x2c, 0x6e, + 0xd3, 0x7b, 0xcb, 0x3c, 0x3d, 0x32, 0xcb, 0x53, 0x7d, 0xea, 0x64, 0x96, 0x67, 0xea, 0x95, 0x92, + 0x77, 0x49, 0xbc, 0x0f, 0x73, 0xea, 0xc9, 0x94, 0x51, 0x8d, 0xf3, 0x46, 0xcb, 0xa8, 0xc6, 0x7d, + 0x7e, 0x45, 0x0b, 0xb3, 0x78, 0x20, 0x0b, 0x63, 0x77, 0xed, 0xcd, 0xe1, 0x18, 0x7b, 0x7b, 0x73, + 0x54, 0x4c, 0xb4, 0xb5, 0xb3, 0x8b, 0xbc, 0xb2, 0xb3, 0x8d, 0xd5, 0xae, 0xec, 0xec, 0xc2, 0xee, + 0xfe, 0xb9, 0xbd, 0x34, 0xe9, 0x30, 0xaf, 0x59, 0x9a, 0xb2, 0xb2, 0x5c, 0xb3, 0x34, 0x56, 0xb9, + 0xd6, 0xbb, 0xf4, 0x76, 0x43, 0xf8, 0xd6, 0xc3, 0x5d, 0x36, 0x17, 0x3f, 0xaa, 0x76, 0x72, 0x8d, + 0xc6, 0xcb, 0x67, 0x35, 0x1b, 0x19, 0xbf, 0x80, 0x65, 0x37, 0xcf, 0x17, 0x37, 0x6a, 0x7e, 0xb2, + 0x59, 0x1e, 0xe4, 0x1f, 0x9d, 0xd1, 0x6a, 0x18, 0xda, 0x42, 0xaa, 0x64, 0x7d, 0x5a, 0x48, 0xa7, + 0x6c, 0x30, 0x2d, 0x64, 0x25, 0xc7, 0x77, 0x79, 0xf2, 0x61, 0x9f, 0x96, 0xc3, 0x39, 0xf2, 0x2f, + 0x9f, 0xd5, 0x5c, 0xbb, 0xd3, 0xc9, 0xf8, 0xbc, 0x34, 0x3d, 0xb3, 0xd2, 0x04, 0xdd, 0xa8, 0x6f, + 0x3c, 0x63, 0xd6, 0x64, 0x4b, 0x6b, 0x66, 0x6d, 0x5b, 0xd4, 0x97, 0xcf, 0x6a, 0xb6, 0x6d, 0x4b, + 0x59, 0x53, 0x35, 0xb6, 0x65, 0xaa, 0x92, 0x6b, 0x6c, 0xcb, 0x74, 0x01, 0xd6, 0xbb, 0x24, 0x76, + 0xa0, 0x67, 0xca, 0xa0, 0xe6, 0x10, 0x54, 0x8b, 0xaf, 0x1b, 0xfd, 0xe9, 0x06, 0xc7, 0xc8, 0xb0, + 0x28, 0xac, 0x7b, 0x87, 0xda, 0x51, 0xfb, 0xf5, 0x9a, 0x16, 0xcb, 0xd0, 0xcf, 0xa9, 0xf2, 0x9b, + 0x39, 0xcb, 0x4e, 0x35, 0x6e, 0xa3, 0x16, 0xcb, 0x02, 0xbc, 0x03, 0x6d, 0xfa, 0x05, 0x88, 0xb0, + 0xfe, 0x03, 0x0a, 0x3d, 0xe8, 0x15, 0x07, 0x67, 0x1b, 0x1f, 0xe3, 0xb5, 0xcd, 0xcc, 0xab, 0x31, + 0x84, 0x99, 0xf9, 0x94, 0x83, 0xf7, 0x2e, 0x89, 0x4f, 0x61, 0xc1, 0x4a, 0x20, 0x85, 0x9e, 0xdc, + 0x74, 0x52, 0xb9, 0xb1, 0x51, 0xd7, 0x64, 0x2f, 0x64, 0x99, 0x01, 0x1a, 0xed, 0x4d, 0xe5, 0x9b, + 0x1b, 0xd7, 0x6b, 0x5a, 0x2c, 0x61, 0x96, 0xca, 0xac, 0x4e, 0x5a, 0x1b, 0x62, 0x2a, 0x8d, 0xac, + 0xf2, 0xb1, 0xb3, 0x40, 0xda, 0xf7, 0x4e, 0xa6, 0x66, 0xf6, 0x7d, 0x5d, 0x76, 0x68, 0xf6, 0x7d, + 0x7d, 0x72, 0x47, 0xfb, 0xbe, 0x92, 0xae, 0x99, 0x7d, 0x5f, 0x9f, 0xf6, 0x99, 0x7d, 0x7f, 0x46, + 0x96, 0xe7, 0x5d, 0x12, 0x8f, 0x30, 0x51, 0x32, 0x8d, 0xa8, 0xb2, 0x57, 0xa6, 0xfb, 0x38, 0x69, + 0xdc, 0xc6, 0xcd, 0xb3, 0x09, 0xce, 0x60, 0xbb, 0x23, 0xe3, 0x17, 0xc3, 0xf6, 0x13, 0xf6, 0x71, + 0xfb, 0x93, 0x38, 0x76, 0x7d, 0x9c, 0x55, 0x7e, 0x73, 0x7d, 0x9c, 0x5d, 0x34, 0x23, 0xc3, 0x5e, + 0xf2, 0xc8, 0x8f, 0xaa, 0x3c, 0x4c, 0xd9, 0xac, 0xca, 0xa3, 0x2c, 0x77, 0x11, 0x8f, 0x4f, 0xb9, + 0x0c, 0xc5, 0xd6, 0xe7, 0xba, 0x4d, 0xec, 0x5a, 0x9e, 0x8d, 0xba, 0x26, 0x33, 0x9f, 0x77, 0xa0, + 0x8d, 0xf9, 0x81, 0x39, 0x69, 0x56, 0xee, 0x60, 0x4e, 0x9a, 0x9d, 0x40, 0xa8, 0x2e, 0x14, 0x2b, + 0x98, 0xfa, 0x94, 0x15, 0x22, 0x5c, 0x71, 0x70, 0x76, 0xd0, 0xa7, 0x7f, 0x83, 0x6f, 0x5c, 0xb8, + 0x53, 0x51, 0x32, 0x41, 0x5f, 0xa5, 0xac, 0xe4, 0x5d, 0x7a, 0x32, 0x47, 0x6f, 0x42, 0x7e, 0xfa, + 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x8e, 0xe9, 0xe0, 0x4e, 0x38, 0x49, 0x00, 0x00, } diff --git a/types/types.proto b/types/types.proto index cede91c4..38d9ae54 100644 --- a/types/types.proto +++ b/types/types.proto @@ -726,6 +726,21 @@ message ServiceUpdateRequest { message ServiceUpdateResponse {} +message PortMappingListRequest { + string podID = 1; +} + +message PortMappingListResponse { + repeated PortMapping portMappings = 1; +} + +message PortMappingModifyRequest { + string podID = 1; + repeated PortMapping portMappings = 2; +} + +message PortMappingModifyResponse {} + message PodStopRequest { string podID = 1; } @@ -871,6 +886,13 @@ service PublicAPI { // ServiceUpdate updates an existing service of a pod rpc ServiceUpdate(ServiceUpdateRequest) returns (ServiceUpdateResponse){} + // PortMappingList get a list of PortMappings + rpc PortMappingList(PortMappingListRequest) returns (PortMappingListResponse) {} + // PortMappingAdd add a list of PortMapping rules to a Pod + rpc PortMappingAdd(PortMappingModifyRequest) returns (PortMappingModifyResponse) {} + // PortMappingDel remove a list of PortMapping rules from a Pod + rpc PortMappingDel(PortMappingModifyRequest) returns (PortMappingModifyResponse) {} + // ImagePull pulls a image from registry rpc ImagePull(ImagePullRequest) returns (stream ImagePullResponse) {} // ImagePush pushes a local image to registry From 0aba3c495dccb3e26400a7296df8086506fc4d31 Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Wed, 6 Sep 2017 19:14:47 +0800 Subject: [PATCH 07/14] add REST server for portmapping Signed-off-by: Wang Xu --- daemon/server.go | 55 +++++++++++++++++++++++++++++++++ errors/errors.go | 12 +++++++ server/router/pod/backend.go | 5 +++ server/router/pod/pod.go | 2 ++ server/router/pod/pod_routes.go | 33 ++++++++++++++++++++ 5 files changed, 107 insertions(+) diff --git a/daemon/server.go b/daemon/server.go index 850f6cd0..5cac0fba 100644 --- a/daemon/server.go +++ b/daemon/server.go @@ -12,6 +12,7 @@ import ( "github.com/docker/engine-api/types" "github.com/golang/glog" "github.com/hyperhq/hyperd/engine" + "github.com/hyperhq/hyperd/errors" "github.com/hyperhq/hyperd/lib/sysinfo" "github.com/hyperhq/hyperd/libmoby/distribution" apitypes "github.com/hyperhq/hyperd/types" @@ -297,6 +298,60 @@ func (daemon *Daemon) CmdCleanPod(podId string) (*engine.Env, error) { return v, nil } +// pod level port mappings API +func (daemon *Daemon) CmdListPortMappings(podId string) (*engine.Env, error) { + p, ok := daemon.PodList.Get(podId) + if !ok { + return nil, errors.ErrPodNotFound.WithArgs(podId) + } + + pms := p.ListPortMappings() + v := &engine.Env{} + v.SetJson("PortMappings", pms) + + return v, nil +} + +func (daemon *Daemon) CmdAddPortMappings(podId string, req []byte) (*engine.Env, error) { + p, ok := daemon.PodList.Get(podId) + if !ok { + return nil, errors.ErrPodNotFound.WithArgs(podId) + } + + var pms []*apitypes.PortMapping + err := json.Unmarshal(req, &pms) + if err != nil { + return nil, errors.ErrBadJsonFormat.WithArgs(err) + } + + err = p.AddPortMapping(pms) + if err != nil { + return nil, err + } + + return &engine.Env{}, nil +} + +func (daemon *Daemon) CmdDeletePortMappings(podId string, req []byte) (*engine.Env, error) { + p, ok := daemon.PodList.Get(podId) + if !ok { + return nil, errors.ErrPodNotFound.WithArgs(podId) + } + + var pms []*apitypes.PortMapping + err := json.Unmarshal(req, &pms) + if err != nil { + return nil, errors.ErrBadJsonFormat.WithArgs(err) + } + + err = p.RemovePortMappingByDest(pms) + if err != nil { + return nil, err + } + + return &engine.Env{}, nil +} + func (daemon *Daemon) CmdImageDelete(name string, force, prune bool) ([]*apitypes.ImageDelete, error) { list, err := daemon.Daemon.ImageDelete(name, force, prune) if err != nil { diff --git a/errors/errors.go b/errors/errors.go index 6646a48f..579f61d5 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -15,6 +15,18 @@ var ( HTTPStatusCode: http.StatusInternalServerError, }) + ErrPodNotFound = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "HYPER_POD_NOT_FOUND", + Message: "Pod %s not found", + HTTPStatusCode: http.StatusNotFound, + }) + + ErrBadJsonFormat = errcode.Register(errGroup, errcode.ErrorDescriptor{ + Value: "HYPER_BAD_JSON_FORMAT", + Message: "failed to parse json: %v", + HTTPStatusCode: http.StatusBadRequest, + }) + ErrSandboxNotExist = errcode.Register(errGroup, errcode.ErrorDescriptor{ Value: "HYPER_SANDBOX_NOT_EXIST", Message: "sandbox does not exist", diff --git a/server/router/pod/backend.go b/server/router/pod/backend.go index a0012538..afcc008f 100644 --- a/server/router/pod/backend.go +++ b/server/router/pod/backend.go @@ -18,4 +18,9 @@ type Backend interface { CmdStopPod(podId, stopVm string) (*engine.Env, error) CmdKillPod(podName, container string, signal int64) (*engine.Env, error) CmdCleanPod(podId string) (*engine.Env, error) + + //port mapping + CmdListPortMappings(podId string) (*engine.Env, error) + CmdAddPortMappings(podId string, pms []byte) (*engine.Env, error) + CmdDeletePortMappings(podId string, pms []byte) (*engine.Env, error) } diff --git a/server/router/pod/pod.go b/server/router/pod/pod.go index 739e1f26..b7067303 100644 --- a/server/router/pod/pod.go +++ b/server/router/pod/pod.go @@ -21,6 +21,7 @@ func NewRouter(b Backend) router.Router { // GET local.NewGetRoute("/pod/info", r.getPodInfo), local.NewGetRoute("/pod/stats", r.getPodStats), + local.NewGetRoute("/pod/{id}/portmappings", r.getPortMappings), local.NewGetRoute("/list", r.getList), // POST local.NewPostRoute("/pod/create", r.postPodCreate), @@ -31,6 +32,7 @@ func NewRouter(b Backend) router.Router { local.NewPostRoute("/pod/pause", r.postPodPause), local.NewPostRoute("/pod/unpause", r.postPodUnpause), // PUT + local.NewPostRoute("/pod/{id}/portmappings/{action}", r.putPortMappings), // DELETE local.NewDeleteRoute("/pod", r.deletePod), } diff --git a/server/router/pod/pod_routes.go b/server/router/pod/pod_routes.go index 884bb693..53fdab32 100644 --- a/server/router/pod/pod_routes.go +++ b/server/router/pod/pod_routes.go @@ -197,3 +197,36 @@ func (p *podRouter) deletePod(ctx context.Context, w http.ResponseWriter, r *htt return env.WriteJSON(w, http.StatusOK) } + +// port mappings +func (p *podRouter) getPortMappings(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + env, err := p.backend.CmdListPortMappings(vars["id"]) + if err != nil { + return err + } + + return env.WriteJSON(w, http.StatusOK) +} + +func (p *podRouter) putPortMappings(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + pms, _ := ioutil.ReadAll(r.Body) + switch vars["action"] { + case "add": + _, err := p.backend.CmdAddPortMappings(vars["id"], pms) + if err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) + case "delete": + _, err := p.backend.CmdDeletePortMappings(vars["id"], pms) + if err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("Only add or delete operation are permitted")) + return nil + } + return nil +} From 4359efa48fcfbeab5fa84dac368c2a415ffd34a6 Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Wed, 6 Sep 2017 19:44:35 +0800 Subject: [PATCH 08/14] add client api sdk for pod level portmapping Signed-off-by: Wang Xu --- client/api/portmapping.go | 62 +++++++++++++++++++++++++++++++++++++++ daemon/server.go | 2 +- 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 client/api/portmapping.go diff --git a/client/api/portmapping.go b/client/api/portmapping.go new file mode 100644 index 00000000..e816e0e8 --- /dev/null +++ b/client/api/portmapping.go @@ -0,0 +1,62 @@ +package api + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/hyperhq/hyperd/types" +) + +type PortMappingList struct { + PortMappings []*types.PortMapping `json:"portMappings"` +} + +func (c *Client) ListPortMappings(podId string) ([]*types.PortMapping, error) { + path := fmt.Sprintf("/pod/%s/portmappings", podId) + + body, code, err := readBody(c.call("GET", path, nil, nil)) + if code == http.StatusNotFound { + return nil, fmt.Errorf("pod %s not found", podId) + } else if err != nil { + return nil, err + } + + var pms PortMappingList + err = json.Unmarshal(body, &pms) + if err != nil { + return nil, err + } + + return pms.PortMappings, nil +} + +func (c *Client) AddPortMappings(podId string, pms []*types.PortMapping) error { + path := fmt.Sprintf("/pod/%s/portmappings/add", podId) + r, code, err := readBody(c.call("PUT", path, pms, nil)) + + if code == http.StatusNoContent || code == http.StatusOK { + return nil + } else if code == http.StatusNotFound { + return fmt.Errorf("pod %s not found", podId) + } else if err != nil { + return err + } else { + return fmt.Errorf("unexpect response code %d: %s", code, string(r)) + } +} + +func (c *Client) DeletePortMappings(podId string, pms []*types.PortMapping) error { + path := fmt.Sprintf("/pod/%s/portmappings/delete", podId) + r, code, err := readBody(c.call("PUT", path, pms, nil)) + + if code == http.StatusNoContent || code == http.StatusOK { + return nil + } else if code == http.StatusNotFound { + return fmt.Errorf("pod %s not found", podId) + } else if err != nil { + return err + } else { + return fmt.Errorf("unexpect response code %d: %s", code, string(r)) + } +} diff --git a/daemon/server.go b/daemon/server.go index 5cac0fba..af9465c0 100644 --- a/daemon/server.go +++ b/daemon/server.go @@ -307,7 +307,7 @@ func (daemon *Daemon) CmdListPortMappings(podId string) (*engine.Env, error) { pms := p.ListPortMappings() v := &engine.Env{} - v.SetJson("PortMappings", pms) + v.SetJson("portMappings", pms) return v, nil } From e89f28f09d15c6c6d531f83b1992ba3e878cb527 Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Wed, 6 Sep 2017 20:48:33 +0800 Subject: [PATCH 09/14] command lines for pod level portmapping 1. run/create now generate pod level portmapping rules 2. ports command (add|delete|ls) was added for portmapping operations Signed-off-by: Wang Xu --- client/api/interface.go | 5 +++ client/common_options.go | 44 +++++++++------------- client/help_darwin.go | 1 + client/help_linux.go | 1 + client/portmapping.go | 79 ++++++++++++++++++++++++++++++++++++++++ types/utils.go | 8 ++++ 6 files changed, 111 insertions(+), 27 deletions(-) create mode 100644 client/portmapping.go diff --git a/client/api/interface.go b/client/api/interface.go index b3d4f8ee..54427038 100644 --- a/client/api/interface.go +++ b/client/api/interface.go @@ -39,6 +39,11 @@ type APIInterface interface { UnpausePod(podId string) error KillPod(pod string, sig int) error + // PortMapping APIs + ListPortMappings(podId string) ([]*types.PortMapping, error) + AddPortMappings(podId string, pms []*types.PortMapping) error + DeletePortMappings(podId string, pms []*types.PortMapping) error + Build(name string, hasBody bool, body io.Reader) (io.ReadCloser, string, error) Commit(container, repo, author, message string, changes []string, pause bool) (string, error) Load(body io.Reader, name string, refs map[string]string) (io.ReadCloser, string, error) diff --git a/client/common_options.go b/client/common_options.go index 5141182c..170f068b 100644 --- a/client/common_options.go +++ b/client/common_options.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" "regexp" - "strconv" "strings" "github.com/docker/docker/pkg/namesgenerator" @@ -98,7 +97,7 @@ func (cli *HyperClient) JsonFromCmdline(container bool, cmdArgs, cmdEnvs, cmdPor image = cmdArgs[0] command = []string{} env = []*apitype.EnvironmentVar{} - ports = []*apitype.UserContainerPort{} + ports = []*apitype.PortMapping{} logOpts = make(map[string]string) labels = make(map[string]string) volumesRef = []*apitype.UserVolumeReference{} @@ -170,7 +169,6 @@ func (cli *HyperClient) JsonFromCmdline(container bool, cmdArgs, cmdEnvs, cmdPor Command: command, Workdir: cmdWorkdir, Entrypoint: entrypoints, - Ports: ports, Envs: env, Volumes: volumesRef, Files: []*apitype.UserFileReference{}, @@ -190,6 +188,7 @@ func (cli *HyperClient) JsonFromCmdline(container bool, cmdArgs, cmdEnvs, cmdPor Type: cmdLogDriver, Config: logOpts, }, + Portmappings: ports, } body = userPod } @@ -265,13 +264,11 @@ func parseVolume(volStr string) (*apitype.UserVolume, *apitype.UserVolumeReferen return &vol, &volRef, nil } -func parsePortMapping(portmap string) (*apitype.UserContainerPort, error) { +func parsePortMapping(portmap string) (*apitype.PortMapping, error) { var ( - port = apitype.UserContainerPort{} - proto string - hPort string - cPort string + tmp *apitype.PortMapping + port *apitype.PortMapping err error ) @@ -279,31 +276,24 @@ func parsePortMapping(portmap string) (*apitype.UserContainerPort, error) { if len(fields) < 2 { return nil, fmt.Errorf("flag needs host port and container port: --publish") } else if len(fields) == 2 { - proto = "tcp" - hPort = fields[0] - cPort = fields[1] + tmp = &apitype.PortMapping{ + Protocol: "tcp", + ContainerPort: fields[1], + HostPort: fields[0], + } } else { - proto = fields[0] - if proto != "tcp" && proto != "udp" { - return nil, fmt.Errorf("flag needs protocol(tcp or udp): --publish") + tmp = &apitype.PortMapping{ + Protocol: fields[0], + ContainerPort: fields[2], + HostPort: fields[1], } - hPort = fields[1] - cPort = fields[2] } - port.Protocol = proto - hp, err := strconv.Atoi(hPort) - port.HostPort = int32(hp) - if err != nil { - return nil, fmt.Errorf("flag needs host port and container port: --publish: %v", err) - } - cp, err := strconv.Atoi(cPort) - port.ContainerPort = int32(cp) + port, err = tmp.Formalize() if err != nil { - return nil, fmt.Errorf("flag needs host port and container port: --publish: %v", err) + return nil, err } - - return &port, nil + return port, nil } func imageToName(image string) string { diff --git a/client/help_darwin.go b/client/help_darwin.go index 8cd1adf5..0408f8a3 100644 --- a/client/help_darwin.go +++ b/client/help_darwin.go @@ -22,6 +22,7 @@ Command: login Register or log in to a Docker registry server logout Log out from a Docker registry server pause Pause a running pod + ports Show or modify port mapping rules pull Pull an image from a Docker registry server push Push an image or a repository to a Docker registry server rm Remove one or more pods or containers diff --git a/client/help_linux.go b/client/help_linux.go index 8cd1adf5..0408f8a3 100644 --- a/client/help_linux.go +++ b/client/help_linux.go @@ -22,6 +22,7 @@ Command: login Register or log in to a Docker registry server logout Log out from a Docker registry server pause Pause a running pod + ports Show or modify port mapping rules pull Pull an image from a Docker registry server push Push an image or a repository to a Docker registry server rm Remove one or more pods or containers diff --git a/client/portmapping.go b/client/portmapping.go new file mode 100644 index 00000000..92c7d6c7 --- /dev/null +++ b/client/portmapping.go @@ -0,0 +1,79 @@ +package client + +import ( + "errors" + "fmt" + "strings" + "text/tabwriter" + + "github.com/hyperhq/hyperd/types" + gflag "github.com/jessevdk/go-flags" +) + +func (cli *HyperClient) HyperCmdPorts(args ...string) error { + var opts struct { + Portmap []string `short:"p" long:"publish" value-name:"[]" default-mask:"-" description:"Publish a container's port to the host, format: -p|--publish [tcp/udp:]hostPort:containerPort (only valid for add and delete)"` + } + var parser = gflag.NewParser(&opts, gflag.Default|gflag.IgnoreUnknown|gflag.PassAfterNonOption) + if len(args) == 0 { + return errors.New("ports command support the follows subcommands: ls|add|delete") + } + cmd := args[0] + + parser.Usage = "ports ls\nports add|delete [OPTIONS] POD\n\nList or modify port mapping rules of a Pod" + args, err := parser.ParseArgs(args[1:]) + if err != nil { + if !strings.Contains(err.Error(), "Usage") { + return err + } else { + return nil + } + } + + if len(args) != 1 { + return errors.New("need a Pod Id as command parameter") + } + + var modFunc func(string, []*types.PortMapping) error + + switch cmd { + case "ls": + pms, err := cli.client.ListPortMappings(args[0]) + if err != nil { + return err + } + w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) + fmt.Fprintln(w, "Protocol\tHost Ports\tContainer Ports") + for _, pm := range pms { + fmt.Fprintf(w, "%s\t%s\t%s\n", pm.Protocol, pm.HostPort, pm.ContainerPort) + } + w.Flush() + return nil + case "add": + modFunc = cli.client.AddPortMappings + case "delete": + modFunc = cli.client.DeletePortMappings + default: + return errors.New("ports command support the follows subcommands: ls|add|delete") + + } + + if len(opts.Portmap) == 0 { + return errors.New("no rules to be add or delete") + } + + pms := make([]*types.PortMapping, 0, len(opts.Portmap)) + for _, o := range opts.Portmap { + pm, err := parsePortMapping(o) + if err != nil { + return fmt.Errorf("failed to parse rule %s: %v", o, err) + } + pms = append(pms, pm) + } + + err = modFunc(args[0], pms) + if err != nil { + return err + } + return nil +} diff --git a/types/utils.go b/types/utils.go index ddb9dc05..7ff44b52 100644 --- a/types/utils.go +++ b/types/utils.go @@ -226,6 +226,14 @@ func (pm *PortMapping) SameDestWith(other *PortMapping) bool { return pm.Protocol == other.Protocol && pm.ContainerPort == other.ContainerPort } +func (pm *PortMapping) Formalize() (*PortMapping, error) { + f, err := readPortMapping(pm) + if err != nil { + return nil, err + } + return f.toSpec(), nil +} + func (pm *_PortMapping) toSpec() *PortMapping { return &PortMapping{ ContainerPort: pm.container.String(), From c0737ded4cd9eeb2544827b1062d67d765a55e44 Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Wed, 20 Sep 2017 11:51:12 -0700 Subject: [PATCH 10/14] fix fmt Signed-off-by: Wang Xu --- client/common_options.go | 6 +++--- client/portmapping.go | 2 +- daemon/pod/persist.go | 6 +++--- daemon/pod/portmappings.go | 9 +++++++-- daemon/pod/provision.go | 1 - networking/portmapping/host_portmapping_linux.go | 2 +- serverrpc/portmapping.go | 1 - 7 files changed, 15 insertions(+), 12 deletions(-) diff --git a/client/common_options.go b/client/common_options.go index 170f068b..3233c1e2 100644 --- a/client/common_options.go +++ b/client/common_options.go @@ -267,9 +267,9 @@ func parseVolume(volStr string) (*apitype.UserVolume, *apitype.UserVolumeReferen func parsePortMapping(portmap string) (*apitype.PortMapping, error) { var ( - tmp *apitype.PortMapping - port *apitype.PortMapping - err error + tmp *apitype.PortMapping + port *apitype.PortMapping + err error ) fields := strings.Split(portmap, ":") diff --git a/client/portmapping.go b/client/portmapping.go index 92c7d6c7..53d7cb58 100644 --- a/client/portmapping.go +++ b/client/portmapping.go @@ -12,7 +12,7 @@ import ( func (cli *HyperClient) HyperCmdPorts(args ...string) error { var opts struct { - Portmap []string `short:"p" long:"publish" value-name:"[]" default-mask:"-" description:"Publish a container's port to the host, format: -p|--publish [tcp/udp:]hostPort:containerPort (only valid for add and delete)"` + Portmap []string `short:"p" long:"publish" value-name:"[]" default-mask:"-" description:"Publish a container's port to the host, format: -p|--publish [tcp/udp:]hostPort:containerPort (only valid for add and delete)"` } var parser = gflag.NewParser(&opts, gflag.Default|gflag.IgnoreUnknown|gflag.PassAfterNonOption) if len(args) == 0 { diff --git a/daemon/pod/persist.go b/daemon/pod/persist.go index c61c89f8..979d5947 100644 --- a/daemon/pod/persist.go +++ b/daemon/pod/persist.go @@ -326,9 +326,9 @@ func (p *XPod) removePodMetaFromDB() error { } func (p *XPod) savePortMapping() error { - pm := &types.PersistPortmappings { - Pod: p.Id(), - ContainerIP: p.containerIP, + pm := &types.PersistPortmappings{ + Pod: p.Id(), + ContainerIP: p.containerIP, PortMappings: p.portMappings, } return saveMessage(p.factory.db, fmt.Sprintf(PMAP_KEY_FMT, p.Id()), pm, p, "port mappings") diff --git a/daemon/pod/portmappings.go b/daemon/pod/portmappings.go index 187d2e18..61c4c1ab 100644 --- a/daemon/pod/portmappings.go +++ b/daemon/pod/portmappings.go @@ -29,7 +29,7 @@ func (p *XPod) initPortMapping() error { pms, err := translatePortMapping(p.portMappings) if err != nil { hlog.Log(ERROR, err) - return nil, err + return err } err = portmapping.SetupPortMaps(p.containerIP, pms) if err != nil { @@ -42,7 +42,12 @@ func (p *XPod) initPortMapping() error { func (p *XPod) flushPortMapping() error { if p.containerIP != "" && len(p.portMappings) > 0 { - err := portmapping.ReleasePortMaps(p.containerIP, p.portMappings) + pms, err := translatePortMapping(p.portMappings) + if err != nil { + hlog.Log(ERROR, err) + return err + } + err = portmapping.ReleasePortMaps(p.containerIP, pms) if err != nil { p.Log(ERROR, "release port mappings failed: %v", err) return err diff --git a/daemon/pod/provision.go b/daemon/pod/provision.go index 40221bc1..c71310a7 100644 --- a/daemon/pod/provision.go +++ b/daemon/pod/provision.go @@ -8,7 +8,6 @@ import ( "github.com/hyperhq/hypercontainer-utils/hlog" "github.com/hyperhq/hyperd/errors" - "github.com/hyperhq/hyperd/networking/portmapping" apitypes "github.com/hyperhq/hyperd/types" "github.com/hyperhq/hyperd/utils" runv "github.com/hyperhq/runv/api" diff --git a/networking/portmapping/host_portmapping_linux.go b/networking/portmapping/host_portmapping_linux.go index 39b1801c..af22c562 100644 --- a/networking/portmapping/host_portmapping_linux.go +++ b/networking/portmapping/host_portmapping_linux.go @@ -46,7 +46,7 @@ func NewPortRange(r string) (*PortRange, error) { return &PortRange{ Begin: int(b), End: int(e), - } + }, nil } // NewPortMapping generate a PortMapping from three strings: proto (tcp or udp, default is tcp), diff --git a/serverrpc/portmapping.go b/serverrpc/portmapping.go index 00ae883a..3c36b107 100644 --- a/serverrpc/portmapping.go +++ b/serverrpc/portmapping.go @@ -58,4 +58,3 @@ func (s *ServerRPC) PortMappingDel(ctx context.Context, req *types.PortMappingMo } return &types.PortMappingModifyResponse{}, nil } - From c98129eb0548822c7ccf68f8b2cf2c465d7c8711 Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Tue, 26 Sep 2017 12:59:37 -0600 Subject: [PATCH 11/14] rules fix for pod level portmapping 1. fix the iptables rules 2. improve the rollback mechanisms 3. don't merge continuous port mapping rules in spec 4. fix the API server Signed-off-by: Wang Xu --- Makefile.am | 2 +- .../portmapping/host_portmapping_linux.go | 61 +++++++++++++++---- server/router/pod/pod.go | 2 +- types/spec_test.go | 20 +++--- types/utils.go | 14 +---- 5 files changed, 62 insertions(+), 37 deletions(-) diff --git a/Makefile.am b/Makefile.am index 5f1d9b51..698fe50e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -26,7 +26,7 @@ if ON_DARWIN SUBDIRS=mac_installer endif -VERSION_PARAM=-ldflags "-X github.com/hyperhq/hyperd/utils.VERSION=$(VERSION) -X github.com/hyperhq/hyperd/utils.GITCOMMIT=`git describe`" +VERSION_PARAM=-ldflags "-X github.com/hyperhq/hyperd/utils.VERSION=$(VERSION) -X github.com/hyperhq/hyperd/utils.GITCOMMIT=`git describe --dirty --always --tags 2> /dev/null || true`" all-local: build-hyperd build-hyperctl build-vmlogd clean-local: diff --git a/networking/portmapping/host_portmapping_linux.go b/networking/portmapping/host_portmapping_linux.go index af22c562..9101efb1 100644 --- a/networking/portmapping/host_portmapping_linux.go +++ b/networking/portmapping/host_portmapping_linux.go @@ -78,6 +78,7 @@ func generateIptablesArgs(containerip string, m *PortMapping) ([]string, []strin proto string from string to string + dport string ) if strings.EqualFold(m.Protocol, "udp") { @@ -90,16 +91,18 @@ func generateIptablesArgs(containerip string, m *PortMapping) ([]string, []strin from = strconv.Itoa(m.FromPorts.Begin) m.FromPorts.End = m.FromPorts.Begin } else if m.FromPorts.End > m.FromPorts.Begin { - from = fmt.Sprintf("%d:%d", m.FromPorts, m.FromPorts.End) + from = fmt.Sprintf("%d:%d", m.FromPorts.Begin, m.FromPorts.End) } else { return []string{}, []string{}, fmt.Errorf("invalid from port range %d-%d", m.FromPorts.Begin, m.FromPorts.End) } if m.ToPorts.End == 0 || m.ToPorts.End == m.ToPorts.Begin { - to = net.JoinHostPort(containerip, strconv.Itoa(m.ToPorts.Begin)) + dport = strconv.Itoa(m.ToPorts.Begin) + to = net.JoinHostPort(containerip, dport) m.ToPorts.End = m.ToPorts.Begin } else if m.ToPorts.End > m.ToPorts.Begin { - to = net.JoinHostPort(containerip, fmt.Sprintf("%d-%d", m.ToPorts, m.ToPorts.End)) + dport = fmt.Sprintf("%d:%d", m.ToPorts.Begin, m.ToPorts.End) + to = net.JoinHostPort(containerip, fmt.Sprintf("%d-%d", m.ToPorts.Begin, m.ToPorts.End)) } else { return []string{}, []string{}, fmt.Errorf("invalid to port range %d-%d", m.ToPorts.Begin, m.ToPorts.End) } @@ -112,42 +115,74 @@ func generateIptablesArgs(containerip string, m *PortMapping) ([]string, []strin } natArgs := []string{"-p", proto, "-m", proto, "--dport", from, "-j", "DNAT", "--to-destination", to} - filterArgs := []string{"-d", containerip, "-p", proto, "-m", proto, "--dport", to, "-j", "ACCEPT"} + filterArgs := []string{"-d", containerip, "-p", proto, "-m", proto, "--dport", dport, "-j", "ACCEPT"} return natArgs, filterArgs, nil } +func parseRawResultOnHyper(output []byte, err error) error { + if err != nil { + return err + } else if len(output) != 0 { + return &iptables.ChainError{Chain: "HYPER", Output: output} + + } + return nil +} + func SetupPortMaps(containerip string, maps []*PortMapping) error { if disableIptables || len(maps) == 0 { return nil } + var ( + revert bool + revertRules = [][]string{} + ) + defer func() { + if revert { + hlog.Log(hlog.WARNING, "revert portmapping rules...") + for _, r := range revertRules { + hlog.Log(hlog.INFO, "revert rule: %v", r) + err := parseRawResultOnHyper(iptables.Raw(r...)) + if err != nil { + hlog.Log(hlog.ERROR, "failed to revert rule: %v", err) + err = nil //just ignore + } + } + } + }() + for _, m := range maps { natArgs, filterArgs, err := generateIptablesArgs(containerip, m) if err != nil { + revert = true return err } //check if this rule has already existed if iptables.PortMapExists("HYPER", natArgs) { - return nil + continue } if iptables.PortMapUsed("HYPER", m.Protocol, m.FromPorts.Begin, m.FromPorts.End) { + revert = true return fmt.Errorf("Host port %v has aleady been used", m.FromPorts) } - err = iptables.OperatePortMap(iptables.Insert, "HYPER", natArgs) + err = parseRawResultOnHyper(iptables.Raw(append([]string{"-t", "nat", "-I", "HYPER"}, natArgs...)...)) if err != nil { - return err + revert = true + return fmt.Errorf("Unable to setup NAT rule in HYPER chain: %s", err) } + revertRules = append(revertRules, append([]string{"-t", "nat", "-D", "HYPER"}, natArgs...)) - if output, err := iptables.Raw(append([]string{"-I", "HYPER"}, filterArgs...)...); err != nil { - return fmt.Errorf("Unable to setup forward rule in HYPER chain: %s", err) - } else if len(output) != 0 { - return &iptables.ChainError{Chain: "HYPER", Output: output} + if err = parseRawResultOnHyper(iptables.Raw(append([]string{"-I", "HYPER"}, filterArgs...)...)); err != nil { + revert = true + return fmt.Errorf("Unable to setup FILTER rule in HYPER chain: %s", err) } + revertRules = append(revertRules, append([]string{"-D", "HYPER"}, filterArgs...)) i := m.FromPorts.Begin j := m.ToPorts.Begin @@ -163,6 +198,7 @@ func SetupPortMaps(containerip string, maps []*PortMapping) error { for i <= m.FromPorts.End { if err = PortMapper.AllocateMap(m.Protocol, i, containerip, j); err != nil { + revert = true return err } i++ @@ -178,6 +214,7 @@ func ReleasePortMaps(containerip string, maps []*PortMapping) error { return nil } +release_loop: for _, m := range maps { if !strings.EqualFold(m.Protocol, "udp") { m.Protocol = "tcp" @@ -191,7 +228,7 @@ func ReleasePortMaps(containerip string, maps []*PortMapping) error { for i := m.FromPorts.Begin; i <= m.FromPorts.End; i++ { err := PortMapper.ReleaseMap(m.Protocol, i) if err != nil { - continue + continue release_loop } } diff --git a/server/router/pod/pod.go b/server/router/pod/pod.go index b7067303..0aa05d89 100644 --- a/server/router/pod/pod.go +++ b/server/router/pod/pod.go @@ -32,7 +32,7 @@ func NewRouter(b Backend) router.Router { local.NewPostRoute("/pod/pause", r.postPodPause), local.NewPostRoute("/pod/unpause", r.postPodUnpause), // PUT - local.NewPostRoute("/pod/{id}/portmappings/{action}", r.putPortMappings), + local.NewPutRoute("/pod/{id}/portmappings/{action}", r.putPortMappings), // DELETE local.NewDeleteRoute("/pod", r.deletePod), } diff --git a/types/spec_test.go b/types/spec_test.go index 01e3be00..a0af820b 100644 --- a/types/spec_test.go +++ b/types/spec_test.go @@ -217,7 +217,7 @@ func TestMergePorts(t *testing.T) { t.Log("> testing empty port mapping list") pms = []*_PortMapping{} - res, err = mergePorts(pms) + res, err = mergeContinuousPorts(pms) if err != nil { t.Fatalf("failed with emport pm list: %v", err) } @@ -241,7 +241,7 @@ func TestMergePorts(t *testing.T) { }, } t.Logf("---[debug] input items %v", pms) - res, err = mergePorts(pms) + res, err = mergeContinuousPorts(pms) if err != nil { t.Fatalf("error with normal pm list: %v", err) } @@ -266,7 +266,7 @@ func TestMergePorts(t *testing.T) { }, } t.Logf("---[debug] input items %v", pms) - res, err = mergePorts(pms) + res, err = mergeContinuousPorts(pms) if err != nil { t.Fatalf("error with normal pm list: %v", err) } @@ -290,7 +290,7 @@ func TestMergePorts(t *testing.T) { }, } t.Logf("---[debug] input items %v", pms) - res, err = mergePorts(pms) + res, err = mergeContinuousPorts(pms) if err != nil { t.Fatalf("error with normal pm list: %v", err) } @@ -315,7 +315,7 @@ func TestMergePorts(t *testing.T) { }, } t.Logf("---[debug] input items %v", pms) - res, err = mergePorts(pms) + res, err = mergeContinuousPorts(pms) if err == nil { t.Fatalf("failed with overlapped pm list: %v", res) } @@ -336,7 +336,7 @@ func TestMergePorts(t *testing.T) { }, } t.Logf("---[debug] input items %v", pms) - res, err = mergePorts(pms) + res, err = mergeContinuousPorts(pms) if err == nil { t.Fatalf("failed with overlapped pm list: %v", res) } @@ -357,7 +357,7 @@ func TestMergePorts(t *testing.T) { }, } t.Logf("---[debug] input items %v", pms) - res, err = mergePorts(pms) + res, err = mergeContinuousPorts(pms) if err == nil { t.Fatalf("failed with overlapped pm list: %v", res) } @@ -379,7 +379,7 @@ func TestMergePorts(t *testing.T) { }, } t.Logf("---[debug] input items %v", pms) - res, err = mergePorts(pms) + res, err = mergeContinuousPorts(pms) if err != nil { t.Fatalf("error with random pm list: %v", err) } @@ -403,7 +403,7 @@ func TestMergePorts(t *testing.T) { }, } t.Logf("---[debug] input items %v", pms) - res, err = mergePorts(pms) + res, err = mergeContinuousPorts(pms) if err == nil { t.Fatalf("didn't found collision in random pm list: %v", res) } @@ -428,7 +428,7 @@ func TestMergePorts(t *testing.T) { }, } t.Logf("---[debug] input items %v", pms) - res, err = mergePorts(pms) + res, err = mergeContinuousPorts(pms) if err != nil { t.Logf("got an error with random pm list: %v", err) t.Log("it's acceptable to fail in this case") diff --git a/types/utils.go b/types/utils.go index 7ff44b52..cb4be4ea 100644 --- a/types/utils.go +++ b/types/utils.go @@ -250,7 +250,7 @@ func (pm *_PortMapping) notDetermined() bool { return !pm.container.isRange() && pm.host.isRange() } -func mergePorts(pms []*_PortMapping) ([]*_PortMapping, error) { +func mergeContinuousPorts(pms []*_PortMapping) ([]*_PortMapping, error) { var ( results = []*_PortMapping{} occupy = map[int]bool{} @@ -364,18 +364,6 @@ func (p *UserPod) MergePortmappings() error { } } - hlog.Log(hlog.TRACE, "TCP ports to be merged: %v", tcpPorts) - tcpPorts, err = mergePorts(tcpPorts) - if err != nil { - return err - } - - hlog.Log(hlog.TRACE, "UDP ports to be merged: %v", udpPorts) - udpPorts, err = mergePorts(udpPorts) - if err != nil { - return err - } - pms := []*PortMapping{} for _, pm := range tcpPorts { pms = append(pms, pm.toSpec()) From ae9c2219d5bd2f6f11a264fc478c1e084b23e422 Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Tue, 26 Sep 2017 14:48:54 -0600 Subject: [PATCH 12/14] update example, add an example pod for portmapping examples/port-mapping.pod ``` { "id": "port-mapping-test", "containers" : [{ "name": "pmtest", "image": "busybox:latest", "command": ["/bin/nc", "-l", "-p", "1300"] }], "resource": { "vcpu": 1, "memory": 128 }, "portmappings": [{ "containerport": "1300-1310", "hostport": "3000-3010", "protocol": "tcp" }] } ``` Signed-off-by: Wang Xu --- examples/file-mapping.pod | 2 +- examples/port-mapping.pod | 18 ++++++++++++++++++ examples/with-volume.pod | 4 ++-- 3 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 examples/port-mapping.pod diff --git a/examples/file-mapping.pod b/examples/file-mapping.pod index 368a0d7b..d649b357 100644 --- a/examples/file-mapping.pod +++ b/examples/file-mapping.pod @@ -17,7 +17,7 @@ "volumes": [{ "name": "resolv.conf", "source": "/etc/resolv.conf", - "driver": "vfs" + "format": "vfs" }], "tty": true } diff --git a/examples/port-mapping.pod b/examples/port-mapping.pod new file mode 100644 index 00000000..c5a0dfd3 --- /dev/null +++ b/examples/port-mapping.pod @@ -0,0 +1,18 @@ +{ + "id": "port-mapping-test", + "containers" : [{ + "name": "pmtest", + "image": "busybox:latest", + "command": ["/bin/nc", "-l", "-p", "1300"] + }], + "resource": { + "vcpu": 1, + "memory": 128 + }, + "portmappings": [{ + "containerport": "1300-1310", + "hostport": "3000-3010", + "protocol": "tcp" + }] +} + diff --git a/examples/with-volume.pod b/examples/with-volume.pod index 114cd860..40aaaccb 100644 --- a/examples/with-volume.pod +++ b/examples/with-volume.pod @@ -36,11 +36,11 @@ "volumes": [{ "name": "log", "source": "", - "driver": "" + "format": "" },{ "name": "tmp", "source": "/tmp", - "driver": "vfs" + "format": "vfs" }], "tty": true } From a530ca13d55553940a4b434c2cee70e5bd54ff34 Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Tue, 26 Sep 2017 15:17:22 -0600 Subject: [PATCH 13/14] improve the command line usage printing for ports command Signed-off-by: Wang Xu --- client/portmapping.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/client/portmapping.go b/client/portmapping.go index 53d7cb58..fd7be39c 100644 --- a/client/portmapping.go +++ b/client/portmapping.go @@ -15,12 +15,14 @@ func (cli *HyperClient) HyperCmdPorts(args ...string) error { Portmap []string `short:"p" long:"publish" value-name:"[]" default-mask:"-" description:"Publish a container's port to the host, format: -p|--publish [tcp/udp:]hostPort:containerPort (only valid for add and delete)"` } var parser = gflag.NewParser(&opts, gflag.Default|gflag.IgnoreUnknown|gflag.PassAfterNonOption) + parser.Usage = "ports ls|add|delete [OPTIONS] POD\n\nList or modify port mapping rules of a Pod\n" + if len(args) == 0 { - return errors.New("ports command support the follows subcommands: ls|add|delete") + parser.WriteHelp(cli.err) + return nil } cmd := args[0] - parser.Usage = "ports ls\nports add|delete [OPTIONS] POD\n\nList or modify port mapping rules of a Pod" args, err := parser.ParseArgs(args[1:]) if err != nil { if !strings.Contains(err.Error(), "Usage") { @@ -30,14 +32,13 @@ func (cli *HyperClient) HyperCmdPorts(args ...string) error { } } - if len(args) != 1 { - return errors.New("need a Pod Id as command parameter") - } - var modFunc func(string, []*types.PortMapping) error switch cmd { case "ls": + if len(args) != 1 { + return errors.New("need a Pod Id as command parameter") + } pms, err := cli.client.ListPortMappings(args[0]) if err != nil { return err @@ -54,10 +55,13 @@ func (cli *HyperClient) HyperCmdPorts(args ...string) error { case "delete": modFunc = cli.client.DeletePortMappings default: - return errors.New("ports command support the follows subcommands: ls|add|delete") - + parser.WriteHelp(cli.err) + return nil } + if len(args) != 1 { + return errors.New("need a Pod Id as command parameter") + } if len(opts.Portmap) == 0 { return errors.New("no rules to be add or delete") } From 5453b703b25685ae771cb5744be82f8f1306d038 Mon Sep 17 00:00:00 2001 From: Wang Xu Date: Wed, 27 Sep 2017 11:13:19 -0600 Subject: [PATCH 14/14] re-enable the in sandbox portmapping with exec in sandbox make it more flexible, update without modify hyperstart protocol Signed-off-by: Wang Xu --- daemon/pod/pod.go | 2 + daemon/pod/portmappings.go | 53 ++++++++- daemon/pod/provision.go | 36 +++--- examples/port-mapping-in-sandbox.pod | 22 ++++ .../portmapping/host_portmapping_linux.go | 11 +- .../portmapping/portmapping_interfaces.go | 44 +++++++ networking/portmapping/sandbox_portmapping.go | 107 ++++++++++++++++++ 7 files changed, 249 insertions(+), 26 deletions(-) create mode 100644 examples/port-mapping-in-sandbox.pod create mode 100644 networking/portmapping/portmapping_interfaces.go create mode 100644 networking/portmapping/sandbox_portmapping.go diff --git a/daemon/pod/pod.go b/daemon/pod/pod.go index b8a14708..2eb08aa8 100644 --- a/daemon/pod/pod.go +++ b/daemon/pod/pod.go @@ -63,6 +63,8 @@ type XPod struct { labels map[string]string resourceLock *sync.Mutex + prestartExecs [][]string + sandbox *hypervisor.Vm factory *PodFactory diff --git a/daemon/pod/portmappings.go b/daemon/pod/portmappings.go index 61c4c1ab..c17b9bad 100644 --- a/daemon/pod/portmappings.go +++ b/daemon/pod/portmappings.go @@ -31,11 +31,20 @@ func (p *XPod) initPortMapping() error { hlog.Log(ERROR, err) return err } - err = portmapping.SetupPortMaps(p.containerIP, pms) + var extPrefix []string + if p.globalSpec.PortmappingWhiteLists != nil && + len(p.globalSpec.PortmappingWhiteLists.InternalNetworks) > 0 && + len(p.globalSpec.PortmappingWhiteLists.ExternalNetworks) > 0 { + extPrefix = p.globalSpec.PortmappingWhiteLists.ExternalNetworks + } + preExec, err := portmapping.SetupPortMaps(p.containerIP, extPrefix, pms) if err != nil { p.Log(ERROR, "failed to setup port mappings: %v", err) return err } + if len(preExec) > 0 { + p.prestartExecs = append(p.prestartExecs, preExec...) + } } return nil } @@ -47,7 +56,7 @@ func (p *XPod) flushPortMapping() error { hlog.Log(ERROR, err) return err } - err = portmapping.ReleasePortMaps(p.containerIP, pms) + _, err = portmapping.ReleasePortMaps(p.containerIP, nil, pms) if err != nil { p.Log(ERROR, "release port mappings failed: %v", err) return err @@ -75,11 +84,29 @@ func (p *XPod) AddPortMapping(spec []*apitypes.PortMapping) error { p.Log(ERROR, "failed to generate port mapping rules: %v", err) return err } - err = portmapping.SetupPortMaps(p.containerIP, pms) + var extPrefix []string + if p.globalSpec.PortmappingWhiteLists != nil && + len(p.globalSpec.PortmappingWhiteLists.InternalNetworks) > 0 && + len(p.globalSpec.PortmappingWhiteLists.ExternalNetworks) > 0 { + extPrefix = p.globalSpec.PortmappingWhiteLists.ExternalNetworks + } + preExec, err := portmapping.SetupPortMaps(p.containerIP, extPrefix, pms) if err != nil { p.Log(ERROR, "failed to apply port mapping rules: %v", err) return err } + if len(preExec) > 0 { + p.prestartExecs = append(p.prestartExecs, preExec...) + if p.sandbox != nil { + for _, ex := range preExec { + _, stderr, err := p.sandbox.HyperstartExecSync(ex, nil) + if err != nil { + p.Log(ERROR, "failed to setup inSandbox mapping: %v [ %s", err, string(stderr)) + return err + } + } + } + } all := make([]*apitypes.PortMapping, len(p.portMappings)+len(spec)) copy(all, spec) @@ -134,11 +161,29 @@ func (p *XPod) removePortMapping(tbr []*apitypes.PortMapping, eq portMappingComp return err } - err = portmapping.ReleasePortMaps(p.containerIP, act) + var extPrefix []string + if p.globalSpec.PortmappingWhiteLists != nil && + len(p.globalSpec.PortmappingWhiteLists.InternalNetworks) > 0 && + len(p.globalSpec.PortmappingWhiteLists.ExternalNetworks) > 0 { + extPrefix = p.globalSpec.PortmappingWhiteLists.ExternalNetworks + } + postExec, err := portmapping.ReleasePortMaps(p.containerIP, extPrefix, act) if err != nil { p.Log(ERROR, "failed to clean up rules: %v", err) return err } + if len(postExec) > 0 { + // don't need to release prestartExec here, it is not persistent + if p.sandbox != nil { + for _, ex := range postExec { + _, stderr, err := p.sandbox.HyperstartExecSync(ex, nil) + if err != nil { + p.Log(ERROR, "failed to setup inSandbox mapping: %v [ %s", err, string(stderr)) + return err + } + } + } + } p.portMappings = other err = p.savePortMapping() diff --git a/daemon/pod/provision.go b/daemon/pod/provision.go index c71310a7..57867035 100644 --- a/daemon/pod/provision.go +++ b/daemon/pod/provision.go @@ -86,19 +86,20 @@ func newXPod(factory *PodFactory, spec *apitypes.UserPod) (*XPod, error) { factory.hosts = HostsCreator(spec.Id) factory.logCreator = initLogCreator(factory, spec) p := &XPod{ - name: spec.Id, - logPrefix: fmt.Sprintf("Pod[%s] ", spec.Id), - globalSpec: spec.CloneGlobalPart(), - containers: make(map[string]*Container), - volumes: make(map[string]*Volume), - interfaces: make(map[string]*Interface), - portMappings: spec.Portmappings, - labels: spec.Labels, - execs: make(map[string]*Exec), - resourceLock: &sync.Mutex{}, - statusLock: &sync.RWMutex{}, - stoppedChan: make(chan bool, 1), - factory: factory, + name: spec.Id, + logPrefix: fmt.Sprintf("Pod[%s] ", spec.Id), + globalSpec: spec.CloneGlobalPart(), + containers: make(map[string]*Container), + volumes: make(map[string]*Volume), + interfaces: make(map[string]*Interface), + portMappings: spec.Portmappings, + labels: spec.Labels, + prestartExecs: [][]string{}, + execs: make(map[string]*Exec), + resourceLock: &sync.Mutex{}, + statusLock: &sync.RWMutex{}, + stoppedChan: make(chan bool, 1), + factory: factory, } p.initCond = sync.NewCond(p.statusLock.RLocker()) return p, nil @@ -484,6 +485,15 @@ func (p *XPod) startAll() error { p.Log(INFO, "start all containers") future := utils.NewFutureSet() + for _, pre := range p.prestartExecs { + p.Log(DEBUG, "run prestart exec %v", pre) + _, stderr, err := p.sandbox.HyperstartExecSync(pre, nil) + if err != nil { + p.Log(ERROR, "failed to execute prestart command %v: %v [ %s", pre, err, string(stderr)) + return err + } + } + for ic, c := range p.containers { future.Add(ic, c.start) } diff --git a/examples/port-mapping-in-sandbox.pod b/examples/port-mapping-in-sandbox.pod new file mode 100644 index 00000000..60a1e3d2 --- /dev/null +++ b/examples/port-mapping-in-sandbox.pod @@ -0,0 +1,22 @@ +{ + "id": "port-mapping-test", + "containers" : [{ + "name": "pmtest", + "image": "busybox:latest", + "command": ["/bin/nc", "-l", "-p", "1300"] + }], + "resource": { + "vcpu": 1, + "memory": 128 + }, + "portmappingWhiteLists":{ + "internalNetworks": ["127.0.0.1/32", "192.168.123.0/24"], + "externalNetworks": ["0.0.0.0/0"] + }, + "portmappings": [{ + "containerport": "1300-1310", + "hostport": "3000-3010", + "protocol": "tcp" + }] +} + diff --git a/networking/portmapping/host_portmapping_linux.go b/networking/portmapping/host_portmapping_linux.go index 9101efb1..53394510 100644 --- a/networking/portmapping/host_portmapping_linux.go +++ b/networking/portmapping/host_portmapping_linux.go @@ -130,11 +130,7 @@ func parseRawResultOnHyper(output []byte, err error) error { return nil } -func SetupPortMaps(containerip string, maps []*PortMapping) error { - if disableIptables || len(maps) == 0 { - return nil - } - +func setupIptablesPortMaps(containerip string, maps []*PortMapping) error { var ( revert bool revertRules = [][]string{} @@ -209,10 +205,7 @@ func SetupPortMaps(containerip string, maps []*PortMapping) error { return nil } -func ReleasePortMaps(containerip string, maps []*PortMapping) error { - if disableIptables || len(maps) == 0 { - return nil - } +func releaseIptablesPortMaps(containerip string, maps []*PortMapping) error { release_loop: for _, m := range maps { diff --git a/networking/portmapping/portmapping_interfaces.go b/networking/portmapping/portmapping_interfaces.go new file mode 100644 index 00000000..ceeb24d8 --- /dev/null +++ b/networking/portmapping/portmapping_interfaces.go @@ -0,0 +1,44 @@ +package portmapping + +func SetupPortMaps(containerip string, externalPrefix []string, maps []*PortMapping) (preExec [][]string, err error) { + if len(maps) == 0 { + return [][]string{}, nil + } + if len(externalPrefix) > 0 { + preExec, err = setupInSandboxMappings(externalPrefix, maps) + if err != nil { + return [][]string{}, err + } + defer func() { + if err != nil { + preExec, _ = releaseInSandboxMappings(externalPrefix, maps) + } + }() + } + if !disableIptables { + err = setupIptablesPortMaps(containerip, maps) + if err != nil { + return [][]string{}, err + } + } + return preExec, nil +} + +func ReleasePortMaps(containerip string, externalPrefix []string, maps []*PortMapping) (postExec [][]string, err error) { + if len(maps) == 0 { + return [][]string{}, nil + } + if len(externalPrefix) > 0 { + postExec, err = releaseInSandboxMappings(externalPrefix, maps) + if err != nil { + return [][]string{}, err + } + } + if !disableIptables { + err = releaseIptablesPortMaps(containerip, maps) + if err != nil { + return [][]string{}, err + } + } + return postExec, nil +} diff --git a/networking/portmapping/sandbox_portmapping.go b/networking/portmapping/sandbox_portmapping.go new file mode 100644 index 00000000..af58d9de --- /dev/null +++ b/networking/portmapping/sandbox_portmapping.go @@ -0,0 +1,107 @@ +package portmapping + +import ( + "fmt" + "strconv" + "strings" +) + +/* Inside hyperstart, there are iptables init job should be done before configure in sandbox rules. + + // iptables -t filter -N hyperstart-INPUT + // iptables -t nat -N hyperstart-PREROUTING + // iptables -t filter -I INPUT -j hyperstart-INPUT + // iptables -t nat -I PREROUTING -j hyperstart-PREROUTING + // iptables -t filter -A hyperstart-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT + // iptables -t filter -A hyperstart-INPUT -p icmp -j ACCEPT + // iptables -t filter -A hyperstart-INPUT -i lo -j ACCEPT + // iptables -t filter -A hyperstart-INPUT -j DROP + // iptables -t nat -A hyperstart-PREROUTING -j RETURN + // sh -c "echo 10485760 > /proc/sys/net/nf_conntrack_max" + // sh -c "echo 300 > /proc/sys/net/netfilter/nf_conntrack_tcp_timeout_established" + + // lan + // iptables -t filter -I hyperstart-INPUT -s %s -j ACCEPT + + These job has been done by hyperstart during initSandbox. +*/ + +func generateRedirectArgs(prefix string, m *PortMapping, insert bool) ([]string, []string, error) { + // outside + //iptables -t nat -I hyperstart-PREROUTING -s %s -p %s -m %s --dport %d -j REDIRECT --to-ports %d" + //iptables -t filter -I hyperstart-INPUT -s %s -p %s -m %s --dport %d -j ACCEPT + var ( + action = "-I" + proto string + dest string + to string + ) + + if !insert { + action = "-D" + } + + if strings.EqualFold(m.Protocol, "udp") { + proto = "udp" + } else { + proto = "tcp" + } + + if m.FromPorts.End == 0 || m.FromPorts.End == m.FromPorts.Begin { + dest = strconv.Itoa(m.FromPorts.Begin) + m.FromPorts.End = m.FromPorts.Begin + } else if m.FromPorts.End > m.FromPorts.Begin { + dest = fmt.Sprintf("%d:%d", m.FromPorts.Begin, m.FromPorts.End) + } else { + return []string{}, []string{}, fmt.Errorf("invalid from port range %d-%d", m.FromPorts.Begin, m.FromPorts.End) + } + + if m.ToPorts.End == 0 || m.ToPorts.End == m.ToPorts.Begin { + to = strconv.Itoa(m.ToPorts.Begin) + m.ToPorts.End = m.ToPorts.Begin + } else if m.ToPorts.End > m.ToPorts.Begin { + to = fmt.Sprintf("%d-%d", m.ToPorts.Begin, m.ToPorts.End) + } else { + return []string{}, []string{}, fmt.Errorf("invalid to port range %d-%d", m.ToPorts.Begin, m.ToPorts.End) + } + + //we may map ports 1:N or N:N, but not M:N (M!=1, M!=N) + hostRange := m.FromPorts.End - m.FromPorts.Begin + containerRange := m.ToPorts.End - m.ToPorts.Begin + if hostRange != 0 && hostRange != containerRange { + return []string{}, []string{}, fmt.Errorf("range mismatch, cannot map ports %s to %s", dest, to) + } + + filterArgs := []string{"iptables", "-t", "filter", action, "hyperstart-INPUT", "-s", prefix, "-p", proto, "-m", proto, "--dport", dest, "-j", "ACCEPT"} + redirectArgs := []string{"iptables", "-t", "nat", action, "hyperstart-PREROUTING", "-s", prefix, "-p", proto, "-m", proto, "--dport", dest, "-j", "REDIRECT", "--to-port", to} + + return redirectArgs, filterArgs, nil +} + +func setupInSandboxMappings(extPrefix []string, maps []*PortMapping) ([][]string, error) { + res := [][]string{} + for _, prefix := range extPrefix { + for _, m := range maps { + redirect, filter, err := generateRedirectArgs(prefix, m, true) + if err != nil { + return nil, err + } + res = append(res, redirect, filter) + } + } + return res, nil +} + +func releaseInSandboxMappings(extPrefix []string, maps []*PortMapping) ([][]string, error) { + res := [][]string{} + for _, prefix := range extPrefix { + for _, m := range maps { + redirect, filter, err := generateRedirectArgs(prefix, m, false) + if err != nil { + return nil, err + } + res = append(res, redirect, filter) + } + } + return res, nil +}