From cd08a85f705a7b67d516137686e16403ea808307 Mon Sep 17 00:00:00 2001 From: Cyril Zhang Date: Wed, 2 Jun 2021 12:45:04 -0700 Subject: [PATCH 1/2] Add racct limit option --- cmd/runj/create.go | 10 ++++++++ cmd/runj/delete.go | 4 +++ jail/limit.go | 57 +++++++++++++++++++++++++++++++++++++++++++ runtimespec/config.go | 25 +++++++++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 jail/limit.go diff --git a/cmd/runj/create.go b/cmd/runj/create.go index 121b4b5..9c3fca4 100644 --- a/cmd/runj/create.go +++ b/cmd/runj/create.go @@ -137,6 +137,16 @@ the console's pseudoterminal`) if err := jail.CreateJail(cmd.Context(), confPath); err != nil { return err } + err = jail.Limit(id, ociConfig) + if err != nil { + return err + } + defer func() { + if err == nil { + return + } + jail.Unlimit(id, ociConfig) + }() err = jail.Mount(ociConfig) if err != nil { return err diff --git a/cmd/runj/delete.go b/cmd/runj/delete.go index 23ba2f8..37bc134 100644 --- a/cmd/runj/delete.go +++ b/cmd/runj/delete.go @@ -59,6 +59,10 @@ func deleteCommand() *cobra.Command { if ociConfig == nil { return errors.New("OCI config is required") } + err = jail.Unlimit(id, ociConfig) + if err != nil { + return err + } err = jail.Unmount(ociConfig) if err != nil { return err diff --git a/jail/limit.go b/jail/limit.go new file mode 100644 index 0000000..c4543af --- /dev/null +++ b/jail/limit.go @@ -0,0 +1,57 @@ +package jail + +import ( + "bytes" + "os/exec" + + "go.sbk.wtf/runj/runtimespec" +) + +// Limit uses rctl to add the rct rules +func Limit(id string, ociConfig *runtimespec.Spec) error { + if ociConfig.FreeBSD == nil { + return nil + } + for _, racctLimit := range ociConfig.FreeBSD.RacctLimits { + rule := makeRCTLRule(id, &racctLimit) + cmd := exec.Command("rctl", "-a", rule) + err := cmd.Run() + if err != nil { + return err + } + } + return nil +} + +// Unlimit uses rctl to remove the rctl rules +func Unlimit(id string, ociConfig *runtimespec.Spec) error { + if ociConfig.FreeBSD == nil { + return nil + } + for _, racctLimit := range ociConfig.FreeBSD.RacctLimits { + rule := makeRCTLRule(id, &racctLimit) + cmd := exec.Command("rctl", "-r", rule) + err := cmd.Run() + if err != nil { + return err + } + } + return nil +} + +func makeRCTLRule(id string, racctLimit *runtimespec.RacctLimit) string { + buf := bytes.Buffer{} + buf.WriteString("jail:") + buf.WriteString(id) + buf.WriteString(":") + buf.WriteString(racctLimit.Resource) + buf.WriteString(":") + buf.WriteString(racctLimit.Action) + buf.WriteString("=") + buf.WriteString(racctLimit.Amount) + if racctLimit.Per != "" { + buf.WriteString("/") + buf.WriteString(racctLimit.Per) + } + return buf.String() +} diff --git a/runtimespec/config.go b/runtimespec/config.go index 4166b59..c7a5b21 100644 --- a/runtimespec/config.go +++ b/runtimespec/config.go @@ -53,6 +53,11 @@ type Spec struct { VM *VM `json:"vm,omitempty" platform:"vm"` */ // End of modification + + // Modification by Cyril Zhang + // FreeBSD is platform-specific configuration for FreeBSD based containers. + FreeBSD *FreeBSD `json:"freebsd,omitempty" platform:"freebsd"` + // End of modification } // Modification by Samuel Karp @@ -135,6 +140,26 @@ type Mount struct { Options []string `json:"options,omitempty"` } +// Modification by Cyril Zhang +// FreeBSD contains platform-specific configuration for FreeBSD based containers. +type FreeBSD struct { + // RacctLimits specifies racct rules to apply to this jail. + RacctLimits []RacctLimit `json:"racct,omitempty"` +} + +// RacctLimit is a racct rule to apply to a jail. +type RacctLimit struct { + // Resource is the resource to set a limit on. + Resource string `json:"resource"` + // Action is what will happen if a process exceeds the allowed amount. + Action string `json:"action"` + // Amount is the allowed amount of the resource. + Amount string `json:"amount"` + // Per defines the entity that the amount applies to. + Per string `json:"per,omitempty"` +} +// End of modification + // Modification by Samuel Karp /* Omitted type definitions for: From 1a04ebdb3696850ea78a98ab5a6b24fa77ada9e2 Mon Sep 17 00:00:00 2001 From: Cyril Zhang Date: Tue, 6 Jul 2021 12:43:37 -0700 Subject: [PATCH 2/2] Use alternative FreeBSD limit data structure --- jail/limit.go | 106 ++++++++++++++++++++++++++++++++++-------- runtimespec/config.go | 65 +++++++++++++++++++++----- 2 files changed, 139 insertions(+), 32 deletions(-) diff --git a/jail/limit.go b/jail/limit.go index c4543af..e67454e 100644 --- a/jail/limit.go +++ b/jail/limit.go @@ -1,7 +1,7 @@ package jail import ( - "bytes" + "fmt" "os/exec" "go.sbk.wtf/runj/runtimespec" @@ -12,8 +12,7 @@ func Limit(id string, ociConfig *runtimespec.Spec) error { if ociConfig.FreeBSD == nil { return nil } - for _, racctLimit := range ociConfig.FreeBSD.RacctLimits { - rule := makeRCTLRule(id, &racctLimit) + for _, rule := range makeRCTLRules(id, ociConfig.FreeBSD.Resources) { cmd := exec.Command("rctl", "-a", rule) err := cmd.Run() if err != nil { @@ -28,8 +27,7 @@ func Unlimit(id string, ociConfig *runtimespec.Spec) error { if ociConfig.FreeBSD == nil { return nil } - for _, racctLimit := range ociConfig.FreeBSD.RacctLimits { - rule := makeRCTLRule(id, &racctLimit) + for _, rule := range makeRCTLRules(id, ociConfig.FreeBSD.Resources) { cmd := exec.Command("rctl", "-r", rule) err := cmd.Run() if err != nil { @@ -39,19 +37,87 @@ func Unlimit(id string, ociConfig *runtimespec.Spec) error { return nil } -func makeRCTLRule(id string, racctLimit *runtimespec.RacctLimit) string { - buf := bytes.Buffer{} - buf.WriteString("jail:") - buf.WriteString(id) - buf.WriteString(":") - buf.WriteString(racctLimit.Resource) - buf.WriteString(":") - buf.WriteString(racctLimit.Action) - buf.WriteString("=") - buf.WriteString(racctLimit.Amount) - if racctLimit.Per != "" { - buf.WriteString("/") - buf.WriteString(racctLimit.Per) - } - return buf.String() +func makeRCTLMemoryRules(id string, memory *runtimespec.FreeBSDMemory) []string { + var rules []string + if memory.Limit != nil { + rules = append(rules, formatRCTLRule(id, "memoryuse", "deny", *memory.Limit)) + } + if memory.Warning != nil { + rules = append(rules, formatRCTLRule(id, "memoryuse", "devctl", *memory.Warning)) + } + if memory.Swap != nil { + rules = append(rules, formatRCTLRule(id, "swapuse", "deny", *memory.Swap)) + } + if memory.SwapWarning != nil { + rules = append(rules, formatRCTLRule(id, "swapuse", "devctl", *memory.SwapWarning)) + } + return rules +} + +func makeRCTLFSIORules(id string, fsio *runtimespec.FreeBSDFSIO) []string { + var rules []string + if fsio.ReadBPS != nil { + rules = append(rules, formatRCTLRule(id, "readbps", "throttle", *fsio.ReadBPS)) + } + if fsio.WriteBPS != nil { + rules = append(rules, formatRCTLRule(id, "writebps", "throttle", *fsio.WriteBPS)) + } + if fsio.ReadIOPS != nil { + rules = append(rules, formatRCTLRule(id, "readiops", "throttle", *fsio.ReadIOPS)) + } + if fsio.WriteIOPS != nil { + rules = append(rules, formatRCTLRule(id, "writeiops", "throttle", *fsio.WriteIOPS)) + } + return rules +} + +func makeRCTLShmRules(id string, shm *runtimespec.FreeBSDShm) []string { + var rules []string + if shm.Count != nil { + rules = append(rules, formatRCTLRule(id, "nshm", "deny", *shm.Count)) + } + if shm.Size != nil { + rules = append(rules, formatRCTLRule(id, "shmsize", "deny", *shm.Size)) + } + return rules +} + +func makeRCTLCPURules(id string, cpu *runtimespec.FreeBSDCPU) []string { + var rules []string + if cpu.Limit != nil { + rules = append(rules, formatRCTLRule(id, "pcpu", "deny", *cpu.Limit)) + } + return rules +} + +func makeRCTLProcessRules(id string, proc *runtimespec.FreeBSDProcess) []string { + var rules []string + if proc.Limit != nil { + rules = append(rules, formatRCTLRule(id, "maxproc", "deny", *proc.Limit)) + } + return rules +} + +func makeRCTLRules(id string, resources *runtimespec.FreeBSDResources) []string { + var rules []string + if resources.Memory != nil { + rules = append(rules, makeRCTLMemoryRules(id, resources.Memory)...) + } + if resources.FSIO != nil { + rules = append(rules, makeRCTLFSIORules(id, resources.FSIO)...) + } + if resources.Shm != nil { + rules = append(rules, makeRCTLShmRules(id, resources.Shm)...) + } + if resources.CPU != nil { + rules = append(rules, makeRCTLCPURules(id, resources.CPU)...) + } + if resources.Process != nil { + rules = append(rules, makeRCTLProcessRules(id, resources.Process)...) + } + return rules +} + +func formatRCTLRule(id string, resource string, action string, amount uint64) string { + return fmt.Sprintf("jail:%v:%v:%v=%v", id, resource, action, amount) } diff --git a/runtimespec/config.go b/runtimespec/config.go index c7a5b21..de29bbb 100644 --- a/runtimespec/config.go +++ b/runtimespec/config.go @@ -143,20 +143,61 @@ type Mount struct { // Modification by Cyril Zhang // FreeBSD contains platform-specific configuration for FreeBSD based containers. type FreeBSD struct { - // RacctLimits specifies racct rules to apply to this jail. - RacctLimits []RacctLimit `json:"racct,omitempty"` + // Resources is resource limits for FreeBSD. + Resources *FreeBSDResources `json:"resources,omitempty"` } -// RacctLimit is a racct rule to apply to a jail. -type RacctLimit struct { - // Resource is the resource to set a limit on. - Resource string `json:"resource"` - // Action is what will happen if a process exceeds the allowed amount. - Action string `json:"action"` - // Amount is the allowed amount of the resource. - Amount string `json:"amount"` - // Per defines the entity that the amount applies to. - Per string `json:"per,omitempty"` +// FreeBSDResources contains a set of resource limits for FreeBSD. +type FreeBSDResources struct { + // Memory is the memory restriction configuration. + Memory *FreeBSDMemory `json:"memory,omitempty"` + // FSIO is the filesystem IO restriction configuration. + FSIO *FreeBSDFSIO `json:"fsIO,omitempty"` + // Shm is the shared memory restriction configuration. + Shm *FreeBSDShm `json:"shm,omitempty"` + // CPU is the CPU restriction configuration. + CPU *FreeBSDCPU `json:"cpu,omitempty"` + // Process is the process restriction configuration. + Process *FreeBSDProcess `json:"process,omitempty"` +} + +type FreeBSDMemory struct { + // Limit is the memory limit (in bytes). + Limit *uint64 `json:"limit,omitempty"` + // Warning is the amount of memory (in bytes) where a warning is sent to devd(8). + Warning *uint64 `json:"warning,omitempty"` + // Swap is the amount of swap that may be used (in bytes). + Swap *uint64 `json:"swap,omitempty"` + // SwapWarning is the amount of swap (in bytes) where a warning is sent to devd(8). + SwapWarning *uint64 `json:"swapWarning,omitempty"` +} + +type FreeBSDFSIO struct { + // ReadBPS is the rate of filesystem reads (in bytes per second) before throttling occurs. + ReadBPS *uint64 `json:"readbps,omitempty"` + // WriteBPS is the rate of filesystem writes (in bytes per second) before throttling occurs. + WriteBPS *uint64 `json:"writebps,omitempty"` + // ReadIOPS is the rate of filesystem read (in operations per second) before throttling occurs. + ReadIOPS *uint64 `json:"readiops,omitempty"` + // WriteBPS is the rate of filesystem writes (in operations per second) before throttling occurs. + WriteIOPS *uint64 `json:"writeiops,omitempty"` +} + +type FreeBSDShm struct { + // Count is the limit of shared memory object count. + Count *uint64 `json:"count,omitempty"` + // Size is the limit of total shared memory object size (in bytes). + Size *uint64 `json:"size,omitempty"` +} + +type FreeBSDCPU struct { + // Limit is limit of CPU usage (in percent of a single CPU). + Limit *uint64 `json:"limit,omitempty"` +} + +type FreeBSDProcess struct { + // Limit is the limit of process count. + Limit *uint64 `json:"limit,omitempty"` } // End of modification