Skip to content

Commit 1aa6528

Browse files
dssengsmira
dsseng
and
smira
committed
fix: make OOM controller more precise by considering separate cgroup PSI
This should reduce false triggers due to high IO activity and similar events increasing global memory PSI despite free memory being available. Also add more details for trigger condition and debugging. Fixes: #12526 Co-authored-by: Andrey Smirnov <andrey.smirnov@siderolabs.com> Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com> Signed-off-by: Dmitrii Sharshakov <dmitry.sharshakov@siderolabs.com>
1 parent f7072c0 commit 1aa6528

File tree

14 files changed

+759
-70
lines changed

14 files changed

+759
-70
lines changed

‎internal/app/machined/pkg/controllers/runtime/internal/oom/oom.go‎

Lines changed: 119 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ package oom
88
import (
99
"fmt"
1010
"io/fs"
11+
"math"
1112
"os"
1213
"path/filepath"
1314
"time"
@@ -67,43 +68,142 @@ func EvaluateTrigger(triggerExpr cel.Expression, evalContext map[string]any) (bo
6768
}
6869

6970
// PopulatePsiToCtx populates the context with PSI data from a cgroup.
70-
func PopulatePsiToCtx(cgroup string, evalContext map[string]any, psi map[string]float64, sampleInterval time.Duration) error {
71-
node, err := cgroups.GetCgroupProperty(cgroup, "memory.pressure")
72-
if err != nil {
73-
return fmt.Errorf("cannot read memory pressure: %w", err)
71+
//
72+
//nolint:gocyclo
73+
func PopulatePsiToCtx(cgroup string, evalContext map[string]any, oldValues map[string]float64, sampleInterval time.Duration) error {
74+
if sampleInterval <= 0 {
75+
return fmt.Errorf("sample interval must be greater than zero")
7476
}
7577

76-
for _, psiType := range []string{"some", "full"} {
77-
for _, span := range []string{"avg10", "avg60", "avg300", "total"} {
78-
spans, ok := node.MemoryPressure[psiType]
79-
if !ok {
80-
return fmt.Errorf("cannot find memory pressure type: type: %s", psiType)
78+
for _, subtree := range []struct {
79+
path string
80+
qos runtime.QoSCgroupClass
81+
}{
82+
{"", -1},
83+
{"init", runtime.QoSCgroupClassSystem},
84+
{"system", runtime.QoSCgroupClassSystem},
85+
{"podruntime", runtime.QoSCgroupClassPodruntime},
86+
{"kubepods/besteffort", runtime.QoSCgroupClassBesteffort},
87+
{"kubepods/burstable", runtime.QoSCgroupClassBurstable},
88+
{"kubepods/guaranteed", runtime.QoSCgroupClassGuaranteed},
89+
} {
90+
node, err := cgroups.GetCgroupProperty(filepath.Join(cgroup, subtree.path), "memory.pressure")
91+
92+
for _, psiType := range []string{"some", "full"} {
93+
for _, span := range []string{"avg10", "avg60", "avg300", "total"} {
94+
value := 0.
95+
96+
// Default non-existent cgroups to all-zero, e.g. during system boot
97+
if err == nil {
98+
value, err = extractPsiEntry(node, psiType, span)
99+
if err != nil {
100+
return err
101+
}
102+
}
103+
104+
// calculate delta
105+
psiPath := subtree.path + "/" + "memory_" + psiType + "_" + span
106+
107+
diff := 0.
108+
if oldValue, ok := oldValues[psiPath]; ok {
109+
diff = (value - oldValue) / sampleInterval.Seconds()
110+
}
111+
112+
oldValues[psiPath] = value
113+
114+
if subtree.qos == -1 {
115+
evalContext["d_memory_"+psiType+"_"+span] = diff
116+
evalContext["memory_"+psiType+"_"+span] = value
117+
} else {
118+
valuesMap, ok := evalContext["qos_memory_"+psiType+"_"+span]
119+
if !ok {
120+
valuesMap = map[int]float64{}
121+
evalContext["qos_memory_"+psiType+"_"+span] = valuesMap
122+
}
123+
124+
valuesMap.(map[int]float64)[int(subtree.qos)] += value
125+
126+
dValuesMap, ok := evalContext["d_qos_memory_"+psiType+"_"+span]
127+
if !ok {
128+
dValuesMap = map[int]float64{}
129+
evalContext["d_qos_memory_"+psiType+"_"+span] = dValuesMap
130+
}
131+
132+
dValuesMap.(map[int]float64)[int(subtree.qos)] += diff
133+
}
81134
}
135+
}
82136

83-
value, ok := spans[span]
84-
if !ok {
85-
return fmt.Errorf("cannot find memory pressure span: span: %s", span)
137+
node = &cgroups.Node{}
138+
// Best effort, if any is not present it will return NaN
139+
cgroups.ReadCgroupfsProperty(node, filepath.Join(cgroup, subtree.path), "memory.current") //nolint:errcheck
140+
cgroups.ReadCgroupfsProperty(node, filepath.Join(cgroup, subtree.path), "memory.max") //nolint:errcheck
141+
cgroups.ReadCgroupfsProperty(node, filepath.Join(cgroup, subtree.path), "memory.peak") //nolint:errcheck
142+
143+
if subtree.qos == -1 {
144+
continue
145+
}
146+
147+
for _, parameter := range []struct {
148+
name string
149+
value float64
150+
}{
151+
{"current", node.MemoryCurrent.Float64()},
152+
{"max", node.MemoryMax.Float64()},
153+
{"peak", node.MemoryPeak.Float64()},
154+
} {
155+
value := parameter.value
156+
// These values cannot be expressed in JSON
157+
if math.IsNaN(value) || math.IsInf(value, 0) {
158+
value = 0.0
86159
}
87160

88-
if !value.IsSet || value.IsMax {
89-
return fmt.Errorf("PSI is not defined")
161+
valuesMap, ok := evalContext["qos_memory_"+parameter.name]
162+
if !ok {
163+
valuesMap = map[int]float64{}
164+
evalContext["qos_memory_"+parameter.name] = valuesMap
90165
}
91166

167+
valuesMap.(map[int]float64)[int(subtree.qos)] += value
168+
169+
oldPath := subtree.path + "/" + "memory_" + parameter.name
170+
92171
diff := 0.
172+
if oldValue, ok := oldValues[oldPath]; ok {
173+
diff = (value - oldValue) / sampleInterval.Seconds()
174+
}
93175

94-
if oldValue, ok := psi["memory_"+psiType+"_"+span]; ok {
95-
diff = (value.Float64() - oldValue) / sampleInterval.Seconds()
176+
dValuesMap, ok := evalContext["d_qos_memory_"+parameter.name]
177+
if !ok {
178+
dValuesMap = map[int]float64{}
179+
evalContext["d_qos_memory_"+parameter.name] = dValuesMap
96180
}
97181

98-
evalContext["d_memory_"+psiType+"_"+span] = diff
99-
evalContext["memory_"+psiType+"_"+span] = value.Float64()
100-
psi["memory_"+psiType+"_"+span] = value.Float64()
182+
dValuesMap.(map[int]float64)[int(subtree.qos)] += diff
101183
}
102184
}
103185

104186
return nil
105187
}
106188

189+
func extractPsiEntry(node *cgroups.Node, psiType string, span string) (float64, error) {
190+
spans, ok := node.MemoryPressure[psiType]
191+
if !ok {
192+
return 0, fmt.Errorf("cannot find memory pressure type: type: %s", psiType)
193+
}
194+
195+
cgValue, ok := spans[span]
196+
if !ok {
197+
return 0, fmt.Errorf("cannot find memory pressure span: span: %s", span)
198+
}
199+
200+
if !cgValue.IsSet || cgValue.IsMax {
201+
return 0, fmt.Errorf("PSI is not defined")
202+
}
203+
204+
return cgValue.Float64(), nil
205+
}
206+
107207
// RankCgroups ranks cgroups using a scoring expression and returns a map.
108208
func RankCgroups(logger *zap.Logger, root string, scoringExpr cel.Expression) map[RankedCgroup]float64 {
109209
ranking := map[RankedCgroup]float64{}

0 commit comments

Comments
 (0)