package health import ( "fmt" "os" "path/filepath" "sort" "strconv" "strings" "sync" "time" ) // machineSampleWindow is the longest gap between two readings that still counts // as "now". CPU busy time and network throughput are differences between two // samples, and the status fragment polls every five seconds — but a page opened // after the panel sat idle for an hour would otherwise average that whole hour // and present it as the current load. Past this the reading only re-baselines, // and the rates say they are still being measured. const machineSampleWindow = time.Minute // Thresholds for the two resources whose exhaustion actually threatens mail: a // fully busy processor slows queue processing, and a machine out of memory has // its processes killed. Network throughput has no comparable threshold — what // counts as a lot depends entirely on the link — so it is reported, not graded. const ( cpuWarnPct = 90.0 memWarnPct = 90.0 memErrorPct = 97.0 ) // Machine is the resource usage of the machine this container runs on: the // status page's answer to "is the server itself under strain", next to the // component checks that answer "is mail flowing". // // The numbers come from the kernel's /proc filesystem, which a container shares // with its host unless it was started with its own namespaces — so CPU and // memory describe the host, while /proc/net/dev describes whatever network // namespace the container is in (its own veth under the default bridge, the // host's interfaces under network_mode: host). type Machine struct { CPU CPU Memory Memory Network Network // Window is the interval the rates were measured over; zero until a // second reading exists. Window time.Duration Status Status } // WindowText names the sampling interval for the card's description. func (m Machine) WindowText() string { if m.Window <= 0 { return "" } return m.Window.Round(time.Second).String() } // CPU is processor load over the sampling window. type CPU struct { // Measured is false until two readings exist to compare; BusyPct means // nothing until it is true. Measured bool BusyPct float64 // Cores and Threads are physical cores and logical CPUs (SMT). Both come // from /proc/cpuinfo when it is readable; otherwise Threads falls back to // the cpuN lines in /proc/stat and Cores matches Threads. Cores int Threads int // Load is the 1/5/15-minute load average, present when /proc/loadavg // could be read. Unlike BusyPct it needs no previous sample. It is kept // for the sampler but is no longer shown on the Status card. Load [3]float64 HasLoad bool Status Status Detail string } // Percent is BusyPct as a whole number, for the element's value // attribute. The bar carries its value in an attribute rather than a width in a // style attribute because the panel's CSP has no inline-style exemption // (security.md). func (c CPU) Percent() int { return percent(c.BusyPct) } // BusyText is the reading as it appears beside the bar. func (c CPU) BusyText() string { return fmt.Sprintf("%d%%", c.Percent()) } // Memory is main memory (and swap, where the machine has any) at the moment of // the reading. Unlike CPU and network it is a level, not a rate, so a single // reading is enough and it is never in the "measuring" state. type Memory struct { Measured bool TotalBytes uint64 AvailableBytes uint64 UsedBytes uint64 UsedPct float64 SwapTotalBytes uint64 SwapUsedBytes uint64 Status Status Detail string } // Percent, UsedText and TotalText render the reading for the template; see // CPU.Percent for why the bar's value travels as an attribute. func (m Memory) Percent() int { return percent(m.UsedPct) } func (m Memory) UsedText() string { return humanBytes(m.UsedBytes) } func (m Memory) TotalText() string { return humanBytes(m.TotalBytes) } func (m Memory) PctText() string { return fmt.Sprintf("%d%%", m.Percent()) } // Interface is one network interface's traffic: the counters since the // interface came up, and the throughput over the sampling window. type Interface struct { Name string RxBytes uint64 TxBytes uint64 RxRate float64 // bytes per second, valid when the Network is Measured TxRate float64 Measured bool } func (i Interface) InText() string { return humanBytes(i.RxBytes) } func (i Interface) OutText() string { return humanBytes(i.TxBytes) } func (i Interface) InRateText() string { return humanRate(i.RxRate) } func (i Interface) OutRateText() string { return humanRate(i.TxRate) } // Network is the traffic across every interface that has carried any, loopback // excluded — loopback traffic is the container talking to itself (the panel to // SQLite, Postfix to its milters) and says nothing about the link. type Network struct { Measured bool Interfaces []Interface RxRate float64 TxRate float64 Status Status Detail string } func (n Network) InRateText() string { return humanRate(n.RxRate) } func (n Network) OutRateText() string { return humanRate(n.TxRate) } // MachineSampler reads those counters. Its zero value is ready to use and it is // safe for concurrent use, but one sampler has to be shared by every caller: // the rates are measured against the reading the previous call left behind, so // a fresh sampler per request would never have anything to compare against. type MachineSampler struct { // procRoot replaces /proc in tests; empty means the real one. procRoot string mu sync.Mutex prevAt time.Time prevCPU cpuTimes prevNet map[string]netCounters } func (m *MachineSampler) root() string { if m.procRoot == "" { return "/proc" } return m.procRoot } // cpuTimes is the aggregate of /proc/stat's "cpu" line: all time accounted for, // and the part of it the processor spent doing nothing. type cpuTimes struct { total uint64 idle uint64 } // netCounters is one interface's byte counters from /proc/net/dev. type netCounters struct { rx uint64 tx uint64 } // Sample reads the current counters and reports usage since the previous call. // Like every other check here it never fails: a counter that cannot be read // becomes an unknown status with an explanation, so a kernel that does not // publish one of these files (or a panel run outside Linux for development) // costs one line of the card rather than the page. func (m *MachineSampler) Sample() Machine { root := m.root() now := time.Now() cpuNow, logical, cpuErr := readCPUTimes(root) cores, threads := readCPUTopology(root) if threads == 0 { threads = logical } if cores == 0 { cores = threads } netNow, netErr := readNetDev(root) m.mu.Lock() prevAt, prevCPU, prevNet := m.prevAt, m.prevCPU, m.prevNet if cpuErr == nil { m.prevCPU = cpuNow } if netErr == nil { m.prevNet = netNow } if cpuErr == nil || netErr == nil { m.prevAt = now } m.mu.Unlock() // A window of zero (two calls in the same instant) would divide by zero; // one longer than machineSampleWindow is no longer a description of now. window := now.Sub(prevAt) fresh := !prevAt.IsZero() && window > 0 && window <= machineSampleWindow mach := Machine{ CPU: cpuUsage(prevCPU, cpuNow, cores, threads, readLoadAvg(root), fresh, cpuErr), Memory: readMemory(root), Network: networkUsage(prevNet, netNow, window, fresh, netErr), } if fresh { mach.Window = window } mach.Status = Worst(mach.CPU.Status, mach.Memory.Status, mach.Network.Status) return mach } // cpuUsage grades the processor over the window. Detail carries only the // core and thread counts — load average and busy prose stay out of the Status // card (see roadmap panel-docs for operator-facing explanation later). func cpuUsage(prev, cur cpuTimes, cores, threads int, load [3]float64, fresh bool, err error) CPU { c := CPU{Cores: cores, Threads: threads} if !isZeroLoad(load) { c.Load, c.HasLoad = load, true } if err != nil { c.Status = StatusUnknown c.Detail = "The kernel's processor counters (/proc/stat) could not be read here." return c } c.Detail = c.contextText() if !fresh || cur.total <= prev.total { c.Status = StatusUnknown return c } total := cur.total - prev.total idle := uint64(0) if cur.idle > prev.idle { idle = cur.idle - prev.idle } if idle > total { idle = total } c.Measured = true c.BusyPct = 100 * float64(total-idle) / float64(total) if c.BusyPct >= cpuWarnPct { c.Status = StatusWarn } else { c.Status = StatusOK } return c } // contextText is the CPU detail column: physical cores and logical threads. func (c CPU) contextText() string { switch { case c.Cores > 0 && c.Threads > 0: return fmt.Sprintf("%d cores · %d threads", c.Cores, c.Threads) case c.Threads > 0: return fmt.Sprintf("%d threads", c.Threads) case c.Cores > 0: return fmt.Sprintf("%d cores", c.Cores) default: return "" } } // readMemory reports main memory from /proc/meminfo. Used is derived from // MemAvailable rather than MemFree: Linux spends every spare page on cache, so // MemFree on a healthy machine is near zero and would report a permanent // emergency. MemAvailable is the kernel's own estimate of what a new workload // could actually get. func readMemory(root string) Memory { var m Memory fields, err := readMeminfo(root) if err != nil { m.Status = StatusUnknown m.Detail = "The kernel's memory counters (/proc/meminfo) could not be read here." return m } total, available := fields["MemTotal"], fields["MemAvailable"] if total == 0 { m.Status = StatusUnknown m.Detail = "/proc/meminfo does not report a total memory size." return m } if available > total { available = total } m.Measured = true m.TotalBytes = total m.AvailableBytes = available m.UsedBytes = total - available m.UsedPct = 100 * float64(m.UsedBytes) / float64(total) m.SwapTotalBytes = fields["SwapTotal"] if swapFree := fields["SwapFree"]; m.SwapTotalBytes > swapFree { m.SwapUsedBytes = m.SwapTotalBytes - swapFree } detail := fmt.Sprintf("%s used of %s.", humanBytes(m.UsedBytes), humanBytes(total)) if m.SwapTotalBytes > 0 { detail += fmt.Sprintf(" Swap: %s of %s.", humanBytes(m.SwapUsedBytes), humanBytes(m.SwapTotalBytes)) } switch { case m.UsedPct >= memErrorPct: m.Status = StatusError m.Detail = detail + " Memory is exhausted; the kernel kills processes to reclaim it, and Postfix or the panel are candidates." case m.UsedPct >= memWarnPct: m.Status = StatusWarn m.Detail = detail + " Little headroom left." default: m.Status = StatusOK m.Detail = detail } return m } // networkUsage turns two readings of the interface counters into throughput. // It never grades: there is no usage figure that is wrong for a mail server, so // the row is informational and only reports "unknown" when the counters are // unreadable. func networkUsage(prev, cur map[string]netCounters, window time.Duration, fresh bool, err error) Network { var n Network if err != nil { n.Status = StatusUnknown n.Detail = "The kernel's network counters (/proc/net/dev) could not be read here." return n } names := make([]string, 0, len(cur)) for name := range cur { names = append(names, name) } // Map order is random, and this table is re-rendered every few seconds: // without a sort the rows would shuffle under the reader. sort.Strings(names) n.Measured = fresh for _, name := range names { c := cur[name] // An interface that has never carried a byte is a veth or a bridge // the deployment happens to have, not part of the mail path. if c.rx == 0 && c.tx == 0 { continue } iface := Interface{Name: name, RxBytes: c.rx, TxBytes: c.tx, Measured: fresh} if fresh { p := prev[name] secs := window.Seconds() // Counters only go up; a drop means the interface (or the // container) was recreated, so there is no rate to report. if c.rx >= p.rx { iface.RxRate = float64(c.rx-p.rx) / secs } if c.tx >= p.tx { iface.TxRate = float64(c.tx-p.tx) / secs } n.RxRate += iface.RxRate n.TxRate += iface.TxRate } n.Interfaces = append(n.Interfaces, iface) } n.Status = StatusOK switch { case len(n.Interfaces) == 0: n.Detail = "No interface outside loopback has carried any traffic." case !fresh: n.Detail = "Measuring — this reading sets the baseline; the next refresh has the throughput." } return n } // readCPUTimes returns the aggregate processor times and the number of cores // from /proc/stat. The times are in USER_HZ ticks, which cancel out because // only their ratio is used. func readCPUTimes(root string) (cpuTimes, int, error) { data, err := os.ReadFile(filepath.Join(root, "stat")) if err != nil { return cpuTimes{}, 0, err } var ( times cpuTimes cores int found bool ) for _, line := range strings.Split(string(data), "\n") { fields := strings.Fields(line) if len(fields) < 5 || !strings.HasPrefix(fields[0], "cpu") { continue } if fields[0] != "cpu" { cores++ // cpu0, cpu1, … — one line per core continue } // user nice system idle iowait irq softirq steal (guest fields are // already counted inside user/nice, so they are left out). for i, f := range fields[1:] { if i >= 8 { break } v, err := strconv.ParseUint(f, 10, 64) if err != nil { return cpuTimes{}, 0, fmt.Errorf("/proc/stat: unreadable cpu time %q", f) } times.total += v // idle plus iowait: both are time the processor had nothing to // run, and separating them tells the reader nothing here. if i == 3 || i == 4 { times.idle += v } } found = true } if !found { return cpuTimes{}, 0, fmt.Errorf("/proc/stat: no aggregate cpu line") } return times, cores, nil } // readCPUTopology returns physical core and logical thread counts from // /proc/cpuinfo. Zeroes mean the file was missing or empty; the caller falls // back to the cpuN count from /proc/stat. func readCPUTopology(root string) (cores, threads int) { data, err := os.ReadFile(filepath.Join(root, "cpuinfo")) if err != nil { return 0, 0 } type coreKey struct{ phys, core int } seen := map[coreKey]bool{} var ( inCPU bool phys = -1 core = -1 idx int ) flush := func() { if !inCPU { return } p, c := phys, core if p < 0 { p = idx } if c < 0 { c = idx } seen[coreKey{p, c}] = true inCPU, phys, core = false, -1, -1 } for _, line := range strings.Split(string(data), "\n") { line = strings.TrimSpace(line) if line == "" { flush() continue } key, val, ok := strings.Cut(line, ":") if !ok { continue } key = strings.TrimSpace(key) val = strings.TrimSpace(val) switch key { case "processor": flush() inCPU = true threads++ idx = threads - 1 if n, err := strconv.Atoi(val); err == nil { idx = n } case "physical id": if n, err := strconv.Atoi(val); err == nil { phys = n } case "core id": if n, err := strconv.Atoi(val); err == nil { core = n } } } flush() if threads == 0 { return 0, 0 } return len(seen), threads } // readLoadAvg reads the 1/5/15-minute load averages. A machine without // /proc/loadavg simply has none reported, so the failure is a zero value rather // than an error. func readLoadAvg(root string) [3]float64 { var load [3]float64 data, err := os.ReadFile(filepath.Join(root, "loadavg")) if err != nil { return load } fields := strings.Fields(string(data)) if len(fields) < 3 { return load } for i := range load { v, err := strconv.ParseFloat(fields[i], 64) if err != nil { return [3]float64{} } load[i] = v } return load } func isZeroLoad(load [3]float64) bool { return load == [3]float64{} } // readMeminfo returns the /proc/meminfo entries this package uses, in bytes. // The file reports kB (kibibytes, despite the label) for these fields. func readMeminfo(root string) (map[string]uint64, error) { data, err := os.ReadFile(filepath.Join(root, "meminfo")) if err != nil { return nil, err } want := map[string]bool{"MemTotal": true, "MemAvailable": true, "SwapTotal": true, "SwapFree": true} out := make(map[string]uint64, len(want)) for _, line := range strings.Split(string(data), "\n") { name, rest, ok := strings.Cut(line, ":") if !ok || !want[name] { continue } fields := strings.Fields(rest) if len(fields) == 0 { continue } v, err := strconv.ParseUint(fields[0], 10, 64) if err != nil { continue } if len(fields) > 1 && strings.EqualFold(fields[1], "kB") { v *= 1024 } out[name] = v } if len(out) == 0 { return nil, fmt.Errorf("/proc/meminfo: no readable fields") } return out, nil } // readNetDev returns each interface's byte counters from /proc/net/dev, loopback // excluded. Each line is " eth0: …" — the // name is separated by a colon, which may or may not have a space after it, so // the split is on the colon and not on whitespace. func readNetDev(root string) (map[string]netCounters, error) { data, err := os.ReadFile(filepath.Join(root, "net", "dev")) if err != nil { return nil, err } out := make(map[string]netCounters) for _, line := range strings.Split(string(data), "\n") { name, rest, ok := strings.Cut(line, ":") name = strings.TrimSpace(name) if !ok || name == "" || name == "lo" || strings.Contains(name, " ") { continue // header lines carry no name, or several words } fields := strings.Fields(rest) if len(fields) < 9 { continue } rx, err1 := strconv.ParseUint(fields[0], 10, 64) tx, err2 := strconv.ParseUint(fields[8], 10, 64) if err1 != nil || err2 != nil { continue } out[name] = netCounters{rx: rx, tx: tx} } return out, nil } // percent rounds a 0–100 reading to a whole number and clamps it, so a bar's // value attribute is always inside the range the element declares. func percent(v float64) int { switch { case v <= 0: return 0 case v >= 100: return 100 default: return int(v + 0.5) } } // humanBytes renders a byte count in the binary units memory and traffic are // conventionally read in. func humanBytes(b uint64) string { const unit = 1024 if b < unit { return fmt.Sprintf("%d B", b) } div, exp := uint64(unit), 0 for n := b / unit; n >= unit && exp < 4; n /= unit { div *= unit exp++ } return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTP"[exp]) } // humanRate renders throughput in bytes per second, to match the totals beside // it rather than the bits per second a link is sold in. func humanRate(perSec float64) string { if perSec < 0 { perSec = 0 } return humanBytes(uint64(perSec+0.5)) + "/s" }