diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bffb47..d09385b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); version ## [Unreleased] +### Added + +- Machine metrics on the status page: a **Machine** card reporting the + processor (busy percentage, core count, load average), memory and swap, and + network throughput and totals per interface, read from the kernel's counters + in `/proc` (`internal/health/machine.go`). CPU and throughput are differences + between two readings, so they are measured against the previous poll of the + status fragment and reported as still being measured until a second reading + exists — a page opened after a long idle stretch re-baselines rather than + presenting that stretch as the current load. A fully busy processor (≥90%) + warns and an exhausted machine (≥97% of memory in use) errors, both counting + towards the page's headline verdict, since either delays or kills the mail + path; throughput is reported and never graded. Counters that cannot be read + — no `/proc` outside Linux — leave the card in place showing "unknown". The + usage bars are `` elements: the panel's CSP has no inline-style + exemption, so a bar's length has to travel on an attribute. + ## [0.5.0] - 2026-08-06 ### Fixed diff --git a/README.md b/README.md index 51c8ff8..02edb38 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,13 @@ service healthy and will mail be accepted?" - **Status** (`/status`) — supervised processes (Postfix, OpenDKIM, panel), TLS certificate validity and expiry, milter socket presence, and a short - Postfix queue summary. The hostname block compares `SELFPOST_HOSTNAME` + Postfix queue summary. The **Machine** card adds the resource usage of the + host underneath — processor (with the load average), memory and swap, and + per-interface network throughput and totals — read from the kernel's + counters; CPU and throughput are measured between refreshes, so they appear + one refresh after the page opens. A fully busy processor or a machine out of + memory is a warning here, because both delay or kill the mail path; + throughput is only reported. The hostname block compares `SELFPOST_HOSTNAME` against the PTR record the internet publishes for this server's IP (forward-confirmed reverse DNS); use *Re-check* after changing DNS. The **Reload configuration** button re-applies OpenDKIM tables and the Postfix diff --git a/docs/architecture.md b/docs/architecture.md index 750064f..4ba89ad 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -133,7 +133,7 @@ unless noted. | `/healthz` | Liveness (no auth) | | `/setup/*` | One-time admin bootstrap | | `/login`, `/logout` | Session auth | -| `/status` | Process, cert, socket, PTR checks | +| `/status` | Process, cert, socket, PTR checks; machine CPU/memory/network | | `/domains`, `/domains/*` | Domain and application CRUD, DKIM, L2 limits | | `/deliveries` | Send log with filters | | `/mail-queue` | Postfix queue view | diff --git a/internal/health/machine.go b/internal/health/machine.go new file mode 100644 index 0000000..37f5b5b --- /dev/null +++ b/internal/health/machine.go @@ -0,0 +1,557 @@ +package health + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// machineSampleWindow is the longest gap between two readings that still counts +// as "now". CPU busy time and network throughput are differences between two +// samples, and the status fragment polls every five seconds — but a page opened +// after the panel sat idle for an hour would otherwise average that whole hour +// and present it as the current load. Past this the reading only re-baselines, +// and the rates say they are still being measured. +const machineSampleWindow = time.Minute + +// Thresholds for the two resources whose exhaustion actually threatens mail: a +// fully busy processor slows queue processing, and a machine out of memory has +// its processes killed. Network throughput has no comparable threshold — what +// counts as a lot depends entirely on the link — so it is reported, not graded. +const ( + cpuWarnPct = 90.0 + memWarnPct = 90.0 + memErrorPct = 97.0 +) + +// Machine is the resource usage of the machine this container runs on: the +// status page's answer to "is the server itself under strain", next to the +// component checks that answer "is mail flowing". +// +// The numbers come from the kernel's /proc filesystem, which a container shares +// with its host unless it was started with its own namespaces — so CPU and +// memory describe the host, while /proc/net/dev describes whatever network +// namespace the container is in (its own veth under the default bridge, the +// host's interfaces under network_mode: host). +type Machine struct { + CPU CPU + Memory Memory + Network Network + // Window is the interval the rates were measured over; zero until a + // second reading exists. + Window time.Duration + Status Status +} + +// WindowText names the sampling interval for the card's description. +func (m Machine) WindowText() string { + if m.Window <= 0 { + return "" + } + return m.Window.Round(time.Second).String() +} + +// CPU is processor load over the sampling window. +type CPU struct { + // Measured is false until two readings exist to compare; BusyPct means + // nothing until it is true. + Measured bool + BusyPct float64 + Cores int + // Load is the 1/5/15-minute load average, present when /proc/loadavg + // could be read. Unlike BusyPct it needs no previous sample, so it is + // there on the very first page load. + Load [3]float64 + HasLoad bool + Status Status + Detail string +} + +// Percent is BusyPct as a whole number, for the element's value +// attribute. The bar carries its value in an attribute rather than a width in a +// style attribute because the panel's CSP has no inline-style exemption +// (security.md). +func (c CPU) Percent() int { return percent(c.BusyPct) } + +// BusyText is the reading as it appears beside the bar. +func (c CPU) BusyText() string { return fmt.Sprintf("%d%%", c.Percent()) } + +// Memory is main memory (and swap, where the machine has any) at the moment of +// the reading. Unlike CPU and network it is a level, not a rate, so a single +// reading is enough and it is never in the "measuring" state. +type Memory struct { + Measured bool + TotalBytes uint64 + AvailableBytes uint64 + UsedBytes uint64 + UsedPct float64 + SwapTotalBytes uint64 + SwapUsedBytes uint64 + Status Status + Detail string +} + +// Percent, UsedText and TotalText render the reading for the template; see +// CPU.Percent for why the bar's value travels as an attribute. +func (m Memory) Percent() int { return percent(m.UsedPct) } +func (m Memory) UsedText() string { return humanBytes(m.UsedBytes) } +func (m Memory) TotalText() string { return humanBytes(m.TotalBytes) } +func (m Memory) PctText() string { return fmt.Sprintf("%d%%", m.Percent()) } + +// Interface is one network interface's traffic: the counters since the +// interface came up, and the throughput over the sampling window. +type Interface struct { + Name string + RxBytes uint64 + TxBytes uint64 + RxRate float64 // bytes per second, valid when the Network is Measured + TxRate float64 + Measured bool +} + +func (i Interface) InText() string { return humanBytes(i.RxBytes) } +func (i Interface) OutText() string { return humanBytes(i.TxBytes) } +func (i Interface) InRateText() string { return humanRate(i.RxRate) } +func (i Interface) OutRateText() string { return humanRate(i.TxRate) } + +// Network is the traffic across every interface that has carried any, loopback +// excluded — loopback traffic is the container talking to itself (the panel to +// SQLite, Postfix to its milters) and says nothing about the link. +type Network struct { + Measured bool + Interfaces []Interface + RxRate float64 + TxRate float64 + Status Status + Detail string +} + +func (n Network) InRateText() string { return humanRate(n.RxRate) } +func (n Network) OutRateText() string { return humanRate(n.TxRate) } + +// MachineSampler reads those counters. Its zero value is ready to use and it is +// safe for concurrent use, but one sampler has to be shared by every caller: +// the rates are measured against the reading the previous call left behind, so +// a fresh sampler per request would never have anything to compare against. +type MachineSampler struct { + // procRoot replaces /proc in tests; empty means the real one. + procRoot string + + mu sync.Mutex + prevAt time.Time + prevCPU cpuTimes + prevNet map[string]netCounters +} + +func (m *MachineSampler) root() string { + if m.procRoot == "" { + return "/proc" + } + return m.procRoot +} + +// cpuTimes is the aggregate of /proc/stat's "cpu" line: all time accounted for, +// and the part of it the processor spent doing nothing. +type cpuTimes struct { + total uint64 + idle uint64 +} + +// netCounters is one interface's byte counters from /proc/net/dev. +type netCounters struct { + rx uint64 + tx uint64 +} + +// Sample reads the current counters and reports usage since the previous call. +// Like every other check here it never fails: a counter that cannot be read +// becomes an unknown status with an explanation, so a kernel that does not +// publish one of these files (or a panel run outside Linux for development) +// costs one line of the card rather than the page. +func (m *MachineSampler) Sample() Machine { + root := m.root() + now := time.Now() + cpuNow, cores, cpuErr := readCPUTimes(root) + netNow, netErr := readNetDev(root) + + m.mu.Lock() + prevAt, prevCPU, prevNet := m.prevAt, m.prevCPU, m.prevNet + if cpuErr == nil { + m.prevCPU = cpuNow + } + if netErr == nil { + m.prevNet = netNow + } + if cpuErr == nil || netErr == nil { + m.prevAt = now + } + m.mu.Unlock() + + // A window of zero (two calls in the same instant) would divide by zero; + // one longer than machineSampleWindow is no longer a description of now. + window := now.Sub(prevAt) + fresh := !prevAt.IsZero() && window > 0 && window <= machineSampleWindow + + mach := Machine{ + CPU: cpuUsage(prevCPU, cpuNow, cores, readLoadAvg(root), fresh, cpuErr), + Memory: readMemory(root), + Network: networkUsage(prevNet, netNow, window, fresh, netErr), + } + if fresh { + mach.Window = window + } + mach.Status = Worst(mach.CPU.Status, mach.Memory.Status, mach.Network.Status) + return mach +} + +// cpuUsage grades the processor over the window. The load average is reported +// alongside it because the two answer different questions — how busy the +// processor was, and how many tasks were waiting for it — and a machine can +// look idle while work queues up behind a slow disk. +func cpuUsage(prev, cur cpuTimes, cores int, load [3]float64, fresh bool, err error) CPU { + c := CPU{Cores: cores} + if !isZeroLoad(load) { + c.Load, c.HasLoad = load, true + } + if err != nil { + c.Status = StatusUnknown + c.Detail = "The kernel's processor counters (/proc/stat) could not be read here." + return c + } + if !fresh || cur.total <= prev.total { + c.Status = StatusUnknown + c.Detail = joinDetail("Measuring — this reading sets the baseline; the next refresh has the figure.", c.contextText()) + return c + } + + total := cur.total - prev.total + idle := uint64(0) + if cur.idle > prev.idle { + idle = cur.idle - prev.idle + } + if idle > total { + idle = total + } + c.Measured = true + c.BusyPct = 100 * float64(total-idle) / float64(total) + + if c.BusyPct >= cpuWarnPct { + c.Status = StatusWarn + c.Detail = joinDetail(c.contextText(), "The processor is close to fully busy, which slows queue processing and every panel page.") + } else { + c.Status = StatusOK + c.Detail = c.contextText() + } + return c +} + +// contextText is the CPU's supporting figures: what the percentage is a +// percentage of, and how deep the run queue is. +func (c CPU) contextText() string { + var parts []string + if c.Cores > 0 { + parts = append(parts, fmt.Sprintf("%d core(s)", c.Cores)) + } + if c.HasLoad { + parts = append(parts, fmt.Sprintf("load average %.2f, %.2f, %.2f", c.Load[0], c.Load[1], c.Load[2])) + } + return strings.Join(parts, " · ") +} + +// readMemory reports main memory from /proc/meminfo. Used is derived from +// MemAvailable rather than MemFree: Linux spends every spare page on cache, so +// MemFree on a healthy machine is near zero and would report a permanent +// emergency. MemAvailable is the kernel's own estimate of what a new workload +// could actually get. +func readMemory(root string) Memory { + var m Memory + fields, err := readMeminfo(root) + if err != nil { + m.Status = StatusUnknown + m.Detail = "The kernel's memory counters (/proc/meminfo) could not be read here." + return m + } + total, available := fields["MemTotal"], fields["MemAvailable"] + if total == 0 { + m.Status = StatusUnknown + m.Detail = "/proc/meminfo does not report a total memory size." + return m + } + if available > total { + available = total + } + + m.Measured = true + m.TotalBytes = total + m.AvailableBytes = available + m.UsedBytes = total - available + m.UsedPct = 100 * float64(m.UsedBytes) / float64(total) + m.SwapTotalBytes = fields["SwapTotal"] + if swapFree := fields["SwapFree"]; m.SwapTotalBytes > swapFree { + m.SwapUsedBytes = m.SwapTotalBytes - swapFree + } + + detail := fmt.Sprintf("%s used of %s; %s available to new work.", + humanBytes(m.UsedBytes), humanBytes(total), humanBytes(available)) + if m.SwapTotalBytes > 0 { + detail += fmt.Sprintf(" Swap: %s of %s.", humanBytes(m.SwapUsedBytes), humanBytes(m.SwapTotalBytes)) + } + switch { + case m.UsedPct >= memErrorPct: + m.Status = StatusError + m.Detail = detail + " Memory is exhausted; the kernel kills processes to reclaim it, and Postfix or the panel are candidates." + case m.UsedPct >= memWarnPct: + m.Status = StatusWarn + m.Detail = detail + " Little headroom left." + default: + m.Status = StatusOK + m.Detail = detail + } + return m +} + +// networkUsage turns two readings of the interface counters into throughput. +// It never grades: there is no usage figure that is wrong for a mail server, so +// the row is informational and only reports "unknown" when the counters are +// unreadable. +func networkUsage(prev, cur map[string]netCounters, window time.Duration, fresh bool, err error) Network { + var n Network + if err != nil { + n.Status = StatusUnknown + n.Detail = "The kernel's network counters (/proc/net/dev) could not be read here." + return n + } + + names := make([]string, 0, len(cur)) + for name := range cur { + names = append(names, name) + } + // Map order is random, and this table is re-rendered every few seconds: + // without a sort the rows would shuffle under the reader. + sort.Strings(names) + + n.Measured = fresh + for _, name := range names { + c := cur[name] + // An interface that has never carried a byte is a veth or a bridge + // the deployment happens to have, not part of the mail path. + if c.rx == 0 && c.tx == 0 { + continue + } + iface := Interface{Name: name, RxBytes: c.rx, TxBytes: c.tx, Measured: fresh} + if fresh { + p := prev[name] + secs := window.Seconds() + // Counters only go up; a drop means the interface (or the + // container) was recreated, so there is no rate to report. + if c.rx >= p.rx { + iface.RxRate = float64(c.rx-p.rx) / secs + } + if c.tx >= p.tx { + iface.TxRate = float64(c.tx-p.tx) / secs + } + n.RxRate += iface.RxRate + n.TxRate += iface.TxRate + } + n.Interfaces = append(n.Interfaces, iface) + } + + n.Status = StatusOK + switch { + case len(n.Interfaces) == 0: + n.Detail = "No interface outside loopback has carried any traffic." + case !fresh: + n.Detail = "Measuring — this reading sets the baseline; the next refresh has the throughput." + } + return n +} + +// readCPUTimes returns the aggregate processor times and the number of cores +// from /proc/stat. The times are in USER_HZ ticks, which cancel out because +// only their ratio is used. +func readCPUTimes(root string) (cpuTimes, int, error) { + data, err := os.ReadFile(filepath.Join(root, "stat")) + if err != nil { + return cpuTimes{}, 0, err + } + var ( + times cpuTimes + cores int + found bool + ) + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) < 5 || !strings.HasPrefix(fields[0], "cpu") { + continue + } + if fields[0] != "cpu" { + cores++ // cpu0, cpu1, … — one line per core + continue + } + // user nice system idle iowait irq softirq steal (guest fields are + // already counted inside user/nice, so they are left out). + for i, f := range fields[1:] { + if i >= 8 { + break + } + v, err := strconv.ParseUint(f, 10, 64) + if err != nil { + return cpuTimes{}, 0, fmt.Errorf("/proc/stat: unreadable cpu time %q", f) + } + times.total += v + // idle plus iowait: both are time the processor had nothing to + // run, and separating them tells the reader nothing here. + if i == 3 || i == 4 { + times.idle += v + } + } + found = true + } + if !found { + return cpuTimes{}, 0, fmt.Errorf("/proc/stat: no aggregate cpu line") + } + return times, cores, nil +} + +// readLoadAvg reads the 1/5/15-minute load averages. A machine without +// /proc/loadavg simply has none reported, so the failure is a zero value rather +// than an error. +func readLoadAvg(root string) [3]float64 { + var load [3]float64 + data, err := os.ReadFile(filepath.Join(root, "loadavg")) + if err != nil { + return load + } + fields := strings.Fields(string(data)) + if len(fields) < 3 { + return load + } + for i := range load { + v, err := strconv.ParseFloat(fields[i], 64) + if err != nil { + return [3]float64{} + } + load[i] = v + } + return load +} + +func isZeroLoad(load [3]float64) bool { + return load == [3]float64{} +} + +// readMeminfo returns the /proc/meminfo entries this package uses, in bytes. +// The file reports kB (kibibytes, despite the label) for these fields. +func readMeminfo(root string) (map[string]uint64, error) { + data, err := os.ReadFile(filepath.Join(root, "meminfo")) + if err != nil { + return nil, err + } + want := map[string]bool{"MemTotal": true, "MemAvailable": true, "SwapTotal": true, "SwapFree": true} + out := make(map[string]uint64, len(want)) + for _, line := range strings.Split(string(data), "\n") { + name, rest, ok := strings.Cut(line, ":") + if !ok || !want[name] { + continue + } + fields := strings.Fields(rest) + if len(fields) == 0 { + continue + } + v, err := strconv.ParseUint(fields[0], 10, 64) + if err != nil { + continue + } + if len(fields) > 1 && strings.EqualFold(fields[1], "kB") { + v *= 1024 + } + out[name] = v + } + if len(out) == 0 { + return nil, fmt.Errorf("/proc/meminfo: no readable fields") + } + return out, nil +} + +// readNetDev returns each interface's byte counters from /proc/net/dev, loopback +// excluded. Each line is " eth0: … …" — the +// name is separated by a colon, which may or may not have a space after it, so +// the split is on the colon and not on whitespace. +func readNetDev(root string) (map[string]netCounters, error) { + data, err := os.ReadFile(filepath.Join(root, "net", "dev")) + if err != nil { + return nil, err + } + out := make(map[string]netCounters) + for _, line := range strings.Split(string(data), "\n") { + name, rest, ok := strings.Cut(line, ":") + name = strings.TrimSpace(name) + if !ok || name == "" || name == "lo" || strings.Contains(name, " ") { + continue // header lines carry no name, or several words + } + fields := strings.Fields(rest) + if len(fields) < 9 { + continue + } + rx, err1 := strconv.ParseUint(fields[0], 10, 64) + tx, err2 := strconv.ParseUint(fields[8], 10, 64) + if err1 != nil || err2 != nil { + continue + } + out[name] = netCounters{rx: rx, tx: tx} + } + return out, nil +} + +// percent rounds a 0–100 reading to a whole number and clamps it, so a bar's +// value attribute is always inside the range the element declares. +func percent(v float64) int { + switch { + case v <= 0: + return 0 + case v >= 100: + return 100 + default: + return int(v + 0.5) + } +} + +// humanBytes renders a byte count in the binary units memory and traffic are +// conventionally read in. +func humanBytes(b uint64) string { + const unit = 1024 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := uint64(unit), 0 + for n := b / unit; n >= unit && exp < 4; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTP"[exp]) +} + +// humanRate renders throughput in bytes per second, to match the totals beside +// it rather than the bits per second a link is sold in. +func humanRate(perSec float64) string { + if perSec < 0 { + perSec = 0 + } + return humanBytes(uint64(perSec+0.5)) + "/s" +} + +// joinDetail joins the non-empty parts of a detail line. +func joinDetail(parts ...string) string { + var kept []string + for _, p := range parts { + if p != "" { + kept = append(kept, p) + } + } + return strings.Join(kept, " ") +} diff --git a/internal/health/machine_test.go b/internal/health/machine_test.go new file mode 100644 index 0000000..e8e3db6 --- /dev/null +++ b/internal/health/machine_test.go @@ -0,0 +1,287 @@ +package health + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +// fakeProc writes a /proc-shaped directory the sampler can be pointed at, so +// the parsing and the arithmetic are tested against known counters instead of +// whatever the machine running the tests happens to be doing. +func fakeProc(t *testing.T, stat, meminfo, loadavg, netdev string) string { + t.Helper() + dir := t.TempDir() + write := func(name, body string) { + if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + write("stat", stat) + write("meminfo", meminfo) + write("loadavg", loadavg) + write(filepath.Join("net", "dev"), netdev) + return dir +} + +const meminfoSample = `MemTotal: 4194304 kB +MemFree: 131072 kB +MemAvailable: 2097152 kB +Buffers: 262144 kB +SwapTotal: 1048576 kB +SwapFree: 524288 kB +` + +const netdevSample = `Inter-| Receive | Transmit + face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed + lo: 500000 1000 0 0 0 0 0 0 500000 1000 0 0 0 0 0 0 + eth0: 1048576 2000 0 0 0 0 0 0 524288 1500 0 0 0 0 0 0 + veth9: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +` + +func TestCPUUsageOverTwoReadings(t *testing.T) { + // 1000 ticks pass, 250 of them idle: 75% busy. + prev := cpuTimes{total: 10000, idle: 8000} + cur := cpuTimes{total: 11000, idle: 8250} + got := cpuUsage(prev, cur, 4, [3]float64{0.5, 0.4, 0.3}, true, nil) + if !got.Measured { + t.Fatalf("reading not marked measured: %+v", got) + } + if got.Percent() != 75 { + t.Errorf("busy = %.2f%% (%s), want 75%%", got.BusyPct, got.BusyText()) + } + if got.Status != StatusOK { + t.Errorf("status = %q, want ok", got.Status) + } + if got.Cores != 4 || !got.HasLoad { + t.Errorf("cores/load not reported: %+v", got) + } +} + +func TestCPUUsageWarnsWhenFullyBusy(t *testing.T) { + got := cpuUsage(cpuTimes{total: 10000, idle: 5000}, cpuTimes{total: 11000, idle: 5010}, 1, [3]float64{}, true, nil) + if got.Status != StatusWarn { + t.Errorf("99%% busy graded %q, want warn (%s)", got.Status, got.Detail) + } + if got.HasLoad { + t.Error("a missing load average should not be reported as zeros") + } +} + +// The first reading has nothing to compare against, and one taken after a long +// idle stretch would describe that stretch rather than now. Both must report +// "unknown" rather than a number the reader would take for the current load. +func TestCPUUsageWithoutAUsableWindow(t *testing.T) { + for _, c := range []struct { + name string + prev, cur cpuTimes + fresh bool + }{ + {"no previous reading", cpuTimes{}, cpuTimes{total: 11000, idle: 8250}, false}, + {"counters did not advance", cpuTimes{total: 11000, idle: 8250}, cpuTimes{total: 11000, idle: 8250}, true}, + } { + got := cpuUsage(c.prev, c.cur, 2, [3]float64{}, c.fresh, nil) + if got.Measured || got.Status != StatusUnknown { + t.Errorf("%s: measured=%v status=%q, want unmeasured/unknown", c.name, got.Measured, got.Status) + } + } + + if got := cpuUsage(cpuTimes{}, cpuTimes{}, 0, [3]float64{}, false, os.ErrNotExist); got.Status != StatusUnknown { + t.Errorf("unreadable /proc/stat: status %q, want unknown", got.Status) + } +} + +func TestReadMemory(t *testing.T) { + dir := fakeProc(t, "cpu 1 1 1 1 1 1 1 1\n", meminfoSample, "0.1 0.2 0.3 1/2 3\n", netdevSample) + got := readMemory(dir) + if !got.Measured { + t.Fatalf("memory not measured: %+v", got) + } + if got.TotalBytes != 4*1024*1024*1024 { + t.Errorf("total = %d bytes (%s), want 4 GiB", got.TotalBytes, got.TotalText()) + } + // 4 GiB total, 2 GiB available to new work: half used, cache included in + // what is available. + if got.Percent() != 50 { + t.Errorf("used = %.1f%% (%s), want 50%%", got.UsedPct, got.PctText()) + } + if got.Status != StatusOK { + t.Errorf("status = %q, want ok (%s)", got.Status, got.Detail) + } + if got.SwapUsedBytes != 512*1024*1024 { + t.Errorf("swap used = %d bytes, want 512 MiB", got.SwapUsedBytes) + } +} + +func TestReadMemoryGrades(t *testing.T) { + cases := []struct { + name string + available string + want Status + }{ + {"plenty free", "MemAvailable: 2097152 kB\n", StatusOK}, + {"little headroom", "MemAvailable: 209715 kB\n", StatusWarn}, + {"exhausted", "MemAvailable: 41943 kB\n", StatusError}, + } + for _, c := range cases { + dir := fakeProc(t, "cpu 1 1 1 1 1 1 1 1\n", "MemTotal: 4194304 kB\n"+c.available, "", netdevSample) + if got := readMemory(dir); got.Status != c.want { + t.Errorf("%s: status %q, want %q (%s)", c.name, got.Status, c.want, got.Detail) + } + } + + if got := readMemory(t.TempDir()); got.Status != StatusUnknown || got.Measured { + t.Errorf("missing /proc/meminfo: status %q measured=%v", got.Status, got.Measured) + } +} + +func TestReadNetDevSkipsLoopbackAndHeaders(t *testing.T) { + dir := fakeProc(t, "cpu 1 1 1 1 1 1 1 1\n", meminfoSample, "", netdevSample) + got, err := readNetDev(dir) + if err != nil { + t.Fatalf("readNetDev: %v", err) + } + if _, ok := got["lo"]; ok { + t.Error("loopback is counted as network traffic") + } + if len(got) != 2 { + t.Fatalf("parsed %d interfaces, want eth0 and veth9: %+v", len(got), got) + } + if got["eth0"].rx != 1048576 || got["eth0"].tx != 524288 { + t.Errorf("eth0 counters = %+v", got["eth0"]) + } +} + +func TestNetworkUsageRates(t *testing.T) { + prev := map[string]netCounters{"eth0": {rx: 1000, tx: 500}} + cur := map[string]netCounters{ + "eth0": {rx: 11000, tx: 5500}, + "veth9": {rx: 0, tx: 0}, // never carried anything: not shown + } + got := networkUsage(prev, cur, 5*time.Second, true, nil) + if len(got.Interfaces) != 1 || got.Interfaces[0].Name != "eth0" { + t.Fatalf("interfaces = %+v, want eth0 only", got.Interfaces) + } + // 10000 bytes in and 5000 out over five seconds. + if got.RxRate != 2000 || got.TxRate != 1000 { + t.Errorf("rates = %.0f in / %.0f out, want 2000/1000", got.RxRate, got.TxRate) + } + if got.InRateText() != "2.0 KiB/s" { + t.Errorf("in rate text = %q", got.InRateText()) + } + if got.Status != StatusOK { + t.Errorf("status = %q, want ok — throughput is reported, not graded", got.Status) + } +} + +// A recreated container (or interface) resets the counters; the drop must not +// be reported as a huge negative or wrapped-around rate. +func TestNetworkUsageIgnoresCounterResets(t *testing.T) { + prev := map[string]netCounters{"eth0": {rx: 1_000_000, tx: 900_000}} + cur := map[string]netCounters{"eth0": {rx: 1000, tx: 900}} + got := networkUsage(prev, cur, 5*time.Second, true, nil) + if got.RxRate != 0 || got.TxRate != 0 { + t.Errorf("rates after a counter reset = %.0f/%.0f, want 0/0", got.RxRate, got.TxRate) + } +} + +func TestNetworkUsageWithoutAPreviousReading(t *testing.T) { + cur := map[string]netCounters{"eth0": {rx: 1000, tx: 500}} + got := networkUsage(nil, cur, 0, false, nil) + if got.Measured { + t.Error("first reading reported as measured") + } + if len(got.Interfaces) != 1 || got.Interfaces[0].InText() != "1000 B" { + t.Errorf("totals should be shown even before a rate exists: %+v", got.Interfaces) + } + if got.Detail == "" { + t.Error("no explanation for the missing rates") + } + + if bad := networkUsage(nil, nil, 0, false, os.ErrNotExist); bad.Status != StatusUnknown { + t.Errorf("unreadable /proc/net/dev: status %q, want unknown", bad.Status) + } +} + +// End to end through the sampler: the first call baselines, the second reports. +func TestMachineSamplerNeedsTwoReadings(t *testing.T) { + dir := fakeProc(t, + "cpu 1000 0 500 8000 500 0 0 0\ncpu0 500 0 250 4000 250 0 0 0\ncpu1 500 0 250 4000 250 0 0 0\n", + meminfoSample, "0.42 0.31 0.20 2/300 1234\n", netdevSample) + m := &MachineSampler{procRoot: dir} + + first := m.Sample() + if first.CPU.Measured || first.CPU.Status != StatusUnknown { + t.Errorf("first sample reported a CPU figure: %+v", first.CPU) + } + if !first.Memory.Measured { + t.Error("memory is a level, not a rate: it must be reported on the first sample") + } + if first.CPU.Cores != 2 { + t.Errorf("cores = %d, want 2", first.CPU.Cores) + } + if !first.CPU.HasLoad || first.CPU.Load[0] != 0.42 { + t.Errorf("load average not read: %+v", first.CPU.Load) + } + // Unknown checks must not drag the card into a warning. + if first.Status != StatusOK { + t.Errorf("overall machine status = %q, want ok", first.Status) + } + + // Second reading: 1000 more ticks, 750 of them idle → 25% busy. + if err := os.WriteFile(filepath.Join(dir, "stat"), + []byte("cpu 1250 0 500 8750 500 0 0 0\ncpu0 625 0 250 4375 250 0 0 0\ncpu1 625 0 250 4375 250 0 0 0\n"), 0o600); err != nil { + t.Fatal(err) + } + second := m.Sample() + if !second.CPU.Measured { + t.Fatalf("second sample still unmeasured: %+v", second.CPU) + } + if second.CPU.Percent() != 25 { + t.Errorf("busy = %s, want 25%%", second.CPU.BusyText()) + } + if !second.Network.Measured || second.Window <= 0 { + t.Errorf("network window not established: measured=%v window=%v", second.Network.Measured, second.Window) + } + if second.WindowText() == "" { + t.Error("no sampling window to show on the card") + } +} + +// Outside Linux — a developer running the panel on their own machine — there is +// no /proc at all. Every metric must degrade to "unknown" rather than failing +// the status page. +func TestMachineSamplerWithoutProc(t *testing.T) { + m := &MachineSampler{procRoot: filepath.Join(t.TempDir(), "absent")} + got := m.Sample() + if got.Status != StatusUnknown { + t.Errorf("status = %q, want unknown", got.Status) + } + for name, st := range map[string]Status{"cpu": got.CPU.Status, "memory": got.Memory.Status, "network": got.Network.Status} { + if st != StatusUnknown { + t.Errorf("%s status = %q, want unknown", name, st) + } + } +} + +func TestHumanBytes(t *testing.T) { + cases := map[uint64]string{ + 0: "0 B", + 999: "999 B", + 1024: "1.0 KiB", + 1536: "1.5 KiB", + 4 * 1024 * 1024 * 1024: "4.0 GiB", + } + for in, want := range cases { + if got := humanBytes(in); got != want { + t.Errorf("humanBytes(%d) = %q, want %q", in, got, want) + } + } + if got := humanRate(0); got != "0 B/s" { + t.Errorf("humanRate(0) = %q", got) + } +} diff --git a/internal/web/handlers_status.go b/internal/web/handlers_status.go index 031a1de..2216652 100644 --- a/internal/web/handlers_status.go +++ b/internal/web/handlers_status.go @@ -40,7 +40,7 @@ func (s *Server) handleStatusRecheck(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/status?rechecked=1", http.StatusSeeOther) } -// statusBody collects the four local checks the fragment renders. Each one +// statusBody collects the local checks the fragment renders. Each one // reports its own problem rather than failing the page, so a broken component // costs one line and not the whole screen. func (s *Server) statusBody() map[string]any { @@ -75,7 +75,13 @@ func (s *Server) statusBody() map[string]any { socketStatus = health.Worst(socketStatus, sock.Status) } - overall := health.Worst(procStatus, queueStatus, cert.Status, socketStatus) + // Resource usage of the machine underneath. It is graded like the rest — + // a processor that is fully busy or a machine out of memory delays or + // kills the mail path — so it counts towards the headline verdict, and + // its rates are measured against the previous poll (internal/health). + machine := s.machine.Sample() + + overall := health.Worst(procStatus, queueStatus, cert.Status, socketStatus, machine.Status) return map[string]any{ "Processes": procs, "ProcessError": procErr != nil, @@ -83,6 +89,7 @@ func (s *Server) statusBody() map[string]any { "QueueSummary": queueSummary(queueText), "QueueError": queueErr, "QueueStatus": queueStatus, + "Machine": machine, "Cert": cert, "Sockets": sockets, "SocketStatus": socketStatus, diff --git a/internal/web/static/panel.css b/internal/web/static/panel.css index 206d15b..e3f3517 100644 --- a/internal/web/static/panel.css +++ b/internal/web/static/panel.css @@ -235,6 +235,20 @@ details form { margin-top: 0.6rem; } .st-warn { background: var(--st-warn-bg); color: var(--st-warn-fg); border-color: var(--st-warn-border); } .st-error { background: var(--st-error-bg); color: var(--st-error-fg); border-color: var(--st-error-border); } .st-unknown { background: var(--st-unknown-bg); color: var(--st-unknown-fg); border-color: var(--st-unknown-border); } +/* Usage bars on the status page's machine card. rather than a div sized + from the reading, because the CSP forbids inline styles (see the note at the + top of this file) and a bar's length has to travel on an attribute. The + element grades itself from low/high/optimum, so the colour matches the + badges' meaning without this file restating the thresholds — and a browser + that does not render meters falls back to the percentage beside it, which is + printed either way. */ +meter { width: 5rem; height: 0.7rem; vertical-align: middle; margin-right: 0.4rem; } +/* The card's own two narrow columns. Cells may break mid word by default (see + the th, td rule above), which the detail column needs and these two must not + have: the resource names and the readings are short, and the long detail + beside them would otherwise win the width and leave "Memory" broken across + two lines. */ +.metric { white-space: nowrap; } .code-row { display: flex; align-items: flex-start; gap: 0.5rem; } .code-row .code { flex: 1; min-width: 0; } /* Compact outlined button: same affordance as the filled one but quiet enough diff --git a/internal/web/templates/status_body.html b/internal/web/templates/status_body.html index 12822ec..81ce2e6 100644 --- a/internal/web/templates/status_body.html +++ b/internal/web/templates/status_body.html @@ -25,6 +25,53 @@ {{end}} +
+

Machine {{.Machine.Status}}

+

Processor, memory and network of the machine this container + runs on, read from the kernel's counters. CPU and throughput are rates, so + they describe{{if .Machine.WindowText}} the {{.Machine.WindowText}}{{end}} + since the previous refresh rather than this instant.

+ + + + + + + + + + + + + + + + + + + +
ResourceUsageDetail
CPU + {{if .Machine.CPU.Measured}} + {{.Machine.CPU.BusyText}} + {{.Machine.CPU.BusyText}} + {{else}}—{{end}} + {{.Machine.CPU.Detail}}
Memory + {{if .Machine.Memory.Measured}} + {{.Machine.Memory.PctText}} + {{.Machine.Memory.PctText}} + {{else}}—{{end}} + {{.Machine.Memory.Detail}}
Network + {{if .Machine.Network.Measured}} + ↓ {{.Machine.Network.InRateText}}
↑ {{.Machine.Network.OutRateText}} + {{else}}—{{end}} + 		+ {{range .Machine.Network.Interfaces}} +
{{.Name}}: {{.InText}} in, {{.OutText}} out{{if .Measured}} (↓ {{.InRateText}} ↑ {{.OutRateText}}){{end}}
+ {{end}} + {{if .Machine.Network.Detail}}
{{.Machine.Network.Detail}}
{{end}} +
+
+

Mail queue {{.QueueStatus}}

{{if .QueueError}} diff --git a/internal/web/templates_test.go b/internal/web/templates_test.go index 63dfa89..0adfa6f 100644 --- a/internal/web/templates_test.go +++ b/internal/web/templates_test.go @@ -211,12 +211,68 @@ func TestLayoutReferencesOnlyEmbeddedAssets(t *testing.T) { } func TestStatusPageRendersEveryCheck(t *testing.T) { + out := renderStatusPage(t, statusPageData()) + for _, want := range []string{ + "opendkim", "FATAL", "Mail queue is empty", "mail.example.com", + "203.0.113.10 → no PTR record", `action="/reload"`, + `hx-get="/status/fragment"`, `class="st st-error"`, + // The machine card: the bars carry their reading in an attribute + // (the CSP rules out sizing them with a style), and the figures are + // printed beside them for anything that does not render a meter. + `Machine `, + "/proc/stat", "/proc/meminfo", "/proc/net/dev", + } { + if !strings.Contains(out, want) { + t.Errorf("degraded machine card is missing %q", want) + } + } +} + +func renderStatusPage(t *testing.T, data map[string]any) string { + t.Helper() tmpl, err := loadTemplates() if err != nil { t.Fatalf("loadTemplates: %v", err) } var buf bytes.Buffer - err = tmpl.pages["status"].ExecuteTemplate(&buf, "layout.html", map[string]any{ + if err := tmpl.pages["status"].ExecuteTemplate(&buf, "layout.html", data); err != nil { + t.Fatalf("execute status page: %v", err) + } + return buf.String() +} + +// statusPageData is one plausible reading of every check the status page shows, +// so a test can render the page and vary the one part it is about. +func statusPageData() map[string]any { + return map[string]any{ "Title": "SelfPost — status", "User": "admin", "Active": "status", @@ -232,6 +288,26 @@ func TestStatusPageRendersEveryCheck(t *testing.T) { NotAfter: time.Now().Add(30 * 24 * time.Hour), DaysLeft: 30, Status: health.StatusOK, Detail: "Valid for another 30 day(s).", }, + "Machine": health.Machine{ + CPU: health.CPU{ + Measured: true, BusyPct: 12.4, Cores: 4, + Load: [3]float64{0.31, 0.24, 0.19}, HasLoad: true, + Status: health.StatusOK, Detail: "4 core(s) · load average 0.31, 0.24, 0.19", + }, + Memory: health.Memory{ + Measured: true, TotalBytes: 4 << 30, UsedBytes: 2 << 30, UsedPct: 50, + Status: health.StatusOK, Detail: "2.0 GiB used of 4.0 GiB; 2.0 GiB available to new work.", + }, + Network: health.Network{ + Measured: true, RxRate: 2048, TxRate: 1024, + Interfaces: []health.Interface{ + {Name: "eth0", RxBytes: 1 << 20, TxBytes: 1 << 19, RxRate: 2048, TxRate: 1024, Measured: true}, + }, + Status: health.StatusOK, + }, + Window: 5 * time.Second, + Status: health.StatusOK, + }, "Sockets": []health.Socket{ {Name: "OpenDKIM", Path: "/run/opendkim/opendkim.sock", Present: true, Status: health.StatusOK, Detail: "Listening."}, }, @@ -244,19 +320,6 @@ func TestStatusPageRendersEveryCheck(t *testing.T) { Detail: "No address has a reverse record.", Records: []string{"203.0.113.10 → no PTR record"}, }, - }) - if err != nil { - t.Fatalf("execute status page: %v", err) - } - out := buf.String() - for _, want := range []string{ - "opendkim", "FATAL", "Mail queue is empty", "mail.example.com", - "203.0.113.10 → no PTR record", `action="/reload"`, - `hx-get="/status/fragment"`, `class="st st-error"`, - } { - if !strings.Contains(out, want) { - t.Errorf("status page is missing %q", want) - } } } diff --git a/internal/web/web.go b/internal/web/web.go index b23b3f7..32bd1bf 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -84,6 +84,11 @@ type Server struct { sessions *sessionStore setup *setupManager dns *dnscheck.Checker + // machine reads the host's CPU, memory and network counters for the + // status page. It has to be one shared sampler for the whole server: + // CPU and throughput are differences between successive readings, so a + // per-request sampler would never have a previous one to subtract. + machine health.MachineSampler loginLimiter *rateLimiter setupLimiter *rateLimiter