release: 0.6.0: Delivery detail page, DNS badge, machine metrics, wide layout
A page per delivery (`/deliveries/{id}`), reached from the *Details* link on
Added:
- A page per delivery (`/deliveries/{id}`), reached from the *Details* link on
- A **DNS** badge in the domain list, one per row, carrying the same
- Machine metrics on the status page: a **Machine** card reporting the
- An index of the current page's own sections in the navigation column, for the
Changed:
- Every panel page is laid out in one column of the same width, so moving
- The mark's small-size variant — the tab icon and the `SP` initials it
- The stamp's `SELF-HOSTED SMTP RELAY` line is set at 11.5/0.15 instead of
- The delivery log lists what identifies a message and nothing else: time,
- Subjects are decoded for display as well as on the way in, so the rows the
… (4 more in CHANGELOG)
Full detail: CHANGELOG.md (0.6.0).
This commit is contained in:
@@ -0,0 +1,557 @@
|
||||
package health
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// machineSampleWindow is the longest gap between two readings that still counts
|
||||
// as "now". CPU busy time and network throughput are differences between two
|
||||
// samples, and the status fragment polls every five seconds — but a page opened
|
||||
// after the panel sat idle for an hour would otherwise average that whole hour
|
||||
// and present it as the current load. Past this the reading only re-baselines,
|
||||
// and the rates say they are still being measured.
|
||||
const machineSampleWindow = time.Minute
|
||||
|
||||
// Thresholds for the two resources whose exhaustion actually threatens mail: a
|
||||
// fully busy processor slows queue processing, and a machine out of memory has
|
||||
// its processes killed. Network throughput has no comparable threshold — what
|
||||
// counts as a lot depends entirely on the link — so it is reported, not graded.
|
||||
const (
|
||||
cpuWarnPct = 90.0
|
||||
memWarnPct = 90.0
|
||||
memErrorPct = 97.0
|
||||
)
|
||||
|
||||
// Machine is the resource usage of the machine this container runs on: the
|
||||
// status page's answer to "is the server itself under strain", next to the
|
||||
// component checks that answer "is mail flowing".
|
||||
//
|
||||
// The numbers come from the kernel's /proc filesystem, which a container shares
|
||||
// with its host unless it was started with its own namespaces — so CPU and
|
||||
// memory describe the host, while /proc/net/dev describes whatever network
|
||||
// namespace the container is in (its own veth under the default bridge, the
|
||||
// host's interfaces under network_mode: host).
|
||||
type Machine struct {
|
||||
CPU CPU
|
||||
Memory Memory
|
||||
Network Network
|
||||
// Window is the interval the rates were measured over; zero until a
|
||||
// second reading exists.
|
||||
Window time.Duration
|
||||
Status Status
|
||||
}
|
||||
|
||||
// WindowText names the sampling interval for the card's description.
|
||||
func (m Machine) WindowText() string {
|
||||
if m.Window <= 0 {
|
||||
return ""
|
||||
}
|
||||
return m.Window.Round(time.Second).String()
|
||||
}
|
||||
|
||||
// CPU is processor load over the sampling window.
|
||||
type CPU struct {
|
||||
// Measured is false until two readings exist to compare; BusyPct means
|
||||
// nothing until it is true.
|
||||
Measured bool
|
||||
BusyPct float64
|
||||
Cores int
|
||||
// Load is the 1/5/15-minute load average, present when /proc/loadavg
|
||||
// could be read. Unlike BusyPct it needs no previous sample, so it is
|
||||
// there on the very first page load.
|
||||
Load [3]float64
|
||||
HasLoad bool
|
||||
Status Status
|
||||
Detail string
|
||||
}
|
||||
|
||||
// Percent is BusyPct as a whole number, for the <meter> element's value
|
||||
// attribute. The bar carries its value in an attribute rather than a width in a
|
||||
// style attribute because the panel's CSP has no inline-style exemption
|
||||
// (security.md).
|
||||
func (c CPU) Percent() int { return percent(c.BusyPct) }
|
||||
|
||||
// BusyText is the reading as it appears beside the bar.
|
||||
func (c CPU) BusyText() string { return fmt.Sprintf("%d%%", c.Percent()) }
|
||||
|
||||
// Memory is main memory (and swap, where the machine has any) at the moment of
|
||||
// the reading. Unlike CPU and network it is a level, not a rate, so a single
|
||||
// reading is enough and it is never in the "measuring" state.
|
||||
type Memory struct {
|
||||
Measured bool
|
||||
TotalBytes uint64
|
||||
AvailableBytes uint64
|
||||
UsedBytes uint64
|
||||
UsedPct float64
|
||||
SwapTotalBytes uint64
|
||||
SwapUsedBytes uint64
|
||||
Status Status
|
||||
Detail string
|
||||
}
|
||||
|
||||
// Percent, UsedText and TotalText render the reading for the template; see
|
||||
// CPU.Percent for why the bar's value travels as an attribute.
|
||||
func (m Memory) Percent() int { return percent(m.UsedPct) }
|
||||
func (m Memory) UsedText() string { return humanBytes(m.UsedBytes) }
|
||||
func (m Memory) TotalText() string { return humanBytes(m.TotalBytes) }
|
||||
func (m Memory) PctText() string { return fmt.Sprintf("%d%%", m.Percent()) }
|
||||
|
||||
// Interface is one network interface's traffic: the counters since the
|
||||
// interface came up, and the throughput over the sampling window.
|
||||
type Interface struct {
|
||||
Name string
|
||||
RxBytes uint64
|
||||
TxBytes uint64
|
||||
RxRate float64 // bytes per second, valid when the Network is Measured
|
||||
TxRate float64
|
||||
Measured bool
|
||||
}
|
||||
|
||||
func (i Interface) InText() string { return humanBytes(i.RxBytes) }
|
||||
func (i Interface) OutText() string { return humanBytes(i.TxBytes) }
|
||||
func (i Interface) InRateText() string { return humanRate(i.RxRate) }
|
||||
func (i Interface) OutRateText() string { return humanRate(i.TxRate) }
|
||||
|
||||
// Network is the traffic across every interface that has carried any, loopback
|
||||
// excluded — loopback traffic is the container talking to itself (the panel to
|
||||
// SQLite, Postfix to its milters) and says nothing about the link.
|
||||
type Network struct {
|
||||
Measured bool
|
||||
Interfaces []Interface
|
||||
RxRate float64
|
||||
TxRate float64
|
||||
Status Status
|
||||
Detail string
|
||||
}
|
||||
|
||||
func (n Network) InRateText() string { return humanRate(n.RxRate) }
|
||||
func (n Network) OutRateText() string { return humanRate(n.TxRate) }
|
||||
|
||||
// MachineSampler reads those counters. Its zero value is ready to use and it is
|
||||
// safe for concurrent use, but one sampler has to be shared by every caller:
|
||||
// the rates are measured against the reading the previous call left behind, so
|
||||
// a fresh sampler per request would never have anything to compare against.
|
||||
type MachineSampler struct {
|
||||
// procRoot replaces /proc in tests; empty means the real one.
|
||||
procRoot string
|
||||
|
||||
mu sync.Mutex
|
||||
prevAt time.Time
|
||||
prevCPU cpuTimes
|
||||
prevNet map[string]netCounters
|
||||
}
|
||||
|
||||
func (m *MachineSampler) root() string {
|
||||
if m.procRoot == "" {
|
||||
return "/proc"
|
||||
}
|
||||
return m.procRoot
|
||||
}
|
||||
|
||||
// cpuTimes is the aggregate of /proc/stat's "cpu" line: all time accounted for,
|
||||
// and the part of it the processor spent doing nothing.
|
||||
type cpuTimes struct {
|
||||
total uint64
|
||||
idle uint64
|
||||
}
|
||||
|
||||
// netCounters is one interface's byte counters from /proc/net/dev.
|
||||
type netCounters struct {
|
||||
rx uint64
|
||||
tx uint64
|
||||
}
|
||||
|
||||
// Sample reads the current counters and reports usage since the previous call.
|
||||
// Like every other check here it never fails: a counter that cannot be read
|
||||
// becomes an unknown status with an explanation, so a kernel that does not
|
||||
// publish one of these files (or a panel run outside Linux for development)
|
||||
// costs one line of the card rather than the page.
|
||||
func (m *MachineSampler) Sample() Machine {
|
||||
root := m.root()
|
||||
now := time.Now()
|
||||
cpuNow, cores, cpuErr := readCPUTimes(root)
|
||||
netNow, netErr := readNetDev(root)
|
||||
|
||||
m.mu.Lock()
|
||||
prevAt, prevCPU, prevNet := m.prevAt, m.prevCPU, m.prevNet
|
||||
if cpuErr == nil {
|
||||
m.prevCPU = cpuNow
|
||||
}
|
||||
if netErr == nil {
|
||||
m.prevNet = netNow
|
||||
}
|
||||
if cpuErr == nil || netErr == nil {
|
||||
m.prevAt = now
|
||||
}
|
||||
m.mu.Unlock()
|
||||
|
||||
// A window of zero (two calls in the same instant) would divide by zero;
|
||||
// one longer than machineSampleWindow is no longer a description of now.
|
||||
window := now.Sub(prevAt)
|
||||
fresh := !prevAt.IsZero() && window > 0 && window <= machineSampleWindow
|
||||
|
||||
mach := Machine{
|
||||
CPU: cpuUsage(prevCPU, cpuNow, cores, readLoadAvg(root), fresh, cpuErr),
|
||||
Memory: readMemory(root),
|
||||
Network: networkUsage(prevNet, netNow, window, fresh, netErr),
|
||||
}
|
||||
if fresh {
|
||||
mach.Window = window
|
||||
}
|
||||
mach.Status = Worst(mach.CPU.Status, mach.Memory.Status, mach.Network.Status)
|
||||
return mach
|
||||
}
|
||||
|
||||
// cpuUsage grades the processor over the window. The load average is reported
|
||||
// alongside it because the two answer different questions — how busy the
|
||||
// processor was, and how many tasks were waiting for it — and a machine can
|
||||
// look idle while work queues up behind a slow disk.
|
||||
func cpuUsage(prev, cur cpuTimes, cores int, load [3]float64, fresh bool, err error) CPU {
|
||||
c := CPU{Cores: cores}
|
||||
if !isZeroLoad(load) {
|
||||
c.Load, c.HasLoad = load, true
|
||||
}
|
||||
if err != nil {
|
||||
c.Status = StatusUnknown
|
||||
c.Detail = "The kernel's processor counters (/proc/stat) could not be read here."
|
||||
return c
|
||||
}
|
||||
if !fresh || cur.total <= prev.total {
|
||||
c.Status = StatusUnknown
|
||||
c.Detail = joinDetail("Measuring — this reading sets the baseline; the next refresh has the figure.", c.contextText())
|
||||
return c
|
||||
}
|
||||
|
||||
total := cur.total - prev.total
|
||||
idle := uint64(0)
|
||||
if cur.idle > prev.idle {
|
||||
idle = cur.idle - prev.idle
|
||||
}
|
||||
if idle > total {
|
||||
idle = total
|
||||
}
|
||||
c.Measured = true
|
||||
c.BusyPct = 100 * float64(total-idle) / float64(total)
|
||||
|
||||
if c.BusyPct >= cpuWarnPct {
|
||||
c.Status = StatusWarn
|
||||
c.Detail = joinDetail(c.contextText(), "The processor is close to fully busy, which slows queue processing and every panel page.")
|
||||
} else {
|
||||
c.Status = StatusOK
|
||||
c.Detail = c.contextText()
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// contextText is the CPU's supporting figures: what the percentage is a
|
||||
// percentage of, and how deep the run queue is.
|
||||
func (c CPU) contextText() string {
|
||||
var parts []string
|
||||
if c.Cores > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d core(s)", c.Cores))
|
||||
}
|
||||
if c.HasLoad {
|
||||
parts = append(parts, fmt.Sprintf("load average %.2f, %.2f, %.2f", c.Load[0], c.Load[1], c.Load[2]))
|
||||
}
|
||||
return strings.Join(parts, " · ")
|
||||
}
|
||||
|
||||
// readMemory reports main memory from /proc/meminfo. Used is derived from
|
||||
// MemAvailable rather than MemFree: Linux spends every spare page on cache, so
|
||||
// MemFree on a healthy machine is near zero and would report a permanent
|
||||
// emergency. MemAvailable is the kernel's own estimate of what a new workload
|
||||
// could actually get.
|
||||
func readMemory(root string) Memory {
|
||||
var m Memory
|
||||
fields, err := readMeminfo(root)
|
||||
if err != nil {
|
||||
m.Status = StatusUnknown
|
||||
m.Detail = "The kernel's memory counters (/proc/meminfo) could not be read here."
|
||||
return m
|
||||
}
|
||||
total, available := fields["MemTotal"], fields["MemAvailable"]
|
||||
if total == 0 {
|
||||
m.Status = StatusUnknown
|
||||
m.Detail = "/proc/meminfo does not report a total memory size."
|
||||
return m
|
||||
}
|
||||
if available > total {
|
||||
available = total
|
||||
}
|
||||
|
||||
m.Measured = true
|
||||
m.TotalBytes = total
|
||||
m.AvailableBytes = available
|
||||
m.UsedBytes = total - available
|
||||
m.UsedPct = 100 * float64(m.UsedBytes) / float64(total)
|
||||
m.SwapTotalBytes = fields["SwapTotal"]
|
||||
if swapFree := fields["SwapFree"]; m.SwapTotalBytes > swapFree {
|
||||
m.SwapUsedBytes = m.SwapTotalBytes - swapFree
|
||||
}
|
||||
|
||||
detail := fmt.Sprintf("%s used of %s; %s available to new work.",
|
||||
humanBytes(m.UsedBytes), humanBytes(total), humanBytes(available))
|
||||
if m.SwapTotalBytes > 0 {
|
||||
detail += fmt.Sprintf(" Swap: %s of %s.", humanBytes(m.SwapUsedBytes), humanBytes(m.SwapTotalBytes))
|
||||
}
|
||||
switch {
|
||||
case m.UsedPct >= memErrorPct:
|
||||
m.Status = StatusError
|
||||
m.Detail = detail + " Memory is exhausted; the kernel kills processes to reclaim it, and Postfix or the panel are candidates."
|
||||
case m.UsedPct >= memWarnPct:
|
||||
m.Status = StatusWarn
|
||||
m.Detail = detail + " Little headroom left."
|
||||
default:
|
||||
m.Status = StatusOK
|
||||
m.Detail = detail
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// networkUsage turns two readings of the interface counters into throughput.
|
||||
// It never grades: there is no usage figure that is wrong for a mail server, so
|
||||
// the row is informational and only reports "unknown" when the counters are
|
||||
// unreadable.
|
||||
func networkUsage(prev, cur map[string]netCounters, window time.Duration, fresh bool, err error) Network {
|
||||
var n Network
|
||||
if err != nil {
|
||||
n.Status = StatusUnknown
|
||||
n.Detail = "The kernel's network counters (/proc/net/dev) could not be read here."
|
||||
return n
|
||||
}
|
||||
|
||||
names := make([]string, 0, len(cur))
|
||||
for name := range cur {
|
||||
names = append(names, name)
|
||||
}
|
||||
// Map order is random, and this table is re-rendered every few seconds:
|
||||
// without a sort the rows would shuffle under the reader.
|
||||
sort.Strings(names)
|
||||
|
||||
n.Measured = fresh
|
||||
for _, name := range names {
|
||||
c := cur[name]
|
||||
// An interface that has never carried a byte is a veth or a bridge
|
||||
// the deployment happens to have, not part of the mail path.
|
||||
if c.rx == 0 && c.tx == 0 {
|
||||
continue
|
||||
}
|
||||
iface := Interface{Name: name, RxBytes: c.rx, TxBytes: c.tx, Measured: fresh}
|
||||
if fresh {
|
||||
p := prev[name]
|
||||
secs := window.Seconds()
|
||||
// Counters only go up; a drop means the interface (or the
|
||||
// container) was recreated, so there is no rate to report.
|
||||
if c.rx >= p.rx {
|
||||
iface.RxRate = float64(c.rx-p.rx) / secs
|
||||
}
|
||||
if c.tx >= p.tx {
|
||||
iface.TxRate = float64(c.tx-p.tx) / secs
|
||||
}
|
||||
n.RxRate += iface.RxRate
|
||||
n.TxRate += iface.TxRate
|
||||
}
|
||||
n.Interfaces = append(n.Interfaces, iface)
|
||||
}
|
||||
|
||||
n.Status = StatusOK
|
||||
switch {
|
||||
case len(n.Interfaces) == 0:
|
||||
n.Detail = "No interface outside loopback has carried any traffic."
|
||||
case !fresh:
|
||||
n.Detail = "Measuring — this reading sets the baseline; the next refresh has the throughput."
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// readCPUTimes returns the aggregate processor times and the number of cores
|
||||
// from /proc/stat. The times are in USER_HZ ticks, which cancel out because
|
||||
// only their ratio is used.
|
||||
func readCPUTimes(root string) (cpuTimes, int, error) {
|
||||
data, err := os.ReadFile(filepath.Join(root, "stat"))
|
||||
if err != nil {
|
||||
return cpuTimes{}, 0, err
|
||||
}
|
||||
var (
|
||||
times cpuTimes
|
||||
cores int
|
||||
found bool
|
||||
)
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 5 || !strings.HasPrefix(fields[0], "cpu") {
|
||||
continue
|
||||
}
|
||||
if fields[0] != "cpu" {
|
||||
cores++ // cpu0, cpu1, … — one line per core
|
||||
continue
|
||||
}
|
||||
// user nice system idle iowait irq softirq steal (guest fields are
|
||||
// already counted inside user/nice, so they are left out).
|
||||
for i, f := range fields[1:] {
|
||||
if i >= 8 {
|
||||
break
|
||||
}
|
||||
v, err := strconv.ParseUint(f, 10, 64)
|
||||
if err != nil {
|
||||
return cpuTimes{}, 0, fmt.Errorf("/proc/stat: unreadable cpu time %q", f)
|
||||
}
|
||||
times.total += v
|
||||
// idle plus iowait: both are time the processor had nothing to
|
||||
// run, and separating them tells the reader nothing here.
|
||||
if i == 3 || i == 4 {
|
||||
times.idle += v
|
||||
}
|
||||
}
|
||||
found = true
|
||||
}
|
||||
if !found {
|
||||
return cpuTimes{}, 0, fmt.Errorf("/proc/stat: no aggregate cpu line")
|
||||
}
|
||||
return times, cores, nil
|
||||
}
|
||||
|
||||
// readLoadAvg reads the 1/5/15-minute load averages. A machine without
|
||||
// /proc/loadavg simply has none reported, so the failure is a zero value rather
|
||||
// than an error.
|
||||
func readLoadAvg(root string) [3]float64 {
|
||||
var load [3]float64
|
||||
data, err := os.ReadFile(filepath.Join(root, "loadavg"))
|
||||
if err != nil {
|
||||
return load
|
||||
}
|
||||
fields := strings.Fields(string(data))
|
||||
if len(fields) < 3 {
|
||||
return load
|
||||
}
|
||||
for i := range load {
|
||||
v, err := strconv.ParseFloat(fields[i], 64)
|
||||
if err != nil {
|
||||
return [3]float64{}
|
||||
}
|
||||
load[i] = v
|
||||
}
|
||||
return load
|
||||
}
|
||||
|
||||
func isZeroLoad(load [3]float64) bool {
|
||||
return load == [3]float64{}
|
||||
}
|
||||
|
||||
// readMeminfo returns the /proc/meminfo entries this package uses, in bytes.
|
||||
// The file reports kB (kibibytes, despite the label) for these fields.
|
||||
func readMeminfo(root string) (map[string]uint64, error) {
|
||||
data, err := os.ReadFile(filepath.Join(root, "meminfo"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
want := map[string]bool{"MemTotal": true, "MemAvailable": true, "SwapTotal": true, "SwapFree": true}
|
||||
out := make(map[string]uint64, len(want))
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
name, rest, ok := strings.Cut(line, ":")
|
||||
if !ok || !want[name] {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(rest)
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
v, err := strconv.ParseUint(fields[0], 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if len(fields) > 1 && strings.EqualFold(fields[1], "kB") {
|
||||
v *= 1024
|
||||
}
|
||||
out[name] = v
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("/proc/meminfo: no readable fields")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// readNetDev returns each interface's byte counters from /proc/net/dev, loopback
|
||||
// excluded. Each line is " eth0: <rx bytes> <rx packets> … <tx bytes> …" — the
|
||||
// name is separated by a colon, which may or may not have a space after it, so
|
||||
// the split is on the colon and not on whitespace.
|
||||
func readNetDev(root string) (map[string]netCounters, error) {
|
||||
data, err := os.ReadFile(filepath.Join(root, "net", "dev"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]netCounters)
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
name, rest, ok := strings.Cut(line, ":")
|
||||
name = strings.TrimSpace(name)
|
||||
if !ok || name == "" || name == "lo" || strings.Contains(name, " ") {
|
||||
continue // header lines carry no name, or several words
|
||||
}
|
||||
fields := strings.Fields(rest)
|
||||
if len(fields) < 9 {
|
||||
continue
|
||||
}
|
||||
rx, err1 := strconv.ParseUint(fields[0], 10, 64)
|
||||
tx, err2 := strconv.ParseUint(fields[8], 10, 64)
|
||||
if err1 != nil || err2 != nil {
|
||||
continue
|
||||
}
|
||||
out[name] = netCounters{rx: rx, tx: tx}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// percent rounds a 0–100 reading to a whole number and clamps it, so a bar's
|
||||
// value attribute is always inside the range the element declares.
|
||||
func percent(v float64) int {
|
||||
switch {
|
||||
case v <= 0:
|
||||
return 0
|
||||
case v >= 100:
|
||||
return 100
|
||||
default:
|
||||
return int(v + 0.5)
|
||||
}
|
||||
}
|
||||
|
||||
// humanBytes renders a byte count in the binary units memory and traffic are
|
||||
// conventionally read in.
|
||||
func humanBytes(b uint64) string {
|
||||
const unit = 1024
|
||||
if b < unit {
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
div, exp := uint64(unit), 0
|
||||
for n := b / unit; n >= unit && exp < 4; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTP"[exp])
|
||||
}
|
||||
|
||||
// humanRate renders throughput in bytes per second, to match the totals beside
|
||||
// it rather than the bits per second a link is sold in.
|
||||
func humanRate(perSec float64) string {
|
||||
if perSec < 0 {
|
||||
perSec = 0
|
||||
}
|
||||
return humanBytes(uint64(perSec+0.5)) + "/s"
|
||||
}
|
||||
|
||||
// joinDetail joins the non-empty parts of a detail line.
|
||||
func joinDetail(parts ...string) string {
|
||||
var kept []string
|
||||
for _, p := range parts {
|
||||
if p != "" {
|
||||
kept = append(kept, p)
|
||||
}
|
||||
}
|
||||
return strings.Join(kept, " ")
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
package health
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeProc writes a /proc-shaped directory the sampler can be pointed at, so
|
||||
// the parsing and the arithmetic are tested against known counters instead of
|
||||
// whatever the machine running the tests happens to be doing.
|
||||
func fakeProc(t *testing.T, stat, meminfo, loadavg, netdev string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
write := func(name, body string) {
|
||||
if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write("stat", stat)
|
||||
write("meminfo", meminfo)
|
||||
write("loadavg", loadavg)
|
||||
write(filepath.Join("net", "dev"), netdev)
|
||||
return dir
|
||||
}
|
||||
|
||||
const meminfoSample = `MemTotal: 4194304 kB
|
||||
MemFree: 131072 kB
|
||||
MemAvailable: 2097152 kB
|
||||
Buffers: 262144 kB
|
||||
SwapTotal: 1048576 kB
|
||||
SwapFree: 524288 kB
|
||||
`
|
||||
|
||||
const netdevSample = `Inter-| Receive | Transmit
|
||||
face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed
|
||||
lo: 500000 1000 0 0 0 0 0 0 500000 1000 0 0 0 0 0 0
|
||||
eth0: 1048576 2000 0 0 0 0 0 0 524288 1500 0 0 0 0 0 0
|
||||
veth9: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
`
|
||||
|
||||
func TestCPUUsageOverTwoReadings(t *testing.T) {
|
||||
// 1000 ticks pass, 250 of them idle: 75% busy.
|
||||
prev := cpuTimes{total: 10000, idle: 8000}
|
||||
cur := cpuTimes{total: 11000, idle: 8250}
|
||||
got := cpuUsage(prev, cur, 4, [3]float64{0.5, 0.4, 0.3}, true, nil)
|
||||
if !got.Measured {
|
||||
t.Fatalf("reading not marked measured: %+v", got)
|
||||
}
|
||||
if got.Percent() != 75 {
|
||||
t.Errorf("busy = %.2f%% (%s), want 75%%", got.BusyPct, got.BusyText())
|
||||
}
|
||||
if got.Status != StatusOK {
|
||||
t.Errorf("status = %q, want ok", got.Status)
|
||||
}
|
||||
if got.Cores != 4 || !got.HasLoad {
|
||||
t.Errorf("cores/load not reported: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCPUUsageWarnsWhenFullyBusy(t *testing.T) {
|
||||
got := cpuUsage(cpuTimes{total: 10000, idle: 5000}, cpuTimes{total: 11000, idle: 5010}, 1, [3]float64{}, true, nil)
|
||||
if got.Status != StatusWarn {
|
||||
t.Errorf("99%% busy graded %q, want warn (%s)", got.Status, got.Detail)
|
||||
}
|
||||
if got.HasLoad {
|
||||
t.Error("a missing load average should not be reported as zeros")
|
||||
}
|
||||
}
|
||||
|
||||
// The first reading has nothing to compare against, and one taken after a long
|
||||
// idle stretch would describe that stretch rather than now. Both must report
|
||||
// "unknown" rather than a number the reader would take for the current load.
|
||||
func TestCPUUsageWithoutAUsableWindow(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
prev, cur cpuTimes
|
||||
fresh bool
|
||||
}{
|
||||
{"no previous reading", cpuTimes{}, cpuTimes{total: 11000, idle: 8250}, false},
|
||||
{"counters did not advance", cpuTimes{total: 11000, idle: 8250}, cpuTimes{total: 11000, idle: 8250}, true},
|
||||
} {
|
||||
got := cpuUsage(c.prev, c.cur, 2, [3]float64{}, c.fresh, nil)
|
||||
if got.Measured || got.Status != StatusUnknown {
|
||||
t.Errorf("%s: measured=%v status=%q, want unmeasured/unknown", c.name, got.Measured, got.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if got := cpuUsage(cpuTimes{}, cpuTimes{}, 0, [3]float64{}, false, os.ErrNotExist); got.Status != StatusUnknown {
|
||||
t.Errorf("unreadable /proc/stat: status %q, want unknown", got.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadMemory(t *testing.T) {
|
||||
dir := fakeProc(t, "cpu 1 1 1 1 1 1 1 1\n", meminfoSample, "0.1 0.2 0.3 1/2 3\n", netdevSample)
|
||||
got := readMemory(dir)
|
||||
if !got.Measured {
|
||||
t.Fatalf("memory not measured: %+v", got)
|
||||
}
|
||||
if got.TotalBytes != 4*1024*1024*1024 {
|
||||
t.Errorf("total = %d bytes (%s), want 4 GiB", got.TotalBytes, got.TotalText())
|
||||
}
|
||||
// 4 GiB total, 2 GiB available to new work: half used, cache included in
|
||||
// what is available.
|
||||
if got.Percent() != 50 {
|
||||
t.Errorf("used = %.1f%% (%s), want 50%%", got.UsedPct, got.PctText())
|
||||
}
|
||||
if got.Status != StatusOK {
|
||||
t.Errorf("status = %q, want ok (%s)", got.Status, got.Detail)
|
||||
}
|
||||
if got.SwapUsedBytes != 512*1024*1024 {
|
||||
t.Errorf("swap used = %d bytes, want 512 MiB", got.SwapUsedBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadMemoryGrades(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
available string
|
||||
want Status
|
||||
}{
|
||||
{"plenty free", "MemAvailable: 2097152 kB\n", StatusOK},
|
||||
{"little headroom", "MemAvailable: 209715 kB\n", StatusWarn},
|
||||
{"exhausted", "MemAvailable: 41943 kB\n", StatusError},
|
||||
}
|
||||
for _, c := range cases {
|
||||
dir := fakeProc(t, "cpu 1 1 1 1 1 1 1 1\n", "MemTotal: 4194304 kB\n"+c.available, "", netdevSample)
|
||||
if got := readMemory(dir); got.Status != c.want {
|
||||
t.Errorf("%s: status %q, want %q (%s)", c.name, got.Status, c.want, got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
if got := readMemory(t.TempDir()); got.Status != StatusUnknown || got.Measured {
|
||||
t.Errorf("missing /proc/meminfo: status %q measured=%v", got.Status, got.Measured)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadNetDevSkipsLoopbackAndHeaders(t *testing.T) {
|
||||
dir := fakeProc(t, "cpu 1 1 1 1 1 1 1 1\n", meminfoSample, "", netdevSample)
|
||||
got, err := readNetDev(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("readNetDev: %v", err)
|
||||
}
|
||||
if _, ok := got["lo"]; ok {
|
||||
t.Error("loopback is counted as network traffic")
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("parsed %d interfaces, want eth0 and veth9: %+v", len(got), got)
|
||||
}
|
||||
if got["eth0"].rx != 1048576 || got["eth0"].tx != 524288 {
|
||||
t.Errorf("eth0 counters = %+v", got["eth0"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetworkUsageRates(t *testing.T) {
|
||||
prev := map[string]netCounters{"eth0": {rx: 1000, tx: 500}}
|
||||
cur := map[string]netCounters{
|
||||
"eth0": {rx: 11000, tx: 5500},
|
||||
"veth9": {rx: 0, tx: 0}, // never carried anything: not shown
|
||||
}
|
||||
got := networkUsage(prev, cur, 5*time.Second, true, nil)
|
||||
if len(got.Interfaces) != 1 || got.Interfaces[0].Name != "eth0" {
|
||||
t.Fatalf("interfaces = %+v, want eth0 only", got.Interfaces)
|
||||
}
|
||||
// 10000 bytes in and 5000 out over five seconds.
|
||||
if got.RxRate != 2000 || got.TxRate != 1000 {
|
||||
t.Errorf("rates = %.0f in / %.0f out, want 2000/1000", got.RxRate, got.TxRate)
|
||||
}
|
||||
if got.InRateText() != "2.0 KiB/s" {
|
||||
t.Errorf("in rate text = %q", got.InRateText())
|
||||
}
|
||||
if got.Status != StatusOK {
|
||||
t.Errorf("status = %q, want ok — throughput is reported, not graded", got.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// A recreated container (or interface) resets the counters; the drop must not
|
||||
// be reported as a huge negative or wrapped-around rate.
|
||||
func TestNetworkUsageIgnoresCounterResets(t *testing.T) {
|
||||
prev := map[string]netCounters{"eth0": {rx: 1_000_000, tx: 900_000}}
|
||||
cur := map[string]netCounters{"eth0": {rx: 1000, tx: 900}}
|
||||
got := networkUsage(prev, cur, 5*time.Second, true, nil)
|
||||
if got.RxRate != 0 || got.TxRate != 0 {
|
||||
t.Errorf("rates after a counter reset = %.0f/%.0f, want 0/0", got.RxRate, got.TxRate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetworkUsageWithoutAPreviousReading(t *testing.T) {
|
||||
cur := map[string]netCounters{"eth0": {rx: 1000, tx: 500}}
|
||||
got := networkUsage(nil, cur, 0, false, nil)
|
||||
if got.Measured {
|
||||
t.Error("first reading reported as measured")
|
||||
}
|
||||
if len(got.Interfaces) != 1 || got.Interfaces[0].InText() != "1000 B" {
|
||||
t.Errorf("totals should be shown even before a rate exists: %+v", got.Interfaces)
|
||||
}
|
||||
if got.Detail == "" {
|
||||
t.Error("no explanation for the missing rates")
|
||||
}
|
||||
|
||||
if bad := networkUsage(nil, nil, 0, false, os.ErrNotExist); bad.Status != StatusUnknown {
|
||||
t.Errorf("unreadable /proc/net/dev: status %q, want unknown", bad.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// End to end through the sampler: the first call baselines, the second reports.
|
||||
func TestMachineSamplerNeedsTwoReadings(t *testing.T) {
|
||||
dir := fakeProc(t,
|
||||
"cpu 1000 0 500 8000 500 0 0 0\ncpu0 500 0 250 4000 250 0 0 0\ncpu1 500 0 250 4000 250 0 0 0\n",
|
||||
meminfoSample, "0.42 0.31 0.20 2/300 1234\n", netdevSample)
|
||||
m := &MachineSampler{procRoot: dir}
|
||||
|
||||
first := m.Sample()
|
||||
if first.CPU.Measured || first.CPU.Status != StatusUnknown {
|
||||
t.Errorf("first sample reported a CPU figure: %+v", first.CPU)
|
||||
}
|
||||
if !first.Memory.Measured {
|
||||
t.Error("memory is a level, not a rate: it must be reported on the first sample")
|
||||
}
|
||||
if first.CPU.Cores != 2 {
|
||||
t.Errorf("cores = %d, want 2", first.CPU.Cores)
|
||||
}
|
||||
if !first.CPU.HasLoad || first.CPU.Load[0] != 0.42 {
|
||||
t.Errorf("load average not read: %+v", first.CPU.Load)
|
||||
}
|
||||
// Unknown checks must not drag the card into a warning.
|
||||
if first.Status != StatusOK {
|
||||
t.Errorf("overall machine status = %q, want ok", first.Status)
|
||||
}
|
||||
|
||||
// Second reading: 1000 more ticks, 750 of them idle → 25% busy.
|
||||
if err := os.WriteFile(filepath.Join(dir, "stat"),
|
||||
[]byte("cpu 1250 0 500 8750 500 0 0 0\ncpu0 625 0 250 4375 250 0 0 0\ncpu1 625 0 250 4375 250 0 0 0\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := m.Sample()
|
||||
if !second.CPU.Measured {
|
||||
t.Fatalf("second sample still unmeasured: %+v", second.CPU)
|
||||
}
|
||||
if second.CPU.Percent() != 25 {
|
||||
t.Errorf("busy = %s, want 25%%", second.CPU.BusyText())
|
||||
}
|
||||
if !second.Network.Measured || second.Window <= 0 {
|
||||
t.Errorf("network window not established: measured=%v window=%v", second.Network.Measured, second.Window)
|
||||
}
|
||||
if second.WindowText() == "" {
|
||||
t.Error("no sampling window to show on the card")
|
||||
}
|
||||
}
|
||||
|
||||
// Outside Linux — a developer running the panel on their own machine — there is
|
||||
// no /proc at all. Every metric must degrade to "unknown" rather than failing
|
||||
// the status page.
|
||||
func TestMachineSamplerWithoutProc(t *testing.T) {
|
||||
m := &MachineSampler{procRoot: filepath.Join(t.TempDir(), "absent")}
|
||||
got := m.Sample()
|
||||
if got.Status != StatusUnknown {
|
||||
t.Errorf("status = %q, want unknown", got.Status)
|
||||
}
|
||||
for name, st := range map[string]Status{"cpu": got.CPU.Status, "memory": got.Memory.Status, "network": got.Network.Status} {
|
||||
if st != StatusUnknown {
|
||||
t.Errorf("%s status = %q, want unknown", name, st)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHumanBytes(t *testing.T) {
|
||||
cases := map[uint64]string{
|
||||
0: "0 B",
|
||||
999: "999 B",
|
||||
1024: "1.0 KiB",
|
||||
1536: "1.5 KiB",
|
||||
4 * 1024 * 1024 * 1024: "4.0 GiB",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := humanBytes(in); got != want {
|
||||
t.Errorf("humanBytes(%d) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
if got := humanRate(0); got != "0 B/s" {
|
||||
t.Errorf("humanRate(0) = %q", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user