release: 0.6.0: Delivery detail page, DNS badge, machine metrics, wide layout
A page per delivery (`/deliveries/{id}`), reached from the *Details* link on
Added:
- A page per delivery (`/deliveries/{id}`), reached from the *Details* link on
- A **DNS** badge in the domain list, one per row, carrying the same
- Machine metrics on the status page: a **Machine** card reporting the
- An index of the current page's own sections in the navigation column, for the
Changed:
- Every panel page is laid out in one column of the same width, so moving
- The mark's small-size variant — the tab icon and the `SP` initials it
- The stamp's `SELF-HOSTED SMTP RELAY` line is set at 11.5/0.15 instead of
- The delivery log lists what identifies a message and nothing else: time,
- Subjects are decoded for display as well as on the way in, so the rows the
… (4 more in CHANGELOG)
Full detail: CHANGELOG.md (0.6.0).
This commit is contained in:
@@ -0,0 +1,557 @@
|
||||
package health
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// machineSampleWindow is the longest gap between two readings that still counts
|
||||
// as "now". CPU busy time and network throughput are differences between two
|
||||
// samples, and the status fragment polls every five seconds — but a page opened
|
||||
// after the panel sat idle for an hour would otherwise average that whole hour
|
||||
// and present it as the current load. Past this the reading only re-baselines,
|
||||
// and the rates say they are still being measured.
|
||||
const machineSampleWindow = time.Minute
|
||||
|
||||
// Thresholds for the two resources whose exhaustion actually threatens mail: a
|
||||
// fully busy processor slows queue processing, and a machine out of memory has
|
||||
// its processes killed. Network throughput has no comparable threshold — what
|
||||
// counts as a lot depends entirely on the link — so it is reported, not graded.
|
||||
const (
|
||||
cpuWarnPct = 90.0
|
||||
memWarnPct = 90.0
|
||||
memErrorPct = 97.0
|
||||
)
|
||||
|
||||
// Machine is the resource usage of the machine this container runs on: the
|
||||
// status page's answer to "is the server itself under strain", next to the
|
||||
// component checks that answer "is mail flowing".
|
||||
//
|
||||
// The numbers come from the kernel's /proc filesystem, which a container shares
|
||||
// with its host unless it was started with its own namespaces — so CPU and
|
||||
// memory describe the host, while /proc/net/dev describes whatever network
|
||||
// namespace the container is in (its own veth under the default bridge, the
|
||||
// host's interfaces under network_mode: host).
|
||||
type Machine struct {
|
||||
CPU CPU
|
||||
Memory Memory
|
||||
Network Network
|
||||
// Window is the interval the rates were measured over; zero until a
|
||||
// second reading exists.
|
||||
Window time.Duration
|
||||
Status Status
|
||||
}
|
||||
|
||||
// WindowText names the sampling interval for the card's description.
|
||||
func (m Machine) WindowText() string {
|
||||
if m.Window <= 0 {
|
||||
return ""
|
||||
}
|
||||
return m.Window.Round(time.Second).String()
|
||||
}
|
||||
|
||||
// CPU is processor load over the sampling window.
|
||||
type CPU struct {
|
||||
// Measured is false until two readings exist to compare; BusyPct means
|
||||
// nothing until it is true.
|
||||
Measured bool
|
||||
BusyPct float64
|
||||
Cores int
|
||||
// Load is the 1/5/15-minute load average, present when /proc/loadavg
|
||||
// could be read. Unlike BusyPct it needs no previous sample, so it is
|
||||
// there on the very first page load.
|
||||
Load [3]float64
|
||||
HasLoad bool
|
||||
Status Status
|
||||
Detail string
|
||||
}
|
||||
|
||||
// Percent is BusyPct as a whole number, for the <meter> element's value
|
||||
// attribute. The bar carries its value in an attribute rather than a width in a
|
||||
// style attribute because the panel's CSP has no inline-style exemption
|
||||
// (security.md).
|
||||
func (c CPU) Percent() int { return percent(c.BusyPct) }
|
||||
|
||||
// BusyText is the reading as it appears beside the bar.
|
||||
func (c CPU) BusyText() string { return fmt.Sprintf("%d%%", c.Percent()) }
|
||||
|
||||
// Memory is main memory (and swap, where the machine has any) at the moment of
|
||||
// the reading. Unlike CPU and network it is a level, not a rate, so a single
|
||||
// reading is enough and it is never in the "measuring" state.
|
||||
type Memory struct {
|
||||
Measured bool
|
||||
TotalBytes uint64
|
||||
AvailableBytes uint64
|
||||
UsedBytes uint64
|
||||
UsedPct float64
|
||||
SwapTotalBytes uint64
|
||||
SwapUsedBytes uint64
|
||||
Status Status
|
||||
Detail string
|
||||
}
|
||||
|
||||
// Percent, UsedText and TotalText render the reading for the template; see
|
||||
// CPU.Percent for why the bar's value travels as an attribute.
|
||||
func (m Memory) Percent() int { return percent(m.UsedPct) }
|
||||
func (m Memory) UsedText() string { return humanBytes(m.UsedBytes) }
|
||||
func (m Memory) TotalText() string { return humanBytes(m.TotalBytes) }
|
||||
func (m Memory) PctText() string { return fmt.Sprintf("%d%%", m.Percent()) }
|
||||
|
||||
// Interface is one network interface's traffic: the counters since the
|
||||
// interface came up, and the throughput over the sampling window.
|
||||
type Interface struct {
|
||||
Name string
|
||||
RxBytes uint64
|
||||
TxBytes uint64
|
||||
RxRate float64 // bytes per second, valid when the Network is Measured
|
||||
TxRate float64
|
||||
Measured bool
|
||||
}
|
||||
|
||||
func (i Interface) InText() string { return humanBytes(i.RxBytes) }
|
||||
func (i Interface) OutText() string { return humanBytes(i.TxBytes) }
|
||||
func (i Interface) InRateText() string { return humanRate(i.RxRate) }
|
||||
func (i Interface) OutRateText() string { return humanRate(i.TxRate) }
|
||||
|
||||
// Network is the traffic across every interface that has carried any, loopback
|
||||
// excluded — loopback traffic is the container talking to itself (the panel to
|
||||
// SQLite, Postfix to its milters) and says nothing about the link.
|
||||
type Network struct {
|
||||
Measured bool
|
||||
Interfaces []Interface
|
||||
RxRate float64
|
||||
TxRate float64
|
||||
Status Status
|
||||
Detail string
|
||||
}
|
||||
|
||||
func (n Network) InRateText() string { return humanRate(n.RxRate) }
|
||||
func (n Network) OutRateText() string { return humanRate(n.TxRate) }
|
||||
|
||||
// MachineSampler reads those counters. Its zero value is ready to use and it is
|
||||
// safe for concurrent use, but one sampler has to be shared by every caller:
|
||||
// the rates are measured against the reading the previous call left behind, so
|
||||
// a fresh sampler per request would never have anything to compare against.
|
||||
type MachineSampler struct {
|
||||
// procRoot replaces /proc in tests; empty means the real one.
|
||||
procRoot string
|
||||
|
||||
mu sync.Mutex
|
||||
prevAt time.Time
|
||||
prevCPU cpuTimes
|
||||
prevNet map[string]netCounters
|
||||
}
|
||||
|
||||
func (m *MachineSampler) root() string {
|
||||
if m.procRoot == "" {
|
||||
return "/proc"
|
||||
}
|
||||
return m.procRoot
|
||||
}
|
||||
|
||||
// cpuTimes is the aggregate of /proc/stat's "cpu" line: all time accounted for,
|
||||
// and the part of it the processor spent doing nothing.
|
||||
type cpuTimes struct {
|
||||
total uint64
|
||||
idle uint64
|
||||
}
|
||||
|
||||
// netCounters is one interface's byte counters from /proc/net/dev.
|
||||
type netCounters struct {
|
||||
rx uint64
|
||||
tx uint64
|
||||
}
|
||||
|
||||
// Sample reads the current counters and reports usage since the previous call.
|
||||
// Like every other check here it never fails: a counter that cannot be read
|
||||
// becomes an unknown status with an explanation, so a kernel that does not
|
||||
// publish one of these files (or a panel run outside Linux for development)
|
||||
// costs one line of the card rather than the page.
|
||||
func (m *MachineSampler) Sample() Machine {
|
||||
root := m.root()
|
||||
now := time.Now()
|
||||
cpuNow, cores, cpuErr := readCPUTimes(root)
|
||||
netNow, netErr := readNetDev(root)
|
||||
|
||||
m.mu.Lock()
|
||||
prevAt, prevCPU, prevNet := m.prevAt, m.prevCPU, m.prevNet
|
||||
if cpuErr == nil {
|
||||
m.prevCPU = cpuNow
|
||||
}
|
||||
if netErr == nil {
|
||||
m.prevNet = netNow
|
||||
}
|
||||
if cpuErr == nil || netErr == nil {
|
||||
m.prevAt = now
|
||||
}
|
||||
m.mu.Unlock()
|
||||
|
||||
// A window of zero (two calls in the same instant) would divide by zero;
|
||||
// one longer than machineSampleWindow is no longer a description of now.
|
||||
window := now.Sub(prevAt)
|
||||
fresh := !prevAt.IsZero() && window > 0 && window <= machineSampleWindow
|
||||
|
||||
mach := Machine{
|
||||
CPU: cpuUsage(prevCPU, cpuNow, cores, readLoadAvg(root), fresh, cpuErr),
|
||||
Memory: readMemory(root),
|
||||
Network: networkUsage(prevNet, netNow, window, fresh, netErr),
|
||||
}
|
||||
if fresh {
|
||||
mach.Window = window
|
||||
}
|
||||
mach.Status = Worst(mach.CPU.Status, mach.Memory.Status, mach.Network.Status)
|
||||
return mach
|
||||
}
|
||||
|
||||
// cpuUsage grades the processor over the window. The load average is reported
|
||||
// alongside it because the two answer different questions — how busy the
|
||||
// processor was, and how many tasks were waiting for it — and a machine can
|
||||
// look idle while work queues up behind a slow disk.
|
||||
func cpuUsage(prev, cur cpuTimes, cores int, load [3]float64, fresh bool, err error) CPU {
|
||||
c := CPU{Cores: cores}
|
||||
if !isZeroLoad(load) {
|
||||
c.Load, c.HasLoad = load, true
|
||||
}
|
||||
if err != nil {
|
||||
c.Status = StatusUnknown
|
||||
c.Detail = "The kernel's processor counters (/proc/stat) could not be read here."
|
||||
return c
|
||||
}
|
||||
if !fresh || cur.total <= prev.total {
|
||||
c.Status = StatusUnknown
|
||||
c.Detail = joinDetail("Measuring — this reading sets the baseline; the next refresh has the figure.", c.contextText())
|
||||
return c
|
||||
}
|
||||
|
||||
total := cur.total - prev.total
|
||||
idle := uint64(0)
|
||||
if cur.idle > prev.idle {
|
||||
idle = cur.idle - prev.idle
|
||||
}
|
||||
if idle > total {
|
||||
idle = total
|
||||
}
|
||||
c.Measured = true
|
||||
c.BusyPct = 100 * float64(total-idle) / float64(total)
|
||||
|
||||
if c.BusyPct >= cpuWarnPct {
|
||||
c.Status = StatusWarn
|
||||
c.Detail = joinDetail(c.contextText(), "The processor is close to fully busy, which slows queue processing and every panel page.")
|
||||
} else {
|
||||
c.Status = StatusOK
|
||||
c.Detail = c.contextText()
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// contextText is the CPU's supporting figures: what the percentage is a
|
||||
// percentage of, and how deep the run queue is.
|
||||
func (c CPU) contextText() string {
|
||||
var parts []string
|
||||
if c.Cores > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d core(s)", c.Cores))
|
||||
}
|
||||
if c.HasLoad {
|
||||
parts = append(parts, fmt.Sprintf("load average %.2f, %.2f, %.2f", c.Load[0], c.Load[1], c.Load[2]))
|
||||
}
|
||||
return strings.Join(parts, " · ")
|
||||
}
|
||||
|
||||
// readMemory reports main memory from /proc/meminfo. Used is derived from
|
||||
// MemAvailable rather than MemFree: Linux spends every spare page on cache, so
|
||||
// MemFree on a healthy machine is near zero and would report a permanent
|
||||
// emergency. MemAvailable is the kernel's own estimate of what a new workload
|
||||
// could actually get.
|
||||
func readMemory(root string) Memory {
|
||||
var m Memory
|
||||
fields, err := readMeminfo(root)
|
||||
if err != nil {
|
||||
m.Status = StatusUnknown
|
||||
m.Detail = "The kernel's memory counters (/proc/meminfo) could not be read here."
|
||||
return m
|
||||
}
|
||||
total, available := fields["MemTotal"], fields["MemAvailable"]
|
||||
if total == 0 {
|
||||
m.Status = StatusUnknown
|
||||
m.Detail = "/proc/meminfo does not report a total memory size."
|
||||
return m
|
||||
}
|
||||
if available > total {
|
||||
available = total
|
||||
}
|
||||
|
||||
m.Measured = true
|
||||
m.TotalBytes = total
|
||||
m.AvailableBytes = available
|
||||
m.UsedBytes = total - available
|
||||
m.UsedPct = 100 * float64(m.UsedBytes) / float64(total)
|
||||
m.SwapTotalBytes = fields["SwapTotal"]
|
||||
if swapFree := fields["SwapFree"]; m.SwapTotalBytes > swapFree {
|
||||
m.SwapUsedBytes = m.SwapTotalBytes - swapFree
|
||||
}
|
||||
|
||||
detail := fmt.Sprintf("%s used of %s; %s available to new work.",
|
||||
humanBytes(m.UsedBytes), humanBytes(total), humanBytes(available))
|
||||
if m.SwapTotalBytes > 0 {
|
||||
detail += fmt.Sprintf(" Swap: %s of %s.", humanBytes(m.SwapUsedBytes), humanBytes(m.SwapTotalBytes))
|
||||
}
|
||||
switch {
|
||||
case m.UsedPct >= memErrorPct:
|
||||
m.Status = StatusError
|
||||
m.Detail = detail + " Memory is exhausted; the kernel kills processes to reclaim it, and Postfix or the panel are candidates."
|
||||
case m.UsedPct >= memWarnPct:
|
||||
m.Status = StatusWarn
|
||||
m.Detail = detail + " Little headroom left."
|
||||
default:
|
||||
m.Status = StatusOK
|
||||
m.Detail = detail
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// networkUsage turns two readings of the interface counters into throughput.
|
||||
// It never grades: there is no usage figure that is wrong for a mail server, so
|
||||
// the row is informational and only reports "unknown" when the counters are
|
||||
// unreadable.
|
||||
func networkUsage(prev, cur map[string]netCounters, window time.Duration, fresh bool, err error) Network {
|
||||
var n Network
|
||||
if err != nil {
|
||||
n.Status = StatusUnknown
|
||||
n.Detail = "The kernel's network counters (/proc/net/dev) could not be read here."
|
||||
return n
|
||||
}
|
||||
|
||||
names := make([]string, 0, len(cur))
|
||||
for name := range cur {
|
||||
names = append(names, name)
|
||||
}
|
||||
// Map order is random, and this table is re-rendered every few seconds:
|
||||
// without a sort the rows would shuffle under the reader.
|
||||
sort.Strings(names)
|
||||
|
||||
n.Measured = fresh
|
||||
for _, name := range names {
|
||||
c := cur[name]
|
||||
// An interface that has never carried a byte is a veth or a bridge
|
||||
// the deployment happens to have, not part of the mail path.
|
||||
if c.rx == 0 && c.tx == 0 {
|
||||
continue
|
||||
}
|
||||
iface := Interface{Name: name, RxBytes: c.rx, TxBytes: c.tx, Measured: fresh}
|
||||
if fresh {
|
||||
p := prev[name]
|
||||
secs := window.Seconds()
|
||||
// Counters only go up; a drop means the interface (or the
|
||||
// container) was recreated, so there is no rate to report.
|
||||
if c.rx >= p.rx {
|
||||
iface.RxRate = float64(c.rx-p.rx) / secs
|
||||
}
|
||||
if c.tx >= p.tx {
|
||||
iface.TxRate = float64(c.tx-p.tx) / secs
|
||||
}
|
||||
n.RxRate += iface.RxRate
|
||||
n.TxRate += iface.TxRate
|
||||
}
|
||||
n.Interfaces = append(n.Interfaces, iface)
|
||||
}
|
||||
|
||||
n.Status = StatusOK
|
||||
switch {
|
||||
case len(n.Interfaces) == 0:
|
||||
n.Detail = "No interface outside loopback has carried any traffic."
|
||||
case !fresh:
|
||||
n.Detail = "Measuring — this reading sets the baseline; the next refresh has the throughput."
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// readCPUTimes returns the aggregate processor times and the number of cores
|
||||
// from /proc/stat. The times are in USER_HZ ticks, which cancel out because
|
||||
// only their ratio is used.
|
||||
func readCPUTimes(root string) (cpuTimes, int, error) {
|
||||
data, err := os.ReadFile(filepath.Join(root, "stat"))
|
||||
if err != nil {
|
||||
return cpuTimes{}, 0, err
|
||||
}
|
||||
var (
|
||||
times cpuTimes
|
||||
cores int
|
||||
found bool
|
||||
)
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 5 || !strings.HasPrefix(fields[0], "cpu") {
|
||||
continue
|
||||
}
|
||||
if fields[0] != "cpu" {
|
||||
cores++ // cpu0, cpu1, … — one line per core
|
||||
continue
|
||||
}
|
||||
// user nice system idle iowait irq softirq steal (guest fields are
|
||||
// already counted inside user/nice, so they are left out).
|
||||
for i, f := range fields[1:] {
|
||||
if i >= 8 {
|
||||
break
|
||||
}
|
||||
v, err := strconv.ParseUint(f, 10, 64)
|
||||
if err != nil {
|
||||
return cpuTimes{}, 0, fmt.Errorf("/proc/stat: unreadable cpu time %q", f)
|
||||
}
|
||||
times.total += v
|
||||
// idle plus iowait: both are time the processor had nothing to
|
||||
// run, and separating them tells the reader nothing here.
|
||||
if i == 3 || i == 4 {
|
||||
times.idle += v
|
||||
}
|
||||
}
|
||||
found = true
|
||||
}
|
||||
if !found {
|
||||
return cpuTimes{}, 0, fmt.Errorf("/proc/stat: no aggregate cpu line")
|
||||
}
|
||||
return times, cores, nil
|
||||
}
|
||||
|
||||
// readLoadAvg reads the 1/5/15-minute load averages. A machine without
|
||||
// /proc/loadavg simply has none reported, so the failure is a zero value rather
|
||||
// than an error.
|
||||
func readLoadAvg(root string) [3]float64 {
|
||||
var load [3]float64
|
||||
data, err := os.ReadFile(filepath.Join(root, "loadavg"))
|
||||
if err != nil {
|
||||
return load
|
||||
}
|
||||
fields := strings.Fields(string(data))
|
||||
if len(fields) < 3 {
|
||||
return load
|
||||
}
|
||||
for i := range load {
|
||||
v, err := strconv.ParseFloat(fields[i], 64)
|
||||
if err != nil {
|
||||
return [3]float64{}
|
||||
}
|
||||
load[i] = v
|
||||
}
|
||||
return load
|
||||
}
|
||||
|
||||
func isZeroLoad(load [3]float64) bool {
|
||||
return load == [3]float64{}
|
||||
}
|
||||
|
||||
// readMeminfo returns the /proc/meminfo entries this package uses, in bytes.
|
||||
// The file reports kB (kibibytes, despite the label) for these fields.
|
||||
func readMeminfo(root string) (map[string]uint64, error) {
|
||||
data, err := os.ReadFile(filepath.Join(root, "meminfo"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
want := map[string]bool{"MemTotal": true, "MemAvailable": true, "SwapTotal": true, "SwapFree": true}
|
||||
out := make(map[string]uint64, len(want))
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
name, rest, ok := strings.Cut(line, ":")
|
||||
if !ok || !want[name] {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(rest)
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
v, err := strconv.ParseUint(fields[0], 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if len(fields) > 1 && strings.EqualFold(fields[1], "kB") {
|
||||
v *= 1024
|
||||
}
|
||||
out[name] = v
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("/proc/meminfo: no readable fields")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// readNetDev returns each interface's byte counters from /proc/net/dev, loopback
|
||||
// excluded. Each line is " eth0: <rx bytes> <rx packets> … <tx bytes> …" — the
|
||||
// name is separated by a colon, which may or may not have a space after it, so
|
||||
// the split is on the colon and not on whitespace.
|
||||
func readNetDev(root string) (map[string]netCounters, error) {
|
||||
data, err := os.ReadFile(filepath.Join(root, "net", "dev"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]netCounters)
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
name, rest, ok := strings.Cut(line, ":")
|
||||
name = strings.TrimSpace(name)
|
||||
if !ok || name == "" || name == "lo" || strings.Contains(name, " ") {
|
||||
continue // header lines carry no name, or several words
|
||||
}
|
||||
fields := strings.Fields(rest)
|
||||
if len(fields) < 9 {
|
||||
continue
|
||||
}
|
||||
rx, err1 := strconv.ParseUint(fields[0], 10, 64)
|
||||
tx, err2 := strconv.ParseUint(fields[8], 10, 64)
|
||||
if err1 != nil || err2 != nil {
|
||||
continue
|
||||
}
|
||||
out[name] = netCounters{rx: rx, tx: tx}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// percent rounds a 0–100 reading to a whole number and clamps it, so a bar's
|
||||
// value attribute is always inside the range the element declares.
|
||||
func percent(v float64) int {
|
||||
switch {
|
||||
case v <= 0:
|
||||
return 0
|
||||
case v >= 100:
|
||||
return 100
|
||||
default:
|
||||
return int(v + 0.5)
|
||||
}
|
||||
}
|
||||
|
||||
// humanBytes renders a byte count in the binary units memory and traffic are
|
||||
// conventionally read in.
|
||||
func humanBytes(b uint64) string {
|
||||
const unit = 1024
|
||||
if b < unit {
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
div, exp := uint64(unit), 0
|
||||
for n := b / unit; n >= unit && exp < 4; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTP"[exp])
|
||||
}
|
||||
|
||||
// humanRate renders throughput in bytes per second, to match the totals beside
|
||||
// it rather than the bits per second a link is sold in.
|
||||
func humanRate(perSec float64) string {
|
||||
if perSec < 0 {
|
||||
perSec = 0
|
||||
}
|
||||
return humanBytes(uint64(perSec+0.5)) + "/s"
|
||||
}
|
||||
|
||||
// joinDetail joins the non-empty parts of a detail line.
|
||||
func joinDetail(parts ...string) string {
|
||||
var kept []string
|
||||
for _, p := range parts {
|
||||
if p != "" {
|
||||
kept = append(kept, p)
|
||||
}
|
||||
}
|
||||
return strings.Join(kept, " ")
|
||||
}
|
||||
Reference in New Issue
Block a user