from-header-binding: the sender check, what it costs, its pages; guide, architecture, security

A message's From: line is held against the application that sends it, under
the rule Postfix holds the envelope sender to. OpenDKIM chooses the key by
that line, so until now an application could have mail signed as another
domain of the server or as the server itself.

The check (internal/milter/sender.go)
- A role of the panel on a socket of its own (SENDER_CHECK_SOCKET), first in
  smtpd_milters and strict: default_action=tempfail. The journal milter keeps
  its socket and stays fail-open. postfix-wrapper waits for the new socket.
- It reads no database: what an application may send as comes from Postfix's
  own sender_login_maps, what the server signs for from OpenDKIM's
  SigningTable. Both are read again when they change; one that cannot be
  read defers mail.
- One From: field, one address, allowed by the envelope's lookup; then no
  other name of the SigningTable may appear anywhere in the raw line.
  Refused: 550 5.7.1 naming the line, reduced to printable ASCII.
- A refusal is a send-log row with reason "from" and the header address
  (migration 0006), written by another goroutine and not waited for. The
  journal's refusals record their reason too (limit, client), and the
  notifier takes only a limit's.

What it costs (docs/plans/from-header-binding.md, Statistics)
- The role marks itself alive every 2 s; at start the gap since the mark is
  downtime, counted from Postfix's start if later and not at all when
  supervisord says Postfix is not running.
- The log tailer counts the connections Postfix turned away: a warning that
  names the socket, then milter-reject "Service unavailable" from the same
  smtpd. Seen on the stand: the client is refused from CONNECT on and leaves
  at AUTH (454), so no MAIL line is ever written.
- sender_check_days, a row per UTC day, kept a year.

Pages
- Health: the card "Sender check", the socket's row, and the cards layout -
  boxes numbered by importance in two stacks read in rows; the left stack
  polls, the three readings of the right one are swapped in by id.
- Outbound log: a refused row says why in its tag (wrong From, wrong client,
  rate-limited); the message's page shows the From: line and what the
  application may send as.
- Help: topic "Sender check", word for word the mockup's.

Found by the stand, fixed here: the names for the second look came from the
sender bindings, and a domain with a key and no application yet was not
among them.

e2e: negative_journal_milter_fail_open is gone - it proved that mail passes
with the panel down, which is no longer so. In its place
sender_check_holds_the_from_line (another domain's From:, the hostname's, a
stranger's, an address off the list, another sender in the display name -
refused; own addresses - delivered) and panel_down_defers_mail (4xx with the
panel stopped, the container alive, mail again after, downtime and one
deferred message on Health, the card warning).

Test server, this tree before the last cosmetic edits (a fixture row, a
fact's size, docs): go vet ok, go test 23 packages ok, make e2e ok (337 s).
Locally after them: internal/web/view ok.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/web/handlers/handlers.go

@@ -28,6 +28,7 @@ type Config struct {
 	TLSCertFile       string
 	OpenDKIMSocket    string
 	JournalSocket     string
+	SenderSocket      string
 	// Level-1 Postfix anvil backstop (env RATE_LIMIT_*), shown in the panel
 	// and used to cap domain/app level-2 ceilings (guide § Rate limiting).
 	RateLimitMessagesPerIP int

internal/web/handlers/handlers_filter_socket_test.go

@@ -25,7 +25,8 @@ func socketsServer(t *testing.T) *Handlers {
 	dir := t.TempDir()
 	h.cfg.OpenDKIMSocket = filepath.Join(dir, "opendkim.sock")
 	h.cfg.JournalSocket = filepath.Join(dir, "journal.sock")
-	for _, path := range []string{h.cfg.OpenDKIMSocket, h.cfg.JournalSocket} {
+	h.cfg.SenderSocket = filepath.Join(dir, "sender.sock")
+	for _, path := range []string{h.cfg.OpenDKIMSocket, h.cfg.SenderSocket, h.cfg.JournalSocket} {
 		l, err := net.Listen("unix", path)
 		if err != nil {
 			t.Skipf("unix sockets unavailable here: %v", err)
@@ -55,19 +56,19 @@ func TestFilterOffAddsNoSocket(t *testing.T) {
 	h := socketsServer(t)
 	h.cfg.InboundEnabled = true
 	h.cfg.InboundAntispamMilter = "" // no filter attached
-	if got := socketsOf(t, h); len(got) != 2 {
-		t.Fatalf("%d sockets with the filter off, want 2: %+v", len(got), got)
+	if got := socketsOf(t, h); len(got) != 3 {
+		t.Fatalf("%d sockets with the filter off, want 3: %+v", len(got), got)
 	}
 
 	// A milter address with the inbound relay off is not a filter either.
 	h.cfg.InboundEnabled = false
 	h.cfg.InboundAntispamMilter = "inet:127.0.0.1:1"
 	got := socketsOf(t, h)
-	if len(got) != 2 {
-		t.Fatalf("%d sockets with the inbound relay off, want 2: %+v", len(got), got)
+	if len(got) != 3 {
+		t.Fatalf("%d sockets with the inbound relay off, want 3: %+v", len(got), got)
 	}
 	card := socketCard(t, h)
-	if card.Value != "2 of 2 answering" || card.Sub != "OpenDKIM, send-log" || card.Level != "" {
+	if card.Value != "3 of 3 answering" || card.Sub != "OpenDKIM, sender-check, send-log" || card.Level != "" {
 		t.Errorf("card = %+v", card)
 	}
 }
@@ -83,7 +84,7 @@ func TestFilterOnAnsweringIsTheThirdSocket(t *testing.T) {
 	h.cfg.InboundAntispamMilter, h.cfg.InboundAntispamAction = "inet:"+l.Addr().String(), "tempfail"
 
 	got := socketsOf(t, h)
-	if len(got) != 3 || got[2].Name != "Spam filter" || got[2].Note != "inbound" || got[2].Path != "inet:"+l.Addr().String() {
+	if len(got) != 4 || got[3].Name != "Spam filter" || got[3].Note != "inbound" || got[3].Path != "inet:"+l.Addr().String() {
 		t.Fatalf("sockets = %+v", got)
 	}
 	for _, s := range got {
@@ -92,7 +93,7 @@ func TestFilterOnAnsweringIsTheThirdSocket(t *testing.T) {
 		}
 	}
 	card := socketCard(t, h)
-	if card.Value != "3 of 3 answering" || card.Sub != "incl. the spam filter" || card.Level != "" {
+	if card.Value != "4 of 4 answering" || card.Sub != "incl. the spam filter" || card.Level != "" {
 		t.Errorf("card = %+v", card)
 	}
 
@@ -132,10 +133,10 @@ func TestFilterDownIsAMissingSocket(t *testing.T) {
 	} {
 		h.cfg.InboundAntispamMilter, h.cfg.InboundAntispamAction = dead, tt.action
 		got := socketsOf(t, h)
-		if len(got) != 3 {
-			t.Fatalf("action %q: %d sockets, want 3", tt.action, len(got))
+		if len(got) != 4 {
+			t.Fatalf("action %q: %d sockets, want 4", tt.action, len(got))
 		}
-		f := got[2]
+		f := got[3]
 		refused := "Connection to " + l.Addr().String() + " refused. " + tt.detail
 		if f.Present || f.Status != tt.status || f.Detail != refused {
 			t.Errorf("action %q: filter = %+v, want status %q, detail %q", tt.action, f, tt.status, refused)
@@ -146,7 +147,7 @@ func TestFilterDownIsAMissingSocket(t *testing.T) {
 			t.Errorf("action %q: SocketStatus = %q, want %q", tt.action, c["SocketStatus"], tt.status)
 		}
 		card := socketCard(t, h)
-		if card.Value != "2 of 3 answering" || card.Level != tt.level || !strings.HasSuffix(card.Sub, "Spam filter") {
+		if card.Value != "3 of 4 answering" || card.Level != tt.level || !strings.HasSuffix(card.Sub, "Spam filter") {
 			t.Errorf("action %q: card = %+v", tt.action, card)
 		}
 
@@ -160,7 +161,7 @@ func TestFilterDownIsAMissingSocket(t *testing.T) {
 		rec := httptest.NewRecorder()
 		h.HandleOverviewFragment(rec, reqAs("/overview/fragment", globalPrincipal))
 		page := rec.Body.String()
-		for _, want := range []string{`>` + tt.stamp + `<`, `Milter sockets</a> (2 of 3 answering)`} {
+		for _, want := range []string{`>` + tt.stamp + `<`, `Milter sockets</a> (3 of 4 answering)`} {
 			if !strings.Contains(page, want) {
 				t.Errorf("action %q: Overview is missing %q", tt.action, want)
 			}

internal/web/handlers/handlers_health_page_test.go

@@ -95,13 +95,13 @@ func TestHealthPageRendersEveryCheck(t *testing.T) {
 	if strings.Contains(out, "Listening") {
 		t.Error("a socket that answers repeats its detail beside the tag")
 	}
-	// The reload control stays outside the polled rows.
-	body, conf := strings.Index(out, `id="health-machine"`), strings.Index(out, `id="configuration"`)
-	if body < 0 || conf < body {
-		t.Fatal("polled rows or the Configuration box missing or out of order")
+	// The reload control stays outside the polled stack, in the other one.
+	body, right, conf := strings.Index(out, `id="health-stack"`), strings.Index(out, `id="sockets"`), strings.Index(out, `id="configuration"`)
+	if body < 0 || right < body || conf < right {
+		t.Fatal("the polled stack or the Configuration box missing or out of order")
 	}
-	if frag := out[body:conf]; strings.Contains(frag, `action="/server/health/reload"`) || !strings.Contains(frag, `id="hostname"`) {
-		t.Error("the hostname box is outside the polled rows, or the reload control inside them")
+	if frag := out[body:right]; strings.Contains(frag, `action="/server/health/reload"`) || !strings.Contains(frag, `id="hostname"`) || !strings.Contains(frag, `id="sender-check"`) {
+		t.Error("the hostname or sender check box is outside the polled stack, or the reload control inside it")
 	}
 }
 
@@ -163,8 +163,9 @@ func TestHealthFragmentCarriesTheSameRows(t *testing.T) {
 		t.Fatalf("GET /server/health/fragment = %d:\n%s", rec.Code, out)
 	}
 	for _, want := range []string{
-		`id="health-machine" data-poll`, `hx-get="/server/health/fragment"`,
-		`id="health-checks" hx-swap-oob="true"`, `id="machine"`, `id="processes"`, `id="certificate"`, `id="sockets"`, `id="hostname"`,
+		`id="health-stack" data-poll`, `hx-get="/server/health/fragment"`,
+		`id="sockets" hx-swap-oob="true"`, `id="certificate" hx-swap-oob="true"`, `id="notification-mail" hx-swap-oob="true"`,
+		`id="machine"`, `id="processes"`, `id="sender-check"`, `id="hostname"`,
 	} {
 		if !strings.Contains(out, want) {
 			t.Errorf("health fragment is missing %q", want)

internal/web/handlers/handlers_monitor.go

@@ -104,6 +104,7 @@ func (h *Handlers) HandleDelivery(w http.ResponseWriter, r *http.Request) {
 	in := view.MessageInput{
 		ID: row.ID, QueueID: row.QueueID, Domain: row.Domain, App: row.AppLogin,
 		From: row.From, To: row.To, Subject: row.Subject, Status: row.Status,
+		Reason: row.Reason, HeaderFrom: row.HeaderFrom,
 		Accepted: row.CreatedAt, Reported: row.UpdatedAt,
 		// Where the row came from, so the way back returns to the page and filters
 		// the operator was looking at rather than the top of an unfiltered log.
@@ -122,13 +123,31 @@ func (h *Handlers) HandleDelivery(w http.ResponseWriter, r *http.Request) {
 		break
 	}
 	page := view.NewOutMessage(h.shellMeta(r), in).
-		WithHistory(deliveryEvents(row, h.cfg.RetryPolicy)).
+		WithHistory(deliveryEvents(row, h.cfg.RetryPolicy, h.sendsAs(row.AppLogin))).
 		// The mail.log lines for this message, and — when there are none — the
 		// reason, which is a normal outcome rather than a failure.
 		WithDeliveryLog(logLines, logNote)
 	h.view.Render(w, http.StatusOK, "out-message", page)
 }
 
+// sendsAs says what an application may send as, in the words of a refusal:
+// any address of its domain, or the addresses of its list. Empty when the
+// login is no application's any more.
+func (h *Handlers) sendsAs(login string) view.Text {
+	a, err := h.store.GetApplicationByLogin(login)
+	if err != nil {
+		return nil
+	}
+	if a.AddressMode == store.AddressModeList {
+		return view.Plain("the addresses of its list")
+	}
+	d, err := h.store.GetDomain(a.DomainID)
+	if err != nil {
+		return nil
+	}
+	return view.Rich("any address of ", view.Mono(d.Name))
+}
+
 // deliveryEvents turns a row's two timestamps into the history the page shows.
 // The journal keeps no event table — a row is created when the message is
 // accepted and updated once when Postfix reports the attempt — so the two
@@ -137,7 +156,10 @@ func (h *Handlers) HandleDelivery(w http.ResponseWriter, r *http.Request) {
 // six hours, then delivered" rather than as two dates in a list of fields.
 // policy supplies the human intervals for deferred and bounced copy, the same
 // strings the Mail queue card prints, so the two cannot drift.
-func deliveryEvents(row store.SendLogRow, policy postfix.RetryPolicy) []view.Step {
+//
+// sendsAs is what the row's application may send as, for the step of a message
+// refused for its From: line; empty when the application is gone.
+func deliveryEvents(row store.SendLogRow, policy postfix.RetryPolicy, sendsAs view.Text) []view.Step {
 	// step states what happened and when; outcome is ok, warn, error or "" for a
 	// step that reports nothing either way, and a zero time is a step that has
 	// not happened yet.
@@ -153,6 +175,22 @@ func deliveryEvents(row store.SendLogRow, policy postfix.RetryPolicy) []view.Ste
 	// A rejected message has no second step, and its first one is not an
 	// acceptance: the journal-milter refused it, so Postfix never queued it.
 	if row.Status == store.StatusRejected {
+		switch row.Reason {
+		case store.ReasonFrom:
+			// The one step that names things: which line, whose, and what that
+			// application may write instead.
+			text := view.Rich(" — ", view.Code("From: "+row.HeaderFrom), " is not an address of application ", view.Mono(row.AppLogin))
+			if len(sendsAs) > 0 {
+				text = append(append(text, view.Inline{Text: ", which sends as "}), sendsAs...)
+			}
+			text = append(text, view.Rich(". The application was told ", view.Code("550 5.7.1"), " and will not try again.")...)
+			s := step(row.CreatedAt, "error", "Refused before queueing", "")
+			s.Text = text
+			return []view.Step{s}
+		case store.ReasonClient:
+			return []view.Step{step(row.CreatedAt, "error", "Refused before queueing",
+				"The application accepts mail only from its listed client addresses, and this message came from another. It was never queued, so there is no queue id and Postfix never attempted delivery.")}
+		}
 		return []view.Step{step(row.CreatedAt, "error", "Refused before queueing",
 			"The journal-milter refused the message under a rate limit. It was never queued, so there is no queue id and Postfix never attempted delivery.")}
 	}
@@ -336,7 +374,7 @@ func (h *Handlers) outLogPage(r *http.Request) (*view.OutLog, error) {
 	for i, row := range rows {
 		table[i] = view.OutLogRow{
 			Time: view.FormatLogTime(row.CreatedAt), From: row.From, To: row.To,
-			Subject: mailhdr.DecodeSubject(row.Subject), Status: view.Tag{Status: row.Status},
+			Subject: mailhdr.DecodeSubject(row.Subject), Status: view.SendLogTag(row.Status, row.Reason),
 			Href: out.DetailHref(row.ID, page),
 		}
 	}

internal/web/handlers/handlers_notifications_test.go

@@ -269,7 +269,7 @@ func TestHealthNotificationMailCard(t *testing.T) {
 }
 
 // With both records in place the card is green. The card is part of the polled
-// fragment, swapped in by its id like the second row.
+// fragment, swapped in by its id.
 func TestHealthNotificationMailCardIsGreenAndPolled(t *testing.T) {
 	h, zone, mail := serverMailStand(t)
 	publishMail(zone, mail)
@@ -279,7 +279,7 @@ func TestHealthNotificationMailCardIsGreenAndPolled(t *testing.T) {
 	h.HandleHealthFragment(rec, auth.RequestWithPrincipal(req, globalPrincipal))
 	out := rec.Body.String()
 	for _, want := range []string{
-		`id="health-mail" hx-swap-oob="true"`,
+		`id="notification-mail" hx-swap-oob="true"`,
 		`<h2>Notification mail</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div>`,
 		`<td><span class="tag is-success is-light">published</span></td>`,
 	} {
@@ -287,7 +287,7 @@ func TestHealthNotificationMailCardIsGreenAndPolled(t *testing.T) {
 			t.Errorf("health fragment is missing %q:\n%s", want, out)
 		}
 	}
-	if strings.Contains(out[strings.Index(out, `id="health-mail"`):], "is-warning") {
+	if strings.Contains(out[strings.Index(out, `id="notification-mail"`):], "is-warning") {
 		t.Error("the card warns while both records are published")
 	}
 }

internal/web/handlers/handlers_status.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -118,15 +119,13 @@ func (h *Handlers) HandleHealth(w http.ResponseWriter, r *http.Request) {
 	h.view.Render(w, http.StatusOK, "health", h.healthView(r, statusFlash(r)))
 }
 
-// HandleHealthFragment is the polled part of Health: the same two rows of
-// boxes the page holds, read again.
+// HandleHealthFragment is the polled part of Health: the boxes the page holds,
+// all but Configuration, read again.
 func (h *Handlers) HandleHealthFragment(w http.ResponseWriter, r *http.Request) {
 	if _, ok := h.requireGlobal(w, r); !ok {
 		return
 	}
-	page := h.healthView(r, "")
-	page.Refresh = true
-	h.view.RenderFragment(w, http.StatusOK, "health_body", page)
+	h.view.RenderFragment(w, http.StatusOK, "health_body", h.healthView(r, "").AsFragment())
 }
 
 // HandleHealthRecheck re-runs the DNS checks of the server ignoring the cache.
@@ -144,7 +143,31 @@ func (h *Handlers) HandleHealthRecheck(w http.ResponseWriter, r *http.Request) {
 func (h *Handlers) healthView(r *http.Request, flash string) *view.Health {
 	page := healthPage(h.shellMeta(r), h.healthChecks(), flash)
 	mail := h.serverMail(false)
-	return page.WithNotificationMail(notify.From(h.cfg.Hostname), mail.rows, mail.status)
+	page.WithNotificationMail(notify.From(h.cfg.Hostname), mail.rows, mail.status)
+	stats, err := h.store.SenderCheckStats(time.Now())
+	if err != nil {
+		logf("panel: health: sender check statistics: %v", err)
+	}
+	rows, since, status := senderCheckCard(stats, time.Now())
+	return page.WithSenderCheck(rows, since, status)
+}
+
+// senderCheckCard is the Sender check card from its statistics. It warns when
+// a message was deferred in the last 24 hours: that is mail an application had
+// to send again. Downtime that deferred nothing is only shown.
+func senderCheckCard(s store.SenderCheckStats, now time.Time) (rows []view.SenderCheckRow, since, status string) {
+	row := func(period string, t store.SenderCheckTotals) view.SenderCheckRow {
+		return view.SenderCheckRow{Period: period, Down: view.Downtime(t.Down), Deferred: strconv.FormatInt(t.Deferred, 10)}
+	}
+	rows = []view.SenderCheckRow{row("Today", s.Today), row("Last 30 days", s.Month), row("Last 12 months", s.Year)}
+	if !s.Since.IsZero() {
+		since = s.Since.Format("2006-01-02")
+	}
+	status = string(health.StatusOK)
+	if !s.DeferredAt.IsZero() && now.Sub(s.DeferredAt) < 24*time.Hour {
+		status = string(health.StatusWarn)
+	}
+	return rows, since, status
 }
 
 // healthPage turns one reading of the checks (healthChecks) into the Health
@@ -397,6 +420,7 @@ func (h *Handlers) healthChecks() map[string]any {
 	cert := health.CheckCertificate(h.cfg.TLSCertFile)
 	sockets := []health.Socket{
 		health.CheckSocket("OpenDKIM", h.cfg.OpenDKIMSocket, true),
+		health.CheckSocket("sender-check", h.cfg.SenderSocket, true),
 		health.CheckSocket("send-log", h.cfg.JournalSocket, false),
 	}
 	if on, milter, action := h.inboundFilter(); on {