from-header-binding: what the reviews had left as found, and address extensions
test / test (push) Canceled after 0s

The owner on the three things the reviews' commit left as found: can they be
fixed? They can.

- Downtime that crossed a UTC midnight was all booked on the day of the
  restart: it is cut at the midnights between (byDay).
- A message with the null envelope sender - a bounce an application writes
  itself - left a refusal row without a domain, which no domain
  administrator would see: the row takes the domain the login is bound to.
- A log read again after an unclean stop counted its deferrals again: while
  the tailer catches up, a warning no later than the last one counted is
  passed over. The mark is kept to the fraction of a second the log gives.

And on the check being stricter than Postfix for an address with an
extension (invoices+x@ passed in the envelope, was refused in the header):
it makes Postfix's second try now, the address without its extension.
recipient_delimiter was the distribution's default; it is set in
postfix-config.sh, since the check depends on it. e2e sends one through.

Test server, this tree: go vet ok, go test 23 packages ok, make e2e ok (339 s).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
mix
2026-10-12 21:29:17 +03:00
co-authored by Claude Opus 5.5
parent 89fd0a6116
commit c817f0833c
13 changed files with 224 additions and 23 deletions
+6
View File
@@ -203,8 +203,14 @@ postconf -e \
# Sender binding (the critical anti-spoofing control). texthash resolves the
# full address first, then the "@domain" wildcard, so both address modes work
# from the same map.
#
# recipient_delimiter is said here and not left to the distribution's main.cf:
# it is what lets an application listed as invoices@ send as invoices+march@,
# and the panel's sender check makes the same second try for the From: line
# (extensionDelimiter in internal/milter/sender.go) - the two must agree.
postconf -e \
"smtpd_sender_login_maps=texthash:${SENDER_LOGIN_MAPS}" \
"recipient_delimiter=+" \
"queue_directory=${QUEUE_DIR}"
# Restrictions: authenticated clients only, no relay to foreign destinations,
+26 -2
View File
@@ -41,8 +41,10 @@ func serveSenderCheck(ctx context.Context, cfg config, st *store.Store) error {
}
if down := senderCheckDowntime(mark, now, postfixStarted); down > 0 {
log.Printf("sender-check: was not answering for %s while Postfix was running", down.Round(time.Second))
if err := st.AddSenderCheckDown(now, down); err != nil {
log.Printf("sender-check: statistics: %v", err)
for _, part := range byDay(now.Add(-down), now) {
if err := st.AddSenderCheckDown(part.day, part.down); err != nil {
log.Printf("sender-check: statistics: %v", err)
}
}
}
keepAlive(ctx, mark, st)
@@ -79,6 +81,28 @@ func senderCheckDowntime(path string, now time.Time, postfixStarted func() (time
return 0
}
// dayPart is the part of a stretch of downtime that fell on one UTC day.
type dayPart struct {
day time.Time
down time.Duration
}
// byDay cuts the time from 'from' to 'to' at the UTC midnights between them,
// so that a night's outage is not all booked on the morning it ended.
func byDay(from, to time.Time) []dayPart {
from, to = from.UTC(), to.UTC()
var parts []dayPart
for from.Before(to) {
next := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.UTC).AddDate(0, 0, 1)
if next.After(to) {
next = to
}
parts = append(parts, dayPart{day: from, down: next.Sub(from)})
from = next
}
return parts
}
// postfixStarted asks supervisord since when Postfix has been running.
func postfixStarted() (time.Time, bool) {
procs, err := health.Processes()
+21
View File
@@ -45,3 +45,24 @@ func TestSenderCheckDowntime(t *testing.T) {
t.Errorf("a mark in the future: %s", d)
}
}
// Downtime is booked on the days it fell on.
func TestByDay(t *testing.T) {
at := func(d, h, m int) time.Time { return time.Date(2026, 10, d, h, m, 0, 0, time.UTC) }
got := byDay(at(12, 23, 50), at(14, 0, 10))
want := []dayPart{{at(12, 23, 50), 10 * time.Minute}, {at(13, 0, 0), 24 * time.Hour}, {at(14, 0, 0), 10 * time.Minute}}
if len(got) != len(want) {
t.Fatalf("parts = %+v", got)
}
for i := range want {
if !got[i].day.Equal(want[i].day) || got[i].down != want[i].down {
t.Errorf("part %d = %+v, want %+v", i, got[i], want[i])
}
}
if one := byDay(at(12, 10, 0), at(12, 10, 0).Add(40*time.Second)); len(one) != 1 || one[0].down != 40*time.Second {
t.Errorf("within a day: %+v", one)
}
if none := byDay(at(12, 10, 0), at(12, 10, 0)); len(none) != 0 {
t.Errorf("no time: %+v", none)
}
}
+8 -5
View File
@@ -234,9 +234,10 @@ server itself.
its domain may write any address of that domain, and one with a list only
the listed ones; everything else is refused - another registered domain,
the server's name, a domain the server knows nothing of. One table for
both senders. Where the lookups differ the check is the stricter one:
Postfix also tries an address without its `+extension` and compares logins
without regard to case.
both senders and one lookup: the address, the address without its
`+extension` (`recipient_delimiter`, which `build/postfix-config.sh` sets
for that reason), then the domain. Postfix compares logins without regard
to case and the check as written, which is how the panel writes them.
- **How the header is read.** Exactly one `From:` field (none, or more than
one, is refused), parsed as an address list that must hold exactly one
address. Then a cruder second look at the raw value: OpenDKIM finds the
@@ -283,8 +284,10 @@ server itself.
`sender_check_days`, a
row per UTC day kept a year, and Server › Health shows today, 30 days and
12 months. The second figure also sees a panel that is running and not
answering, which the mark does not. A log read twice after an unclean stop
can count a message twice.
answering, which the mark does not. Downtime is cut at the UTC midnights
it crossed. A log read again after an unclean stop does not count again:
while the tailer catches up, a warning no later than the last one counted
is passed over.
- **Why it is not a process of its own.** Decided with the owner: measure
first. The role has a socket of its own so that the journal stays
fail-open, and moving it out later changes where it is started and nothing
+4
View File
@@ -1097,6 +1097,10 @@ listed for it. Since `2.4.0` that rule covers both senders a message has:
- the **`From:` line** of the message - the one a mail program shows and the
one the DKIM signature is chosen by - which the panel checks.
An address with an extension counts as the address: an application listed as
`invoices@…` may also send as `invoices+march@…`, in the envelope and in
`From:` alike.
A message whose `From:` is anything else is refused with `550 5.7.1` and a
text naming the line: another of your domains, the server's own name, an
address at a domain that is not on this server, a line with two addresses,
+2 -2
View File
@@ -146,5 +146,5 @@ the fourth was not.
- [x] Panel: Health (card, sockets row, layout), Outbound log tags, the message's page, help
- [x] guide.md, architecture.md, security.md (the accepted risk goes), schema-migrations.md
- [x] `go vet`, `go test`, e2e: refused for another domain's `From:`, for the hostname's and for an address off the list; accepted for its own (on the test server: vet ok, 23 packages ok, e2e ok - `sender_check_holds_the_from_line`, `panel_down_defers_mail`)
- [x] Technical review — **Opus** (2026-10-12; one Critical, fixed: a folded `From:` line - what a mail library makes of a long display name - was refused for good. Also fixed: a display name in a charset Go does not convert was refused; the statistics ran before the check served, and a hung `supervisorctl` would have kept it from serving; the count of deferred mail missed a check that stops answering in the middle of a message, and is now taken at Postfix's warning; a card whose figures could not be read said ok; a killed container's stale socket satisfied the wrapper's wait; outside quoted strings the line may now hold one `@` only. Left as found: downtime that crosses midnight is booked on the day of the restart; a message with the null envelope sender leaves a row without a domain; a log read again after an unclean stop counts its deferrals again)
- [x] Security review — **Fable** (2026-10-12; no Critical or High. The same folded line, as its one Medium. Fixed: another sender's name in an encoded display name passed the second look; a client could arm the deferred counter by writing the socket's name where Postfix quotes it in a warning; a refused message wrote a row per recipient, a thousand at most - now one. Recorded: the check's lookup is stricter than Postfix's in two places, address extensions and the case of logins; what a display name looks like to a person is not promised. Confirmed from the code: no path to the signer around the check, and every failure of the check defers)
- [x] Technical review — **Opus** (2026-10-12; one Critical, fixed: a folded `From:` line - what a mail library makes of a long display name - was refused for good. Also fixed: a display name in a charset Go does not convert was refused; the statistics ran before the check served, and a hung `supervisorctl` would have kept it from serving; the count of deferred mail missed a check that stops answering in the middle of a message, and is now taken at Postfix's warning; a card whose figures could not be read said ok; a killed container's stale socket satisfied the wrapper's wait; outside quoted strings the line may now hold one `@` only. Three were first left as found and fixed the same evening at the owner's question: downtime is cut at the UTC midnights it crossed; the row of a message with the null envelope sender takes the domain the login is bound to; a log read again after an unclean stop does not count again what it counted before)
- [x] Security review — **Fable** (2026-10-12; no Critical or High. The same folded line, as its one Medium. Fixed: another sender's name in an encoded display name passed the second look; a client could arm the deferred counter by writing the socket's name where Postfix quotes it in a warning; a refused message wrote a row per recipient, a thousand at most - now one. Recorded: what a display name looks like to a person is not promised. The review also found the check stricter than Postfix for an address with an extension (`invoices+x@`, allowed in the envelope and refused in the header); the owner asked for it to work, and the check now makes Postfix's second try, with `recipient_delimiter` set explicitly. Confirmed from the code: no path to the signer around the check, and every failure of the check defers)
+1 -1
View File
@@ -153,7 +153,7 @@ func Run(ctx context.Context, path string, st StatusStore, retention RetentionDa
journal.handle(line, now, catchingUp)
}
if deferred != nil {
deferred.handle(line, now)
deferred.handle(line, now, catchingUp)
}
keeper.add(line, now, catchingUp)
queueID, recipient, status, ok := parseDelivery(line)
+24 -3
View File
@@ -33,9 +33,11 @@ import (
// "milter unix:<socket>: can't read … reply packet header" (Postfix's
// milter8.c; not seen on the stand).
// SenderCheckStore is where the count goes. *store.Store satisfies it.
// SenderCheckStore is where the count goes, and where the time of the last
// counted line is kept. *store.Store satisfies it.
type SenderCheckStore interface {
AddSenderCheckDeferred(at time.Time, n int64) error
SenderCheckDeferredAt() (time.Time, error)
}
// SenderCheck turns the counting on: the store and the socket Postfix was
@@ -60,10 +62,22 @@ type deferredCounter struct {
// listening, and a reply that did not come.
connect, broken string
failed map[string]bool // smtpd pid -> its current client was counted already
// last is the time of the latest line counted, by this run or an earlier
// one. While the tailer catches up after a start it may be reading lines
// it had read before an unclean stop; a warning no later than this was
// counted then. mail.log's stamps are whole seconds, so a warning of the
// very second of the last one counted can be passed over - once, at a
// restart - which is the cheaper mistake than counting a stretch twice.
last time.Time
}
func newDeferredCounter(sc *SenderCheck) *deferredCounter {
last, err := sc.Store.SenderCheckDeferredAt()
if err != nil {
log.Printf("log-tailer: sender check statistics: %v", err)
}
return &deferredCounter{
last: last,
st: sc.Store,
connect: "warning: connect to Milter service unix:" + sc.Socket + ":",
broken: "warning: milter unix:" + sc.Socket + ":",
@@ -71,8 +85,9 @@ func newDeferredCounter(sc *SenderCheck) *deferredCounter {
}
}
// handle looks at one mail.log line.
func (c *deferredCounter) handle(line string, now time.Time) {
// handle looks at one mail.log line. catchingUp is true until the tailer has
// read to the end of the log for the first time.
func (c *deferredCounter) handle(line string, now time.Time, catchingUp bool) {
at, rest := lineTime(line, now)
m := smtpdLineRe.FindStringSubmatch(rest)
if m == nil {
@@ -94,6 +109,12 @@ func (c *deferredCounter) handle(line string, now time.Time) {
c.failed = make(map[string]bool)
}
c.failed[pid] = true
if catchingUp && !at.After(c.last) {
return
}
if at.After(c.last) {
c.last = at
}
if err := c.st.AddSenderCheckDeferred(at, 1); err != nil {
log.Printf("log-tailer: sender check statistics: %v", err)
}
+39 -1
View File
@@ -7,13 +7,19 @@ import (
type deferredStore struct {
days map[string]int64
last time.Time
}
func (d *deferredStore) SenderCheckDeferredAt() (time.Time, error) { return d.last, nil }
func (d *deferredStore) AddSenderCheckDeferred(at time.Time, n int64) error {
if d.days == nil {
d.days = make(map[string]int64)
}
d.days[at.UTC().Format("2006-01-02")] += n
if at.After(d.last) {
d.last = at
}
return nil
}
@@ -65,9 +71,41 @@ func TestDeferredCounter(t *testing.T) {
// cleanup that writes it.
`2026-10-13T00:01:00.000000+00:00 mail postfix/cleanup[1195]: B6B484831946: milter-reject: END-OF-MESSAGE from unknown[10.77.0.1]: 5.7.1 From: director@other.e2e.test - not an address this application sends as; from=<alerts@sender.e2e.test> to=<rcpt@sink.e2e.test> proto=ESMTP helo=<x>`,
} {
c.handle(line, now)
c.handle(line, now, false)
}
if len(st.days) != 2 || st.days["2026-10-12"] != 1 || st.days["2026-10-13"] != 1 {
t.Fatalf("deferred per day = %v, want one on each of the two days", st.days)
}
}
// After an unclean stop the tailer reads again some lines it had read. What
// was counted then is not counted now; what was written while the panel was
// down - the very lines this count exists for - is.
func TestDeferredCounterDoesNotCountALogReadAgain(t *testing.T) {
st := &deferredStore{}
now := time.Date(2026, 10, 12, 18, 0, 0, 0, time.UTC)
outage := func(pid, at string) []string {
return []string{
at + `0+00:00 mail postfix/smtpd[` + pid + `]: connect from unknown[10.77.0.1]`,
at + `1+00:00 mail postfix/smtpd[` + pid + `]: warning: connect to Milter service unix:/run/selfpost/sender.sock: Connection refused`,
}
}
first := append(outage("11", "2026-10-12T16:00:00.00000"), outage("12", "2026-10-12T16:00:05.00000")...)
c := newDeferredCounter(&SenderCheck{Store: st, Socket: "/run/selfpost/sender.sock"})
for _, line := range first {
c.handle(line, now, false)
}
if st.days["2026-10-12"] != 2 {
t.Fatalf("first run counted %d, want 2", st.days["2026-10-12"])
}
// A new run: the same lines again, then what the outage added.
c = newDeferredCounter(&SenderCheck{Store: st, Socket: "/run/selfpost/sender.sock"})
for _, line := range append(first, outage("13", "2026-10-12T16:00:09.00000")...) {
c.handle(line, now, true)
}
if st.days["2026-10-12"] != 3 {
t.Fatalf("after the restart the day has %d, want 3", st.days["2026-10-12"])
}
}
+61 -8
View File
@@ -31,9 +31,9 @@ import (
// answer, Postfix defers mail. It therefore depends on as little as it can.
// It reads no database: what an application may send as comes from the file
// of sender bindings Postfix itself reads for the envelope, so the two go by
// one table. Where the lookups differ the check is the stricter: Postfix
// also tries an address without its "+extension" and compares logins without
// regard to case. The send-log row of a refusal is written by another goroutine and
// one table, by one lookup: the address, the address without its
// "+extension", the domain. (Postfix compares logins without regard to case;
// the check compares them as written, which is how the panel writes them.) The send-log row of a refusal is written by another goroutine and
// is not waited for.
// tableFile is a table of the mail path that the panel rewrites and the check
@@ -69,9 +69,10 @@ func (f *tableFile) changed() (lines []string, again bool, err error) {
// sender - an address, or "@domain" for any address of a domain - with the
// logins that may use it.
type Bindings struct {
mu sync.Mutex
file tableFile
logins map[string]map[string]bool // sender -> logins
mu sync.Mutex
file tableFile
logins map[string]map[string]bool // sender -> logins
domainOf map[string]string // login -> the domain it sends as
}
// NewBindings reads the file at path on first use.
@@ -86,6 +87,7 @@ func (b *Bindings) fresh() error {
return err
}
logins := make(map[string]map[string]bool)
domains := make(map[string]string)
for _, line := range lines {
// texthash: <key><whitespace><value>, the value a comma-separated
// list of logins (internal/postfix renderSenderLoginMaps).
@@ -102,10 +104,13 @@ func (b *Bindings) fresh() error {
for _, l := range strings.Split(strings.Join(fields[1:], ""), ",") {
if l != "" {
set[l] = true
if _, known := domains[l]; !known {
domains[l] = domainOf(sender)
}
}
}
}
b.logins = logins
b.logins, b.domainOf = logins, domains
return nil
}
@@ -122,12 +127,48 @@ func (b *Bindings) Allowed(login, addr string) (bool, error) {
if b.logins[addr][login] {
return true, nil
}
// Postfix's second try: the address without its extension, so that an
// application listed as invoices@ may also send as invoices+march@
// (recipient_delimiter in build/postfix-config.sh).
if bare := withoutExtension(addr); bare != addr && b.logins[bare][login] {
return true, nil
}
if d := domainOf(addr); d != "" && b.logins["@"+d][login] {
return true, nil
}
return false, nil
}
// extensionDelimiter is Postfix's recipient_delimiter, which
// build/postfix-config.sh sets to this.
const extensionDelimiter = "+"
// withoutExtension drops the extension of an address's local part: what
// follows its first delimiter. "user+tag@example.org" is "user@example.org";
// an address whose local part begins with the delimiter has no extension.
func withoutExtension(addr string) string {
at := strings.LastIndexByte(addr, '@')
if at < 0 {
return addr
}
if i := strings.Index(addr[:at], extensionDelimiter); i > 0 {
return addr[:i] + addr[at:]
}
return addr
}
// DomainOf returns the domain login sends as, from the first of its bindings
// in the file's order; "" for a login that has none. An application belongs
// to one domain, so all of its bindings name the same one.
func (b *Bindings) DomainOf(login string) string {
b.mu.Lock()
defer b.mu.Unlock()
if b.fresh() != nil {
return ""
}
return b.domainOf[login]
}
// Signers is OpenDKIM's SigningTable as the check reads it: the names this
// server signs for - every sending domain, with applications or without, and
// the server's own name.
@@ -319,7 +360,7 @@ func (s *senderSession) record(shown string) {
to += fmt.Sprintf(" and %d more", more)
}
e := store.SendLogEntry{
Domain: domainOf(s.from),
Domain: s.domain(),
AppLogin: s.login,
From: s.from,
To: to,
@@ -365,6 +406,18 @@ func unquoted(v string) string {
return sb.String()
}
// domain is the sending domain of the refused message for its row in the
// log: the envelope sender's, which Postfix has held against the login - or,
// for the null sender of a bounce an application writes itself, which Postfix
// holds against nobody, the domain the login is bound to. Without it the row
// would belong to no domain and its administrator would never see it.
func (s *senderSession) domain() string {
if d := domainOf(s.from); d != "" {
return d
}
return s.bind.DomainOf(s.login)
}
// mentionedDomains returns, lower-cased, what follows every "@" of v as far
// as it reads as a host name.
func mentionedDomains(v string) []string {
+17
View File
@@ -94,6 +94,8 @@ func TestSenderCheck(t *testing.T) {
{"own domain, any address, with a name", "shop", [][2]string{from(`"The Shop" <Orders@Shop.Example.Org>`)}, true, ""},
{"second application of the domain", "shop2", [][2]string{from("x@shop.example.org")}, true, ""},
{"a listed address", "billing", [][2]string{from("Invoices <invoices@example.org>")}, true, ""},
{"a listed address with an extension", "billing", [][2]string{from("invoices+2026-march@example.org")}, true, ""},
{"an extension, any address of the domain", "shop", [][2]string{from("orders+48213@shop.example.org")}, true, ""},
{"header name in another case", "shop", [][2]string{{"FROM", "a@shop.example.org"}}, true, ""},
{"its own name inside the display name", "shop", [][2]string{from(`"shop@shop.example.org" <a@shop.example.org>`)}, true, ""},
{"an encoded display name", "shop", [][2]string{from("=?utf-8?b?0JzQsNCz0LDQt9C40L0=?= <a@shop.example.org>")}, true, ""},
@@ -107,6 +109,8 @@ func TestSenderCheck(t *testing.T) {
{"the server's own name", "shop", [][2]string{from("postmaster@mail.example.org")}, false, "not an address this application sends as"},
{"a stranger's domain", "shop", [][2]string{from("someone@gmail.example")}, false, "not an address this application sends as"},
{"an address off the list", "billing", [][2]string{from("director@example.org")}, false, "not an address this application sends as"},
{"an address off the list, with an extension", "billing", [][2]string{from("director+invoices@example.org")}, false, "not an address this application sends as"},
{"a listed name as the extension of another", "billing", [][2]string{from("+invoices@example.org")}, false, "not an address this application sends as"},
{"a subdomain of its own", "shop", [][2]string{from("a@sub.shop.example.org")}, false, "not an address this application sends as"},
{"no From: line", "shop", [][2]string{{"Subject", "hello"}}, false, "no From: line"},
{"two From: lines", "shop", [][2]string{from("a@shop.example.org"), from("director@example.org")}, false, "more than one From: line"},
@@ -254,3 +258,16 @@ func TestSenderCheckRefusalIsOneRow(t *testing.T) {
t.Fatalf("rows = %+v", rows)
}
}
// A message with the null envelope sender - a bounce an application writes
// itself - is judged like any other, and its row belongs to the domain the
// login is bound to.
func TestSenderCheckRefusalWithNullSenderHasItsDomain(t *testing.T) {
bind := NewBindings(writeMaps(t, senderMaps))
for login, want := range map[string]string{"shop": "shop.example.org", "billing": "example.org"} {
_, rows := offer(t, bind, login, "", [2]string{"From", "someone@gmail.example"})
if len(rows) != 1 || rows[0].Domain != want || rows[0].From != "" {
t.Errorf("%s: rows = %+v, want one of domain %s", login, rows, want)
}
}
}
+14 -1
View File
@@ -88,13 +88,26 @@ func (s *Store) AddSenderCheckDeferred(at time.Time, n int64) error {
if err != nil {
return err
}
stamp := at.UTC().Format(time.RFC3339)
// To the fraction of a second the log gives, and of a fixed width so that
// the strings compare as the times do.
stamp := at.UTC().Format("2006-01-02T15:04:05.000000Z")
if stamp > last {
return s.SetSetting(settingSenderCheckDeferredAt, stamp)
}
return nil
}
// SenderCheckDeferredAt is when a message was last counted as deferred; zero
// if none was.
func (s *Store) SenderCheckDeferredAt() (time.Time, error) {
v, err := s.GetSetting(settingSenderCheckDeferredAt)
if err != nil || v == "" {
return time.Time{}, err
}
at, _ := time.Parse(time.RFC3339Nano, v)
return at, nil
}
// SenderCheckStats sums the days up to now: today, the last 30 days and the
// last 12 months, each including today.
func (s *Store) SenderCheckStats(now time.Time) (SenderCheckStats, error) {
+1
View File
@@ -40,6 +40,7 @@ func testSenderCheck(t *testing.T, sc *scenario) {
{"own address", wildcard, "alerts@" + senderDomain, true},
{"another address of the domain, with a name", wildcard, `"The Shop" <orders@` + senderDomain + `>`, true},
{"the listed address", list, "billing@" + senderDomain, true},
{"the listed address with an extension, in envelope and header", who{listLogin, listPassword, "billing+march@" + senderDomain}, "billing+march@" + senderDomain, true},
// What a mail library does with a long display name: the line is
// folded, and Postfix hands the milter both of its lines.
{"a long name, folded onto a second line", wildcard, "\"The Shop, its orders and its returns desk\"\r\n <orders@" + senderDomain + ">", true},