Phase 6: journal-milter + send-log status tailer + retention

Implement the structured send log (spec 7.3), the project's highest-risk component since a milter bug can break the relay itself. - internal/milter: go-milter v0.4.1 journal-milter. Per-connection session collects SASL login, From, recipients and Subject across callbacks and writes one send_log "queued" row per (queue-id, recipient) at EOM (spec 7.3.3). Monitoring only: callbacks return Continue/Accept, recorder errors are logged never propagated, so it can never block mail. - internal/logtail: polling mail.log tailer with rotation handling (inode change / truncation), parses sent/deferred/bounced/expired by queue-id + recipient and advances rows; background retention sweep prunes rows past SEND_LOG_RETENTION_DAYS (default 90) at startup and every 6h. - internal/store/sendlog.go: InsertQueued, UpdateStatus (case-insensitive recipient match), DeleteSendLogBefore + status constants. - cmd/panel: open the store once and share it across http/milter/tailer; replace the journal/logtail stubs with the real roles. - build/postfix-config.sh: bounded milter timeouts (15/15/30s) so a hung milter also fails open in seconds, not the 300s default. Fix found in-container: SASL login (app_login) was empty because go-milter keys macros exactly as Postfix sends them, and multi-character macro names arrive brace-wrapped ({auth_authen}); the SASL-less Phase 0 spike could not observe this. Added a brace-tolerant macro lookup. Verified on selfpost.mixfed.ru: gofmt/vet/unit tests green; container e2e records rows with correct fields and advances status via the tailer; fail-open confirmed for both an unreachable and a hung milter; retention prunes at start. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-13 22:58:34 +03:00
parent 2dbd8d009e
commit dc08ccbf7c
13 changed files with 1033 additions and 51 deletions
@@ -0,0 +1,174 @@
+// Package milter implements the SelfPost journal-milter: a lightweight milter
+// (spec 7.3) attached to Postfix's smtpd_milters alongside OpenDKIM. On the
+// receive path it reads the SASL login, From, recipients and Subject of each
+// accepted message and records one send-log row per (queue-id, recipient),
+// giving the panel a structured, filterable history that raw mail.log cannot.
+//
+// It is monitoring only: it never rejects, and every callback returns Continue
+// or Accept so a failure of this milter can never block the relay. Postfix is
+// configured with default_action=accept for this milter's socket, so even a
+// crash or hang fails open (spec 7.3).
+package milter
+
+import (
+	"context"
+	"log"
+	"net"
+	"net/textproto"
+	"strings"
+
+	"github.com/emersion/go-milter"
+
+	"codeberg.org/mix/selfpost/internal/store"
+)
+
+// Recorder persists queued send-log entries. *store.Store satisfies it; tests
+// substitute a fake.
+type Recorder interface {
+	InsertQueued(e store.SendLogEntry) error
+}
+
+// session accumulates the fields of one message as the milter callbacks fire.
+// Milter macros arrive per-stage and do not accumulate, so each value is
+// captured at the stage that carries it (spec 7.3 / Phase 0 spike): SASL login
+// and From at MAIL, each recipient at RCPT, Subject in the headers, and the
+// queue-id at end-of-message. go-milter creates one session per connection; a
+// connection may carry several messages, so per-message fields are reset at
+// MailFrom (the start of every transaction).
+type session struct {
+	milter.NoOpMilter
+	rec Recorder
+
+	clientIP string // captured once per connection
+
+	login   string
+	from    string
+	rcpts   []string
+	subject string
+}
+
+// Connect captures the client IP, which comes from the addr parameter rather
+// than a macro (the {client_addr} macro was empty in the spike). It is the
+// rate-limit key for Phase 8; here it is recorded for completeness.
+func (s *session) Connect(host, family string, port uint16, addr net.IP, m *milter.Modifier) (milter.Response, error) {
+	if addr != nil {
+		s.clientIP = addr.String()
+	}
+	return milter.RespContinue, nil
+}
+
+// MailFrom starts a new message: reset per-message state, then capture the
+// envelope sender and the SASL login ({auth_authen}, carried by the MAIL-stage
+// macros).
+func (s *session) MailFrom(from string, m *milter.Modifier) (milter.Response, error) {
+	s.from = cleanAddress(from)
+	s.login = macro(m, "auth_authen")
+	s.rcpts = nil
+	s.subject = ""
+	return milter.RespContinue, nil
+}
+
+// RcptTo records each recipient. Postfix calls this once per recipient, which
+// is what lets the journal keep a separate row per (queue-id, recipient).
+func (s *session) RcptTo(rcpt string, m *milter.Modifier) (milter.Response, error) {
+	s.rcpts = append(s.rcpts, cleanAddress(rcpt))
+	return milter.RespContinue, nil
+}
+
+// Header captures the Subject. Only the first Subject header is kept.
+func (s *session) Header(name, value string, m *milter.Modifier) (milter.Response, error) {
+	if s.subject == "" && textproto.CanonicalMIMEHeaderKey(name) == "Subject" {
+		s.subject = value
+	}
+	return milter.RespContinue, nil
+}
+
+// Body fires at end-of-message, when the queue-id macro {i} is set and the
+// message is about to be committed to the queue. This is where the "queued"
+// rows are written. We accept (this milter is done) without ever rejecting.
+func (s *session) Body(m *milter.Modifier) (milter.Response, error) {
+	s.record(macro(m, "i"))
+	return milter.RespAccept, nil
+}
+
+// macro reads a milter macro, tolerating Postfix's convention of wrapping
+// multi-character macro names in curly braces (e.g. {auth_authen}) while
+// single-character names (e.g. i) arrive bare. go-milter stores whatever name
+// Postfix sends verbatim, so a lookup must try both forms — this is exactly the
+// distinction the SASL-less Phase 0 spike could not observe.
+func macro(m *milter.Modifier, name string) string {
+	if v, ok := m.Macros[name]; ok {
+		return v
+	}
+	return m.Macros["{"+name+"}"]
+}
+
+// record writes one send-log row per recipient. Failures are logged, never
+// propagated: journalling must not affect mail acceptance (spec 7.3).
+func (s *session) record(queueID string) {
+	domain := domainOf(s.from)
+	rcpts := s.rcpts
+	if len(rcpts) == 0 {
+		// No recipient seen (unusual) — still record the message so it is
+		// visible in the log rather than silently dropped.
+		rcpts = []string{""}
+	}
+	for _, to := range rcpts {
+		err := s.rec.InsertQueued(store.SendLogEntry{
+			QueueID:  queueID,
+			Domain:   domain,
+			AppLogin: s.login,
+			From:     s.from,
+			To:       to,
+			Subject:  s.subject,
+		})
+		if err != nil {
+			log.Printf("journal-milter: record %s -> %s: %v", queueID, to, err)
+		}
+	}
+}
+
+// cleanAddress strips the angle brackets and any ESMTP parameters Postfix may
+// pass with an address, leaving the bare mailbox.
+func cleanAddress(a string) string {
+	a = strings.TrimSpace(a)
+	if i := strings.IndexByte(a, ' '); i >= 0 { // drop "addr SIZE=… BODY=…" params
+		a = a[:i]
+	}
+	a = strings.TrimPrefix(a, "<")
+	a = strings.TrimSuffix(a, ">")
+	return a
+}
+
+// domainOf returns the lower-cased domain of an email address, or "" if there
+// is no domain part. Sender binding (Phase 4) guarantees the From domain equals
+// the application's domain, so this is the sending domain (spec 7.3).
+func domainOf(addr string) string {
+	if i := strings.LastIndexByte(addr, '@'); i >= 0 {
+		return strings.ToLower(addr[i+1:])
+	}
+	return ""
+}
+
+// Serve runs the journal-milter on ln until ctx is cancelled. Each connection
+// gets a fresh session bound to rec. It returns nil on a clean shutdown.
+func Serve(ctx context.Context, ln net.Listener, rec Recorder) error {
+	srv := &milter.Server{
+		NewMilter: func() milter.Milter { return &session{rec: rec} },
+		Actions:   0,                // read-only: we make no message modifications
+		Protocol:  milter.OptNoBody, // the journal needs headers/EOM, not the body
+	}
+
+	go func() {
+		<-ctx.Done()
+		_ = srv.Close()
+	}()
+
+	if err := srv.Serve(ln); err != nil {
+		if ctx.Err() != nil {
+			return nil // expected: Close() during shutdown unblocks Serve
+		}
+		return err
+	}
+	return nil
+}
@@ -0,0 +1,156 @@
+package milter
+
+import (
+	"errors"
+	"net"
+	"testing"
+
+	"github.com/emersion/go-milter"
+
+	"codeberg.org/mix/selfpost/internal/store"
+)
+
+// fakeRecorder captures inserts and can be made to fail, to prove the milter
+// swallows recorder errors and still accepts the message.
+type fakeRecorder struct {
+	entries []store.SendLogEntry
+	fail    bool
+}
+
+func (f *fakeRecorder) InsertQueued(e store.SendLogEntry) error {
+	if f.fail {
+		return errors.New("boom")
+	}
+	f.entries = append(f.entries, e)
+	return nil
+}
+
+func mods(kv map[string]string) *milter.Modifier {
+	return &milter.Modifier{Macros: kv}
+}
+
+// drive replays a typical message through one session and returns the recorder.
+func drive(t *testing.T, rec Recorder) *session {
+	t.Helper()
+	s := &session{rec: rec}
+	if _, err := s.Connect("localhost", "tcp4", 0, net.ParseIP("203.0.113.7"), mods(nil)); err != nil {
+		t.Fatalf("Connect: %v", err)
+	}
+	if _, err := s.MailFrom("noreply@example.com", mods(map[string]string{"auth_authen": "app1"})); err != nil {
+		t.Fatalf("MailFrom: %v", err)
+	}
+	if _, err := s.RcptTo("<a@example.net>", mods(nil)); err != nil {
+		t.Fatalf("RcptTo: %v", err)
+	}
+	if _, err := s.RcptTo("b@example.net", mods(nil)); err != nil {
+		t.Fatalf("RcptTo: %v", err)
+	}
+	if _, err := s.Header("Subject", "Hello there", mods(nil)); err != nil {
+		t.Fatalf("Header: %v", err)
+	}
+	if _, err := s.Body(mods(map[string]string{"i": "ABC123"})); err != nil {
+		t.Fatalf("Body: %v", err)
+	}
+	return s
+}
+
+func TestSessionRecordsRowPerRecipient(t *testing.T) {
+	rec := &fakeRecorder{}
+	s := drive(t, rec)
+
+	if s.clientIP != "203.0.113.7" {
+		t.Fatalf("clientIP = %q, want 203.0.113.7", s.clientIP)
+	}
+	if len(rec.entries) != 2 {
+		t.Fatalf("want 2 entries, got %d: %+v", len(rec.entries), rec.entries)
+	}
+	got := rec.entries[0]
+	want := store.SendLogEntry{
+		QueueID:  "ABC123",
+		Domain:   "example.com",
+		AppLogin: "app1",
+		From:     "noreply@example.com",
+		To:       "a@example.net", // angle brackets stripped
+		Subject:  "Hello there",
+	}
+	if got != want {
+		t.Fatalf("entry[0]\n got %+v\nwant %+v", got, want)
+	}
+	if rec.entries[1].To != "b@example.net" {
+		t.Fatalf("entry[1].To = %q", rec.entries[1].To)
+	}
+}
+
+func TestBodyAcceptsEvenWhenRecorderFails(t *testing.T) {
+	rec := &fakeRecorder{fail: true}
+	s := &session{rec: rec}
+	_, _ = s.MailFrom("x@example.com", mods(map[string]string{"auth_authen": "app1"}))
+	_, _ = s.RcptTo("y@example.net", mods(nil))
+	resp, err := s.Body(mods(map[string]string{"i": "Q9"}))
+	if err != nil {
+		t.Fatalf("Body returned error, must fail open: %v", err)
+	}
+	if resp != milter.RespAccept {
+		t.Fatalf("Body response = %v, want Accept", resp)
+	}
+}
+
+// A single connection may carry several messages; the second must not inherit
+// the first's recipients or subject.
+func TestSessionResetsBetweenMessages(t *testing.T) {
+	rec := &fakeRecorder{}
+	s := &session{rec: rec}
+
+	_, _ = s.MailFrom("a@example.com", mods(map[string]string{"auth_authen": "app1"}))
+	_, _ = s.RcptTo("one@example.net", mods(nil))
+	_, _ = s.Header("Subject", "first", mods(nil))
+	_, _ = s.Body(mods(map[string]string{"i": "Q1"}))
+
+	_, _ = s.MailFrom("b@example.com", mods(map[string]string{"auth_authen": "app2"}))
+	_, _ = s.RcptTo("two@example.net", mods(nil))
+	_, _ = s.Body(mods(map[string]string{"i": "Q2"}))
+
+	if len(rec.entries) != 2 {
+		t.Fatalf("want 2 entries, got %d", len(rec.entries))
+	}
+	second := rec.entries[1]
+	if second.QueueID != "Q2" || second.To != "two@example.net" || second.Subject != "" || second.AppLogin != "app2" {
+		t.Fatalf("second message leaked state: %+v", second)
+	}
+}
+
+// Postfix sends multi-character macro names wrapped in braces ({auth_authen},
+// {i} for some versions), so the milter must resolve those too — this is the
+// case the SASL-less spike missed and that produced empty app_login at first.
+func TestBracedMacros(t *testing.T) {
+	rec := &fakeRecorder{}
+	s := &session{rec: rec}
+	_, _ = s.MailFrom("app@example.com", mods(map[string]string{"{auth_authen}": "app1"}))
+	_, _ = s.RcptTo("to@example.net", mods(nil))
+	_, _ = s.Body(mods(map[string]string{"{i}": "QBRACE"}))
+
+	if len(rec.entries) != 1 {
+		t.Fatalf("want 1 entry, got %d", len(rec.entries))
+	}
+	e := rec.entries[0]
+	if e.AppLogin != "app1" {
+		t.Fatalf("AppLogin = %q, want app1 (braced {auth_authen} not resolved)", e.AppLogin)
+	}
+	if e.QueueID != "QBRACE" {
+		t.Fatalf("QueueID = %q, want QBRACE (braced {i} not resolved)", e.QueueID)
+	}
+}
+
+func TestDomainOf(t *testing.T) {
+	cases := map[string]string{
+		"user@Example.COM": "example.com",
+		"no-domain":        "",
+		"":                 "",
+		"a@b@c.com":        "c.com",
+	}
+	for in, want := range cases {
+		if got := domainOf(in); got != want {
+			t.Fatalf("domainOf(%q) = %q, want %q", in, got, want)
+		}
+	}
+}