release: 0.1.0: Phases 0-11: panel, Postfix/DKIM, milter, backup, Docker image

Initial feature-complete implementation of the v1.0 specification (phases 0-11 of `docs/implementation-plan.md`).
Added:
  - Panel (Go, single static binary) with SQLite persistence, one-time
  - Domain management with per-domain DKIM (RSA-2048, generated in pure Go) and
  - Application (sender identity) management: SASL credentials via `sasldb2`,
  - Full Postfix relay config generated from env at container start: SMTPS 465,
  - Journal milter (pure Go, `go-milter`) recording every send to `send_log`;
  … (6 more in CHANGELOG)

Full detail: CHANGELOG.md (0.1.0).
This commit is contained in:
mix
2026-09-22 15:24:51 +03:00
commit 8ce050ef7f
157 changed files with 17631 additions and 0 deletions
+188
View File
@@ -0,0 +1,188 @@
// Package milter implements the SelfPost journal-milter: a lightweight milter
// (spec 7.3) attached to Postfix's smtpd_milters alongside OpenDKIM. On the
// receive path it reads the SASL login, From, recipients and Subject of each
// accepted message and records one send-log row per (queue-id, recipient),
// giving the panel a structured, filterable history that raw mail.log cannot.
//
// It is monitoring only: it never rejects, and every callback returns Continue
// or Accept so a failure of this milter can never block the relay. Postfix is
// configured with default_action=accept for this milter's socket, so even a
// crash or hang fails open (spec 7.3).
package milter
import (
"context"
"log"
"net"
"net/textproto"
"strings"
"time"
"github.com/emersion/go-milter"
"codeberg.org/mix/selfpost/internal/store"
)
// Store is the persistence the milter needs on the receive path: recording
// accepted messages (spec 7.3) and, for level-2 rate limiting (spec 7.4),
// looking up the configured limits and counting recent messages. *store.Store
// satisfies it; tests substitute a fake.
type Store interface {
InsertQueued(e store.SendLogEntry) error
InsertRejected(e store.SendLogEntry) error
RateLimit(scope, ref string) (store.RateLimit, bool, error)
CountMessages(scope, ref string, since time.Time) (int64, error)
}
// session accumulates the fields of one message as the milter callbacks fire.
// Milter macros arrive per-stage and do not accumulate, so each value is
// captured at the stage that carries it (spec 7.3 / Phase 0 spike): SASL login
// and From at MAIL, each recipient at RCPT, Subject in the headers, and the
// queue-id at end-of-message. go-milter creates one session per connection; a
// connection may carry several messages, so per-message fields are reset at
// MailFrom (the start of every transaction).
type session struct {
milter.NoOpMilter
rec Store
clientIP string // captured once per connection
login string
from string
rcpts []string
subject string
}
// Connect captures the client IP, which comes from the addr parameter rather
// than a macro (the {client_addr} macro was empty in the spike). It is the
// rate-limit key for Phase 8; here it is recorded for completeness.
func (s *session) Connect(host, family string, port uint16, addr net.IP, m *milter.Modifier) (milter.Response, error) {
if addr != nil {
s.clientIP = addr.String()
}
return milter.RespContinue, nil
}
// MailFrom starts a new message: reset per-message state, then capture the
// envelope sender and the SASL login ({auth_authen}, carried by the MAIL-stage
// macros). This is also the earliest stage where both the sending domain (from
// the sender) and the application (the login) are known, so the level-2 rate
// limit is enforced here: over the limit, the message is refused with a 4xx
// tempfail before recipients are even offered (spec 7.4). Enforcement is
// fail-open — see overLimit.
func (s *session) MailFrom(from string, m *milter.Modifier) (milter.Response, error) {
s.from = cleanAddress(from)
s.login = macro(m, "auth_authen")
s.rcpts = nil
s.subject = ""
if s.overLimit() {
s.recordRejected()
return milter.RespTempFail, nil
}
return milter.RespContinue, nil
}
// RcptTo records each recipient. Postfix calls this once per recipient, which
// is what lets the journal keep a separate row per (queue-id, recipient).
func (s *session) RcptTo(rcpt string, m *milter.Modifier) (milter.Response, error) {
s.rcpts = append(s.rcpts, cleanAddress(rcpt))
return milter.RespContinue, nil
}
// Header captures the Subject. Only the first Subject header is kept.
func (s *session) Header(name, value string, m *milter.Modifier) (milter.Response, error) {
if s.subject == "" && textproto.CanonicalMIMEHeaderKey(name) == "Subject" {
s.subject = value
}
return milter.RespContinue, nil
}
// Body fires at end-of-message, when the queue-id macro {i} is set and the
// message is about to be committed to the queue. This is where the "queued"
// rows are written. We accept (this milter is done) without ever rejecting.
func (s *session) Body(m *milter.Modifier) (milter.Response, error) {
s.record(macro(m, "i"))
return milter.RespAccept, nil
}
// macro reads a milter macro, tolerating Postfix's convention of wrapping
// multi-character macro names in curly braces (e.g. {auth_authen}) while
// single-character names (e.g. i) arrive bare. go-milter stores whatever name
// Postfix sends verbatim, so a lookup must try both forms — this is exactly the
// distinction the SASL-less Phase 0 spike could not observe.
func macro(m *milter.Modifier, name string) string {
if v, ok := m.Macros[name]; ok {
return v
}
return m.Macros["{"+name+"}"]
}
// record writes one send-log row per recipient. Failures are logged, never
// propagated: journalling must not affect mail acceptance (spec 7.3).
func (s *session) record(queueID string) {
domain := domainOf(s.from)
rcpts := s.rcpts
if len(rcpts) == 0 {
// No recipient seen (unusual) — still record the message so it is
// visible in the log rather than silently dropped.
rcpts = []string{""}
}
for _, to := range rcpts {
err := s.rec.InsertQueued(store.SendLogEntry{
QueueID: queueID,
Domain: domain,
AppLogin: s.login,
From: s.from,
To: to,
Subject: s.subject,
})
if err != nil {
log.Printf("journal-milter: record %s -> %s: %v", queueID, to, err)
}
}
}
// cleanAddress strips the angle brackets and any ESMTP parameters Postfix may
// pass with an address, leaving the bare mailbox.
func cleanAddress(a string) string {
a = strings.TrimSpace(a)
if i := strings.IndexByte(a, ' '); i >= 0 { // drop "addr SIZE=… BODY=…" params
a = a[:i]
}
a = strings.TrimPrefix(a, "<")
a = strings.TrimSuffix(a, ">")
return a
}
// domainOf returns the lower-cased domain of an email address, or "" if there
// is no domain part. Sender binding (Phase 4) guarantees the From domain equals
// the application's domain, so this is the sending domain (spec 7.3).
func domainOf(addr string) string {
if i := strings.LastIndexByte(addr, '@'); i >= 0 {
return strings.ToLower(addr[i+1:])
}
return ""
}
// Serve runs the journal-milter on ln until ctx is cancelled. Each connection
// gets a fresh session bound to rec. It returns nil on a clean shutdown.
func Serve(ctx context.Context, ln net.Listener, rec Store) error {
srv := &milter.Server{
NewMilter: func() milter.Milter { return &session{rec: rec} },
Actions: 0, // read-only: we make no message modifications
Protocol: milter.OptNoBody, // the journal needs headers/EOM, not the body
}
go func() {
<-ctx.Done()
_ = srv.Close()
}()
if err := srv.Serve(ln); err != nil {
if ctx.Err() != nil {
return nil // expected: Close() during shutdown unblocks Serve
}
return err
}
return nil
}
+317
View File
@@ -0,0 +1,317 @@
package milter
import (
"errors"
"net"
"testing"
"time"
"github.com/emersion/go-milter"
"codeberg.org/mix/selfpost/internal/store"
)
// fakeRecorder captures inserts and can be made to fail, to prove the milter
// swallows recorder errors and still accepts the message. By default it reports
// no configured rate limit, so the level-2 check is inert unless a test sets
// limits (see fakeRecorder fields).
type fakeRecorder struct {
entries []store.SendLogEntry
rejected []store.SendLogEntry
fail bool
// limits, keyed by "scope|ref", drive the level-2 rate-limit tests. counts
// gives the recent-message count returned for a "scope|ref". lookupErr and
// countErr force the store errors that must fail open.
limits map[string]store.RateLimit
counts map[string]int64
lookupErr error
countErr error
}
func (f *fakeRecorder) InsertQueued(e store.SendLogEntry) error {
if f.fail {
return errors.New("boom")
}
f.entries = append(f.entries, e)
return nil
}
func (f *fakeRecorder) InsertRejected(e store.SendLogEntry) error {
f.rejected = append(f.rejected, e)
return nil
}
func (f *fakeRecorder) RateLimit(scope, ref string) (store.RateLimit, bool, error) {
if f.lookupErr != nil {
return store.RateLimit{}, false, f.lookupErr
}
rl, ok := f.limits[scope+"|"+ref]
return rl, ok, nil
}
func (f *fakeRecorder) CountMessages(scope, ref string, _ time.Time) (int64, error) {
if f.countErr != nil {
return 0, f.countErr
}
return f.counts[scope+"|"+ref], nil
}
func mods(kv map[string]string) *milter.Modifier {
return &milter.Modifier{Macros: kv}
}
// drive replays a typical message through one session and returns the recorder.
func drive(t *testing.T, rec Store) *session {
t.Helper()
s := &session{rec: rec}
if _, err := s.Connect("localhost", "tcp4", 0, net.ParseIP("203.0.113.7"), mods(nil)); err != nil {
t.Fatalf("Connect: %v", err)
}
if _, err := s.MailFrom("noreply@example.com", mods(map[string]string{"auth_authen": "app1"})); err != nil {
t.Fatalf("MailFrom: %v", err)
}
if _, err := s.RcptTo("<a@example.net>", mods(nil)); err != nil {
t.Fatalf("RcptTo: %v", err)
}
if _, err := s.RcptTo("b@example.net", mods(nil)); err != nil {
t.Fatalf("RcptTo: %v", err)
}
if _, err := s.Header("Subject", "Hello there", mods(nil)); err != nil {
t.Fatalf("Header: %v", err)
}
if _, err := s.Body(mods(map[string]string{"i": "ABC123"})); err != nil {
t.Fatalf("Body: %v", err)
}
return s
}
func TestSessionRecordsRowPerRecipient(t *testing.T) {
rec := &fakeRecorder{}
s := drive(t, rec)
if s.clientIP != "203.0.113.7" {
t.Fatalf("clientIP = %q, want 203.0.113.7", s.clientIP)
}
if len(rec.entries) != 2 {
t.Fatalf("want 2 entries, got %d: %+v", len(rec.entries), rec.entries)
}
got := rec.entries[0]
want := store.SendLogEntry{
QueueID: "ABC123",
Domain: "example.com",
AppLogin: "app1",
From: "noreply@example.com",
To: "a@example.net", // angle brackets stripped
Subject: "Hello there",
}
if got != want {
t.Fatalf("entry[0]\n got %+v\nwant %+v", got, want)
}
if rec.entries[1].To != "b@example.net" {
t.Fatalf("entry[1].To = %q", rec.entries[1].To)
}
}
func TestBodyAcceptsEvenWhenRecorderFails(t *testing.T) {
rec := &fakeRecorder{fail: true}
s := &session{rec: rec}
_, _ = s.MailFrom("x@example.com", mods(map[string]string{"auth_authen": "app1"}))
_, _ = s.RcptTo("y@example.net", mods(nil))
resp, err := s.Body(mods(map[string]string{"i": "Q9"}))
if err != nil {
t.Fatalf("Body returned error, must fail open: %v", err)
}
if resp != milter.RespAccept {
t.Fatalf("Body response = %v, want Accept", resp)
}
}
// A single connection may carry several messages; the second must not inherit
// the first's recipients or subject.
func TestSessionResetsBetweenMessages(t *testing.T) {
rec := &fakeRecorder{}
s := &session{rec: rec}
_, _ = s.MailFrom("a@example.com", mods(map[string]string{"auth_authen": "app1"}))
_, _ = s.RcptTo("one@example.net", mods(nil))
_, _ = s.Header("Subject", "first", mods(nil))
_, _ = s.Body(mods(map[string]string{"i": "Q1"}))
_, _ = s.MailFrom("b@example.com", mods(map[string]string{"auth_authen": "app2"}))
_, _ = s.RcptTo("two@example.net", mods(nil))
_, _ = s.Body(mods(map[string]string{"i": "Q2"}))
if len(rec.entries) != 2 {
t.Fatalf("want 2 entries, got %d", len(rec.entries))
}
second := rec.entries[1]
if second.QueueID != "Q2" || second.To != "two@example.net" || second.Subject != "" || second.AppLogin != "app2" {
t.Fatalf("second message leaked state: %+v", second)
}
}
// Postfix sends multi-character macro names wrapped in braces ({auth_authen},
// {i} for some versions), so the milter must resolve those too — this is the
// case the SASL-less spike missed and that produced empty app_login at first.
func TestBracedMacros(t *testing.T) {
rec := &fakeRecorder{}
s := &session{rec: rec}
_, _ = s.MailFrom("app@example.com", mods(map[string]string{"{auth_authen}": "app1"}))
_, _ = s.RcptTo("to@example.net", mods(nil))
_, _ = s.Body(mods(map[string]string{"{i}": "QBRACE"}))
if len(rec.entries) != 1 {
t.Fatalf("want 1 entry, got %d", len(rec.entries))
}
e := rec.entries[0]
if e.AppLogin != "app1" {
t.Fatalf("AppLogin = %q, want app1 (braced {auth_authen} not resolved)", e.AppLogin)
}
if e.QueueID != "QBRACE" {
t.Fatalf("QueueID = %q, want QBRACE (braced {i} not resolved)", e.QueueID)
}
}
// limitAt is the client IP the rate-limit tests connect from; the limits below
// register it so the differentiated check applies.
const limitIP = "203.0.113.7"
func activeLimit(ips ...string) store.RateLimit {
return store.RateLimit{AllowedIPs: ips, MaxMessages: 5, WindowSeconds: 3600}
}
// mailFrom drives just the connect + MAIL FROM stages and returns the response,
// which is where the level-2 limit is enforced.
func mailFrom(t *testing.T, rec Store, ip, from, login string) milter.Response {
t.Helper()
s := &session{rec: rec}
if _, err := s.Connect("h", "tcp4", 0, net.ParseIP(ip), mods(nil)); err != nil {
t.Fatalf("Connect: %v", err)
}
resp, err := s.MailFrom(from, mods(map[string]string{"auth_authen": login}))
if err != nil {
t.Fatalf("MailFrom: %v", err)
}
return resp
}
func TestRateLimitRefusesWhenDomainOverLimit(t *testing.T) {
rec := &fakeRecorder{
limits: map[string]store.RateLimit{
store.RateLimitScopeDomain + "|example.com": activeLimit(limitIP),
},
counts: map[string]int64{store.RateLimitScopeDomain + "|example.com": 5}, // == max
}
if resp := mailFrom(t, rec, limitIP, "a@example.com", "app1"); resp != milter.RespTempFail {
t.Fatalf("over-limit MAIL FROM = %v, want TempFail (4xx)", resp)
}
if len(rec.rejected) != 1 || rec.rejected[0].Domain != "example.com" {
t.Fatalf("want one rejected send-log row for example.com, got %+v", rec.rejected)
}
}
func TestRateLimitRefusesWhenAppOverLimit(t *testing.T) {
rec := &fakeRecorder{
limits: map[string]store.RateLimit{
store.RateLimitScopeApp + "|app1": activeLimit(limitIP),
},
counts: map[string]int64{store.RateLimitScopeApp + "|app1": 9}, // over max
}
if resp := mailFrom(t, rec, limitIP, "a@example.com", "app1"); resp != milter.RespTempFail {
t.Fatalf("over app limit = %v, want TempFail", resp)
}
}
func TestRateLimitAllowsUnderLimit(t *testing.T) {
rec := &fakeRecorder{
limits: map[string]store.RateLimit{
store.RateLimitScopeDomain + "|example.com": activeLimit(limitIP),
},
counts: map[string]int64{store.RateLimitScopeDomain + "|example.com": 4}, // < max
}
if resp := mailFrom(t, rec, limitIP, "a@example.com", "app1"); resp != milter.RespContinue {
t.Fatalf("under limit = %v, want Continue", resp)
}
if len(rec.rejected) != 0 {
t.Fatalf("under limit must not record a rejection: %+v", rec.rejected)
}
}
func TestRateLimitIgnoresUnregisteredIP(t *testing.T) {
rec := &fakeRecorder{
limits: map[string]store.RateLimit{
store.RateLimitScopeDomain + "|example.com": activeLimit("198.51.100.1"), // not limitIP
},
counts: map[string]int64{store.RateLimitScopeDomain + "|example.com": 999},
}
// The sender's IP is not in the domain's registered set, so level-2 does not
// apply even though the count is huge (level-1 anvil would still cover it).
if resp := mailFrom(t, rec, limitIP, "a@example.com", "app1"); resp != milter.RespContinue {
t.Fatalf("unregistered IP = %v, want Continue (level-2 n/a)", resp)
}
}
func TestRateLimitInactiveWithoutCeiling(t *testing.T) {
rec := &fakeRecorder{
// IP registered but no ceiling/window: an inert draft, must not enforce.
limits: map[string]store.RateLimit{
store.RateLimitScopeDomain + "|example.com": {AllowedIPs: []string{limitIP}},
},
counts: map[string]int64{store.RateLimitScopeDomain + "|example.com": 999},
}
if resp := mailFrom(t, rec, limitIP, "a@example.com", "app1"); resp != milter.RespContinue {
t.Fatalf("inactive limit = %v, want Continue", resp)
}
}
func TestRateLimitFailsOpenOnLookupError(t *testing.T) {
rec := &fakeRecorder{lookupErr: errors.New("db down")}
if resp := mailFrom(t, rec, limitIP, "a@example.com", "app1"); resp != milter.RespContinue {
t.Fatalf("lookup error = %v, want Continue (fail-open)", resp)
}
}
func TestRateLimitFailsOpenOnCountError(t *testing.T) {
rec := &fakeRecorder{
limits: map[string]store.RateLimit{
store.RateLimitScopeDomain + "|example.com": activeLimit(limitIP),
},
countErr: errors.New("db down"),
}
if resp := mailFrom(t, rec, limitIP, "a@example.com", "app1"); resp != milter.RespContinue {
t.Fatalf("count error = %v, want Continue (fail-open)", resp)
}
}
func TestRateLimitNoIPKeyDoesNotApply(t *testing.T) {
rec := &fakeRecorder{
limits: map[string]store.RateLimit{
store.RateLimitScopeDomain + "|example.com": activeLimit(limitIP),
},
counts: map[string]int64{store.RateLimitScopeDomain + "|example.com": 999},
}
// A session with no client IP (e.g. local submission) cannot be keyed.
s := &session{rec: rec}
resp, err := s.MailFrom("a@example.com", mods(map[string]string{"auth_authen": "app1"}))
if err != nil {
t.Fatalf("MailFrom: %v", err)
}
if resp != milter.RespContinue {
t.Fatalf("no-IP session = %v, want Continue", resp)
}
}
func TestDomainOf(t *testing.T) {
cases := map[string]string{
"user@Example.COM": "example.com",
"no-domain": "",
"": "",
"a@b@c.com": "c.com",
}
for in, want := range cases {
if got := domainOf(in); got != want {
t.Fatalf("domainOf(%q) = %q, want %q", in, got, want)
}
}
}
+70
View File
@@ -0,0 +1,70 @@
package milter
import (
"log"
"time"
"codeberg.org/mix/selfpost/internal/store"
)
// overLimit reports whether the message currently being received should be
// refused under a level-2 differentiated limit (spec 7.4). It checks the
// domain-level and application-level limits in turn; either being exceeded is
// enough to refuse.
//
// It is deliberately fail-open: any store error, or the absence of a usable
// limit, is treated as "not over limit" so a malfunction of the level-2 limiter
// can never block mail — Postfix's level-1 anvil limit (spec 5) remains the
// backstop, and it does not depend on this milter at all. Only a clean count at
// or above a configured ceiling returns true.
func (s *session) overLimit() bool {
if s.clientIP == "" {
return false // no client IP to key on; level-2 does not apply
}
checks := []struct{ scope, ref string }{
{store.RateLimitScopeDomain, domainOf(s.from)},
{store.RateLimitScopeApp, s.login},
}
for _, c := range checks {
if c.ref == "" {
continue
}
rl, ok, err := s.rec.RateLimit(c.scope, c.ref)
if err != nil {
log.Printf("journal-milter: rate-limit lookup %s %q: %v (fail-open)", c.scope, c.ref, err)
continue
}
// No limit configured, an inert draft, or a client IP outside the
// registered set: the differentiated limit does not apply here.
if !ok || !rl.Active() || !rl.AllowsIP(s.clientIP) {
continue
}
since := time.Now().Add(-time.Duration(rl.WindowSeconds) * time.Second)
n, err := s.rec.CountMessages(c.scope, c.ref, since)
if err != nil {
log.Printf("journal-milter: rate-limit count %s %q: %v (fail-open)", c.scope, c.ref, err)
continue
}
if n >= int64(rl.MaxMessages) {
log.Printf("journal-milter: %s %q over limit: %d/%d in %ds from %s — refusing 4xx",
c.scope, c.ref, n, rl.MaxMessages, rl.WindowSeconds, s.clientIP)
return true
}
}
return false
}
// recordRejected writes a send-log row for a message refused by a level-2 limit
// (spec 7.4, "опционально фиксирует ... для видимости в UI"), so the rejection
// shows up in the monitoring screen. Only MAIL-stage fields are known; the write
// is best-effort and never affects the response.
func (s *session) recordRejected() {
err := s.rec.InsertRejected(store.SendLogEntry{
Domain: domainOf(s.from),
AppLogin: s.login,
From: s.from,
})
if err != nil {
log.Printf("journal-milter: record rejected %s: %v", s.from, err)
}
}