feat: log-tailer offset persistence + in-flight L2 rate-limit accounting (code-review.md § Phase 3)

- logtail: persist the read position (offset + fingerprint of the log's
  first 512 bytes) in a new logtail_state table (migration 0003) and
  resume from it on start, so delivery lines written while the panel was
  down are parsed instead of skipped and their send-log rows no longer
  stay "queued" forever. Fingerprint mismatch (rotated/recreated while
  down) reads the file from the start — re-parsing is idempotent; a
  first-ever start with nothing stored still begins at end-of-file.
  Writes are throttled to one per 5s, forced on rotation and shutdown.

- milter: count messages that passed the level-2 check but have not
  reached the send log yet (internal/milter/inflight.go), so concurrent
  SMTP sessions cannot each spend the same last slot. A literal
  count+insert transaction, as the review suggested, is not possible:
  the count happens at MAIL FROM and the insert at end-of-message.
  Reservations are released after the insert, on ABORT, and after a
  10-minute TTL — a client that drops mid-transaction must not be able
  to hold a slot, since the limiter is fail-open by design.

Docs: architecture.md (log tailer, persistence, L2 counting),
security.md and roadmap.md (restart gap closed, container recreate
remains), CHANGELOG, progress.md, code-review.md.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-06 17:12:37 +03:00
parent 0093878eea
commit a92d583053
15 changed files with 683 additions and 40 deletions
+124
View File
@@ -331,6 +331,130 @@ func TestRateLimitNoIPKeyDoesNotApply(t *testing.T) {
}
}
// mailFromIn is mailFrom with an explicit shared in-flight registry, so a test
// can play several concurrent SMTP sessions of one process against each other.
func mailFromIn(t *testing.T, rec Store, fl *inflight, ip, from, login string) (*session, milter.Response) {
t.Helper()
s := &session{rec: rec, flight: fl}
if _, err := s.Connect("h", "tcp4", 0, net.ParseIP(ip), mods(nil)); err != nil {
t.Fatalf("Connect: %v", err)
}
resp, err := s.MailFrom(from, mods(map[string]string{"auth_authen": login}))
if err != nil {
t.Fatalf("MailFrom: %v", err)
}
return s, resp
}
func limitedRecorder(count int64) *fakeRecorder {
return &fakeRecorder{
limits: map[string]store.RateLimit{
store.RateLimitScopeDomain + "|example.com": activeLimit(limitIP),
},
counts: map[string]int64{store.RateLimitScopeDomain + "|example.com": count},
}
}
// Messages between MAIL FROM and end-of-message are not in the send log yet, so
// counting the stored rows alone lets concurrent sessions each pass the same
// check and overshoot the ceiling. The last free slot may only be taken once.
func TestRateLimitCountsInFlightMessages(t *testing.T) {
rec := limitedRecorder(4) // one below the ceiling of 5
fl := &inflight{}
if _, resp := mailFromIn(t, rec, fl, limitIP, "a@example.com", "app1"); resp != milter.RespContinue {
t.Fatalf("first message = %v, want Continue (4/5 stored)", resp)
}
// Same window, nothing written yet: the first message holds the fifth slot.
if _, resp := mailFromIn(t, rec, fl, limitIP, "b@example.com", "app1"); resp != milter.RespTempFail {
t.Fatalf("concurrent message = %v, want TempFail (would overshoot)", resp)
}
if len(rec.rejected) != 1 {
t.Fatalf("want one rejected send-log row, got %+v", rec.rejected)
}
}
// Once the message is recorded the stored count sees it, so its reservation
// must be given back — otherwise it would be counted twice and the ceiling
// would drift closed.
func TestReservationReleasedAtEndOfMessage(t *testing.T) {
rec := limitedRecorder(4)
fl := &inflight{}
s, resp := mailFromIn(t, rec, fl, limitIP, "a@example.com", "app1")
if resp != milter.RespContinue {
t.Fatalf("first message = %v, want Continue", resp)
}
if _, err := s.Body(mods(map[string]string{"i": "Q1"})); err != nil {
t.Fatalf("Body: %v", err)
}
if n := fl.count(store.RateLimitScopeDomain+"|example.com", time.Now().Add(-time.Hour)); n != 0 {
t.Fatalf("in-flight count after EOM = %d, want 0", n)
}
}
// A transaction the client abandons (RSET, or a Postfix-side rejection) never
// reaches the send log, so its slot must not stay claimed.
func TestReservationReleasedOnAbort(t *testing.T) {
rec := limitedRecorder(4)
fl := &inflight{}
s, resp := mailFromIn(t, rec, fl, limitIP, "a@example.com", "app1")
if resp != milter.RespContinue {
t.Fatalf("first message = %v, want Continue", resp)
}
if err := s.Abort(mods(nil)); err != nil {
t.Fatalf("Abort: %v", err)
}
if _, resp := mailFromIn(t, rec, fl, limitIP, "b@example.com", "app1"); resp != milter.RespContinue {
t.Fatalf("after abort = %v, want Continue (slot released)", resp)
}
}
// A refused message must not leave the slots it claimed for the limits checked
// before the one that tripped, or every refusal would tighten the ceiling.
func TestRefusalReleasesEarlierReservation(t *testing.T) {
rec := &fakeRecorder{
limits: map[string]store.RateLimit{
store.RateLimitScopeDomain + "|example.com": activeLimit(limitIP),
store.RateLimitScopeApp + "|app1": activeLimit(limitIP),
},
counts: map[string]int64{
store.RateLimitScopeDomain + "|example.com": 0, // domain: plenty of room
store.RateLimitScopeApp + "|app1": 5, // app: at the ceiling
},
}
fl := &inflight{}
if _, resp := mailFromIn(t, rec, fl, limitIP, "a@example.com", "app1"); resp != milter.RespTempFail {
t.Fatalf("app over limit = %v, want TempFail", resp)
}
if n := fl.count(store.RateLimitScopeDomain+"|example.com", time.Now().Add(-time.Hour)); n != 0 {
t.Fatalf("domain reservation left behind after refusal: %d", n)
}
}
// The in-flight count only covers the limit's own window: a reservation older
// than it (a session stuck mid-DATA for longer than the window) must not be
// counted against a window it no longer belongs to.
func TestInflightIgnoresReservationsOutsideWindow(t *testing.T) {
fl := &inflight{}
r := fl.reserve("domain|example.com")
r.at = time.Now().Add(-time.Minute)
if n := fl.count("domain|example.com", time.Now().Add(-time.Hour)); n != 1 {
t.Fatalf("count inside window = %d, want 1", n)
}
if n := fl.count("domain|example.com", time.Now().Add(-time.Second)); n != 0 {
t.Fatalf("count outside window = %d, want 0", n)
}
// Past the TTL the reservation is dropped even for a wide window, so a
// client that vanished after MAIL FROM cannot hold a slot forever.
r.at = time.Now().Add(-2 * reservationTTL)
if n := fl.count("domain|example.com", time.Now().Add(-3*reservationTTL)); n != 0 {
t.Fatalf("expired reservation still counted: %d", n)
}
}
func TestDomainOf(t *testing.T) {
cases := map[string]string{
"user@Example.COM": "example.com",