feat: log-tailer offset persistence + in-flight L2 rate-limit accounting (code-review.md § Phase 3)
- logtail: persist the read position (offset + fingerprint of the log's first 512 bytes) in a new logtail_state table (migration 0003) and resume from it on start, so delivery lines written while the panel was down are parsed instead of skipped and their send-log rows no longer stay "queued" forever. Fingerprint mismatch (rotated/recreated while down) reads the file from the start — re-parsing is idempotent; a first-ever start with nothing stored still begins at end-of-file. Writes are throttled to one per 5s, forced on rotation and shutdown. - milter: count messages that passed the level-2 check but have not reached the send log yet (internal/milter/inflight.go), so concurrent SMTP sessions cannot each spend the same last slot. A literal count+insert transaction, as the review suggested, is not possible: the count happens at MAIL FROM and the insert at end-of-message. Reservations are released after the insert, on ABORT, and after a 10-minute TTL — a client that drops mid-transaction must not be able to hold a slot, since the limiter is fail-open by design. Docs: architecture.md (log tailer, persistence, L2 counting), security.md and roadmap.md (restart gap closed, container recreate remains), CHANGELOG, progress.md, code-review.md. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -331,6 +331,130 @@ func TestRateLimitNoIPKeyDoesNotApply(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// mailFromIn is mailFrom with an explicit shared in-flight registry, so a test
|
||||
// can play several concurrent SMTP sessions of one process against each other.
|
||||
func mailFromIn(t *testing.T, rec Store, fl *inflight, ip, from, login string) (*session, milter.Response) {
|
||||
t.Helper()
|
||||
s := &session{rec: rec, flight: fl}
|
||||
if _, err := s.Connect("h", "tcp4", 0, net.ParseIP(ip), mods(nil)); err != nil {
|
||||
t.Fatalf("Connect: %v", err)
|
||||
}
|
||||
resp, err := s.MailFrom(from, mods(map[string]string{"auth_authen": login}))
|
||||
if err != nil {
|
||||
t.Fatalf("MailFrom: %v", err)
|
||||
}
|
||||
return s, resp
|
||||
}
|
||||
|
||||
func limitedRecorder(count int64) *fakeRecorder {
|
||||
return &fakeRecorder{
|
||||
limits: map[string]store.RateLimit{
|
||||
store.RateLimitScopeDomain + "|example.com": activeLimit(limitIP),
|
||||
},
|
||||
counts: map[string]int64{store.RateLimitScopeDomain + "|example.com": count},
|
||||
}
|
||||
}
|
||||
|
||||
// Messages between MAIL FROM and end-of-message are not in the send log yet, so
|
||||
// counting the stored rows alone lets concurrent sessions each pass the same
|
||||
// check and overshoot the ceiling. The last free slot may only be taken once.
|
||||
func TestRateLimitCountsInFlightMessages(t *testing.T) {
|
||||
rec := limitedRecorder(4) // one below the ceiling of 5
|
||||
fl := &inflight{}
|
||||
|
||||
if _, resp := mailFromIn(t, rec, fl, limitIP, "a@example.com", "app1"); resp != milter.RespContinue {
|
||||
t.Fatalf("first message = %v, want Continue (4/5 stored)", resp)
|
||||
}
|
||||
// Same window, nothing written yet: the first message holds the fifth slot.
|
||||
if _, resp := mailFromIn(t, rec, fl, limitIP, "b@example.com", "app1"); resp != milter.RespTempFail {
|
||||
t.Fatalf("concurrent message = %v, want TempFail (would overshoot)", resp)
|
||||
}
|
||||
if len(rec.rejected) != 1 {
|
||||
t.Fatalf("want one rejected send-log row, got %+v", rec.rejected)
|
||||
}
|
||||
}
|
||||
|
||||
// Once the message is recorded the stored count sees it, so its reservation
|
||||
// must be given back — otherwise it would be counted twice and the ceiling
|
||||
// would drift closed.
|
||||
func TestReservationReleasedAtEndOfMessage(t *testing.T) {
|
||||
rec := limitedRecorder(4)
|
||||
fl := &inflight{}
|
||||
|
||||
s, resp := mailFromIn(t, rec, fl, limitIP, "a@example.com", "app1")
|
||||
if resp != milter.RespContinue {
|
||||
t.Fatalf("first message = %v, want Continue", resp)
|
||||
}
|
||||
if _, err := s.Body(mods(map[string]string{"i": "Q1"})); err != nil {
|
||||
t.Fatalf("Body: %v", err)
|
||||
}
|
||||
if n := fl.count(store.RateLimitScopeDomain+"|example.com", time.Now().Add(-time.Hour)); n != 0 {
|
||||
t.Fatalf("in-flight count after EOM = %d, want 0", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A transaction the client abandons (RSET, or a Postfix-side rejection) never
|
||||
// reaches the send log, so its slot must not stay claimed.
|
||||
func TestReservationReleasedOnAbort(t *testing.T) {
|
||||
rec := limitedRecorder(4)
|
||||
fl := &inflight{}
|
||||
|
||||
s, resp := mailFromIn(t, rec, fl, limitIP, "a@example.com", "app1")
|
||||
if resp != milter.RespContinue {
|
||||
t.Fatalf("first message = %v, want Continue", resp)
|
||||
}
|
||||
if err := s.Abort(mods(nil)); err != nil {
|
||||
t.Fatalf("Abort: %v", err)
|
||||
}
|
||||
if _, resp := mailFromIn(t, rec, fl, limitIP, "b@example.com", "app1"); resp != milter.RespContinue {
|
||||
t.Fatalf("after abort = %v, want Continue (slot released)", resp)
|
||||
}
|
||||
}
|
||||
|
||||
// A refused message must not leave the slots it claimed for the limits checked
|
||||
// before the one that tripped, or every refusal would tighten the ceiling.
|
||||
func TestRefusalReleasesEarlierReservation(t *testing.T) {
|
||||
rec := &fakeRecorder{
|
||||
limits: map[string]store.RateLimit{
|
||||
store.RateLimitScopeDomain + "|example.com": activeLimit(limitIP),
|
||||
store.RateLimitScopeApp + "|app1": activeLimit(limitIP),
|
||||
},
|
||||
counts: map[string]int64{
|
||||
store.RateLimitScopeDomain + "|example.com": 0, // domain: plenty of room
|
||||
store.RateLimitScopeApp + "|app1": 5, // app: at the ceiling
|
||||
},
|
||||
}
|
||||
fl := &inflight{}
|
||||
if _, resp := mailFromIn(t, rec, fl, limitIP, "a@example.com", "app1"); resp != milter.RespTempFail {
|
||||
t.Fatalf("app over limit = %v, want TempFail", resp)
|
||||
}
|
||||
if n := fl.count(store.RateLimitScopeDomain+"|example.com", time.Now().Add(-time.Hour)); n != 0 {
|
||||
t.Fatalf("domain reservation left behind after refusal: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// The in-flight count only covers the limit's own window: a reservation older
|
||||
// than it (a session stuck mid-DATA for longer than the window) must not be
|
||||
// counted against a window it no longer belongs to.
|
||||
func TestInflightIgnoresReservationsOutsideWindow(t *testing.T) {
|
||||
fl := &inflight{}
|
||||
r := fl.reserve("domain|example.com")
|
||||
r.at = time.Now().Add(-time.Minute)
|
||||
|
||||
if n := fl.count("domain|example.com", time.Now().Add(-time.Hour)); n != 1 {
|
||||
t.Fatalf("count inside window = %d, want 1", n)
|
||||
}
|
||||
if n := fl.count("domain|example.com", time.Now().Add(-time.Second)); n != 0 {
|
||||
t.Fatalf("count outside window = %d, want 0", n)
|
||||
}
|
||||
// Past the TTL the reservation is dropped even for a wide window, so a
|
||||
// client that vanished after MAIL FROM cannot hold a slot forever.
|
||||
r.at = time.Now().Add(-2 * reservationTTL)
|
||||
if n := fl.count("domain|example.com", time.Now().Add(-3*reservationTTL)); n != 0 {
|
||||
t.Fatalf("expired reservation still counted: %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDomainOf(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"user@Example.COM": "example.com",
|
||||
|
||||
Reference in New Issue
Block a user