logtail: the deferred counter remembers a place in the log, not a second (stage 2b)
test / test (push) Canceled after 0s

The counter passed over, while catching up after a start, every warning no
later than the last one counted - by time; mail.log stamps whole seconds,
so a second warning of that second, later in the file, was lost.

- The tailer gives each line its place (Place: the file's fingerprint and
  the offset just past the line): follow tracks the bytes the reader hands
  over, readRotated likewise in mail.log.1. Run hands tr.line to the
  counter.
- store.AddSenderCheckDeferred(at, n, place) counts, moves the
  last-deferral mark (never back - an upsert WHERE) and saves the place
  (settings sender_check_counted_place) in one transaction.
- Catching up, a warning in the file of the last one counted is passed over
  by place (end <= counted end); in another file - a rotated log read again
  - by time, as before. The fingerprint carries the inode, so the place
  stays true across a rotation.

Tests: two warnings of one second, an unclean stop between them, all lines
read again - each counted once; the store saves the place with the count
and keeps the latest deferral time. Existing counter tests pass a zero
place (time rule).

CHANGELOG [Unreleased], audit plan line ticked, roadmap 24/62.
Evidence: go vet ./... clean; go test ./internal/store, ./internal/logtail ok
(but for the Windows-only test); the server run follows before the push.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
mix
2026-10-14 16:28:13 +03:00
co-authored by Claude Opus 5.5
parent 4678015bbd
commit c452606175
9 changed files with 204 additions and 45 deletions
+50 -6
View File
@@ -6,13 +6,17 @@ import (
)
type deferredStore struct {
days map[string]int64
last time.Time
days map[string]int64
last time.Time
place string
}
func (d *deferredStore) SenderCheckDeferredAt() (time.Time, error) { return d.last, nil }
func (d *deferredStore) AddSenderCheckDeferred(at time.Time, n int64) error {
func (d *deferredStore) SenderCheckCountedPlace() (string, error) { return d.place, nil }
func (d *deferredStore) AddSenderCheckDeferred(at time.Time, n int64, place string) error {
d.place = place
if d.days == nil {
d.days = make(map[string]int64)
}
@@ -77,7 +81,7 @@ func TestDeferredCounter(t *testing.T) {
// cleanup that writes it.
`2026-10-13T00:01:00.000000+00:00 mail postfix/cleanup[1195]: B6B484831946: milter-reject: END-OF-MESSAGE from unknown[10.77.0.1]: 5.7.1 From: director@other.e2e.test - not an address this application sends as; from=<alerts@sender.e2e.test> to=<rcpt@sink.e2e.test> proto=ESMTP helo=<x>`,
} {
c.handle(line, now, false)
c.handle(line, now, false, Place{})
}
if len(st.days) != 2 || st.days["2026-10-12"] != 1 || st.days["2026-10-13"] != 2 {
t.Fatalf("deferred per day = %v, want one on the first day and two on the second", st.days)
@@ -100,7 +104,7 @@ func TestDeferredCounterDoesNotCountALogReadAgain(t *testing.T) {
c := newDeferredCounter(&SenderCheck{Store: st, Socket: "/run/selfpost/sender.sock"})
for _, line := range first {
c.handle(line, now, false)
c.handle(line, now, false, Place{})
}
if st.days["2026-10-12"] != 2 {
t.Fatalf("first run counted %d, want 2", st.days["2026-10-12"])
@@ -109,9 +113,49 @@ func TestDeferredCounterDoesNotCountALogReadAgain(t *testing.T) {
// A new run: the same lines again, then what the outage added.
c = newDeferredCounter(&SenderCheck{Store: st, Socket: "/run/selfpost/sender.sock"})
for _, line := range append(first, outage("13", "2026-10-12T16:00:09.00000")...) {
c.handle(line, now, true)
c.handle(line, now, true, Place{})
}
if st.days["2026-10-12"] != 3 {
t.Fatalf("after the restart the day has %d, want 3", st.days["2026-10-12"])
}
}
// Two warnings of the same second, the panel stopped uncleanly between them:
// the next start reads both again, and the place of the first - not its
// second - tells them apart. Each is counted once (audit 2026-10-12, stage
// 2b; a time alone passed the second over).
func TestDeferredCounterRemembersAPlaceNotASecond(t *testing.T) {
st := &deferredStore{}
now := time.Date(2026, 10, 12, 18, 0, 0, 0, time.UTC)
lines := []string{
`2026-10-12T16:00:00+00:00 mail postfix/smtpd[11]: connect from unknown[10.77.0.1]`,
`2026-10-12T16:00:01+00:00 mail postfix/smtpd[11]: warning: connect to Milter service unix:/run/selfpost/sender.sock: Connection refused`,
`2026-10-12T16:00:01+00:00 mail postfix/smtpd[12]: connect from unknown[10.77.0.1]`,
`2026-10-12T16:00:01+00:00 mail postfix/smtpd[12]: warning: connect to Milter service unix:/run/selfpost/sender.sock: Connection refused`,
}
// The place each line ends at, in one file.
places := make([]Place, len(lines))
end := int64(0)
for i, l := range lines {
end += int64(len(l)) + 1
places[i] = Place{FP: "abc:1.2", End: end}
}
c := newDeferredCounter(&SenderCheck{Store: st, Socket: "/run/selfpost/sender.sock"})
for i := 0; i < 2; i++ { // stopped after the first warning
c.handle(lines[i], now, false, places[i])
}
if st.days["2026-10-12"] != 1 {
t.Fatalf("before the stop: %d, want 1", st.days["2026-10-12"])
}
// Started again: the saved offset was from before the stop, so all four
// lines are read again.
c = newDeferredCounter(&SenderCheck{Store: st, Socket: "/run/selfpost/sender.sock"})
for i, l := range lines {
c.handle(l, now, true, places[i])
}
if st.days["2026-10-12"] != 2 {
t.Fatalf("after the restart the day has %d, want 2", st.days["2026-10-12"])
}
}