logtail: the deferred counter remembers a place in the log, not a second (stage 2b)
test / test (push) Canceled after 0s
test / test (push) Canceled after 0s
The counter passed over, while catching up after a start, every warning no later than the last one counted - by time; mail.log stamps whole seconds, so a second warning of that second, later in the file, was lost. - The tailer gives each line its place (Place: the file's fingerprint and the offset just past the line): follow tracks the bytes the reader hands over, readRotated likewise in mail.log.1. Run hands tr.line to the counter. - store.AddSenderCheckDeferred(at, n, place) counts, moves the last-deferral mark (never back - an upsert WHERE) and saves the place (settings sender_check_counted_place) in one transaction. - Catching up, a warning in the file of the last one counted is passed over by place (end <= counted end); in another file - a rotated log read again - by time, as before. The fingerprint carries the inode, so the place stays true across a rotation. Tests: two warnings of one second, an unclean stop between them, all lines read again - each counted once; the store saves the place with the count and keeps the latest deferral time. Existing counter tests pass a zero place (time rule). CHANGELOG [Unreleased], audit plan line ticked, roadmap 24/62. Evidence: go vet ./... clean; go test ./internal/store, ./internal/logtail ok (but for the Windows-only test); the server run follows before the push. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -162,7 +162,8 @@ func Run(ctx context.Context, path string, st StatusStore, retention RetentionDa
|
||||
}
|
||||
|
||||
keeper := newLineKeeper(st)
|
||||
return follow(ctx, path, &tracker{st: st, path: path}, caughtUp, keeper.flush, func(line string) {
|
||||
tr := &tracker{st: st, path: path}
|
||||
return follow(ctx, path, tr, caughtUp, keeper.flush, func(line string) {
|
||||
// Until the first read reaches end-of-file the lines are those written
|
||||
// while the panel was down - and, after an unclean stop, some it had
|
||||
// read already.
|
||||
@@ -177,7 +178,7 @@ func Run(ctx context.Context, path string, st StatusStore, retention RetentionDa
|
||||
journal.handle(line, now, catchingUp)
|
||||
}
|
||||
if deferred != nil {
|
||||
deferred.handle(line, now, catchingUp)
|
||||
deferred.handle(line, now, catchingUp, tr.line)
|
||||
}
|
||||
keeper.add(line, now, catchingUp)
|
||||
queueID, recipient, status, ok := parseDelivery(line)
|
||||
@@ -569,6 +570,9 @@ func follow(ctx context.Context, path string, tr *tracker, caughtUp chan struct{
|
||||
r *bufio.Reader
|
||||
info os.FileInfo
|
||||
pending string
|
||||
// consumed is the offset just past what the reader has handed over,
|
||||
// pending included: where a complete line ends.
|
||||
consumed int64
|
||||
)
|
||||
openAt := func(offset int64, whence int) error {
|
||||
nf, err := os.Open(path)
|
||||
@@ -588,6 +592,7 @@ func follow(ctx context.Context, path string, tr *tracker, caughtUp chan struct{
|
||||
f.Close()
|
||||
}
|
||||
f, r, info, pending = nf, bufio.NewReader(nf), ni, ""
|
||||
consumed = offset
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -602,9 +607,12 @@ func follow(ctx context.Context, path string, tr *tracker, caughtUp chan struct{
|
||||
case <-time.After(pollInterval):
|
||||
}
|
||||
}
|
||||
if _, err := f.Seek(tr.resume(f), io.SeekStart); err != nil {
|
||||
start := tr.resume(f)
|
||||
if _, err := f.Seek(start, io.SeekStart); err != nil {
|
||||
log.Printf("log-tailer: seek %s: %v", path, err)
|
||||
start = 0
|
||||
}
|
||||
consumed = start
|
||||
r.Reset(f) // the reader buffered from the pre-seek position
|
||||
// Rotated while the panel was down: the end of the old file first.
|
||||
if tr.readRotated(handle) {
|
||||
@@ -620,6 +628,7 @@ func follow(ctx context.Context, path string, tr *tracker, caughtUp chan struct{
|
||||
defer flush()
|
||||
for {
|
||||
line, err := r.ReadString('\n')
|
||||
consumed += int64(len(line))
|
||||
if err == io.EOF {
|
||||
pending += line // hold the partial line until it completes
|
||||
return
|
||||
@@ -630,6 +639,7 @@ func follow(ctx context.Context, path string, tr *tracker, caughtUp chan struct{
|
||||
}
|
||||
full := pending + line
|
||||
pending = ""
|
||||
tr.line = Place{FP: tr.fp, End: consumed}
|
||||
handle(strings.TrimRight(full, "\r\n"))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user