mix/selfpost
1
0
Fork 0
Code Issues Pull Requests Actions 21 Packages Projects Releases Wiki Activity

delivery-log-storage: findings of the technical and security reviews
test / test (push) Canceled after 0s
Details

Browse Source 
Neither review (read-only agents, Opus and Fable, over ae06d8b..815236a) has
a Critical or High finding. What they found and what was done:

- A stranger on port 25 could grow the table: every queue-id line of inbound
  mail was stored and pruned an hour or more later, his own message-id among
  them. The tailer now leaves out the lines of a queue id that
  postfix/inbound/smtpd opened, until qmgr removes the message or another
  listener opens one under the id; 4096 such ids are remembered.
- A commit, so a disk sync, per stored line on the connection the journal
  milter shares, and a backlog written line by line. Lines are now written in
  one transaction per pass over the log, 500 at most, before the read offset
  is saved (InsertSendLogLines; follow() calls the keeper's flush).
- The prune asked the send log once per line and ran hourly. It asks once per
  queue id and runs at start and with the six-hourly sweep; with port-25 mail
  left out there is little without a row to wait for.
- The 4096-byte cut could split a character; it keeps characters whole.
- security.md did not say that a backup now carries these lines, and the plan
  said the time window excludes a reused queue id where it only limits the
  chance to the row's queue lifetime. Both are corrected; the collision is
  recorded as an accepted risk with its trigger.
- Tests that did not prove their comments: the prune is now tested with the
  sweep, a rejected row next to the prune, batching and the change from
  catching up to following, port-25 mail left out and its id reused.

Left as found and written down in the plan: lines the tailer never reads
(a file rotated away while the panel was down), and a message in the queue
across the upgrade, which shows only its later lines.

On the test server, for this tree: go vet clean, go test ./... 22 packages
ok, make e2e ok.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
mix
2026-10-12 10:41:04 +03:00
co-authored by Claude Opus 5.5
parent 815236a607
commit a990e7cc23
14 changed files with 456 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/logtail/lines_test.go
+102 -12
View File
@@ -20,7 +20,7 @@ func TestQueueLineHead(t *testing.T) {
{"port-25 smtpd", "mail postfix/inbound/smtpd[41]: 4A1B2C3D: client=mx.example.net[203.0.113.9]", "4A1B2C3D"},
{"cleanup", "mail postfix/cleanup[52]: 41E862C00D9E: message-id=<1@example.com>", "41E862C00D9E"},
{"qmgr removes it", "mail postfix/qmgr[10]: 41E862C00D9E: removed", "41E862C00D9E"},
{"another program", "mail opendkim[12]: 41E862C00D9E: DKIM-Signature field added (s=sp, d=example.com)", "41E862C00D9E"},
{"a bounce is sent: the id at the head, not the new message's", "mail postfix/bounce[30]: 41E862C00D9E: sender non-delivery notification: 5F00AA12", "41E862C00D9E"},
{"long queue id", "mail postfix/qmgr[10]: 3sJ8Xh2Cgqz4Xy1: removed", "3sJ8Xh2Cgqz4Xy1"},
{"refused before queueing", "mail postfix/inbound/smtpd[41]: NOQUEUE: reject: RCPT from mx.example.net[203.0.113.9]: 550 5.1.1 <n@in.example.com>: Recipient address rejected; from=<a@example.net> to=<n@in.example.com> proto=ESMTP helo=<mx.example.net>", ""},
{"a warning", "mail postfix/smtpd[10]: warning: hostname mx.example.net does not resolve to address 203.0.113.9", ""},
@@ -38,7 +38,7 @@ func TestQueueLineHead(t *testing.T) {
for _, c := range cases {
got := ""
if m := queueLineRe.FindStringSubmatch(c.rest); m != nil {
got = m[1]
got = m[2]
}
if got != c.want {
t.Errorf("%s: queue id %q, want %q", c.name, got, c.want)
@@ -47,14 +47,19 @@ func TestQueueLineHead(t *testing.T) {
}
// A line is dated by its own stamp, not by when the tailer got to it, and is
// kept as the log has it.
func TestStoreLineDatesByTheLine(t *testing.T) {
// kept as the log has it. Nothing is written until the pass over the log ends.
func TestLineKeeperDatesByTheLine(t *testing.T) {
cs := &captureStore{}
k := newLineKeeper(cs)
now := time.Date(2026, 10, 12, 18, 0, 0, 0, time.UTC)
line := "Oct 12 09:00:00 mail postfix/qmgr[7]: 41E862C00D9E: removed"
storeLine(cs, line, now, false)
storeLine(cs, "Oct 12 09:00:01 mail postfix/smtpd[7]: connect from unknown[203.0.113.7]", now, false)
k.add(line, now, false)
k.add("Oct 12 09:00:01 mail postfix/smtpd[7]: connect from unknown[203.0.113.7]", now, false)
if len(cs.lines) != 0 {
t.Fatalf("written before the pass ended: %v", cs.kept())
}
k.flush()
if len(cs.lines) != 1 {
t.Fatalf("kept %d lines, want 1: %v", len(cs.lines), cs.kept())
@@ -68,6 +73,91 @@ func TestStoreLineDatesByTheLine(t *testing.T) {
}
}
// A backlog is written in batches of lineBatch, not held whole, and a pass
// that changes from catching up to following does not write the later lines
// under the earlier rule.
func TestLineKeeperWritesInBatches(t *testing.T) {
cs := &captureStore{}
k := newLineKeeper(cs)
now := time.Date(2026, 10, 12, 18, 0, 0, 0, time.UTC)
line := func(i int) string {
return "host postfix/qmgr[7]: 41E862C00D9E: from=<a@example.com>, size=" + itoa(i) + ", nrcpt=1 (queue active)"
}
for i := 0; i < lineBatch+10; i++ {
k.add(line(i), now, true)
}
if len(cs.lines) != lineBatch || cs.batches != 1 {
t.Fatalf("after %d lines: %d written in %d batches, want %d in 1", lineBatch+10, len(cs.lines), cs.batches, lineBatch)
}
// The same line again, now as one read while following: it is a new line,
// and the ten still held from the catch-up are written first, under theirs.
k.add(line(0), now, false)
k.flush()
if got, want := len(cs.lines), lineBatch+11; got != want {
t.Errorf("%d lines written, want %d", got, want)
}
if cs.batches != 3 {
t.Errorf("%d batches, want 3", cs.batches)
}
}
// Mail that comes in on port 25 is a stranger's and never gets a send-log row:
// none of its lines are kept, from the smtpd that opened the queue id to
// qmgr's "removed". The id is then anybody's again.
func TestLineKeeperLeavesOutPort25Mail(t *testing.T) {
cs := &captureStore{}
k := newLineKeeper(cs)
now := time.Date(2026, 10, 12, 18, 0, 0, 0, time.UTC)
const (
outClient = "host postfix/submission/smtpd[10]: 41E862C00D9E: client=app.example.com[203.0.113.7], sasl_method=PLAIN, sasl_username=app"
outSent = "host postfix/smtp[26]: 41E862C00D9E: to=<a@example.net>, relay=mx.example.net[203.0.113.9]:25, dsn=2.0.0, status=sent (250 OK)"
reused = "host postfix/submission/smtpd[10]: 4A1B2C3D: client=app.example.com[203.0.113.7], sasl_method=PLAIN, sasl_username=app"
bounce = "host postfix/cleanup[52]: 5F00AA12: message-id=<20261012180000.5F00AA12@mail.example.com>"
)
for _, line := range []string{
"host postfix/inbound/smtpd[41]: 4A1B2C3D: client=mx.example.net[198.51.100.9]",
outClient,
"host postfix/cleanup[52]: 4A1B2C3D: message-id=<whatever-the-stranger-wrote@example.net>",
"host postfix/qmgr[10]: 4A1B2C3D: from=<a@example.net>, size=900, nrcpt=1 (queue active)",
outSent,
"host postfix/smtp[27]: 4A1B2C3D: to=<b@in.example.com>, relay=upstream.example.com[192.0.2.1]:25, dsn=2.0.0, status=sent (250 OK)",
"host postfix/qmgr[10]: 4A1B2C3D: removed",
// The id comes round again for a submitted message, and a bounce has no
// smtpd at all: both are kept.
reused,
bounce,
} {
k.add(line, now, false)
}
k.flush()
want := []string{"41E862C00D9E|" + outClient, "41E862C00D9E|" + outSent, "4A1B2C3D|" + reused, "5F00AA12|" + bounce}
if got := cs.kept(); strings.Join(got, "\n") != strings.Join(want, "\n") {
t.Errorf("kept:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
}
// A port-25 id that a submission listener opens before qmgr's "removed" was
// seen (the line was lost, the panel restarted) is the submitted message's.
k.add("host postfix/inbound/smtpd[41]: 6C00BB34: client=mx.example.net[198.51.100.9]", now, false)
again := "host postfix/smtps/smtpd[11]: 6C00BB34: client=app.example.com[203.0.113.7], sasl_method=PLAIN, sasl_username=app"
k.add(again, now, false)
k.flush()
if got := cs.kept(); got[len(got)-1] != "6C00BB34|"+again {
t.Errorf("a reused port-25 id was not kept for the submitted message: %v", got[len(got)-1])
}
// What is remembered of port 25 has a ceiling.
for i := 0; i < port25IDs+50; i++ {
k.rememberPort25("7D" + itoa(100000+i))
}
if len(k.port25) > port25IDs {
t.Errorf("%d port-25 ids remembered, ceiling is %d", len(k.port25), port25IDs)
}
}
// The tailer keeps queue-id lines as it follows the log, and after an unclean
// stop - its saved place lies before lines it had already read - it does not
// keep them a second time.
@@ -115,13 +205,13 @@ func TestRunKeepsQueueLinesOnce(t *testing.T) {
}
}
// Lines without a send-log row are removed at the start, with every send-log
// sweep and on their own shorter cadence, and only once they are old enough
// for the row to have been written.
// Lines without a send-log row are removed at the start and after every
// send-log sweep, and only once they are old enough for the row to have been
// written.
func TestRetentionLoopPrunesLines(t *testing.T) {
old := linePruneInterval
linePruneInterval = 20 * time.Millisecond
t.Cleanup(func() { linePruneInterval = old })
old := retentionInterval
retentionInterval = 20 * time.Millisecond
t.Cleanup(func() { retentionInterval = old })
cs := &captureStore{}
ctx, cancel := context.WithCancel(context.Background())