inbound-antispam-panel: refused inbound mail from mail.log; retention

Step 4/12 of docs/plans/inbound-antispam-panel.md. The log tailer writes an inbound_spam_log row for mail port 25 refused (internal/logtail/inbound.go): a "reject:" line of the port-25 smtpd is Postfix's own policy; a milter-reject / milter-hold / milter-discard line of cleanup is the filter's verdict. Only a recipient at an inbound domain is a row, so "Relay access denied" for a stranger's domain is not. The row is dated by its log line; the reply text is cut to 255 bytes of printable ASCII; a reply starting with 4 is a tempfail. To tell port 25 from 465 / 587 the listener now logs as postfix/inbound (syslog_name). cleanup logs under one name for every listener, so its verdict counts only for a queue id the port-25 smtpd was seen to open (4096 remembered). Lines are matched from the head, after timestamp and host: their tail is the sender's own text. Retention: PruneInboundSpam every six hours - older than the inbound_spam_log_retention_days setting (seeded from INBOUND_SPAM_LOG_RETENTION_DAYS, 90, range 7-365), then down to 10 000 rows. The setting has no field in the panel yet. Known limits, in architecture.md: a log line names one recipient, so a message refused at end-of-message is one row; a line re-read after an unclean stop would be a second row. Checked on the test server, uncommitted tree: go vet ./... ok, go test ./... ok (22 packages), make e2e ok - the unknown recipient at the inbound domain refused on port 25 has its row (reject by postfix), the stranger's domain has none. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-11 22:42:25 +03:00
parent 94e94490af
commit 27fd355e53
18 changed files with 744 additions and 25 deletions
@@ -37,18 +37,25 @@ type InboundSpamEntry struct {
 	Decision      string
 	Engine        string
 	Detail        string
+	// At is when it happened; zero means now. The log tailer dates a refusal
+	// by its mail.log line, which it may read long after.
+	At time.Time
 }

 // InsertInboundSpam records one journal row. It is called from the inbound
 // journal milter's hot path and from the log tailer, so it returns any error
 // for the caller to log; neither of them may hold up mail over it.
 func (s *Store) InsertInboundSpam(e InboundSpamEntry) error {
+	at := e.At
+	if at.IsZero() {
+		at = time.Now()
+	}
 	_, err := s.db.Exec(
 		`INSERT INTO inbound_spam_log
 			(inbound_domain, client_ip, from_addr, to_addr, subject, decision, engine, detail, created_at)
 		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
 		e.InboundDomain, e.ClientIP, e.From, e.To, e.Subject, e.Decision, e.Engine, e.Detail,
-		time.Now().UTC().Format(time.RFC3339),
+		at.UTC().Format(time.RFC3339),
 	)
 	if err != nil {
 		return fmt.Errorf("insert inbound_spam_log: %w", err)
@@ -56,6 +63,24 @@ func (s *Store) InsertInboundSpam(e InboundSpamEntry) error {
 	return nil
 }

+// PruneInboundSpam deletes journal rows older than cutoff and then, if more
+// than keep rows are left, the oldest of those. It returns how many went.
+func (s *Store) PruneInboundSpam(cutoff time.Time, keep int) (int64, error) {
+	res, err := s.db.Exec("DELETE FROM inbound_spam_log WHERE created_at < ?", cutoff.UTC().Format(time.RFC3339))
+	if err != nil {
+		return 0, fmt.Errorf("prune inbound_spam_log by age: %w", err)
+	}
+	n, _ := res.RowsAffected()
+	res, err = s.db.Exec(
+		`DELETE FROM inbound_spam_log WHERE id <=
+			(SELECT id FROM inbound_spam_log ORDER BY id DESC LIMIT 1 OFFSET ?)`, keep)
+	if err != nil {
+		return n, fmt.Errorf("prune inbound_spam_log by count: %w", err)
+	}
+	m, _ := res.RowsAffected()
+	return n + m, nil
+}
+
 // IsInboundDomain reports whether name is a configured inbound domain. Port 25
 // also takes mail that is not the relay's - DMARC reports for the server's own
 // name - and the journal keeps rows for inbound domains only.
@@ -59,3 +59,102 @@ func TestIsInboundDomain(t *testing.T) {
 		}
 	}
 }
+
+func TestInsertInboundSpamKeepsItsOwnTime(t *testing.T) {
+	st := openTestStore(t)
+	at := time.Date(2026, 10, 1, 9, 30, 0, 0, time.FixedZone("msk", 3*3600))
+	err := st.InsertInboundSpam(InboundSpamEntry{
+		InboundDomain: "in.example.com", ClientIP: "203.0.113.7", To: "a@in.example.com",
+		Decision: SpamDecisionReject, Engine: SpamEnginePostfix, At: at,
+	})
+	if err != nil {
+		t.Fatalf("InsertInboundSpam: %v", err)
+	}
+	var created string
+	if err := st.db.QueryRow("SELECT created_at FROM inbound_spam_log").Scan(&created); err != nil || created != "2026-10-01T06:30:00Z" {
+		t.Errorf("created_at = %q, %v", created, err)
+	}
+}
+
+func TestPruneInboundSpam(t *testing.T) {
+	st := openTestStore(t)
+	now := time.Now().UTC()
+	add := func(to string, age time.Duration) {
+		t.Helper()
+		err := st.InsertInboundSpam(InboundSpamEntry{
+			InboundDomain: "in.example.com", ClientIP: "203.0.113.7", To: to,
+			Decision: SpamDecisionReject, Engine: SpamEnginePostfix, At: now.Add(-age),
+		})
+		if err != nil {
+			t.Fatalf("InsertInboundSpam: %v", err)
+		}
+	}
+	left := func() string {
+		t.Helper()
+		rows, err := st.db.Query("SELECT to_addr FROM inbound_spam_log ORDER BY id")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer rows.Close()
+		var out string
+		for rows.Next() {
+			var to string
+			if err := rows.Scan(&to); err != nil {
+				t.Fatal(err)
+			}
+			out += to + " "
+		}
+		return out
+	}
+
+	add("old", 100*24*time.Hour)
+	add("a", 3*time.Hour)
+	add("b", 2*time.Hour)
+	add("c", time.Hour)
+
+	// Age first: nothing else goes while the rest fits.
+	if n, err := st.PruneInboundSpam(now.AddDate(0, 0, -90), 10); err != nil || n != 1 {
+		t.Fatalf("prune by age = %d, %v", n, err)
+	}
+	if got := left(); got != "a b c " {
+		t.Fatalf("left = %q", got)
+	}
+	// Then the cap, oldest first.
+	if n, err := st.PruneInboundSpam(now.AddDate(0, 0, -90), 2); err != nil || n != 1 {
+		t.Fatalf("prune by count = %d, %v", n, err)
+	}
+	if got := left(); got != "b c " {
+		t.Fatalf("left = %q", got)
+	}
+	if n, err := st.PruneInboundSpam(now.AddDate(0, 0, -90), 2); err != nil || n != 0 {
+		t.Errorf("a second prune removed %d, %v", n, err)
+	}
+}
+
+func TestInboundSpamLogRetentionDays(t *testing.T) {
+	st := openTestStore(t)
+	if days, err := st.GetInboundSpamLogRetentionDays(30); err != nil || days != 30 {
+		t.Errorf("unset, env 30: %d, %v", days, err)
+	}
+	if days, err := st.GetInboundSpamLogRetentionDays(3); err != nil || days != InboundSpamLogRetentionDaysDefault {
+		t.Errorf("unset, env out of range: %d, %v", days, err)
+	}
+	if err := st.EnsureInboundSpamLogRetentionDays(30); err != nil {
+		t.Fatalf("Ensure: %v", err)
+	}
+	if err := st.EnsureInboundSpamLogRetentionDays(60); err != nil {
+		t.Fatalf("Ensure again: %v", err)
+	}
+	if days, _ := st.GetInboundSpamLogRetentionDays(90); days != 30 {
+		t.Errorf("after Ensure(30), Ensure(60): %d, want the first seed kept", days)
+	}
+	if err := st.SetInboundSpamLogRetentionDays(6); err == nil {
+		t.Error("6 days was accepted")
+	}
+	if err := st.SetInboundSpamLogRetentionDays(365); err != nil {
+		t.Errorf("365 days: %v", err)
+	}
+	if days, _ := st.GetInboundSpamLogRetentionDays(90); days != 365 {
+		t.Errorf("after Set(365): %d", days)
+	}
+}
@@ -77,6 +77,72 @@ func (s *Store) EnsureSendLogRetentionDays(envDefault int) error {
 	return s.SetSendLogRetentionDays(sendLogRetentionFallback(envDefault))
 }

+const (
+	// InboundSpamLogRetentionDaysKey is the settings table key for the
+	// inbound antispam journal's retention. Same range as the send log's.
+	InboundSpamLogRetentionDaysKey = "inbound_spam_log_retention_days"
+	InboundSpamLogRetentionDaysMin = 7
+	InboundSpamLogRetentionDaysMax = 365
+	// InboundSpamLogRetentionDaysDefault matches
+	// INBOUND_SPAM_LOG_RETENTION_DAYS when unset.
+	InboundSpamLogRetentionDaysDefault = 90
+)
+
+// ErrInboundSpamLogRetentionDaysOutOfRange is returned when the inbound
+// journal's retention is outside the allowed panel range.
+var ErrInboundSpamLogRetentionDaysOutOfRange = errors.New("inbound journal retention days out of range")
+
+// ValidateInboundSpamLogRetentionDays checks the panel-allowed window.
+func ValidateInboundSpamLogRetentionDays(days int) error {
+	if days < InboundSpamLogRetentionDaysMin || days > InboundSpamLogRetentionDaysMax {
+		return fmt.Errorf("%w: must be between %d and %d days", ErrInboundSpamLogRetentionDaysOutOfRange, InboundSpamLogRetentionDaysMin, InboundSpamLogRetentionDaysMax)
+	}
+	return nil
+}
+
+func inboundSpamLogRetentionFallback(envDefault int) int {
+	if ValidateInboundSpamLogRetentionDays(envDefault) == nil {
+		return envDefault
+	}
+	return InboundSpamLogRetentionDaysDefault
+}
+
+// GetInboundSpamLogRetentionDays returns the effective retention window of
+// the inbound journal. When the setting is missing or invalid, envDefault is
+// used (INBOUND_SPAM_LOG_RETENTION_DAYS).
+func (s *Store) GetInboundSpamLogRetentionDays(envDefault int) (int, error) {
+	raw, err := s.GetSetting(InboundSpamLogRetentionDaysKey)
+	if err != nil {
+		return 0, err
+	}
+	days, err := strconv.Atoi(raw)
+	if err != nil || ValidateInboundSpamLogRetentionDays(days) != nil {
+		return inboundSpamLogRetentionFallback(envDefault), nil
+	}
+	return days, nil
+}
+
+// SetInboundSpamLogRetentionDays persists the inbound journal's retention.
+func (s *Store) SetInboundSpamLogRetentionDays(days int) error {
+	if err := ValidateInboundSpamLogRetentionDays(days); err != nil {
+		return err
+	}
+	return s.SetSetting(InboundSpamLogRetentionDaysKey, strconv.Itoa(days))
+}
+
+// EnsureInboundSpamLogRetentionDays seeds the setting from envDefault when it
+// has never been written.
+func (s *Store) EnsureInboundSpamLogRetentionDays(envDefault int) error {
+	raw, err := s.GetSetting(InboundSpamLogRetentionDaysKey)
+	if err != nil {
+		return err
+	}
+	if raw != "" {
+		return nil
+	}
+	return s.SetInboundSpamLogRetentionDays(inboundSpamLogRetentionFallback(envDefault))
+}
+
 // GetSetting returns a settings value or empty string when missing.
 func (s *Store) GetSetting(key string) (string, error) {
 	var value string