inbound-antispam-panel: Filter lists, kind column, matches, note on a sheet
Follows the design commit before it (owner's words quoted there). The lists' tables are Value, Kind, Matched and Remove. The kind tag opens the entry's sheet: note, day added, how often it decided a message in the days the log keeps, and the last match. Matches are read off the journal on each view, nothing is stored (inbound.Lists.Usage, store.ListSpamListHits). A row counts for an entry when the list decided the message - a rejected row whose reply is the deny rule's message, an accepted row whose verdict carries the allow rule's symbol for that kind - and the entry fits the row's sender or client address the way rspamd matches it: an address whole, a domain as the sender's domain, a network containing the client (an IPv4-mapped address as IPv4). A failure to count shows the lists with "never" and logs why. For that the journal milter keeps the lists' own symbols ahead of its six-symbol cap: rspamd writes symbols heaviest first, so an allow entry's -100 came last and was cut behind any six others. The plan gains "Filter list usage": what counting from the journal implies, and - the owner asked for it in writing - the conditions under which a counter kept with each entry becomes worth a migration (the journal cut by its row cap and not by age; a short retention; counts that must outlive the journal; a slow page). multimap.conf says which of its texts SelfPost reads back. Tests: Usage over every kind on both lists and the rows that are nobody's; the milter's symbol order; the store query; the page and its sheets; a handler test from journal rows to the numbers on the page. Seen: the fixture page with a sheet open, headless Edge at 1360 px - the same as the accepted drawing. Checked on the test server, uncommitted tree: go vet ./... ok, go test ./... ok (22 packages), make e2e ok. Not seen in a real run: a count above zero. It needs an entry added in the panel, which I cannot sign in to, and then mail that the list decides; in particular the deny rule's message arriving in mail.log as the reply of a milter-reject line is what the trial with rspamc showed, not yet what Postfix logged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -21,6 +21,7 @@ type ListStore interface {
|
||||
AddSpamListEntry(list, kind, value, note string) (store.SpamListEntry, error)
|
||||
ListSpamListEntries() ([]store.SpamListEntry, error)
|
||||
DeleteSpamListEntry(id int64) error
|
||||
ListSpamListHits(marks ...string) ([]store.SpamListHit, error)
|
||||
}
|
||||
|
||||
// Lists owns the allow and deny lists of the external spam filter: the rows in
|
||||
@@ -236,3 +237,115 @@ func normalizeNote(raw string) (string, error) {
|
||||
}
|
||||
return note, nil
|
||||
}
|
||||
|
||||
// What a list leaves in the journal when it decides a message - the two ends
|
||||
// of deploy/antispam/rspamd/multimap.conf this package depends on. A deny rule
|
||||
// refuses with a message, which the log tailer files as the reply of a
|
||||
// rejected row. An allow rule adds a symbol named after the kind of entry,
|
||||
// which the journal milter keeps in the verdict of an accepted row.
|
||||
const (
|
||||
denyReplyMark = "this server's deny list"
|
||||
allowMark = "SELFPOST_ALLOW_"
|
||||
)
|
||||
|
||||
var allowSymbol = map[string]string{
|
||||
store.SpamEntryIP: "SELFPOST_ALLOW_IP(",
|
||||
store.SpamEntryAddress: "SELFPOST_ALLOW_FROM(",
|
||||
store.SpamEntryDomain: "SELFPOST_ALLOW_DOMAIN(",
|
||||
}
|
||||
|
||||
// Usage is how often an entry decided a message, as far back as the journal
|
||||
// reaches, and when it last did. The zero value is an entry that never did.
|
||||
type Usage struct {
|
||||
Hits int
|
||||
Last time.Time
|
||||
}
|
||||
|
||||
// Usage counts, for every entry, the journal rows it decided. Nothing is
|
||||
// stored for this: the journal already says which list decided a message and
|
||||
// who the message was from, and an entry is matched against that the way the
|
||||
// filter matches it - an address whole, a domain as the sender's domain, an
|
||||
// address or network against the client's. So the count reaches back as far
|
||||
// as the journal does (its retention, or less when it is full), and a row
|
||||
// counts for every entry of the deciding list that fits it: the journal does
|
||||
// not say which of two fitting entries the filter met first.
|
||||
//
|
||||
// A counter kept with each entry would not depend on the journal; the plan
|
||||
// (docs/plans/inbound-antispam-panel.md, Filter list usage) says when that is
|
||||
// worth a migration.
|
||||
func (l *Lists) Usage() (map[int64]Usage, error) {
|
||||
entries, err := l.store.ListSpamListEntries()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[int64]Usage, len(entries))
|
||||
if len(entries) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
hits, err := l.store.ListSpamListHits(denyReplyMark, allowMark)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type matcher struct {
|
||||
id int64
|
||||
list string
|
||||
kind string
|
||||
value string
|
||||
prefix netip.Prefix
|
||||
}
|
||||
matchers := make([]matcher, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
m := matcher{id: e.ID, list: e.List, kind: e.Kind, value: e.Value}
|
||||
if e.Kind == store.SpamEntryIP {
|
||||
if p, err := netip.ParsePrefix(e.Value); err == nil {
|
||||
m.prefix = p
|
||||
} else if a, err := netip.ParseAddr(e.Value); err == nil {
|
||||
m.prefix = netip.PrefixFrom(a, a.BitLen())
|
||||
} else {
|
||||
continue // not an entry this package wrote
|
||||
}
|
||||
}
|
||||
matchers = append(matchers, m)
|
||||
}
|
||||
for _, h := range hits {
|
||||
from := strings.ToLower(h.From)
|
||||
domain := ""
|
||||
if i := strings.LastIndexByte(from, '@'); i >= 0 {
|
||||
domain = from[i+1:]
|
||||
}
|
||||
client, clientErr := netip.ParseAddr(h.ClientIP)
|
||||
client = client.Unmap()
|
||||
denied := h.Decision != store.SpamDecisionAccept && strings.Contains(h.Detail, denyReplyMark)
|
||||
for _, m := range matchers {
|
||||
switch m.list {
|
||||
case store.SpamListDeny:
|
||||
if !denied {
|
||||
continue
|
||||
}
|
||||
case store.SpamListAllow:
|
||||
if h.Decision != store.SpamDecisionAccept || !strings.Contains(h.Detail, allowSymbol[m.kind]) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
fits := false
|
||||
switch m.kind {
|
||||
case store.SpamEntryAddress:
|
||||
fits = from == m.value
|
||||
case store.SpamEntryDomain:
|
||||
fits = domain == m.value
|
||||
case store.SpamEntryIP:
|
||||
fits = clientErr == nil && m.prefix.Contains(client)
|
||||
}
|
||||
if !fits {
|
||||
continue
|
||||
}
|
||||
u := out[m.id]
|
||||
u.Hits++
|
||||
if h.CreatedAt.After(u.Last) {
|
||||
u.Last = h.CreatedAt
|
||||
}
|
||||
out[m.id] = u
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -276,3 +276,95 @@ func TestListsLastWritten(t *testing.T) {
|
||||
t.Errorf("LastWritten counts a stray file: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// How often an entry decided a message is read off the journal: the list that
|
||||
// decided, and whether the entry fits who the message was from.
|
||||
func TestListsUsage(t *testing.T) {
|
||||
l, st, _ := testLists(t)
|
||||
add := func(list, kind, value string) int64 {
|
||||
t.Helper()
|
||||
e, err := l.Add(list, kind, value, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Add(%s, %s, %s): %v", list, kind, value, err)
|
||||
}
|
||||
return e.ID
|
||||
}
|
||||
denyDomain := add(store.SpamListDeny, store.SpamEntryDomain, "evil.example")
|
||||
denyAddr := add(store.SpamListDeny, store.SpamEntryAddress, "spammer@evil.example")
|
||||
denyNet := add(store.SpamListDeny, store.SpamEntryIP, "198.51.100.0/24")
|
||||
denyV6 := add(store.SpamListDeny, store.SpamEntryIP, "2001:db8::/32")
|
||||
allowAddr := add(store.SpamListAllow, store.SpamEntryAddress, "billing@vendor.example")
|
||||
allowDomain := add(store.SpamListAllow, store.SpamEntryDomain, "vendor.example")
|
||||
allowIP := add(store.SpamListAllow, store.SpamEntryIP, "203.0.113.5")
|
||||
unused := add(store.SpamListAllow, store.SpamEntryDomain, "quiet.example")
|
||||
|
||||
const denied = "5.7.1 Refused by this server's deny list"
|
||||
at := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
|
||||
hour := 0
|
||||
row := func(from, ip, decision, engine, detail string) {
|
||||
t.Helper()
|
||||
hour++
|
||||
err := st.InsertInboundSpam(store.InboundSpamEntry{
|
||||
InboundDomain: "in.example.com", To: "a@in.example.com", From: from, ClientIP: ip,
|
||||
Decision: decision, Engine: engine, Detail: detail, At: at.Add(time.Duration(hour) * time.Hour),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// hour 1: the address and its domain are both on the deny list, and both fit.
|
||||
row("Spammer@Evil.Example", "192.0.2.9", store.SpamDecisionReject, store.SpamEngineRspamd, denied)
|
||||
// hour 2: another address of the domain.
|
||||
row("other@evil.example", "192.0.2.9", store.SpamDecisionReject, store.SpamEngineRspamd, denied)
|
||||
// hour 3, 4: the networks, v4 (as Postfix may log it, mapped) and v6.
|
||||
row("x@elsewhere.example", "::ffff:198.51.100.77", store.SpamDecisionReject, store.SpamEngineRspamd, denied)
|
||||
row("x@elsewhere.example", "2001:db8::1", store.SpamDecisionReject, store.SpamEngineRspamd, denied)
|
||||
// hour 5: a subdomain is not the domain.
|
||||
row("x@sub.evil.example", "192.0.2.9", store.SpamDecisionReject, store.SpamEngineRspamd, denied)
|
||||
// hour 6: allowed by its address and its domain - two symbols, two entries.
|
||||
row("billing@vendor.example", "192.0.2.50", store.SpamDecisionAccept, store.SpamEngineRspamd,
|
||||
"no action, score -199.00 / 15.00; SELFPOST_ALLOW_DOMAIN(-100.00) SELFPOST_ALLOW_FROM(-100.00)")
|
||||
// hour 7: allowed by its domain alone.
|
||||
row("accounts@vendor.example", "192.0.2.50", store.SpamDecisionAccept, store.SpamEngineRspamd,
|
||||
"no action, score -99.00 / 15.00; SELFPOST_ALLOW_DOMAIN(-100.00)")
|
||||
// hour 8: allowed by the client address; the sender's domain is not why.
|
||||
row("x@quiet.example", "203.0.113.5", store.SpamDecisionAccept, store.SpamEngineRspamd,
|
||||
"no action, score -99.00 / 15.00; SELFPOST_ALLOW_IP(-100.00)")
|
||||
// Nobody's: the filter's own refusal, an accept with no list symbol, a
|
||||
// refusal by Postfix, and a sender who spells the mark in his own address.
|
||||
row("spammer@evil.example", "192.0.2.9", store.SpamDecisionReject, store.SpamEngineRspamd, "5.7.1 Spam message rejected")
|
||||
row("billing@vendor.example", "192.0.2.50", store.SpamDecisionAccept, store.SpamEngineRspamd, "no action, score 0.10 / 15.00")
|
||||
row("spammer@evil.example", "198.51.100.1", store.SpamDecisionReject, store.SpamEnginePostfix, "550 5.1.1 <nobody@in.example.com>: this server's deny list")
|
||||
row("billing@vendor.example", "203.0.113.5", store.SpamDecisionAccept, store.SpamEngineSelfPost, "")
|
||||
|
||||
usage, err := l.Usage()
|
||||
if err != nil {
|
||||
t.Fatalf("Usage: %v", err)
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
id int64
|
||||
hits int
|
||||
last int // hour
|
||||
}{
|
||||
"deny domain": {denyDomain, 2, 2}, "deny address": {denyAddr, 1, 1},
|
||||
"deny network": {denyNet, 1, 3}, "deny v6 network": {denyV6, 1, 4},
|
||||
"allow address": {allowAddr, 1, 6}, "allow domain": {allowDomain, 2, 7}, "allow ip": {allowIP, 1, 8},
|
||||
"unused": {unused, 0, 0},
|
||||
} {
|
||||
got := usage[c.id]
|
||||
wantLast := time.Time{}
|
||||
if c.hits > 0 {
|
||||
wantLast = at.Add(time.Duration(c.last) * time.Hour)
|
||||
}
|
||||
if got.Hits != c.hits || !got.Last.Equal(wantLast) {
|
||||
t.Errorf("%s: %d hits, last %v; want %d, %v", name, got.Hits, got.Last, c.hits, wantLast)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListsUsageWithNoEntries(t *testing.T) {
|
||||
l, _, _ := testLists(t)
|
||||
if usage, err := l.Usage(); err != nil || len(usage) != 0 {
|
||||
t.Errorf("Usage = %v, %v", usage, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user