inbound-antispam-panel: Filter lists, kind column, matches, note on a sheet
Follows the design commit before it (owner's words quoted there). The lists' tables are Value, Kind, Matched and Remove. The kind tag opens the entry's sheet: note, day added, how often it decided a message in the days the log keeps, and the last match. Matches are read off the journal on each view, nothing is stored (inbound.Lists.Usage, store.ListSpamListHits). A row counts for an entry when the list decided the message - a rejected row whose reply is the deny rule's message, an accepted row whose verdict carries the allow rule's symbol for that kind - and the entry fits the row's sender or client address the way rspamd matches it: an address whole, a domain as the sender's domain, a network containing the client (an IPv4-mapped address as IPv4). A failure to count shows the lists with "never" and logs why. For that the journal milter keeps the lists' own symbols ahead of its six-symbol cap: rspamd writes symbols heaviest first, so an allow entry's -100 came last and was cut behind any six others. The plan gains "Filter list usage": what counting from the journal implies, and - the owner asked for it in writing - the conditions under which a counter kept with each entry becomes worth a migration (the journal cut by its row cap and not by age; a short retention; counts that must outlive the journal; a slow page). multimap.conf says which of its texts SelfPost reads back. Tests: Usage over every kind on both lists and the rows that are nobody's; the milter's symbol order; the store query; the page and its sheets; a handler test from journal rows to the numbers on the page. Seen: the fixture page with a sheet open, headless Edge at 1360 px - the same as the accepted drawing. Checked on the test server, uncommitted tree: go vet ./... ok, go test ./... ok (22 packages), make e2e ok. Not seen in a real run: a count above zero. It needs an entry added in the panel, which I cannot sign in to, and then mail that the list decides; in particular the deny rule's message arriving in mail.log as the reply of a milter-reject line is what the trial with rspamc showed, not yet what Postfix logged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -167,12 +167,21 @@ func (h *Handlers) renderFilterLists(w http.ResponseWriter, r *http.Request, sta
|
||||
http.Error(w, "internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// How often each entry decided a message is counted from the journal. A
|
||||
// failure there is not a reason to hide the lists: they are shown without
|
||||
// the counts' worth - every entry "never" - and the log says why.
|
||||
usage, err := h.lists.Usage()
|
||||
if err != nil {
|
||||
logf("panel: filter lists: usage: %v", err)
|
||||
}
|
||||
on, _, _ := h.inboundFilter()
|
||||
in := make([]view.FilterEntryInput, len(entries))
|
||||
for i, e := range entries {
|
||||
in[i] = view.FilterEntryInput{ID: e.ID, List: e.List, Kind: e.Kind, Value: e.Value, Note: e.Note, Added: e.CreatedAt}
|
||||
in[i] = view.FilterEntryInput{ID: e.ID, List: e.List, Kind: e.Kind, Value: e.Value, Note: e.Note, Added: e.CreatedAt,
|
||||
Hits: usage[e.ID].Hits, LastHit: usage[e.ID].Last}
|
||||
}
|
||||
page := view.NewFilterLists(h.shellMeta(r), view.FilterListsInput{FilterOn: on, LastWritten: h.lists.LastWritten()}).
|
||||
page := view.NewFilterLists(h.shellMeta(r), view.FilterListsInput{
|
||||
FilterOn: on, LastWritten: h.lists.LastWritten(), RetentionDays: h.inboundSpamLogRetentionDays()}).
|
||||
WithEntries(in).
|
||||
WithResult(flash, formErr)
|
||||
// A blank list or kind leaves the form on its first choice.
|
||||
|
||||
@@ -364,9 +364,10 @@ func TestFilterListsAddAndRemove(t *testing.T) {
|
||||
has(t, "lists", page,
|
||||
`<div class="notification is-success is-light">`, `Entry added.`,
|
||||
`<h2>Deny</h2><div class="sp-end">1 entry · wins over Allow</div>`, `<h2>Allow</h2><div class="sp-end">1 entry</div>`,
|
||||
`bulk.example <span class="tag is-light">domain</span></td><td class="sp-muted">newsletter</td>`,
|
||||
`198.51.100.0/24 <span class="tag is-light">ip</span></td><td class="sp-muted">—</td>`,
|
||||
`<td class="sp-muted sp-nowrap sp-small">`+time.Now().UTC().Format("2006-01-02")+`</td>`,
|
||||
`<td class="sp-mono sp-small">bulk.example</td><td><button type="button" class="tag is-light" popovertarget="entry-`,
|
||||
`<tr><th>Note</th><td>newsletter</td></tr><tr><th>Added</th><td class="sp-mono sp-small">`+time.Now().UTC().Format("2006-01-02")+`</td></tr><tr><th>Matched</th><td>never in the last 90 days</td></tr>`,
|
||||
`<td class="sp-mono sp-small">198.51.100.0/24</td>`, `<tr><th>Note</th><td>—</td></tr>`,
|
||||
`<td class="sp-muted sp-nowrap">never</td>`,
|
||||
)
|
||||
lacks(t, "lists", page, `No sender is turned away`)
|
||||
|
||||
@@ -635,3 +636,39 @@ func TestHelpHasTheInboundLogAndFilterListsTopics(t *testing.T) {
|
||||
has(t, "log", s.get(h.HandleInboundLog, "/inbound/log", globalPrincipal).Body.String(), `href="/help#inbound-log"`)
|
||||
has(t, "lists", s.get(h.HandleFilterLists, "/inbound/filter-lists", globalPrincipal).Body.String(), `href="/help#filter-lists"`)
|
||||
}
|
||||
|
||||
// An entry's count comes from the journal: a row counts for the entry that
|
||||
// fits it on the list that decided it.
|
||||
func TestFilterListsCountMatchesFromTheJournal(t *testing.T) {
|
||||
s := newInboundLogStand(t)
|
||||
for _, e := range [][3]string{{"deny", "domain", "bulk.example"}, {"allow", "address", "billing@vendor.example"}, {"deny", "ip", "203.0.113.0/24"}} {
|
||||
if _, err := s.h.lists.Add(e[0], e[1], e[2], ""); err != nil {
|
||||
t.Fatalf("Add %v: %v", e, err)
|
||||
}
|
||||
}
|
||||
at := time.Date(2026, 10, 1, 9, 30, 0, 0, time.UTC)
|
||||
for i, row := range []store.InboundSpamEntry{
|
||||
{From: "promo@bulk.example", ClientIP: "198.51.100.9", Decision: store.SpamDecisionReject, Engine: store.SpamEngineRspamd, Detail: "5.7.1 Refused by this server's deny list"},
|
||||
{From: "other@BULK.example", ClientIP: "198.51.100.9", Decision: store.SpamDecisionReject, Engine: store.SpamEngineRspamd, Detail: "5.7.1 Refused by this server's deny list"},
|
||||
{From: "billing@vendor.example", ClientIP: "192.0.2.5", Decision: store.SpamDecisionAccept, Engine: store.SpamEngineRspamd, Detail: "no action, score -99.00 / 15.00; SELFPOST_ALLOW_FROM(-100.00)"},
|
||||
// refused by the filter for its own reasons: nobody's match
|
||||
{From: "promo@bulk.example", ClientIP: "198.51.100.9", Decision: store.SpamDecisionReject, Engine: store.SpamEngineRspamd, Detail: "5.7.1 Spam message rejected"},
|
||||
} {
|
||||
row.InboundDomain, row.To, row.At = "a.example.com", "alice@a.example.com", at.Add(time.Duration(i)*time.Hour)
|
||||
if err := s.st.InsertInboundSpam(row); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
page := s.get(s.h.HandleFilterLists, "/inbound/filter-lists", globalPrincipal).Body.String()
|
||||
has(t, "lists", page,
|
||||
`<tr><th>Matched</th><td>2 in the last 90 days</td></tr><tr><th>Last match</th><td class="sp-mono sp-small">2026-10-01 10:30</td></tr>`,
|
||||
`<td class="sp-nowrap">2</td>`,
|
||||
`<tr><th>Matched</th><td>1 in the last 90 days</td></tr><tr><th>Last match</th><td class="sp-mono sp-small">2026-10-01 11:30</td></tr>`,
|
||||
`<td class="sp-nowrap">1</td>`,
|
||||
// the network nobody came from
|
||||
`<td class="sp-muted sp-nowrap">never</td>`,
|
||||
)
|
||||
if got := strings.Count(page, `<td class="sp-muted sp-nowrap">never</td>`); got != 1 {
|
||||
t.Errorf("%d entries say never, want the one network", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -450,12 +450,13 @@ func inLogFixture() *InLog {
|
||||
// allowed, and a filter that reads them.
|
||||
func inFilterListsFixture() *FilterLists {
|
||||
day := func(m time.Month, d int) time.Time { return time.Date(2026, m, d, 0, 0, 0, 0, time.UTC) }
|
||||
return NewFilterLists(admin(), FilterListsInput{FilterOn: true, LastWritten: time.Date(2026, 9, 11, 8, 40, 0, 0, time.UTC)}).
|
||||
at := func(d, h, m int) time.Time { return time.Date(2026, 9, d, h, m, 0, 0, time.UTC) }
|
||||
return NewFilterLists(admin(), FilterListsInput{FilterOn: true, LastWritten: time.Date(2026, 9, 11, 8, 40, 0, 0, time.UTC), RetentionDays: 90}).
|
||||
WithEntries([]FilterEntryInput{
|
||||
{ID: 1, List: "deny", Kind: "domain", Value: "bulk.example", Note: "newsletter nobody subscribed to", Added: day(9, 2)},
|
||||
{ID: 2, List: "deny", Kind: "ip", Value: "198.51.100.0/24", Note: "dictionary attack on lists.example.org", Added: day(8, 19)},
|
||||
{ID: 1, List: "deny", Kind: "domain", Value: "bulk.example", Note: "newsletter nobody subscribed to", Added: day(9, 2), Hits: 41, LastHit: at(21, 13, 57)},
|
||||
{ID: 2, List: "deny", Kind: "ip", Value: "198.51.100.0/24", Note: "dictionary attack on lists.example.org", Added: day(8, 19), Hits: 1203, LastHit: at(21, 9, 12)},
|
||||
{ID: 3, List: "deny", Kind: "address", Value: "win@lottery.example", Added: day(7, 30)},
|
||||
{ID: 4, List: "allow", Kind: "domain", Value: "partner.example", Note: "invoices were scored as spam", Added: day(9, 11)},
|
||||
{ID: 5, List: "allow", Kind: "address", Value: "noreply@github.example", Note: "security advisories", Added: day(6, 4)},
|
||||
{ID: 4, List: "allow", Kind: "domain", Value: "partner.example", Note: "invoices were scored as spam", Added: day(9, 11), Hits: 7, LastHit: at(19, 16, 40)},
|
||||
{ID: 5, List: "allow", Kind: "address", Value: "noreply@github.example", Note: "security advisories", Added: day(6, 4), Hits: 18, LastHit: at(21, 13, 30)},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -205,7 +205,7 @@ var inboundTopics = []HelpTopic{
|
||||
Strong("Deny wins"), ".",
|
||||
Br(), Br(),
|
||||
"SelfPost does not apply the lists itself: it writes them as files under ", Code("/data/antispam/"),
|
||||
", and rspamd rereads a file within seconds of a change. With the spam filter off the lists are kept and can be edited, but nothing reads them. The sender matched is the envelope sender — the one in the log — not the From line a mail program shows."),
|
||||
", and rspamd rereads a file within seconds of a change. With the spam filter off the lists are kept and can be edited, but nothing reads them. The sender matched is the envelope sender — the one in the log — not the From line a mail program shows. Matched counts the messages in the Inbound log that the entry decided, so it reaches back as far as the log does; never means not once in that time, and such an entry is one to consider removing."),
|
||||
}},
|
||||
}
|
||||
|
||||
|
||||
@@ -46,6 +46,8 @@ type FilterLists struct {
|
||||
|
||||
Where Box
|
||||
WhereFacts []Fact
|
||||
|
||||
retentionDays int
|
||||
}
|
||||
|
||||
// FilterList is one of the two boxes of entries.
|
||||
@@ -55,24 +57,29 @@ type FilterList struct {
|
||||
Empty EmptyState
|
||||
}
|
||||
|
||||
// FilterRow is one entry of a list: the value with the kind of entry as a tag,
|
||||
// the operator's note, the day it was added and the way to remove it.
|
||||
// FilterRow is one entry of a list: the value, the kind of entry as a tag that
|
||||
// opens the entry's sheet (the note, the day it was added, how often it decided
|
||||
// a message and when last), how often it decided, and the way to remove it.
|
||||
// Matched is empty for an entry that never did; the table then says "never".
|
||||
type FilterRow struct {
|
||||
Value string
|
||||
Kind Tag
|
||||
Note string
|
||||
Added string
|
||||
Remove RowAction
|
||||
Value string
|
||||
Entry Sheet
|
||||
Matched string
|
||||
Remove RowAction
|
||||
}
|
||||
|
||||
// FilterEntryInput is what the page knows of one entry of a list.
|
||||
// FilterEntryInput is what the page knows of one entry of a list. Hits and
|
||||
// LastHit are counted from the Inbound log, so they reach back as far as it
|
||||
// does.
|
||||
type FilterEntryInput struct {
|
||||
ID int64
|
||||
List string
|
||||
Kind string
|
||||
Value string
|
||||
Note string
|
||||
Added time.Time
|
||||
ID int64
|
||||
List string
|
||||
Kind string
|
||||
Value string
|
||||
Note string
|
||||
Added time.Time
|
||||
Hits int
|
||||
LastHit time.Time
|
||||
}
|
||||
|
||||
// FilterListsInput is the state of the filter the lists are written for: whether
|
||||
@@ -80,6 +87,9 @@ type FilterEntryInput struct {
|
||||
type FilterListsInput struct {
|
||||
FilterOn bool
|
||||
LastWritten time.Time
|
||||
// RetentionDays is how long the Inbound log keeps a row: how far back an
|
||||
// entry's matches are counted.
|
||||
RetentionDays int
|
||||
}
|
||||
|
||||
// NewFilterLists builds the page. The form starts on the Deny list for a sender
|
||||
@@ -113,6 +123,7 @@ func NewFilterLists(m Meta, in FilterListsInput) *FilterLists {
|
||||
}
|
||||
p.Deny.Box.End = Plain(plural(0, "entry", "entries") + " · wins over Allow")
|
||||
p.Allow.Box.End = Plain(plural(0, "entry", "entries"))
|
||||
p.retentionDays = in.RetentionDays
|
||||
p.setWhere(in)
|
||||
return p
|
||||
}
|
||||
@@ -144,10 +155,25 @@ func (p *FilterLists) setWhere(in FilterListsInput) {
|
||||
func (p *FilterLists) WithEntries(entries []FilterEntryInput) *FilterLists {
|
||||
p.Deny.Rows, p.Allow.Rows = nil, nil
|
||||
for _, e := range entries {
|
||||
row := FilterRow{
|
||||
Value: e.Value, Kind: Tag{Label: e.Kind}, Note: e.Note, Added: e.Added.UTC().Format("2006-01-02"),
|
||||
Remove: RowAction{Label: "Remove", Post: inboundFilterLists + "/" + strconv.FormatInt(e.ID, 10) + "/delete", Danger: true},
|
||||
id := strconv.FormatInt(e.ID, 10)
|
||||
window := "in the last " + plural(int64(p.retentionDays), "day", "days")
|
||||
sheet := []SheetRow{
|
||||
{Label: "Note", Lines: []string{orDash(e.Note)}},
|
||||
{Label: "Added", Lines: []string{e.Added.UTC().Format("2006-01-02")}, Mono: true},
|
||||
}
|
||||
row := FilterRow{
|
||||
Value: e.Value,
|
||||
Remove: RowAction{Label: "Remove", Post: inboundFilterLists + "/" + id + "/delete", Danger: true},
|
||||
}
|
||||
if e.Hits > 0 {
|
||||
row.Matched = strconv.Itoa(e.Hits)
|
||||
sheet = append(sheet,
|
||||
SheetRow{Label: "Matched", Lines: []string{row.Matched + " " + window}},
|
||||
SheetRow{Label: "Last match", Lines: []string{e.LastHit.UTC().Format("2006-01-02 15:04")}, Mono: true})
|
||||
} else {
|
||||
sheet = append(sheet, SheetRow{Label: "Matched", Lines: []string{"never " + window}})
|
||||
}
|
||||
row.Entry = Sheet{ID: "entry-" + id, Tag: Tag{Label: e.Kind}, Title: e.Value, Rows: sheet}
|
||||
switch e.List {
|
||||
case "deny":
|
||||
p.Deny.Rows = append(p.Deny.Rows, row)
|
||||
|
||||
@@ -183,18 +183,27 @@ func TestFilterListsShowsTheFormTheListsAndWhereTheyGo(t *testing.T) {
|
||||
`href="/help#filter-lists" title="How the lists are applied"`,
|
||||
`<h2>Deny</h2><div class="sp-end">3 entries · wins over Allow</div>`,
|
||||
`<h2>Allow</h2><div class="sp-end">2 entries</div>`,
|
||||
`<td class="sp-mono sp-small">bulk.example <span class="tag is-light">domain</span></td><td class="sp-muted">newsletter nobody subscribed to</td><td class="sp-muted sp-nowrap sp-small">2026-09-02</td>`,
|
||||
`198.51.100.0/24 <span class="tag is-light">ip</span>`,
|
||||
`win@lottery.example <span class="tag is-light">address</span></td><td class="sp-muted">—</td>`,
|
||||
`<thead><tr><th>Value</th><th>Kind</th><th>Matched</th><th></th></tr></thead>`,
|
||||
// the kind is a tag that opens the entry's sheet; the count is beside it
|
||||
`<td class="sp-mono sp-small">bulk.example</td><td><button type="button" class="tag is-light" popovertarget="entry-1">domain</button><div class="sp-sheet" id="entry-1" popover>`,
|
||||
`<div class="sp-sheet-head"><span class="tag is-light">domain</span><h2>bulk.example</h2>`,
|
||||
`<tr><th>Note</th><td>newsletter nobody subscribed to</td></tr><tr><th>Added</th><td class="sp-mono sp-small">2026-09-02</td></tr><tr><th>Matched</th><td>41 in the last 90 days</td></tr><tr><th>Last match</th><td class="sp-mono sp-small">2026-09-21 13:57</td></tr></tbody>`,
|
||||
`</div></td><td class="sp-nowrap">41</td><td class="sp-actions">`,
|
||||
`<td class="sp-nowrap">1203</td>`,
|
||||
// an entry that never decided anything says so, in the row and on its sheet
|
||||
`<td class="sp-mono sp-small">win@lottery.example</td><td><button type="button" class="tag is-light" popovertarget="entry-3">address</button>`,
|
||||
`<tr><th>Note</th><td>—</td></tr><tr><th>Added</th><td class="sp-mono sp-small">2026-07-30</td></tr><tr><th>Matched</th><td>never in the last 90 days</td></tr></tbody>`,
|
||||
`</div></td><td class="sp-muted sp-nowrap">never</td><td class="sp-actions">`,
|
||||
`<form method="post" action="/inbound/filter-lists/3/delete"><button type="submit" class="has-text-danger">Remove</button></form>`,
|
||||
`<h2>Where the lists go</h2><div class="sp-end"><span class="tag is-success is-light">applied</span></div>`,
|
||||
`rspamd, from map files it rereads within seconds of a change`,
|
||||
`<span class="sp-mono">/data/antispam/</span> — six, one for each list and kind of entry`,
|
||||
`2026-09-11 08:40 UTC`, `Deny wins, whichever entries they are.`,
|
||||
)
|
||||
if got := strings.Count(out, `<table`); got != 2 {
|
||||
t.Errorf("%d tables, want the two lists", got)
|
||||
if got := strings.Count(out, `<table class="table is-fullwidth is-hoverable">`); got != 2 {
|
||||
t.Errorf("%d list tables, want the two lists", got)
|
||||
}
|
||||
pageLacks(t, "Filter lists", out, `<th>Note</th><th>`, `<th>Added</th><th>`)
|
||||
}
|
||||
|
||||
// With no filter attached the lists are still the operator's to edit, and the
|
||||
@@ -213,8 +222,8 @@ func TestFilterListsOffAndEmpty(t *testing.T) {
|
||||
`No sender is turned away by hand.`,
|
||||
)
|
||||
pageLacks(t, "Filter lists", out, `rspamd, from map files`, `is-success is-light">applied`, `No sender is let through by hand.`)
|
||||
if got := strings.Count(out, `<table`); got != 1 {
|
||||
t.Errorf("%d tables, want the one list that has entries", got)
|
||||
if got := strings.Count(out, `<table class="table is-fullwidth is-hoverable">`); got != 1 {
|
||||
t.Errorf("%d list tables, want the one list that has entries", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
{{/* Inbound > Filter lists (mockup in-filter-lists.html): the allow and deny
|
||||
lists that correct the spam filter by hand, for the whole instance, and
|
||||
where they go. Global role only. The form adds an entry; each row of a list
|
||||
removes its own. */}}
|
||||
removes its own, says how often the entry decided a message, and opens a
|
||||
sheet with the entry's note from the tag of its kind. */}}
|
||||
{{define "content"}}
|
||||
{{template "page_head" .Head}}
|
||||
{{if .Flash}}{{template "flash" .Flash}}{{end}}
|
||||
@@ -54,9 +55,9 @@
|
||||
table, or what is absent when the list is empty. */}}
|
||||
{{define "filter_list"}}{{template "box_open" .Box}}
|
||||
{{if .Rows}}<div class="table-container"><table class="table is-fullwidth is-hoverable">
|
||||
<thead><tr><th>Value</th><th>Note</th><th>Added</th><th></th></tr></thead>
|
||||
<thead><tr><th>Value</th><th>Kind</th><th>Matched</th><th></th></tr></thead>
|
||||
<tbody>{{range .Rows}}
|
||||
<tr><td class="sp-mono sp-small">{{.Value}} {{template "tag" .Kind}}</td>{{if .Note}}<td class="sp-muted">{{.Note}}</td>{{else}}<td class="sp-muted">—</td>{{end}}<td class="sp-muted sp-nowrap sp-small">{{.Added}}</td><td class="sp-actions">{{template "row_action" .Remove}}</td></tr>{{end}}
|
||||
<tr><td class="sp-mono sp-small">{{.Value}}</td><td>{{template "sheet_tag" .Entry}}</td>{{if .Matched}}<td class="sp-nowrap">{{.Matched}}</td>{{else}}<td class="sp-muted sp-nowrap">never</td>{{end}}<td class="sp-actions">{{template "row_action" .Remove}}</td></tr>{{end}}
|
||||
</tbody>
|
||||
</table></div>
|
||||
{{else}}{{template "empty_state" .Empty}}
|
||||
|
||||
Reference in New Issue
Block a user