inbound-antispam-panel: filter lists, written as rspamd maps

Step 5/12 of docs/plans/inbound-antispam-panel.md.

Store: AddSpamListEntry, ListSpamListEntries, DeleteSpamListEntry.
internal/inbound/lists.go validates and normalises an entry - sender
address, sender domain, client address or network (canonical text, an
IPv4-mapped address as IPv4, /0 refused) - and writes the lists as six
map files under /data/antispam (ANTISPAM_DIR), each atomically, 0644 in
a 0755 directory: the filter is another user in another container. All
six are written on every change and on every start with the relay on,
empty ones included. An entry whose maps could not be written is taken
back out of the table. The writer refuses a stored value with
whitespace, a newline or # in it.

The plan's allow.map / deny.map became one file per list and entry
type: to rspamd an IP map and a sender map are different map types.

deploy/antispam/rspamd/multimap.conf holds the rules; the fragment
mounts it and ./data/antispam into the sidecar. Deny is a prefilter
reject, allow a symbol weighing -100. Tried in a throwaway rspamd 3.11
container on the test server:
  - with allow as a prefilter accept, an address on both lists was
    accepted and an IP on both was refused - the rule that ran first
    decided; with the weight, deny won in all five mixed cases;
  - IPv4, IPv6, address (any case), domain (not its subdomains) match;
  - a rewritten file is reread in about ten seconds, no restart;
  - files that appear after rspamd started are loaded the same way.

No page edits the lists yet (step 6).

Checked on the test server, uncommitted tree: go vet ./... ok,
go test ./... ok (22 packages), make e2e ok - with the relay on
/data/antispam is 755 and holds the six maps, 644, empty.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/inbound/lists.go

@@ -0,0 +1,224 @@
+package inbound
+
+import (
+	"fmt"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/mixeme/selfpost/internal/atomicfile"
+	"github.com/mixeme/selfpost/internal/configsafe"
+	"github.com/mixeme/selfpost/internal/store"
+)
+
+// ListStore is the slice of the store the filter lists need. *store.Store
+// satisfies it.
+type ListStore interface {
+	AddSpamListEntry(list, kind, value, note string) (store.SpamListEntry, error)
+	ListSpamListEntries() ([]store.SpamListEntry, error)
+	DeleteSpamListEntry(id int64) error
+}
+
+// Lists owns the allow and deny lists of the external spam filter: the rows in
+// SQLite and the map files the filter reads. SelfPost does not consult the
+// lists itself - it writes them out, six files in one directory, one entry a
+// line, and the filter (rspamd, deploy/antispam/rspamd/multimap.conf) rereads
+// a file when it changes. Which list wins when both match is the filter's
+// configuration: deny.
+type Lists struct {
+	store ListStore
+	dir   string
+}
+
+// NewLists builds the filter lists over the store and the map directory
+// (/data/antispam).
+func NewLists(st ListStore, dir string) *Lists {
+	return &Lists{store: st, dir: dir}
+}
+
+// noteMaxRunes caps the operator's comment on an entry.
+const noteMaxRunes = 200
+
+// mapFiles names the file of every (list, kind). An IP map and a sender map
+// are different things to the filter, and a domain entry is matched against
+// another part of the sender than an address, so each has its own file.
+var mapFiles = map[[2]string]string{
+	{store.SpamListAllow, store.SpamEntryIP}:      "allow_ip.map",
+	{store.SpamListAllow, store.SpamEntryAddress}: "allow_from.map",
+	{store.SpamListAllow, store.SpamEntryDomain}:  "allow_domain.map",
+	{store.SpamListDeny, store.SpamEntryIP}:       "deny_ip.map",
+	{store.SpamListDeny, store.SpamEntryAddress}:  "deny_from.map",
+	{store.SpamListDeny, store.SpamEntryDomain}:   "deny_domain.map",
+}
+
+// Entries returns every entry of both lists.
+func (l *Lists) Entries() ([]store.SpamListEntry, error) {
+	return l.store.ListSpamListEntries()
+}
+
+// Add validates an entry, stores it normalised and rewrites the maps. A
+// refusal is a *ValidationError; an entry already on that list is
+// store.ErrSpamListEntryExists.
+func (l *Lists) Add(list, kind, rawValue, rawNote string) (store.SpamListEntry, error) {
+	if list != store.SpamListAllow && list != store.SpamListDeny {
+		return store.SpamListEntry{}, invalid("choose the allow or the deny list")
+	}
+	value, err := normalizeListValue(kind, rawValue)
+	if err != nil {
+		return store.SpamListEntry{}, err
+	}
+	note, err := normalizeNote(rawNote)
+	if err != nil {
+		return store.SpamListEntry{}, err
+	}
+	e, err := l.store.AddSpamListEntry(list, kind, value, note)
+	if err != nil {
+		return store.SpamListEntry{}, err
+	}
+	if err := l.Sync(); err != nil {
+		// An entry the filter never got is worse than no entry: the page
+		// would show a rule that does nothing.
+		_ = l.store.DeleteSpamListEntry(e.ID)
+		return store.SpamListEntry{}, err
+	}
+	return e, nil
+}
+
+// Delete removes an entry and rewrites the maps.
+func (l *Lists) Delete(id int64) error {
+	if err := l.store.DeleteSpamListEntry(id); err != nil {
+		return err
+	}
+	return l.Sync()
+}
+
+// Sync writes all six map files from SQLite, each atomically, so the filter
+// reads a whole old file or a whole new one. Every file is written even when
+// empty: the filter's configuration names all six.
+func (l *Lists) Sync() error {
+	entries, err := l.store.ListSpamListEntries()
+	if err != nil {
+		return err
+	}
+	lines := make(map[string][]string, len(mapFiles))
+	for _, name := range mapFiles {
+		lines[name] = nil
+	}
+	for _, e := range entries {
+		name, ok := mapFiles[[2]string{e.List, e.Kind}]
+		if !ok {
+			return fmt.Errorf("filter list entry %d: unknown list %q or kind %q", e.ID, e.List, e.Kind)
+		}
+		// Values were validated on the way in; this is the writer's own guard
+		// against a row that got into the table some other way. In a map file a
+		// newline ends the entry, whitespace starts its value and # a comment.
+		if err := configsafe.Token("filter list entry", e.Value, " \t\r\n#"); err != nil {
+			return err
+		}
+		lines[name] = append(lines[name], e.Value)
+	}
+	// The filter runs under another user in another container: the directory
+	// and the files are readable by anyone. They hold addresses, not secrets.
+	if err := os.MkdirAll(l.dir, 0o755); err != nil {
+		return fmt.Errorf("create %s: %w", l.dir, err)
+	}
+	for name, vals := range lines {
+		sort.Strings(vals)
+		var b strings.Builder
+		b.WriteString("# Written by SelfPost (Inbound > Filter lists). Do not edit: the next change in the panel replaces this file.\n")
+		for _, v := range vals {
+			b.WriteString(v)
+			b.WriteByte('\n')
+		}
+		if err := atomicfile.Write(filepath.Join(l.dir, name), []byte(b.String()), 0o644); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// normalizeListValue validates one entry and returns the form it is stored and
+// written in: lower case for an address or a domain, the canonical text of an
+// address or a network for an IP.
+func normalizeListValue(kind, raw string) (string, error) {
+	v := strings.ToLower(strings.TrimSpace(raw))
+	switch kind {
+	case store.SpamEntryAddress:
+		if v == "" {
+			return "", invalid("sender address is required")
+		}
+		if len(v) > 254 {
+			return "", invalid("sender address must be at most 254 characters")
+		}
+		at := strings.LastIndexByte(v, '@')
+		if at <= 0 || at >= len(v)-1 {
+			return "", invalid("%q is not a valid email address", v)
+		}
+		local, domain := v[:at], v[at+1:]
+		if local[0] == '.' || local[len(local)-1] == '.' || len(local) > 64 {
+			return "", invalid("%q: invalid local part", v)
+		}
+		for i := 0; i < len(local); i++ {
+			c := local[i]
+			lower := c >= 'a' && c <= 'z'
+			digit := c >= '0' && c <= '9'
+			// '=' for the rewritten senders of forwarders (SRS0=…).
+			if !lower && !digit && !strings.ContainsRune("._+-=", rune(c)) {
+				return "", invalid("%q: local part contains invalid characters", v)
+			}
+		}
+		if err := checkDomain(domain); err != nil {
+			return "", err
+		}
+		return v, nil
+	case store.SpamEntryDomain:
+		// "@example.com" is how a domain is often written in such a list.
+		v = strings.TrimPrefix(v, "@")
+		if err := checkDomain(v); err != nil {
+			return "", err
+		}
+		return v, nil
+	case store.SpamEntryIP:
+		if v == "" {
+			return "", invalid("address or network is required")
+		}
+		if strings.Contains(v, "/") {
+			p, err := netip.ParsePrefix(v)
+			if err != nil {
+				return "", invalid("%q is not a network in CIDR form (e.g. 192.0.2.0/24)", v)
+			}
+			if p.Addr().Is4In6() {
+				return "", invalid("%q: write an IPv4 network as IPv4 (e.g. 192.0.2.0/24)", v)
+			}
+			if p.Bits() == 0 {
+				return "", invalid("%q is every address there is", v)
+			}
+			return p.Masked().String(), nil
+		}
+		a, err := netip.ParseAddr(v)
+		if err != nil || a.Zone() != "" {
+			return "", invalid("%q is not an IP address or a network in CIDR form", v)
+		}
+		return a.Unmap().String(), nil
+	default:
+		return "", invalid("choose what the entry is: a sender address, a sender domain or a client address")
+	}
+}
+
+// normalizeNote trims the operator's comment and refuses what a single line of
+// text cannot be.
+func normalizeNote(raw string) (string, error) {
+	note := strings.TrimSpace(raw)
+	if utf8.RuneCountInString(note) > noteMaxRunes {
+		return "", invalid("note must be at most %d characters", noteMaxRunes)
+	}
+	for _, r := range note {
+		if r < 0x20 || r == 0x7f || r == utf8.RuneError {
+			return "", invalid("note must be one line of plain text")
+		}
+	}
+	return note, nil
+}

internal/inbound/lists_test.go

@@ -0,0 +1,237 @@
+package inbound
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/mixeme/selfpost/internal/store"
+)
+
+func testLists(t *testing.T) (*Lists, *store.Store, string) {
+	t.Helper()
+	st, err := store.Open(filepath.Join(t.TempDir(), "test.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { st.Close() })
+	dir := filepath.Join(t.TempDir(), "antispam")
+	return NewLists(st, dir), st, dir
+}
+
+// mapLines reads a map file without its comment line.
+func mapLines(t *testing.T, dir, name string) []string {
+	t.Helper()
+	raw, err := os.ReadFile(filepath.Join(dir, name))
+	if err != nil {
+		t.Fatalf("read %s: %v", name, err)
+	}
+	var out []string
+	for _, line := range strings.Split(strings.TrimRight(string(raw), "\n"), "\n") {
+		if !strings.HasPrefix(line, "#") {
+			out = append(out, line)
+		}
+	}
+	return out
+}
+
+func TestNormalizeListValue(t *testing.T) {
+	for _, c := range []struct{ kind, raw, want string }{
+		{store.SpamEntryAddress, " Billing@Vendor.Example ", "billing@vendor.example"},
+		{store.SpamEntryAddress, "srs0=ab12=cd=example.org=user@forwarder.example", "srs0=ab12=cd=example.org=user@forwarder.example"},
+		{store.SpamEntryDomain, "@Evil.Example", "evil.example"},
+		{store.SpamEntryDomain, "sub.evil.example", "sub.evil.example"},
+		{store.SpamEntryIP, "192.0.2.7", "192.0.2.7"},
+		{store.SpamEntryIP, "192.0.2.77/24", "192.0.2.0/24"},
+		{store.SpamEntryIP, "::ffff:192.0.2.7", "192.0.2.7"},
+		{store.SpamEntryIP, "2001:DB8:0:0::1", "2001:db8::1"},
+		{store.SpamEntryIP, "2001:db8:1::5/48", "2001:db8:1::/48"},
+	} {
+		got, err := normalizeListValue(c.kind, c.raw)
+		if err != nil || got != c.want {
+			t.Errorf("normalizeListValue(%s, %q) = %q, %v; want %q", c.kind, c.raw, got, err, c.want)
+		}
+	}
+
+	for _, c := range []struct{ kind, raw string }{
+		{store.SpamEntryAddress, ""},
+		{store.SpamEntryAddress, "no-at-sign"},
+		{store.SpamEntryAddress, "@example.com"},
+		{store.SpamEntryAddress, "a@localhost"},
+		{store.SpamEntryAddress, "a b@example.com"},
+		{store.SpamEntryAddress, "a#b@example.com"},
+		{store.SpamEntryAddress, "a\nb@example.com"},
+		{store.SpamEntryAddress, ".a@example.com"},
+		{store.SpamEntryDomain, ""},
+		{store.SpamEntryDomain, "example"},
+		{store.SpamEntryDomain, "exa mple.com"},
+		{store.SpamEntryDomain, "example.com # all of it"},
+		{store.SpamEntryDomain, "user@example.com"},
+		{store.SpamEntryIP, ""},
+		{store.SpamEntryIP, "example.com"},
+		{store.SpamEntryIP, "192.0.2.300"},
+		{store.SpamEntryIP, "192.0.2.0/33"},
+		{store.SpamEntryIP, "0.0.0.0/0"},
+		{store.SpamEntryIP, "::/0"},
+		{store.SpamEntryIP, "fe80::1%eth0"},
+		{store.SpamEntryIP, "::ffff:192.0.2.0/120"},
+		{store.SpamEntryIP, "192.0.2.7 192.0.2.8"},
+		{"subject", "pills"},
+		{"", "x"},
+	} {
+		got, err := normalizeListValue(c.kind, c.raw)
+		var ve *ValidationError
+		if err == nil || !errors.As(err, &ve) {
+			t.Errorf("normalizeListValue(%q, %q) = %q, %v; want a ValidationError", c.kind, c.raw, got, err)
+		}
+	}
+}
+
+func TestListsAddWritesTheMaps(t *testing.T) {
+	l, _, dir := testLists(t)
+	for _, e := range []struct{ list, kind, value string }{
+		{store.SpamListAllow, store.SpamEntryAddress, "Billing@Vendor.Example"},
+		{store.SpamListAllow, store.SpamEntryAddress, "accounts@vendor.example"},
+		{store.SpamListDeny, store.SpamEntryDomain, "@evil.example"},
+		{store.SpamListDeny, store.SpamEntryIP, "198.51.100.9/24"},
+		// The same value on both lists is two entries: deny wins in the filter.
+		{store.SpamListDeny, store.SpamEntryAddress, "billing@vendor.example"},
+	} {
+		if _, err := l.Add(e.list, e.kind, e.value, " why "); err != nil {
+			t.Fatalf("Add(%s, %s, %q): %v", e.list, e.kind, e.value, err)
+		}
+	}
+
+	want := map[string]string{
+		"allow_from.map":   "accounts@vendor.example billing@vendor.example",
+		"allow_domain.map": "",
+		"allow_ip.map":     "",
+		"deny_from.map":    "billing@vendor.example",
+		"deny_domain.map":  "evil.example",
+		"deny_ip.map":      "198.51.100.0/24",
+	}
+	for name, lines := range want {
+		if got := strings.Join(mapLines(t, dir, name), " "); got != lines {
+			t.Errorf("%s = %q, want %q", name, got, lines)
+		}
+		// The filter is another user in another container.
+		if info, err := os.Stat(filepath.Join(dir, name)); err != nil {
+			t.Errorf("stat %s: %v", name, err)
+		} else if info.Mode().Perm()&0o004 == 0 && os.PathSeparator == '/' {
+			t.Errorf("%s is mode %o, not readable by the filter", name, info.Mode().Perm())
+		}
+	}
+	if files, _ := os.ReadDir(dir); len(files) != len(want) {
+		t.Errorf("the directory holds %d files, want %d (no temp file left behind)", len(files), len(want))
+	}
+
+	entries, err := l.Entries()
+	if err != nil || len(entries) != 5 {
+		t.Fatalf("Entries = %d, %v", len(entries), err)
+	}
+	if entries[0].Note != "why" {
+		t.Errorf("note = %q, want it trimmed", entries[0].Note)
+	}
+}
+
+func TestListsRefusals(t *testing.T) {
+	l, _, dir := testLists(t)
+	if _, err := l.Add(store.SpamListAllow, store.SpamEntryIP, "192.0.2.7", ""); err != nil {
+		t.Fatalf("Add: %v", err)
+	}
+	// Written differently, the same entry.
+	if _, err := l.Add(store.SpamListAllow, store.SpamEntryIP, "::ffff:192.0.2.7", ""); !errors.Is(err, store.ErrSpamListEntryExists) {
+		t.Errorf("the same address again: %v, want ErrSpamListEntryExists", err)
+	}
+	var ve *ValidationError
+	for name, add := range map[string]func() error{
+		"an unknown list": func() error { _, err := l.Add("block", store.SpamEntryIP, "192.0.2.8", ""); return err },
+		"a bad value":     func() error { _, err := l.Add(store.SpamListDeny, store.SpamEntryIP, "nope", ""); return err },
+		"a long note": func() error {
+			_, err := l.Add(store.SpamListDeny, store.SpamEntryIP, "192.0.2.8", strings.Repeat("x", noteMaxRunes+1))
+			return err
+		},
+		"a note of two lines": func() error {
+			_, err := l.Add(store.SpamListDeny, store.SpamEntryIP, "192.0.2.8", "one\ntwo")
+			return err
+		},
+	} {
+		if err := add(); !errors.As(err, &ve) {
+			t.Errorf("%s: %v, want a ValidationError", name, err)
+		}
+	}
+	if got := strings.Join(mapLines(t, dir, "deny_ip.map"), " "); got != "" {
+		t.Errorf("deny_ip.map = %q after refusals only", got)
+	}
+	if entries, _ := l.Entries(); len(entries) != 1 {
+		t.Errorf("entries = %d, want the first one alone", len(entries))
+	}
+}
+
+func TestListsDelete(t *testing.T) {
+	l, _, dir := testLists(t)
+	e, err := l.Add(store.SpamListDeny, store.SpamEntryDomain, "evil.example", "")
+	if err != nil {
+		t.Fatalf("Add: %v", err)
+	}
+	if err := l.Delete(e.ID); err != nil {
+		t.Fatalf("Delete: %v", err)
+	}
+	if got := mapLines(t, dir, "deny_domain.map"); len(got) != 0 {
+		t.Errorf("deny_domain.map = %v after the delete", got)
+	}
+	if err := l.Delete(e.ID); !errors.Is(err, store.ErrSpamListEntryNotFound) {
+		t.Errorf("a second delete: %v", err)
+	}
+}
+
+// An entry the filter never got must not stay on the page.
+func TestListsAddRollsBackWhenTheMapsCannotBeWritten(t *testing.T) {
+	st, err := store.Open(filepath.Join(t.TempDir(), "test.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer st.Close()
+	// A file where the directory should be.
+	blocked := filepath.Join(t.TempDir(), "antispam")
+	if err := os.WriteFile(blocked, nil, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	l := NewLists(st, blocked)
+	if _, err := l.Add(store.SpamListDeny, store.SpamEntryDomain, "evil.example", ""); err == nil {
+		t.Fatal("Add succeeded with nowhere to write the maps")
+	}
+	if entries, _ := l.Entries(); len(entries) != 0 {
+		t.Errorf("the entry stayed: %+v", entries)
+	}
+}
+
+// The writer's own guard: a row that got into the table some other way is not
+// written into a map.
+func TestListsSyncRefusesAnUnsafeRow(t *testing.T) {
+	l, st, dir := testLists(t)
+	if _, err := st.AddSpamListEntry(store.SpamListDeny, store.SpamEntryDomain, "evil.example\ngood.example", ""); err != nil {
+		t.Fatalf("AddSpamListEntry: %v", err)
+	}
+	if err := l.Sync(); err == nil {
+		t.Fatal("Sync wrote a value with a newline in it")
+	}
+	if _, err := os.Stat(filepath.Join(dir, "deny_domain.map")); err == nil {
+		t.Error("a map file was written")
+	}
+}
+
+// With nothing on the lists all six files are there, empty.
+func TestListsSyncWritesEmptyMaps(t *testing.T) {
+	l, _, dir := testLists(t)
+	if err := l.Sync(); err != nil {
+		t.Fatalf("Sync: %v", err)
+	}
+	for _, name := range mapFiles {
+		if got := mapLines(t, dir, name); len(got) != 0 {
+			t.Errorf("%s = %v", name, got)
+		}
+	}
+}

internal/store/inboundspam.go

@@ -95,3 +95,86 @@ func (s *Store) IsInboundDomain(name string) (bool, error) {
 	}
 	return true, nil
 }
+
+// Lists and entry kinds of the filter lists. Kept in sync with the CHECK
+// constraints of inbound_spam_list.
+const (
+	SpamListAllow = "allow"
+	SpamListDeny  = "deny"
+
+	SpamEntryAddress = "address" // sender address
+	SpamEntryDomain  = "domain"  // sender domain
+	SpamEntryIP      = "ip"      // client address or network
+)
+
+// ErrSpamListEntryExists is returned when the entry is already on that list.
+var ErrSpamListEntryExists = errors.New("filter list entry already exists")
+
+// ErrSpamListEntryNotFound is returned when a filter list entry id does not exist.
+var ErrSpamListEntryNotFound = errors.New("filter list entry not found")
+
+// SpamListEntry is one allow or deny entry for the external filter. Value is
+// stored normalised by the caller (internal/inbound), which is what makes the
+// uniqueness of an entry on its list mean anything.
+type SpamListEntry struct {
+	ID        int64
+	List      string
+	Kind      string
+	Value     string
+	Note      string
+	CreatedAt time.Time
+}
+
+// AddSpamListEntry inserts an entry. The caller must have validated it.
+func (s *Store) AddSpamListEntry(list, kind, value, note string) (SpamListEntry, error) {
+	now := time.Now().UTC()
+	res, err := s.db.Exec(
+		"INSERT INTO inbound_spam_list (list_type, entry_type, value, note, created_at) VALUES (?, ?, ?, ?, ?)",
+		list, kind, value, note, now.Format(time.RFC3339),
+	)
+	if err != nil {
+		if isUniqueViolation(err) {
+			return SpamListEntry{}, ErrSpamListEntryExists
+		}
+		return SpamListEntry{}, fmt.Errorf("insert inbound_spam_list: %w", err)
+	}
+	id, err := res.LastInsertId()
+	if err != nil {
+		return SpamListEntry{}, fmt.Errorf("filter list entry id: %w", err)
+	}
+	return SpamListEntry{ID: id, List: list, Kind: kind, Value: value, Note: note, CreatedAt: now.Truncate(time.Second)}, nil
+}
+
+// ListSpamListEntries returns every entry of both lists: by list, then kind,
+// then value.
+func (s *Store) ListSpamListEntries() ([]SpamListEntry, error) {
+	rows, err := s.db.Query(
+		"SELECT id, list_type, entry_type, value, note, created_at FROM inbound_spam_list ORDER BY list_type, entry_type, value")
+	if err != nil {
+		return nil, fmt.Errorf("list inbound_spam_list: %w", err)
+	}
+	defer rows.Close()
+	var out []SpamListEntry
+	for rows.Next() {
+		var e SpamListEntry
+		var created string
+		if err := rows.Scan(&e.ID, &e.List, &e.Kind, &e.Value, &e.Note, &created); err != nil {
+			return nil, fmt.Errorf("scan inbound_spam_list: %w", err)
+		}
+		e.CreatedAt, _ = time.Parse(time.RFC3339, created)
+		out = append(out, e)
+	}
+	return out, rows.Err()
+}
+
+// DeleteSpamListEntry removes an entry by id.
+func (s *Store) DeleteSpamListEntry(id int64) error {
+	res, err := s.db.Exec("DELETE FROM inbound_spam_list WHERE id = ?", id)
+	if err != nil {
+		return fmt.Errorf("delete inbound_spam_list: %w", err)
+	}
+	if n, _ := res.RowsAffected(); n == 0 {
+		return ErrSpamListEntryNotFound
+	}
+	return nil
+}