diff --git a/CHANGELOG.md b/CHANGELOG.md index 36d5540..87a928f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); version ## [Unreleased] +### Added + +- Panel **Settings** (global administrator): **Send log retention (days)** — + how long delivery journal rows on `/deliveries` are kept (7–365). Stored in + SQLite `settings`; `SEND_LOG_RETENTION_DAYS` seeds the initial default only. + The log-tailer re-reads the value every prune cycle (no restart). + +### Changed + +- Deliveries list and delivery detail pages show the configured retention + instead of a hardcoded ninety-day message. + ## [1.4.0] - 2026-08-17 Optional inbound relay (backup-MX / forwarder) on port 25, off by default. diff --git a/cmd/panel/httpserver.go b/cmd/panel/httpserver.go index f760b3d..2d57bbf 100644 --- a/cmd/panel/httpserver.go +++ b/cmd/panel/httpserver.go @@ -96,8 +96,9 @@ func newPanel(cfg config, st *store.Store) (*web.Server, error) { DNSResolvers: cfg.dnsResolvers, RateLimitMessagesPerIP: cfg.rateLimitMessagesPerIP, RateLimitWindowSeconds: cfg.rateLimitWindowSeconds, - RetryPolicy: retryPolicy, - InboundEnabled: cfg.inboundEnabled, + RetryPolicy: retryPolicy, + InboundEnabled: cfg.inboundEnabled, + SendLogRetentionEnvDefault: cfg.retentionDays, }, cfg.setupTokenPath) } diff --git a/cmd/panel/main.go b/cmd/panel/main.go index a0bd5d5..2b7a4db 100644 --- a/cmd/panel/main.go +++ b/cmd/panel/main.go @@ -249,6 +249,10 @@ func run() error { } } + if err := st.EnsureSendLogRetentionDays(cfg.retentionDays); err != nil { + return fmt.Errorf("bootstrap send-log retention: %w", err) + } + var wg sync.WaitGroup errc := make(chan error, 3) @@ -258,7 +262,16 @@ func run() error { }{ {"http", func(ctx context.Context) error { return serveHTTP(ctx, cfg, st) }}, {"journal-milter", func(ctx context.Context) error { return serveJournal(ctx, cfg, st) }}, - {"log-tailer", func(ctx context.Context) error { return logtail.Run(ctx, cfg.mailLog, st, cfg.retentionDays) }}, + {"log-tailer", func(ctx context.Context) error { + return logtail.Run(ctx, cfg.mailLog, st, func() int { + days, err := st.GetSendLogRetentionDays(cfg.retentionDays) + if err != nil { + log.Printf("send-log retention: %v", err) + return cfg.retentionDays + } + return days + }) + }}, } for _, r := range roles { diff --git a/docs/guide.md b/docs/guide.md index 7794171..e6c1076 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -196,7 +196,7 @@ expected to set; defaults match the code exactly. | `INBOUND_MESSAGE_SIZE_LIMIT` | Maximum message size in bytes on inbound smtpd (`message_size_limit`). | `26214400` (25 MiB) | `.env` | | `RATE_LIMIT_MESSAGES_PER_IP` | Level-1 backstop: maximum messages one client IP may submit per window (Postfix `smtpd_client_message_rate_limit`). See [Rate limiting — level 1](#rate-limiting--level-1-ip-backstop). | `100` | `.env` | | `RATE_LIMIT_WINDOW_SECONDS` | Level-1 window length in seconds (Postfix `anvil_rate_time_unit`). | `3600` | `.env` | -| `SEND_LOG_RETENTION_DAYS` | Days of send-log history kept before the background sweep deletes rows — the main driver of `/data` growth over time. | `90` | `.env` | +| `SEND_LOG_RETENTION_DAYS` | Initial default for how many days of send-log history are kept before the background sweep deletes rows — the main driver of `/data` growth over time. After the first panel start, change retention on **Settings** (global administrator); the env value is only used to seed SQLite when the setting has never been saved. | `90` | `.env` | | `PANEL_SESSION_IDLE_DAYS` | Sliding idle timeout for the panel login session, in days. There is no absolute cap: an admin who keeps coming back stays signed in indefinitely. | `7` | `.env` | | `SELFPOST_DNS_RESOLVERS` | Comma-separated recursive resolvers the panel's PTR/SPF/DKIM/DMARC checks query directly (so they report what the internet sees, not what this host's stub resolver synthesises). | `1.1.1.1:53`, `8.8.8.8:53`, `9.9.9.9:53` when unset | `.env` | | `TRUSTED_PROXY_CIDR` | Comma-separated CIDRs (bare IPs allowed) of reverse proxies allowed to supply `X-Forwarded-For` for login, setup, and account-change rate-limiting. **Leave unset unless you know the exact address of your reverse proxy.** A wrong value lets a client spoof its rate-limit key by sending a forged `X-Forwarded-For` header — the panel trusts the last hop only when the TCP peer matches one of these CIDRs. Behind the default Apache host-network setup this is typically the Docker bridge gateway, e.g. `172.18.0.1`. | *(empty — XFF ignored)* | `.env` | @@ -347,8 +347,10 @@ shows after manual edits under `/data`. `/settings` changes the signed-in user's username and/or password. **Global administrators** also set the panel-wide default DMARC report address (`rua=`) offered when a domain doesn't set its own — see -[Domain-level DNS](#domain-level-dns-spf-dkim-dmarc). Application SASL logins -are separate and are not changed here. +[Domain-level DNS](#domain-level-dns-spf-dkim-dmarc) — and how many days +**Deliveries** rows are kept before the background sweep deletes them (7–365 +days; takes effect on the next six-hour prune cycle without a container +restart). Application SASL logins are separate and are not changed here. ### Users @@ -741,8 +743,9 @@ administrators see them here because they cannot open Mail queue. Under both sit the `mail.log` lines for its queue id: the connection to the receiving server, the server's reply, and the status that reply was filed as. Rows outlive `mail.log`, so an older message's lines may have rotated -away; the page says so. Retention is controlled by -`SEND_LOG_RETENTION_DAYS`. +away; the page says so. Retention is set on **Settings** (global +administrator); `SEND_LOG_RETENTION_DAYS` in `.env` is only the initial +default until it is changed there. ### Exporting and importing a single domain diff --git a/docs/plans/send-log-retention.md b/docs/plans/send-log-retention.md index b7b4e91..30f2361 100644 --- a/docs/plans/send-log-retention.md +++ b/docs/plans/send-log-retention.md @@ -1,6 +1,6 @@ # Plan: send-log-retention -**Status:** candidate +**Status:** done **Date:** 2026-08-17 **Version:** `1.x` MINOR; no schema migration required (uses existing `settings` table). @@ -129,11 +129,11 @@ Target version cut: **`1.5.0`** (MINOR). One commit per step; code only after roadmap status is **agreed**. See [development.md](../development.md) § Plan checklists. -- [ ] `GetSendLogRetentionDays` / `SetSetting` key `send_log_retention_days` (7–365) — **Opus** -- [ ] Bootstrap from `SEND_LOG_RETENTION_DAYS` when settings empty — **Opus** -- [ ] `logtail.retentionLoop`: read setting each prune cycle — **Opus** -- [ ] Settings card on `/settings` (`settings.html`) — **Sonnet** -- [ ] Remove hardcoded «ninety days» in handlers and templates — **Sonnet** -- [ ] Tests: save/load, range, loop without restart — **Sonnet** -- [ ] [guide.md](../guide.md) — **Sonnet** -- [ ] `go vet`, `go test` on touched packages — **Haiku** +- [x] `GetSendLogRetentionDays` / `SetSetting` key `send_log_retention_days` (7–365) — **Opus** +- [x] Bootstrap from `SEND_LOG_RETENTION_DAYS` when settings empty — **Opus** +- [x] `logtail.retentionLoop`: read setting each prune cycle — **Opus** +- [x] Settings card on `/settings` (`settings.html`) — **Sonnet** +- [x] Remove hardcoded «ninety days» in handlers and templates — **Sonnet** +- [x] Tests: save/load, range, loop without restart — **Sonnet** +- [x] [guide.md](../guide.md) — **Sonnet** +- [x] `go vet`, `go test` on touched packages — **Haiku** diff --git a/docs/roadmap.md b/docs/roadmap.md index 886cf32..da33293 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -28,7 +28,7 @@ in `git log` and [CHANGELOG.md](../CHANGELOG.md). | ID | Topic | Status | Progress | Plan | |---|---|---|---|---| -| send-log-retention | Send-log retention days in panel Settings | candidate | 0/8 | [plans/send-log-retention.md](plans/send-log-retention.md) | +| send-log-retention | Send-log retention days in panel Settings | done | 8/8 | [plans/send-log-retention.md](plans/send-log-retention.md) | | domain-stats-auto-ratelimit | 30-day send stats + auto level-2 rate limit | candidate | 0/11 | [plans/domain-stats-auto-ratelimit.md](plans/domain-stats-auto-ratelimit.md) | | contributing | `CONTRIBUTING.md` | candidate | — | — | | dmarc-reports | DMARC aggregate report ingestion and panel UI | candidate | 0/8 | [plans/dmarc-reports.md](plans/dmarc-reports.md) | diff --git a/internal/logtail/logtail.go b/internal/logtail/logtail.go index bc3ffad..a294c73 100644 --- a/internal/logtail/logtail.go +++ b/internal/logtail/logtail.go @@ -39,17 +39,15 @@ type StatusStore interface { // is a var so tests can shorten it. var pollInterval = time.Second +// retentionInterval is how often the retention sweep runs (also once at +// startup). It is a var so tests can shorten it. +var retentionInterval = 6 * time.Hour + // queueIDs lists the messages Postfix currently holds, for the reconcile sweep. // It is a var so tests can answer without a running Postfix. var queueIDs = postfix.QueueIDs const ( - // retentionInterval is how often the retention sweep runs (also once at - // startup). The window itself is configurable; the cadence need not be. - retentionInterval = 6 * time.Hour - // defaultRetentionDays applies when the configured value is unset/invalid - // (guide § Environment variables: SEND_LOG_RETENTION_DAYS). - defaultRetentionDays = 90 // reconcileInterval is how often the sweep compares stuck rows against the // Postfix queue, and reconcileGrace how long a row is left alone first. // The grace covers the ordinary lag between the milter writing the row and @@ -59,6 +57,10 @@ const ( reconcileGrace = 2 * time.Minute ) +// RetentionDays returns the send-log retention window in days. The log-tailer +// calls it on every prune cycle so a panel change takes effect without restart. +type RetentionDays func() int + // deliveryRe matches a Postfix delivery line and captures queue-id, recipient // and status, e.g. // @@ -98,11 +100,11 @@ func parseDelivery(line string) (queueID, recipient, status string, ok bool) { } // Run follows path and updates send-log statuses until ctx is cancelled, while -// a background sweep prunes rows older than retentionDays. Reading resumes at +// a background sweep prunes rows older than retention(). Reading resumes at // the offset the previous run persisted, so a restart parses the delivery lines // written while the panel was down. It returns nil on a clean shutdown. -func Run(ctx context.Context, path string, st StatusStore, retentionDays int) error { - go retentionLoop(ctx, st, retentionDays) +func Run(ctx context.Context, path string, st StatusStore, retention RetentionDays) error { + go retentionLoop(ctx, st, retention) // The reconcile sweep must not run against a backlog the tailer has not // read yet: on a restart the log holds the very lines that resolve the rows @@ -192,11 +194,12 @@ func reconcile(st StatusStore, cutoff time.Time) { } // retentionLoop prunes expired send-log rows immediately and then periodically. -func retentionLoop(ctx context.Context, st StatusStore, retentionDays int) { - if retentionDays <= 0 { - retentionDays = defaultRetentionDays - } +func retentionLoop(ctx context.Context, st StatusStore, retention RetentionDays) { prune := func() { + retentionDays := retention() + if retentionDays <= 0 { + retentionDays = store.SendLogRetentionDaysDefault + } cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays) n, err := st.DeleteSendLogBefore(cutoff) if err != nil { diff --git a/internal/logtail/logtail_test.go b/internal/logtail/logtail_test.go index 1273d48..fa42b01 100644 --- a/internal/logtail/logtail_test.go +++ b/internal/logtail/logtail_test.go @@ -212,7 +212,7 @@ func TestFollowTailsAndRotates(t *testing.T) { cs := &captureStore{} ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) - go func() { done <- Run(ctx, path, cs, 90) }() + go func() { done <- Run(ctx, path, cs, func() int { return 90 }) }() // Give follow() time to open at EOF (it seeks to end immediately on start, // so the seed line above is ignored), then append a delivery line. @@ -295,7 +295,7 @@ func startRun(t *testing.T, path string, cs *captureStore) func() { t.Helper() ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) - go func() { done <- Run(ctx, path, cs, 90) }() + go func() { done <- Run(ctx, path, cs, func() int { return 90 }) }() // follow() opens and seeks on start; give it a moment before the caller // appends, so the append is not raced by the initial open. time.Sleep(50 * time.Millisecond) diff --git a/internal/logtail/retention_test.go b/internal/logtail/retention_test.go new file mode 100644 index 0000000..78b7296 --- /dev/null +++ b/internal/logtail/retention_test.go @@ -0,0 +1,83 @@ +package logtail + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/mixeme/selfpost/internal/store" +) + +type retentionProbeStore struct { + mu sync.Mutex + cutoff time.Time +} + +func (s *retentionProbeStore) UpdateStatus(string, string, string) (int64, error) { + return 0, nil +} + +func (s *retentionProbeStore) ListQueuedOlderThan(time.Time) ([]store.QueuedDelivery, error) { + return nil, nil +} + +func (s *retentionProbeStore) DeleteSendLogBefore(cutoff time.Time) (int64, error) { + s.mu.Lock() + s.cutoff = cutoff + s.mu.Unlock() + return 0, nil +} + +func (s *retentionProbeStore) LogtailState(string) (store.LogtailState, bool, error) { + return store.LogtailState{}, false, nil +} + +func (s *retentionProbeStore) SaveLogtailState(string, store.LogtailState) error { + return nil +} + +func (s *retentionProbeStore) lastCutoff() time.Time { + s.mu.Lock() + defer s.mu.Unlock() + return s.cutoff +} + +// retentionLoop must re-read the configured window every cycle so a panel +// change takes effect without restarting the process. +func TestRetentionLoopUsesUpdatedValue(t *testing.T) { + old := retentionInterval + retentionInterval = 20 * time.Millisecond + t.Cleanup(func() { retentionInterval = old }) + + var days atomic.Int32 + days.Store(30) + + st := &retentionProbeStore{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go retentionLoop(ctx, st, func() int { return int(days.Load()) }) + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if age := time.Since(st.lastCutoff()); age < 31*24*time.Hour && age > 29*24*time.Hour { + break + } + time.Sleep(5 * time.Millisecond) + } + if age := time.Since(st.lastCutoff()); age < 29*24*time.Hour || age > 31*24*time.Hour { + t.Fatalf("first prune cutoff age %v, want about 30 days", age) + } + + days.Store(7) + deadline = time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if age := time.Since(st.lastCutoff()); age < 8*24*time.Hour && age > 6*24*time.Hour { + return + } + time.Sleep(5 * time.Millisecond) + } + age := time.Since(st.lastCutoff()) + t.Fatalf("second prune cutoff age %v, want about 7 days", age) +} diff --git a/internal/store/settings.go b/internal/store/settings.go index b1b4d42..22a61b4 100644 --- a/internal/store/settings.go +++ b/internal/store/settings.go @@ -4,8 +4,79 @@ import ( "database/sql" "errors" "fmt" + "strconv" ) +const ( + // SendLogRetentionDaysKey is the settings table key for delivery-journal + // retention (architecture.md § Persistence). + SendLogRetentionDaysKey = "send_log_retention_days" + SendLogRetentionDaysMin = 7 + SendLogRetentionDaysMax = 365 + // SendLogRetentionDaysDefault matches SEND_LOG_RETENTION_DAYS when unset. + SendLogRetentionDaysDefault = 90 +) + +// ErrSendLogRetentionDaysOutOfRange is returned when retention is outside the +// allowed panel range. +var ErrSendLogRetentionDaysOutOfRange = errors.New("send log retention days out of range") + +// ValidateSendLogRetentionDays checks the panel-allowed retention window. +func ValidateSendLogRetentionDays(days int) error { + if days < SendLogRetentionDaysMin || days > SendLogRetentionDaysMax { + return fmt.Errorf("%w: must be between %d and %d days", ErrSendLogRetentionDaysOutOfRange, SendLogRetentionDaysMin, SendLogRetentionDaysMax) + } + return nil +} + +func sendLogRetentionFallback(envDefault int) int { + if envDefault > 0 { + if err := ValidateSendLogRetentionDays(envDefault); err == nil { + return envDefault + } + } + return SendLogRetentionDaysDefault +} + +// GetSendLogRetentionDays returns the effective retention window. When the +// setting is missing or invalid, envDefault is used (guide § Environment +// variables: SEND_LOG_RETENTION_DAYS). +func (s *Store) GetSendLogRetentionDays(envDefault int) (int, error) { + raw, err := s.GetSetting(SendLogRetentionDaysKey) + if err != nil { + return 0, err + } + if raw == "" { + return sendLogRetentionFallback(envDefault), nil + } + days, err := strconv.Atoi(raw) + if err != nil || ValidateSendLogRetentionDays(days) != nil { + return sendLogRetentionFallback(envDefault), nil + } + return days, nil +} + +// SetSendLogRetentionDays persists the panel-configured retention window. +func (s *Store) SetSendLogRetentionDays(days int) error { + if err := ValidateSendLogRetentionDays(days); err != nil { + return err + } + return s.SetSetting(SendLogRetentionDaysKey, strconv.Itoa(days)) +} + +// EnsureSendLogRetentionDays seeds the setting from envDefault when it has +// never been written (first panel start after upgrade). +func (s *Store) EnsureSendLogRetentionDays(envDefault int) error { + raw, err := s.GetSetting(SendLogRetentionDaysKey) + if err != nil { + return err + } + if raw != "" { + return nil + } + return s.SetSendLogRetentionDays(sendLogRetentionFallback(envDefault)) +} + // GetSetting returns a settings value or empty string when missing. func (s *Store) GetSetting(key string) (string, error) { var value string diff --git a/internal/store/settings_test.go b/internal/store/settings_test.go new file mode 100644 index 0000000..1159a57 --- /dev/null +++ b/internal/store/settings_test.go @@ -0,0 +1,87 @@ +package store + +import ( + "errors" + "testing" +) + +func TestSendLogRetentionDaysSaveLoad(t *testing.T) { + st := openTestStore(t) + + if err := st.SetSendLogRetentionDays(45); err != nil { + t.Fatalf("SetSendLogRetentionDays: %v", err) + } + got, err := st.GetSendLogRetentionDays(90) + if err != nil { + t.Fatalf("GetSendLogRetentionDays: %v", err) + } + if got != 45 { + t.Fatalf("got %d, want 45", got) + } +} + +func TestSendLogRetentionDaysRejectsOutOfRange(t *testing.T) { + st := openTestStore(t) + + for _, days := range []int{0, 6, 366, -1} { + if err := st.SetSendLogRetentionDays(days); err == nil { + t.Fatalf("SetSendLogRetentionDays(%d) want error", days) + } else if !errors.Is(err, ErrSendLogRetentionDaysOutOfRange) { + t.Fatalf("SetSendLogRetentionDays(%d) = %v, want ErrSendLogRetentionDaysOutOfRange", days, err) + } + } +} + +func TestSendLogRetentionDaysBootstrapFromEnv(t *testing.T) { + st := openTestStore(t) + + if err := st.EnsureSendLogRetentionDays(120); err != nil { + t.Fatalf("EnsureSendLogRetentionDays: %v", err) + } + got, err := st.GetSendLogRetentionDays(90) + if err != nil { + t.Fatalf("GetSendLogRetentionDays: %v", err) + } + if got != 120 { + t.Fatalf("got %d, want 120", got) + } + + // Second call is a no-op. + if err := st.EnsureSendLogRetentionDays(30); err != nil { + t.Fatalf("EnsureSendLogRetentionDays again: %v", err) + } + got, err = st.GetSendLogRetentionDays(90) + if err != nil { + t.Fatalf("GetSendLogRetentionDays: %v", err) + } + if got != 120 { + t.Fatalf("after second ensure got %d, want 120", got) + } +} + +func TestSendLogRetentionDaysMissingUsesEnvDefault(t *testing.T) { + st := openTestStore(t) + + got, err := st.GetSendLogRetentionDays(60) + if err != nil { + t.Fatalf("GetSendLogRetentionDays: %v", err) + } + if got != 60 { + t.Fatalf("got %d, want 60", got) + } +} + +func TestSendLogRetentionDaysInvalidStoredFallsBack(t *testing.T) { + st := openTestStore(t) + + if err := st.SetSetting(SendLogRetentionDaysKey, "not-a-number"); err != nil { + t.Fatalf("SetSetting: %v", err) + } + got, err := st.GetSendLogRetentionDays(60) + if err != nil { + t.Fatalf("GetSendLogRetentionDays: %v", err) + } + if got != 60 { + t.Fatalf("got %d, want env fallback 60", got) + } +} diff --git a/internal/web/handlers/handlers.go b/internal/web/handlers/handlers.go index db6787a..f44c1f3 100644 --- a/internal/web/handlers/handlers.go +++ b/internal/web/handlers/handlers.go @@ -38,6 +38,9 @@ type Config struct { // InboundEnabled mirrors INBOUND_RELAY_ENABLE: the inbound panel and // routes exist only when this is true. InboundEnabled bool + // SendLogRetentionEnvDefault is SEND_LOG_RETENTION_DAYS at panel start; used + // as bootstrap and fallback when the settings row is missing or invalid. + SendLogRetentionEnvDefault int } // Handlers holds dependencies for authenticated panel routes. diff --git a/internal/web/handlers/handlers_monitor.go b/internal/web/handlers/handlers_monitor.go index 31e56ee..4292d79 100644 --- a/internal/web/handlers/handlers_monitor.go +++ b/internal/web/handlers/handlers_monitor.go @@ -108,10 +108,11 @@ func (h *Handlers) HandleDelivery(w http.ResponseWriter, r *http.Request) { row.Subject = mailhdr.DecodeSubject(row.Subject) logRows, logNote := h.deliveryLog(row) h.view.Render(w, http.StatusOK, "delivery", map[string]any{ - "Title": "SelfPost — delivery", - "User": auth.CurrentUser(r), - "Active": "deliveries", - "IsGlobal": p.IsGlobal(), + "Title": "SelfPost — delivery", + "User": auth.CurrentUser(r), + "Active": "deliveries", + "IsGlobal": p.IsGlobal(), + "SendLogRetentionDays": h.sendLogRetentionDays(), "Row": row, // The status in the panel's own badge vocabulary, so the headline reads // the same way as every other health signal in the panel. @@ -270,10 +271,8 @@ func (h *Handlers) deliveryLog(row store.SendLogRow) ([]deliveryLogRow, string) return nil, "Could not read the mail log." } if len(lines) == 0 { - // Send-log rows outlive mail.log: retention is ninety days by default - // and rotation keeps fourteen files, so an older message having nothing - // left to show is the normal end state, not a fault. - return nil, "Nothing for this queue id in the current mail log. Its lines have most likely been rotated away." + days := h.sendLogRetentionDays() + return nil, fmt.Sprintf("Nothing for this queue id in the current mail log. Its lines have most likely been rotated away (send-log rows are kept for %d days).", days) } out := make([]deliveryLogRow, len(lines)) for i, line := range lines { @@ -386,17 +385,18 @@ func (h *Handlers) sendLogData(r *http.Request) (map[string]any, error) { lastPage = int((total + sendLogPageSize - 1) / sendLogPageSize) } return map[string]any{ - "Rows": view, - "FilterDomains": domainNames, - "FilterApps": logins, - "FilterDomain": filter.Domain, - "FilterApp": filter.AppLogin, - "Page": page, - "PrevPage": page - 1, - "NextPage": page + 1, - "LastPage": lastPage, - "HasPrev": page > 1, - "HasNext": page < lastPage, + "Rows": view, + "FilterDomains": domainNames, + "FilterApps": logins, + "FilterDomain": filter.Domain, + "FilterApp": filter.AppLogin, + "Page": page, + "PrevPage": page - 1, + "NextPage": page + 1, + "LastPage": lastPage, + "HasPrev": page > 1, + "HasNext": page < lastPage, + "SendLogRetentionDays": h.sendLogRetentionDays(), }, nil } diff --git a/internal/web/handlers/handlers_monitor_test.go b/internal/web/handlers/handlers_monitor_test.go index 15db362..cd39ec9 100644 --- a/internal/web/handlers/handlers_monitor_test.go +++ b/internal/web/handlers/handlers_monitor_test.go @@ -191,12 +191,19 @@ func TestDeliveryPageKeepsAnUnstampedLogLineWhole(t *testing.T) { // queue at all. Neither is a fault, so neither may render as an error. func TestDeliveryPageExplainsAnEmptyDeliveryLog(t *testing.T) { h, row := serverWithDelivery(t) + h.cfg.SendLogRetentionEnvDefault = 60 + if err := h.store.SetSendLogRetentionDays(60); err != nil { + t.Fatalf("SetSendLogRetentionDays: %v", err) + } h.cfg.MailLogPath = filepath.Join(t.TempDir(), "mail.log") // never created out := getBody(t, h.HandleDelivery, "/deliveries/"+itoa(row.ID)) if !strings.Contains(out, "rotated away") { t.Errorf("delivery page does not explain the empty delivery log:\n%s", out) } + if !strings.Contains(out, "kept for 60 days") { + t.Errorf("delivery page does not show configured retention:\n%s", out) + } if strings.Contains(out, `class="error"`) || strings.Contains(out, "Could not read the mail log") { t.Errorf("an aged-out delivery log is reported as a failure:\n%s", out) } diff --git a/internal/web/handlers/handlers_settings.go b/internal/web/handlers/handlers_settings.go index 210cb8e..194112f 100644 --- a/internal/web/handlers/handlers_settings.go +++ b/internal/web/handlers/handlers_settings.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net/http" + "strconv" "strings" "time" @@ -29,7 +30,7 @@ func (h *Handlers) HandleSettings(w http.ResponseWriter, r *http.Request) { http.Error(w, "internal error", http.StatusInternalServerError) return } - h.renderSettings(w, r, http.StatusOK, "", u.Username, u.DMARCReportEmail, p.IsGlobal()) + h.renderSettings(w, r, http.StatusOK, "", u.Username, u.DMARCReportEmail, h.sendLogRetentionDays(), p.IsGlobal()) case http.MethodPost: h.submitSettings(w, r) default: @@ -38,7 +39,7 @@ func (h *Handlers) HandleSettings(w http.ResponseWriter, r *http.Request) { } } -func (h *Handlers) renderSettings(w http.ResponseWriter, r *http.Request, status int, formErr, formUsername, formDMARCEmail string, showDMARC bool) { +func (h *Handlers) renderSettings(w http.ResponseWriter, r *http.Request, status int, formErr, formUsername, formDMARCEmail string, formSendLogRetentionDays int, showDMARC bool) { var reportAuth dnscheck.Result if showDMARC && formDMARCEmail != "" { if hub := dnscheck.EmailDomain(formDMARCEmail); hub != "" { @@ -52,6 +53,7 @@ func (h *Handlers) renderSettings(w http.ResponseWriter, r *http.Request, status data["Active"] = "settings" data["FormUsername"] = formUsername data["FormDMARCEmail"] = formDMARCEmail + data["FormSendLogRetentionDays"] = formSendLogRetentionDays data["ShowDMARC"] = showDMARC data["ReportAuthName"] = dnscheck.ReportAuthRecordName(dnscheck.EmailDomain(formDMARCEmail)) data["ReportAuthExample"] = dnscheck.ReportAuthExample() @@ -74,6 +76,8 @@ func settingsFlash(r *http.Request) string { return "Username and password changed. Any other signed-in sessions were signed out." case "email": return "DMARC report address updated." + case "retention": + return "Send log retention updated." case "username-email": return "Username and DMARC report address updated." case "password-email": @@ -86,23 +90,23 @@ func settingsFlash(r *http.Request) string { } func (h *Handlers) submitSettings(w http.ResponseWriter, r *http.Request) { - if !h.auth.AllowLoginAttempt(r) { - p, _ := h.principal(r) - h.renderSettings(w, r, http.StatusTooManyRequests, - "Too many attempts. Please wait and try again.", auth.CurrentUser(r), "", p.IsGlobal()) - return - } - if err := r.ParseForm(); err != nil { - p, _ := h.principal(r) - h.renderSettings(w, r, http.StatusBadRequest, "Invalid form submission.", auth.CurrentUser(r), "", p.IsGlobal()) - return - } - p, ok := h.principal(r) if !ok { http.Error(w, "internal error", http.StatusInternalServerError) return } + currentRetention := h.sendLogRetentionDays() + + if !h.auth.AllowLoginAttempt(r) { + h.renderSettings(w, r, http.StatusTooManyRequests, + "Too many attempts. Please wait and try again.", auth.CurrentUser(r), "", currentRetention, p.IsGlobal()) + return + } + if err := r.ParseForm(); err != nil { + h.renderSettings(w, r, http.StatusBadRequest, "Invalid form submission.", auth.CurrentUser(r), "", currentRetention, p.IsGlobal()) + return + } + user, err := h.store.GetUser(p.ID) if err != nil { logf("panel: settings: get user failed: %v", err) @@ -115,49 +119,66 @@ func (h *Handlers) submitSettings(w http.ResponseWriter, r *http.Request) { password := r.PostFormValue("new_password") confirm := r.PostFormValue("new_password_confirm") dmarcEmail := strings.TrimSpace(r.PostFormValue("dmarc_report_email")) + formRetention := currentRetention if !p.IsGlobal() { dmarcEmail = user.DMARCReportEmail + } else if raw := strings.TrimSpace(r.PostFormValue("send_log_retention_days")); raw != "" { + parsed, err := strconv.Atoi(raw) + if err != nil { + h.renderSettings(w, r, http.StatusBadRequest, "Send log retention must be a whole number of days.", username, dmarcEmail, currentRetention, true) + return + } + formRetention = parsed } if username == "" { username = user.Username } if err := bcrypt.CompareHashAndPassword([]byte(user.PasswordHash), []byte(current)); err != nil { - h.renderSettings(w, r, http.StatusUnauthorized, "Current password is incorrect.", username, dmarcEmail, p.IsGlobal()) + h.renderSettings(w, r, http.StatusUnauthorized, "Current password is incorrect.", username, dmarcEmail, formRetention, p.IsGlobal()) return } renaming := username != user.Username if renaming { if err := validate.Username(username); err != nil { - h.renderSettings(w, r, http.StatusBadRequest, err.Error(), username, dmarcEmail, p.IsGlobal()) + h.renderSettings(w, r, http.StatusBadRequest, err.Error(), username, dmarcEmail, formRetention, p.IsGlobal()) return } } if p.IsGlobal() { if err := validate.Email(dmarcEmail); err != nil { - h.renderSettings(w, r, http.StatusBadRequest, err.Error(), username, dmarcEmail, true) + h.renderSettings(w, r, http.StatusBadRequest, err.Error(), username, dmarcEmail, formRetention, true) return } } emailChanging := p.IsGlobal() && dmarcEmail != user.DMARCReportEmail + retentionChanging := false + if p.IsGlobal() && formRetention != currentRetention { + if err := store.ValidateSendLogRetentionDays(formRetention); err != nil { + h.renderSettings(w, r, http.StatusBadRequest, err.Error(), username, dmarcEmail, formRetention, true) + return + } + retentionChanging = true + } + repassword := password != "" || confirm != "" if repassword { if password != confirm { - h.renderSettings(w, r, http.StatusBadRequest, "New passwords do not match.", username, dmarcEmail, p.IsGlobal()) + h.renderSettings(w, r, http.StatusBadRequest, "New passwords do not match.", username, dmarcEmail, formRetention, p.IsGlobal()) return } if err := validate.AdminPassword(password); err != nil { - h.renderSettings(w, r, http.StatusBadRequest, err.Error(), username, dmarcEmail, p.IsGlobal()) + h.renderSettings(w, r, http.StatusBadRequest, err.Error(), username, dmarcEmail, formRetention, p.IsGlobal()) return } } - if !renaming && !repassword && !emailChanging { + if !renaming && !repassword && !emailChanging && !retentionChanging { h.renderSettings(w, r, http.StatusBadRequest, - "Nothing to change: enter a new username, password, or DMARC report address.", username, dmarcEmail, p.IsGlobal()) + "Nothing to change: enter a new username, password, DMARC report address, or send log retention.", username, dmarcEmail, formRetention, p.IsGlobal()) return } @@ -167,25 +188,36 @@ func (h *Handlers) submitSettings(w http.ResponseWriter, r *http.Request) { if err != nil { logf("panel: settings: hashing password failed: %v", err) h.renderSettings(w, r, http.StatusInternalServerError, - "Internal error. Please try again.", username, dmarcEmail, p.IsGlobal()) + "Internal error. Please try again.", username, dmarcEmail, formRetention, p.IsGlobal()) return } hash = string(newHash) } - if err := h.store.UpdateUser(user.ID, username, hash, dmarcEmail); err != nil { - logf("panel: settings: update user failed: %v", err) - msg := "Could not save the changes. Please check the logs and try again." - if errors.Is(err, store.ErrUserNotFound) { - msg = "There is no user account to update." - } - if errors.Is(err, store.ErrUserExists) { - msg = "That username is already in use." - h.renderSettings(w, r, http.StatusConflict, msg, username, dmarcEmail, p.IsGlobal()) + if renaming || repassword || emailChanging { + if err := h.store.UpdateUser(user.ID, username, hash, dmarcEmail); err != nil { + logf("panel: settings: update user failed: %v", err) + msg := "Could not save the changes. Please check the logs and try again." + if errors.Is(err, store.ErrUserNotFound) { + msg = "There is no user account to update." + } + if errors.Is(err, store.ErrUserExists) { + msg = "That username is already in use." + h.renderSettings(w, r, http.StatusConflict, msg, username, dmarcEmail, formRetention, p.IsGlobal()) + return + } + h.renderSettings(w, r, http.StatusInternalServerError, msg, username, dmarcEmail, formRetention, p.IsGlobal()) + return + } + } + + if retentionChanging { + if err := h.store.SetSendLogRetentionDays(formRetention); err != nil { + logf("panel: settings: set send-log retention failed: %v", err) + h.renderSettings(w, r, http.StatusInternalServerError, + "Could not save send log retention. Please check the logs and try again.", username, dmarcEmail, formRetention, true) return } - h.renderSettings(w, r, http.StatusInternalServerError, msg, username, dmarcEmail, p.IsGlobal()) - return } if token, ok := h.auth.SessionToken(r); ok { @@ -197,25 +229,48 @@ func (h *Handlers) submitSettings(w http.ResponseWriter, r *http.Request) { } } - logf("panel: user %d settings updated (username: %t, password: %t, dmarc email: %t)", user.ID, renaming, repassword, emailChanging) - http.Redirect(w, r, "/settings?updated="+updatedFlag(renaming, repassword, emailChanging), http.StatusSeeOther) + logf("panel: user %d settings updated (username: %t, password: %t, dmarc email: %t, retention: %t)", user.ID, renaming, repassword, emailChanging, retentionChanging) + http.Redirect(w, r, "/settings?updated="+updatedFlag(renaming, repassword, emailChanging, retentionChanging), http.StatusSeeOther) } -func updatedFlag(renamed, repassword, emailChanged bool) string { - switch { - case renamed && repassword && emailChanged: +func updatedFlag(renamed, repassword, emailChanged, retentionChanged bool) string { + changed := 0 + if renamed { + changed++ + } + if repassword { + changed++ + } + if emailChanged { + changed++ + } + if retentionChanged { + changed++ + } + if changed > 1 { return "all" - case renamed && emailChanged: - return "username-email" - case repassword && emailChanged: - return "password-email" - case renamed && repassword: - return "both" + } + switch { case renamed: return "username" case repassword: return "password" - default: + case emailChanged: return "email" + default: + return "retention" } } + +// sendLogRetentionDays returns the effective delivery-journal retention window. +func (h *Handlers) sendLogRetentionDays() int { + days, err := h.store.GetSendLogRetentionDays(h.cfg.SendLogRetentionEnvDefault) + if err != nil { + logf("panel: send-log retention: %v", err) + if h.cfg.SendLogRetentionEnvDefault > 0 { + return h.cfg.SendLogRetentionEnvDefault + } + return store.SendLogRetentionDaysDefault + } + return days +} diff --git a/internal/web/handlers/handlers_settings_test.go b/internal/web/handlers/handlers_settings_test.go new file mode 100644 index 0000000..167a27c --- /dev/null +++ b/internal/web/handlers/handlers_settings_test.go @@ -0,0 +1,111 @@ +package handlers + +import ( + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "strings" + "testing" + + "github.com/mixeme/selfpost/internal/dnscheck" + "github.com/mixeme/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/web/auth" + "golang.org/x/crypto/bcrypt" +) + +func TestSettingsPageShowsSendLogRetention(t *testing.T) { + h, _ := settingsServer(t) + if err := h.store.SetSendLogRetentionDays(45); err != nil { + t.Fatalf("SetSendLogRetentionDays: %v", err) + } + + out := getBody(t, h.HandleSettings, "/settings") + for _, want := range []string{ + `id="deliveries-retention"`, + `name="send_log_retention_days"`, + `value="45"`, + "Send log retention", + } { + if !strings.Contains(out, want) { + t.Errorf("settings page missing %q:\n%s", want, out) + } + } +} + +func TestSubmitSettingsSavesSendLogRetention(t *testing.T) { + h, password := settingsServer(t) + + values := url.Values{ + "username": {"admin"}, + "current_password": {password}, + "send_log_retention_days": {"120"}, + } + req := httptest.NewRequest(http.MethodPost, "/settings", strings.NewReader(values.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req = auth.RequestWithPrincipal(req, globalPrincipal) + + rec := httptest.NewRecorder() + h.HandleSettings(rec, req) + if rec.Code != http.StatusSeeOther { + t.Fatalf("POST /settings = %d, want 303:\n%s", rec.Code, rec.Body.String()) + } + + got, err := h.store.GetSendLogRetentionDays(90) + if err != nil { + t.Fatalf("GetSendLogRetentionDays: %v", err) + } + if got != 120 { + t.Fatalf("retention = %d, want 120", got) + } +} + +func TestSubmitSettingsRejectsOutOfRangeRetention(t *testing.T) { + h, password := settingsServer(t) + + values := url.Values{ + "username": {"admin"}, + "current_password": {password}, + "send_log_retention_days": {"3"}, + } + req := httptest.NewRequest(http.MethodPost, "/settings", strings.NewReader(values.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req = auth.RequestWithPrincipal(req, globalPrincipal) + + rec := httptest.NewRecorder() + h.HandleSettings(rec, req) + if rec.Code != http.StatusBadRequest { + t.Fatalf("POST /settings = %d, want 400:\n%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "between 7 and 365") { + t.Errorf("expected range error in body:\n%s", rec.Body.String()) + } +} + +func settingsServer(t *testing.T) (*Handlers, string) { + t.Helper() + st, err := store.Open(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("open store: %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + + const password = "correct-password-here!" + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.MinCost) + if err != nil { + t.Fatalf("hash password: %v", err) + } + if _, err := st.CreateUser("admin", string(hash), store.RoleGlobal, nil); err != nil { + t.Fatalf("create user: %v", err) + } + + v := mustView(t) + a := auth.New(st, auth.Config{}, v, filepath.Join(t.TempDir(), "setup-token")) + return &Handlers{ + store: st, + view: v, + auth: a, + dns: dnscheck.New(nil), + cfg: Config{SendLogRetentionEnvDefault: 90}, + }, password +} diff --git a/internal/web/view/templates/deliveries.html b/internal/web/view/templates/deliveries.html index 7930e51..807c7ee 100644 --- a/internal/web/view/templates/deliveries.html +++ b/internal/web/view/templates/deliveries.html @@ -8,6 +8,10 @@ {{define "content"}}

Deliveries

+

Rows older than {{.SendLogRetentionDays}} days are deleted +automatically.{{if .IsGlobal}} Change retention on +Settings.{{end}}

+

Filter

diff --git a/internal/web/view/templates/settings.html b/internal/web/view/templates/settings.html index c83310a..9b3b15f 100644 --- a/internal/web/view/templates/settings.html +++ b/internal/web/view/templates/settings.html @@ -77,10 +77,23 @@
+
+

Deliveries

+

How long rows on Deliveries are + kept before the background sweep deletes them — the main driver of + /data growth over time. Does not affect mail.log + rotation (fourteen daily files).

+ + + +
+ -

Leave both new-password fields empty to change the username -or DMARC address only. Changing the password signs out every other session; -this one stays signed in.

+

Leave both new-password fields empty to change the username, +DMARC address, or send log retention only. Changing the password signs out +every other session; this one stays signed in.

{{else}}
diff --git a/internal/web/view/templates_test.go b/internal/web/view/templates_test.go index abe0c24..0b54af4 100644 --- a/internal/web/view/templates_test.go +++ b/internal/web/view/templates_test.go @@ -300,11 +300,15 @@ func TestSettingsPageDocumentsRateLimits(t *testing.T) { if !strings.Contains(src, `id="rate-limits"`) { t.Error("settings should include a sending rate limits card") } + if !strings.Contains(src, `id="deliveries-retention"`) { + t.Error("settings should include a send log retention card for global administrators") + } for _, want := range []string{ "RATE_LIMIT_MESSAGES_PER_IP", "Level 2 — domain", "trusted IPs", "{{.L1Messages}} messages / {{.L1Window}} seconds", + `name="send_log_retention_days"`, } { if !strings.Contains(src, want) { t.Errorf("settings rate limits card missing %q", want) diff --git a/internal/web/web.go b/internal/web/web.go index 3dd0a9f..aff8530 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -85,6 +85,8 @@ type Config struct { RetryPolicy postfix.RetryPolicy // InboundEnabled mirrors INBOUND_RELAY_ENABLE. InboundEnabled bool + // SendLogRetentionEnvDefault is SEND_LOG_RETENTION_DAYS at panel start. + SendLogRetentionEnvDefault int } // Server is the panel HTTP application. @@ -124,8 +126,9 @@ func New(st *store.Store, domains *domain.Service, apps *app.Service, inboundSvc JournalSocket: cfg.JournalSocket, RateLimitMessagesPerIP: cfg.RateLimitMessagesPerIP, RateLimitWindowSeconds: cfg.RateLimitWindowSeconds, - RetryPolicy: cfg.RetryPolicy, - InboundEnabled: cfg.InboundEnabled, + RetryPolicy: cfg.RetryPolicy, + InboundEnabled: cfg.InboundEnabled, + SendLogRetentionEnvDefault: cfg.SendLogRetentionEnvDefault, }, v, dnscheck.New(cfg.DNSResolvers), &health.MachineSampler{}, a) return &Server{cfg: cfg, auth: a, handlers: h}, nil }