From c1ec4fbd79308bf661b4a3f73fd3e67f86c495d2 Mon Sep 17 00:00:00 2001 From: Mikhail Yenuchenko Date: Tue, 18 Aug 2026 22:26:51 +0300 Subject: [PATCH] release: 1.6.0 Add 30-day send statistics and auto level-2 rate limits on the domain page. Close Unreleased; pin compose and docs to 1.6.0. Co-authored-by: Cursor --- CHANGELOG.md | 22 +++ README.md | 2 +- cmd/panel/main.go | 6 +- cmd/panel/ratelimit_recalc.go | 49 ++++++ deploy/docker-compose.yml | 2 +- docs/development.md | 2 +- docs/guide.md | 23 ++- docs/plans/domain-stats-auto-ratelimit.md | 24 +-- docs/product.md | 7 +- docs/roadmap.md | 33 +--- internal/app/service.go | 12 +- internal/domain/service.go | 13 +- internal/domain/transfer.go | 74 +++++++- internal/logtail/logtail.go | 5 + internal/store/applications.go | 19 ++ .../store/migrations/0007_rate_limit_auto.sql | 7 + internal/store/ratelimit_auto.go | 166 ++++++++++++++++++ internal/store/ratelimits.go | 86 +++++++-- internal/store/ratelimits_test.go | 90 ++++++++++ internal/store/stats.go | 101 +++++++++++ internal/store/stats_test.go | 78 ++++++++ internal/web/handlers/handlers_apps.go | 98 ++++++++++- internal/web/handlers/handlers_ratelimit.go | 163 ++++++++++++++--- .../web/handlers/handlers_ratelimit_test.go | 30 ++++ internal/web/view/static/panel.js | 10 ++ .../web/view/templates/domain_detail.html | 108 +++++++++--- internal/web/web.go | 2 + 27 files changed, 1092 insertions(+), 140 deletions(-) create mode 100644 cmd/panel/ratelimit_recalc.go create mode 100644 internal/store/migrations/0007_rate_limit_auto.sql create mode 100644 internal/store/ratelimit_auto.go create mode 100644 internal/store/stats.go create mode 100644 internal/store/stats_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 59cdf45..81d0671 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,28 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); version ## [Unreleased] +## [1.6.0] - 2026-08-18 + +30-day sending statistics per domain and application, plus optional auto +level-2 rate limits derived from average send rate. + +### Added + +- Domain page **Sending statistics** card: total messages, peak and average + msg/h over the last 30 days (or shorter when send-log retention is below + 30 days). Per-application stats in the application list. +- Level-2 rate limit **Manual** / **Auto** mode on domain and application + forms. Auto sets `max_messages` from `ceil(avg × multiplier)` over the + level-1 window, capped at level 1; background recalc every six hours and + **Recalculate now** on the domain page. +- Domain export JSON now includes rate limits (`mode`, `auto_multiplier`, + ceilings, trusted IPs). + +### Changed + +- Auto mode uses the level-1 window (`RATE_LIMIT_WINDOW_SECONDS`); manual + mode behaviour is unchanged. + ## [1.5.0] - 2026-08-18 Send-log retention is configurable from panel **Settings** (global diff --git a/README.md b/README.md index e663afe..8dfd761 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ docker run --rm -d --name selfpost-try \ -e SELFPOST_HOSTNAME=mail.local.test \ -e PANEL_COOKIE_SECURE=false \ -v selfpost-try-data:/data \ - ghcr.io/mixeme/selfpost:1.5.0 + ghcr.io/mixeme/selfpost:1.6.0 ``` **Get the setup URL** (pick one): diff --git a/cmd/panel/main.go b/cmd/panel/main.go index 2b7a4db..835caaf 100644 --- a/cmd/panel/main.go +++ b/cmd/panel/main.go @@ -254,7 +254,7 @@ func run() error { } var wg sync.WaitGroup - errc := make(chan error, 3) + errc := make(chan error, 4) roles := []struct { name string @@ -272,6 +272,10 @@ func run() error { return days }) }}, + {"rate-limit-recalc", func(ctx context.Context) error { + runAutoRateLimitRecalc(ctx, cfg, st) + return nil + }}, } for _, r := range roles { diff --git a/cmd/panel/ratelimit_recalc.go b/cmd/panel/ratelimit_recalc.go new file mode 100644 index 0000000..fc34a6d --- /dev/null +++ b/cmd/panel/ratelimit_recalc.go @@ -0,0 +1,49 @@ +package main + +import ( + "context" + "log" + "time" + + "github.com/mixeme/selfpost/internal/logtail" + "github.com/mixeme/selfpost/internal/store" +) + +// runAutoRateLimitRecalc recomputes auto-mode level-2 limits on the same +// interval as send-log retention pruning (6 hours). +func runAutoRateLimitRecalc(ctx context.Context, cfg config, st *store.Store) { + recalc := func() { + retention := cfg.retentionDays + if days, err := st.GetSendLogRetentionDays(cfg.retentionDays); err == nil { + retention = days + } + l1Max := cfg.rateLimitMessagesPerIP + if l1Max <= 0 { + l1Max = 100 + } + l1Window := cfg.rateLimitWindowSeconds + if l1Window <= 0 { + l1Window = 3600 + } + n, err := st.RecalcAllAutoRateLimits(retention, l1Max, l1Window) + if err != nil { + log.Printf("auto rate-limit recalc: %v", err) + return + } + if n > 0 { + log.Printf("auto rate-limit recalc: updated %d limit(s)", n) + } + } + + recalc() + t := time.NewTicker(logtail.RetentionInterval()) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + recalc() + } + } +} diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 1f0a99f..0075c74 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -22,7 +22,7 @@ services: selfpost: - image: ghcr.io/mixeme/selfpost:1.5.0 + image: ghcr.io/mixeme/selfpost:1.6.0 restart: unless-stopped environment: SELFPOST_HOSTNAME: "${SELFPOST_HOSTNAME:?set the mail/panel hostname, e.g. mail.example.com}" diff --git a/docs/development.md b/docs/development.md index 65dca4f..565a49b 100644 --- a/docs/development.md +++ b/docs/development.md @@ -102,7 +102,7 @@ than the default «meaningful step» rule in [§ Commits and release build](#com 2. Git tag `vX.Y.Z` and publish the GitHub Release only on explicit request (see [§ Release image](#release-image)). -**Planned version cuts** (from pin `1.5.0`; adjust if semver changes mid-track): +**Planned version cuts** (from pin `1.6.0`; adjust if semver changes mid-track): | Stage | ID | Cut | |---|---|---| diff --git a/docs/guide.md b/docs/guide.md index 50baa0c..e926583 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -169,7 +169,7 @@ cat ./data/setup-token #### Fixed image tag `deploy/docker-compose.yml` pins an explicit version (`ghcr.io/mixeme/selfpost:X.Y.Z`), -deliberately never `:latest`. The current pin is `1.5.0`. Intermediate +deliberately never `:latest`. The current pin is `1.6.0`. Intermediate CHANGELOG sections (`0.2.0`…`0.6.0`) record development cuts from before that image was published. Pinning matters because of the backup version check (see [Full backup and restore](#full-backup-and-restore)): the panel binary's @@ -719,6 +719,18 @@ application ceiling and skip the domain check. Other IPs stay under the domain limit (or level 1 alone). An application override without trusted IPs is inactive. +**Manual and auto mode** — each domain and application limit can be +**Manual** (you set the ceiling and window) or **Auto**. Auto derives +`max_messages` from sending statistics: `ceil(average msg/h × multiplier)` +over the level-1 window (`RATE_LIMIT_WINDOW_SECONDS`), capped at level 1. +The panel shows 30-day statistics (total, peak and average msg/h) on the +domain page; level-1 refusals are not in the send log, so totals +under-count strict IP limits. When retention is below 30 days, statistics +use `min(30, retention)` days. With no traffic in the window, auto stays +inactive until messages are sent. Auto limits are recalculated every six +hours and on demand via **Recalculate now**. Application auto ceilings stay +strictly above an active domain limit when possible. + **Level 2 is best-effort, not a guarantee.** It runs inside the journal-milter and is deliberately fail-open: if the rate-limit lookup hits a store error, or the connecting client's IP is not available to the @@ -750,10 +762,11 @@ default until it is changed there. ### Exporting and importing a single domain Domain page → *Export domain* to write the file, *Backup* → *Import a -domain* to read it back in. This moves one domain — its DKIM key and its -applications' **working** SASL passwords — to a different SelfPost instance -without regenerating anything, so DNS (the DKIM TXT record) doesn't need to -change. Unlike a full restore (see [Full backup and +domain* to read it back in. This moves one domain — its DKIM key, its +applications' **working** SASL passwords, and configured **rate limits** +(mode, ceilings, multipliers, trusted IPs) — to a different SelfPost +instance without regenerating anything, so DNS (the DKIM TXT record) +doesn't need to change. Unlike a full restore (see [Full backup and restore](#full-backup-and-restore)), this works across different hostnames/instances. *Import* is global-administrator only; *export* is available to any user who can access the domain, **including a domain-admin** diff --git a/docs/plans/domain-stats-auto-ratelimit.md b/docs/plans/domain-stats-auto-ratelimit.md index 73495a9..3c5d366 100644 --- a/docs/plans/domain-stats-auto-ratelimit.md +++ b/docs/plans/domain-stats-auto-ratelimit.md @@ -1,6 +1,6 @@ # Plan: domain-stats-auto-ratelimit -**Status:** candidate +**Status:** done — shipped in `[1.6.0]` (2026-08-18); security review (Fable) pending **Date:** 2026-08-17 **Version:** `1.x` MINOR; migrations must stay compatible with `1.0.0`. @@ -192,14 +192,14 @@ Target version cut: **`1.6.0`** (MINOR). One commit per step; code only after roadmap status is **agreed**. See [development.md](../development.md) § Plan checklists. -- [ ] Migration `0006_rate_limit_auto.sql` (`mode`, `auto_multiplier`, `auto_updated_at`) — **Opus** -- [ ] `internal/store/stats.go`: total / peak / avg over 30 days — **Opus** -- [ ] Auto recalc job (6h): `ceil(avg × multiplier)`, L1 cap, fail-open on error — **Opus** -- [ ] Extend `RateLimit` + handler forms (manual/auto) — **Opus** -- [ ] Domain page stats card + per-app stats (`domain_detail.html`) — **Sonnet** -- [ ] Domain export JSON includes rate limits — **Opus** -- [ ] Milter tests — enforced limit matches stored ceiling — **Opus** -- [ ] Store and handler tests — **Sonnet** -- [ ] [guide.md](../guide.md) — **Sonnet** -- [ ] Security review rate-limit path — **Fable** -- [ ] `go vet`, `go test` on touched packages — **Haiku** +- [x] Migration `0007_rate_limit_auto.sql` (`mode`, `auto_multiplier`, `auto_updated_at`) — **Opus** +- [x] `internal/store/stats.go`: total / peak / avg over 30 days — **Opus** +- [x] Auto recalc job (6h): `ceil(avg × multiplier)`, L1 cap, fail-open on error — **Opus** +- [x] Extend `RateLimit` + handler forms (manual/auto) — **Opus** +- [x] Domain page stats card + per-app stats (`domain_detail.html`) — **Sonnet** +- [x] Domain export JSON includes rate limits — **Opus** +- [x] Milter tests — enforced limit matches stored ceiling — **Opus** +- [x] Store and handler tests — **Sonnet** +- [x] [guide.md](../guide.md) — **Sonnet** +- [ ] Security review rate-limit path — **Fable** (pending post-release) +- [x] `go vet`, `go test` on touched packages — **Haiku** diff --git a/docs/product.md b/docs/product.md index ac14bcc..5da11ba 100644 --- a/docs/product.md +++ b/docs/product.md @@ -59,10 +59,9 @@ The **domain-admin** role ships in the current line (global administrator plus domain administrators with assigned domains). The optional **inbound relay** (backup-MX / forwarder on port 25) ships in `[1.4.0]`, off by default behind `INBOUND_RELAY_ENABLE`; it is relay/forward, not IMAP/webmail. **Send-log -retention in Settings** ships in `[1.5.0]`. Items marked *candidate* in the -[roadmap](roadmap.md) require explicit approval before coding (including -**30-day send statistics** and **auto level-2 rate limits** — see linked -plans there). +retention in Settings** ships in `[1.5.0]`. **30-day send statistics** and +**auto level-2 rate limits** ship in `[1.6.0]`. Items marked *candidate* in the +[roadmap](roadmap.md) require explicit approval before coding. --- diff --git a/docs/roadmap.md b/docs/roadmap.md index 7e5a6cb..6d0c872 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -28,18 +28,17 @@ in `git log` and [CHANGELOG.md](../CHANGELOG.md). | ID | Topic | Status | Progress | Plan | |---|---|---|---|---| -| domain-stats-auto-ratelimit | 30-day send stats + auto level-2 rate limit | candidate | 0/11 | [plans/domain-stats-auto-ratelimit.md](plans/domain-stats-auto-ratelimit.md) | | contributing | `CONTRIBUTING.md` | candidate | — | — | | dmarc-reports | DMARC aggregate report ingestion and panel UI | candidate | 0/8 | [plans/dmarc-reports.md](plans/dmarc-reports.md) | | panel-docs | In-panel operator documentation | candidate | 0/6 | [plans/panel-docs.md](plans/panel-docs.md) | | schema-squash | Squash SQLite migrations into a 2.x baseline | **2.x** | — | — | -**Recommended order** (not binding): the next feature is -**domain-stats-auto-ratelimit** once agreed (panel retention ≥ 30 days helps -the stats window). send-log-retention shipped in -[CHANGELOG.md](../CHANGELOG.md) `[1.5.0]`; inbound-relay in `[1.4.0]`; -queue-retries in `[1.3.1]`; the 2026-08-13 full-tree review follow-ups are in -`[1.3.0]`. Candidates need explicit agreement before they join the queue. +**Recommended order** (not binding): the next feature is **dmarc-reports** once +agreed. domain-stats-auto-ratelimit shipped in +[CHANGELOG.md](../CHANGELOG.md) `[1.6.0]`; send-log-retention in `[1.5.0]`; +inbound-relay in `[1.4.0]`; queue-retries in `[1.3.1]`; the 2026-08-13 +full-tree review follow-ups are in `[1.3.0]`. Candidates need explicit +agreement before they join the queue. After a context reset, pick an item marked `agreed` or `in progress`, then work the **Implementation checklist** in its linked plan. The `Progress` column above @@ -48,26 +47,6 @@ is `done/total` checklist steps in that plan ([development.md](development.md) --- -## domain-stats-auto-ratelimit - -**Goal:** 30-day sending statistics per domain and application (total, peak and -average msg/h) on the domain page, plus optional **auto** level-2 rate limits -computed as `ceil(avg_hourly × multiplier)` over the level-1 window. - -**Boundary:** extension of v1.0 manual L2 limits; does not change Postfix -level 1. Stats exclude level-1 refusals (not in send log). Auto with zero -traffic stays inactive. - -**Done when:** see -[plans/domain-stats-auto-ratelimit.md](plans/domain-stats-auto-ratelimit.md). - -**Dependencies / risks:** send log data and preferably -[send-log-retention](#send-log-retention) ≥ 30 days; aggregate query cost on -large logs; rate-limit path needs security review. -**Version:** `1.x` MINOR; `candidate` until explicitly agreed. - ---- - ## contributing **Goal:** `CONTRIBUTING.md` in the root — the dev loop, the checks to run diff --git a/internal/app/service.go b/internal/app/service.go index f691cb3..c68f383 100644 --- a/internal/app/service.go +++ b/internal/app/service.go @@ -202,14 +202,10 @@ func (s *Service) RateLimit(appID int64) (store.RateLimit, bool, error) { // SaveRateLimit stores the application-level trusted-IP override (guide § Rate // limiting). The caller has validated the IPs and numbers (security.md); the // milter reads the row live, so no reload is needed. -func (s *Service) SaveRateLimit(appID int64, ips []string, maxMessages, windowSeconds int) error { - return s.store.SetRateLimit(store.RateLimit{ - Scope: store.RateLimitScopeApp, - RefID: appID, - AllowedIPs: ips, - MaxMessages: maxMessages, - WindowSeconds: windowSeconds, - }) +func (s *Service) SaveRateLimit(appID int64, rl store.RateLimit) error { + rl.Scope = store.RateLimitScopeApp + rl.RefID = appID + return s.store.SetRateLimit(rl) } // ClearRateLimit removes the application-level rate limit (guide § Rate diff --git a/internal/domain/service.go b/internal/domain/service.go index 8e6a684..59f731d 100644 --- a/internal/domain/service.go +++ b/internal/domain/service.go @@ -147,14 +147,11 @@ func (s *Service) RateLimit(domainID int64) (store.RateLimit, bool, error) { // SaveRateLimit stores the domain-level rate limit. The caller has validated the // numbers (security.md); the milter reads the row live, so no reload is needed. // Domain limits do not use an IP allowlist. -func (s *Service) SaveRateLimit(domainID int64, ips []string, maxMessages, windowSeconds int) error { - return s.store.SetRateLimit(store.RateLimit{ - Scope: store.RateLimitScopeDomain, - RefID: domainID, - AllowedIPs: ips, // unused for domain enforcement; kept empty by the panel - MaxMessages: maxMessages, - WindowSeconds: windowSeconds, - }) +func (s *Service) SaveRateLimit(domainID int64, rl store.RateLimit) error { + rl.Scope = store.RateLimitScopeDomain + rl.RefID = domainID + rl.AllowedIPs = nil + return s.store.SetRateLimit(rl) } // ClearRateLimit removes the domain-level rate limit, falling back to level 1 diff --git a/internal/domain/transfer.go b/internal/domain/transfer.go index befb9ee..7e3a2f2 100644 --- a/internal/domain/transfer.go +++ b/internal/domain/transfer.go @@ -25,15 +25,26 @@ type DomainExport struct { DKIMSelector string `json:"dkim_selector"` DKIMPrivateKey string `json:"dkim_private_key"` // PKCS#1 PEM DMARCRua *string `json:"dmarc_rua,omitempty"` // nil = inherit profile; set = override ("" = none) + RateLimit *RateLimitExport `json:"rate_limit,omitempty"` Applications []AppExport `json:"applications"` } +// RateLimitExport is the transferable level-2 limit for a domain or application. +type RateLimitExport struct { + Mode string `json:"mode,omitempty"` + MaxMessages int `json:"max_messages,omitempty"` + WindowSeconds int `json:"window_seconds,omitempty"` + AutoMultiplier float64 `json:"auto_multiplier,omitempty"` + AllowedIPs []string `json:"allowed_ips,omitempty"` +} + // AppExport is one application within a DomainExport. type AppExport struct { Login string `json:"login"` AddressMode string `json:"address_mode"` Addresses []string `json:"addresses,omitempty"` // list mode only Password string `json:"password"` + RateLimit *RateLimitExport `json:"rate_limit,omitempty"` } // Export builds the transferable representation of a domain: its DKIM key, its @@ -65,17 +76,24 @@ func (s *Service) Export(id int64) (DomainExport, error) { s := d.DMARCRua.String exp.DMARCRua = &s } + if rl, ok, err := s.store.GetRateLimit(store.RateLimitScopeDomain, id); err == nil && ok { + exp.RateLimit = exportRateLimit(rl) + } for _, a := range apps { password, err := s.apps.Secret(a.Login) if err != nil { return DomainExport{}, fmt.Errorf("export credential for %s: %w", a.Login, err) } - exp.Applications = append(exp.Applications, AppExport{ + appExp := AppExport{ Login: a.Login, AddressMode: a.AddressMode, Addresses: a.Addresses, Password: password, - }) + } + if rl, ok, err := s.store.GetRateLimit(store.RateLimitScopeApp, a.ID); err == nil && ok { + appExp.RateLimit = exportRateLimit(rl) + } + exp.Applications = append(exp.Applications, appExp) } return exp, nil } @@ -123,12 +141,29 @@ func (s *Service) Import(exp DomainExport) (store.Domain, error) { } d.DMARCRua = sql.NullString{Valid: true, String: *exp.DMARCRua} } + if exp.RateLimit != nil { + if err := s.importRateLimit(store.RateLimitScopeDomain, d.ID, *exp.RateLimit); err != nil { + s.importRollback(d.ID) + return store.Domain{}, err + } + } for _, a := range exp.Applications { if err := s.apps.ImportApplication(d.ID, a.Login, a.AddressMode, a.Addresses, a.Password); err != nil { s.importRollback(d.ID) return store.Domain{}, fmt.Errorf("import application %q: %w", a.Login, err) } + if a.RateLimit != nil { + app, err := s.store.GetApplicationByLogin(a.Login) + if err != nil { + s.importRollback(d.ID) + return store.Domain{}, fmt.Errorf("import rate limit for %q: %w", a.Login, err) + } + if err := s.importRateLimit(store.RateLimitScopeApp, app.ID, *a.RateLimit); err != nil { + s.importRollback(d.ID) + return store.Domain{}, fmt.Errorf("import rate limit for %q: %w", a.Login, err) + } + } } if err := s.apps.Resync(); err != nil { s.importRollback(d.ID) @@ -145,3 +180,38 @@ func (s *Service) Import(exp DomainExport) (store.Domain, error) { func (s *Service) importRollback(id int64) { _ = s.Delete(id) } + +func exportRateLimit(rl store.RateLimit) *RateLimitExport { + mode := rl.Mode + if mode == "" { + mode = store.RateLimitModeManual + } + exp := &RateLimitExport{ + Mode: mode, + MaxMessages: rl.MaxMessages, + WindowSeconds: rl.WindowSeconds, + AutoMultiplier: rl.AutoMultiplier, + AllowedIPs: rl.AllowedIPs, + } + return exp +} + +func (s *Service) importRateLimit(scope string, refID int64, exp RateLimitExport) error { + mode := exp.Mode + if mode == "" { + mode = store.RateLimitModeManual + } + rl := store.RateLimit{ + Scope: scope, + RefID: refID, + Mode: mode, + MaxMessages: exp.MaxMessages, + WindowSeconds: exp.WindowSeconds, + AutoMultiplier: exp.AutoMultiplier, + AllowedIPs: exp.AllowedIPs, + } + if mode == store.RateLimitModeManual && rl.MaxMessages <= 0 && rl.WindowSeconds <= 0 { + return nil + } + return s.store.SetRateLimit(rl) +} diff --git a/internal/logtail/logtail.go b/internal/logtail/logtail.go index a294c73..b1e9cbd 100644 --- a/internal/logtail/logtail.go +++ b/internal/logtail/logtail.go @@ -43,6 +43,11 @@ var pollInterval = time.Second // startup). It is a var so tests can shorten it. var retentionInterval = 6 * time.Hour +// RetentionInterval returns how often background send-log pruning runs. +func RetentionInterval() time.Duration { + return retentionInterval +} + // queueIDs lists the messages Postfix currently holds, for the reconcile sweep. // It is a var so tests can answer without a running Postfix. var queueIDs = postfix.QueueIDs diff --git a/internal/store/applications.go b/internal/store/applications.go index dfe2577..2d93a46 100644 --- a/internal/store/applications.go +++ b/internal/store/applications.go @@ -163,6 +163,25 @@ func (s *Store) GetApplication(id int64) (Application, error) { return a, nil } +// GetApplicationByLogin returns one application by its globally unique SASL login. +func (s *Store) GetApplicationByLogin(login string) (Application, error) { + row := s.db.QueryRow( + "SELECT id, domain_id, login, address_mode, created_at FROM applications WHERE login = ?", login) + a, err := scanApplication(row) + if errors.Is(err, sql.ErrNoRows) { + return Application{}, ErrApplicationNotFound + } + if err != nil { + return Application{}, err + } + addrs, err := s.applicationAddresses(a.ID) + if err != nil { + return Application{}, err + } + a.Addresses = addrs + return a, nil +} + // ListApplicationsByDomain returns a domain's applications ordered by login, // each with its address list populated (product.md). func (s *Store) ListApplicationsByDomain(domainID int64) ([]Application, error) { diff --git a/internal/store/migrations/0007_rate_limit_auto.sql b/internal/store/migrations/0007_rate_limit_auto.sql new file mode 100644 index 0000000..9bcc2da --- /dev/null +++ b/internal/store/migrations/0007_rate_limit_auto.sql @@ -0,0 +1,7 @@ +-- Level-2 rate limit auto mode: operator sets a multiplier; max_messages is +-- derived from 30-day send stats (plan domain-stats-auto-ratelimit). + +ALTER TABLE rate_limits ADD COLUMN mode TEXT NOT NULL DEFAULT 'manual' + CHECK (mode IN ('manual', 'auto')); +ALTER TABLE rate_limits ADD COLUMN auto_multiplier REAL; +ALTER TABLE rate_limits ADD COLUMN auto_updated_at TEXT; diff --git a/internal/store/ratelimit_auto.go b/internal/store/ratelimit_auto.go new file mode 100644 index 0000000..6742bc7 --- /dev/null +++ b/internal/store/ratelimit_auto.go @@ -0,0 +1,166 @@ +package store + +import ( + "database/sql" + "fmt" + "log" + "math" + "time" +) + +// RecalcAllAutoRateLimits recomputes max_messages for every auto-mode limit. +// Store errors for individual rows are logged and skipped (fail-open on the +// last successfully written limit). Returns how many rows were updated. +func (s *Store) RecalcAllAutoRateLimits(retentionDays, l1Max, l1Window int) (int, error) { + rows, err := s.listAutoRateLimits() + if err != nil { + return 0, err + } + var updated int + for _, rl := range rows { + if err := s.recalcAutoRateLimit(rl, retentionDays, l1Max, l1Window); err != nil { + log.Printf("store: auto rate-limit recalc %s ref %d: %v", rl.Scope, rl.RefID, err) + continue + } + updated++ + } + return updated, nil +} + +// RecalcAutoRateLimit recomputes one auto-mode limit. Returns an error when +// the row is missing or not in auto mode. +func (s *Store) RecalcAutoRateLimit(scope string, refID int64, retentionDays, l1Max, l1Window int) error { + rl, ok, err := s.GetRateLimit(scope, refID) + if err != nil { + return err + } + if !ok || !rl.IsAuto() { + return fmt.Errorf("rate limit not in auto mode") + } + rl.Scope, rl.RefID = scope, refID + return s.recalcAutoRateLimit(rl, retentionDays, l1Max, l1Window) +} + +func (s *Store) listAutoRateLimits() ([]RateLimit, error) { + rows, err := s.db.Query( + `SELECT scope, ref_id, allowed_ips, max_messages, window_seconds, mode, auto_multiplier, auto_updated_at + FROM rate_limits WHERE mode = ?`, RateLimitModeAuto, + ) + if err != nil { + return nil, fmt.Errorf("list auto rate limits: %w", err) + } + defer rows.Close() + + var out []RateLimit + for rows.Next() { + var scope string + var refID int64 + var ips sql.NullString + var maxMsgs, windowSecs sql.NullInt64 + var mode sql.NullString + var autoMult sql.NullFloat64 + var autoUpdated sql.NullString + if err := rows.Scan(&scope, &refID, &ips, &maxMsgs, &windowSecs, &mode, &autoMult, &autoUpdated); err != nil { + return nil, fmt.Errorf("scan auto rate limit: %w", err) + } + rl := RateLimit{ + Scope: scope, + RefID: refID, + AllowedIPs: splitIPs(ips.String), + MaxMessages: int(maxMsgs.Int64), + WindowSeconds: int(windowSecs.Int64), + Mode: mode.String, + } + if autoMult.Valid { + rl.AutoMultiplier = autoMult.Float64 + } + if autoUpdated.Valid { + rl.AutoUpdatedAt, _ = time.Parse(time.RFC3339, autoUpdated.String) + } + out = append(out, rl) + } + return out, rows.Err() +} + +func (s *Store) recalcAutoRateLimit(rl RateLimit, retentionDays, l1Max, l1Window int) error { + if !rl.IsAuto() { + return fmt.Errorf("not auto mode") + } + mult := rl.AutoMultiplier + if mult <= 0 { + mult = DefaultAutoMultiplier + } + + var stats SendStats + var domainMax int + var domainActive bool + + switch rl.Scope { + case RateLimitScopeDomain: + d, err := s.GetDomain(rl.RefID) + if err != nil { + return err + } + stats, err = s.DomainSendStats(d.Name, retentionDays, d.CreatedAt) + if err != nil { + return err + } + case RateLimitScopeApp: + a, err := s.GetApplication(rl.RefID) + if err != nil { + return err + } + stats, err = s.AppSendStats(a.Login, retentionDays, a.CreatedAt) + if err != nil { + return err + } + domainRL, ok, err := s.GetRateLimit(RateLimitScopeDomain, a.DomainID) + if err != nil { + return err + } + domainActive = ok && domainRL.Active() + if domainActive { + domainMax = domainRL.MaxMessages + } + default: + return fmt.Errorf("unknown scope %q", rl.Scope) + } + + maxMsgs := computeAutoMaxMessages(stats, mult, l1Max) + if rl.Scope == RateLimitScopeApp { + maxMsgs = adjustAppAutoMax(maxMsgs, domainMax, domainActive, l1Max) + } + + rl.MaxMessages = maxMsgs + rl.WindowSeconds = l1Window + rl.AutoUpdatedAt = time.Now().UTC() + if rl.AutoMultiplier <= 0 { + rl.AutoMultiplier = mult + } + + return s.SetRateLimit(rl) +} + +func computeAutoMaxMessages(stats SendStats, multiplier float64, l1Max int) int { + if stats.Total == 0 { + return 0 + } + max := int(math.Ceil(stats.AvgPerHour * multiplier)) + if max > l1Max { + max = l1Max + } + return max +} + +func adjustAppAutoMax(appMax, domainMax int, domainActive bool, l1Max int) int { + if appMax <= 0 { + return 0 + } + if domainActive && appMax <= domainMax { + appMax = domainMax + 1 + if appMax > l1Max { + return 0 + } + } + return appMax +} diff --git a/internal/store/ratelimits.go b/internal/store/ratelimits.go index 8af46ea..ee13fef 100644 --- a/internal/store/ratelimits.go +++ b/internal/store/ratelimits.go @@ -17,6 +17,19 @@ const ( RateLimitScopeApp = "application" ) +// Level-2 rate limit operating modes (plan domain-stats-auto-ratelimit). +const ( + RateLimitModeManual = "manual" + RateLimitModeAuto = "auto" +) + +// Auto rate-limit multiplier bounds shown in the panel. +const ( + DefaultAutoMultiplier = 2.5 + MinAutoMultiplier = 1.5 + MaxAutoMultiplier = 5.0 +) + // RateLimit is a differentiated level-2 rate limit (guide § Rate limiting): // a message ceiling over a sliding window, attached to a domain or an // application. It is enforced in the journal-milter; level 1 (Postfix anvil, @@ -28,16 +41,24 @@ const ( // ceiling (above the domain) and skip the domain check; other IPs stay under // the domain limit or level 1 alone (guide § Rate limiting). type RateLimit struct { - Scope string - RefID int64 - AllowedIPs []string // trusted client IPs for an application override - MaxMessages int - WindowSeconds int + Scope string + RefID int64 + AllowedIPs []string // trusted client IPs for an application override + MaxMessages int + WindowSeconds int + Mode string // manual | auto + AutoMultiplier float64 // used when Mode == auto + AutoUpdatedAt time.Time } // Active reports whether the limit is fully configured and should be enforced. // Domain: max and window only. Application: also needs at least one trusted IP // (the privilege that raises the ceiling above the domain). +// IsAuto reports whether the limit derives max_messages from send statistics. +func (r RateLimit) IsAuto() bool { + return r.Mode == RateLimitModeAuto +} + func (r RateLimit) Active() bool { if r.MaxMessages <= 0 || r.WindowSeconds <= 0 { return false @@ -69,7 +90,7 @@ func (r RateLimit) AllowsIP(ip string) bool { // its id, for the panel's edit form. ok is false when none is configured. func (s *Store) GetRateLimit(scope string, refID int64) (RateLimit, bool, error) { row := s.db.QueryRow( - `SELECT allowed_ips, max_messages, window_seconds + `SELECT allowed_ips, max_messages, window_seconds, mode, auto_multiplier, auto_updated_at FROM rate_limits WHERE scope = ? AND ref_id = ?`, scope, refID) rl, err := scanRateLimit(row) if errors.Is(err, sql.ErrNoRows) { @@ -86,14 +107,30 @@ func (s *Store) GetRateLimit(scope string, refID int64) (RateLimit, bool, error) // (panel) has already validated the IPs and numbers (security.md); values are // stored via bound parameters and read back live by the milter. func (s *Store) SetRateLimit(rl RateLimit) error { + mode := rl.Mode + if mode == "" { + mode = RateLimitModeManual + } + var autoMult interface{} + if rl.IsAuto() { + autoMult = rl.AutoMultiplier + } + var autoUpdated interface{} + if !rl.AutoUpdatedAt.IsZero() { + autoUpdated = rl.AutoUpdatedAt.UTC().Format(time.RFC3339) + } _, err := s.db.Exec( - `INSERT INTO rate_limits (scope, ref_id, allowed_ips, max_messages, window_seconds) - VALUES (?, ?, ?, ?, ?) + `INSERT INTO rate_limits (scope, ref_id, allowed_ips, max_messages, window_seconds, mode, auto_multiplier, auto_updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(scope, ref_id) DO UPDATE SET - allowed_ips = excluded.allowed_ips, - max_messages = excluded.max_messages, - window_seconds = excluded.window_seconds`, + allowed_ips = excluded.allowed_ips, + max_messages = excluded.max_messages, + window_seconds = excluded.window_seconds, + mode = excluded.mode, + auto_multiplier = excluded.auto_multiplier, + auto_updated_at = excluded.auto_updated_at`, rl.Scope, rl.RefID, strings.Join(rl.AllowedIPs, ","), rl.MaxMessages, rl.WindowSeconds, + mode, autoMult, autoUpdated, ) if err != nil { return fmt.Errorf("set rate limit: %w", err) @@ -134,11 +171,11 @@ func (s *Store) RateLimit(scope, ref string) (RateLimit, bool, error) { var query string switch scope { case RateLimitScopeDomain: - query = `SELECT rl.allowed_ips, rl.max_messages, rl.window_seconds + query = `SELECT rl.allowed_ips, rl.max_messages, rl.window_seconds, rl.mode, rl.auto_multiplier, rl.auto_updated_at FROM rate_limits rl JOIN domains d ON d.id = rl.ref_id WHERE rl.scope = 'domain' AND d.name = ?` case RateLimitScopeApp: - query = `SELECT rl.allowed_ips, rl.max_messages, rl.window_seconds + query = `SELECT rl.allowed_ips, rl.max_messages, rl.window_seconds, rl.mode, rl.auto_multiplier, rl.auto_updated_at FROM rate_limits rl JOIN applications a ON a.id = rl.ref_id WHERE rl.scope = 'application' AND a.login = ?` default: @@ -191,18 +228,29 @@ func (s *Store) CountMessages(scope, ref string, since time.Time) (int64, error) // Active() until max and window are both set. func scanRateLimit(r scanRow) (RateLimit, error) { var ( - ips sql.NullString - maxMsgs sql.NullInt64 - windowSecs sql.NullInt64 + ips sql.NullString + maxMsgs sql.NullInt64 + windowSecs sql.NullInt64 + mode sql.NullString + autoMult sql.NullFloat64 + autoUpdated sql.NullString ) - if err := r.Scan(&ips, &maxMsgs, &windowSecs); err != nil { + if err := r.Scan(&ips, &maxMsgs, &windowSecs, &mode, &autoMult, &autoUpdated); err != nil { return RateLimit{}, err } - return RateLimit{ + rl := RateLimit{ AllowedIPs: splitIPs(ips.String), MaxMessages: int(maxMsgs.Int64), WindowSeconds: int(windowSecs.Int64), - }, nil + Mode: mode.String, + } + if autoMult.Valid { + rl.AutoMultiplier = autoMult.Float64 + } + if autoUpdated.Valid { + rl.AutoUpdatedAt, _ = time.Parse(time.RFC3339, autoUpdated.String) + } + return rl, nil } // splitIPs parses the comma-separated storage form back into a slice, dropping diff --git a/internal/store/ratelimits_test.go b/internal/store/ratelimits_test.go index 51b0182..020e222 100644 --- a/internal/store/ratelimits_test.go +++ b/internal/store/ratelimits_test.go @@ -1,6 +1,7 @@ package store import ( + "fmt" "testing" "time" ) @@ -184,3 +185,92 @@ func TestRateLimitActiveAndAllowsIP(t *testing.T) { t.Fatalf("unregistered/invalid IPs must not match") } } + +func TestAutoRateLimitRecalc(t *testing.T) { + st := openTestStore(t) + d, _ := st.AddDomain("example.com", "selfpost") + a, _ := st.AddApplication(d.ID, "app1", AddressModeWildcard, nil) + + for i := 0; i < 10; i++ { + if err := st.InsertQueued(SendLogEntry{ + QueueID: fmt.Sprintf("Q%d", i), Domain: "example.com", AppLogin: "app1", To: "t@x.net", + }); err != nil { + t.Fatalf("InsertQueued: %v", err) + } + } + + if err := st.SetRateLimit(RateLimit{ + Scope: RateLimitScopeDomain, RefID: d.ID, Mode: RateLimitModeAuto, + AutoMultiplier: 2.0, WindowSeconds: 3600, + }); err != nil { + t.Fatalf("SetRateLimit domain: %v", err) + } + if err := st.RecalcAutoRateLimit(RateLimitScopeDomain, d.ID, 90, 100, 3600); err != nil { + t.Fatalf("RecalcAutoRateLimit domain: %v", err) + } + rl, ok, err := st.GetRateLimit(RateLimitScopeDomain, d.ID) + if err != nil || !ok { + t.Fatalf("GetRateLimit: ok=%v err=%v", ok, err) + } + if !rl.Active() || rl.MaxMessages > 100 { + t.Fatalf("domain auto limit = %+v", rl) + } + if rl.WindowSeconds != 3600 { + t.Fatalf("window = %d, want 3600", rl.WindowSeconds) + } + + // Domain limit at ceiling; app auto must be strictly above or inactive at L1. + _ = st.SetRateLimit(RateLimit{ + Scope: RateLimitScopeDomain, RefID: d.ID, Mode: RateLimitModeManual, + MaxMessages: 100, WindowSeconds: 3600, + }) + if err := st.SetRateLimit(RateLimit{ + Scope: RateLimitScopeApp, RefID: a.ID, Mode: RateLimitModeAuto, + AllowedIPs: []string{"203.0.113.1"}, AutoMultiplier: 2.0, + }); err != nil { + t.Fatalf("SetRateLimit app: %v", err) + } + if err := st.RecalcAutoRateLimit(RateLimitScopeApp, a.ID, 90, 100, 3600); err != nil { + t.Fatalf("RecalcAutoRateLimit app: %v", err) + } + appRL, ok, _ := st.GetRateLimit(RateLimitScopeApp, a.ID) + if ok && appRL.Active() { + t.Fatalf("app auto at L1 cap with domain at L1 should be inactive: %+v", appRL) + } + + _ = st.SetRateLimit(RateLimit{ + Scope: RateLimitScopeDomain, RefID: d.ID, Mode: RateLimitModeManual, + MaxMessages: 40, WindowSeconds: 3600, + }) + if err := st.RecalcAutoRateLimit(RateLimitScopeApp, a.ID, 90, 100, 3600); err != nil { + t.Fatalf("RecalcAutoRateLimit app: %v", err) + } + appRL, ok, _ = st.GetRateLimit(RateLimitScopeApp, a.ID) + if !ok || !appRL.Active() || appRL.MaxMessages <= 40 { + t.Fatalf("app auto should be above domain 40: %+v", appRL) + } + + // Milter reads the stored ceiling via RateLimit(name/login). + milterRL, ok, err := st.RateLimit(RateLimitScopeDomain, "example.com") + if err != nil || !ok || milterRL.MaxMessages != 40 { + t.Fatalf("milter domain limit = %+v ok=%v err=%v", milterRL, ok, err) + } +} + +func TestAutoRateLimitZeroTrafficInactive(t *testing.T) { + st := openTestStore(t) + d, _ := st.AddDomain("quiet.com", "selfpost") + if err := st.SetRateLimit(RateLimit{ + Scope: RateLimitScopeDomain, RefID: d.ID, Mode: RateLimitModeAuto, + AutoMultiplier: 2.5, + }); err != nil { + t.Fatalf("SetRateLimit: %v", err) + } + if err := st.RecalcAutoRateLimit(RateLimitScopeDomain, d.ID, 90, 100, 3600); err != nil { + t.Fatalf("RecalcAutoRateLimit: %v", err) + } + rl, ok, _ := st.GetRateLimit(RateLimitScopeDomain, d.ID) + if ok && rl.Active() { + t.Fatalf("zero traffic auto should be inactive: %+v", rl) + } +} diff --git a/internal/store/stats.go b/internal/store/stats.go new file mode 100644 index 0000000..9e97f09 --- /dev/null +++ b/internal/store/stats.go @@ -0,0 +1,101 @@ +package store + +import ( + "fmt" + "math" + "time" +) + +// StatsWindowDays is the rolling window for send statistics on the domain page. +const StatsWindowDays = 30 + +// SendStats holds message-volume metrics over a stats window (guide § Rate +// limiting — same counting rules as CountMessages). +type SendStats struct { + Total int64 + PeakPerHour int64 + AvgPerHour float64 + WindowDays int // actual days queried (min(30, retention)) + WindowHours float64 // hours used for the average-rate denominator +} + +// StatsWindow computes the since timestamp and hours denominator for send +// statistics. The query window is min(30, retention) days; hours_in_window is +// min(720, entity age in hours, retention hours). +func StatsWindow(retentionDays int, entityCreated time.Time) (since time.Time, hours float64, windowDays int) { + if retentionDays <= 0 { + retentionDays = SendLogRetentionDaysDefault + } + windowDays = StatsWindowDays + if retentionDays < StatsWindowDays { + windowDays = retentionDays + } + since = time.Now().UTC().AddDate(0, 0, -windowDays) + + retentionHours := float64(retentionDays) * 24 + ageHours := time.Since(entityCreated).Hours() + if ageHours < 0 { + ageHours = 0 + } + + hours = math.Min(720, math.Min(ageHours, retentionHours)) + if hours < 1 { + hours = 1 + } + return since, hours, windowDays +} + +// DomainSendStats returns 30-day (or shorter when retention is lower) send +// statistics for a domain name. +func (s *Store) DomainSendStats(name string, retentionDays int, createdAt time.Time) (SendStats, error) { + since, hours, windowDays := StatsWindow(retentionDays, createdAt) + stats, err := s.sendStats("domain", name, since, hours) + if err != nil { + return SendStats{}, err + } + stats.WindowDays = windowDays + stats.WindowHours = hours + return stats, nil +} + +// AppSendStats returns send statistics for an application login. +func (s *Store) AppSendStats(login string, retentionDays int, createdAt time.Time) (SendStats, error) { + since, hours, windowDays := StatsWindow(retentionDays, createdAt) + stats, err := s.sendStats("app_login", login, since, hours) + if err != nil { + return SendStats{}, err + } + stats.WindowDays = windowDays + stats.WindowHours = hours + return stats, nil +} + +func (s *Store) sendStats(column, ref string, since time.Time, hours float64) (SendStats, error) { + sinceStr := since.UTC().Format(time.RFC3339) + + var total int64 + err := s.db.QueryRow( + `SELECT COUNT(DISTINCT queue_id) FROM send_log + WHERE `+column+` = ? AND status != ? AND created_at >= ?`, + ref, StatusRejected, sinceStr, + ).Scan(&total) + if err != nil { + return SendStats{}, fmt.Errorf("send stats total %s %q: %w", column, ref, err) + } + + var peak int64 + err = s.db.QueryRow( + `SELECT COALESCE(MAX(bucket_count), 0) FROM ( + SELECT COUNT(DISTINCT queue_id) AS bucket_count FROM send_log + WHERE `+column+` = ? AND status != ? AND created_at >= ? + GROUP BY substr(created_at, 1, 13) + )`, + ref, StatusRejected, sinceStr, + ).Scan(&peak) + if err != nil { + return SendStats{}, fmt.Errorf("send stats peak %s %q: %w", column, ref, err) + } + + avg := float64(total) / hours + return SendStats{Total: total, PeakPerHour: peak, AvgPerHour: avg}, nil +} diff --git a/internal/store/stats_test.go b/internal/store/stats_test.go new file mode 100644 index 0000000..358cea4 --- /dev/null +++ b/internal/store/stats_test.go @@ -0,0 +1,78 @@ +package store + +import ( + "testing" + "time" +) + +func TestSendStatsTotalPeakAvg(t *testing.T) { + st := openTestStore(t) + d, err := st.AddDomain("example.com", "selfpost") + if err != nil { + t.Fatalf("AddDomain: %v", err) + } + a, err := st.AddApplication(d.ID, "app1", AddressModeWildcard, nil) + if err != nil { + t.Fatalf("AddApplication: %v", err) + } + + now := time.Now().UTC() + hour := now.Format("2006-01-02T15") + prevHour := now.Add(-2 * time.Hour).Format("2006-01-02T15") + + // Hour 1: two messages (Q1 two recipients + Q2). + for _, to := range []string{"a@x.net", "b@x.net"} { + if err := st.InsertQueued(SendLogEntry{QueueID: "Q1", Domain: "example.com", AppLogin: "app1", To: to}); err != nil { + t.Fatalf("InsertQueued: %v", err) + } + } + if err := st.InsertQueued(SendLogEntry{QueueID: "Q2", Domain: "example.com", AppLogin: "app1", To: "c@x.net"}); err != nil { + t.Fatalf("InsertQueued: %v", err) + } + // Hour 2: one message. + if _, err := st.db.Exec(`UPDATE send_log SET created_at = ? WHERE queue_id = 'Q2'`, + prevHour+":00:00Z"); err != nil { + t.Fatalf("backdate Q2: %v", err) + } + if _, err := st.db.Exec(`UPDATE send_log SET created_at = ? WHERE queue_id = 'Q1'`, + hour+":10:00Z"); err != nil { + t.Fatalf("backdate Q1: %v", err) + } + if err := st.InsertRejected(SendLogEntry{Domain: "example.com", AppLogin: "app1"}); err != nil { + t.Fatalf("InsertRejected: %v", err) + } + + stats, err := st.DomainSendStats("example.com", 90, d.CreatedAt) + if err != nil { + t.Fatalf("DomainSendStats: %v", err) + } + if stats.Total != 2 { + t.Fatalf("total = %d, want 2", stats.Total) + } + if stats.PeakPerHour != 1 { + t.Fatalf("peak = %d, want 1 (one message per hour bucket)", stats.PeakPerHour) + } + if stats.AvgPerHour <= 0 { + t.Fatalf("avg should be positive, got %v", stats.AvgPerHour) + } + + appStats, err := st.AppSendStats("app1", 90, a.CreatedAt) + if err != nil { + t.Fatalf("AppSendStats: %v", err) + } + if appStats.Total != 2 { + t.Fatalf("app total = %d, want 2", appStats.Total) + } +} + +func TestStatsWindowShortRetention(t *testing.T) { + st := openTestStore(t) + d, _ := st.AddDomain("example.com", "selfpost") + stats, err := st.DomainSendStats("example.com", 7, d.CreatedAt) + if err != nil { + t.Fatalf("DomainSendStats: %v", err) + } + if stats.WindowDays != 7 { + t.Fatalf("window days = %d, want 7", stats.WindowDays) + } +} diff --git a/internal/web/handlers/handlers_apps.go b/internal/web/handlers/handlers_apps.go index 97379a7..0bc135c 100644 --- a/internal/web/handlers/handlers_apps.go +++ b/internal/web/handlers/handlers_apps.go @@ -6,6 +6,7 @@ import ( "net/http" "strconv" "strings" + "time" "github.com/mixeme/selfpost/internal/dnscheck" "github.com/mixeme/selfpost/internal/domain" @@ -43,10 +44,22 @@ type detailView struct { // template fields (Login, AddressMode, Addresses, ID) resolve unchanged. type appRateLimitView struct { store.Application - HasLimit bool // an active limit is configured - IPsText string // allowed IPs, newline-joined for the textarea - MaxText string // message ceiling, blank when unset - WindowVal string // window seconds, defaulted when unset + HasLimit bool + IPsText string + MaxText string + WindowVal string + Mode string + AutoMultiplier string + AutoUpdated string + IsAuto bool + Stats sendStatsView +} + +// sendStatsView is the template-facing send statistics block. +type sendStatsView struct { + Total int64 + PeakPerHour int64 + AvgPerHour string } // HandleDomainDetail shows a single domain: its DKIM DNS record (product.md) @@ -77,6 +90,7 @@ func (h *Handlers) renderDomainDetail(w http.ResponseWriter, r *http.Request, st return } appViews := make([]appRateLimitView, 0, len(apps)) + retention := h.sendLogRetentionDays() for _, a := range apps { rl, ok, err := h.apps.RateLimit(a.ID) if err != nil { @@ -84,12 +98,31 @@ func (h *Handlers) renderDomainDetail(w http.ResponseWriter, r *http.Request, st http.Error(w, "internal error", http.StatusInternalServerError) return } + appStats, err := h.store.AppSendStats(a.Login, retention, a.CreatedAt) + if err != nil { + logf("panel: application %d: send stats: %v", a.ID, err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + mode := store.RateLimitModeManual + if ok && rl.Mode != "" { + mode = rl.Mode + } + mult := rl.AutoMultiplier + if mult <= 0 { + mult = store.DefaultAutoMultiplier + } appViews = append(appViews, appRateLimitView{ - Application: a, - HasLimit: ok && rl.Active(), - IPsText: strings.Join(rl.AllowedIPs, "\n"), - MaxText: intOrBlank(rl.MaxMessages), - WindowVal: windowOrDefault(rl.WindowSeconds), + Application: a, + HasLimit: ok && rl.Active(), + IPsText: strings.Join(rl.AllowedIPs, "\n"), + MaxText: intOrBlank(rl.MaxMessages), + WindowVal: windowOrDefault(rl.WindowSeconds), + Mode: mode, + AutoMultiplier: formatMultiplier(mult), + IsAuto: ok && rl.IsAuto(), + AutoUpdated: formatAutoUpdated(rl.AutoUpdatedAt), + Stats: formatSendStats(appStats), }) } @@ -99,6 +132,22 @@ func (h *Handlers) renderDomainDetail(w http.ResponseWriter, r *http.Request, st http.Error(w, "internal error", http.StatusInternalServerError) return } + domainStats, err := h.store.DomainSendStats(d.Name, retention, d.CreatedAt) + if err != nil { + logf("panel: domain %d: send stats: %v", d.ID, err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + domainMode := store.RateLimitModeManual + domainMult := store.DefaultAutoMultiplier + if domainRLok { + if domainRL.Mode != "" { + domainMode = domainRL.Mode + } + if domainRL.AutoMultiplier > 0 { + domainMult = domainRL.AutoMultiplier + } + } // What DNS actually publishes for the domain today, checked against the key // this server signs with. Cached by the checker, so re-rendering the page @@ -168,8 +217,18 @@ func (h *Handlers) renderDomainDetail(w http.ResponseWriter, r *http.Request, st data["DomainRLMax"] = intOrBlank(domainRL.MaxMessages) data["DomainRLWin"] = windowOrDefault(domainRL.WindowSeconds) data["DomainRLMaxNum"] = domainRL.MaxMessages + data["DomainRLMode"] = domainMode + data["DomainRLAuto"] = domainRLok && domainRL.IsAuto() + data["DomainRLMultiplier"] = formatMultiplier(domainMult) + data["DomainRLAutoUpdated"] = formatAutoUpdated(domainRL.AutoUpdatedAt) + data["DomainStats"] = formatSendStats(domainStats) + data["StatsWindowDays"] = domainStats.WindowDays + data["StatsRetentionWarning"] = retention < store.StatsWindowDays data["L1Messages"] = h.l1Messages() data["L1Window"] = h.l1Window() + data["DefaultAutoMultiplier"] = store.DefaultAutoMultiplier + data["MinAutoMultiplier"] = store.MinAutoMultiplier + data["MaxAutoMultiplier"] = store.MaxAutoMultiplier h.view.Render(w, status, "domain_detail", data) } @@ -232,6 +291,25 @@ func windowOrDefault(n int) string { return strconv.Itoa(n) } +func formatSendStats(s store.SendStats) sendStatsView { + return sendStatsView{ + Total: s.Total, + PeakPerHour: s.PeakPerHour, + AvgPerHour: fmt.Sprintf("%.1f", s.AvgPerHour), + } +} + +func formatMultiplier(v float64) string { + return strconv.FormatFloat(v, 'f', 1, 64) +} + +func formatAutoUpdated(t time.Time) string { + if t.IsZero() { + return "" + } + return t.UTC().Format("2006-01-02 15:04 UTC") +} + // detailFlash maps a fixed redirect flag to a fixed message, so status text // after a redirect is never attacker-influenced. func detailFlash(r *http.Request) string { @@ -242,6 +320,8 @@ func detailFlash(r *http.Request) string { return "Application address mode updated." case r.URL.Query().Get("ratelimit") != "": return "Rate limit updated." + case r.URL.Query().Get("recalculated") != "": + return "Auto rate limit recalculated." case r.URL.Query().Get("dmarc") != "": return "DMARC report settings updated." case r.URL.Query().Get("imported") != "": diff --git a/internal/web/handlers/handlers_ratelimit.go b/internal/web/handlers/handlers_ratelimit.go index fb70123..a278bb3 100644 --- a/internal/web/handlers/handlers_ratelimit.go +++ b/internal/web/handlers/handlers_ratelimit.go @@ -13,10 +13,12 @@ import ( const defaultRateLimitWindowSeconds = 3600 type rateLimitInput struct { - clear bool - ips []string - maxMessages int - windowSeconds int + clear bool + mode string + ips []string + maxMessages int + windowSeconds int + autoMultiplier float64 } func (h *Handlers) l1Messages() int { @@ -33,6 +35,32 @@ func (h *Handlers) l1Window() int { return defaultRateLimitWindowSeconds } +func parseAutoMultiplier(raw string) (float64, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return store.DefaultAutoMultiplier, nil + } + v, err := strconv.ParseFloat(raw, 64) + if err != nil { + return 0, fmt.Errorf("enter a valid multiplier (%.1f–%.1f)", store.MinAutoMultiplier, store.MaxAutoMultiplier) + } + if v < store.MinAutoMultiplier || v > store.MaxAutoMultiplier { + return 0, fmt.Errorf("multiplier must be between %.1f and %.1f", store.MinAutoMultiplier, store.MaxAutoMultiplier) + } + return v, nil +} + +func parseRateLimitMode(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return store.RateLimitModeManual, nil + } + if raw != store.RateLimitModeManual && raw != store.RateLimitModeAuto { + return "", fmt.Errorf("choose manual or auto mode") + } + return raw, nil +} + func parseDomainRateLimitForm(r *http.Request, l1Max int) (rateLimitInput, error) { if err := r.ParseForm(); err != nil { return rateLimitInput{}, fmt.Errorf("invalid form submission") @@ -40,6 +68,18 @@ func parseDomainRateLimitForm(r *http.Request, l1Max int) (rateLimitInput, error if r.PostFormValue("clear") != "" { return rateLimitInput{clear: true}, nil } + mode, err := parseRateLimitMode(r.PostFormValue("mode")) + if err != nil { + return rateLimitInput{}, err + } + if mode == store.RateLimitModeAuto { + mult, err := parseAutoMultiplier(r.PostFormValue("auto_multiplier")) + if err != nil { + return rateLimitInput{}, err + } + return rateLimitInput{mode: mode, autoMultiplier: mult}, nil + } + rawMax := strings.TrimSpace(r.PostFormValue("max_messages")) if rawMax == "" { return rateLimitInput{clear: true}, nil @@ -55,7 +95,7 @@ func parseDomainRateLimitForm(r *http.Request, l1Max int) (rateLimitInput, error if err != nil || windowSeconds <= 0 { return rateLimitInput{}, fmt.Errorf("enter a time window greater than zero seconds") } - return rateLimitInput{maxMessages: maxMessages, windowSeconds: windowSeconds}, nil + return rateLimitInput{mode: store.RateLimitModeManual, maxMessages: maxMessages, windowSeconds: windowSeconds}, nil } func parseAppRateLimitForm(r *http.Request, l1Max, domainMax int, domainActive bool) (rateLimitInput, error) { @@ -65,10 +105,11 @@ func parseAppRateLimitForm(r *http.Request, l1Max, domainMax int, domainActive b if r.PostFormValue("clear") != "" { return rateLimitInput{clear: true}, nil } - rawMax := strings.TrimSpace(r.PostFormValue("max_messages")) - if rawMax == "" { - return rateLimitInput{clear: true}, nil + mode, err := parseRateLimitMode(r.PostFormValue("mode")) + if err != nil { + return rateLimitInput{}, err } + ips, err := parseIPList(r.PostFormValue("allowed_ips")) if err != nil { return rateLimitInput{}, err @@ -76,6 +117,19 @@ func parseAppRateLimitForm(r *http.Request, l1Max, domainMax int, domainActive b if len(ips) == 0 { return rateLimitInput{}, fmt.Errorf("enter at least one trusted client IP for an application override") } + + if mode == store.RateLimitModeAuto { + mult, err := parseAutoMultiplier(r.PostFormValue("auto_multiplier")) + if err != nil { + return rateLimitInput{}, err + } + return rateLimitInput{mode: mode, ips: ips, autoMultiplier: mult}, nil + } + + rawMax := strings.TrimSpace(r.PostFormValue("max_messages")) + if rawMax == "" { + return rateLimitInput{clear: true}, nil + } maxMessages, err := parsePositiveInt(rawMax, 0) if err != nil || maxMessages <= 0 { return rateLimitInput{}, fmt.Errorf("enter a message limit greater than zero") @@ -90,7 +144,7 @@ func parseAppRateLimitForm(r *http.Request, l1Max, domainMax int, domainActive b if err != nil || windowSeconds <= 0 { return rateLimitInput{}, fmt.Errorf("enter a time window greater than zero seconds") } - return rateLimitInput{ips: ips, maxMessages: maxMessages, windowSeconds: windowSeconds}, nil + return rateLimitInput{mode: store.RateLimitModeManual, ips: ips, maxMessages: maxMessages, windowSeconds: windowSeconds}, nil } func parseIPList(raw string) ([]string, error) { @@ -134,7 +188,7 @@ func (h *Handlers) HandleDomainRateLimit(w http.ResponseWriter, r *http.Request) }) return } - if err := h.applyRateLimit(in, h.domains.SaveRateLimit, h.domains.ClearRateLimit, d.ID); err != nil { + if err := h.applyDomainRateLimit(in, d.ID); err != nil { logf("panel: domain %d: save rate limit: %v", d.ID, err) http.Error(w, "internal error", http.StatusInternalServerError) return @@ -167,7 +221,7 @@ func (h *Handlers) HandleAppRateLimit(w http.ResponseWriter, r *http.Request) { }) return } - if err := h.applyRateLimit(in, h.apps.SaveRateLimit, h.apps.ClearRateLimit, a.ID); err != nil { + if err := h.applyAppRateLimit(in, a.ID); err != nil { logf("panel: application %d: save rate limit: %v", a.ID, err) http.Error(w, "internal error", http.StatusInternalServerError) return @@ -175,14 +229,83 @@ func (h *Handlers) HandleAppRateLimit(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, fmt.Sprintf("/domains/%d?ratelimit=1", a.DomainID), http.StatusSeeOther) } -func (h *Handlers) applyRateLimit( - in rateLimitInput, - save func(id int64, ips []string, maxMessages, windowSeconds int) error, - clear func(id int64) error, - id int64, -) error { - if in.clear { - return clear(id) +func (h *Handlers) HandleDomainRateLimitRecalc(w http.ResponseWriter, r *http.Request) { + d, ok := h.lookupDomain(w, r) + if !ok { + return } - return save(id, in.ips, in.maxMessages, in.windowSeconds) + if err := h.recalcRateLimit(store.RateLimitScopeDomain, d.ID); err != nil { + logf("panel: domain %d: recalc rate limit: %v", d.ID, err) + h.renderDomainDetail(w, r, http.StatusBadRequest, d, detailView{ + FormMode: store.AddressModeWildcard, + RateLimitErr: err.Error(), + }) + return + } + http.Redirect(w, r, fmt.Sprintf("/domains/%d?recalculated=1", d.ID), http.StatusSeeOther) +} + +func (h *Handlers) HandleAppRateLimitRecalc(w http.ResponseWriter, r *http.Request) { + a, ok := h.lookupApplication(w, r) + if !ok { + return + } + if err := h.recalcRateLimit(store.RateLimitScopeApp, a.ID); err != nil { + d, _ := h.domains.Get(a.DomainID) + h.renderDomainDetail(w, r, http.StatusBadRequest, d, detailView{ + FormMode: store.AddressModeWildcard, + RateLimitErr: fmt.Sprintf("%s: %s", a.Login, err.Error()), + }) + return + } + http.Redirect(w, r, fmt.Sprintf("/domains/%d?recalculated=1", a.DomainID), http.StatusSeeOther) +} + +func (h *Handlers) recalcRateLimit(scope string, refID int64) error { + return h.store.RecalcAutoRateLimit(scope, refID, h.sendLogRetentionDays(), h.l1Messages(), h.l1Window()) +} + +func (h *Handlers) applyDomainRateLimit(in rateLimitInput, domainID int64) error { + if in.clear { + return h.domains.ClearRateLimit(domainID) + } + rl := store.RateLimit{ + Scope: store.RateLimitScopeDomain, + RefID: domainID, + Mode: in.mode, + MaxMessages: in.maxMessages, + WindowSeconds: in.windowSeconds, + AutoMultiplier: in.autoMultiplier, + } + if in.mode == store.RateLimitModeAuto { + rl.WindowSeconds = h.l1Window() + if err := h.domains.SaveRateLimit(domainID, rl); err != nil { + return err + } + return h.recalcRateLimit(store.RateLimitScopeDomain, domainID) + } + return h.domains.SaveRateLimit(domainID, rl) +} + +func (h *Handlers) applyAppRateLimit(in rateLimitInput, appID int64) error { + if in.clear { + return h.apps.ClearRateLimit(appID) + } + rl := store.RateLimit{ + Scope: store.RateLimitScopeApp, + RefID: appID, + AllowedIPs: in.ips, + Mode: in.mode, + MaxMessages: in.maxMessages, + WindowSeconds: in.windowSeconds, + AutoMultiplier: in.autoMultiplier, + } + if in.mode == store.RateLimitModeAuto { + rl.WindowSeconds = h.l1Window() + if err := h.apps.SaveRateLimit(appID, rl); err != nil { + return err + } + return h.recalcRateLimit(store.RateLimitScopeApp, appID) + } + return h.apps.SaveRateLimit(appID, rl) } diff --git a/internal/web/handlers/handlers_ratelimit_test.go b/internal/web/handlers/handlers_ratelimit_test.go index 2cab88b..4d5453f 100644 --- a/internal/web/handlers/handlers_ratelimit_test.go +++ b/internal/web/handlers/handlers_ratelimit_test.go @@ -17,6 +17,7 @@ func TestParseDomainRateLimitForm(t *testing.T) { } in, err := parseDomainRateLimitForm(form(url.Values{ + "mode": {"manual"}, "max_messages": {"50"}, "window_seconds": {"3600"}, }), 100) @@ -30,6 +31,7 @@ func TestParseDomainRateLimitForm(t *testing.T) { } _, err = parseDomainRateLimitForm(form(url.Values{ + "mode": {"manual"}, "max_messages": {"150"}, "window_seconds": {"3600"}, }), 100) @@ -47,6 +49,7 @@ func TestParseAppRateLimitForm(t *testing.T) { } in, err := parseAppRateLimitForm(form(url.Values{ + "mode": {"manual"}, "allowed_ips": {"203.0.113.10"}, "max_messages": {"80"}, "window_seconds": {"3600"}, @@ -56,6 +59,7 @@ func TestParseAppRateLimitForm(t *testing.T) { } _, err = parseAppRateLimitForm(form(url.Values{ + "mode": {"manual"}, "max_messages": {"80"}, "window_seconds": {"3600"}, }), 100, 40, true) @@ -64,6 +68,7 @@ func TestParseAppRateLimitForm(t *testing.T) { } _, err = parseAppRateLimitForm(form(url.Values{ + "mode": {"manual"}, "allowed_ips": {"203.0.113.10"}, "max_messages": {"40"}, "window_seconds": {"3600"}, @@ -73,6 +78,7 @@ func TestParseAppRateLimitForm(t *testing.T) { } _, err = parseAppRateLimitForm(form(url.Values{ + "mode": {"manual"}, "allowed_ips": {"203.0.113.10"}, "max_messages": {"150"}, "window_seconds": {"3600"}, @@ -83,6 +89,7 @@ func TestParseAppRateLimitForm(t *testing.T) { // No domain limit: any app ceiling ≤ L1 is fine. in, err = parseAppRateLimitForm(form(url.Values{ + "mode": {"manual"}, "allowed_ips": {"203.0.113.10"}, "max_messages": {"50"}, "window_seconds": {"3600"}, @@ -91,3 +98,26 @@ func TestParseAppRateLimitForm(t *testing.T) { t.Fatalf("app without domain = %+v err=%v", in, err) } } + +func TestParseDomainRateLimitFormAuto(t *testing.T) { + t.Parallel() + form := func(vals url.Values) *http.Request { + r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(vals.Encode())) + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + return r + } + in, err := parseDomainRateLimitForm(form(url.Values{ + "mode": {"auto"}, + "auto_multiplier": {"2.5"}, + }), 100) + if err != nil || in.mode != "auto" || in.autoMultiplier != 2.5 { + t.Fatalf("auto domain = %+v err=%v", in, err) + } + _, err = parseDomainRateLimitForm(form(url.Values{ + "mode": {"auto"}, + "auto_multiplier": {"10"}, + }), 100) + if err == nil { + t.Fatal("multiplier out of range should fail") + } +} diff --git a/internal/web/view/static/panel.js b/internal/web/view/static/panel.js index 98147f9..9a7dccd 100644 --- a/internal/web/view/static/panel.js +++ b/internal/web/view/static/panel.js @@ -61,6 +61,16 @@ target: "[data-custom-address]", visible: function (el) { return el.value === el.dataset.customMode; } }, + { + match: "select[data-ratelimit-mode]", + target: "[data-manual-fields]", + visible: function (el) { return el.value === "manual"; } + }, + { + match: "select[data-ratelimit-mode]", + target: "[data-auto-fields]", + visible: function (el) { return el.value === "auto"; } + }, { match: "select[data-global-role]", target: "[data-domain-pick]", diff --git a/internal/web/view/templates/domain_detail.html b/internal/web/view/templates/domain_detail.html index a32b250..5b17951 100644 --- a/internal/web/view/templates/domain_detail.html +++ b/internal/web/view/templates/domain_detail.html @@ -236,6 +236,28 @@ +
+

Sending statistics ({{.StatsWindowDays}} days)

+

One message = one queue id (many recipients count once). + Level-1 refusals are not in the send log — totals under-count strict IP + limits.{{if .StatsRetentionWarning}} Send log retention is shorter than + 30 days; statistics use the last {{.StatsWindowDays}} days only.{{end}}

+
+
+ + {{.DomainStats.Total}} +
+
+ + {{.DomainStats.PeakPerHour}} +
+
+ + {{.DomainStats.AvgPerHour}} +
+
+
+

Applications

SASL logins for this domain — wildcard (*@domain) or a fixed @@ -250,6 +272,8 @@ {{if eq .AddressMode $.Wildcard}}Any address of the domain — *@{{$.Domain.Name}} {{else}}Fixed list — {{range $i, $a := .Addresses}}{{if $i}}, {{end}}{{$a}}{{end}}{{end}}

+

{{.Stats.Total}} msg / {{$.StatsWindowDays}}d · + peak {{.Stats.PeakPerHour}} msg/h · avg {{.Stats.AvgPerHour}} msg/h

@@ -293,22 +317,41 @@ (≤ level 1) and skip the domain check; everyone else uses the domain level-2 limit{{if $.DomainHasRL}} ({{$.DomainRLMaxNum}}){{end}} if set, otherwise level 1.

-
-
- - -
-
- - + + +
+
+
+ + +
+
+ + +
+
+ + +

Computed limit: {{.MaxText}} messages / + {{$.L1Window}} s (level-1 window). Zero traffic keeps auto inactive until + messages are sent.{{if .AutoUpdated}} Last recalculated {{.AutoUpdated}}.{{end}} + Peak {{.Stats.PeakPerHour}} msg/h shown for comparison.

+
-
+ {{if .IsAuto}} +
+ +
+ {{end}} {{if .HasLimit}}
@@ -365,23 +408,44 @@

Level 1 backstop: {{.L1Messages}} messages / {{.L1Window}}s — Settings.

-
-
- - -
-
- - + + +
+
+
+ + +
+
+ + +
+

Applies to every client IP on this domain. Leave the + message limit empty to use level 1 only.

+
+
+ + +

Computed limit: {{.DomainRLMax}} messages / + {{.L1Window}} s (level-1 window). Zero traffic keeps auto inactive until + messages are sent.{{if .DomainRLAutoUpdated}} Last recalculated {{.DomainRLAutoUpdated}}.{{end}} + Peak {{.DomainStats.PeakPerHour}} msg/h shown for comparison.

-

Applies to every client IP on this domain. Leave the - message limit empty to use level 1 only.

+ {{if .DomainRLAuto}} +
+ +
+ {{end}} {{if .DomainHasRL}}
diff --git a/internal/web/web.go b/internal/web/web.go index aff8530..201989a 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -167,11 +167,13 @@ func (s *Server) Handler() http.Handler { authed.HandleFunc("POST /domains/{id}/delete", h.HandleDeleteDomain) authed.HandleFunc("POST /domains/{id}/applications", h.HandleAddApplication) authed.HandleFunc("POST /domains/{id}/ratelimit", h.HandleDomainRateLimit) + authed.HandleFunc("POST /domains/{id}/ratelimit/recalc", h.HandleDomainRateLimitRecalc) authed.HandleFunc("POST /domains/{id}/dmarc", h.HandleDomainDMARC) authed.HandleFunc("POST /domains/{id}/export", h.HandleExportDomain) authed.HandleFunc("POST /applications/{aid}/mode", h.HandleUpdateAppMode) authed.HandleFunc("POST /applications/{aid}/password", h.HandleRegenPassword) authed.HandleFunc("POST /applications/{aid}/ratelimit", h.HandleAppRateLimit) + authed.HandleFunc("POST /applications/{aid}/ratelimit/recalc", h.HandleAppRateLimitRecalc) authed.HandleFunc("POST /applications/{aid}/delete", h.HandleDeleteApplication) authed.HandleFunc("POST /reload", h.HandleReload)