release: 1.6.0
test / test (push) Waiting to run

Add 30-day send statistics and auto level-2 rate limits on the domain page. Close Unreleased; pin compose and docs to 1.6.0.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-08-18 22:26:51 +03:00
parent 8538b8d5f6
commit c1ec4fbd79
27 changed files with 1092 additions and 140 deletions
+19
View File
@@ -163,6 +163,25 @@ func (s *Store) GetApplication(id int64) (Application, error) {
return a, nil
}
// GetApplicationByLogin returns one application by its globally unique SASL login.
func (s *Store) GetApplicationByLogin(login string) (Application, error) {
row := s.db.QueryRow(
"SELECT id, domain_id, login, address_mode, created_at FROM applications WHERE login = ?", login)
a, err := scanApplication(row)
if errors.Is(err, sql.ErrNoRows) {
return Application{}, ErrApplicationNotFound
}
if err != nil {
return Application{}, err
}
addrs, err := s.applicationAddresses(a.ID)
if err != nil {
return Application{}, err
}
a.Addresses = addrs
return a, nil
}
// ListApplicationsByDomain returns a domain's applications ordered by login,
// each with its address list populated (product.md).
func (s *Store) ListApplicationsByDomain(domainID int64) ([]Application, error) {
@@ -0,0 +1,7 @@
-- Level-2 rate limit auto mode: operator sets a multiplier; max_messages is
-- derived from 30-day send stats (plan domain-stats-auto-ratelimit).
ALTER TABLE rate_limits ADD COLUMN mode TEXT NOT NULL DEFAULT 'manual'
CHECK (mode IN ('manual', 'auto'));
ALTER TABLE rate_limits ADD COLUMN auto_multiplier REAL;
ALTER TABLE rate_limits ADD COLUMN auto_updated_at TEXT;
+166
View File
@@ -0,0 +1,166 @@
package store
import (
"database/sql"
"fmt"
"log"
"math"
"time"
)
// RecalcAllAutoRateLimits recomputes max_messages for every auto-mode limit.
// Store errors for individual rows are logged and skipped (fail-open on the
// last successfully written limit). Returns how many rows were updated.
func (s *Store) RecalcAllAutoRateLimits(retentionDays, l1Max, l1Window int) (int, error) {
rows, err := s.listAutoRateLimits()
if err != nil {
return 0, err
}
var updated int
for _, rl := range rows {
if err := s.recalcAutoRateLimit(rl, retentionDays, l1Max, l1Window); err != nil {
log.Printf("store: auto rate-limit recalc %s ref %d: %v", rl.Scope, rl.RefID, err)
continue
}
updated++
}
return updated, nil
}
// RecalcAutoRateLimit recomputes one auto-mode limit. Returns an error when
// the row is missing or not in auto mode.
func (s *Store) RecalcAutoRateLimit(scope string, refID int64, retentionDays, l1Max, l1Window int) error {
rl, ok, err := s.GetRateLimit(scope, refID)
if err != nil {
return err
}
if !ok || !rl.IsAuto() {
return fmt.Errorf("rate limit not in auto mode")
}
rl.Scope, rl.RefID = scope, refID
return s.recalcAutoRateLimit(rl, retentionDays, l1Max, l1Window)
}
func (s *Store) listAutoRateLimits() ([]RateLimit, error) {
rows, err := s.db.Query(
`SELECT scope, ref_id, allowed_ips, max_messages, window_seconds, mode, auto_multiplier, auto_updated_at
FROM rate_limits WHERE mode = ?`, RateLimitModeAuto,
)
if err != nil {
return nil, fmt.Errorf("list auto rate limits: %w", err)
}
defer rows.Close()
var out []RateLimit
for rows.Next() {
var scope string
var refID int64
var ips sql.NullString
var maxMsgs, windowSecs sql.NullInt64
var mode sql.NullString
var autoMult sql.NullFloat64
var autoUpdated sql.NullString
if err := rows.Scan(&scope, &refID, &ips, &maxMsgs, &windowSecs, &mode, &autoMult, &autoUpdated); err != nil {
return nil, fmt.Errorf("scan auto rate limit: %w", err)
}
rl := RateLimit{
Scope: scope,
RefID: refID,
AllowedIPs: splitIPs(ips.String),
MaxMessages: int(maxMsgs.Int64),
WindowSeconds: int(windowSecs.Int64),
Mode: mode.String,
}
if autoMult.Valid {
rl.AutoMultiplier = autoMult.Float64
}
if autoUpdated.Valid {
rl.AutoUpdatedAt, _ = time.Parse(time.RFC3339, autoUpdated.String)
}
out = append(out, rl)
}
return out, rows.Err()
}
func (s *Store) recalcAutoRateLimit(rl RateLimit, retentionDays, l1Max, l1Window int) error {
if !rl.IsAuto() {
return fmt.Errorf("not auto mode")
}
mult := rl.AutoMultiplier
if mult <= 0 {
mult = DefaultAutoMultiplier
}
var stats SendStats
var domainMax int
var domainActive bool
switch rl.Scope {
case RateLimitScopeDomain:
d, err := s.GetDomain(rl.RefID)
if err != nil {
return err
}
stats, err = s.DomainSendStats(d.Name, retentionDays, d.CreatedAt)
if err != nil {
return err
}
case RateLimitScopeApp:
a, err := s.GetApplication(rl.RefID)
if err != nil {
return err
}
stats, err = s.AppSendStats(a.Login, retentionDays, a.CreatedAt)
if err != nil {
return err
}
domainRL, ok, err := s.GetRateLimit(RateLimitScopeDomain, a.DomainID)
if err != nil {
return err
}
domainActive = ok && domainRL.Active()
if domainActive {
domainMax = domainRL.MaxMessages
}
default:
return fmt.Errorf("unknown scope %q", rl.Scope)
}
maxMsgs := computeAutoMaxMessages(stats, mult, l1Max)
if rl.Scope == RateLimitScopeApp {
maxMsgs = adjustAppAutoMax(maxMsgs, domainMax, domainActive, l1Max)
}
rl.MaxMessages = maxMsgs
rl.WindowSeconds = l1Window
rl.AutoUpdatedAt = time.Now().UTC()
if rl.AutoMultiplier <= 0 {
rl.AutoMultiplier = mult
}
return s.SetRateLimit(rl)
}
func computeAutoMaxMessages(stats SendStats, multiplier float64, l1Max int) int {
if stats.Total == 0 {
return 0
}
max := int(math.Ceil(stats.AvgPerHour * multiplier))
if max > l1Max {
max = l1Max
}
return max
}
func adjustAppAutoMax(appMax, domainMax int, domainActive bool, l1Max int) int {
if appMax <= 0 {
return 0
}
if domainActive && appMax <= domainMax {
appMax = domainMax + 1
if appMax > l1Max {
return 0
}
}
return appMax
}
+67 -19
View File
@@ -17,6 +17,19 @@ const (
RateLimitScopeApp = "application"
)
// Level-2 rate limit operating modes (plan domain-stats-auto-ratelimit).
const (
RateLimitModeManual = "manual"
RateLimitModeAuto = "auto"
)
// Auto rate-limit multiplier bounds shown in the panel.
const (
DefaultAutoMultiplier = 2.5
MinAutoMultiplier = 1.5
MaxAutoMultiplier = 5.0
)
// RateLimit is a differentiated level-2 rate limit (guide § Rate limiting):
// a message ceiling over a sliding window, attached to a domain or an
// application. It is enforced in the journal-milter; level 1 (Postfix anvil,
@@ -28,16 +41,24 @@ const (
// ceiling (above the domain) and skip the domain check; other IPs stay under
// the domain limit or level 1 alone (guide § Rate limiting).
type RateLimit struct {
Scope string
RefID int64
AllowedIPs []string // trusted client IPs for an application override
MaxMessages int
WindowSeconds int
Scope string
RefID int64
AllowedIPs []string // trusted client IPs for an application override
MaxMessages int
WindowSeconds int
Mode string // manual | auto
AutoMultiplier float64 // used when Mode == auto
AutoUpdatedAt time.Time
}
// Active reports whether the limit is fully configured and should be enforced.
// Domain: max and window only. Application: also needs at least one trusted IP
// (the privilege that raises the ceiling above the domain).
// IsAuto reports whether the limit derives max_messages from send statistics.
func (r RateLimit) IsAuto() bool {
return r.Mode == RateLimitModeAuto
}
func (r RateLimit) Active() bool {
if r.MaxMessages <= 0 || r.WindowSeconds <= 0 {
return false
@@ -69,7 +90,7 @@ func (r RateLimit) AllowsIP(ip string) bool {
// its id, for the panel's edit form. ok is false when none is configured.
func (s *Store) GetRateLimit(scope string, refID int64) (RateLimit, bool, error) {
row := s.db.QueryRow(
`SELECT allowed_ips, max_messages, window_seconds
`SELECT allowed_ips, max_messages, window_seconds, mode, auto_multiplier, auto_updated_at
FROM rate_limits WHERE scope = ? AND ref_id = ?`, scope, refID)
rl, err := scanRateLimit(row)
if errors.Is(err, sql.ErrNoRows) {
@@ -86,14 +107,30 @@ func (s *Store) GetRateLimit(scope string, refID int64) (RateLimit, bool, error)
// (panel) has already validated the IPs and numbers (security.md); values are
// stored via bound parameters and read back live by the milter.
func (s *Store) SetRateLimit(rl RateLimit) error {
mode := rl.Mode
if mode == "" {
mode = RateLimitModeManual
}
var autoMult interface{}
if rl.IsAuto() {
autoMult = rl.AutoMultiplier
}
var autoUpdated interface{}
if !rl.AutoUpdatedAt.IsZero() {
autoUpdated = rl.AutoUpdatedAt.UTC().Format(time.RFC3339)
}
_, err := s.db.Exec(
`INSERT INTO rate_limits (scope, ref_id, allowed_ips, max_messages, window_seconds)
VALUES (?, ?, ?, ?, ?)
`INSERT INTO rate_limits (scope, ref_id, allowed_ips, max_messages, window_seconds, mode, auto_multiplier, auto_updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(scope, ref_id) DO UPDATE SET
allowed_ips = excluded.allowed_ips,
max_messages = excluded.max_messages,
window_seconds = excluded.window_seconds`,
allowed_ips = excluded.allowed_ips,
max_messages = excluded.max_messages,
window_seconds = excluded.window_seconds,
mode = excluded.mode,
auto_multiplier = excluded.auto_multiplier,
auto_updated_at = excluded.auto_updated_at`,
rl.Scope, rl.RefID, strings.Join(rl.AllowedIPs, ","), rl.MaxMessages, rl.WindowSeconds,
mode, autoMult, autoUpdated,
)
if err != nil {
return fmt.Errorf("set rate limit: %w", err)
@@ -134,11 +171,11 @@ func (s *Store) RateLimit(scope, ref string) (RateLimit, bool, error) {
var query string
switch scope {
case RateLimitScopeDomain:
query = `SELECT rl.allowed_ips, rl.max_messages, rl.window_seconds
query = `SELECT rl.allowed_ips, rl.max_messages, rl.window_seconds, rl.mode, rl.auto_multiplier, rl.auto_updated_at
FROM rate_limits rl JOIN domains d ON d.id = rl.ref_id
WHERE rl.scope = 'domain' AND d.name = ?`
case RateLimitScopeApp:
query = `SELECT rl.allowed_ips, rl.max_messages, rl.window_seconds
query = `SELECT rl.allowed_ips, rl.max_messages, rl.window_seconds, rl.mode, rl.auto_multiplier, rl.auto_updated_at
FROM rate_limits rl JOIN applications a ON a.id = rl.ref_id
WHERE rl.scope = 'application' AND a.login = ?`
default:
@@ -191,18 +228,29 @@ func (s *Store) CountMessages(scope, ref string, since time.Time) (int64, error)
// Active() until max and window are both set.
func scanRateLimit(r scanRow) (RateLimit, error) {
var (
ips sql.NullString
maxMsgs sql.NullInt64
windowSecs sql.NullInt64
ips sql.NullString
maxMsgs sql.NullInt64
windowSecs sql.NullInt64
mode sql.NullString
autoMult sql.NullFloat64
autoUpdated sql.NullString
)
if err := r.Scan(&ips, &maxMsgs, &windowSecs); err != nil {
if err := r.Scan(&ips, &maxMsgs, &windowSecs, &mode, &autoMult, &autoUpdated); err != nil {
return RateLimit{}, err
}
return RateLimit{
rl := RateLimit{
AllowedIPs: splitIPs(ips.String),
MaxMessages: int(maxMsgs.Int64),
WindowSeconds: int(windowSecs.Int64),
}, nil
Mode: mode.String,
}
if autoMult.Valid {
rl.AutoMultiplier = autoMult.Float64
}
if autoUpdated.Valid {
rl.AutoUpdatedAt, _ = time.Parse(time.RFC3339, autoUpdated.String)
}
return rl, nil
}
// splitIPs parses the comma-separated storage form back into a slice, dropping
+90
View File
@@ -1,6 +1,7 @@
package store
import (
"fmt"
"testing"
"time"
)
@@ -184,3 +185,92 @@ func TestRateLimitActiveAndAllowsIP(t *testing.T) {
t.Fatalf("unregistered/invalid IPs must not match")
}
}
func TestAutoRateLimitRecalc(t *testing.T) {
st := openTestStore(t)
d, _ := st.AddDomain("example.com", "selfpost")
a, _ := st.AddApplication(d.ID, "app1", AddressModeWildcard, nil)
for i := 0; i < 10; i++ {
if err := st.InsertQueued(SendLogEntry{
QueueID: fmt.Sprintf("Q%d", i), Domain: "example.com", AppLogin: "app1", To: "t@x.net",
}); err != nil {
t.Fatalf("InsertQueued: %v", err)
}
}
if err := st.SetRateLimit(RateLimit{
Scope: RateLimitScopeDomain, RefID: d.ID, Mode: RateLimitModeAuto,
AutoMultiplier: 2.0, WindowSeconds: 3600,
}); err != nil {
t.Fatalf("SetRateLimit domain: %v", err)
}
if err := st.RecalcAutoRateLimit(RateLimitScopeDomain, d.ID, 90, 100, 3600); err != nil {
t.Fatalf("RecalcAutoRateLimit domain: %v", err)
}
rl, ok, err := st.GetRateLimit(RateLimitScopeDomain, d.ID)
if err != nil || !ok {
t.Fatalf("GetRateLimit: ok=%v err=%v", ok, err)
}
if !rl.Active() || rl.MaxMessages > 100 {
t.Fatalf("domain auto limit = %+v", rl)
}
if rl.WindowSeconds != 3600 {
t.Fatalf("window = %d, want 3600", rl.WindowSeconds)
}
// Domain limit at ceiling; app auto must be strictly above or inactive at L1.
_ = st.SetRateLimit(RateLimit{
Scope: RateLimitScopeDomain, RefID: d.ID, Mode: RateLimitModeManual,
MaxMessages: 100, WindowSeconds: 3600,
})
if err := st.SetRateLimit(RateLimit{
Scope: RateLimitScopeApp, RefID: a.ID, Mode: RateLimitModeAuto,
AllowedIPs: []string{"203.0.113.1"}, AutoMultiplier: 2.0,
}); err != nil {
t.Fatalf("SetRateLimit app: %v", err)
}
if err := st.RecalcAutoRateLimit(RateLimitScopeApp, a.ID, 90, 100, 3600); err != nil {
t.Fatalf("RecalcAutoRateLimit app: %v", err)
}
appRL, ok, _ := st.GetRateLimit(RateLimitScopeApp, a.ID)
if ok && appRL.Active() {
t.Fatalf("app auto at L1 cap with domain at L1 should be inactive: %+v", appRL)
}
_ = st.SetRateLimit(RateLimit{
Scope: RateLimitScopeDomain, RefID: d.ID, Mode: RateLimitModeManual,
MaxMessages: 40, WindowSeconds: 3600,
})
if err := st.RecalcAutoRateLimit(RateLimitScopeApp, a.ID, 90, 100, 3600); err != nil {
t.Fatalf("RecalcAutoRateLimit app: %v", err)
}
appRL, ok, _ = st.GetRateLimit(RateLimitScopeApp, a.ID)
if !ok || !appRL.Active() || appRL.MaxMessages <= 40 {
t.Fatalf("app auto should be above domain 40: %+v", appRL)
}
// Milter reads the stored ceiling via RateLimit(name/login).
milterRL, ok, err := st.RateLimit(RateLimitScopeDomain, "example.com")
if err != nil || !ok || milterRL.MaxMessages != 40 {
t.Fatalf("milter domain limit = %+v ok=%v err=%v", milterRL, ok, err)
}
}
func TestAutoRateLimitZeroTrafficInactive(t *testing.T) {
st := openTestStore(t)
d, _ := st.AddDomain("quiet.com", "selfpost")
if err := st.SetRateLimit(RateLimit{
Scope: RateLimitScopeDomain, RefID: d.ID, Mode: RateLimitModeAuto,
AutoMultiplier: 2.5,
}); err != nil {
t.Fatalf("SetRateLimit: %v", err)
}
if err := st.RecalcAutoRateLimit(RateLimitScopeDomain, d.ID, 90, 100, 3600); err != nil {
t.Fatalf("RecalcAutoRateLimit: %v", err)
}
rl, ok, _ := st.GetRateLimit(RateLimitScopeDomain, d.ID)
if ok && rl.Active() {
t.Fatalf("zero traffic auto should be inactive: %+v", rl)
}
}
+101
View File
@@ -0,0 +1,101 @@
package store
import (
"fmt"
"math"
"time"
)
// StatsWindowDays is the rolling window for send statistics on the domain page.
const StatsWindowDays = 30
// SendStats holds message-volume metrics over a stats window (guide § Rate
// limiting — same counting rules as CountMessages).
type SendStats struct {
Total int64
PeakPerHour int64
AvgPerHour float64
WindowDays int // actual days queried (min(30, retention))
WindowHours float64 // hours used for the average-rate denominator
}
// StatsWindow computes the since timestamp and hours denominator for send
// statistics. The query window is min(30, retention) days; hours_in_window is
// min(720, entity age in hours, retention hours).
func StatsWindow(retentionDays int, entityCreated time.Time) (since time.Time, hours float64, windowDays int) {
if retentionDays <= 0 {
retentionDays = SendLogRetentionDaysDefault
}
windowDays = StatsWindowDays
if retentionDays < StatsWindowDays {
windowDays = retentionDays
}
since = time.Now().UTC().AddDate(0, 0, -windowDays)
retentionHours := float64(retentionDays) * 24
ageHours := time.Since(entityCreated).Hours()
if ageHours < 0 {
ageHours = 0
}
hours = math.Min(720, math.Min(ageHours, retentionHours))
if hours < 1 {
hours = 1
}
return since, hours, windowDays
}
// DomainSendStats returns 30-day (or shorter when retention is lower) send
// statistics for a domain name.
func (s *Store) DomainSendStats(name string, retentionDays int, createdAt time.Time) (SendStats, error) {
since, hours, windowDays := StatsWindow(retentionDays, createdAt)
stats, err := s.sendStats("domain", name, since, hours)
if err != nil {
return SendStats{}, err
}
stats.WindowDays = windowDays
stats.WindowHours = hours
return stats, nil
}
// AppSendStats returns send statistics for an application login.
func (s *Store) AppSendStats(login string, retentionDays int, createdAt time.Time) (SendStats, error) {
since, hours, windowDays := StatsWindow(retentionDays, createdAt)
stats, err := s.sendStats("app_login", login, since, hours)
if err != nil {
return SendStats{}, err
}
stats.WindowDays = windowDays
stats.WindowHours = hours
return stats, nil
}
func (s *Store) sendStats(column, ref string, since time.Time, hours float64) (SendStats, error) {
sinceStr := since.UTC().Format(time.RFC3339)
var total int64
err := s.db.QueryRow(
`SELECT COUNT(DISTINCT queue_id) FROM send_log
WHERE `+column+` = ? AND status != ? AND created_at >= ?`,
ref, StatusRejected, sinceStr,
).Scan(&total)
if err != nil {
return SendStats{}, fmt.Errorf("send stats total %s %q: %w", column, ref, err)
}
var peak int64
err = s.db.QueryRow(
`SELECT COALESCE(MAX(bucket_count), 0) FROM (
SELECT COUNT(DISTINCT queue_id) AS bucket_count FROM send_log
WHERE `+column+` = ? AND status != ? AND created_at >= ?
GROUP BY substr(created_at, 1, 13)
)`,
ref, StatusRejected, sinceStr,
).Scan(&peak)
if err != nil {
return SendStats{}, fmt.Errorf("send stats peak %s %q: %w", column, ref, err)
}
avg := float64(total) / hours
return SendStats{Total: total, PeakPerHour: peak, AvgPerHour: avg}, nil
}
+78
View File
@@ -0,0 +1,78 @@
package store
import (
"testing"
"time"
)
func TestSendStatsTotalPeakAvg(t *testing.T) {
st := openTestStore(t)
d, err := st.AddDomain("example.com", "selfpost")
if err != nil {
t.Fatalf("AddDomain: %v", err)
}
a, err := st.AddApplication(d.ID, "app1", AddressModeWildcard, nil)
if err != nil {
t.Fatalf("AddApplication: %v", err)
}
now := time.Now().UTC()
hour := now.Format("2006-01-02T15")
prevHour := now.Add(-2 * time.Hour).Format("2006-01-02T15")
// Hour 1: two messages (Q1 two recipients + Q2).
for _, to := range []string{"a@x.net", "b@x.net"} {
if err := st.InsertQueued(SendLogEntry{QueueID: "Q1", Domain: "example.com", AppLogin: "app1", To: to}); err != nil {
t.Fatalf("InsertQueued: %v", err)
}
}
if err := st.InsertQueued(SendLogEntry{QueueID: "Q2", Domain: "example.com", AppLogin: "app1", To: "c@x.net"}); err != nil {
t.Fatalf("InsertQueued: %v", err)
}
// Hour 2: one message.
if _, err := st.db.Exec(`UPDATE send_log SET created_at = ? WHERE queue_id = 'Q2'`,
prevHour+":00:00Z"); err != nil {
t.Fatalf("backdate Q2: %v", err)
}
if _, err := st.db.Exec(`UPDATE send_log SET created_at = ? WHERE queue_id = 'Q1'`,
hour+":10:00Z"); err != nil {
t.Fatalf("backdate Q1: %v", err)
}
if err := st.InsertRejected(SendLogEntry{Domain: "example.com", AppLogin: "app1"}); err != nil {
t.Fatalf("InsertRejected: %v", err)
}
stats, err := st.DomainSendStats("example.com", 90, d.CreatedAt)
if err != nil {
t.Fatalf("DomainSendStats: %v", err)
}
if stats.Total != 2 {
t.Fatalf("total = %d, want 2", stats.Total)
}
if stats.PeakPerHour != 1 {
t.Fatalf("peak = %d, want 1 (one message per hour bucket)", stats.PeakPerHour)
}
if stats.AvgPerHour <= 0 {
t.Fatalf("avg should be positive, got %v", stats.AvgPerHour)
}
appStats, err := st.AppSendStats("app1", 90, a.CreatedAt)
if err != nil {
t.Fatalf("AppSendStats: %v", err)
}
if appStats.Total != 2 {
t.Fatalf("app total = %d, want 2", appStats.Total)
}
}
func TestStatsWindowShortRetention(t *testing.T) {
st := openTestStore(t)
d, _ := st.AddDomain("example.com", "selfpost")
stats, err := st.DomainSendStats("example.com", 7, d.CreatedAt)
if err != nil {
t.Fatalf("DomainSendStats: %v", err)
}
if stats.WindowDays != 7 {
t.Fatalf("window days = %d, want 7", stats.WindowDays)
}
}