audit 3: every per-message question of the send log has its index
test / test (push) Canceled after 0s
test / test (push) Canceled after 0s
Migration 0010 (DDL only; 0009 went to stage 2b). EXPLAIN QUERY PLAN of
every send_log statement showed, beyond M2 as written (stage 2 had already
moved the by-domain counts onto (domain_id, created_at)):
- the tailer's sweep of open rows (every 5 min) and its count scanned the
table: partial idx_send_log_open (created_at) WHERE status IN ('queued',
'deferred'), the statuses literal in the statements, ORDER BY +id (else
SQLite walks the table in id order to skip a sort);
- AttachRefusalLine's NOT EXISTS (... refusal_line = ?1) scanned the table
for every refusal line: partial idx_send_log_refusal_line, the query
carrying the index's WHERE; its row found by idx_send_log_rejected
(status literal, +queue_id) instead of every refusal without a queue id;
- an application's level-2 count went through its domain's index:
+domain_id in appByLogin;
- a page of one domain's or one application's log sorted all of it:
idx_send_log_domain_id, idx_send_log_app_login (id order; several domains
still sort, top-N). idx_send_log_domain stays for the global filter.
The statements are constants and builders (countMessagesQuery,
peakPerHourQuery, listOpenQuery, countOpenQuery, sendLogPageQuery,
attachRefusalLineQuery, limitRefusalsQuery) and
TestSendLogQueriesUseTheirIndexes explains exactly those: the index each
must use, no SCAN of send_log, no sort where none is wanted; the literal
statuses checked against the constants. It replaces the hand-written plan
check of TestLimitRefusalsSince.
Tested on the test server: go vet + go test of 24 packages, make e2e (all
pass) on this tree.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+46
-24
@@ -209,6 +209,20 @@ func (s *Store) ExpireMessage(queueID string) (int64, error) {
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// openRows selects the rows still open, StatusQueued and StatusDeferred
|
||||
// written as literals: a term the same as the WHERE of the partial index
|
||||
// idx_send_log_open (migration 0010) is what lets SQLite use it
|
||||
// (https://sqlite.org/partialindex.html § 3), and a bound parameter is not.
|
||||
const openRows = `status IN ('queued', 'deferred') AND queue_id <> ''`
|
||||
|
||||
// listOpenQuery is ListOpenOlderThan's. The unary + on the order keeps SQLite
|
||||
// from walking the whole table in id order instead of the index
|
||||
// (https://sqlite.org/optoverview.html § 18): the open rows are few, and
|
||||
// sorting them is nothing.
|
||||
const listOpenQuery = `SELECT id, queue_id, to_addr FROM send_log
|
||||
WHERE ` + openRows + ` AND created_at < ?
|
||||
ORDER BY +id`
|
||||
|
||||
// QueuedDelivery is a send-log row still waiting for a delivery result,
|
||||
// reduced to what the log-tailer's reconcile sweep needs to look it up in the
|
||||
// Postfix queue and, failing that, to close it (architecture.md § Log tailer).
|
||||
@@ -232,12 +246,7 @@ type QueuedDelivery struct {
|
||||
// BINARY collation, memcmp() (https://sqlite.org/datatype3.html § 7), and
|
||||
// time.RFC3339 of a UTC time is fixed-width, YYYY-MM-DDTHH:MM:SSZ.
|
||||
func (s *Store) ListOpenOlderThan(cutoff time.Time) ([]QueuedDelivery, error) {
|
||||
rows, err := s.db.Query(
|
||||
`SELECT id, queue_id, to_addr FROM send_log
|
||||
WHERE status IN (?, ?) AND queue_id <> '' AND created_at < ?
|
||||
ORDER BY id`,
|
||||
StatusQueued, StatusDeferred, cutoff.UTC().Format(time.RFC3339),
|
||||
)
|
||||
rows, err := s.db.Query(listOpenQuery, cutoff.UTC().Format(time.RFC3339))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list queued send_log rows: %w", err)
|
||||
}
|
||||
@@ -284,6 +293,23 @@ type SendLogRow struct {
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
// attachRefusalLineQuery is AttachRefusalLine's. The refused row is found by
|
||||
// its time among the refused rows
|
||||
// (idx_send_log_rejected: the status a literal, and the unary + keeps
|
||||
// SQLite from walking every refusal without a queue id instead), and the
|
||||
// line by idx_send_log_refusal_line (migration 0010; the second term is that
|
||||
// index's WHERE) - https://sqlite.org/partialindex.html § 3,
|
||||
// optoverview.html § 18.
|
||||
const attachRefusalLineQuery = `UPDATE send_log SET refusal_line = ?1 WHERE id = (
|
||||
SELECT id FROM send_log
|
||||
WHERE status = 'rejected' AND +queue_id = '' AND refusal_line = ''
|
||||
AND reason IN ('', ?2, ?3) AND from_addr = ?4 COLLATE NOCASE AND client_ip = ?5
|
||||
AND app_login = ?9 COLLATE NOCASE
|
||||
AND created_at BETWEEN ?6 AND ?7
|
||||
ORDER BY ABS(julianday(created_at) - julianday(?8)), id
|
||||
LIMIT 1)
|
||||
AND NOT EXISTS (SELECT 1 FROM send_log WHERE refusal_line = ?1 AND refusal_line <> '')`
|
||||
|
||||
// AttachRefusalLine puts Postfix's line about a refusal at MAIL FROM on the
|
||||
// row the journal milter wrote for it (audit 2026-10-12, stage 2b). Such a
|
||||
// refusal has no queue id, and the line names no login, so the row is found
|
||||
@@ -302,17 +328,8 @@ type SendLogRow struct {
|
||||
// § 2); NOCASE folds ASCII only (datatype3.html § 7.1); ?1 is bound once and
|
||||
// used twice (lang_expr.html § 4).
|
||||
func (s *Store) AttachRefusalLine(clientIP, login, from string, at time.Time, line string) (bool, error) {
|
||||
res, err := s.db.Exec(
|
||||
`UPDATE send_log SET refusal_line = ?1 WHERE id = (
|
||||
SELECT id FROM send_log
|
||||
WHERE status = ?2 AND queue_id = '' AND refusal_line = ''
|
||||
AND reason IN ('', ?3, ?4) AND from_addr = ?5 COLLATE NOCASE AND client_ip = ?6
|
||||
AND app_login = ?10 COLLATE NOCASE
|
||||
AND created_at BETWEEN ?7 AND ?8
|
||||
ORDER BY ABS(julianday(created_at) - julianday(?9)), id
|
||||
LIMIT 1)
|
||||
AND NOT EXISTS (SELECT 1 FROM send_log WHERE refusal_line = ?1)`,
|
||||
line, StatusRejected, ReasonLimit, ReasonClient, from, clientIP,
|
||||
res, err := s.db.Exec(attachRefusalLineQuery,
|
||||
line, ReasonLimit, ReasonClient, from, clientIP,
|
||||
at.Add(-refusalLineWindow).UTC().Format(time.RFC3339),
|
||||
at.Add(refusalLineWindow).UTC().Format(time.RFC3339),
|
||||
at.UTC().Format(time.RFC3339),
|
||||
@@ -409,19 +426,24 @@ type SendLogFilter struct {
|
||||
AllDomains bool
|
||||
}
|
||||
|
||||
// sendLogPageQuery is a page of the rows where selects, newest first. A page
|
||||
// of one domain or one application reads its rows off idx_send_log_domain_id
|
||||
// or idx_send_log_app_login (migration 0010), already in id order; a scope of
|
||||
// several domains is sorted, keeping only the page.
|
||||
func sendLogPageQuery(where string) string {
|
||||
return `SELECT id, queue_id, domain, COALESCE(domain_id, 0), app_login, from_addr, to_addr, subject, status, reason, header_from, created_at, updated_at
|
||||
FROM send_log` + where + `
|
||||
ORDER BY id DESC
|
||||
LIMIT ? OFFSET ?`
|
||||
}
|
||||
|
||||
// QuerySendLog returns send-log rows matching filter, newest first, for the
|
||||
// monitoring screen's server-side pagination (product.md's send-log view).
|
||||
// The filter's authorization scope is mandatory: see SendLogFilter.
|
||||
func (s *Store) QuerySendLog(filter SendLogFilter, limit, offset int) ([]SendLogRow, error) {
|
||||
where, args := sendLogWhere(filter)
|
||||
args = append(args, limit, offset)
|
||||
rows, err := s.db.Query(
|
||||
`SELECT id, queue_id, domain, COALESCE(domain_id, 0), app_login, from_addr, to_addr, subject, status, reason, header_from, created_at, updated_at
|
||||
FROM send_log`+where+`
|
||||
ORDER BY id DESC
|
||||
LIMIT ? OFFSET ?`,
|
||||
args...,
|
||||
)
|
||||
rows, err := s.db.Query(sendLogPageQuery(where), args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query send_log: %w", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user