From 6db37047110cce1af87a9f6898a4fa4320b52b02 Mon Sep 17 00:00:00 2001 From: Mix Date: Sun, 18 Oct 2026 23:17:14 +0300 Subject: [PATCH] =?UTF-8?q?queue-table:=20Outbound=20=E2=80=BA=20Queue=20a?= =?UTF-8?q?s=20the=20drawn=20table,=20Inbound=20=E2=80=BA=20Queue=20built?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner, 2026-10-18: "Фиаско, что очередь не была сделана. Как мы это допустили? Исправляем!" The mockups out-queue and in-queue (design 629de5e, guards 8525e8e). - postfix: postqueue -j gives each message's size and each recipient's delay_reason (src/postqueue/showq_json.c, format_json(): "delay_reason" only when LEN(why) > 0); QueueListing reads it with the page's 5 s, QueueMessages keeps the notifier's 30 s. - view: one Queue page type for both directions (queue.go), one polled box (queue_body.html): a row per recipient, oldest first, cut at 200 with a foot; the head "N messages · size", "Refreshes on its own" while empty; Last reply is the delay reason, or where the message is (active, hold, incoming) before there was one. Inbound has no Details: an inbound message has no page of its own. - handlers: one read of the queue, split by the recipient's domain against the inbound domains; an outbound recipient is linked to its send-log row (store.SendLogRefsByQueueID, through idx_send_log_queue_id, plan pinned in TestSendLogQueriesUseTheirIndexes), matched without regard to case, else the message's first row. Both pages are the global role's; the inbound one 404s with the relay off. Routes /inbound/queue (+ /fragment), menu, touch menu and sibling strip. - Overview: Queues · Both empty · outbound and inbound with the relay on. - Help: the queue topic rewritten, an Inbound queue topic (word for word with the mockup; domain administrators do not see it, as with the lists). - TestUnbuiltScreensAreOnTheRoadmap and its rule in development.md: an exempted screen must be named in roadmap.md; queue-table leaves the roadmap; guide, architecture, panel-redesign.md updated. Tests: go vet; go test ./internal/web/... ./internal/store ./internal/postfix ./internal/notify pass. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 26 +++ docs/architecture.md | 2 +- docs/development.md | 9 + docs/guide.md | 17 +- docs/plans/panel-redesign.md | 9 +- docs/roadmap.md | 20 -- internal/notify/notify.go | 2 +- internal/notify/notify_test.go | 6 +- internal/postfix/queuejson.go | 50 ++-- internal/postfix/queuejson_test.go | 14 +- internal/store/queryplan_test.go | 2 + internal/store/sendlog.go | 54 +++++ internal/store/sendlog_test.go | 26 +++ internal/web/handlers/handlers_monitor.go | 120 +++++++++- .../web/handlers/handlers_monitor_test.go | 3 + internal/web/handlers/handlers_queue_test.go | 110 +++++++++ internal/web/handlers/handlers_status.go | 28 ++- internal/web/handlers/handlers_status_test.go | 7 + internal/web/view/components_test.go | 8 +- internal/web/view/fixtures_test.go | 25 +- internal/web/view/help.go | 14 +- internal/web/view/inbound_log_test.go | 9 +- internal/web/view/outbound_queue.go | 82 ------- internal/web/view/outbound_queue_test.go | 60 ----- internal/web/view/queue.go | 221 ++++++++++++++++++ internal/web/view/queue_test.go | 123 ++++++++++ internal/web/view/templates/in_queue.html | 14 ++ internal/web/view/templates/layout.html | 7 +- internal/web/view/templates/out_queue.html | 8 +- .../web/view/templates/out_queue_body.html | 6 - internal/web/view/templates/queue_body.html | 13 ++ internal/web/view/unbuilt_screens_test.go | 32 +++ internal/web/view/view.go | 5 +- internal/web/web.go | 7 +- 34 files changed, 893 insertions(+), 246 deletions(-) create mode 100644 internal/web/handlers/handlers_queue_test.go delete mode 100644 internal/web/view/outbound_queue.go delete mode 100644 internal/web/view/outbound_queue_test.go create mode 100644 internal/web/view/queue.go create mode 100644 internal/web/view/queue_test.go create mode 100644 internal/web/view/templates/in_queue.html delete mode 100644 internal/web/view/templates/out_queue_body.html create mode 100644 internal/web/view/templates/queue_body.html create mode 100644 internal/web/view/unbuilt_screens_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 381a6b5..884d2b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,32 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); version ## [Unreleased] +### Added + +- **Inbound › Queue** (`/inbound/queue`, global role, with the inbound relay + on): mail accepted on port 25 that its upstream has not yet taken, a row + per recipient with the upstream's last reply - the screen `in-queue`, drawn + for 2.0 and not built until now. Help has its topic. + +### Changed + +- **Outbound › Queue is the table the mockup draws** instead of + `postqueue -p`'s text: a row per recipient, oldest first - queue id, when + it arrived, sender, recipient, and the last reply of the receiving server + or why no connection was made - with *Details* to the message in the + Outbound log when it has a row there. The box head counts the messages and + their size; past 200 recipients the oldest 200 are listed and a line says + how many wait. Read from `postqueue -j` (Postfix 3.10.13, + `src/postqueue/showq_json.c`: `message_size`, and a recipient's + `delay_reason` when it has one). A recipient of an inbound domain is on + Inbound › Queue instead. +- **Overview's queue card** is *Queues*, "Both empty", "outbound and inbound" + while the inbound relay is on, as the mockup has it: Postfix keeps one queue + for both. +- `TestUnbuiltScreensAreOnTheRoadmap`: a screen the guards let pass unbuilt + must be named in `docs/roadmap.md`. The two queues had been deferred on + 2026-10-09 with no roadmap row and were forgotten; development.md says so. + ## [2.15.0] - 2026-10-18 The two proposals the security audit left as product choices. Migration diff --git a/docs/architecture.md b/docs/architecture.md index 75af945..b8366b6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -863,7 +863,7 @@ only reads (the delete confirmation at `GET …/delete` renders a form). | Outbound › Log, Queue | `GET /outbound/log` · `/outbound/log/fragment` · `/outbound/log/{id}` · `GET /outbound/queue` · `/outbound/queue/fragment` | | Outbound › DMARC reports | `GET /outbound/dmarc` · `…/dmarc/domains/{id}` · `…/dmarc/reports/{id}` — registered only with `DMARC_REPORTS_ENABLE=true` | | Inbound › Domains | `GET, POST /inbound/domains` · `/inbound/domains/{id}` · `POST …/dns-recheck`, `…/upstream`, `…/recipients` · `GET, POST …/delete` — registered only with `INBOUND_RELAY_ENABLE=true` | -| Inbound › Log, Filter lists | `GET /inbound/log` · `GET, POST /inbound/filter-lists` · `POST /inbound/filter-lists/{id}/delete` — registered only with `INBOUND_RELAY_ENABLE=true` | +| Inbound › Log, Queue, Filter lists | `GET /inbound/log` · `GET /inbound/queue` · `/inbound/queue/fragment` · `GET, POST /inbound/filter-lists` · `POST /inbound/filter-lists/{id}/delete` — registered only with `INBOUND_RELAY_ENABLE=true` | | Server | `GET /server/health` (+ `/fragment`) · `POST /server/health/recheck`, `/server/health/reload`, `/server/health/test-message` · `GET /server/log` (+ `/fragment`) · `GET, POST /server/backup` · `POST /server/backup/import` · `/server/users`, `…/new`, `…/{uid}`, `…/{uid}/delete` · `GET, POST /server/settings` · `POST /server/settings/dns-recheck` · `GET /server/components` | | Account | `GET /account` · `POST /account/profile`, `/account/password`, `/account/email/confirm` | | User menu | `GET /help` · `POST /logout` (public mux) | diff --git a/docs/development.md b/docs/development.md index 4467c4a..3c20d47 100644 --- a/docs/development.md +++ b/docs/development.md @@ -451,6 +451,15 @@ that changes the mockups, a guard or the script touches nothing else under [plans/panel-redesign.md](plans/panel-redesign.md) § The contract and § Enforcement have the full rules. +**A drawn screen that is not built has a row.** A screen the mockups draw and +the panel does not build, or builds otherwise (the guards' `deferredScreens` +and `outlineExempt`), is named in [roadmap.md](roadmap.md) - a row of the index, +or an idea under *Ideas* - in the same commit that exempts it. +`TestUnbuiltScreensAreOnTheRoadmap` fails otherwise. The queue as a table and +the Inbound queue were drawn for 2.0, deferred on 2026-10-09 with no row of +their own, and then forgotten for nine releases, the guards green because they +had been told to be (owner, 2026-10-18: "Фиаско"); built in `2.16.0`. + ### End-to-end (container suite) Separate Go module `test/e2e/`; **not** included in the main module's diff --git a/docs/guide.md b/docs/guide.md index 4d7d4df..fa5770c 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -447,7 +447,7 @@ changes. |---|---|---| | Overview | Overview | `/overview` | | Outbound | Domains · Log · Queue · DMARC reports | `/outbound/domains`, `/outbound/log`, `/outbound/queue`, `/outbound/dmarc` | -| Inbound | Domains · Log · Filter lists (global role) | `/inbound/domains`, `/inbound/log`, `/inbound/filter-lists` | +| Inbound | Domains · Log · Queue and Filter lists (global role) | `/inbound/domains`, `/inbound/log`, `/inbound/queue`, `/inbound/filter-lists` | | Server | Health · System log · Backup · Users · Settings | `/server/health`, `/server/log`, `/server/backup`, `/server/users`, `/server/settings` | | User menu (your name) | Account · Help · Sign out | `/account`, `/help`, `POST /logout` | @@ -564,14 +564,25 @@ past that the page shows the last result). *Greeting* is the `220` line Postfix ### Queue and System log - **Outbound › Queue** (`/outbound/queue`, global role) — live view of - messages Postfix is still trying to deliver, deferring or holding, as - `postqueue -p` lists them. It only shows: nothing there deletes or forces a message. A box + messages Postfix is still trying to deliver, deferring or holding, a row + per recipient, oldest first: queue id, when it arrived, sender, recipient, + and the last reply of the receiving server or why no connection was made + (from `postqueue -j`). *Details* opens the message in the Outbound log when + it has a row there; the panel's own mail and bounces have none. The box + head counts the messages and their size; past 200 recipients the oldest 200 + are listed and a line says how many there are - `postqueue -p` in the + container lists them all. It only shows: nothing there deletes or forces a message. A box states this instance's retry policy — first retry delay, later backoff cap, how long a message stays in the queue — from `postconf -h`, read once when the panel starts. A `postconf -e` override inside the container is visible after the next panel (or container) restart. There is no maximum attempt count: Postfix retries until the message is delivered or the queue lifetime runs out. +- **Inbound › Queue** (`/inbound/queue`, global role, with the inbound relay + on) — the same Postfix queue, the recipients of inbound domains: mail + accepted on port 25 that its upstream has not yet taken, with the + upstream's last reply. A recipient belongs here when its domain is an + inbound domain, and to Outbound › Queue otherwise. - **Server › System log** (`/server/log`) — tail of `/data/log/mail.log` (Postfix and related daemon lines). The log rotates daily (14 files kept) with a `postfix reload` after each rotation; a background loop checks every diff --git a/docs/plans/panel-redesign.md b/docs/plans/panel-redesign.md index 726af00..66a0a7e 100644 --- a/docs/plans/panel-redesign.md +++ b/docs/plans/panel-redesign.md @@ -95,7 +95,7 @@ only. | Outbound: domain, applications, settings, log, DMARC | all domains | assigned outbound domains | | Inbound: domain (upstream, recipients), log | all domains | assigned inbound domains | | Add / delete a domain, either direction | yes | no | -| Queues, either direction | yes | no, until the queue is a filterable table | +| Queues, either direction | yes | no — the queue is the instance's; a domain's rows for its administrator would be a decision of its own | | Inbound › Quarantine | all domains | assigned inbound domains | | Inbound › Filter lists, Server › Preflight | yes | no — the lists are instance-wide | | Account, Help | yes | yes | @@ -215,8 +215,10 @@ Rules that come with the rename: **Needs its own roadmap decision before it is built** (the mockups show the target; none of this is smuggled in with the restyle): -- Queue as a table — parse `postqueue -j` instead of printing `postqueue -p`. -- Inbound queue — the same queue filtered by recipient domain. +- Queue as a table — parse `postqueue -j` instead of printing `postqueue -p` + (built in `2.16.0`). +- Inbound queue — the same queue filtered by recipient domain (built in + `2.16.0`). - Inbound log — rides on `inbound-antispam-panel` (`2.1.0`). - Server › Health — a route for the tables removed from Overview, and the home of *Reload configuration* (today a card on `/status`). @@ -236,7 +238,6 @@ Drawn and not built (`deferredScreens` in the guard tests): | Screen | Item | State | |---|---|---| -| `in-queue` | [queue-table](../roadmap.md#queue-table): the queue as a table from `postqueue -j`, and an Inbound › Queue | undecided; Outbound › Queue shows postqueue's listing in a log pane (`outlineExempt`) | | `in-quarantine`, `in-quarantine-message` | inbound-quarantine, an idea off the roadmap ([its notes](inbound-quarantine.md)) | its open questions stay open; the mockup assumes SelfPost-held mail, release to upstream, 14 days, delegated like the rest of Inbound | The sibling strip under the navbar holds five or six entries per group without diff --git a/docs/roadmap.md b/docs/roadmap.md index 85e8052..aeea19a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -30,7 +30,6 @@ directory). |---|---|---|---|---| | security-audit-2026-10-17 | Stage H of the 2026-10-17 security audit: the test server's host (S1, S2, S5) and its DNS (S3) | stages 0-6 shipped `2.14.4`, the product side of S3 `2.15.0`; stage H the owner's | 27/29 | [security-audit-2026-10-17.md](security-audit-2026-10-17.md) § Stage H | | go-milter-upstream | An issue and a PR for go-milter: its header map outlives a message and is unbounded | prepared 2026-10-16, the owner posts | — | [third_party/go-milter/PATCHES.md](../third_party/go-milter/PATCHES.md) | -| queue-table | Outbound › Queue as the table the mockup draws, from `postqueue -j`; an Inbound › Queue (`in-queue`) | candidate, undecided since 2026-10-09 | — | — | After a context reset, pick an item marked `agreed` or `in progress`, then work the **Implementation checklist** in its linked plan. The `Progress` column above @@ -93,25 +92,6 @@ option), `third_party/go-milter` goes and `go.mod`'s `replace` with it. --- -## queue-table - -**Problem:** the 2.0 mockups draw Outbound › Queue as a table and an -Inbound › Queue (`in-queue`); the redesign kept both out as decisions of -their own (owner, 2026-10-09). Outbound › Queue shows `postqueue -p`'s own -listing in a log pane, and the mockup's `in-queue` is drawn but not built -(`deferredScreens`, `outlineExempt` in the guard tests). - -**Goal:** the outbound queue as a table from `postqueue -j` - a row per -message with its sender, recipients, age and last reason - and, if the owner -wants it, the same for mail port 25 holds for an upstream. - -**Recommendation:** the outbound table first, read-only - about 4 h with its -tests, since the notifier already reads `postqueue -j` for each message's -arrival time; the inbound queue and any action on a row (retry, delete) only -if the owner asks for them, each drawn mockup-first. - ---- - ## Dropped, with the condition for return Items the owner took off the roadmap. Each names what would bring it back; diff --git a/internal/notify/notify.go b/internal/notify/notify.go index cbf6e90..8b75b85 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -541,7 +541,7 @@ func (n *Notifier) ownMail(m postfix.QueuedMessage) bool { return true } for _, rcpt := range m.Recipients { - if strings.EqualFold(rcpt, own) { + if strings.EqualFold(rcpt.Address, own) { return true } } diff --git a/internal/notify/notify_test.go b/internal/notify/notify_test.go index 382de4c..2e91d63 100644 --- a/internal/notify/notify_test.go +++ b/internal/notify/notify_test.go @@ -460,12 +460,12 @@ func TestQueueLeavesOutTheNotifiersOwnMail(t *testing.T) { b := newBench(t) old := b.now.Add(-4 * 24 * time.Hour) b.q = []postfix.QueuedMessage{ - {ID: "NOTE0001", Sender: "postmaster@mail.example.org", Recipients: []string{"gone@example.net"}, Arrival: old}, - {ID: "BOUNCE01", Sender: "", Recipients: []string{"Postmaster@Mail.Example.Org"}, Arrival: old}, + {ID: "NOTE0001", Sender: "postmaster@mail.example.org", Recipients: []postfix.QueuedRecipient{{Address: "gone@example.net"}}, Arrival: old}, + {ID: "BOUNCE01", Sender: "", Recipients: []postfix.QueuedRecipient{{Address: "Postmaster@Mail.Example.Org"}}, Arrival: old}, } want(t, "only its own mail is old", b.hour()) - b.q = append(b.q, postfix.QueuedMessage{ID: "APP00001", Sender: "", Recipients: []string{"app@example.com"}, Arrival: old}) + b.q = append(b.q, postfix.QueuedMessage{ID: "APP00001", Sender: "", Recipients: []postfix.QueuedRecipient{{Address: "app@example.com"}}, Arrival: old}) if got := b.hour(); len(got) != 1 { t.Errorf("an application's bounce is old too: %v, want one message", got) } diff --git a/internal/postfix/queuejson.go b/internal/postfix/queuejson.go index 5a428e4..3ef078c 100644 --- a/internal/postfix/queuejson.go +++ b/internal/postfix/queuejson.go @@ -11,7 +11,7 @@ import ( ) // QueuedMessage is one message Postfix is holding, as `postqueue -j` reports -// it: where it is, and since when. +// it: where it is, since when, and what became of each recipient's last try. // // Postfix 3.10.13, src/postqueue/showq_json.c, format_json(): one object and a // newline per message - "queue_name", "queue_id", "arrival_time" (seconds, @@ -22,21 +22,31 @@ type QueuedMessage struct { ID string Queue string // "deferred", "active", "hold", "incoming", "maildrop" Sender string - Recipients []string + Recipients []QueuedRecipient Arrival time.Time + Size int64 // bytes, "message_size" } -// queueJSONTimeout is longer than the listing for a page gets: the notifier -// asks once an hour and can wait, and a queue large enough to be slow to list -// is the one that must not go unreported. +// QueuedRecipient is one recipient of a queued message. DelayReason is why its +// last delivery attempt did not finish - the remote server's reply, or why no +// connection was made - and empty when there was none yet (showq_json.c writes +// "delay_reason" only for a recipient that has one). +type QueuedRecipient struct { + Address string + DelayReason string +} + +// queueJSONTimeout is longer than the listing for a page gets (postqueueTimeout): +// the notifier asks once an hour and can wait, and a queue large enough to be +// slow to list is the one that must not go unreported. const queueJSONTimeout = 30 * time.Second -// readQueueJSON runs `postqueue -j`. It is a var so tests can answer without a -// running Postfix, the way readRetryConf is. +// readQueueJSON runs `postqueue -j` with a deadline. It is a var so tests can +// answer without a running Postfix, the way readRetryConf is. var readQueueJSON = postqueueJSON -func postqueueJSON() ([]byte, error) { - ctx, cancel := context.WithTimeout(context.Background(), queueJSONTimeout) +func postqueueJSON(timeout time.Duration) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() // A fixed flag and no user input, never through a shell (security.md). cmd := exec.CommandContext(ctx, "postqueue", "-j") @@ -46,7 +56,7 @@ func postqueueJSON() ([]byte, error) { cmd.Stderr = errOut err := cmd.Run() if ctx.Err() == context.DeadlineExceeded { - return nil, fmt.Errorf("postqueue -j timed out after %s", queueJSONTimeout) + return nil, fmt.Errorf("postqueue -j timed out after %s", timeout) } if err != nil { return nil, fmt.Errorf("postqueue -j: %w: %s", err, strings.TrimSpace(errOut.buf.String())) @@ -63,7 +73,18 @@ func postqueueJSON() ([]byte, error) { // for the notification about mail that has waited too long. An error means the // queue could not be listed and says nothing about any message. func QueueMessages() ([]QueuedMessage, error) { - out, err := readQueueJSON() + out, err := readQueueJSON(queueJSONTimeout) + if err != nil { + return nil, err + } + return parseQueueJSON(out) +} + +// QueueListing is QueueMessages for a page: the Outbound and Inbound queues +// (architecture.md § Panel HTTP surface). A page waits as long as the +// `postqueue -p` listing did, not the notifier's half minute. +func QueueListing() ([]QueuedMessage, error) { + out, err := readQueueJSON(postqueueTimeout) if err != nil { return nil, err } @@ -86,9 +107,11 @@ func parseQueueJSON(out []byte) ([]QueuedMessage, error) { QueueName string `json:"queue_name"` QueueID string `json:"queue_id"` ArrivalTime int64 `json:"arrival_time"` + MessageSize int64 `json:"message_size"` Sender string `json:"sender"` Recipients []struct { - Address string `json:"address"` + Address string `json:"address"` + DelayReason string `json:"delay_reason"` } `json:"recipients"` } if err := json.Unmarshal([]byte(line), &e); err != nil || e.QueueID == "" || e.ArrivalTime <= 0 { @@ -99,9 +122,10 @@ func parseQueueJSON(out []byte) ([]QueuedMessage, error) { Queue: e.QueueName, Sender: e.Sender, Arrival: time.Unix(e.ArrivalTime, 0).UTC(), + Size: e.MessageSize, } for _, r := range e.Recipients { - m.Recipients = append(m.Recipients, r.Address) + m.Recipients = append(m.Recipients, QueuedRecipient{Address: r.Address, DelayReason: r.DelayReason}) } msgs = append(msgs, m) } diff --git a/internal/postfix/queuejson_test.go b/internal/postfix/queuejson_test.go index b0e328e..95f479b 100644 --- a/internal/postfix/queuejson_test.go +++ b/internal/postfix/queuejson_test.go @@ -28,10 +28,15 @@ not json at all !got[0].Arrival.Equal(time.Unix(1791000000, 0)) { t.Errorf("first message = %+v", got[0]) } - if len(got[0].Recipients) != 1 || got[0].Recipients[0] != "a@example.net" { + if len(got[0].Recipients) != 1 || got[0].Recipients[0] != (QueuedRecipient{"a@example.net", "connect to mx.example.net[203.0.113.9]:25: Connection timed out"}) { t.Errorf("first message's recipients = %v", got[0].Recipients) } - if got[1].ID != "5F00AA12" || got[1].Queue != "active" { + if got[0].Size != 446 { + t.Errorf("first message's size = %d, want 446", got[0].Size) + } + // A recipient not tried yet has no delay_reason (showq_json.c writes it + // only when there is one). + if got[1].ID != "5F00AA12" || got[1].Queue != "active" || got[1].Recipients[0] != (QueuedRecipient{Address: "b@example.net"}) { t.Errorf("second message = %+v", got[1]) } if empty, err := parseQueueJSON(nil); err != nil || len(empty) != 0 { @@ -50,9 +55,12 @@ not json at all func TestQueueMessagesReportsAFailedListing(t *testing.T) { old := readQueueJSON t.Cleanup(func() { readQueueJSON = old }) - readQueueJSON = func() ([]byte, error) { return nil, errors.New("postqueue: Permission denied") } + readQueueJSON = func(time.Duration) ([]byte, error) { return nil, errors.New("postqueue: Permission denied") } if msgs, err := QueueMessages(); err == nil { t.Fatalf("no error for a failed listing (got %v)", msgs) } + if msgs, err := QueueListing(); err == nil { + t.Fatalf("no error for a failed listing on a page (got %v)", msgs) + } } diff --git a/internal/store/queryplan_test.go b/internal/store/queryplan_test.go index 9ca50ec..71a8dd0 100644 --- a/internal/store/queryplan_test.go +++ b/internal/store/queryplan_test.go @@ -61,6 +61,8 @@ func TestSendLogQueriesUseTheirIndexes(t *testing.T) { []string{"SEARCH send_log USING INDEX idx_send_log_domain_id_created_at (domain_id=? AND created_at>?)"}, true}, {"busiest hour of an application", peakPerHourQuery(appByLogin), []any{"app1", StatusRejected, since}, []string{"SEARCH send_log USING INDEX idx_send_log_app_login_created_at (app_login=? AND created_at>?)"}, true}, + {"the rows of queued messages", sendLogRefsQuery(2), []any{"4XcB7k2Jm9z1", "5F00AA12"}, + []string{"SEARCH send_log USING INDEX idx_send_log_queue_id (queue_id=?)"}, true}, {"open rows", listOpenQuery, []any{since}, []string{"SEARCH send_log USING INDEX idx_send_log_open (created_at= refs["Q1"][1].ID { + t.Errorf("refs = %+v", refs) + } + if refs, err := s.SendLogRefsByQueueID(nil); err != nil || len(refs) != 0 { + t.Errorf("no ids: %v, %v", refs, err) + } +} diff --git a/internal/web/handlers/handlers_monitor.go b/internal/web/handlers/handlers_monitor.go index 3f23f3a..2cb1365 100644 --- a/internal/web/handlers/handlers_monitor.go +++ b/internal/web/handlers/handlers_monitor.go @@ -9,6 +9,7 @@ import ( "slices" "sort" "strconv" + "strings" "time" "github.com/mixeme/selfpost/internal/logtail" @@ -461,9 +462,8 @@ func (h *Handlers) HandleMailQueue(w http.ResponseWriter, r *http.Request) { if _, ok := h.requireGlobal(w, r); !ok { return } - policy := h.cfg.RetryPolicy - h.view.Render(w, http.StatusOK, "out-queue", h.outQueuePage(r). - WithRetryPolicy(policy.FirstRetry(), policy.BackoffCap(), policy.QueueLifetime(), policy.FromDefaults)) + out, _, errText := h.queueEntries() + h.view.Render(w, http.StatusOK, "out-queue", h.withRetryPolicy(view.NewOutQueue(h.shellMeta(r), out, errText))) } // HandleMailQueueBody serves the HTMX polling fragment for the queue view. @@ -471,14 +471,116 @@ func (h *Handlers) HandleMailQueueBody(w http.ResponseWriter, r *http.Request) { if _, ok := h.requireGlobal(w, r); !ok { return } - h.view.RenderFragment(w, http.StatusOK, "out_queue_body", h.outQueuePage(r)) + out, _, errText := h.queueEntries() + h.view.RenderFragment(w, http.StatusOK, "queue_body", view.NewOutQueue(h.shellMeta(r), out, errText)) } -// outQueuePage reads the queue and gives it to the view; the page and its -// fragment draw the same box from it. -func (h *Handlers) outQueuePage(r *http.Request) *view.OutQueue { - out, errText := readQueue() - return view.NewOutQueue(h.shellMeta(r), out, errText) +// HandleInboundQueue renders the Inbound queue page: the recipients of inbound +// domains still waiting for their upstream. Like the Outbound queue it is the +// global role's (requireInboundGlobal), and 404 while the relay is off. +func (h *Handlers) HandleInboundQueue(w http.ResponseWriter, r *http.Request) { + if _, ok := h.requireInboundGlobal(w, r); !ok { + return + } + _, in, errText := h.queueEntries() + h.view.Render(w, http.StatusOK, "in-queue", h.withRetryPolicy(view.NewInQueue(h.shellMeta(r), in, errText))) +} + +// HandleInboundQueueBody serves the HTMX polling fragment of the Inbound queue. +func (h *Handlers) HandleInboundQueueBody(w http.ResponseWriter, r *http.Request) { + if _, ok := h.requireInboundGlobal(w, r); !ok { + return + } + _, in, errText := h.queueEntries() + h.view.RenderFragment(w, http.StatusOK, "queue_body", view.NewInQueue(h.shellMeta(r), in, errText)) +} + +func (h *Handlers) withRetryPolicy(p *view.Queue) *view.Queue { + policy := h.cfg.RetryPolicy + return p.WithRetryPolicy(policy.FirstRetry(), policy.BackoffCap(), policy.QueueLifetime(), policy.FromDefaults) +} + +// queueListing reads what Postfix holds. It is a var so tests can answer +// without a running Postfix. +var queueListing = postfix.QueueListing + +// queueEntries reads the queue once and splits it by the recipient's domain: +// a recipient of an inbound domain waits for its upstream (Inbound › Queue), +// every other one for a receiving server (Outbound › Queue) - the panel's own +// mail and bounces included. A recipient of the Outbound queue is linked to +// its send-log row when it has one. errText is what the page says when the +// queue cannot be read; the reason goes to the log, not to the page. +func (h *Handlers) queueEntries() (out, in []view.QueueEntry, errText string) { + const unreadable = "Could not read the mail queue." + msgs, err := queueListing() + if err != nil { + logf("panel: postqueue -j: %v", err) + return nil, nil, unreadable + } + inbound := map[string]bool{} + if h.cfg.InboundEnabled { + domains, err := h.store.ListInboundDomains() + if err != nil { + // Without the list, a recipient cannot be told to be inbound; + // putting it under Outbound would show it on the wrong page. + logf("panel: queue: inbound domains: %v", err) + return nil, nil, unreadable + } + for _, d := range domains { + inbound[strings.ToLower(d.Name)] = true + } + } + ids := make([]string, 0, len(msgs)) + for _, m := range msgs { + ids = append(ids, m.ID) + } + refs, err := h.store.SendLogRefsByQueueID(ids) + if err != nil { + logf("panel: queue: send-log rows: %v", err) // the rows still show, without Details + } + for _, m := range msgs { + for _, rcpt := range m.Recipients { + e := view.QueueEntry{ + ID: m.ID, Queue: m.Queue, Arrival: m.Arrival, Size: m.Size, + Sender: m.Sender, Recipient: rcpt.Address, Reason: rcpt.DelayReason, + } + if inbound[recipientDomain(rcpt.Address)] { + in = append(in, e) + continue + } + e.Href = sendLogHref(refs[m.ID], rcpt.Address) + out = append(out, e) + } + } + return out, in, "" +} + +// recipientDomain is the domain of an address, lower-cased: what Postfix +// matches relay_domains against. +func recipientDomain(addr string) string { + at := strings.LastIndexByte(addr, '@') + if at < 0 { + return "" + } + return strings.ToLower(addr[at+1:]) +} + +// sendLogHref is the page of the send-log row of a waiting recipient: the row +// of that recipient, or, when the queue names it otherwise than the row does +// (a canonical map rewrote it; the row holds the RCPT TO as sent), the +// message's first row. "" when the message has no row. +func sendLogHref(refs []store.SendLogRef, rcpt string) string { + if len(refs) == 0 { + return "" + } + ref := refs[0] + for _, r := range refs { + if strings.EqualFold(r.To, rcpt) { + ref = r + break + } + } + return "/outbound/log/" + strconv.FormatInt(ref.ID, 10) } // readQueue runs postqueue -p, returning a friendly message instead of the diff --git a/internal/web/handlers/handlers_monitor_test.go b/internal/web/handlers/handlers_monitor_test.go index 40c3533..3603df7 100644 --- a/internal/web/handlers/handlers_monitor_test.go +++ b/internal/web/handlers/handlers_monitor_test.go @@ -510,6 +510,7 @@ func fixtureRetryPolicy() postfix.RetryPolicy { // The retry policy sits on the page itself, outside the HTMX poll, and prints // whatever policy was cached on Config — never a live postconf. func TestMailQueueShowsRetryPolicyCard(t *testing.T) { + stubQueue(t, nil, nil) h := &Handlers{view: mustView(t), cfg: Config{Version: "test", RetryPolicy: fixtureRetryPolicy()}} out := getBody(t, h.HandleMailQueue, "/outbound/queue") @@ -533,6 +534,7 @@ func TestMailQueueShowsRetryPolicyCard(t *testing.T) { } func TestMailQueueBodyOmitsRetryPolicyCard(t *testing.T) { + stubQueue(t, nil, nil) h := &Handlers{view: mustView(t), cfg: Config{RetryPolicy: fixtureRetryPolicy()}} out := getBody(t, h.HandleMailQueueBody, "/outbound/queue/fragment") @@ -545,6 +547,7 @@ func TestMailQueueBodyOmitsRetryPolicyCard(t *testing.T) { } func TestMailQueueNotesCompiledInFallback(t *testing.T) { + stubQueue(t, nil, nil) h := &Handlers{view: mustView(t), cfg: Config{RetryPolicy: postfix.DefaultRetryPolicy()}} out := getBody(t, h.HandleMailQueue, "/outbound/queue") diff --git a/internal/web/handlers/handlers_queue_test.go b/internal/web/handlers/handlers_queue_test.go new file mode 100644 index 0000000..047e95a --- /dev/null +++ b/internal/web/handlers/handlers_queue_test.go @@ -0,0 +1,110 @@ +package handlers + +import ( + "errors" + "net/http" + "strconv" + "strings" + "testing" + "time" + + "github.com/mixeme/selfpost/internal/postfix" + "github.com/mixeme/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/web/auth" +) + +// The two queues are one Postfix queue split by the recipient's domain. Their +// markup is held to the mockups by the guards in internal/web/view; what is +// checked here is the split, the link to the send-log row, and who may look. + +// stubQueue answers queueListing with msgs (or err) for the test. +func stubQueue(t *testing.T, msgs []postfix.QueuedMessage, err error) { + t.Helper() + old := queueListing + t.Cleanup(func() { queueListing = old }) + queueListing = func() ([]postfix.QueuedMessage, error) { return msgs, err } +} + +func TestQueuesSplitByRecipientDomain(t *testing.T) { + s := newInboundLogStand(t) // inbound domains a.example.com, b.example.com + if err := s.st.InsertQueued(store.SendLogEntry{QueueID: "OUT00001", Domain: "shop.example.org", AppLogin: "shop", + From: "alerts@shop.example.org", To: "noc@example.net", Subject: "Hi"}); err != nil { + t.Fatal(err) + } + rows, err := s.st.QuerySendLog(store.SendLogFilter{AllDomains: true}, 1, 0) + if err != nil || len(rows) != 1 { + t.Fatalf("query: %v (%d rows)", err, len(rows)) + } + at := time.Date(2026, 10, 18, 9, 0, 0, 0, time.UTC) + stubQueue(t, []postfix.QueuedMessage{ + {ID: "OUT00001", Queue: "deferred", Sender: "alerts@shop.example.org", Arrival: at, Size: 3174, + Recipients: []postfix.QueuedRecipient{{Address: "NOC@example.net", DelayReason: "451 4.7.1 Greylisted"}}}, + {ID: "IN000001", Queue: "deferred", Sender: "someone@example.net", Arrival: at, Size: 5000, + Recipients: []postfix.QueuedRecipient{{Address: "alice@A.example.com", DelayReason: "connect to upstream.example.com[192.0.2.7]:25: Connection refused"}}}, + {ID: "NOTE0001", Queue: "active", Sender: "postmaster@mail.example.org", Arrival: at, Size: 900, + Recipients: []postfix.QueuedRecipient{{Address: "admin@example.net"}}}, + }, nil) + + out := s.get(s.h.HandleMailQueue, "/outbound/queue", globalPrincipal).Body.String() + has(t, "Outbound queue", out, "OUT00001", "451 4.7.1 Greylisted", + `Details`, // matched without regard to case + "NOTE0001", "Being delivered now", "2 messages") + if strings.Contains(out, "IN000001") { + t.Error("a recipient of an inbound domain is on the Outbound queue") + } + if strings.Count(out, ">Details") != 1 { + t.Error("the panel's own message has no send-log row and should have no Details link") + } + + in := s.get(s.h.HandleInboundQueue, "/inbound/queue", globalPrincipal).Body.String() + has(t, "Inbound queue", in, "IN000001", "Connection refused", "1 message · 4.9 kB") + for _, other := range []string{"OUT00001", "NOTE0001", ">Details"} { + if strings.Contains(in, other) { + t.Errorf("the Inbound queue shows %q", other) + } + } + + frag := s.get(s.h.HandleInboundQueueBody, "/inbound/queue/fragment", globalPrincipal).Body.String() + if !strings.HasPrefix(frag, `
`, `