Compare commits
77
Commits
v2.0.0
...
8b9798884a
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8b9798884a | ||
|
|
80bab92c8c | ||
|
|
937b41ddb7 | ||
|
|
cc2ca3c968 | ||
|
|
70b4eb5585 | ||
|
|
86665f08e6 | ||
|
|
c3d99b1f15 | ||
|
|
b9353db507 | ||
|
|
7fdf09839f | ||
|
|
6d9273f418 | ||
|
|
e3bcafb6d9 | ||
|
|
2a96008803 | ||
|
|
8f226bd82f | ||
|
|
c817f0833c | ||
|
|
89fd0a6116 | ||
|
|
c171fb5bc7 | ||
|
|
8542e673a8 | ||
|
|
cbff3e7e33 | ||
|
|
972516d1c4 | ||
|
|
ce7b09a65c | ||
|
|
f09b60aaa2 | ||
|
|
400aa76737 | ||
|
|
b905f3e560 | ||
|
|
c9027381e5 | ||
|
|
50fd6b859f | ||
|
|
06ad174fb9 | ||
|
|
87fa0a4381 | ||
|
|
eff8010d39 | ||
|
|
394eb67c4c | ||
|
|
972a9f825c | ||
|
|
a990e7cc23 | ||
|
|
815236a607 | ||
|
|
a6845ae948 | ||
|
|
1e6836b3c1 | ||
|
|
fc184ecc01 | ||
|
|
81df131217 | ||
|
|
3d17206063 | ||
|
|
4468aa8dd7 | ||
|
|
977d63a2ea | ||
|
|
ae06d8b47e | ||
|
|
1f44e40382 | ||
|
|
d398f488e2 | ||
|
|
89774ab847 | ||
|
|
c731a98a0f | ||
|
|
5f86457848 | ||
|
|
00feb7c1c7 | ||
|
|
d2a778edc0 | ||
|
|
7192c3861e | ||
|
|
e0ccff257d | ||
|
|
c1da784193 | ||
|
|
9d330bed77 | ||
|
|
970c8ca8da | ||
|
|
19156814b8 | ||
|
|
d2b97b3963 | ||
|
|
d4efdb483d | ||
|
|
56c5dfee40 | ||
|
|
82d4a209be | ||
|
|
737611a07e | ||
|
|
eea40bcd49 | ||
|
|
27fd355e53 | ||
|
|
94e94490af | ||
|
|
28ec784ae2 | ||
|
|
a3cc52d6d4 | ||
|
|
d65d632505 | ||
|
|
dd0fefdcb7 | ||
|
|
75ebbf6a57 | ||
|
|
ede3783734 | ||
|
|
d70371a50c | ||
|
|
21e961cb29 | ||
|
|
82f212ab39 | ||
|
|
b55b33ae92 | ||
|
|
8c9a71e6a0 | ||
|
|
646ec1e149 | ||
|
|
8496b289ec | ||
|
|
4dd46ea355 | ||
|
|
505260e227 | ||
|
|
09d9625f32 |
@@ -4,6 +4,9 @@
|
||||
.gitattributes
|
||||
dev
|
||||
docs
|
||||
# test/e2e/.stage holds what an e2e run leaves behind (DKIM keys, sasldb2);
|
||||
# the e2e's own sink image is built from test/e2e/sink as its own context.
|
||||
test
|
||||
bin
|
||||
*.exe
|
||||
README.md
|
||||
|
||||
@@ -89,8 +89,27 @@ jobs:
|
||||
cache-dependency-path: test/e2e/go.sum
|
||||
|
||||
- name: e2e (gates publishing — see docs/development.md)
|
||||
# The e2e builds selfpost:e2e again through compose.override.yml, and
|
||||
# that rebuild is the image pushed below. Without the version here it
|
||||
# was stamped "e2e" (audit 2026-10-12, H1); the suite now checks the
|
||||
# stamp against this variable.
|
||||
env:
|
||||
SELFPOST_E2E_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
run: cd test/e2e && go test -v -timeout 20m ./...
|
||||
|
||||
- name: Check the version stamp of the image about to be pushed
|
||||
env:
|
||||
WANT: ${{ needs.prepare.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for bin in panel selfpost-backup; do
|
||||
got=$(docker run --rm --entrypoint "$bin" selfpost:e2e -version)
|
||||
if [ "$got" != "$WANT" ]; then
|
||||
echo "::error::selfpost:e2e $bin -version = '$got', want '$WANT'; refusing to push"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Log in to ghcr.io
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
|
||||
@@ -24,6 +24,9 @@ jobs:
|
||||
with:
|
||||
go-version: "1.26"
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
go.sum
|
||||
test/e2e/go.sum
|
||||
|
||||
- name: Format
|
||||
run: |
|
||||
@@ -39,3 +42,13 @@ jobs:
|
||||
|
||||
- name: Test
|
||||
run: go test ./...
|
||||
|
||||
# The e2e module runs only in release.yml, after a release is published;
|
||||
# a type error there would first show up then. Compile it here without
|
||||
# running it: `go test -run` would still run TestMain, which brings up
|
||||
# the whole stand.
|
||||
- name: Compile the e2e
|
||||
working-directory: test/e2e
|
||||
run: |
|
||||
go vet ./...
|
||||
go test -c -o /dev/null .
|
||||
|
||||
+376
@@ -5,6 +5,382 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); version
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- **A release image says which release it is.** The e2e gate in
|
||||
`release.yml` rebuilt the image it was about to push with its own
|
||||
`VERSION=e2e`, so every published image printed `e2e` for `-version` and in
|
||||
the panel's footer, and the backup/restore version check compared a
|
||||
backup's version with `e2e` - any backup restored into any release (audit
|
||||
2026-10-12, H1). The gate now builds with the release's version
|
||||
(`SELFPOST_E2E_VERSION`), the e2e checks `panel -version` and
|
||||
`selfpost-backup -version` against it, and `release.yml` checks the stamp
|
||||
again on the image right before pushing it. Images already published are
|
||||
left as they are.
|
||||
- **The documents no longer promise an image that is not there.** README,
|
||||
the guide and the compose file pinned `ghcr.io/mixeme/selfpost:2.4.1`, but
|
||||
no 2.x image has been published (ghcr.io holds `1.0.0` and `1.3.0`). They
|
||||
now say so and show how to build the image locally under the pinned name,
|
||||
which Compose and `docker run` then use without pulling (audit 2026-10-12,
|
||||
M6). The pin stays: the next publication makes it true.
|
||||
- **Dependencies raised**: `modernc.org/sqlite` 1.53.0 → 1.60.1 (with
|
||||
`modernc.org/libc` 1.77.1), `golang.org/x/crypto` 0.54.0 → 0.58.0,
|
||||
`golang.org/x/sys` 0.47.0 → 0.49.0. The three published `x/crypto` fixes
|
||||
were in code SelfPost does not call (audit 2026-10-12). development.md says
|
||||
the image is rebuilt after each Go patch release, which is where the
|
||||
standard library's security fixes come from.
|
||||
- **The e2e module is compiled on every push** (`test.yml`: `go vet` and
|
||||
`go test -c` in `test/e2e`), so a type error there no longer first shows
|
||||
up in `release.yml` after a release is published (audit 2026-10-12, M20).
|
||||
- **`test/` is out of the image's build context** (`.dockerignore`): a
|
||||
developer build after an e2e run sent the stand's leftovers - DKIM keys,
|
||||
`sasldb2` - to the builder (audit 2026-10-12, L33).
|
||||
|
||||
## [2.4.1] - 2026-10-12
|
||||
|
||||
Documentation only; no product code changes.
|
||||
|
||||
### Added
|
||||
|
||||
- **A full-tree code audit** at `2.4.0`, with its plan of work:
|
||||
[docs/audit-2026-10-12.md](docs/audit-2026-10-12.md). Six zones read file
|
||||
by file against what `architecture.md` promises; what the readers could not
|
||||
settle from source was settled on the test server. Four high findings - the
|
||||
published release images are stamped `VERSION=e2e` because the e2e gate
|
||||
rebuilds the image it is about to push, so the backup version check has
|
||||
never checked anything; a renewed TLS certificate is not picked up while
|
||||
the container runs; the queue-reconcile sweep can mark delivered mail
|
||||
`bounced` while the log tailer is blind; application logins are
|
||||
case-sensitive to the panel and case-insensitive to Postfix - twenty-one
|
||||
medium, thirty-four low, the doc drift and the test gaps, and what was
|
||||
checked and found sound. The plan is six stages, each a cut of its own;
|
||||
nothing in it is agreed yet. The published `1.x` tags and images stay as
|
||||
they are (owner, 2026-10-12): fixes go forward only.
|
||||
|
||||
## [2.4.0] - 2026-10-12
|
||||
|
||||
### Security
|
||||
|
||||
- **An application signs only as itself.** OpenDKIM chooses the key by the
|
||||
`From:` header, and Postfix held only the envelope sender against the
|
||||
login: an application of one sending domain could write another registered
|
||||
domain, or the server's own name, into `From:` and have the message signed
|
||||
with that name's key (security review of `2.3.0`, where it was recorded as
|
||||
an accepted risk). The panel now holds the header to what the envelope is
|
||||
held to - any address of the application's domain, or the addresses of its
|
||||
list, with or without a `+extension` - and refuses anything else with
|
||||
`550 5.7.1`, a domain this server does not sign for included
|
||||
([plans/from-header-binding.md](docs/plans/from-header-binding.md)).
|
||||
- **The check is strict.** It is the first milter of the chain, on a socket
|
||||
of its own (`SENDER_CHECK_SOCKET`), attached with
|
||||
`default_action=tempfail`: while the panel does not answer, Postfix defers
|
||||
mail and accepts nothing unchecked. The journal milter keeps its socket and
|
||||
stays fail-open.
|
||||
|
||||
### Added
|
||||
|
||||
- **Server › Health counts what the strict check costs** in a card of its
|
||||
own, *Sender check*: for how long the panel did not answer while Postfix
|
||||
was running, and how many submissions Postfix turned away because of it -
|
||||
today, in the last 30 days and in the last 12 months (UTC days, kept a
|
||||
year). The card warns when mail was deferred in the last 24 hours. The
|
||||
numbers are there to decide whether the check needs a process of its own.
|
||||
- **A refused row of the Outbound log says why**, in its tag: `wrong From`,
|
||||
`wrong client` (the application's client IP list) or `rate-limited`; a row
|
||||
refused before this version says `rejected`. A message refused for its
|
||||
`From:` line shows on its page what the line said and what the application
|
||||
may send as. Help has a *Sender check* topic.
|
||||
- Migration `0006_sender_check.sql` (DDL only): `send_log.reason`,
|
||||
`send_log.header_from`, `sender_check_days`.
|
||||
|
||||
### Changed
|
||||
|
||||
- **Mail with a `From:` the application is not bound to is refused.** What
|
||||
stops working on upgrade: an application sending with the `From:` of
|
||||
another of your domains (give that domain an application); an application
|
||||
with a list whose envelope and `From:` addresses differ (add the second to
|
||||
the list); a form that puts its visitor's address into `From:` (use
|
||||
`Reply-To:`). The line is read strictly: one `From:` field, one address,
|
||||
one `@` outside a quoted display name; a name with a comma or brackets has
|
||||
to be quoted. Guide § Who an application may send as.
|
||||
- **With the panel down, mail is deferred, not accepted.** Until now a
|
||||
stopped panel cost the send log and the level-2 limits and nothing else.
|
||||
- **Server › Health is laid out as two stacks of boxes numbered by
|
||||
importance**, read left to right and top to bottom, each stack closed up:
|
||||
Processes, Milter sockets, Sender check, TLS certificate, Hostname and
|
||||
reverse DNS, Notification mail, Machine, Configuration. On a phone they
|
||||
come in that order.
|
||||
- A refused message is one row of the log, naming its first recipient and
|
||||
how many more there were.
|
||||
- `recipient_delimiter` is set to `+` by the image instead of being left to
|
||||
the distribution's default; the start removes milter sockets a killed
|
||||
container left behind.
|
||||
|
||||
## [2.3.0] - 2026-10-12
|
||||
|
||||
### Added
|
||||
|
||||
- **Schema for notifications** (migration `0004_notifications.sql`, DDL only):
|
||||
a user's own switch, `users.notify`, on for everyone; `notify_state`, what
|
||||
the panel has already written to its users about. The instance switch is a
|
||||
settings key, on unless turned off
|
||||
([plans/panel-notifications.md](docs/plans/panel-notifications.md)).
|
||||
- **The panel writes to its users when something needs them.** Four things,
|
||||
each said once: a sending domain's DKIM, SPF or DMARC record stops matching
|
||||
(after two hourly checks in a row, and once more when it matches again); a
|
||||
level-2 rate limit refuses mail (at most once a day per sending domain); the TLS
|
||||
certificate is 14 days from expiry, and 3; a message has waited in the
|
||||
queue for more than three days (and once more when none has). Global users
|
||||
are told all of it, domain users about their own domains. Sent from
|
||||
`postmaster@<SELFPOST_HOSTNAME>` by local submission, one message per
|
||||
recipient; not rows of the Outbound log and counted against no limit.
|
||||
- **The server's own name is signed.** The panel generates a DKIM key for
|
||||
`SELFPOST_HOSTNAME` and OpenDKIM signs mail submitted locally as that name
|
||||
(`non_smtpd_milters`), so that notifications arrive. The name is not a
|
||||
sending domain: no application can use it as an envelope sender.
|
||||
- **Notifications in the panel.** Account has *Send me notifications* beside
|
||||
the e-mail; Server › Settings has the switch for everyone and the two DNS
|
||||
records of the server's own name, DKIM and SPF, with the values to publish;
|
||||
Server › Health has a *Notification mail* card with the state of each. Help
|
||||
has a Notifications topic. Every user starts with notifications on, and so
|
||||
does the instance.
|
||||
|
||||
### Changed
|
||||
|
||||
- **An account e-mail may be at a public mail host.** Account and Server ›
|
||||
Users refused addresses at Gmail, Outlook, Proton and the like, a rule that
|
||||
belongs to DMARC report addresses. A user's own address is now where the
|
||||
panel writes to them.
|
||||
- **A lookup that does not answer is not an SPF mismatch.** When the address
|
||||
lookup behind an `a` or `mx` mechanism timed out, a domain's SPF record was
|
||||
shown as not authorising this server. It is now a warning that the check
|
||||
could not tell.
|
||||
|
||||
### Security
|
||||
|
||||
- Technical and security reviews of 2026-10-12 of the notifications, before
|
||||
release: no Critical finding. The notification about refused mail was built
|
||||
on the domain a client writes in `MAIL FROM`; it is now built on the
|
||||
registry. Recorded in [security.md](docs/security.md) as accepted, with a
|
||||
roadmap candidate (from-header-binding): an application can have a message
|
||||
signed as another registered domain, or as the server's own name, by
|
||||
writing it into `From:` - the first part of which has always been so.
|
||||
|
||||
## [2.2.0] - 2026-10-12
|
||||
|
||||
### Added
|
||||
|
||||
- **Schema: `send_log_lines`** (migration `0003_send_log_lines.sql`, DDL only).
|
||||
The table that will hold what Postfix wrote about a message for as long as
|
||||
its send-log row lives, with the store methods to write a line, to write it
|
||||
once, to read the lines of a row and to prune
|
||||
([plans/delivery-log-storage.md](docs/plans/delivery-log-storage.md)).
|
||||
- **The log tailer keeps what Postfix writes about queued messages.** A
|
||||
`mail.log` line headed by a queue id is stored with the line's own time, one
|
||||
transaction per pass over the log; after an unclean stop a line read twice
|
||||
is stored once. Mail that came in on port 25 is left out: a stranger's
|
||||
message never has a send-log row, and what he can make Postfix log is not
|
||||
written to the database. Lines whose queue id has no send-log row - the row
|
||||
was pruned, or the message never had one (bounces) - are deleted once they
|
||||
are an hour old, at start and with every six-hourly send-log sweep.
|
||||
|
||||
### Changed
|
||||
|
||||
- **A message's page keeps its delivery log after `mail.log` rotates.** The
|
||||
*Delivery log* box on Outbound › Log › a message was filled by searching
|
||||
the current `mail.log`, which is rotated daily, so it was empty for anything
|
||||
not sent today while the row itself lives for ninety days. The page now
|
||||
reads the lines the tailer stored: they are there for as long as the row
|
||||
is. Postfix reuses queue ids, so a row is given only the lines dated within
|
||||
its own queue lifetime. A message accepted before this version shows its
|
||||
lines only while they are still in the current file, as before; the note
|
||||
shown when there are none and the *Outbound log* help topic say so.
|
||||
|
||||
### Security
|
||||
|
||||
- **A full backup now carries the log lines of messages in the send log** -
|
||||
client addresses, SASL logins, message ids, receivers' replies - where
|
||||
`mail.log` itself is not in a backup. Recorded in
|
||||
[security.md](docs/security.md) with the accepted risk of a queue-id
|
||||
collision showing one message's lines on another's page. Technical and
|
||||
security reviews of 2026-10-12: no Critical or High finding.
|
||||
|
||||
## [2.1.0] - 2026-10-12
|
||||
|
||||
### Security
|
||||
|
||||
- **The level-2 rate limit and an application's client IP list held only for
|
||||
the first message of a connection.** After a refused `MAIL FROM`, a second
|
||||
one on the same connection was accepted, and so was every message after the
|
||||
first that a connection carried: an application over its limit could go on
|
||||
sending, and one restricted to certain client addresses could send from any
|
||||
other with a retry. In the code since `1.9.0` (the client IP list) and
|
||||
earlier (the limit); found by the technical review of 2026-10-12 and
|
||||
reproduced on the test stand. Postfix now gives the panel's milters the
|
||||
client address with every message, the limit no longer depends on it, and an
|
||||
application with a client IP list is refused when no address is known.
|
||||
- Reviews of the inbound antispam feature before its release (technical and
|
||||
security, 2026-10-12; nothing Critical or High in the feature itself). Fixed:
|
||||
a client's HELO name could supply the sender, recipient and reply of an
|
||||
inbound log row; the log's row cap was applied only every six hours; rows of
|
||||
a connection's later messages had no client address; two changes of the
|
||||
filter lists at once could leave the filter's files behind the page; a held
|
||||
message was logged twice; after an unclean stop of the panel the refusals of
|
||||
its last seconds were logged twice; with the spam filter down, a sender's
|
||||
own verdict headers were recorded as the filter's. Recorded as accepted in
|
||||
[docs/security.md](docs/security.md): an allow entry for a sender address
|
||||
trusts an address anyone can claim; the list files are readable by anyone on
|
||||
the host.
|
||||
- `INBOUND_ANTISPAM_MILTER` and `INBOUND_JOURNAL_MILTER_SOCKET` are checked
|
||||
character by character at start; the check looked at their first characters
|
||||
only.
|
||||
|
||||
### Changed
|
||||
|
||||
- **Port 25 logs as `postfix/inbound`.** Its lines in the mail log (Server ›
|
||||
System log) now read `postfix/inbound/smtpd[…]` where they read
|
||||
`postfix/smtpd[…]`; the submission ports keep `postfix/smtpd`. This is how a
|
||||
refusal on port 25 is told from one on 465 / 587.
|
||||
|
||||
### Added
|
||||
|
||||
- Schema: migration `0002_inbound_spam_log.sql`, the first after the 2.0
|
||||
baseline - two empty tables for the inbound antispam journal and the filter
|
||||
lists (inbound-antispam-panel). A 2.0.x database is migrated when the panel
|
||||
starts.
|
||||
- **Inbound mail is recorded.** With the inbound relay on, the panel runs a
|
||||
second milter, the last one on port 25, and writes a row for every recipient
|
||||
of a message that was accepted for an inbound domain: client address,
|
||||
envelope, subject and - with a spam filter attached - the filter's action,
|
||||
score and heaviest symbols. It only watches: it refuses nothing and Postfix
|
||||
accepts mail when it is down.
|
||||
- **Refused inbound mail is recorded too**, from the mail log: a recipient
|
||||
Postfix refused at an inbound domain (unknown user, sender domain not
|
||||
found), and a message the spam filter rejected, deferred, held or
|
||||
discarded. The row is dated by its log line and carries the reply text. A
|
||||
refusal to relay for a domain that is not an inbound one is not recorded.
|
||||
- The inbound journal is pruned every six hours: rows older than 90 days
|
||||
(`INBOUND_SPAM_LOG_RETENTION_DAYS`, 7-365) and anything beyond 10 000 rows.
|
||||
- The antispam compose fragment mounts `antispam/rspamd/milter_headers.conf`
|
||||
into the rspamd sidecar, which makes it write its verdict into
|
||||
`X-Rspamd-Action` and `X-Spamd-Result`. **Copy `deploy/antispam/rspamd/`
|
||||
next to the fragment before starting it** - Docker creates a directory in
|
||||
place of a missing file and the sidecar then fails to read its
|
||||
configuration.
|
||||
- **Allow and deny lists for the spam filter**, kept by SelfPost and applied
|
||||
by rspamd: sender addresses, sender domains, client addresses and networks.
|
||||
They are written as six files under `data/antispam/`, which the fragment
|
||||
mounts into the sidecar together with `antispam/rspamd/multimap.conf`;
|
||||
rspamd rereads a changed file by itself. Deny wins over allow. The files
|
||||
are there from the first start with the inbound relay on.
|
||||
- **Inbound › Log**: the journal as a page - time, sender and client address,
|
||||
recipient, subject and decision, filtered by domain, decision and day. The
|
||||
decision opens a sheet over the page with who decided and why: the filter's
|
||||
verdict and symbols, or the reply a refusal was given. A user with the
|
||||
*domain* role sees the rows of the inbound domains assigned to them. For the
|
||||
global role a row links to the filter lists with the sender filled in:
|
||||
*Deny sender* on accepted mail, *Allow sender* on mail the filter refused -
|
||||
unless the sender is spelt in a way the lists do not take.
|
||||
- **Inbound › Filter lists** (global role): add and remove allow and deny
|
||||
entries; the page says where the lists go and whether the filter is on to
|
||||
read them. Each entry shows how often it decided a message - *Matched*, or
|
||||
*never* - and its kind opens a sheet with the note, the day it was added and
|
||||
the last match, so that entries nothing matches any more can be found and
|
||||
removed. The count is read from the inbound log and reaches back as far as
|
||||
the log does. A network of one address (`192.0.2.7/32`) is stored as the
|
||||
address, so it cannot be on a list twice.
|
||||
- **Server › Settings**: *Keep inbound log rows, days*, shown with the inbound
|
||||
relay on.
|
||||
- Help: *Inbound log* and *Filter lists*.
|
||||
- New setting `ANTISPAM_DIR` (default `/data/antispam`).
|
||||
- **The spam filter signs what it judged.** rspamd adds `X-SelfPost-Filter` to
|
||||
every message it lets through - an HMAC of the message's queue id under a
|
||||
key SelfPost creates in `data/antispam/filter.key` - and the inbound log
|
||||
shows a verdict only next to a signature that fits. Nothing to configure,
|
||||
but the sidecar needs the new `antispam/rspamd/milter_headers.conf` and a
|
||||
restart to read it; until then accepted mail is logged as *selfpost*, with
|
||||
no verdict.
|
||||
- A full backup carries the inbound log, the filter lists and their files
|
||||
under `data/antispam/`. It does not carry the `antispam/` directory with the
|
||||
compose fragment and the rspamd configuration - copy that from
|
||||
`deploy/antispam/` when restoring.
|
||||
- New setting `INBOUND_JOURNAL_MILTER_SOCKET` (default
|
||||
`/run/selfpost/inbound-journal.sock`).
|
||||
|
||||
## [2.0.1] - 2026-10-11
|
||||
|
||||
### Changed
|
||||
|
||||
- **A message that is not a report this relay keeps is dropped at once.**
|
||||
One that cannot be read as an aggregate report, reports on a domain not
|
||||
sent from here, or came to another domain's report address used to be put
|
||||
off and offered again by Postfix for days, counted as a parse failure each
|
||||
time. It is now accepted and discarded - no retry, and no bounce, since the
|
||||
envelope sender of such a message is whatever its author wrote. It counts
|
||||
as one parse failure and the reason is in the mail log, on the message's
|
||||
own delivery line (Server › System log, `dropped: …`). Only a failure of
|
||||
the database is still retried.
|
||||
- **A full backup no longer holds the Postfix queue**, from the panel or from
|
||||
`selfpost-backup`. It is taken while the instance runs, and a copy of a
|
||||
queue Postfix is working on can hold a message twice or not at all. Mail
|
||||
still waiting for delivery is not carried over; a restored instance starts
|
||||
with an empty queue. To keep the queue, stop the instance and `tar` the
|
||||
project directory.
|
||||
- DMARC reports are kept 500 **for each domain** (and 90 days), not 500 for
|
||||
the whole server - see Security.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **Full backup from the panel works.** Since the Postfix queue moved under
|
||||
`/data` in 1.3.0 the button answered with a 200 and ten bytes: the archive
|
||||
includes the queue, Postfix keeps the queue readable by its own user alone,
|
||||
and the panel is not that user ("open /data/postfix/queue/active:
|
||||
permission denied" in the log). `selfpost-backup` run with `docker exec`
|
||||
was not affected - it runs as root. The queue is no longer part of a backup
|
||||
(see Changed), so the two archives are the same again. Everything is read
|
||||
through before the download starts, so a failure is a message on the page
|
||||
and not a file cut short. One thing the panel may still be unable to read
|
||||
is the private key in `certs/`: it then refuses and says what to do -
|
||||
`chgrp 998 certs/privkey.pem && chmod 640 certs/privkey.pem` on the host,
|
||||
998 being the container's `selfpost` group, whose id is now fixed in the
|
||||
image.
|
||||
|
||||
### Security
|
||||
|
||||
Review of the DMARC report path (Fable, 2026-10-11), which started to run for
|
||||
the first time in 2.0.0. Nothing Critical or High.
|
||||
|
||||
- **A report is bounded before it is stored.** The XML comes from anyone who
|
||||
knows a report address, and its 8 MiB limit left room for some 160 000
|
||||
records or a single name of megabytes - rows in the database and cells of
|
||||
the report page. A report holds at most 5 000 records (a larger one is
|
||||
dropped), every text field is cut to 255 bytes, and a record's count is
|
||||
capped so the sums cannot wrap.
|
||||
- **One domain's reports cannot push out another's.** The cap of 500 was for
|
||||
the whole server, so a stranger filing reports under one domain's address
|
||||
could evict every other domain's history. It is counted per domain now.
|
||||
- The delivery command runs one at a time, not up to a hundred in parallel.
|
||||
- The reason written to the mail log for a dropped message is cut to 200
|
||||
bytes and anything outside printable ASCII is replaced: parts of it are the
|
||||
sender's own bytes (a header line, a file name inside an archive).
|
||||
- `SELFPOST_DATA_DIR` / `SELFPOST_DB_PATH` with whitespace or `$` are refused
|
||||
at start when DMARC ingest is on, instead of handing the delivery command
|
||||
another path and so an empty database of its own.
|
||||
|
||||
### Added
|
||||
|
||||
- e2e: the run now ends on port 25 with both optional features on. A DMARC
|
||||
report is mailed to a domain's hosted address and has to appear under
|
||||
Outbound › DMARC reports; a report on another domain and a message with no
|
||||
report are accepted and dropped; mail for an inbound domain reaches its
|
||||
upstream and other recipients are refused. Until now the suite only read
|
||||
back the generated configuration of these features.
|
||||
- e2e: full backup and restore. The run takes a backup from the panel, throws
|
||||
the installation away, unpacks the archive into the empty project directory
|
||||
and starts it: the administrator signs in, the domain has the same DKIM key,
|
||||
and mail sent by the application verifies against the record published
|
||||
before the backup.
|
||||
|
||||
## [2.0.0] - 2026-10-11
|
||||
|
||||
The panel redesign. A breaking release: every panel URL changed and the
|
||||
|
||||
@@ -106,13 +106,23 @@ One container, panel at `http://127.0.0.1:8080` — no reverse proxy, no TLS
|
||||
files, no compose files. Good for clicking through the UI on your machine;
|
||||
outbound mail will not reach the real internet without DNS, PTR, and port 25.
|
||||
|
||||
> **The 2.x line is not published on ghcr.io yet** (the newest image there is
|
||||
> `1.3.0`). Until it is, build the image from this checkout under the name the
|
||||
> commands below and `deploy/docker-compose.yml` use; Docker then takes the
|
||||
> local image and pulls nothing:
|
||||
>
|
||||
> ```sh
|
||||
> docker build -f build/Dockerfile --build-arg VERSION=2.4.1 \
|
||||
> -t ghcr.io/mixeme/selfpost:2.4.1 .
|
||||
> ```
|
||||
|
||||
```sh
|
||||
docker run --rm -d --name selfpost-try \
|
||||
-p 127.0.0.1:8080:8080 \
|
||||
-e SELFPOST_HOSTNAME=mail.local.test \
|
||||
-e PANEL_COOKIE_SECURE=false \
|
||||
-v selfpost-try-data:/data \
|
||||
ghcr.io/mixeme/selfpost:2.0.0
|
||||
ghcr.io/mixeme/selfpost:2.4.1
|
||||
```
|
||||
|
||||
**Get the setup URL** (pick one):
|
||||
|
||||
+7
-2
@@ -62,7 +62,10 @@ RUN useradd --system --no-create-home --shell /usr/sbin/nologin panel
|
||||
# entrypoint.sh) — lets OpenDKIM read the panel-owned keys and lets Postfix read
|
||||
# the sasldb2/sender map, and lets the panel reach the supervisor control socket
|
||||
# to signal OpenDKIM/Postfix reloads without any process running as root.
|
||||
RUN groupadd --system selfpost \
|
||||
# The id is fixed, and it is the one the group has always had: an operator
|
||||
# gives the TLS private key on the host to this group by number, so that the
|
||||
# panel can put it into a full backup (guide § Taking a full backup).
|
||||
RUN groupadd --system --gid 998 selfpost \
|
||||
&& usermod -aG selfpost panel \
|
||||
&& usermod -aG selfpost opendkim \
|
||||
&& usermod -aG selfpost postfix
|
||||
@@ -73,7 +76,9 @@ RUN groupadd --system selfpost \
|
||||
# with `no-new-privileges`, which disables setgid transitions, so the panel
|
||||
# would always see "Permission denied". Making `panel` a real member of postdrop
|
||||
# grants the same access without relying on a setgid escalation the hardening
|
||||
# deliberately forbids. postdrop membership is read-side only: it does not let
|
||||
# deliberately forbids. The same membership is what lets the panel submit its
|
||||
# own notifications with sendmail(1) - postdrop, too, is setgid and would fail
|
||||
# to write the maildrop directory (internal/notify/mailer.go). It does not let
|
||||
# the panel bypass any Postfix restriction that a local user does not already
|
||||
# have through the world-executable sendmail.
|
||||
RUN usermod -aG postdrop panel
|
||||
|
||||
+16
-2
@@ -112,6 +112,15 @@ for f in relay_domains transport relay_recipients tls_policy; do
|
||||
done
|
||||
chown panel:selfpost /data/postfix
|
||||
chmod 2750 /data/postfix
|
||||
|
||||
# Filter lists for the external spam filter (written by the panel when
|
||||
# INBOUND_RELAY_ENABLE=true, read by the antispam sidecar, which mounts this
|
||||
# directory). The sidecar runs under a user of its own in another container,
|
||||
# so the directory and its files are readable by anyone: they hold sender
|
||||
# addresses and networks, no secrets.
|
||||
mkdir -p /data/antispam
|
||||
chown panel:selfpost /data/antispam
|
||||
chmod 0755 /data/antispam
|
||||
chown panel:selfpost /data/postfix/sender_login_maps
|
||||
chmod 0640 /data/postfix/sender_login_maps
|
||||
|
||||
@@ -133,8 +142,8 @@ chmod 2750 /data/log
|
||||
find /data/log -type f -exec chmod 0640 {} +
|
||||
|
||||
# Milter socket directories. Postfix (user `postfix`) must actually CONNECT to
|
||||
# both milter sockets — OpenDKIM's and the panel's journal-milter — not just
|
||||
# probe them at start-up. The sockets are created by the opendkim and panel
|
||||
# the milter sockets — OpenDKIM's and the panel's — not just probe them at
|
||||
# start-up. The sockets are created by the opendkim and panel
|
||||
# users respectively, so bridge them to `postfix` through the shared `selfpost`
|
||||
# group: group-owned + setgid dirs mean each socket created inside inherits
|
||||
# group `selfpost`, and group-traversable (2750) lets postfix reach it. Without
|
||||
@@ -144,6 +153,11 @@ mkdir -p /run/opendkim /run/selfpost
|
||||
chown opendkim:selfpost /run/opendkim
|
||||
chown panel:selfpost /run/selfpost
|
||||
chmod 2750 /run/opendkim /run/selfpost
|
||||
# A container that was killed leaves its sockets behind, and a file nobody
|
||||
# listens on would satisfy postfix-wrapper.sh's wait: Postfix would start
|
||||
# before the sender check does and defer mail until it is there. The check's
|
||||
# alive mark stays - it is how the downtime of a restart is told.
|
||||
rm -f /run/opendkim/*.sock /run/selfpost/*.sock
|
||||
|
||||
# Generate the outbound-relay Postfix configuration from the environment (spec
|
||||
# 5). Kept out of the image build so cert paths, rate limits, hostname and the
|
||||
|
||||
+81
-6
@@ -93,10 +93,13 @@ fi
|
||||
RATE_MSGS="${RATE_LIMIT_MESSAGES_PER_IP:-100}"
|
||||
RATE_WINDOW="${RATE_LIMIT_WINDOW_SECONDS:-3600}"
|
||||
|
||||
# Milter sockets: OpenDKIM (signing, strict) and the panel journal-milter
|
||||
# (monitoring, fail-open). Fixed container paths, matched by postfix-wrapper.sh.
|
||||
# Milter sockets: OpenDKIM (signing, strict), the panel's sender check
|
||||
# (strict) and its journal-milter (monitoring, fail-open). Fixed container
|
||||
# paths, matched by postfix-wrapper.sh.
|
||||
OPENDKIM_SOCK="${OPENDKIM_SOCKET:-/run/opendkim/opendkim.sock}"
|
||||
JOURNAL_SOCK="${JOURNAL_MILTER_SOCKET:-/run/selfpost/journal.sock}"
|
||||
# The panel's sender check (the From: line against the login, strict).
|
||||
SENDER_SOCK="${SENDER_CHECK_SOCKET:-/run/selfpost/sender.sock}"
|
||||
|
||||
# Persistent panel-managed sender map; texthash needs no postmap, so the
|
||||
# unprivileged panel can rewrite it and just ask for a reload.
|
||||
@@ -117,6 +120,9 @@ SUBMISSION_ENABLE="${SUBMISSION_ENABLE:-false}"
|
||||
INBOUND_RELAY_ENABLE="${INBOUND_RELAY_ENABLE:-false}"
|
||||
INBOUND_ANTISPAM_MILTER="${INBOUND_ANTISPAM_MILTER:-}"
|
||||
INBOUND_ANTISPAM_MILTER_ACTION="${INBOUND_ANTISPAM_MILTER_ACTION:-accept}"
|
||||
# The panel's inbound journal milter (monitoring, fail-open), on port 25 only
|
||||
# and only while the inbound relay is on. Matched by postfix-wrapper.sh.
|
||||
INBOUND_JOURNAL_SOCK="${INBOUND_JOURNAL_MILTER_SOCKET:-/run/selfpost/inbound-journal.sock}"
|
||||
INBOUND_RATE_LIMIT_MESSAGES_PER_IP="${INBOUND_RATE_LIMIT_MESSAGES_PER_IP:-20}"
|
||||
INBOUND_MESSAGE_SIZE_LIMIT="${INBOUND_MESSAGE_SIZE_LIMIT:-26214400}"
|
||||
RELAY_DOMAINS_MAP="${POSTFIX_RELAY_DOMAINS:-/data/postfix/relay_domains}"
|
||||
@@ -197,8 +203,14 @@ postconf -e \
|
||||
# Sender binding (the critical anti-spoofing control). texthash resolves the
|
||||
# full address first, then the "@domain" wildcard, so both address modes work
|
||||
# from the same map.
|
||||
#
|
||||
# recipient_delimiter is said here and not left to the distribution's main.cf:
|
||||
# it is what lets an application listed as invoices@ send as invoices+march@,
|
||||
# and the panel's sender check makes the same second try for the From: line
|
||||
# (extensionDelimiter in internal/milter/sender.go) - the two must agree.
|
||||
postconf -e \
|
||||
"smtpd_sender_login_maps=texthash:${SENDER_LOGIN_MAPS}" \
|
||||
"recipient_delimiter=+" \
|
||||
"queue_directory=${QUEUE_DIR}"
|
||||
|
||||
# Restrictions: authenticated clients only, no relay to foreign destinations,
|
||||
@@ -221,11 +233,30 @@ postconf -e \
|
||||
# if it is unreachable, defer rather than send unsigned). The journal-milter is
|
||||
# monitoring only and is fail-open (default_action=accept): its failure must
|
||||
# never block the relay. Per-milter settings use Postfix 3.0+ brace syntax.
|
||||
#
|
||||
# The sender check comes first and is strict as well: it holds the From: line
|
||||
# against the login, which is what OpenDKIM then chooses the key by, so a
|
||||
# message it could not judge must not reach the signer. While the panel does
|
||||
# not answer on that socket, mail is deferred (docs/security.md).
|
||||
#
|
||||
# non_smtpd_milters is the chain for mail submitted locally with sendmail(1) -
|
||||
# the panel's own notifications, sent as postmaster@ the hostname. OpenDKIM
|
||||
# signs them with the key of the server's own name; the journal-milter is not
|
||||
# on this chain, since they are not an application's mail. Bounces Postfix
|
||||
# generates itself do not pass it (internal_mail_filter_classes is empty).
|
||||
postconf -e \
|
||||
"milter_protocol=6" \
|
||||
"milter_default_action=tempfail" \
|
||||
"smtpd_milters={ unix:${OPENDKIM_SOCK}, default_action=tempfail }, { unix:${JOURNAL_SOCK}, default_action=accept }" \
|
||||
"non_smtpd_milters="
|
||||
"smtpd_milters={ unix:${SENDER_SOCK}, default_action=tempfail }, { unix:${OPENDKIM_SOCK}, default_action=tempfail }, { unix:${JOURNAL_SOCK}, default_action=accept }" \
|
||||
"non_smtpd_milters={ unix:${OPENDKIM_SOCK}, default_action=tempfail }"
|
||||
|
||||
# What Postfix tells a milter with MAIL FROM: its defaults plus the client's
|
||||
# address. The panel's milters judge every message by that address (an
|
||||
# application's client-address list) and record it (the inbound journal), and
|
||||
# the address given once at connect is not there for a connection's later
|
||||
# messages - see MailFrom in internal/milter/milter.go.
|
||||
postconf -e \
|
||||
"milter_mail_macros=i {auth_type} {auth_authen} {auth_author} {mail_addr} {mail_host} {mail_mailer} {client_addr}"
|
||||
|
||||
# Bounded milter timeouts: a *hung* milter (socket accepts but never replies)
|
||||
# must fail open just like a crash, not stall mail acceptance until the Postfix
|
||||
@@ -291,11 +322,23 @@ if [ "${INBOUND_RELAY_ENABLE}" = "true" ] || [ "${DMARC_REPORTS_ENABLE}" = "true
|
||||
# attribute" on every report. They go into the service line itself.
|
||||
# ${recipient} is expanded by pipe(8), not by the shell: the command
|
||||
# learns which report address the message came to from its argument.
|
||||
postconf -M 'dmarc-ingest/unix=dmarc-ingest unix - n n - - pipe flags=Rq user=panel argv=/usr/local/bin/dmarc-ingest ${recipient}'
|
||||
# One at a time (the 1 before "pipe"): reports are in no hurry, every run
|
||||
# writes to the database the panel writes to, and a hundred of them in
|
||||
# parallel - Postfix's default - is what a flood of reports would start.
|
||||
postconf -M 'dmarc-ingest/unix=dmarc-ingest unix - n n - 1 pipe flags=Rq user=panel argv=/usr/local/bin/dmarc-ingest ${recipient}'
|
||||
# One recipient per run, so ${recipient} is one argument.
|
||||
postconf -e "dmarc-ingest_destination_recipient_limit=1"
|
||||
# pipe(8) starts the command with export_environment only, so what the
|
||||
# command reads from the container's environment is named there.
|
||||
# export_environment is a list split at whitespace in which Postfix
|
||||
# expands "$": a path with either would reach the command as another
|
||||
# path, and it would quietly open a database of its own.
|
||||
case "${SELFPOST_DATA_DIR:-}${SELFPOST_DB_PATH:-}" in
|
||||
*[[:space:]]* | *'$'*)
|
||||
echo "postfix-config: SELFPOST_DATA_DIR and SELFPOST_DB_PATH must not contain whitespace or \$ when DMARC_REPORTS_ENABLE=true" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
DMARC_INGEST_ENV="DMARC_REPORTS_ENABLE=true"
|
||||
[ -z "${SELFPOST_DATA_DIR:-}" ] || DMARC_INGEST_ENV="${DMARC_INGEST_ENV} SELFPOST_DATA_DIR=${SELFPOST_DATA_DIR}"
|
||||
[ -z "${SELFPOST_DB_PATH:-}" ] || DMARC_INGEST_ENV="${DMARC_INGEST_ENV} SELFPOST_DB_PATH=${SELFPOST_DB_PATH}"
|
||||
@@ -345,10 +388,18 @@ if [ "${INBOUND_RELAY_ENABLE}" = "true" ] || [ "${DMARC_REPORTS_ENABLE}" = "true
|
||||
# would file inbound mail into the outbound send log under the *sender's*
|
||||
# domain — polluting Deliveries and letting a forged From: consume that
|
||||
# domain's level-2 rate-limit budget. So smtpd_milters is always overridden
|
||||
# here: either the optional antispam milter, or nothing at all.
|
||||
# here: the optional antispam milter, then - with the inbound relay on - the
|
||||
# panel's inbound journal milter, or nothing at all.
|
||||
INBOUND_MILTERS=""
|
||||
if [ "${INBOUND_RELAY_ENABLE}" = "true" ] && [ -n "${INBOUND_ANTISPAM_MILTER}" ]; then
|
||||
# A glob's "*" matches anything, so the shape alone would pass
|
||||
# "inet:a:1},{unix:/elsewhere": characters outside the set are refused
|
||||
# first, then the shape is checked.
|
||||
case "${INBOUND_ANTISPAM_MILTER}" in
|
||||
*[!A-Za-z0-9._:/-]*)
|
||||
echo "FATAL: INBOUND_ANTISPAM_MILTER must be inet:host:port or unix:/path, got: ${INBOUND_ANTISPAM_MILTER}" >&2
|
||||
exit 1
|
||||
;;
|
||||
inet:[A-Za-z0-9._-]*:[0-9]* | unix:/[A-Za-z0-9._/-]* ) ;;
|
||||
*)
|
||||
echo "FATAL: INBOUND_ANTISPAM_MILTER must be inet:host:port or unix:/path, got: ${INBOUND_ANTISPAM_MILTER}" >&2
|
||||
@@ -364,6 +415,26 @@ if [ "${INBOUND_RELAY_ENABLE}" = "true" ] || [ "${DMARC_REPORTS_ENABLE}" = "true
|
||||
esac
|
||||
INBOUND_MILTERS="{${INBOUND_ANTISPAM_MILTER},default_action=${INBOUND_ANTISPAM_MILTER_ACTION}}"
|
||||
fi
|
||||
if [ "${INBOUND_RELAY_ENABLE}" = "true" ]; then
|
||||
# Last in the chain: Postfix hands a message to its milters one after
|
||||
# another at end-of-message, so this one reads the verdict headers the
|
||||
# filter added and sees only what the filter let through. It watches and
|
||||
# never refuses, and must never hold up someone else's mail: accept when
|
||||
# it is down. A comma, not a space, between the two - see
|
||||
# master_cf_set_smtp_inet_o.
|
||||
case "${INBOUND_JOURNAL_SOCK}" in
|
||||
*[!A-Za-z0-9._/-]*)
|
||||
echo "FATAL: INBOUND_JOURNAL_MILTER_SOCKET must be an absolute path, got: ${INBOUND_JOURNAL_SOCK}" >&2
|
||||
exit 1
|
||||
;;
|
||||
/[A-Za-z0-9._/-]*) ;;
|
||||
*)
|
||||
echo "FATAL: INBOUND_JOURNAL_MILTER_SOCKET must be an absolute path, got: ${INBOUND_JOURNAL_SOCK}" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
INBOUND_MILTERS="${INBOUND_MILTERS:+${INBOUND_MILTERS},}{unix:${INBOUND_JOURNAL_SOCK},default_action=accept}"
|
||||
fi
|
||||
|
||||
PORT25_RATE="${INBOUND_RATE_LIMIT_MESSAGES_PER_IP}"
|
||||
PORT25_SIZE="${INBOUND_MESSAGE_SIZE_LIMIT}"
|
||||
@@ -380,8 +451,12 @@ if [ "${INBOUND_RELAY_ENABLE}" = "true" ] || [ "${DMARC_REPORTS_ENABLE}" = "true
|
||||
fi
|
||||
fi
|
||||
|
||||
# syslog_name: this listener's lines read postfix/inbound/smtpd, the
|
||||
# submission ports' postfix/smtpd. The log tailer files a refusal on port 25
|
||||
# into the inbound journal and has to tell the two apart.
|
||||
postconf -M "smtp/inet=smtp inet n - n - - smtpd"
|
||||
postconf -P \
|
||||
"smtp/inet/syslog_name=postfix/inbound" \
|
||||
"smtp/inet/smtpd_sasl_auth_enable=no" \
|
||||
"smtp/inet/smtpd_tls_auth_only=no" \
|
||||
"smtp/inet/smtpd_sender_login_maps=" \
|
||||
|
||||
@@ -2,22 +2,31 @@
|
||||
# Postfix start wrapper: solves the cold-start race where Postfix would try to
|
||||
# reach the milter sockets before they are listening.
|
||||
#
|
||||
# It blocks until BOTH milter sockets — OpenDKIM and the panel's journal-milter
|
||||
# — are present, then execs `postfix start-fg`. If they are not ready within the
|
||||
# It blocks until the milter sockets — OpenDKIM, the panel's sender check and
|
||||
# journal-milter and, with the inbound relay on, its inbound journal milter —
|
||||
# are present,
|
||||
# then execs `postfix start-fg`. If they are not ready within the
|
||||
# timeout it exits non-zero WITHOUT starting Postfix, so supervisord/Docker see
|
||||
# an explicit startup failure instead of a relay running blind.
|
||||
#
|
||||
# This handles cold start only. Runtime milter failures after a successful start
|
||||
# are governed by fail-open (milter_default_action).
|
||||
# This handles cold start only. Once Postfix runs, a milter that does not
|
||||
# answer costs what its default_action says (build/postfix-config.sh): mail is
|
||||
# deferred for OpenDKIM and the sender check, and passes the journal-milter.
|
||||
set -eu
|
||||
|
||||
OPENDKIM_SOCK="${OPENDKIM_SOCKET:-/run/opendkim/opendkim.sock}"
|
||||
JOURNAL_SOCK="${JOURNAL_MILTER_SOCKET:-/run/selfpost/journal.sock}"
|
||||
SENDER_SOCK="${SENDER_CHECK_SOCKET:-/run/selfpost/sender.sock}"
|
||||
TIMEOUT="${MILTER_WAIT_TIMEOUT:-30}"
|
||||
INTERVAL=1
|
||||
|
||||
SOCKS="$OPENDKIM_SOCK $JOURNAL_SOCK $SENDER_SOCK"
|
||||
if [ "${INBOUND_RELAY_ENABLE:-false}" = "true" ]; then
|
||||
SOCKS="$SOCKS ${INBOUND_JOURNAL_MILTER_SOCKET:-/run/selfpost/inbound-journal.sock}"
|
||||
fi
|
||||
|
||||
elapsed=0
|
||||
for sock in "$OPENDKIM_SOCK" "$JOURNAL_SOCK"; do
|
||||
for sock in $SOCKS; do
|
||||
while [ ! -S "$sock" ]; do
|
||||
if [ "$elapsed" -ge "$TIMEOUT" ]; then
|
||||
echo "postfix-wrapper: timed out after ${TIMEOUT}s waiting for milter socket $sock" >&2
|
||||
@@ -29,5 +38,5 @@ for sock in "$OPENDKIM_SOCK" "$JOURNAL_SOCK"; do
|
||||
echo "postfix-wrapper: milter socket ready: $sock"
|
||||
done
|
||||
|
||||
echo "postfix-wrapper: both milter sockets ready, starting postfix"
|
||||
echo "postfix-wrapper: milter sockets ready, starting postfix"
|
||||
exec postfix start-fg
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
; SelfPost process supervision.
|
||||
;
|
||||
; Start ORDER is enforced by priority=: OpenDKIM, then the panel (which opens
|
||||
; the journal-milter socket), then the Postfix wrapper — which additionally
|
||||
; blocks until both milter sockets are ready before starting Postfix.
|
||||
; the sender check's and the journal-milter's sockets), then the Postfix
|
||||
; wrapper — which additionally blocks until the milter sockets are ready
|
||||
; before starting Postfix.
|
||||
;
|
||||
; If any managed process exhausts its restart retries (FATAL), the crashexit
|
||||
; event listener brings the whole container down so Docker's restart policy can
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -28,14 +29,47 @@ func runDMARCIngestMode() error {
|
||||
|
||||
recipient := ingestRecipient(os.Args[1:])
|
||||
if err := dmarc.IngestMessage(st, io.LimitReader(os.Stdin, maxIngestStdin), recipient, time.Now().UTC()); err != nil {
|
||||
if !errors.Is(err, dmarc.ErrRejected) {
|
||||
return err
|
||||
}
|
||||
// Not a report this relay keeps. It is dropped: deferring would have
|
||||
// Postfix try it again for days, and a bounce would go to an envelope
|
||||
// sender anyone can write - the report addresses are public.
|
||||
if incrErr := st.IncrDMARCParseFailures(); incrErr != nil {
|
||||
log.Printf("dmarc ingest: record failure: %v", incrErr)
|
||||
}
|
||||
return err
|
||||
logRejected(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// logRejected leaves the reason in the mail log. Postfix puts the start of
|
||||
// what a delivery command wrote - about a hundred bytes - into its own line
|
||||
// for the message ("status=sent (delivered via dmarc-ingest service (...))"),
|
||||
// so the reason goes out first and bare: no prefix, no date, no recipient,
|
||||
// all of which that line already has. postlog is no use here: pipe(8) starts
|
||||
// the command without the group that may write to Postfix's log socket.
|
||||
func logRejected(reason error) {
|
||||
fmt.Fprintln(os.Stderr, "dropped: "+logSafe(strings.TrimPrefix(reason.Error(), dmarc.ErrRejected.Error()+": ")))
|
||||
}
|
||||
|
||||
// logSafe makes a reason fit for a log line. Parts of it are the sender's own
|
||||
// bytes - a header line, a file name inside an archive, a piece of XML - so it
|
||||
// is cut short and anything that is not printable ASCII becomes "?", whatever
|
||||
// Postfix would have done with it.
|
||||
func logSafe(s string) string {
|
||||
const max = 200
|
||||
if len(s) > max {
|
||||
s = s[:max]
|
||||
}
|
||||
return strings.Map(func(r rune) rune {
|
||||
if r < 0x20 || r > 0x7e {
|
||||
return '?'
|
||||
}
|
||||
return r
|
||||
}, s)
|
||||
}
|
||||
|
||||
// ingestRecipient is the report address the message was delivered to. Postfix's
|
||||
// pipe(8) hands it over as an argument (${recipient} in the service's argv,
|
||||
// build/postfix-config.sh) - it sets no RECIPIENT variable, that is local(8).
|
||||
|
||||
@@ -18,3 +18,17 @@ func TestIngestRecipient(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogSafe(t *testing.T) {
|
||||
got := logSafe("zip entry \"a\nstatus=bounced\tx\" \x1b[31m é")
|
||||
if want := "zip entry \"a?status=bounced?x\" ?[31m ?"; got != want {
|
||||
t.Errorf("logSafe = %q, want %q", got, want)
|
||||
}
|
||||
long := make([]byte, 1000)
|
||||
for i := range long {
|
||||
long[i] = 'a'
|
||||
}
|
||||
if got := logSafe(string(long)); len(got) != 200 {
|
||||
t.Errorf("logSafe kept %d bytes of 1000, want 200", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ var documentedInternal = []string{
|
||||
"SELFPOST_SETUP_TOKEN_FILE",
|
||||
"PANEL_HTTP_ADDR",
|
||||
"JOURNAL_MILTER_SOCKET",
|
||||
"SENDER_CHECK_SOCKET",
|
||||
"MAIL_LOG",
|
||||
"PANEL_COOKIE_SECURE",
|
||||
"OPENDKIM_SOCKET",
|
||||
@@ -70,6 +71,7 @@ var loadConfigKeys = []string{
|
||||
"SELFPOST_DATA_DIR",
|
||||
"PANEL_HTTP_ADDR",
|
||||
"JOURNAL_MILTER_SOCKET",
|
||||
"SENDER_CHECK_SOCKET",
|
||||
"MAIL_LOG",
|
||||
"SEND_LOG_RETENTION_DAYS",
|
||||
"SELFPOST_DB_PATH",
|
||||
@@ -104,6 +106,7 @@ var buildScriptKeys = []string{
|
||||
"RATE_LIMIT_WINDOW_SECONDS",
|
||||
"OPENDKIM_SOCKET",
|
||||
"JOURNAL_MILTER_SOCKET",
|
||||
"SENDER_CHECK_SOCKET",
|
||||
"MAIL_LOG",
|
||||
"POSTFIX_SENDER_LOGIN_MAPS",
|
||||
"POSTFIX_QUEUE_DIR",
|
||||
|
||||
+15
-2
@@ -24,6 +24,7 @@ type mailStack struct {
|
||||
Domains *domain.Service
|
||||
Apps *app.Service
|
||||
Inbound *inbound.Service
|
||||
Lists *inbound.Lists
|
||||
DMARC *dmarc.Service
|
||||
pf *postfix.Postfix
|
||||
odk *domain.OpenDKIM
|
||||
@@ -36,11 +37,15 @@ func newMailStack(cfg config, st *store.Store) *mailStack {
|
||||
odk := domain.NewOpenDKIM(cfg.opendkimDir)
|
||||
apps := app.NewService(st, app.NewSASLDB(cfg.saslDBPath, cfg.saslRealm), pf)
|
||||
domains := domain.NewService(st, odk, apps, cfg.dkimSelectorDef)
|
||||
// The panel's notifications are sent as postmaster@ the server's own name,
|
||||
// and signed as it.
|
||||
domains.SetServerName(cfg.hostname)
|
||||
inb := inbound.NewService(st, pf)
|
||||
dmarcSvc := dmarc.NewService(st, pf, cfg.hostname, cfg.dmarcEnabled)
|
||||
return &mailStack{
|
||||
Domains: domains, Apps: apps, Inbound: inb, DMARC: dmarcSvc,
|
||||
pf: pf, odk: odk, inboundEnabled: cfg.inboundEnabled, dmarcEnabled: cfg.dmarcEnabled,
|
||||
Lists: inbound.NewLists(st, cfg.antispamDir),
|
||||
pf: pf, odk: odk, inboundEnabled: cfg.inboundEnabled, dmarcEnabled: cfg.dmarcEnabled,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,6 +62,11 @@ func (m *mailStack) Resync() error {
|
||||
if err := m.Inbound.Resync(); err != nil {
|
||||
return fmt.Errorf("inbound maps resync: %w", err)
|
||||
}
|
||||
// The filter's configuration names all six list files; they exist from
|
||||
// the first start, empty until an entry is added.
|
||||
if err := m.Lists.Sync(); err != nil {
|
||||
return fmt.Errorf("filter lists resync: %w", err)
|
||||
}
|
||||
}
|
||||
if m.dmarcEnabled {
|
||||
if err := m.DMARC.Resync(); err != nil {
|
||||
@@ -91,7 +101,7 @@ func newPanel(cfg config, st *store.Store) (*web.Server, error) {
|
||||
// effective config, including a manual override; the panel keeps this
|
||||
// snapshot for the process lifetime (architecture.md).
|
||||
retryPolicy := postfix.LoadRetryPolicy()
|
||||
return web.New(st, ms.Domains, ms.Apps, ms.Inbound, ms.DMARC, web.Config{
|
||||
return web.New(st, ms.Domains, ms.Apps, ms.Inbound, ms.Lists, ms.DMARC, web.Config{
|
||||
Hostname: cfg.hostname,
|
||||
CookieSecure: cfg.cookieSecure,
|
||||
SubmissionEnabled: cfg.submissionEnabled,
|
||||
@@ -104,6 +114,7 @@ func newPanel(cfg config, st *store.Store) (*web.Server, error) {
|
||||
TLSCertFile: cfg.tlsCertFile,
|
||||
OpenDKIMSocket: cfg.opendkimSocket,
|
||||
JournalSocket: cfg.journalSocket,
|
||||
SenderSocket: cfg.senderSocket,
|
||||
SessionIdleDays: cfg.sessionIdleDays,
|
||||
DNSResolvers: cfg.dnsResolvers,
|
||||
RateLimitMessagesPerIP: cfg.rateLimitMessagesPerIP,
|
||||
@@ -114,6 +125,8 @@ func newPanel(cfg config, st *store.Store) (*web.Server, error) {
|
||||
InboundAntispamAction: cfg.inboundAntispamAction,
|
||||
DMARCEnabled: cfg.dmarcEnabled,
|
||||
SendLogRetentionEnvDefault: cfg.retentionDays,
|
||||
|
||||
InboundSpamLogRetentionEnvDefault: cfg.inboundSpamLogRetentionDays,
|
||||
}, cfg.setupTokenPath)
|
||||
}
|
||||
|
||||
|
||||
+39
-9
@@ -8,6 +8,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/inbound"
|
||||
"github.com/mixeme/selfpost/internal/milter"
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
@@ -15,21 +16,52 @@ import (
|
||||
// serveJournal opens the journal-milter Unix socket and runs the real milter
|
||||
// (architecture.md § Mail path), recording accepted messages into the send
|
||||
// log. Socket lifecycle (creation, stale cleanup, group permissions) lives
|
||||
// here; the protocol handler lives in internal/milter.
|
||||
// in listenMilter; the protocol handler lives in internal/milter.
|
||||
func serveJournal(ctx context.Context, cfg config, st *store.Store) error {
|
||||
socketPath := cfg.journalSocket
|
||||
if err := os.MkdirAll(filepath.Dir(socketPath), 0o755); err != nil {
|
||||
ln, err := listenMilter(cfg.journalSocket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("journal-milter listening on %s", cfg.journalSocket)
|
||||
return milter.Serve(ctx, ln, st)
|
||||
}
|
||||
|
||||
// serveInboundJournal runs the inbound journal milter, the last milter on
|
||||
// port 25 (internal/milter/inbound.go). It exists only while the inbound relay
|
||||
// is on: build/postfix-config.sh attaches the socket under the same condition.
|
||||
func serveInboundJournal(ctx context.Context, cfg config, st *store.Store) error {
|
||||
ln, err := listenMilter(cfg.inboundJournalSocket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// With a filter attached, the key it signs with. Without the key no
|
||||
// verdict can be told from a sender's own, so none is read: the journal
|
||||
// still records the mail, and the log says why it shows no verdicts.
|
||||
key := ""
|
||||
if cfg.inboundAntispamMilter != "" {
|
||||
if key, err = inbound.FilterKey(cfg.antispamDir); err != nil {
|
||||
log.Printf("inbound-journal-milter: no key for the filter's signature, verdicts will not be recorded: %v", err)
|
||||
key = ""
|
||||
}
|
||||
}
|
||||
log.Printf("inbound-journal-milter listening on %s", cfg.inboundJournalSocket)
|
||||
return milter.ServeInbound(ctx, ln, st, key)
|
||||
}
|
||||
|
||||
// listenMilter opens a milter's Unix socket where Postfix can reach it.
|
||||
func listenMilter(socketPath string) (net.Listener, error) {
|
||||
if err := os.MkdirAll(filepath.Dir(socketPath), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Clear a stale socket left behind by an unclean shutdown, otherwise the
|
||||
// listen below fails with "address already in use".
|
||||
if err := os.Remove(socketPath); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ln, err := net.Listen("unix", socketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
// Postfix (user `postfix`) connects to this socket as a milter and needs
|
||||
// group access. The socket inherits group `selfpost` from the setgid parent
|
||||
@@ -37,9 +69,7 @@ func serveJournal(ctx context.Context, cfg config, st *store.Store) error {
|
||||
// it (connecting to a Unix socket needs write permission on the node).
|
||||
if err := os.Chmod(socketPath, 0o660); err != nil {
|
||||
ln.Close()
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Printf("journal-milter listening on %s", socketPath)
|
||||
return milter.Serve(ctx, ln, st)
|
||||
return ln, nil
|
||||
}
|
||||
|
||||
+64
-4
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/mixeme/selfpost/internal/backup"
|
||||
"github.com/mixeme/selfpost/internal/buildinfo"
|
||||
"github.com/mixeme/selfpost/internal/dnscheck"
|
||||
"github.com/mixeme/selfpost/internal/inbound"
|
||||
"github.com/mixeme/selfpost/internal/logtail"
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
@@ -59,6 +60,8 @@ func main() {
|
||||
type config struct {
|
||||
httpAddr string
|
||||
journalSocket string
|
||||
// The sender check's socket; the default mirrors build/postfix-config.sh.
|
||||
senderSocket string
|
||||
mailLog string
|
||||
retentionDays int
|
||||
|
||||
@@ -89,6 +92,9 @@ type config struct {
|
||||
saslRealm string
|
||||
postfixDir string
|
||||
deployRoot string
|
||||
// Where the filter lists are written as map files for the external spam
|
||||
// filter; the antispam compose fragment mounts it into the sidecar.
|
||||
antispamDir string
|
||||
|
||||
inboundEnabled bool
|
||||
// The optional milter on the inbound listener (INBOUND_ANTISPAM_MILTER) and
|
||||
@@ -96,7 +102,12 @@ type config struct {
|
||||
// what attaches it; the panel only says so. See antispamAction.
|
||||
inboundAntispamMilter string
|
||||
inboundAntispamAction string
|
||||
dmarcEnabled bool
|
||||
// The inbound journal milter's socket; the default mirrors
|
||||
// build/postfix-config.sh. Served only while the inbound relay is on.
|
||||
inboundJournalSocket string
|
||||
// Retention of the inbound journal until Server > Settings says otherwise.
|
||||
inboundSpamLogRetentionDays int
|
||||
dmarcEnabled bool
|
||||
}
|
||||
|
||||
func loadConfig() config {
|
||||
@@ -104,6 +115,7 @@ func loadConfig() config {
|
||||
return config{
|
||||
httpAddr: envDefault("PANEL_HTTP_ADDR", ":8080"),
|
||||
journalSocket: envDefault("JOURNAL_MILTER_SOCKET", "/run/selfpost/journal.sock"),
|
||||
senderSocket: envDefault("SENDER_CHECK_SOCKET", "/run/selfpost/sender.sock"),
|
||||
// Postfix's delivery log, under /data so the lines that resolve a
|
||||
// "queued" send-log row outlive the container. The default must match
|
||||
// maillog_file in build/postfix-config.sh.
|
||||
@@ -157,11 +169,16 @@ func loadConfig() config {
|
||||
saslDBPath: envDefault("SASL_DB_PATH", filepath.Join(dataDir, "sasl", "sasldb2")),
|
||||
saslRealm: saslRealm(),
|
||||
postfixDir: envDefault("POSTFIX_DIR", filepath.Join(dataDir, "postfix")),
|
||||
deployRoot: envDefault("SELFPOST_DEPLOY_ROOT", "/selfpost-deploy"),
|
||||
|
||||
antispamDir: envDefault("ANTISPAM_DIR", filepath.Join(dataDir, "antispam")),
|
||||
deployRoot: envDefault("SELFPOST_DEPLOY_ROOT", "/selfpost-deploy"),
|
||||
// Optional inbound relay (backup-MX / forwarder). Off unless exactly "true".
|
||||
inboundEnabled: os.Getenv("INBOUND_RELAY_ENABLE") == "true",
|
||||
inboundAntispamMilter: os.Getenv("INBOUND_ANTISPAM_MILTER"),
|
||||
inboundAntispamAction: antispamAction(os.Getenv("INBOUND_ANTISPAM_MILTER_ACTION")),
|
||||
inboundJournalSocket: envDefault("INBOUND_JOURNAL_MILTER_SOCKET", "/run/selfpost/inbound-journal.sock"),
|
||||
|
||||
inboundSpamLogRetentionDays: envInt("INBOUND_SPAM_LOG_RETENTION_DAYS", store.InboundSpamLogRetentionDaysDefault),
|
||||
// Optional DMARC aggregate ingest on port 25. Off unless exactly "true".
|
||||
dmarcEnabled: os.Getenv("DMARC_REPORTS_ENABLE") == "true",
|
||||
}
|
||||
@@ -289,12 +306,46 @@ func run() error {
|
||||
}
|
||||
}
|
||||
|
||||
// The filter lists as files, on every start: the spam filter's
|
||||
// configuration names all six, and a data directory from before the lists
|
||||
// existed has none. Not fatal - the lists are the filter's business, and
|
||||
// the panel has mail to relay either way.
|
||||
if cfg.inboundEnabled && !restored {
|
||||
if err := inbound.NewLists(st, cfg.antispamDir).Sync(); err != nil {
|
||||
log.Printf("filter lists bootstrap: %v", err)
|
||||
}
|
||||
}
|
||||
// The key the filter signs with has to be in the shared directory before
|
||||
// the filter reads it, whether or not a filter is attached yet.
|
||||
if cfg.inboundEnabled {
|
||||
if _, err := inbound.FilterKey(cfg.antispamDir); err != nil {
|
||||
log.Printf("filter key bootstrap: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := st.EnsureSendLogRetentionDays(cfg.retentionDays); err != nil {
|
||||
return fmt.Errorf("bootstrap send-log retention: %w", err)
|
||||
}
|
||||
// With the inbound relay on, the tailer also writes the inbound journal's
|
||||
// rows for refused mail and prunes that journal.
|
||||
var inboundTail *logtail.Inbound
|
||||
if cfg.inboundEnabled {
|
||||
if err := st.EnsureInboundSpamLogRetentionDays(cfg.inboundSpamLogRetentionDays); err != nil {
|
||||
return fmt.Errorf("bootstrap inbound journal retention: %w", err)
|
||||
}
|
||||
inboundTail = &logtail.Inbound{Store: st, Retention: func() int {
|
||||
days, err := st.GetInboundSpamLogRetentionDays(cfg.inboundSpamLogRetentionDays)
|
||||
if err != nil {
|
||||
log.Printf("inbound journal retention: %v", err)
|
||||
return cfg.inboundSpamLogRetentionDays
|
||||
}
|
||||
return days
|
||||
}}
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errc := make(chan error, 4)
|
||||
// Room for every role to report a failure without blocking: seven at most.
|
||||
errc := make(chan error, 7)
|
||||
|
||||
roles := []struct {
|
||||
name string
|
||||
@@ -302,6 +353,7 @@ func run() error {
|
||||
}{
|
||||
{"http", func(ctx context.Context) error { return serveHTTP(ctx, cfg, st) }},
|
||||
{"journal-milter", func(ctx context.Context) error { return serveJournal(ctx, cfg, st) }},
|
||||
{"sender-check", func(ctx context.Context) error { return serveSenderCheck(ctx, cfg, st) }},
|
||||
{"log-tailer", func(ctx context.Context) error {
|
||||
return logtail.Run(ctx, cfg.mailLog, st, func() int {
|
||||
days, err := st.GetSendLogRetentionDays(cfg.retentionDays)
|
||||
@@ -310,12 +362,20 @@ func run() error {
|
||||
return cfg.retentionDays
|
||||
}
|
||||
return days
|
||||
})
|
||||
}, inboundTail, &logtail.SenderCheck{Store: st, Socket: cfg.senderSocket})
|
||||
}},
|
||||
{"rate-limit-recalc", func(ctx context.Context) error {
|
||||
runAutoRateLimitRecalc(ctx, cfg, st)
|
||||
return nil
|
||||
}},
|
||||
{"notifier", func(ctx context.Context) error { return runNotifier(ctx, cfg, st) }},
|
||||
}
|
||||
|
||||
if cfg.inboundEnabled {
|
||||
roles = append(roles, struct {
|
||||
name string
|
||||
fn func(context.Context) error
|
||||
}{"inbound-journal-milter", func(ctx context.Context) error { return serveInboundJournal(ctx, cfg, st) }})
|
||||
}
|
||||
|
||||
for _, r := range roles {
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/dnscheck"
|
||||
"github.com/mixeme/selfpost/internal/health"
|
||||
"github.com/mixeme/selfpost/internal/notify"
|
||||
"github.com/mixeme/selfpost/internal/postfix"
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
|
||||
// runNotifier writes to the panel's users when something needs them
|
||||
// (architecture.md § Notifications), until ctx is cancelled. It reads what the
|
||||
// pages read - a domain's published records against the key this server signs
|
||||
// with, the certificate file, Postfix's queue - with a DNS checker of its own,
|
||||
// so that its hourly look is never a page's cached one.
|
||||
func runNotifier(ctx context.Context, cfg config, st *store.Store) error {
|
||||
ms := newMailStack(cfg, st)
|
||||
dns := dnscheck.New(cfg.dnsResolvers)
|
||||
|
||||
// OpenDKIM's tables are rebuilt when a domain is added or deleted. On the
|
||||
// first start of a version that signs the server's own name they do not
|
||||
// have its line yet. A failure here costs the signature, not the message.
|
||||
if rebuilt, err := ms.Domains.EnsureServerSigned(); err != nil {
|
||||
log.Printf("notifier: sign as %s: %v", cfg.hostname, err)
|
||||
} else if rebuilt {
|
||||
log.Printf("notifier: OpenDKIM now signs mail of %s", cfg.hostname)
|
||||
}
|
||||
|
||||
n := notify.New(st, notify.Checks{
|
||||
Domain: func(d store.Domain) (dnscheck.Domain, error) {
|
||||
rec, err := ms.Domains.DKIMRecord(d)
|
||||
if err != nil {
|
||||
return dnscheck.Domain{}, err
|
||||
}
|
||||
// The server's own addresses are cached for a minute: one lookup
|
||||
// for a pass over all the domains.
|
||||
srv := dns.Server(cfg.hostname, false)
|
||||
return dns.Domain(dnscheck.Query{
|
||||
Name: d.Name,
|
||||
Selector: d.DKIMSelector,
|
||||
ExpectedDKIM: rec.Value,
|
||||
Hostname: srv.Hostname,
|
||||
ServerIPs: srv.IPs,
|
||||
DMARCReportEmail: d.DMARCRua,
|
||||
}, true), nil
|
||||
},
|
||||
Certificate: func() health.Certificate { return health.CheckCertificate(cfg.tlsCertFile) },
|
||||
Queue: postfix.QueueMessages,
|
||||
}, notify.NewMailer(cfg.hostname), cfg.hostname)
|
||||
return n.Run(ctx)
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/domain"
|
||||
"github.com/mixeme/selfpost/internal/health"
|
||||
"github.com/mixeme/selfpost/internal/milter"
|
||||
"github.com/mixeme/selfpost/internal/postfix"
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
|
||||
// aliveEvery is how often the sender check marks itself alive. The gap a
|
||||
// restart finds since the last mark is the check's downtime, so the mark's
|
||||
// age is the error of that figure.
|
||||
const aliveEvery = 2 * time.Second
|
||||
|
||||
// serveSenderCheck runs the sender check (internal/milter/sender.go) on a
|
||||
// socket of its own. Postfix attaches that socket strictly, so while this
|
||||
// role is not serving, mail is deferred: what that costs is counted here and
|
||||
// by the log tailer, and shown on Server > Health
|
||||
// (docs/plans/from-header-binding.md § Statistics).
|
||||
func serveSenderCheck(ctx context.Context, cfg config, st *store.Store) error {
|
||||
ln, err := listenMilter(cfg.senderSocket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("sender-check listening on %s", cfg.senderSocket)
|
||||
|
||||
// Statistics are the database's business and supervisord's, and the check
|
||||
// is neither: they are kept beside it, and nothing of them is waited for
|
||||
// before it serves.
|
||||
go func() {
|
||||
now := time.Now()
|
||||
mark := cfg.senderSocket + ".alive"
|
||||
if err := st.StartSenderCheckStats(now); err != nil {
|
||||
log.Printf("sender-check: statistics: %v", err)
|
||||
}
|
||||
if down := senderCheckDowntime(mark, now, postfixStarted); down > 0 {
|
||||
log.Printf("sender-check: was not answering for %s while Postfix was running", down.Round(time.Second))
|
||||
for _, part := range byDay(now.Add(-down), now) {
|
||||
if err := st.AddSenderCheckDown(part.day, part.down); err != nil {
|
||||
log.Printf("sender-check: statistics: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
keepAlive(ctx, mark, st)
|
||||
}()
|
||||
|
||||
// The two tables the mail path itself goes by: whom Postfix lets send as
|
||||
// what, and what OpenDKIM signs.
|
||||
bind := milter.NewBindings(postfix.New(cfg.postfixDir).SenderLoginMapsPath())
|
||||
signers := milter.NewSigners(domain.NewOpenDKIM(cfg.opendkimDir).SigningTablePath())
|
||||
return milter.ServeSender(ctx, ln, bind, signers, st)
|
||||
}
|
||||
|
||||
// senderCheckDowntime is for how long the check was not serving before now
|
||||
// while Postfix was: the time since the mark at path was last set, or since
|
||||
// Postfix started if that was later. No mark means the container is new, and
|
||||
// a Postfix that is not running means mail was stopped for another reason -
|
||||
// a rollout, a reboot - and neither is the check's downtime.
|
||||
func senderCheckDowntime(path string, now time.Time, postfixStarted func() (time.Time, bool)) time.Duration {
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
from := fi.ModTime()
|
||||
started, running := postfixStarted()
|
||||
if !running {
|
||||
return 0
|
||||
}
|
||||
if started.After(from) {
|
||||
from = started
|
||||
}
|
||||
if d := now.Sub(from); d > 0 {
|
||||
return d
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// dayPart is the part of a stretch of downtime that fell on one UTC day.
|
||||
type dayPart struct {
|
||||
day time.Time
|
||||
down time.Duration
|
||||
}
|
||||
|
||||
// byDay cuts the time from 'from' to 'to' at the UTC midnights between them,
|
||||
// so that a night's outage is not all booked on the morning it ended.
|
||||
func byDay(from, to time.Time) []dayPart {
|
||||
from, to = from.UTC(), to.UTC()
|
||||
var parts []dayPart
|
||||
for from.Before(to) {
|
||||
next := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.UTC).AddDate(0, 0, 1)
|
||||
if next.After(to) {
|
||||
next = to
|
||||
}
|
||||
parts = append(parts, dayPart{day: from, down: next.Sub(from)})
|
||||
from = next
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
// postfixStarted asks supervisord since when Postfix has been running.
|
||||
func postfixStarted() (time.Time, bool) {
|
||||
procs, err := health.Processes()
|
||||
if err != nil {
|
||||
// Said aloud: the downtime of this start is then not counted.
|
||||
log.Printf("sender-check: statistics: when Postfix started is not known: %v", err)
|
||||
return time.Time{}, false
|
||||
}
|
||||
for _, p := range procs {
|
||||
if p.Name != "postfix" || p.State != "RUNNING" {
|
||||
continue
|
||||
}
|
||||
if up, ok := health.Uptime(p.Detail); ok {
|
||||
return time.Now().Add(-up), true
|
||||
}
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
|
||||
// keepAlive sets the mark until ctx ends, and once a day drops the days of
|
||||
// the statistics that are older than a year.
|
||||
func keepAlive(ctx context.Context, path string, st *store.Store) {
|
||||
touch := func() {
|
||||
now := time.Now()
|
||||
if err := os.Chtimes(path, now, now); err != nil {
|
||||
if err := os.WriteFile(path, nil, 0o644); err != nil {
|
||||
log.Printf("sender-check: alive mark: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
touch()
|
||||
tick := time.NewTicker(aliveEvery)
|
||||
defer tick.Stop()
|
||||
prune := time.NewTicker(24 * time.Hour)
|
||||
defer prune.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
touch()
|
||||
case <-prune.C:
|
||||
if err := st.PruneSenderCheckDays(time.Now()); err != nil {
|
||||
log.Printf("sender-check: statistics: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The check's downtime is the time since its last mark, under a Postfix that
|
||||
// was running: a new container has no mark, and a Postfix that is not up, or
|
||||
// came up later than the mark, was not kept waiting by the check for all of
|
||||
// that time.
|
||||
func TestSenderCheckDowntime(t *testing.T) {
|
||||
now := time.Date(2026, 10, 12, 18, 0, 0, 0, time.UTC)
|
||||
mark := filepath.Join(t.TempDir(), "sender.sock.alive")
|
||||
set := func(at time.Time) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(mark, nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chtimes(mark, at, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
postfix := func(started time.Time, running bool) func() (time.Time, bool) {
|
||||
return func() (time.Time, bool) { return started, running }
|
||||
}
|
||||
|
||||
if d := senderCheckDowntime(mark, now, postfix(now.Add(-time.Hour), true)); d != 0 {
|
||||
t.Errorf("no mark: %s", d)
|
||||
}
|
||||
set(now.Add(-40 * time.Second))
|
||||
if d := senderCheckDowntime(mark, now, postfix(now.Add(-time.Hour), true)); d != 40*time.Second {
|
||||
t.Errorf("Postfix up throughout: %s, want 40s", d)
|
||||
}
|
||||
if d := senderCheckDowntime(mark, now, postfix(now.Add(-10*time.Second), true)); d != 10*time.Second {
|
||||
t.Errorf("Postfix started within the gap: %s, want 10s", d)
|
||||
}
|
||||
if d := senderCheckDowntime(mark, now, postfix(time.Time{}, false)); d != 0 {
|
||||
t.Errorf("Postfix not running: %s", d)
|
||||
}
|
||||
set(now.Add(time.Minute)) // a clock set back
|
||||
if d := senderCheckDowntime(mark, now, postfix(now.Add(-time.Hour), true)); d != 0 {
|
||||
t.Errorf("a mark in the future: %s", d)
|
||||
}
|
||||
}
|
||||
|
||||
// Downtime is booked on the days it fell on.
|
||||
func TestByDay(t *testing.T) {
|
||||
at := func(d, h, m int) time.Time { return time.Date(2026, 10, d, h, m, 0, 0, time.UTC) }
|
||||
got := byDay(at(12, 23, 50), at(14, 0, 10))
|
||||
want := []dayPart{{at(12, 23, 50), 10 * time.Minute}, {at(13, 0, 0), 24 * time.Hour}, {at(14, 0, 0), 10 * time.Minute}}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("parts = %+v", got)
|
||||
}
|
||||
for i := range want {
|
||||
if !got[i].day.Equal(want[i].day) || got[i].down != want[i].down {
|
||||
t.Errorf("part %d = %+v, want %+v", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
if one := byDay(at(12, 10, 0), at(12, 10, 0).Add(40*time.Second)); len(one) != 1 || one[0].down != 40*time.Second {
|
||||
t.Errorf("within a day: %+v", one)
|
||||
}
|
||||
if none := byDay(at(12, 10, 0), at(12, 10, 0)); len(none) != 0 {
|
||||
t.Errorf("no time: %+v", none)
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,10 @@ DMARC_REPORTS_ENABLE=false
|
||||
# INBOUND_ANTISPAM_MILTER=inet:antispam:11332
|
||||
# INBOUND_ANTISPAM_MILTER_ACTION=accept
|
||||
|
||||
# Days the inbound journal keeps a row (7-365) until the panel says otherwise;
|
||||
# it never holds more than 10 000 rows.
|
||||
# INBOUND_SPAM_LOG_RETENTION_DAYS=90
|
||||
|
||||
# Coarse per-client-IP limit and message size on inbound smtpd (port 25).
|
||||
# INBOUND_RATE_LIMIT_MESSAGES_PER_IP=20
|
||||
# INBOUND_MESSAGE_SIZE_LIMIT=26214400
|
||||
|
||||
@@ -27,3 +27,10 @@ services:
|
||||
image: rspamd/rspamd:3.11
|
||||
restart: unless-stopped
|
||||
# No host ports: only SelfPost talks to the milter on the compose network.
|
||||
volumes:
|
||||
# The verdict headers SelfPost's inbound journal reads.
|
||||
- ./antispam/rspamd/milter_headers.conf:/etc/rspamd/local.d/milter_headers.conf:ro
|
||||
# The allow and deny lists kept in the panel: the rules, and the files
|
||||
# SelfPost writes them to.
|
||||
- ./antispam/rspamd/multimap.conf:/etc/rspamd/local.d/multimap.conf:ro
|
||||
- ./data/antispam:/etc/rspamd/selfpost:ro
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# Mounted into the sidecar as /etc/rspamd/local.d/milter_headers.conf.
|
||||
#
|
||||
# rspamd writes its verdict into two headers of every message it lets through:
|
||||
# the action it chose and the score with the symbols behind it. SelfPost's
|
||||
# inbound journal reads them (Inbound > Log shows them next to the message),
|
||||
# and the upstream mailbox server receives them with the mail.
|
||||
#
|
||||
# rspamd removes headers of these names that a sender wrote himself before it
|
||||
# adds its own. Without this file the journal still lists accepted mail, with
|
||||
# no verdict next to it.
|
||||
#
|
||||
# The third header is the proof that the other two are rspamd's. The filter is
|
||||
# attached so that mail still flows when it is down, and then nothing removes
|
||||
# what a sender wrote: he could send his own verdict. So rspamd signs each
|
||||
# message - an HMAC-SHA256 of the message's queue id under a key SelfPost
|
||||
# generates and leaves in the directory it shares with this container - and
|
||||
# SelfPost reads a verdict only next to a signature it can recompute. The
|
||||
# signature travels on with the mail and is of no use to whoever sees it: it
|
||||
# fits that one queue id.
|
||||
use = ["x-spamd-result", "x-rspamd-action", "selfpost-signature"];
|
||||
|
||||
custom {
|
||||
selfpost-signature = <<EOD
|
||||
return function(task, common_meta)
|
||||
local remove = { ['X-SelfPost-Filter'] = 0 }
|
||||
-- The key is read for every message and kept nowhere: SelfPost may make a
|
||||
-- new one (the file was lost, a backup was restored), and a key remembered
|
||||
-- here would go on signing what SelfPost no longer accepts until someone
|
||||
-- restarted this container. The file is one short line.
|
||||
local key
|
||||
local f = io.open('/etc/rspamd/selfpost/filter.key', 'r')
|
||||
if f then
|
||||
key = f:read('*l')
|
||||
f:close()
|
||||
end
|
||||
local qid = task:get_queue_id()
|
||||
if not key or key == '' or not qid or qid == '' then
|
||||
return nil, {}, remove, {}
|
||||
end
|
||||
local hash = require "rspamd_cryptobox_hash"
|
||||
local sig = hash.create_specific_keyed(key, 'sha256', qid):hex()
|
||||
return nil, { ['X-SelfPost-Filter'] = sig }, remove, {}
|
||||
end
|
||||
EOD;
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
# Mounted into the sidecar as /etc/rspamd/local.d/multimap.conf.
|
||||
#
|
||||
# The allow and deny lists kept in the panel (Inbound > Filter lists). SelfPost
|
||||
# writes them as six files under data/antispam/, which the fragment mounts at
|
||||
# /etc/rspamd/selfpost; rspamd rereads a file when it changes, no restart.
|
||||
#
|
||||
# Deny wins over allow, whichever entries match. A denied sender is refused
|
||||
# before anything else is looked at (a prefilter). An allowed one is not
|
||||
# accepted outright - rspamd would then let whichever of the two rules ran
|
||||
# first decide - but given a weight no spam score outweighs, which a refusal
|
||||
# by the deny list still overrides.
|
||||
#
|
||||
# A domain entry is that domain, not its subdomains.
|
||||
#
|
||||
# SelfPost counts how often an entry decided a message by what these rules
|
||||
# leave behind: the deny message below and the names of the allow symbols.
|
||||
# Change either and Inbound > Filter lists shows "never" for every entry.
|
||||
|
||||
SELFPOST_DENY_IP {
|
||||
type = "ip";
|
||||
map = "/etc/rspamd/selfpost/deny_ip.map";
|
||||
prefilter = true;
|
||||
action = "reject";
|
||||
message = "Refused by this server's deny list";
|
||||
description = "Client address on the deny list";
|
||||
}
|
||||
SELFPOST_DENY_FROM {
|
||||
type = "from";
|
||||
map = "/etc/rspamd/selfpost/deny_from.map";
|
||||
prefilter = true;
|
||||
action = "reject";
|
||||
message = "Refused by this server's deny list";
|
||||
description = "Sender address on the deny list";
|
||||
}
|
||||
SELFPOST_DENY_DOMAIN {
|
||||
type = "from";
|
||||
filter = "email:domain";
|
||||
map = "/etc/rspamd/selfpost/deny_domain.map";
|
||||
prefilter = true;
|
||||
action = "reject";
|
||||
message = "Refused by this server's deny list";
|
||||
description = "Sender domain on the deny list";
|
||||
}
|
||||
|
||||
SELFPOST_ALLOW_IP {
|
||||
type = "ip";
|
||||
map = "/etc/rspamd/selfpost/allow_ip.map";
|
||||
score = -100.0;
|
||||
description = "Client address on the allow list";
|
||||
}
|
||||
SELFPOST_ALLOW_FROM {
|
||||
type = "from";
|
||||
map = "/etc/rspamd/selfpost/allow_from.map";
|
||||
score = -100.0;
|
||||
description = "Sender address on the allow list";
|
||||
}
|
||||
SELFPOST_ALLOW_DOMAIN {
|
||||
type = "from";
|
||||
filter = "email:domain";
|
||||
map = "/etc/rspamd/selfpost/allow_domain.map";
|
||||
score = -100.0;
|
||||
description = "Sender domain on the allow list";
|
||||
}
|
||||
@@ -18,10 +18,12 @@
|
||||
# The image tag below is FIXED on purpose: backup restore compares the manifest
|
||||
# version against the running binary's version, so ":latest" would make that
|
||||
# check meaningless. Bump the tag deliberately when you want to upgrade.
|
||||
# The 2.x line is not on ghcr.io yet: build it locally under this exact name
|
||||
# (README, Quick start) and Compose will use it without pulling.
|
||||
|
||||
services:
|
||||
selfpost:
|
||||
image: ghcr.io/mixeme/selfpost:2.0.0
|
||||
image: ghcr.io/mixeme/selfpost:2.4.1
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
SELFPOST_HOSTNAME: "${SELFPOST_HOSTNAME:?set the mail/panel hostname, e.g. mail.example.com}"
|
||||
@@ -47,6 +49,7 @@ services:
|
||||
# How long the send log keeps rows before the background sweep deletes
|
||||
# them — the main driver of /data growth over time.
|
||||
SEND_LOG_RETENTION_DAYS: "${SEND_LOG_RETENTION_DAYS:-90}"
|
||||
INBOUND_SPAM_LOG_RETENTION_DAYS: "${INBOUND_SPAM_LOG_RETENTION_DAYS:-90}"
|
||||
# Sliding idle timeout for the panel login session, in days.
|
||||
PANEL_SESSION_IDLE_DAYS: "${PANEL_SESSION_IDLE_DAYS:-7}"
|
||||
# Resolvers the panel's PTR/SPF/DKIM/DMARC checks query directly, so they
|
||||
|
||||
+293
-24
@@ -27,13 +27,13 @@ Managed programs ([build/supervisord.conf](../build/supervisord.conf)):
|
||||
| Program | User | Priority | Role |
|
||||
|---|---|---|---|
|
||||
| `opendkim` | root → `opendkim` (`UserID` in opendkim.conf) | 100 | DKIM signing milter |
|
||||
| `panel` | panel | 200 | HTTP UI + journal-milter + log-tailer + rate-limit recalc |
|
||||
| `postfix` | root (wrapper) | 300 | MTA — started only after both milter sockets exist |
|
||||
| `panel` | panel | 200 | HTTP UI + sender check + journal-milter + log-tailer + rate-limit recalc; with the inbound relay on also the inbound journal milter |
|
||||
| `postfix` | root (wrapper) | 300 | MTA — started only after the milter sockets exist (OpenDKIM's, the sender check's, the journal milter's and, with the inbound relay on, the inbound journal milter's) |
|
||||
| `postfix-reload` | root | — | On-demand `postfix reload` (autostart off) |
|
||||
| `cert-reload` | root | 400 | Daily `postfix reload` for renewed TLS certs |
|
||||
| `logrotate` | root | 400 | Periodic `mail.log` rotation |
|
||||
|
||||
Start order: OpenDKIM → panel (opens journal-milter socket) → Postfix wrapper
|
||||
Start order: OpenDKIM → panel (opens the sender check's and the journal-milter's sockets) → Postfix wrapper
|
||||
polls unix sockets (timeout `MILTER_WAIT_TIMEOUT`, default 30s) then
|
||||
`postfix start-fg`.
|
||||
|
||||
@@ -50,6 +50,7 @@ panel, and postfix are RUNNING; Docker `HEALTHCHECK` uses the same probe.
|
||||
```
|
||||
Client ──TLS+SASL──► Postfix (465 smtps, optional 587 submission)
|
||||
│
|
||||
├─► sender check (From: against the login, tempfail on failure)
|
||||
├─► OpenDKIM milter (sign, tempfail on failure)
|
||||
├─► journal-milter (send log + L2 rate limits, fail-open)
|
||||
└─► outbound MX delivery (port 25 client)
|
||||
@@ -74,10 +75,61 @@ smtp/inet` removes Debian's stock smtpd). Outbound delivery still uses the
|
||||
`smtpd_tls_security_level=encrypt`.
|
||||
- **25/smtp inet** — when `INBOUND_RELAY_ENABLE=true` and/or
|
||||
`DMARC_REPORTS_ENABLE=true`. No SASL, no OpenDKIM, no journal-milter —
|
||||
`smtpd_milters` is always overridden on this service (to the antispam milter
|
||||
or to nothing), because main.cf's submission chain would otherwise let a
|
||||
signing outage defer relayed mail and would file inbound mail into the
|
||||
outbound send log under the sender's domain.
|
||||
`smtpd_milters` is always overridden on this service, because main.cf's
|
||||
submission chain would otherwise let a signing outage defer relayed mail and
|
||||
would file inbound mail into the outbound send log under the sender's
|
||||
domain. With the inbound relay on the chain is the optional antispam milter,
|
||||
then the panel's **inbound journal milter** (unix socket
|
||||
`INBOUND_JOURNAL_MILTER_SOCKET`, default
|
||||
`/run/selfpost/inbound-journal.sock`, `default_action=accept`); with DMARC
|
||||
ingest alone it is empty. Postfix hands a message to its milters one after
|
||||
another at end-of-message, each seeing what the one before added, so the
|
||||
journal milter sees only mail the filter let through and reads the filter's
|
||||
verdict headers (`X-Rspamd-Action`, `X-Spamd-Result`) off it. It writes one
|
||||
`inbound_spam_log` row per recipient at an inbound domain - a DMARC report
|
||||
address is not one - and never refuses anything
|
||||
([internal/milter/inbound.go](../internal/milter/inbound.go)). The headers
|
||||
are read only while a filter is attached, and only next to the filter's
|
||||
signature: a third header, `X-SelfPost-Filter`, the HMAC-SHA256 of the
|
||||
message's queue id under the key in `/data/antispam/filter.key`
|
||||
(`inbound.FilterKey` makes it on first start; rspamd reads it through the
|
||||
mounted directory and signs in a custom routine of `milter_headers.conf`).
|
||||
With the filter down - it is attached fail-open - a sender's own verdict
|
||||
headers carry no signature and the row is that of mail nobody judged.
|
||||
Refused mail never reaches this milter.
|
||||
Refusals come from `mail.log`: this listener logs as `postfix/inbound`
|
||||
(`syslog_name`), and the log tailer turns a `reject:` line of
|
||||
`postfix/inbound/smtpd` (Postfix's own policy) and a `milter-reject` /
|
||||
`milter-hold` / `milter-discard` line of `cleanup` (the filter's verdict,
|
||||
counted only for a queue id that listener opened) into a row, dated by the
|
||||
line, when the recipient is at an inbound domain
|
||||
([internal/logtail/inbound.go](../internal/logtail/inbound.go)). A line
|
||||
names one recipient, so a message refused at end-of-message is one row
|
||||
whatever its recipient count. The tailer saves its place every five seconds,
|
||||
so after an unclean stop it reads some lines again; the lines it reads on
|
||||
start are written only if the same row is not there already. The
|
||||
journal is pruned every six hours: rows
|
||||
older than the `inbound_spam_log_retention_days` setting, then down to
|
||||
10 000.
|
||||
**Filter lists** (`inbound_spam_list`): allow and deny entries for the
|
||||
external filter - sender address, sender domain, client address or network -
|
||||
validated and normalised by `internal/inbound` (`lists.go`) and written as
|
||||
six map files under `/data/antispam/` (`ANTISPAM_DIR`), each atomically, on
|
||||
every change and on every start with the relay on. SelfPost never consults
|
||||
them. The sidecar mounts the directory and rspamd's multimap rules
|
||||
([deploy/antispam/rspamd/multimap.conf](../deploy/antispam/rspamd/multimap.conf))
|
||||
read them: a deny entry is a prefilter `reject`, an allow entry a symbol
|
||||
weighing -100. Allow is deliberately not a prefilter `accept`: with two
|
||||
prefilters rspamd lets the one that ran first decide, and deny then won or
|
||||
lost by rule order (seen on rspamd 3.11). A refusal by the deny list
|
||||
overrides any score, so deny wins whichever entries match.
|
||||
How often an entry decided a message is not stored: `Lists.Usage` reads it
|
||||
off the journal on each view of Filter lists - the rows a list decided (the
|
||||
deny rule's reply, the allow rule's symbol, which the journal milter keeps
|
||||
ahead of its six-symbol cap) matched against the entries as rspamd matches
|
||||
them. When that stops being good enough is written in
|
||||
[plans/inbound-antispam-panel.md](plans/inbound-antispam-panel.md) § Filter
|
||||
list usage.
|
||||
**Inbound relay** (`INBOUND_RELAY_ENABLE=true`): accepts only
|
||||
`relay_domains` + `relay_recipient_maps` (`reject_unauth_destination`,
|
||||
`reject_unlisted_recipient`). Maps under `/data/postfix/` (`relay_domains`,
|
||||
@@ -93,7 +145,9 @@ smtp/inet` removes Debian's stock smtpd). Outbound delivery still uses the
|
||||
`INBOUND_RATE_LIMIT_MESSAGES_PER_IP` and message size uses the greater of
|
||||
`INBOUND_MESSAGE_SIZE_LIMIT` and `DMARC_MESSAGE_SIZE_LIMIT`.
|
||||
- **No open relay** — `permit_sasl_authenticated`, `reject_unauth_destination`;
|
||||
`smtpd_sender_login_maps` + `reject_sender_login_mismatch`.
|
||||
`smtpd_sender_login_maps` + `reject_sender_login_mismatch`. That holds the
|
||||
envelope sender; the `From:` header is held to the same table by the
|
||||
panel's sender check (§ Panel roles).
|
||||
- **Level-1 rate limit** — `smtpd_client_message_rate_limit` /
|
||||
`anvil_rate_time_unit` from `RATE_LIMIT_*` env vars; independent of milter.
|
||||
- **Chroot disabled** for all services (DNS/TLS inside container).
|
||||
@@ -105,9 +159,23 @@ smtp/inet` removes Debian's stock smtpd). Outbound delivery still uses the
|
||||
Per-domain keys under `/data/opendkim/keys`; `KeyTable` / `SigningTable` maintained
|
||||
by the panel. Socket `/run/opendkim/opendkim.sock`.
|
||||
|
||||
The server's own name (`SELFPOST_HOSTNAME`) has a key and a line in both
|
||||
tables too, without being a row of the registry
|
||||
(`domain.Service.SetServerName`): the panel's notifications are sent as
|
||||
`postmaster@` that name and signed with it. No application can use it as an
|
||||
envelope sender - the sender map is built from the registry - but the
|
||||
`From:` header is not held against the login, so an application can still
|
||||
have a message signed as the server ([security.md](security.md) § Accepted
|
||||
risks). A sending domain of the same name shares its key and selector with
|
||||
the server; deleting that domain leaves the server signed under the default
|
||||
selector, which is the same key only if that was the domain's. At its start
|
||||
the notifier makes sure the tables have the line and name the selector the
|
||||
panel shows (`EnsureServerSigned`): tables are otherwise rebuilt only when a
|
||||
domain is added or deleted.
|
||||
|
||||
### Panel binary ([cmd/panel](../cmd/panel))
|
||||
|
||||
One process, four roles:
|
||||
One process, five roles (six with the inbound relay on):
|
||||
|
||||
1. **HTTP server** — `:8080` (`PANEL_HTTP_ADDR`); HTTPS terminated by reverse
|
||||
proxy only. On start it runs `postconf -h` once for the deferred-mail retry
|
||||
@@ -118,7 +186,9 @@ One process, four roles:
|
||||
numbers; they never call `postconf` per request. If `postconf` is missing,
|
||||
the panel logs a warning and uses Postfix 3.x compiled-in defaults
|
||||
(`300s` / `4000s` / `5d` / `0`) with a muted note on the box.
|
||||
2. **journal-milter** — unix socket `JOURNAL_MILTER_SOCKET`; records From/To/
|
||||
2. **sender check** — unix socket `SENDER_CHECK_SOCKET`, first in the chain
|
||||
and **strict** (`default_action=tempfail`): § Sender check below.
|
||||
3. **journal-milter** — unix socket `JOURNAL_MILTER_SOCKET`; records From/To/
|
||||
Subject/SASL user at DATA; enforces level-2 rate limits; **fail-open**
|
||||
(`default_action=accept`) so milter failure does not stop mail. Domain
|
||||
ceilings apply to every client IP; an application ceiling **overrides** the
|
||||
@@ -130,13 +200,157 @@ One process, four roles:
|
||||
this process has admitted but not yet written (`internal/milter/inflight.go`),
|
||||
so concurrent sessions cannot each spend the same last slot; a reservation
|
||||
is released at end-of-message, on ABORT, or after a 10-minute TTL.
|
||||
3. **log-tailer** — follows `MAIL_LOG`, updates send-log delivery status by
|
||||
4. **log-tailer** — follows `MAIL_LOG`, updates send-log delivery status by
|
||||
queue-id. Send-log `queued → sent` transitions depend on this goroutine alone
|
||||
(`UpdateStatus` is only called from [internal/logtail](../internal/logtail/logtail.go)).
|
||||
4. **rate-limit recalc** — every six hours (and on demand from Domain settings and
|
||||
It also keeps the lines Postfix writes under a queue id, for the message's
|
||||
page (§ Log tailer, *Stored lines*), and counts the mail Postfix deferred
|
||||
because the sender check did not answer.
|
||||
5. **rate-limit recalc** — every six hours (and on demand from Domain settings and
|
||||
the application form), recomputes level-2 **Auto** rate limits from send-log statistics.
|
||||
6. **notifier** — writes to the panel's users when something needs them
|
||||
(§ Notifications below).
|
||||
|
||||
Milter chain in Postfix: OpenDKIM (tempfail) then journal (accept on failure).
|
||||
Milter chain in Postfix: the sender check (tempfail), OpenDKIM (tempfail),
|
||||
then the journal (accept on failure). Mail submitted locally with
|
||||
`sendmail(1)` — the notifier's — passes OpenDKIM alone (`non_smtpd_milters`):
|
||||
it is signed, it is the panel's own and so needs no check of whose it is, and
|
||||
it is not an application's mail to be journalled or limited. Bounces Postfix
|
||||
generates pass no milter.
|
||||
|
||||
### Sender check
|
||||
|
||||
[internal/milter/sender.go](../internal/milter/sender.go) holds a message's
|
||||
`From:` header against the application that sends it
|
||||
([plans/from-header-binding.md](plans/from-header-binding.md)). Postfix holds
|
||||
the envelope sender against the SASL login, and OpenDKIM chooses the signing
|
||||
key by the header; until `2.4.0` nothing connected the two, and an
|
||||
application could have mail signed as another domain of the server or as the
|
||||
server itself.
|
||||
|
||||
- **The rule is the envelope's.** The check reads the file Postfix reads for
|
||||
`smtpd_sender_login_maps` and makes the same lookup: the address itself is
|
||||
bound to the login, or `@domain` is. So an application set to any address of
|
||||
its domain may write any address of that domain, and one with a list only
|
||||
the listed ones; everything else is refused - another registered domain,
|
||||
the server's name, a domain the server knows nothing of. One table for
|
||||
both senders and one lookup: the address, the address without its
|
||||
`+extension` (`recipient_delimiter`, which `build/postfix-config.sh` sets
|
||||
for that reason), then the domain. Postfix compares logins without regard
|
||||
to case and the check as written, which is how the panel writes them.
|
||||
- **How the header is read.** Exactly one `From:` field (none, or more than
|
||||
one, is refused), parsed as an address list that must hold exactly one
|
||||
address. Then a cruder second look at the raw value: OpenDKIM finds the
|
||||
address with a parser of its own, and wherever the two could read a line
|
||||
differently - a display name, a comment, a quoted string - the other
|
||||
address would have to be spelled out in it. So no `@name` may appear there
|
||||
that is another name the server signs for; those names are read from
|
||||
OpenDKIM's own `SigningTable`; the display name is looked at once more
|
||||
as decoded, since a mail program shows `=?utf-8?q?director=40example.org?=`
|
||||
as an address though no `@` is written in it. And, needing no knowledge of
|
||||
that parser:
|
||||
outside quoted strings the line has exactly one `@`. A folded line is
|
||||
unfolded first, and a display name in a charset Go does not convert is
|
||||
passed through unread - only the address is judged.
|
||||
- **The verdict** comes at the end of the headers: `550 5.7.1`, permanent,
|
||||
naming the line, as a reply code of the milter's own. What the client wrote
|
||||
is reduced to printable ASCII and cut before it is said back, logged or
|
||||
stored. A message with no SASL login is refused as well; the submission
|
||||
ports do not let one this far.
|
||||
- **Strict, and therefore small.** Postfix attaches the socket with
|
||||
`default_action=tempfail`: while the panel does not answer on it within the
|
||||
milter timeouts, mail is deferred, not accepted unchecked. The role
|
||||
therefore reads no database. Its two inputs are files the panel rewrites by
|
||||
rename and the mail path itself goes by; they are read again when their
|
||||
time or size has changed, and a file that cannot be read defers mail - a
|
||||
table from before would let a deleted application go on sending. The
|
||||
send-log row of a refusal (`status=rejected`, `reason=from`,
|
||||
`header_from`) is handed to another goroutine through a buffer and is not
|
||||
waited for. It is one row for the message, naming the first recipient and
|
||||
how many more there were: a row per recipient would let one refused message
|
||||
write a thousand.
|
||||
- **What it costs is counted.** The role marks itself alive every two
|
||||
seconds (the socket's path with `.alive`, under `/run`, so a new container
|
||||
starts without one). At start the time since the mark is the check's
|
||||
downtime - counted from Postfix's own start if that was later, and not at
|
||||
all when supervisord says Postfix is not running, since mail was then
|
||||
stopped for another reason. The log tailer counts the mail: Postfix
|
||||
writes a warning naming the socket and then, from the same smtpd process,
|
||||
`milter-reject: … 451 4.7.1 Service unavailable` from the connection on
|
||||
(the client is told `454 4.3.0` at `AUTH` and leaves), so what is counted
|
||||
is a connection turned away, once - a client brings a message when it
|
||||
connects. The reject line does not name the milter and the journal milter's
|
||||
level-2 refusal reads the same, hence the warning. Both go into
|
||||
`sender_check_days`, a
|
||||
row per UTC day kept a year, and Server › Health shows today, 30 days and
|
||||
12 months. The second figure also sees a panel that is running and not
|
||||
answering, which the mark does not. Downtime is cut at the UTC midnights
|
||||
it crossed. A log read again after an unclean stop does not count again:
|
||||
while the tailer catches up, a warning no later than the last one counted
|
||||
is passed over.
|
||||
- **Why it is not a process of its own.** Decided with the owner: measure
|
||||
first. The role has a socket of its own so that the journal stays
|
||||
fail-open, and moving it out later changes where it is started and nothing
|
||||
else.
|
||||
|
||||
### Notifications
|
||||
|
||||
[internal/notify](../internal/notify/notify.go) reports four things to the
|
||||
users of the panel: a sending domain's DNS record that stopped matching, a
|
||||
level-2 rate limit that refused mail, the TLS certificate about to expire, a
|
||||
message more than three days in the queue. The rules of when a message goes
|
||||
out are the owner's and are written in
|
||||
[plans/panel-notifications.md](plans/panel-notifications.md) § When a message
|
||||
is sent; the operator's view is guide § Notifications.
|
||||
|
||||
- **Readings.** Once an hour, starting three minutes after the process does:
|
||||
every sending domain's records through a DNS checker of the notifier's own,
|
||||
forced past the cache (`cmd/panel/notifier.go` builds the same query a
|
||||
domain's page does); the certificate file; `postqueue -j` for the arrival
|
||||
time of each queued message. Once a minute: the send log's `rejected` rows
|
||||
of the last day, through a partial index of their own.
|
||||
- **A refused row is not believed whole.** Its domain is the one in
|
||||
`MAIL FROM`, which the client typed and Postfix has not yet held against
|
||||
the login when the milter refuses; the same status is written when an
|
||||
application is refused for its client address or its `From:` line. Since
|
||||
`2.4.0` a row says which (`send_log.reason`) and only a limit's rows are
|
||||
taken; a row from before has no reason. So refusals are counted by
|
||||
the domain the login is registered under, only for logins that exist, and
|
||||
only while a level-2 limit is set on the login or its domain
|
||||
(`Store.LimitRefusalsSince`). One subject per sending domain, whichever of
|
||||
its applications are refused.
|
||||
- **State** is a row per subject in `notify_state` (`dns:<domain>`, `cert`,
|
||||
`queue`, `limit:<domain>`), which is what makes each thing be said
|
||||
once and a restart say nothing again. A DNS subject goes *pending* on the
|
||||
first bad check and *open*, with a message, on a later one at least 45
|
||||
minutes after it - a count alone would let the last check before a restart
|
||||
and the first after it stand for "an hour apart"; back the same way. The
|
||||
certificate's row remembers which threshold was said for which expiry. A
|
||||
deleted domain's row goes with it.
|
||||
- **Said means sent.** A problem that arises while no user can be told, or
|
||||
while Postfix does not take the message, stays unsaid and is told at a
|
||||
later check. Its ending, with nobody left to tell, is simply over. If one
|
||||
recipient's copy fails and another's goes out, the subject is said.
|
||||
- **The notifier's own mail is not a subject.** A notification waiting for a
|
||||
mailbox that does not answer, or its bounce coming back to
|
||||
`postmaster@<hostname>` - which receives no mail - is left out of the
|
||||
queue check.
|
||||
- **A ceiling** of 60 messages an hour stands over all of it.
|
||||
- **No information is not a finding.** A lookup that did not answer, a queue
|
||||
that cannot be listed, a certificate that cannot be read move no state.
|
||||
- **Recipients** come from the store at the moment of sending: global users
|
||||
for everything, domain users for subjects that are a domain's; only users
|
||||
with an e-mail and their own switch on. One message per recipient, so users
|
||||
do not see each other's addresses.
|
||||
- **The instance switch** (settings key `notifications`, on unless turned
|
||||
off) stops the checks themselves, not only the mail: no state moves while
|
||||
it is off.
|
||||
- **Sending** is `sendmail -i -f postmaster@<hostname> -- <address>` as the
|
||||
unprivileged panel user, which is a member of `postdrop` and so can write
|
||||
the maildrop directory without the setgid bit `no-new-privileges` forbids.
|
||||
A message that could not be handed over leaves the state where it was and
|
||||
is tried at the next check. The text names the page to open and has no
|
||||
link: the panel does not know its own address.
|
||||
|
||||
### Log tailer and `mail.log` rotation
|
||||
|
||||
@@ -183,15 +397,66 @@ leaves every row untouched rather than closing them all. Now that the log
|
||||
survives the container, reaching this path means the lines are gone for good —
|
||||
rotated past fourteen files while the panel was down, or deleted.
|
||||
|
||||
**Stored lines** (`send_log_lines`, since 2.2.0;
|
||||
[internal/logtail/lines.go](../internal/logtail/lines.go)). A send-log row
|
||||
lives for the retention window, ninety days by default, and `mail.log` for a
|
||||
day before it is rotated; a message's page (`/outbound/log/{id}`) shows what
|
||||
Postfix wrote about it, so the tailer keeps those lines in the database as it
|
||||
reads them:
|
||||
|
||||
- **What is kept:** a line headed by a queue id — after the timestamp, the
|
||||
host name and `program[pid]:` — with that id, the line's own time and the
|
||||
line as written, cut to 4096 bytes. The id is read from the head of the line
|
||||
only: the tail is the far end's text. `NOQUEUE` and `warning` stand in the
|
||||
same place and are not ids; the pattern accepts Postfix's short form
|
||||
(capital hexadecimal) and its long form (no vowels).
|
||||
- **Port-25 mail is left out.** A queue id opened by `postfix/inbound/smtpd`
|
||||
is a stranger's message and never gets a send-log row; none of its lines
|
||||
are stored, up to qmgr's `removed` or until another listener opens a
|
||||
message under the id (4096 such ids are remembered). What a stranger can
|
||||
make Postfix log is therefore not written to the database at his rate.
|
||||
- **No check for a row at write time.** A message's first lines (`client=`,
|
||||
`message-id=`) are logged before the journal milter writes its row at
|
||||
end-of-message, so the tailer cannot ask whether the id is in the send log.
|
||||
Lines that never get a row — bounces Postfix generates, port-25 mail met in
|
||||
mid-flight after a restart — are kept for a while.
|
||||
- **Written per pass, not per line:** the lines of one pass over the log go
|
||||
in one transaction (`InsertSendLogLines`), 500 at most, before the read
|
||||
offset is saved. A commit per line would be a disk sync per line on the
|
||||
connection the journal milter shares.
|
||||
- **Retention is one rule:** a line older than an hour whose queue id no
|
||||
send-log row carries is deleted (`PruneSendLogLines`), at start and after
|
||||
every six-hourly send-log sweep. That removes the lines of rows the sweep
|
||||
deleted and the lines that never had a row. The send log is asked once per
|
||||
queue id; the statement still walks the table, about a second per million
|
||||
lines, with the connection taken meanwhile.
|
||||
- **Which lines are a row's.** Postfix reuses a queue id once the message has
|
||||
left the queue, and a row outlives that by months. `SendLogLines` gives a row
|
||||
the lines of its queue id dated from one hour before the row was written to
|
||||
the queue lifetime (the larger of `maximal_queue_lifetime` and
|
||||
`bounce_queue_lifetime`) plus a day after it. That limits a chance reuse of
|
||||
the id to those days; it does not exclude it. A message with several
|
||||
recipients is several rows under one id, and each row's page shows all of
|
||||
the message's lines.
|
||||
- **Read twice, stored once.** The read offset is saved every few seconds, so
|
||||
after an unclean stop some lines are read again; while the tailer is
|
||||
catching up at start it writes a line only if the same line is not there.
|
||||
- **Not stored:** what the tailer never read — the tail of a file rotated away
|
||||
while the panel was down.
|
||||
- **Size:** six to ten lines of about 200 bytes per message — more than the row
|
||||
itself. The table is in the database and therefore in a full backup, which
|
||||
`log/` is not ([security.md](security.md) § Backup and domain export).
|
||||
|
||||
**Two one-shot reads** sit beside the follow loop and are unrelated to it, both
|
||||
serving panel pages on request: `TailLines` (the last *n* lines, for
|
||||
`/server/log`) and `QueueLines` (the lines carrying one queue-id, for
|
||||
`/outbound/log/{id}`). `QueueLines` scans a bounded tail of the current file —
|
||||
`/server/log`) and `QueueLines` (the lines carrying one queue-id). `QueueLines`
|
||||
is what a message's page falls back to when no lines are stored for its row —
|
||||
a message accepted before 2.2.0. It scans a bounded tail of the current file —
|
||||
finding a message's lines means reading rather than seeking — and matches the id
|
||||
anchored on the character before it, since queue ids are hexadecimal runs and a
|
||||
shorter one is regularly the tail of a longer one. Send-log rows outlive the log
|
||||
(retention 90 days, rotation 14 files), so an empty result is the expected end
|
||||
state for an older message and the page reports it as such, not as a failure.
|
||||
shorter one is regularly the tail of a longer one. For such a message an empty
|
||||
result is the expected end state once the log has rotated, and the page reports
|
||||
it as such, not as a failure.
|
||||
|
||||
---
|
||||
|
||||
@@ -531,16 +796,17 @@ full backups also archive the operator's `./certs` PEM files when present.
|
||||
|
||||
### Database
|
||||
|
||||
The schema is one embedded baseline, `internal/store/migrations/0001_init.sql`,
|
||||
applied by `store.Open` into an empty file; `PRAGMA user_version` is `1` and
|
||||
The schema is an embedded baseline, `internal/store/migrations/0001_init.sql`,
|
||||
and the migrations after it (`0002_inbound_spam_log.sql`), applied by
|
||||
`store.Open` in file order; `PRAGMA user_version` counts the files applied and
|
||||
`PRAGMA application_id` is `0x53503230` ("SP20"). There is no 1.x migration
|
||||
chain. **2.0 starts from an empty data directory:** a database with a schema
|
||||
version but without the 2.x `application_id` — what every 1.x file looks like —
|
||||
is refused at start with `ErrForeignSchema` and left untouched, as is one whose
|
||||
version is newer than the build knows. A restored 1.x backup fails the same
|
||||
way. Domains are carried across with a domain export, whose format is not the
|
||||
database. Later schema changes are new files (`0002_*.sql`, …); once `2.0.0` is
|
||||
released the baseline is never edited. History and the per-table list:
|
||||
database. A schema change is a new file; the baseline has not been edited
|
||||
since `2.0.0`. History and the per-table list:
|
||||
[schema-migrations.md](schema-migrations.md).
|
||||
|
||||
The tables that carry the panel's model:
|
||||
@@ -565,6 +831,7 @@ The tables that carry the panel's model:
|
||||
(send-log retention). Nothing that belongs to a user.
|
||||
- `applications` (with `auth_ip_restrict` / `auth_allowed_ips`),
|
||||
`application_addresses`, `rate_limits` (no `allowed_ips` column), `send_log`,
|
||||
`send_log_lines` (what Postfix logged under a queue id — § Log tailer),
|
||||
`logtail_state`, `inbound_domains`, `inbound_transports`,
|
||||
`inbound_recipients`, `dmarc_reports`, `dmarc_report_records`.
|
||||
|
||||
@@ -572,7 +839,8 @@ The panel is the only writer; SQLite runs in WAL mode on one connection (see
|
||||
[Panel binary](#panel-binary-cmdpanel)), which is why a full backup snapshots
|
||||
with `VACUUM INTO` instead of copying the file.
|
||||
|
||||
**Rotation:** send-log retention `SEND_LOG_RETENTION_DAYS` (default 90);
|
||||
**Rotation:** send-log retention `SEND_LOG_RETENTION_DAYS` (default 90), which
|
||||
the stored log lines follow;
|
||||
`mail.log` via logrotate (14 rotated files, check every 6h, rename +
|
||||
`postfix reload` in `postrotate` — see § Log tailer above).
|
||||
|
||||
@@ -625,7 +893,8 @@ unsupported rather than as a missing doc:
|
||||
- **Panel paths and tuning:** `SELFPOST_DATA_DIR` (`/data`), `SELFPOST_DB_PATH`
|
||||
(`/data/selfpost.db`), `SELFPOST_SETUP_TOKEN_FILE`
|
||||
(`/data/setup-token`), `PANEL_HTTP_ADDR` (`:8080`),
|
||||
`JOURNAL_MILTER_SOCKET` (`/run/selfpost/journal.sock`), `MAIL_LOG`
|
||||
`JOURNAL_MILTER_SOCKET` (`/run/selfpost/journal.sock`),
|
||||
`SENDER_CHECK_SOCKET` (`/run/selfpost/sender.sock`), `MAIL_LOG`
|
||||
(`/data/log/mail.log` — read by the panel and written by Postfix, so a change
|
||||
here has to be matched in `build/postfix-config.sh`),
|
||||
`PANEL_COOKIE_SECURE` (`true`), `OPENDKIM_SOCKET`
|
||||
|
||||
@@ -82,6 +82,8 @@
|
||||
<div class="field"><label class="label" for="username">Username</label><div class="control has-icons-left"><input class="input" id="username" name="username" value="admin" autocomplete="username" spellcheck="false" required><span class="icon is-left"><i class="ti ti-user"></i></span></div></div>
|
||||
<div class="field"><label class="label" for="email">E-mail</label><div class="control has-icons-left"><input class="input" id="email" name="email" type="email" value="mix@example.org" autocomplete="email"><span class="icon is-left"><i class="ti ti-mail"></i></span></div>
|
||||
<p class="help">Your own address. A domain's report address can be filled with it from Domain settings.</p></div>
|
||||
<div class="field"><label class="checkbox"><input type="checkbox" name="notify" checked> Send me notifications</label>
|
||||
<p class="help">To this address, when something needs you: a DNS record of a domain stops matching, a rate limit refuses mail, the TLS certificate is about to expire, a message has waited in the queue for three days. <a href="help.html#notifications">What is sent and when</a></p></div>
|
||||
<button type="submit" class="button is-primary">Save profile</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
@@ -78,7 +78,7 @@
|
||||
<div class="box">
|
||||
<div class="sp-box-head"><span class="sp-no">01</span><h2>Full backup</h2><a class="sp-help" href="help.html#backup" title="Backups, encryption and restore"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-body">
|
||||
<p class="mb-3"><code>data/</code> (database, DKIM keys, credentials, the queue), <code>docker-compose.yml</code>, <code>.env</code> and <code>certs/</code> in one archive. Restore into an empty project directory and start the <strong>same SelfPost version</strong>. The reverse-proxy vhost is not included.</p>
|
||||
<p class="mb-3"><code>data/</code> (database, DKIM keys, credentials), <code>docker-compose.yml</code>, <code>.env</code> and <code>certs/</code> in one archive. The Postfix queue is left out: it cannot be copied reliably while the server is running. Restore into an empty project directory and start the <strong>same SelfPost version</strong>. The reverse-proxy vhost is not included.</p>
|
||||
<form class="sp-form" method="post" action="#">
|
||||
<div class="field"><label class="checkbox"><input type="checkbox" name="encrypt" checked> Encrypt with a password (<code>.spbk</code>)</label></div>
|
||||
<div class="sp-pair">
|
||||
|
||||
@@ -37,7 +37,7 @@ SIGNED_OUT = {"sp-signin", "sp-signin-brand", "sp-signin-form", "sp-foot"}
|
||||
# Selectors allowed to set a width. Everything else gets its width from the
|
||||
# layout it sits in — that is the whole point of the redesign.
|
||||
WIDTH_OK = (".navbar-brand img", ".sp-head .sp-lead", ".sp-head .sp-grow", ".sp-postmark", ".table td.sp-clip",
|
||||
".sp-h-icon", ".sp-signin", ".sp-timeline li::before", ".sp-perf")
|
||||
".sp-h-icon", ".sp-signin", ".sp-timeline li::before", ".sp-perf", ".sp-sheet")
|
||||
|
||||
|
||||
class Node:
|
||||
|
||||
@@ -84,13 +84,14 @@
|
||||
<dl class="sp-facts">
|
||||
<div><dt>list</dt><dd>page_head, then full-width boxes: the add form first, the table second. <a href="out-domains.html">Example</a></dd></div>
|
||||
<div><dt>detail</dt><dd>page_head, then <code>columns</code>: <code>is-3</code> side menu + <code>is-9</code> boxes. <a href="out-domain.html">Example</a></dd></div>
|
||||
<div><dt>cards</dt><dd>page_head, then <code>columns sp-cards</code>: two <code>is-6</code> stacks of numbered boxes read in rows, odd on the left and even on the right, each stack closed up. On a phone the boxes come in the order of their numbers. The two stacks under this box are laid out so. <a href="health.html">Example</a></dd></div>
|
||||
<div><dt>form</dt><dd>page_head, then boxes in equal <code>columns</code>, one submit row under them. <a href="out-app.html">Example</a></dd></div>
|
||||
<div><dt>signed out</dt><dd>The split brand / form screen, no navbar. <a href="login.html">Example</a></dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="columns">
|
||||
<div class="columns sp-cards">
|
||||
<div class="column is-6">
|
||||
<div class="box">
|
||||
<div class="sp-box-head"><span class="sp-no">01</span><h2>postmark</h2><div class="sp-end">Worst status of the page's checks</div></div>
|
||||
@@ -114,6 +115,7 @@
|
||||
<tr><td>error, fail, mismatch, bounced</td><td><span class="tag is-danger is-light">fail</span></td><td class="sp-mono sp-small">tag is-danger is-light</td></tr>
|
||||
<tr><td>unknown, off, neutral counts</td><td><span class="tag is-light">off</span></td><td class="sp-mono sp-small">tag is-light</td></tr>
|
||||
<tr><td>held: quarantine · role: global</td><td><span class="tag is-primary is-light">quarantine</span></td><td class="sp-mono sp-small">tag is-primary is-light</td></tr>
|
||||
<tr><td>a tag that opens a <code>sheet</code></td><td><button type="button" class="tag is-danger is-light" popovertarget="kit-sheet">reject</button><div class="sp-sheet" id="kit-sheet" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-danger is-light">reject</span><h2>sheet</h2><button type="button" popovertarget="kit-sheet" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>What it holds</th><td>What does not fit the row, as name and value</td></tr><tr><th>How it closes</th><td class="sp-mono sp-small">Esc<br>a click outside<br>the cross</td></tr></tbody></table></div></div></div></div></td><td class="sp-mono sp-small">button.tag + sp-sheet (popover)</td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
|
||||
@@ -80,7 +80,7 @@
|
||||
<div><dt>Last report</dt><dd>2026-09-21 06:12 UTC</dd></div>
|
||||
<div><dt>This week</dt><dd>14 kept · 0 parse failures</dd></div>
|
||||
<div><dt>Hosted addresses</dt><dd class="sp-mono sp-small">dmarc-reports+<domain>@mail.example.org</dd></div>
|
||||
<div><dt>Retention</dt><dd>500 reports or 90 days</dd></div>
|
||||
<div><dt>Retention</dt><dd>500 reports a domain or 90 days</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -74,23 +74,10 @@
|
||||
<form method="post" action="#"><button type="submit" class="button"><i class="ti ti-refresh"></i>Re-check DNS</button></form>
|
||||
</div>
|
||||
|
||||
<div class="columns">
|
||||
<div class="column is-6">
|
||||
<div class="box" id="machine">
|
||||
<div class="sp-box-head"><span class="sp-no">01</span><h2>Machine</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div><a class="sp-help" href="help.html#checks" title="What these readings mean"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="table-container"><table class="table is-fullwidth">
|
||||
<thead><tr><th>Resource</th><th>Usage</th><th>Detail</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>CPU</td><td><progress class="progress is-small is-success mb-1" value="12" max="100"></progress>12 %</td><td class="sp-muted">2 cores, load 0.21</td></tr>
|
||||
<tr><td>Memory</td><td><progress class="progress is-small is-success mb-1" value="41" max="100"></progress>41 %</td><td class="sp-muted">812 MB of 1.9 GB</td></tr>
|
||||
<tr><td>Network</td><td class="sp-nowrap">↓ 18 kB/s · ↑ 44 kB/s</td><td class="sp-muted">eth0: 2.1 GB in, 6.4 GB out</td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="columns sp-cards">
|
||||
<div class="column is-6">
|
||||
<div class="box" id="processes">
|
||||
<div class="sp-box-head"><span class="sp-no">02</span><h2>Processes</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div></div>
|
||||
<div class="sp-box-head"><span class="sp-no">01</span><h2>Processes</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div></div>
|
||||
<div class="table-container"><table class="table is-fullwidth">
|
||||
<thead><tr><th>Program</th><th>State</th><th>Detail</th></tr></thead>
|
||||
<tbody>
|
||||
@@ -102,37 +89,20 @@
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="columns">
|
||||
<div class="column is-4">
|
||||
<div class="box" id="certificate">
|
||||
<div class="sp-box-head"><span class="sp-no">03</span><h2>TLS certificate</h2><div class="sp-end"><span class="tag is-warning is-light">warn</span></div><a class="sp-help" href="help.html#certificate" title="When it warns and what to do"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-body">
|
||||
<dl class="sp-facts">
|
||||
<div><dt>Expires</dt><dd class="sp-mono">2026-10-03 09:14 UTC</dd></div>
|
||||
<div><dt>Names</dt><dd class="sp-mono">mail.example.org</dd></div>
|
||||
</dl>
|
||||
<p class="mt-4 has-text-danger">Expires in 12 days — renewal has not run.</p>
|
||||
<p class="sp-muted">Renewal normally runs 30 days before expiry. Check that port 80 is reachable and look for <code>acme</code> lines in the <a href="system-log.html">system log</a>.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="column is-4">
|
||||
<div class="box" id="sockets">
|
||||
<div class="sp-box-head"><span class="sp-no">04</span><h2>Milter sockets</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div></div>
|
||||
<div class="box" id="sender-check">
|
||||
<div class="sp-box-head"><span class="sp-no">03</span><h2>Sender check</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div><a class="sp-help" href="help.html#sender-check" title="What is checked and what these numbers are for"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="table-container"><table class="table is-fullwidth">
|
||||
<thead><tr><th>Milter</th><th>State</th></tr></thead>
|
||||
<thead><tr><th>Period</th><th>Panel not answering</th><th>Messages deferred</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>opendkim<br><span class="sp-muted sp-mono sp-small">inet:127.0.0.1:8891</span></td><td><span class="tag is-success is-light">ok</span></td></tr>
|
||||
<tr><td>journal-milter<br><span class="sp-muted sp-mono sp-small">unix:/run/selfpost/journal.sock</span></td><td><span class="tag is-success is-light">ok</span></td></tr>
|
||||
<tr><td>rspamd <span class="sp-muted">· inbound spam filter</span><br><span class="sp-muted sp-mono sp-small">inet:antispam:11332</span></td><td><span class="tag is-success is-light">ok</span></td></tr>
|
||||
<tr><td>Today</td><td>0 s</td><td>0</td></tr>
|
||||
<tr><td>Last 30 days</td><td>4 s</td><td>0</td></tr>
|
||||
<tr><td>Last 12 months</td><td>41 s</td><td>3</td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
<div class="sp-box-body">
|
||||
<p class="sp-muted">The panel holds each message's <code>From:</code> against the application that sends it. While the panel does not answer, Postfix defers mail instead of accepting it unchecked. Time is counted only while Postfix itself was running. Counted since <span class="sp-mono">2026-10-13</span>.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="column is-4">
|
||||
<div class="box" id="hostname">
|
||||
<div class="sp-box-head"><span class="sp-no">05</span><h2>Hostname and reverse DNS</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div><a class="sp-help" href="help.html#rdns" title="Forward and reverse DNS"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-body">
|
||||
@@ -142,14 +112,62 @@
|
||||
</dl>
|
||||
</div>
|
||||
</div>
|
||||
<div class="box" id="machine">
|
||||
<div class="sp-box-head"><span class="sp-no">07</span><h2>Machine</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div><a class="sp-help" href="help.html#checks" title="What these readings mean"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="table-container"><table class="table is-fullwidth">
|
||||
<thead><tr><th>Resource</th><th>Usage</th><th>Detail</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>CPU</td><td><progress class="progress is-small is-success mb-1" value="12" max="100"></progress>12 %</td><td class="sp-muted">2 cores, load 0.21</td></tr>
|
||||
<tr><td>Memory</td><td><progress class="progress is-small is-success mb-1" value="41" max="100"></progress>41 %</td><td class="sp-muted">812 MB of 1.9 GB</td></tr>
|
||||
<tr><td>Network</td><td class="sp-nowrap">↓ 18 kB/s · ↑ 44 kB/s</td><td class="sp-muted">eth0: 2.1 GB in, 6.4 GB out</td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="box" id="configuration">
|
||||
<div class="sp-box-head"><span class="sp-no">06</span><h2>Configuration</h2><div class="sp-end">Safe to run at any time</div></div>
|
||||
<div class="sp-box-body">
|
||||
<p class="mb-3">Regenerates the OpenDKIM and Postfix configuration from the database and reloads both daemons. Use it after editing the files by hand, after restoring a backup, or when the running configuration looks out of step with the domain and application lists. It does not touch the queue or the TLS certificate.</p>
|
||||
<form method="post" action="#"><button type="submit" class="button"><i class="ti ti-reload"></i>Reload configuration</button></form>
|
||||
<div class="column is-6">
|
||||
<div class="box" id="sockets">
|
||||
<div class="sp-box-head"><span class="sp-no">02</span><h2>Milter sockets</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div></div>
|
||||
<div class="table-container"><table class="table is-fullwidth">
|
||||
<thead><tr><th>Milter</th><th>State</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>opendkim<br><span class="sp-muted sp-mono sp-small">inet:127.0.0.1:8891</span></td><td><span class="tag is-success is-light">ok</span></td></tr>
|
||||
<tr><td>journal-milter<br><span class="sp-muted sp-mono sp-small">unix:/run/selfpost/journal.sock</span></td><td><span class="tag is-success is-light">ok</span></td></tr>
|
||||
<tr><td>sender-check<br><span class="sp-muted sp-mono sp-small">unix:/run/selfpost/sender.sock</span></td><td><span class="tag is-success is-light">ok</span></td></tr>
|
||||
<tr><td>rspamd <span class="sp-muted">· inbound spam filter</span><br><span class="sp-muted sp-mono sp-small">inet:antispam:11332</span></td><td><span class="tag is-success is-light">ok</span></td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
<div class="box" id="certificate">
|
||||
<div class="sp-box-head"><span class="sp-no">04</span><h2>TLS certificate</h2><div class="sp-end"><span class="tag is-warning is-light">warn</span></div><a class="sp-help" href="help.html#certificate" title="When it warns and what to do"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-body">
|
||||
<dl class="sp-facts">
|
||||
<div><dt>Expires</dt><dd class="sp-mono">2026-10-03 09:14 UTC</dd></div>
|
||||
<div><dt>Names</dt><dd class="sp-mono">mail.example.org</dd></div>
|
||||
</dl>
|
||||
<p class="mt-4 has-text-danger">Expires in 12 days — renewal has not run.</p>
|
||||
<p class="sp-muted">Renewal normally runs 30 days before expiry. Check that port 80 is reachable and look for <code>acme</code> lines in the <a href="system-log.html">system log</a>.</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="box" id="notification-mail">
|
||||
<div class="sp-box-head"><span class="sp-no">06</span><h2>Notification mail</h2><div class="sp-end"><span class="tag is-warning is-light">warn</span></div><a class="sp-help" href="help.html#notifications" title="What the panel sends and what makes it arrive"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="table-container"><table class="table is-fullwidth">
|
||||
<thead><tr><th>Record</th><th>State</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>DKIM<br><span class="sp-muted sp-mono sp-small">sp2026._domainkey.mail.example.org</span></td><td><span class="tag is-success is-light">published</span></td></tr>
|
||||
<tr><td>SPF<br><span class="sp-muted sp-mono sp-small">mail.example.org</span></td><td><span class="tag is-warning is-light">missing</span></td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
<div class="sp-box-body">
|
||||
<p class="sp-muted">Sent from <span class="sp-mono">postmaster@mail.example.org</span>. What to publish is under <a href="settings.html#notification-mail">Settings</a>.</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="box" id="configuration">
|
||||
<div class="sp-box-head"><span class="sp-no">08</span><h2>Configuration</h2><div class="sp-end">Safe to run at any time</div></div>
|
||||
<div class="sp-box-body">
|
||||
<p class="mb-3">Regenerates the OpenDKIM and Postfix configuration from the database and reloads both daemons. Use it after editing the files by hand, after restoring a backup, or when the running configuration looks out of step with the domain and application lists. It does not touch the queue or the TLS certificate.</p>
|
||||
<form method="post" action="#"><button type="submit" class="button"><i class="ti ti-reload"></i>Reload configuration</button></form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -86,6 +86,7 @@
|
||||
<ul class="menu-list">
|
||||
<li><a href="#dns"><i class="ti ti-world-check"></i>DNS records</a></li>
|
||||
<li><a href="#apps"><i class="ti ti-apps"></i>Applications</a></li>
|
||||
<li><a href="#sender-check"><i class="ti ti-shield-check"></i>Sender check</a></li>
|
||||
<li><a href="#limits"><i class="ti ti-gauge"></i>Rate limits</a></li>
|
||||
<li><a href="#log"><i class="ti ti-list-details"></i>Outbound log</a></li>
|
||||
<li><a href="#queue"><i class="ti ti-stack-2"></i>Queue and retries</a></li>
|
||||
@@ -97,12 +98,15 @@
|
||||
<li><a href="#upstream"><i class="ti ti-arrow-forward-up"></i>Upstream</a></li>
|
||||
<li><a href="#recipients"><i class="ti ti-address-book"></i>Valid recipients</a></li>
|
||||
<li><a href="#filter"><i class="ti ti-shield-check"></i>Spam filter</a></li>
|
||||
<li><a href="#inbound-log"><i class="ti ti-list-details"></i>Inbound log</a></li>
|
||||
<li><a href="#filter-lists"><i class="ti ti-filter"></i>Filter lists</a></li>
|
||||
</ul>
|
||||
<p class="menu-label">Server</p>
|
||||
<ul class="menu-list">
|
||||
<li><a href="#backup"><i class="ti ti-archive"></i>Backup and moving a domain</a></li>
|
||||
<li><a href="#users"><i class="ti ti-users"></i>Users and roles</a></li>
|
||||
<li><a href="#settings"><i class="ti ti-settings"></i>Settings</a></li>
|
||||
<li><a href="#notifications"><i class="ti ti-mail"></i>Notifications</a></li>
|
||||
</ul>
|
||||
<p class="menu-label">Account</p>
|
||||
<ul class="menu-list">
|
||||
@@ -121,8 +125,9 @@
|
||||
<div class="sp-box-head"><span class="sp-no">02</span><h2>Outbound</h2></div>
|
||||
<div class="sp-help-topic" id="dns"><h3>DNS records</h3><p>DKIM proves the message was signed by this relay, SPF lists the addresses allowed to send for the domain, DMARC tells receivers what to do when both fail and where to report. Publish all three exactly as shown; merge SPF into an existing record rather than adding a second one. A domain's status is the worst of its three records, and results are cached for a few minutes, so use Re-check DNS after publishing. The SPF check is shallow: it looks for this server's address in the record itself and does not follow <code>include:</code> or <code>redirect=</code>, so a record that relies on them is reported as a warning, not a failure. <code>p=none</code> does not affect delivery; tighten to <code>quarantine</code>, then <code>reject</code>, once the reports look clean. A domain's report address is set in its Domain settings.</p></div>
|
||||
<div class="sp-help-topic" id="apps"><h3>Applications</h3><p>An application is one SASL login, unique across all domains. It may send as any address of its domain or only as listed addresses, and, with a list of client IPs set, only from those addresses. The password is shown once at creation or regeneration and is not stored in readable form.</p></div>
|
||||
<div class="sp-help-topic" id="sender-check"><h3>Sender check</h3><p>An application sends only as itself, and that holds for both senders a message has. The envelope sender is held against the login by Postfix. The <code>From:</code> line, the one a mail program shows and the one the DKIM signature is chosen by, is held against it by the panel, under the same rule: an application set to any address of its domain may write any address of that domain, and one with a list of addresses only those. A message with any other <code>From:</code> - another of your domains, the server's own name, an address somewhere else, or none at all - is refused for good with <code>550 5.7.1</code>. It is a rejected row of the Outbound log, and its page says which address was refused. This is what keeps one application from getting mail signed with another domain's key.<br><br>The check is strict. While the panel does not answer, Postfix does not accept mail unchecked: it replies with a temporary error and the application sends again later. Server › Health counts what that costs: for how long the panel did not answer while Postfix itself was running, and how many messages were deferred because of it. The card warns when a message was deferred in the last 24 hours. If the numbers stay at nothing, the check is where it should be; if they do not, it belongs in a process of its own.</p></div>
|
||||
<div class="sp-help-topic" id="limits"><h3>Rate limits</h3><p>Level 1 is a per-IP ceiling set in <code>.env</code>. Level 2 is optional, per domain and per application, entered by hand or computed from the last 30 days of sending, and can never exceed level 1. An application's own limit replaces its domain's.</p></div>
|
||||
<div class="sp-help-topic" id="log"><h3>Outbound log</h3><p>One row per message and recipient, with the status Postfix last reported: queued (accepted, nothing reported yet), sent (the receiving server took it; what the mailbox does next is not reported back), deferred (not taken yet, Postfix tries again), bounced (failed for good, or the queue lifetime ran out) and rejected (refused before queueing under a level-2 rate limit, so it never reached Postfix). A message's page lists these steps and the lines Postfix wrote about it in <code>mail.log</code>, found by queue id; that file is rotated daily and about two weeks are kept, so an old message may show none. Rows are deleted after the retention set in Server › Settings, 90 days by default. A domain administrator sees only the rows of their own domains.</p></div>
|
||||
<div class="sp-help-topic" id="log"><h3>Outbound log</h3><p>One row per message and recipient, with the status Postfix last reported: queued (accepted, nothing reported yet), sent (the receiving server took it; what the mailbox does next is not reported back), deferred (not taken yet, Postfix tries again), bounced (failed for good, or the queue lifetime ran out) and rejected (refused before queueing, so it never reached the queue: under a level-2 rate limit, from a client address the application does not allow, or for a <code>From:</code> line that is not the application's - the message's page says which). A message's page lists these steps and the lines Postfix wrote about it in <code>mail.log</code>, found by queue id. The lines are kept with the row: they stay on the page after that file has rotated, for as long as the row does. A message accepted before version 2.2.0 shows them only while they are still in the current file. Rows are deleted after the retention set in Server › Settings, 90 days by default, and their lines with them. A domain administrator sees only the rows of their own domains.</p></div>
|
||||
<div class="sp-help-topic" id="queue"><h3>Queue and retries</h3><p>The page lists, as <code>postqueue -p</code> prints it, the messages Postfix has accepted and not yet delivered: mail being sent right now and mail deferred for a later attempt. Only the global role can open it, and it only shows; nothing here deletes or forces a message. A deferred message is tried again after the first retry delay, then at gaps that double up to a cap, until the receiving server takes it or the queue lifetime runs out and it bounces; there is no attempt counter, and the box on the page shows this server's values. A queue that keeps growing means a receiver is refusing or not answering: open the message from the Outbound log to read the reason Postfix logged.</p></div>
|
||||
<div class="sp-help-topic" id="dmarc"><h3>DMARC reports</h3><p>An aggregate report is the summary a receiving provider sends, usually daily, to the <code>rua=</code> address of your DMARC record: which servers sent mail as your domain and whether SPF or DKIM passed for it. A domain's report address is one of three choices, set in its Domain settings: no reports (the default for a new domain), the SelfPost hosted address (an address on this server whose reports are parsed and shown under DMARC reports; offered only while hosted reports are enabled on the server), or an address typed for the domain (an ordinary mailbox that receives the raw XML; a button fills in your own profile e-mail). When the typed address is on another domain than the one sending, that domain must publish a report-authorization record, or providers will not send; the domain's page shows it with the other DNS records and checks it. In the tables pass means SPF or DKIM passed and aligned, fail means neither did, and your own server shows as <em>this relay</em>. A fail from any other source is someone else sending as your domain — a forwarder, an application you forgot, or abuse — so do not tighten <code>p=</code> until that sender is fixed or removed. A domain administrator reads the reports of their own domains.</p></div>
|
||||
</div>
|
||||
@@ -132,16 +137,19 @@
|
||||
<div class="sp-help-topic" id="upstream"><h3>Upstream</h3><p>The upstream is the server that actually holds the mailboxes. Give its host and port — used exactly as written, with no MX lookup — and choose the TLS for the hop to it: Opportunistic uses TLS when the upstream offers it and otherwise sends in the clear, Required holds the mail until a TLS session is possible, Off never uses TLS. While the upstream does not answer, Postfix keeps the mail and retries as for any other mail.</p></div>
|
||||
<div class="sp-help-topic" id="recipients"><h3>Valid recipients</h3><p>Choose whom this domain accepts. Listed addresses only: the relay accepts exactly the addresses you list, one per line or comma-separated, and refuses any other while the sender is still connected, so it never has to send a bounce for them. Any recipient: everything at the domain is accepted and passed on, and an address the upstream then rejects comes back as a bounce from this server to a sender who is often forged — backscatter. Prefer a list, and keep it in step with the mailboxes upstream.</p></div>
|
||||
<div class="sp-help-topic" id="filter"><h3>Spam filter</h3><p>The spam filter belongs to the server, not to a domain. The server's administrator attaches it in the server's <code>.env</code>, and from then on every inbound domain is screened by it before mail is handed on; the box shows on or off and has nothing to switch. Off means mail goes to the upstream unfiltered. While the filter is down, mail is either accepted unfiltered (the default) or deferred, so the sending server tries again later, as the server's setting says; the box states which.</p></div>
|
||||
<div class="sp-help-topic" id="inbound-log"><h3>Inbound log</h3><p>One row per recipient of a message offered on port 25 for an inbound domain, with what was decided: <span class="tag is-success is-light">accept</span> — handed on to the upstream; <span class="tag is-danger is-light">reject</span> — refused for good; <span class="tag is-warning is-light">tempfail</span> — the sender was asked to come back later; <span class="tag is-primary is-light">quarantine</span> — held by the filter. <em>Why</em> names who decided. <code>rspamd</code> is the spam filter, with its action, the score against the reject threshold and the heaviest symbols; <code>postfix</code> is the relay's own rule — an unknown recipient, a sender domain that does not exist — with its reply; <code>selfpost</code> means nobody judged the message: it was relayed with no filter attached. Accept says what this server did, not what the upstream did with the message afterwards.<br><br>Accepted mail is recorded as it passes; refused mail is read from <code>mail.log</code>, so its row has no subject, and a message refused after its text was sent is one row however many recipients it had. A refusal to relay for a domain that is not an inbound one is every scanner on the internet and is not recorded. Rows are kept for the number of days set in Settings, and never more than 10 000.</p></div>
|
||||
<div class="sp-help-topic" id="filter-lists"><h3>Filter lists</h3><p>Two lists for the whole server that correct the spam filter by hand. An entry is a sender address, a sender domain — that domain, not its subdomains — or a client address or network in CIDR form. A sender on the Deny list is refused outright. A sender on the Allow list is given a weight no spam score outweighs, so the filter lets it through. The sender's address is whatever the sending server says it is - anyone can claim to be a sender you allow - so allow a client address or network where you can, and a sender address only for mail you would rather receive forged than lose. When entries of both lists match one message, <strong>Deny wins</strong>.<br><br>SelfPost does not apply the lists itself: it writes them as files under <code>/data/antispam/</code>, and rspamd rereads a file within seconds of a change. With the spam filter off the lists are kept and can be edited, but nothing reads them. The sender matched is the envelope sender — the one in the log — not the From line a mail program shows. Matched counts the messages in the Inbound log that the entry decided, so it reaches back as far as the log does; never means not once in that time, and such an entry is one to consider removing.</p></div>
|
||||
</div>
|
||||
<div class="box">
|
||||
<div class="sp-box-head"><span class="sp-no">04</span><h2>Server</h2></div>
|
||||
<div class="sp-help-topic" id="backup"><h3>Backup and moving a domain</h3><p>Full backup downloads one archive of the whole instance: <code>data/</code> (the database, DKIM keys, application credentials and the queue), <code>docker-compose.yml</code>, <code>.env</code> and <code>certs/</code>. It holds secrets, so leave Encrypt with a password ticked: the file is sealed as <code>.spbk</code>, the password is not stored, and without it nothing can be recovered. To restore, unpack it into an empty project directory (decrypt a <code>.spbk</code> first with <code>selfpost-backup -decrypt</code>) and start the same SelfPost version that made it; the panel refuses data from another version. The reverse-proxy configuration and <code>mail.log</code> are not included. One domain moves on its own: Export domain on its settings page writes a <code>.json</code> (or encrypted <code>.spde</code>) with its DKIM key and its applications' passwords, and Import a domain here reads it into another instance, so the published DNS needs no change.</p></div>
|
||||
<div class="sp-help-topic" id="backup"><h3>Backup and moving a domain</h3><p>Full backup downloads one archive of the whole instance: <code>data/</code> (the database, DKIM keys and application credentials), <code>docker-compose.yml</code>, <code>.env</code> and <code>certs/</code>. The Postfix queue is not in it. The backup is taken while the server is running, and Postfix keeps moving messages through the queue, so a copy could hold a message twice or miss it; mail still waiting for delivery is therefore not carried over, and a restored server starts with an empty queue. To keep the queue, stop the container and archive the project directory instead. The private key in <code>certs/</code> has to be readable by the panel: give it to group 998 on the host (<code>chgrp 998</code>, <code>chmod 640</code>). Until then the panel refuses to make a backup and names the file, rather than hand over an archive without the key. It holds secrets, so leave Encrypt with a password ticked: the file is sealed as <code>.spbk</code>, the password is not stored, and without it nothing can be recovered. To restore, unpack it into an empty project directory (decrypt a <code>.spbk</code> first with <code>selfpost-backup -decrypt</code>) and start the same SelfPost version that made it; the panel refuses data from another version. The reverse-proxy configuration and <code>mail.log</code> are not included. One domain moves on its own: Export domain on its settings page writes a <code>.json</code> (or encrypted <code>.spde</code>) with its DKIM key and its applications' passwords, and Import a domain here reads it into another instance, so the published DNS needs no change.</p></div>
|
||||
<div class="sp-help-topic" id="users"><h3>Users and roles</h3><p>The two roles are a reach, not a rank: every user is an administrator and signs in with a username and password. Global reaches the whole server — Overview, Server, the queue and every domain — and is the only role that adds or deletes domains and users. Domain reaches only the domains assigned to it, outbound and inbound separately (the same name on both lists is two assignments); <em>All</em> includes domains added later, and a domain user needs at least one domain or <em>All</em> on either list. The only global user cannot be demoted or deleted, and you cannot delete the account you are signed in with. A new password set here signs that user out of their other sessions.</p></div>
|
||||
<div class="sp-help-topic" id="settings"><h3>Settings</h3><p>Settings holds what is true of the whole instance. Log retention is how many days the Outbound log keeps its rows, from 7 to 365 and 90 by default; a sweep every six hours deletes older ones, and it is the main driver of how fast <code>/data</code> grows. It does not change the rotation of <code>mail.log</code>, which is daily with about two weeks kept. The sending rate limit shown under it is level 1, the per-client-IP ceiling set in <code>.env</code>: changing it needs a restart, and no level-2 limit can exceed it.</p></div>
|
||||
<div class="sp-help-topic" id="settings"><h3>Settings</h3><p>Settings holds what is true of the whole instance. Log retention is how many days the Outbound log keeps its rows, from 7 to 365 and 90 by default; a sweep every six hours deletes older ones, and it is the main driver of how fast <code>/data</code> grows. With the inbound relay on there is a second number, for the Inbound log, with the same range. It does not change the rotation of <code>mail.log</code>, which is daily with about two weeks kept. The sending rate limit shown under it is level 1, the per-client-IP ceiling set in <code>.env</code>: changing it needs a restart, and no level-2 limit can exceed it. Notifications are turned on or off here for everyone.</p></div>
|
||||
<div class="sp-help-topic" id="notifications"><h3>Notifications</h3><p>The panel writes to its users when something needs them, from <code>postmaster@</code> the server's hostname. Four things are reported. A sending domain's SPF, DKIM or DMARC record stops matching: sent once two hourly checks in a row have found it so, and once more when the record matches again. A level-2 rate limit refuses mail: sent at the first refusal, then at most once a day for the same limit. The TLS certificate is 14 days from expiry, and again at 3 days if it is still the same certificate. A message has waited in the queue for more than three days, with one more message when none is left. A global user is told all of it; a domain administrator is told about the DNS records and limits of their own domains.<br><br>Each user turns notifications on or off under Account and needs an e-mail there; Server › Settings turns them off for everyone. They are the panel's own mail: not rows of the Outbound log, and not counted against any limit.<br><br>For that mail to arrive, the server's own name needs two DNS records of its own, as each of your domains does: a DKIM key, which the panel generates and signs notifications with, and an SPF record, <code>v=spf1 a -all</code>. Server › Settings gives both with the values to publish, and Server › Health shows whether they are in place. Without them notifications are still sent, and a receiver may file them as spam. Mail of your domains does not depend on either.</p></div>
|
||||
</div>
|
||||
<div class="box">
|
||||
<div class="sp-box-head"><span class="sp-no">05</span><h2>Account</h2></div>
|
||||
<div class="sp-help-topic" id="account"><h3>Your account</h3><p>Account is your own, whatever your role: your username and e-mail, and your password. Changing the password signs out every other session of this account and keeps the one you are using; applications have their own logins and are not affected.</p></div>
|
||||
<div class="sp-help-topic" id="account"><h3>Your account</h3><p>Account is your own, whatever your role: your username and e-mail, whether the panel sends you notifications, and your password. Changing the password signs out every other session of this account and keeps the one you are using; applications have their own logins and are not affected.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -74,7 +74,7 @@
|
||||
</div>
|
||||
|
||||
<div class="box">
|
||||
<div class="sp-box-head"><span class="sp-no">01</span><h2>Add an entry</h2><a class="sp-help" href="help.html#inbound" title="How the lists are applied"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-head"><span class="sp-no">01</span><h2>Add an entry</h2><a class="sp-help" href="help.html#filter-lists" title="How the lists are applied"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-body">
|
||||
<form class="sp-form" method="post" action="#">
|
||||
<div class="columns">
|
||||
@@ -84,7 +84,7 @@
|
||||
</div>
|
||||
<div class="column is-3">
|
||||
<div class="field"><label class="label" for="etype">Entry is</label>
|
||||
<div class="control"><div class="select is-fullwidth"><select id="etype" name="entry_type"><option>a sender address</option><option>a sender domain</option><option>a client IP or CIDR</option></select></div></div></div>
|
||||
<div class="control"><div class="select is-fullwidth"><select id="etype" name="entry_type"><option>a sender address</option><option>a sender domain</option><option>a client address or network</option></select></div></div></div>
|
||||
</div>
|
||||
<div class="column is-6">
|
||||
<div class="field"><label class="label" for="value">Value</label>
|
||||
@@ -103,11 +103,11 @@
|
||||
<div class="box">
|
||||
<div class="sp-box-head"><span class="sp-no">02</span><h2>Deny</h2><div class="sp-end">3 entries · wins over Allow</div></div>
|
||||
<div class="table-container"><table class="table is-fullwidth is-hoverable">
|
||||
<thead><tr><th>Value</th><th>Note</th><th>Added</th><th></th></tr></thead>
|
||||
<thead><tr><th>Value</th><th>Kind</th><th>Matched</th><th></th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="sp-mono sp-small">@bulk.example <span class="tag is-light">domain</span></td><td class="sp-muted">newsletter nobody subscribed to</td><td class="sp-muted sp-nowrap sp-small">2026-09-02</td><td class="sp-actions"><a class="has-text-danger" href="#">Remove</a></td></tr>
|
||||
<tr><td class="sp-mono sp-small">198.51.100.0/24 <span class="tag is-light">ip</span></td><td class="sp-muted">dictionary attack on lists.example.org</td><td class="sp-muted sp-nowrap sp-small">2026-08-19</td><td class="sp-actions"><a class="has-text-danger" href="#">Remove</a></td></tr>
|
||||
<tr><td class="sp-mono sp-small">win@lottery.example <span class="tag is-light">address</span></td><td class="sp-muted">—</td><td class="sp-muted sp-nowrap sp-small">2026-07-30</td><td class="sp-actions"><a class="has-text-danger" href="#">Remove</a></td></tr>
|
||||
<tr><td class="sp-mono sp-small">bulk.example</td><td><button type="button" class="tag is-light" popovertarget="deny-1">domain</button><div class="sp-sheet" id="deny-1" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-light">domain</span><h2>bulk.example</h2><button type="button" popovertarget="deny-1" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Note</th><td>newsletter nobody subscribed to</td></tr><tr><th>Added</th><td class="sp-mono sp-small">2026-09-02</td></tr><tr><th>Matched</th><td>41 in the last 90 days</td></tr><tr><th>Last match</th><td class="sp-mono sp-small">2026-09-21 13:57</td></tr></tbody></table></div></div></div></div></td><td class="sp-nowrap">41</td><td class="sp-actions"><form method="post" action="#"><button type="submit" class="has-text-danger">Remove</button></form></td></tr>
|
||||
<tr><td class="sp-mono sp-small">198.51.100.0/24</td><td><button type="button" class="tag is-light" popovertarget="deny-2">ip</button><div class="sp-sheet" id="deny-2" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-light">ip</span><h2>198.51.100.0/24</h2><button type="button" popovertarget="deny-2" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Note</th><td>dictionary attack on lists.example.org</td></tr><tr><th>Added</th><td class="sp-mono sp-small">2026-08-19</td></tr><tr><th>Matched</th><td>1203 in the last 90 days</td></tr><tr><th>Last match</th><td class="sp-mono sp-small">2026-09-21 09:12</td></tr></tbody></table></div></div></div></div></td><td class="sp-nowrap">1203</td><td class="sp-actions"><form method="post" action="#"><button type="submit" class="has-text-danger">Remove</button></form></td></tr>
|
||||
<tr><td class="sp-mono sp-small">win@lottery.example</td><td><button type="button" class="tag is-light" popovertarget="deny-3">address</button><div class="sp-sheet" id="deny-3" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-light">address</span><h2>win@lottery.example</h2><button type="button" popovertarget="deny-3" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Note</th><td>—</td></tr><tr><th>Added</th><td class="sp-mono sp-small">2026-07-30</td></tr><tr><th>Matched</th><td>never in the last 90 days</td></tr></tbody></table></div></div></div></div></td><td class="sp-muted sp-nowrap">never</td><td class="sp-actions"><form method="post" action="#"><button type="submit" class="has-text-danger">Remove</button></form></td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
@@ -116,10 +116,10 @@
|
||||
<div class="box">
|
||||
<div class="sp-box-head"><span class="sp-no">03</span><h2>Allow</h2><div class="sp-end">2 entries</div></div>
|
||||
<div class="table-container"><table class="table is-fullwidth is-hoverable">
|
||||
<thead><tr><th>Value</th><th>Note</th><th>Added</th><th></th></tr></thead>
|
||||
<thead><tr><th>Value</th><th>Kind</th><th>Matched</th><th></th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="sp-mono sp-small">@partner.example <span class="tag is-light">domain</span></td><td class="sp-muted">invoices were scored as spam</td><td class="sp-muted sp-nowrap sp-small">2026-09-11</td><td class="sp-actions"><a class="has-text-danger" href="#">Remove</a></td></tr>
|
||||
<tr><td class="sp-mono sp-small">noreply@github.example <span class="tag is-light">address</span></td><td class="sp-muted">security advisories</td><td class="sp-muted sp-nowrap sp-small">2026-06-04</td><td class="sp-actions"><a class="has-text-danger" href="#">Remove</a></td></tr>
|
||||
<tr><td class="sp-mono sp-small">partner.example</td><td><button type="button" class="tag is-light" popovertarget="allow-1">domain</button><div class="sp-sheet" id="allow-1" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-light">domain</span><h2>partner.example</h2><button type="button" popovertarget="allow-1" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Note</th><td>invoices were scored as spam</td></tr><tr><th>Added</th><td class="sp-mono sp-small">2026-09-11</td></tr><tr><th>Matched</th><td>7 in the last 90 days</td></tr><tr><th>Last match</th><td class="sp-mono sp-small">2026-09-19 16:40</td></tr></tbody></table></div></div></div></div></td><td class="sp-nowrap">7</td><td class="sp-actions"><form method="post" action="#"><button type="submit" class="has-text-danger">Remove</button></form></td></tr>
|
||||
<tr><td class="sp-mono sp-small">noreply@github.example</td><td><button type="button" class="tag is-light" popovertarget="allow-2">address</button><div class="sp-sheet" id="allow-2" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-light">address</span><h2>noreply@github.example</h2><button type="button" popovertarget="allow-2" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Note</th><td>security advisories</td></tr><tr><th>Added</th><td class="sp-mono sp-small">2026-06-04</td></tr><tr><th>Matched</th><td>18 in the last 90 days</td></tr><tr><th>Last match</th><td class="sp-mono sp-small">2026-09-21 13:30</td></tr></tbody></table></div></div></div></div></td><td class="sp-nowrap">18</td><td class="sp-actions"><form method="post" action="#"><button type="submit" class="has-text-danger">Remove</button></form></td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
@@ -130,10 +130,10 @@
|
||||
<div class="sp-box-head"><span class="sp-no">04</span><h2>Where the lists go</h2><div class="sp-end"><span class="tag is-success is-light">applied</span></div></div>
|
||||
<div class="sp-box-body">
|
||||
<dl class="sp-facts">
|
||||
<div><dt>Applied by</dt><dd>rspamd, from map files</dd></div>
|
||||
<div><dt>Files</dt><dd class="sp-mono sp-small">/data/antispam/allow.map<br>/data/antispam/deny.map</dd></div>
|
||||
<div><dt>Applied by</dt><dd>rspamd, from map files it rereads within seconds of a change</dd></div>
|
||||
<div><dt>Files</dt><dd><span class="sp-mono">/data/antispam/</span> — six, one for each list and kind of entry</dd></div>
|
||||
<div><dt>Last written</dt><dd>2026-09-11 08:40 UTC</dd></div>
|
||||
<div><dt>When the filter is off</dt><dd class="sp-muted">The lists are kept and editable, but nothing reads them until <code>INBOUND_ANTISPAM_MILTER</code> is set.</dd></div>
|
||||
<div><dt>When both lists match</dt><dd>Deny wins, whichever entries they are.</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -69,7 +69,7 @@
|
||||
<div class="sp-grow">
|
||||
<p class="sp-kicker">Inbound</p>
|
||||
<h1 class="title is-3">Inbound log</h1>
|
||||
<p class="sp-lead">What port 25 decided about each message, one row per recipient. Accepts come from the inbound journal, rejects from <code>mail.log</code>; the filter's reasons appear when the spam filter is on.</p>
|
||||
<p class="sp-lead">What port 25 decided about each message, one row per recipient. Accepted mail is recorded as it passes, refused mail is read from <code>mail.log</code>; the filter's verdict appears when the spam filter is on. Rows older than 90 days are deleted — change that in <a href="settings.html">Settings</a>.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -83,19 +83,20 @@
|
||||
<input class="input" type="date" name="to" value="2026-09-21" aria-label="To date">
|
||||
<button type="submit" class="button"><i class="ti ti-filter"></i>Filter</button>
|
||||
</form>
|
||||
<a class="sp-help" href="help.html#inbound-log" title="What the decisions mean"><i class="ti ti-help-circle"></i></a>
|
||||
</div>
|
||||
<div class="table-container"><table class="table is-fullwidth is-hoverable">
|
||||
<thead><tr><th>Time, UTC</th><th>From · client</th><th>To</th><th>Subject</th><th>Decision</th><th>Why</th><th></th></tr></thead>
|
||||
<thead><tr><th>Time, UTC</th><th>From</th><th>To</th><th>Subject</th><th>Decision</th><th></th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 14:00:18</td><td>t.berg@<wbr>uni.example<br><span class="sp-mono sp-muted sp-small">192.0.2.77</span></td><td>dev@<wbr>lists.example.org</td><td class="sp-clip">Re: [dev] patch series v3</td><td><span class="tag is-success is-light">accept</span></td><td class="sp-muted sp-small">rspamd · score 0.4</td><td class="sp-actions"><a href="in-filter-lists.html">Deny sender</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:59:02</td><td>invoice@<wbr>partner-billing.example<br><span class="sp-mono sp-muted sp-small">203.0.113.140</span></td><td>owner@<wbr>lists.example.org</td><td class="sp-clip">Outstanding invoice — action required</td><td><span class="tag is-primary is-light">quarantine</span></td><td class="sp-muted sp-small">rspamd · score 9.8 · PHISHING, FORGED_SENDER</td><td class="sp-actions"><a href="in-quarantine-message.html">Review</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:57:41</td><td>promo@<wbr>bulk.example<br><span class="sp-mono sp-muted sp-small">198.51.100.90</span></td><td>sales@<wbr>lists.example.org</td><td class="sp-clip sp-muted">—</td><td><span class="tag is-danger is-light">reject</span></td><td class="sp-muted sp-small">postfix · 550 5.1.1 unknown recipient</td><td class="sp-actions"><a href="in-filter-lists.html">Deny sender</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:51:30</td><td>billing@<wbr>partner.example<br><span class="sp-mono sp-muted sp-small">198.51.100.12</span></td><td>owner@<wbr>lists.example.org</td><td class="sp-clip">Invoice 2026-0917</td><td><span class="tag is-danger is-light">reject</span></td><td class="sp-muted sp-small">rspamd · score 16.2 · BAYES_SPAM</td><td class="sp-actions"><a href="in-filter-lists.html">Allow sender</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:44:02</td><td>hr@<wbr>partner.example<br><span class="sp-mono sp-muted sp-small">192.0.2.31</span></td><td>jobs@<wbr>acme.io</td><td class="sp-clip">Application: backend engineer</td><td><span class="tag is-warning is-light">tempfail</span></td><td class="sp-muted sp-small">selfpost · upstream 10.0.4.12 timed out</td><td class="sp-actions"></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:30:55</td><td>noreply@<wbr>github.example<br><span class="sp-mono sp-muted sp-small">192.0.2.200</span></td><td>security@<wbr>lists.example.org</td><td class="sp-clip">[advisory] new report</td><td><span class="tag is-success is-light">accept</span></td><td class="sp-muted sp-small">rspamd · allow list</td><td class="sp-actions"></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 14:00:18</td><td>t.berg@<wbr>uni.example<br><span class="sp-mono sp-muted sp-small">192.0.2.77</span></td><td>dev@<wbr>lists.example.org</td><td class="sp-clip">Re: [dev] patch series v3</td><td><button type="button" class="tag is-success is-light" popovertarget="why-1">accept</button><div class="sp-sheet" id="why-1" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-success is-light">accept</span><h2>Why</h2><button type="button" popovertarget="why-1" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Decided by</th><td>rspamd</td></tr><tr><th>Verdict</th><td>no action, score 0.40 / 15.00</td></tr><tr><th>Subject</th><td>Re: [dev] patch series v3</td></tr></tbody></table></div></div></div></div></td><td class="sp-actions"><a href="in-filter-lists.html">Deny sender</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:59:02</td><td>invoice@<wbr>partner-billing.example<br><span class="sp-mono sp-muted sp-small">203.0.113.140</span></td><td>owner@<wbr>lists.example.org</td><td class="sp-clip">Outstanding invoice — action required</td><td><button type="button" class="tag is-success is-light" popovertarget="why-2">accept</button><div class="sp-sheet" id="why-2" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-success is-light">accept</span><h2>Why</h2><button type="button" popovertarget="why-2" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Decided by</th><td>rspamd</td></tr><tr><th>Verdict</th><td>add header, score 9.80 / 15.00</td></tr><tr><th>Symbols</th><td class="sp-mono sp-small">PHISHING(5.00)<br>FORGED_SENDER(3.00)<br>R_SPF_FAIL(1.00)</td></tr><tr><th>Subject</th><td>Outstanding invoice — action required</td></tr></tbody></table></div></div></div></div></td><td class="sp-actions"><a href="in-filter-lists.html">Deny sender</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:57:41</td><td>promo@<wbr>bulk.example<br><span class="sp-mono sp-muted sp-small">198.51.100.90</span></td><td>sales@<wbr>lists.example.org</td><td class="sp-clip sp-muted">—</td><td><button type="button" class="tag is-danger is-light" popovertarget="why-3">reject</button><div class="sp-sheet" id="why-3" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-danger is-light">reject</span><h2>Why</h2><button type="button" popovertarget="why-3" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Decided by</th><td>postfix</td></tr><tr><th>Reply</th><td class="sp-mono sp-small">550 5.1.1 <sales@lists.example.org>: Recipient address rejected: User unknown in relay recipient table</td></tr></tbody></table></div></div></div></div></td><td class="sp-actions"></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:51:30</td><td>billing@<wbr>partner.example<br><span class="sp-mono sp-muted sp-small">198.51.100.12</span></td><td>owner@<wbr>lists.example.org</td><td class="sp-clip sp-muted">—</td><td><button type="button" class="tag is-danger is-light" popovertarget="why-4">reject</button><div class="sp-sheet" id="why-4" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-danger is-light">reject</span><h2>Why</h2><button type="button" popovertarget="why-4" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Decided by</th><td>rspamd</td></tr><tr><th>Reply</th><td class="sp-mono sp-small">5.7.1 Spam message rejected</td></tr></tbody></table></div></div></div></div></td><td class="sp-actions"><a href="in-filter-lists.html">Allow sender</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:44:02</td><td>hr@<wbr>partner.example<br><span class="sp-mono sp-muted sp-small">192.0.2.31</span></td><td>jobs@<wbr>acme.io</td><td class="sp-clip sp-muted">—</td><td><button type="button" class="tag is-warning is-light" popovertarget="why-5">tempfail</button><div class="sp-sheet" id="why-5" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-warning is-light">tempfail</span><h2>Why</h2><button type="button" popovertarget="why-5" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Decided by</th><td>rspamd</td></tr><tr><th>Reply</th><td class="sp-mono sp-small">4.7.1 Try again later</td></tr></tbody></table></div></div></div></div></td><td class="sp-actions"><a href="in-filter-lists.html">Allow sender</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:30:55</td><td>noreply@<wbr>github.example<br><span class="sp-mono sp-muted sp-small">192.0.2.200</span></td><td>security@<wbr>lists.example.org</td><td class="sp-clip">[advisory] new report</td><td><button type="button" class="tag is-success is-light" popovertarget="why-6">accept</button><div class="sp-sheet" id="why-6" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-success is-light">accept</span><h2>Why</h2><button type="button" popovertarget="why-6" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Decided by</th><td>rspamd</td></tr><tr><th>Verdict</th><td>no action, score -98.10 / 15.00</td></tr><tr><th>Symbols</th><td class="sp-mono sp-small">SELFPOST_ALLOW_FROM(-100.00)<br>R_DKIM_ALLOW(-0.20)</td></tr><tr><th>Subject</th><td>[advisory] new report</td></tr></tbody></table></div></div></div></div></td><td class="sp-actions"><a href="in-filter-lists.html">Deny sender</a></td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
<div class="sp-box-foot"><span>Page 1 of 38</span><a href="#">Older →</a><span class="sp-end">Allow / Deny sender opens <a href="in-filter-lists.html">Filter lists</a> with the entry filled in · global administrator only</span></div>
|
||||
<div class="sp-box-foot"><span>Page 1 of 38</span><a href="#">Older →</a><span class="sp-end">A decision opens its reason · Allow / Deny sender opens <a href="in-filter-lists.html">Filter lists</a> with the entry filled in</span></div>
|
||||
</div>
|
||||
|
||||
<footer class="sp-foot">SelfPost 1.8.0 · © 2026 Mix · <a href="#">License (AGPL-3.0)</a> · <a href="#">Source</a> · No warranty</footer>
|
||||
|
||||
@@ -91,6 +91,7 @@
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 14:01:50</td><td>shop@<wbr>shop.example.org</td><td>anna.lind@<wbr>outlook.com</td><td class="sp-clip">Your order #48212 has shipped</td><td><span class="tag is-success is-light">sent</span></td><td class="sp-actions"><a href="out-message.html">Details</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:58:07</td><td>alerts@<wbr>example.org</td><td>noc@<wbr>example.org</td><td class="sp-clip">[FIRING] disk usage above 90 % on db-2</td><td><span class="tag is-warning is-light">deferred</span></td><td class="sp-actions"><a href="out-message.html">Details</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:41:19</td><td>no-reply@<wbr>notify.acme.io</td><td>j.doe@<wbr>nonexistent.example</td><td class="sp-clip">Reset your password</td><td><span class="tag is-danger is-light">bounced</span></td><td class="sp-actions"><a href="out-message.html">Details</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:40:55</td><td>shop@<wbr>shop.example.org</td><td>k.berg@<wbr>web.de</td><td class="sp-clip">A message from the director</td><td><span class="tag is-danger is-light">wrong From</span></td><td class="sp-actions"><a href="out-message.html">Details</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:40:02</td><td>prod@<wbr>example.org</td><td>billing@<wbr>partner.example</td><td class="sp-clip">Invoice 2026-0917</td><td><span class="tag is-light">rate-limited</span></td><td class="sp-actions"><a href="out-message.html">Details</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:12:44</td><td>shop@<wbr>shop.example.org</td><td>p.novak@<wbr>seznam.cz</td><td class="sp-clip">We received your return request</td><td><span class="tag is-success is-light">sent</span></td><td class="sp-actions"><a href="out-message.html">Details</a></td></tr>
|
||||
</tbody>
|
||||
|
||||
@@ -79,36 +79,22 @@
|
||||
"columns": [
|
||||
{
|
||||
"is-6": [
|
||||
"box[Processes]:table",
|
||||
"box[Sender check]:table+body",
|
||||
"box[Hostname and reverse DNS]:body",
|
||||
"box[Machine]:table"
|
||||
]
|
||||
},
|
||||
{
|
||||
"is-6": [
|
||||
"box[Processes]:table"
|
||||
"box[Milter sockets]:table",
|
||||
"box[TLS certificate]:body",
|
||||
"box[Notification mail]:table+body",
|
||||
"box[Configuration]:body"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"columns": [
|
||||
{
|
||||
"is-4": [
|
||||
"box[TLS certificate]:body"
|
||||
]
|
||||
},
|
||||
{
|
||||
"is-4": [
|
||||
"box[Milter sockets]:table"
|
||||
]
|
||||
},
|
||||
{
|
||||
"is-4": [
|
||||
"box[Hostname and reverse DNS]:body"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"box[Configuration]:body"
|
||||
}
|
||||
],
|
||||
"help": [
|
||||
"head",
|
||||
@@ -122,9 +108,9 @@
|
||||
{
|
||||
"is-9": [
|
||||
"box[Overview]:help×3",
|
||||
"box[Outbound]:help×6",
|
||||
"box[Inbound]:help×4",
|
||||
"box[Server]:help×3",
|
||||
"box[Outbound]:help×7",
|
||||
"box[Inbound]:help×6",
|
||||
"box[Server]:help×4",
|
||||
"box[Account]:help"
|
||||
]
|
||||
}
|
||||
@@ -406,12 +392,14 @@
|
||||
"columns": [
|
||||
{
|
||||
"is-6": [
|
||||
"box[Log retention]:body"
|
||||
"box[Log retention]:body",
|
||||
"box[Sending rate limits]:body"
|
||||
]
|
||||
},
|
||||
{
|
||||
"is-6": [
|
||||
"box[Sending rate limits]:body"
|
||||
"box[Notifications]:body",
|
||||
"box[Notification mail]:record×2"
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -82,11 +82,12 @@
|
||||
<div class="field"><label class="label" for="ret">Keep outbound log rows, days</label>
|
||||
<div class="control"><input class="input" id="ret" name="send_log_retention_days" type="number" min="7" max="365" value="30" required></div>
|
||||
<p class="help">The main driver of <code>/data</code> growth. Does not affect <code>mail.log</code> rotation (fourteen daily files).</p></div>
|
||||
<div class="field"><label class="label" for="inret">Keep inbound log rows, days</label>
|
||||
<div class="control"><input class="input" id="inret" name="inbound_spam_log_retention_days" type="number" min="7" max="365" value="90" required></div>
|
||||
<p class="help">The inbound log also never holds more than 10 000 rows: refused mail is logged at the rate strangers knock on port 25.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="column is-6">
|
||||
<div class="box" id="rate-limits">
|
||||
<div class="sp-box-head"><span class="sp-no">02</span><h2>Sending rate limits</h2><div class="sp-end">Read-only</div><a class="sp-help" href="help.html#limits" title="Levels 1 and 2"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-body">
|
||||
@@ -97,6 +98,43 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="column is-6">
|
||||
<div class="box" id="notifications">
|
||||
<div class="sp-box-head"><span class="sp-no">03</span><h2>Notifications</h2><a class="sp-help" href="help.html#notifications" title="What is sent and when"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-body">
|
||||
<div class="field"><label class="checkbox"><input type="checkbox" name="notify" checked> Send notification e-mails</label>
|
||||
<p class="help">From <span class="sp-mono">postmaster@mail.example.org</span>, to every user who has an e-mail and has not turned them off under <a href="account.html">Account</a>.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="box" id="notification-mail">
|
||||
<div class="sp-box-head"><span class="sp-no">04</span><h2>Notification mail</h2><div class="sp-end">DNS records of the server's own name</div><a class="sp-help" href="help.html#notifications" title="What the panel sends and what makes it arrive"><i class="ti ti-help-circle"></i></a></div>
|
||||
|
||||
<div class="sp-record">
|
||||
<div class="sp-record-head"><h3>DKIM</h3><span class="tag is-success is-light">published</span><small>Not a secret · selector <code>sp2026</code></small></div>
|
||||
<div class="sp-row">
|
||||
<div class="field"><label class="label">Host / name</label>
|
||||
<div class="field has-addons"><div class="control is-expanded"><input class="input" readonly value="sp2026._domainkey.mail.example.org"></div><div class="control"><button type="button" class="button"><i class="ti ti-copy"></i>Copy</button></div></div></div>
|
||||
<div class="field"><label class="label">Type</label><div class="control"><input class="input" readonly value="TXT"></div></div>
|
||||
</div>
|
||||
<div class="field"><label class="label">Value to publish</label>
|
||||
<div class="field has-addons"><div class="control is-expanded"><textarea class="textarea" rows="5" readonly>v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAq7n2Lw0c5vXeR1dYt8uJb3mKs6pHf9aZoWg4iNy2TqCx0lVjE7rUh5sBk1dMa8fPz3oGc6yXw9tIe4nQv2uL…IDAQAB</textarea></div><div class="control"><button type="button" class="button"><i class="ti ti-copy"></i>Copy</button></div></div></div>
|
||||
</div>
|
||||
|
||||
<div class="sp-record">
|
||||
<div class="sp-record-head"><h3>SPF</h3><span class="tag is-warning is-light">missing</span><small>For the server's own name, not for your domains</small></div>
|
||||
<div class="sp-row">
|
||||
<div class="field"><label class="label">Host / name</label>
|
||||
<div class="field has-addons"><div class="control is-expanded"><input class="input" readonly value="mail.example.org"></div><div class="control"><button type="button" class="button"><i class="ti ti-copy"></i>Copy</button></div></div></div>
|
||||
<div class="field"><label class="label">Type</label><div class="control"><input class="input" readonly value="TXT"></div></div>
|
||||
</div>
|
||||
<div class="field"><label class="label">Value to publish</label>
|
||||
<div class="field has-addons"><div class="control is-expanded"><input class="input" readonly value="v=spf1 a -all"></div><div class="control"><button type="button" class="button"><i class="ti ti-copy"></i>Copy</button></div></div></div>
|
||||
<p class="help">Notifications are sent without it, and a receiver may file them as spam.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="buttons"><button type="submit" class="button is-primary">Save settings</button></div>
|
||||
</form>
|
||||
|
||||
@@ -16,6 +16,8 @@
|
||||
<div class="field"><label class="label" for="username">Username</label><div class="control has-icons-left"><input class="input" id="username" name="username" value="admin" autocomplete="username" spellcheck="false" required><span class="icon is-left"><i class="ti ti-user"></i></span></div></div>
|
||||
<div class="field"><label class="label" for="email">E-mail</label><div class="control has-icons-left"><input class="input" id="email" name="email" type="email" value="mix@example.org" autocomplete="email"><span class="icon is-left"><i class="ti ti-mail"></i></span></div>
|
||||
<p class="help">Your own address. A domain's report address can be filled with it from Domain settings.</p></div>
|
||||
<div class="field"><label class="checkbox"><input type="checkbox" name="notify" checked> Send me notifications</label>
|
||||
<p class="help">To this address, when something needs you: a DNS record of a domain stops matching, a rate limit refuses mail, the TLS certificate is about to expire, a message has waited in the queue for three days. <a href="help.html#notifications">What is sent and when</a></p></div>
|
||||
<button type="submit" class="button is-primary">Save profile</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
<div class="box">
|
||||
<div class="sp-box-head"><span class="sp-no">01</span><h2>Full backup</h2><a class="sp-help" href="help.html#backup" title="Backups, encryption and restore"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-body">
|
||||
<p class="mb-3"><code>data/</code> (database, DKIM keys, credentials, the queue), <code>docker-compose.yml</code>, <code>.env</code> and <code>certs/</code> in one archive. Restore into an empty project directory and start the <strong>same SelfPost version</strong>. The reverse-proxy vhost is not included.</p>
|
||||
<p class="mb-3"><code>data/</code> (database, DKIM keys, credentials), <code>docker-compose.yml</code>, <code>.env</code> and <code>certs/</code> in one archive. The Postfix queue is left out: it cannot be copied reliably while the server is running. Restore into an empty project directory and start the <strong>same SelfPost version</strong>. The reverse-proxy vhost is not included.</p>
|
||||
<form class="sp-form" method="post" action="#">
|
||||
<div class="field"><label class="checkbox"><input type="checkbox" name="encrypt" checked> Encrypt with a password (<code>.spbk</code>)</label></div>
|
||||
<div class="sp-pair">
|
||||
|
||||
@@ -18,13 +18,14 @@
|
||||
<dl class="sp-facts">
|
||||
<div><dt>list</dt><dd>page_head, then full-width boxes: the add form first, the table second. <a href="out-domains.html">Example</a></dd></div>
|
||||
<div><dt>detail</dt><dd>page_head, then <code>columns</code>: <code>is-3</code> side menu + <code>is-9</code> boxes. <a href="out-domain.html">Example</a></dd></div>
|
||||
<div><dt>cards</dt><dd>page_head, then <code>columns sp-cards</code>: two <code>is-6</code> stacks of numbered boxes read in rows, odd on the left and even on the right, each stack closed up. On a phone the boxes come in the order of their numbers. The two stacks under this box are laid out so. <a href="health.html">Example</a></dd></div>
|
||||
<div><dt>form</dt><dd>page_head, then boxes in equal <code>columns</code>, one submit row under them. <a href="out-app.html">Example</a></dd></div>
|
||||
<div><dt>signed out</dt><dd>The split brand / form screen, no navbar. <a href="login.html">Example</a></dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="columns">
|
||||
<div class="columns sp-cards">
|
||||
<div class="column is-6">
|
||||
<div class="box">
|
||||
<div class="sp-box-head"><span class="sp-no">01</span><h2>postmark</h2><div class="sp-end">Worst status of the page's checks</div></div>
|
||||
@@ -48,6 +49,7 @@
|
||||
<tr><td>error, fail, mismatch, bounced</td><td><span class="tag is-danger is-light">fail</span></td><td class="sp-mono sp-small">tag is-danger is-light</td></tr>
|
||||
<tr><td>unknown, off, neutral counts</td><td><span class="tag is-light">off</span></td><td class="sp-mono sp-small">tag is-light</td></tr>
|
||||
<tr><td>held: quarantine · role: global</td><td><span class="tag is-primary is-light">quarantine</span></td><td class="sp-mono sp-small">tag is-primary is-light</td></tr>
|
||||
<tr><td>a tag that opens a <code>sheet</code></td><td><button type="button" class="tag is-danger is-light" popovertarget="kit-sheet">reject</button><div class="sp-sheet" id="kit-sheet" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-danger is-light">reject</span><h2>sheet</h2><button type="button" popovertarget="kit-sheet" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>What it holds</th><td>What does not fit the row, as name and value</td></tr><tr><th>How it closes</th><td class="sp-mono sp-small">Esc<br>a click outside<br>the cross</td></tr></tbody></table></div></div></div></div></td><td class="sp-mono sp-small">button.tag + sp-sheet (popover)</td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
<div><dt>Last report</dt><dd>2026-09-21 06:12 UTC</dd></div>
|
||||
<div><dt>This week</dt><dd>14 kept · 0 parse failures</dd></div>
|
||||
<div><dt>Hosted addresses</dt><dd class="sp-mono sp-small">dmarc-reports+<domain>@mail.example.org</dd></div>
|
||||
<div><dt>Retention</dt><dd>500 reports or 90 days</dd></div>
|
||||
<div><dt>Retention</dt><dd>500 reports a domain or 90 days</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -8,23 +8,10 @@
|
||||
<form method="post" action="#"><button type="submit" class="button"><i class="ti ti-refresh"></i>Re-check DNS</button></form>
|
||||
</div>
|
||||
|
||||
<div class="columns">
|
||||
<div class="column is-6">
|
||||
<div class="box" id="machine">
|
||||
<div class="sp-box-head"><span class="sp-no">01</span><h2>Machine</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div><a class="sp-help" href="help.html#checks" title="What these readings mean"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="table-container"><table class="table is-fullwidth">
|
||||
<thead><tr><th>Resource</th><th>Usage</th><th>Detail</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>CPU</td><td><progress class="progress is-small is-success mb-1" value="12" max="100"></progress>12 %</td><td class="sp-muted">2 cores, load 0.21</td></tr>
|
||||
<tr><td>Memory</td><td><progress class="progress is-small is-success mb-1" value="41" max="100"></progress>41 %</td><td class="sp-muted">812 MB of 1.9 GB</td></tr>
|
||||
<tr><td>Network</td><td class="sp-nowrap">↓ 18 kB/s · ↑ 44 kB/s</td><td class="sp-muted">eth0: 2.1 GB in, 6.4 GB out</td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="columns sp-cards">
|
||||
<div class="column is-6">
|
||||
<div class="box" id="processes">
|
||||
<div class="sp-box-head"><span class="sp-no">02</span><h2>Processes</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div></div>
|
||||
<div class="sp-box-head"><span class="sp-no">01</span><h2>Processes</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div></div>
|
||||
<div class="table-container"><table class="table is-fullwidth">
|
||||
<thead><tr><th>Program</th><th>State</th><th>Detail</th></tr></thead>
|
||||
<tbody>
|
||||
@@ -36,37 +23,20 @@
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="columns">
|
||||
<div class="column is-4">
|
||||
<div class="box" id="certificate">
|
||||
<div class="sp-box-head"><span class="sp-no">03</span><h2>TLS certificate</h2><div class="sp-end"><span class="tag is-warning is-light">warn</span></div><a class="sp-help" href="help.html#certificate" title="When it warns and what to do"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-body">
|
||||
<dl class="sp-facts">
|
||||
<div><dt>Expires</dt><dd class="sp-mono">2026-10-03 09:14 UTC</dd></div>
|
||||
<div><dt>Names</dt><dd class="sp-mono">mail.example.org</dd></div>
|
||||
</dl>
|
||||
<p class="mt-4 has-text-danger">Expires in 12 days — renewal has not run.</p>
|
||||
<p class="sp-muted">Renewal normally runs 30 days before expiry. Check that port 80 is reachable and look for <code>acme</code> lines in the <a href="system-log.html">system log</a>.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="column is-4">
|
||||
<div class="box" id="sockets">
|
||||
<div class="sp-box-head"><span class="sp-no">04</span><h2>Milter sockets</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div></div>
|
||||
<div class="box" id="sender-check">
|
||||
<div class="sp-box-head"><span class="sp-no">03</span><h2>Sender check</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div><a class="sp-help" href="help.html#sender-check" title="What is checked and what these numbers are for"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="table-container"><table class="table is-fullwidth">
|
||||
<thead><tr><th>Milter</th><th>State</th></tr></thead>
|
||||
<thead><tr><th>Period</th><th>Panel not answering</th><th>Messages deferred</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>opendkim<br><span class="sp-muted sp-mono sp-small">inet:127.0.0.1:8891</span></td><td><span class="tag is-success is-light">ok</span></td></tr>
|
||||
<tr><td>journal-milter<br><span class="sp-muted sp-mono sp-small">unix:/run/selfpost/journal.sock</span></td><td><span class="tag is-success is-light">ok</span></td></tr>
|
||||
<tr><td>rspamd <span class="sp-muted">· inbound spam filter</span><br><span class="sp-muted sp-mono sp-small">inet:antispam:11332</span></td><td><span class="tag is-success is-light">ok</span></td></tr>
|
||||
<tr><td>Today</td><td>0 s</td><td>0</td></tr>
|
||||
<tr><td>Last 30 days</td><td>4 s</td><td>0</td></tr>
|
||||
<tr><td>Last 12 months</td><td>41 s</td><td>3</td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
<div class="sp-box-body">
|
||||
<p class="sp-muted">The panel holds each message's <code>From:</code> against the application that sends it. While the panel does not answer, Postfix defers mail instead of accepting it unchecked. Time is counted only while Postfix itself was running. Counted since <span class="sp-mono">2026-10-13</span>.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="column is-4">
|
||||
<div class="box" id="hostname">
|
||||
<div class="sp-box-head"><span class="sp-no">05</span><h2>Hostname and reverse DNS</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div><a class="sp-help" href="help.html#rdns" title="Forward and reverse DNS"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-body">
|
||||
@@ -76,13 +46,61 @@
|
||||
</dl>
|
||||
</div>
|
||||
</div>
|
||||
<div class="box" id="machine">
|
||||
<div class="sp-box-head"><span class="sp-no">07</span><h2>Machine</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div><a class="sp-help" href="help.html#checks" title="What these readings mean"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="table-container"><table class="table is-fullwidth">
|
||||
<thead><tr><th>Resource</th><th>Usage</th><th>Detail</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>CPU</td><td><progress class="progress is-small is-success mb-1" value="12" max="100"></progress>12 %</td><td class="sp-muted">2 cores, load 0.21</td></tr>
|
||||
<tr><td>Memory</td><td><progress class="progress is-small is-success mb-1" value="41" max="100"></progress>41 %</td><td class="sp-muted">812 MB of 1.9 GB</td></tr>
|
||||
<tr><td>Network</td><td class="sp-nowrap">↓ 18 kB/s · ↑ 44 kB/s</td><td class="sp-muted">eth0: 2.1 GB in, 6.4 GB out</td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="box" id="configuration">
|
||||
<div class="sp-box-head"><span class="sp-no">06</span><h2>Configuration</h2><div class="sp-end">Safe to run at any time</div></div>
|
||||
<div class="sp-box-body">
|
||||
<p class="mb-3">Regenerates the OpenDKIM and Postfix configuration from the database and reloads both daemons. Use it after editing the files by hand, after restoring a backup, or when the running configuration looks out of step with the domain and application lists. It does not touch the queue or the TLS certificate.</p>
|
||||
<form method="post" action="#"><button type="submit" class="button"><i class="ti ti-reload"></i>Reload configuration</button></form>
|
||||
<div class="column is-6">
|
||||
<div class="box" id="sockets">
|
||||
<div class="sp-box-head"><span class="sp-no">02</span><h2>Milter sockets</h2><div class="sp-end"><span class="tag is-success is-light">ok</span></div></div>
|
||||
<div class="table-container"><table class="table is-fullwidth">
|
||||
<thead><tr><th>Milter</th><th>State</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>opendkim<br><span class="sp-muted sp-mono sp-small">inet:127.0.0.1:8891</span></td><td><span class="tag is-success is-light">ok</span></td></tr>
|
||||
<tr><td>journal-milter<br><span class="sp-muted sp-mono sp-small">unix:/run/selfpost/journal.sock</span></td><td><span class="tag is-success is-light">ok</span></td></tr>
|
||||
<tr><td>sender-check<br><span class="sp-muted sp-mono sp-small">unix:/run/selfpost/sender.sock</span></td><td><span class="tag is-success is-light">ok</span></td></tr>
|
||||
<tr><td>rspamd <span class="sp-muted">· inbound spam filter</span><br><span class="sp-muted sp-mono sp-small">inet:antispam:11332</span></td><td><span class="tag is-success is-light">ok</span></td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
<div class="box" id="certificate">
|
||||
<div class="sp-box-head"><span class="sp-no">04</span><h2>TLS certificate</h2><div class="sp-end"><span class="tag is-warning is-light">warn</span></div><a class="sp-help" href="help.html#certificate" title="When it warns and what to do"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-body">
|
||||
<dl class="sp-facts">
|
||||
<div><dt>Expires</dt><dd class="sp-mono">2026-10-03 09:14 UTC</dd></div>
|
||||
<div><dt>Names</dt><dd class="sp-mono">mail.example.org</dd></div>
|
||||
</dl>
|
||||
<p class="mt-4 has-text-danger">Expires in 12 days — renewal has not run.</p>
|
||||
<p class="sp-muted">Renewal normally runs 30 days before expiry. Check that port 80 is reachable and look for <code>acme</code> lines in the <a href="system-log.html">system log</a>.</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="box" id="notification-mail">
|
||||
<div class="sp-box-head"><span class="sp-no">06</span><h2>Notification mail</h2><div class="sp-end"><span class="tag is-warning is-light">warn</span></div><a class="sp-help" href="help.html#notifications" title="What the panel sends and what makes it arrive"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="table-container"><table class="table is-fullwidth">
|
||||
<thead><tr><th>Record</th><th>State</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>DKIM<br><span class="sp-muted sp-mono sp-small">sp2026._domainkey.mail.example.org</span></td><td><span class="tag is-success is-light">published</span></td></tr>
|
||||
<tr><td>SPF<br><span class="sp-muted sp-mono sp-small">mail.example.org</span></td><td><span class="tag is-warning is-light">missing</span></td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
<div class="sp-box-body">
|
||||
<p class="sp-muted">Sent from <span class="sp-mono">postmaster@mail.example.org</span>. What to publish is under <a href="settings.html#notification-mail">Settings</a>.</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="box" id="configuration">
|
||||
<div class="sp-box-head"><span class="sp-no">08</span><h2>Configuration</h2><div class="sp-end">Safe to run at any time</div></div>
|
||||
<div class="sp-box-body">
|
||||
<p class="mb-3">Regenerates the OpenDKIM and Postfix configuration from the database and reloads both daemons. Use it after editing the files by hand, after restoring a backup, or when the running configuration looks out of step with the domain and application lists. It does not touch the queue or the TLS certificate.</p>
|
||||
<form method="post" action="#"><button type="submit" class="button"><i class="ti ti-reload"></i>Reload configuration</button></form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
<ul class="menu-list">
|
||||
<li><a href="#dns"><i class="ti ti-world-check"></i>DNS records</a></li>
|
||||
<li><a href="#apps"><i class="ti ti-apps"></i>Applications</a></li>
|
||||
<li><a href="#sender-check"><i class="ti ti-shield-check"></i>Sender check</a></li>
|
||||
<li><a href="#limits"><i class="ti ti-gauge"></i>Rate limits</a></li>
|
||||
<li><a href="#log"><i class="ti ti-list-details"></i>Outbound log</a></li>
|
||||
<li><a href="#queue"><i class="ti ti-stack-2"></i>Queue and retries</a></li>
|
||||
@@ -31,12 +32,15 @@
|
||||
<li><a href="#upstream"><i class="ti ti-arrow-forward-up"></i>Upstream</a></li>
|
||||
<li><a href="#recipients"><i class="ti ti-address-book"></i>Valid recipients</a></li>
|
||||
<li><a href="#filter"><i class="ti ti-shield-check"></i>Spam filter</a></li>
|
||||
<li><a href="#inbound-log"><i class="ti ti-list-details"></i>Inbound log</a></li>
|
||||
<li><a href="#filter-lists"><i class="ti ti-filter"></i>Filter lists</a></li>
|
||||
</ul>
|
||||
<p class="menu-label">Server</p>
|
||||
<ul class="menu-list">
|
||||
<li><a href="#backup"><i class="ti ti-archive"></i>Backup and moving a domain</a></li>
|
||||
<li><a href="#users"><i class="ti ti-users"></i>Users and roles</a></li>
|
||||
<li><a href="#settings"><i class="ti ti-settings"></i>Settings</a></li>
|
||||
<li><a href="#notifications"><i class="ti ti-mail"></i>Notifications</a></li>
|
||||
</ul>
|
||||
<p class="menu-label">Account</p>
|
||||
<ul class="menu-list">
|
||||
@@ -55,8 +59,9 @@
|
||||
<div class="sp-box-head"><span class="sp-no">02</span><h2>Outbound</h2></div>
|
||||
<div class="sp-help-topic" id="dns"><h3>DNS records</h3><p>DKIM proves the message was signed by this relay, SPF lists the addresses allowed to send for the domain, DMARC tells receivers what to do when both fail and where to report. Publish all three exactly as shown; merge SPF into an existing record rather than adding a second one. A domain's status is the worst of its three records, and results are cached for a few minutes, so use Re-check DNS after publishing. The SPF check is shallow: it looks for this server's address in the record itself and does not follow <code>include:</code> or <code>redirect=</code>, so a record that relies on them is reported as a warning, not a failure. <code>p=none</code> does not affect delivery; tighten to <code>quarantine</code>, then <code>reject</code>, once the reports look clean. A domain's report address is set in its Domain settings.</p></div>
|
||||
<div class="sp-help-topic" id="apps"><h3>Applications</h3><p>An application is one SASL login, unique across all domains. It may send as any address of its domain or only as listed addresses, and, with a list of client IPs set, only from those addresses. The password is shown once at creation or regeneration and is not stored in readable form.</p></div>
|
||||
<div class="sp-help-topic" id="sender-check"><h3>Sender check</h3><p>An application sends only as itself, and that holds for both senders a message has. The envelope sender is held against the login by Postfix. The <code>From:</code> line, the one a mail program shows and the one the DKIM signature is chosen by, is held against it by the panel, under the same rule: an application set to any address of its domain may write any address of that domain, and one with a list of addresses only those. A message with any other <code>From:</code> - another of your domains, the server's own name, an address somewhere else, or none at all - is refused for good with <code>550 5.7.1</code>. It is a rejected row of the Outbound log, and its page says which address was refused. This is what keeps one application from getting mail signed with another domain's key.<br><br>The check is strict. While the panel does not answer, Postfix does not accept mail unchecked: it replies with a temporary error and the application sends again later. Server › Health counts what that costs: for how long the panel did not answer while Postfix itself was running, and how many messages were deferred because of it. The card warns when a message was deferred in the last 24 hours. If the numbers stay at nothing, the check is where it should be; if they do not, it belongs in a process of its own.</p></div>
|
||||
<div class="sp-help-topic" id="limits"><h3>Rate limits</h3><p>Level 1 is a per-IP ceiling set in <code>.env</code>. Level 2 is optional, per domain and per application, entered by hand or computed from the last 30 days of sending, and can never exceed level 1. An application's own limit replaces its domain's.</p></div>
|
||||
<div class="sp-help-topic" id="log"><h3>Outbound log</h3><p>One row per message and recipient, with the status Postfix last reported: queued (accepted, nothing reported yet), sent (the receiving server took it; what the mailbox does next is not reported back), deferred (not taken yet, Postfix tries again), bounced (failed for good, or the queue lifetime ran out) and rejected (refused before queueing under a level-2 rate limit, so it never reached Postfix). A message's page lists these steps and the lines Postfix wrote about it in <code>mail.log</code>, found by queue id; that file is rotated daily and about two weeks are kept, so an old message may show none. Rows are deleted after the retention set in Server › Settings, 90 days by default. A domain administrator sees only the rows of their own domains.</p></div>
|
||||
<div class="sp-help-topic" id="log"><h3>Outbound log</h3><p>One row per message and recipient, with the status Postfix last reported: queued (accepted, nothing reported yet), sent (the receiving server took it; what the mailbox does next is not reported back), deferred (not taken yet, Postfix tries again), bounced (failed for good, or the queue lifetime ran out) and rejected (refused before queueing, so it never reached the queue: under a level-2 rate limit, from a client address the application does not allow, or for a <code>From:</code> line that is not the application's - the message's page says which). A message's page lists these steps and the lines Postfix wrote about it in <code>mail.log</code>, found by queue id. The lines are kept with the row: they stay on the page after that file has rotated, for as long as the row does. A message accepted before version 2.2.0 shows them only while they are still in the current file. Rows are deleted after the retention set in Server › Settings, 90 days by default, and their lines with them. A domain administrator sees only the rows of their own domains.</p></div>
|
||||
<div class="sp-help-topic" id="queue"><h3>Queue and retries</h3><p>The page lists, as <code>postqueue -p</code> prints it, the messages Postfix has accepted and not yet delivered: mail being sent right now and mail deferred for a later attempt. Only the global role can open it, and it only shows; nothing here deletes or forces a message. A deferred message is tried again after the first retry delay, then at gaps that double up to a cap, until the receiving server takes it or the queue lifetime runs out and it bounces; there is no attempt counter, and the box on the page shows this server's values. A queue that keeps growing means a receiver is refusing or not answering: open the message from the Outbound log to read the reason Postfix logged.</p></div>
|
||||
<div class="sp-help-topic" id="dmarc"><h3>DMARC reports</h3><p>An aggregate report is the summary a receiving provider sends, usually daily, to the <code>rua=</code> address of your DMARC record: which servers sent mail as your domain and whether SPF or DKIM passed for it. A domain's report address is one of three choices, set in its Domain settings: no reports (the default for a new domain), the SelfPost hosted address (an address on this server whose reports are parsed and shown under DMARC reports; offered only while hosted reports are enabled on the server), or an address typed for the domain (an ordinary mailbox that receives the raw XML; a button fills in your own profile e-mail). When the typed address is on another domain than the one sending, that domain must publish a report-authorization record, or providers will not send; the domain's page shows it with the other DNS records and checks it. In the tables pass means SPF or DKIM passed and aligned, fail means neither did, and your own server shows as <em>this relay</em>. A fail from any other source is someone else sending as your domain — a forwarder, an application you forgot, or abuse — so do not tighten <code>p=</code> until that sender is fixed or removed. A domain administrator reads the reports of their own domains.</p></div>
|
||||
</div>
|
||||
@@ -66,16 +71,19 @@
|
||||
<div class="sp-help-topic" id="upstream"><h3>Upstream</h3><p>The upstream is the server that actually holds the mailboxes. Give its host and port — used exactly as written, with no MX lookup — and choose the TLS for the hop to it: Opportunistic uses TLS when the upstream offers it and otherwise sends in the clear, Required holds the mail until a TLS session is possible, Off never uses TLS. While the upstream does not answer, Postfix keeps the mail and retries as for any other mail.</p></div>
|
||||
<div class="sp-help-topic" id="recipients"><h3>Valid recipients</h3><p>Choose whom this domain accepts. Listed addresses only: the relay accepts exactly the addresses you list, one per line or comma-separated, and refuses any other while the sender is still connected, so it never has to send a bounce for them. Any recipient: everything at the domain is accepted and passed on, and an address the upstream then rejects comes back as a bounce from this server to a sender who is often forged — backscatter. Prefer a list, and keep it in step with the mailboxes upstream.</p></div>
|
||||
<div class="sp-help-topic" id="filter"><h3>Spam filter</h3><p>The spam filter belongs to the server, not to a domain. The server's administrator attaches it in the server's <code>.env</code>, and from then on every inbound domain is screened by it before mail is handed on; the box shows on or off and has nothing to switch. Off means mail goes to the upstream unfiltered. While the filter is down, mail is either accepted unfiltered (the default) or deferred, so the sending server tries again later, as the server's setting says; the box states which.</p></div>
|
||||
<div class="sp-help-topic" id="inbound-log"><h3>Inbound log</h3><p>One row per recipient of a message offered on port 25 for an inbound domain, with what was decided: <span class="tag is-success is-light">accept</span> — handed on to the upstream; <span class="tag is-danger is-light">reject</span> — refused for good; <span class="tag is-warning is-light">tempfail</span> — the sender was asked to come back later; <span class="tag is-primary is-light">quarantine</span> — held by the filter. <em>Why</em> names who decided. <code>rspamd</code> is the spam filter, with its action, the score against the reject threshold and the heaviest symbols; <code>postfix</code> is the relay's own rule — an unknown recipient, a sender domain that does not exist — with its reply; <code>selfpost</code> means nobody judged the message: it was relayed with no filter attached. Accept says what this server did, not what the upstream did with the message afterwards.<br><br>Accepted mail is recorded as it passes; refused mail is read from <code>mail.log</code>, so its row has no subject, and a message refused after its text was sent is one row however many recipients it had. A refusal to relay for a domain that is not an inbound one is every scanner on the internet and is not recorded. Rows are kept for the number of days set in Settings, and never more than 10 000.</p></div>
|
||||
<div class="sp-help-topic" id="filter-lists"><h3>Filter lists</h3><p>Two lists for the whole server that correct the spam filter by hand. An entry is a sender address, a sender domain — that domain, not its subdomains — or a client address or network in CIDR form. A sender on the Deny list is refused outright. A sender on the Allow list is given a weight no spam score outweighs, so the filter lets it through. The sender's address is whatever the sending server says it is - anyone can claim to be a sender you allow - so allow a client address or network where you can, and a sender address only for mail you would rather receive forged than lose. When entries of both lists match one message, <strong>Deny wins</strong>.<br><br>SelfPost does not apply the lists itself: it writes them as files under <code>/data/antispam/</code>, and rspamd rereads a file within seconds of a change. With the spam filter off the lists are kept and can be edited, but nothing reads them. The sender matched is the envelope sender — the one in the log — not the From line a mail program shows. Matched counts the messages in the Inbound log that the entry decided, so it reaches back as far as the log does; never means not once in that time, and such an entry is one to consider removing.</p></div>
|
||||
</div>
|
||||
<div class="box">
|
||||
<div class="sp-box-head"><span class="sp-no">04</span><h2>Server</h2></div>
|
||||
<div class="sp-help-topic" id="backup"><h3>Backup and moving a domain</h3><p>Full backup downloads one archive of the whole instance: <code>data/</code> (the database, DKIM keys, application credentials and the queue), <code>docker-compose.yml</code>, <code>.env</code> and <code>certs/</code>. It holds secrets, so leave Encrypt with a password ticked: the file is sealed as <code>.spbk</code>, the password is not stored, and without it nothing can be recovered. To restore, unpack it into an empty project directory (decrypt a <code>.spbk</code> first with <code>selfpost-backup -decrypt</code>) and start the same SelfPost version that made it; the panel refuses data from another version. The reverse-proxy configuration and <code>mail.log</code> are not included. One domain moves on its own: Export domain on its settings page writes a <code>.json</code> (or encrypted <code>.spde</code>) with its DKIM key and its applications' passwords, and Import a domain here reads it into another instance, so the published DNS needs no change.</p></div>
|
||||
<div class="sp-help-topic" id="backup"><h3>Backup and moving a domain</h3><p>Full backup downloads one archive of the whole instance: <code>data/</code> (the database, DKIM keys and application credentials), <code>docker-compose.yml</code>, <code>.env</code> and <code>certs/</code>. The Postfix queue is not in it. The backup is taken while the server is running, and Postfix keeps moving messages through the queue, so a copy could hold a message twice or miss it; mail still waiting for delivery is therefore not carried over, and a restored server starts with an empty queue. To keep the queue, stop the container and archive the project directory instead. The private key in <code>certs/</code> has to be readable by the panel: give it to group 998 on the host (<code>chgrp 998</code>, <code>chmod 640</code>). Until then the panel refuses to make a backup and names the file, rather than hand over an archive without the key. It holds secrets, so leave Encrypt with a password ticked: the file is sealed as <code>.spbk</code>, the password is not stored, and without it nothing can be recovered. To restore, unpack it into an empty project directory (decrypt a <code>.spbk</code> first with <code>selfpost-backup -decrypt</code>) and start the same SelfPost version that made it; the panel refuses data from another version. The reverse-proxy configuration and <code>mail.log</code> are not included. One domain moves on its own: Export domain on its settings page writes a <code>.json</code> (or encrypted <code>.spde</code>) with its DKIM key and its applications' passwords, and Import a domain here reads it into another instance, so the published DNS needs no change.</p></div>
|
||||
<div class="sp-help-topic" id="users"><h3>Users and roles</h3><p>The two roles are a reach, not a rank: every user is an administrator and signs in with a username and password. Global reaches the whole server — Overview, Server, the queue and every domain — and is the only role that adds or deletes domains and users. Domain reaches only the domains assigned to it, outbound and inbound separately (the same name on both lists is two assignments); <em>All</em> includes domains added later, and a domain user needs at least one domain or <em>All</em> on either list. The only global user cannot be demoted or deleted, and you cannot delete the account you are signed in with. A new password set here signs that user out of their other sessions.</p></div>
|
||||
<div class="sp-help-topic" id="settings"><h3>Settings</h3><p>Settings holds what is true of the whole instance. Log retention is how many days the Outbound log keeps its rows, from 7 to 365 and 90 by default; a sweep every six hours deletes older ones, and it is the main driver of how fast <code>/data</code> grows. It does not change the rotation of <code>mail.log</code>, which is daily with about two weeks kept. The sending rate limit shown under it is level 1, the per-client-IP ceiling set in <code>.env</code>: changing it needs a restart, and no level-2 limit can exceed it.</p></div>
|
||||
<div class="sp-help-topic" id="settings"><h3>Settings</h3><p>Settings holds what is true of the whole instance. Log retention is how many days the Outbound log keeps its rows, from 7 to 365 and 90 by default; a sweep every six hours deletes older ones, and it is the main driver of how fast <code>/data</code> grows. With the inbound relay on there is a second number, for the Inbound log, with the same range. It does not change the rotation of <code>mail.log</code>, which is daily with about two weeks kept. The sending rate limit shown under it is level 1, the per-client-IP ceiling set in <code>.env</code>: changing it needs a restart, and no level-2 limit can exceed it. Notifications are turned on or off here for everyone.</p></div>
|
||||
<div class="sp-help-topic" id="notifications"><h3>Notifications</h3><p>The panel writes to its users when something needs them, from <code>postmaster@</code> the server's hostname. Four things are reported. A sending domain's SPF, DKIM or DMARC record stops matching: sent once two hourly checks in a row have found it so, and once more when the record matches again. A level-2 rate limit refuses mail: sent at the first refusal, then at most once a day for the same limit. The TLS certificate is 14 days from expiry, and again at 3 days if it is still the same certificate. A message has waited in the queue for more than three days, with one more message when none is left. A global user is told all of it; a domain administrator is told about the DNS records and limits of their own domains.<br><br>Each user turns notifications on or off under Account and needs an e-mail there; Server › Settings turns them off for everyone. They are the panel's own mail: not rows of the Outbound log, and not counted against any limit.<br><br>For that mail to arrive, the server's own name needs two DNS records of its own, as each of your domains does: a DKIM key, which the panel generates and signs notifications with, and an SPF record, <code>v=spf1 a -all</code>. Server › Settings gives both with the values to publish, and Server › Health shows whether they are in place. Without them notifications are still sent, and a receiver may file them as spam. Mail of your domains does not depend on either.</p></div>
|
||||
</div>
|
||||
<div class="box">
|
||||
<div class="sp-box-head"><span class="sp-no">05</span><h2>Account</h2></div>
|
||||
<div class="sp-help-topic" id="account"><h3>Your account</h3><p>Account is your own, whatever your role: your username and e-mail, and your password. Changing the password signs out every other session of this account and keeps the one you are using; applications have their own logins and are not affected.</p></div>
|
||||
<div class="sp-help-topic" id="account"><h3>Your account</h3><p>Account is your own, whatever your role: your username and e-mail, whether the panel sends you notifications, and your password. Changing the password signs out every other session of this account and keeps the one you are using; applications have their own logins and are not affected.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
</div>
|
||||
|
||||
<div class="box">
|
||||
<div class="sp-box-head"><span class="sp-no">01</span><h2>Add an entry</h2><a class="sp-help" href="help.html#inbound" title="How the lists are applied"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-head"><span class="sp-no">01</span><h2>Add an entry</h2><a class="sp-help" href="help.html#filter-lists" title="How the lists are applied"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-body">
|
||||
<form class="sp-form" method="post" action="#">
|
||||
<div class="columns">
|
||||
@@ -18,7 +18,7 @@
|
||||
</div>
|
||||
<div class="column is-3">
|
||||
<div class="field"><label class="label" for="etype">Entry is</label>
|
||||
<div class="control"><div class="select is-fullwidth"><select id="etype" name="entry_type"><option>a sender address</option><option>a sender domain</option><option>a client IP or CIDR</option></select></div></div></div>
|
||||
<div class="control"><div class="select is-fullwidth"><select id="etype" name="entry_type"><option>a sender address</option><option>a sender domain</option><option>a client address or network</option></select></div></div></div>
|
||||
</div>
|
||||
<div class="column is-6">
|
||||
<div class="field"><label class="label" for="value">Value</label>
|
||||
@@ -37,11 +37,11 @@
|
||||
<div class="box">
|
||||
<div class="sp-box-head"><span class="sp-no">02</span><h2>Deny</h2><div class="sp-end">3 entries · wins over Allow</div></div>
|
||||
<div class="table-container"><table class="table is-fullwidth is-hoverable">
|
||||
<thead><tr><th>Value</th><th>Note</th><th>Added</th><th></th></tr></thead>
|
||||
<thead><tr><th>Value</th><th>Kind</th><th>Matched</th><th></th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="sp-mono sp-small">@bulk.example <span class="tag is-light">domain</span></td><td class="sp-muted">newsletter nobody subscribed to</td><td class="sp-muted sp-nowrap sp-small">2026-09-02</td><td class="sp-actions"><a class="has-text-danger" href="#">Remove</a></td></tr>
|
||||
<tr><td class="sp-mono sp-small">198.51.100.0/24 <span class="tag is-light">ip</span></td><td class="sp-muted">dictionary attack on lists.example.org</td><td class="sp-muted sp-nowrap sp-small">2026-08-19</td><td class="sp-actions"><a class="has-text-danger" href="#">Remove</a></td></tr>
|
||||
<tr><td class="sp-mono sp-small">win@lottery.example <span class="tag is-light">address</span></td><td class="sp-muted">—</td><td class="sp-muted sp-nowrap sp-small">2026-07-30</td><td class="sp-actions"><a class="has-text-danger" href="#">Remove</a></td></tr>
|
||||
<tr><td class="sp-mono sp-small">bulk.example</td><td><button type="button" class="tag is-light" popovertarget="deny-1">domain</button><div class="sp-sheet" id="deny-1" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-light">domain</span><h2>bulk.example</h2><button type="button" popovertarget="deny-1" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Note</th><td>newsletter nobody subscribed to</td></tr><tr><th>Added</th><td class="sp-mono sp-small">2026-09-02</td></tr><tr><th>Matched</th><td>41 in the last 90 days</td></tr><tr><th>Last match</th><td class="sp-mono sp-small">2026-09-21 13:57</td></tr></tbody></table></div></div></div></div></td><td class="sp-nowrap">41</td><td class="sp-actions"><form method="post" action="#"><button type="submit" class="has-text-danger">Remove</button></form></td></tr>
|
||||
<tr><td class="sp-mono sp-small">198.51.100.0/24</td><td><button type="button" class="tag is-light" popovertarget="deny-2">ip</button><div class="sp-sheet" id="deny-2" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-light">ip</span><h2>198.51.100.0/24</h2><button type="button" popovertarget="deny-2" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Note</th><td>dictionary attack on lists.example.org</td></tr><tr><th>Added</th><td class="sp-mono sp-small">2026-08-19</td></tr><tr><th>Matched</th><td>1203 in the last 90 days</td></tr><tr><th>Last match</th><td class="sp-mono sp-small">2026-09-21 09:12</td></tr></tbody></table></div></div></div></div></td><td class="sp-nowrap">1203</td><td class="sp-actions"><form method="post" action="#"><button type="submit" class="has-text-danger">Remove</button></form></td></tr>
|
||||
<tr><td class="sp-mono sp-small">win@lottery.example</td><td><button type="button" class="tag is-light" popovertarget="deny-3">address</button><div class="sp-sheet" id="deny-3" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-light">address</span><h2>win@lottery.example</h2><button type="button" popovertarget="deny-3" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Note</th><td>—</td></tr><tr><th>Added</th><td class="sp-mono sp-small">2026-07-30</td></tr><tr><th>Matched</th><td>never in the last 90 days</td></tr></tbody></table></div></div></div></div></td><td class="sp-muted sp-nowrap">never</td><td class="sp-actions"><form method="post" action="#"><button type="submit" class="has-text-danger">Remove</button></form></td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
@@ -50,10 +50,10 @@
|
||||
<div class="box">
|
||||
<div class="sp-box-head"><span class="sp-no">03</span><h2>Allow</h2><div class="sp-end">2 entries</div></div>
|
||||
<div class="table-container"><table class="table is-fullwidth is-hoverable">
|
||||
<thead><tr><th>Value</th><th>Note</th><th>Added</th><th></th></tr></thead>
|
||||
<thead><tr><th>Value</th><th>Kind</th><th>Matched</th><th></th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="sp-mono sp-small">@partner.example <span class="tag is-light">domain</span></td><td class="sp-muted">invoices were scored as spam</td><td class="sp-muted sp-nowrap sp-small">2026-09-11</td><td class="sp-actions"><a class="has-text-danger" href="#">Remove</a></td></tr>
|
||||
<tr><td class="sp-mono sp-small">noreply@github.example <span class="tag is-light">address</span></td><td class="sp-muted">security advisories</td><td class="sp-muted sp-nowrap sp-small">2026-06-04</td><td class="sp-actions"><a class="has-text-danger" href="#">Remove</a></td></tr>
|
||||
<tr><td class="sp-mono sp-small">partner.example</td><td><button type="button" class="tag is-light" popovertarget="allow-1">domain</button><div class="sp-sheet" id="allow-1" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-light">domain</span><h2>partner.example</h2><button type="button" popovertarget="allow-1" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Note</th><td>invoices were scored as spam</td></tr><tr><th>Added</th><td class="sp-mono sp-small">2026-09-11</td></tr><tr><th>Matched</th><td>7 in the last 90 days</td></tr><tr><th>Last match</th><td class="sp-mono sp-small">2026-09-19 16:40</td></tr></tbody></table></div></div></div></div></td><td class="sp-nowrap">7</td><td class="sp-actions"><form method="post" action="#"><button type="submit" class="has-text-danger">Remove</button></form></td></tr>
|
||||
<tr><td class="sp-mono sp-small">noreply@github.example</td><td><button type="button" class="tag is-light" popovertarget="allow-2">address</button><div class="sp-sheet" id="allow-2" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-light">address</span><h2>noreply@github.example</h2><button type="button" popovertarget="allow-2" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Note</th><td>security advisories</td></tr><tr><th>Added</th><td class="sp-mono sp-small">2026-06-04</td></tr><tr><th>Matched</th><td>18 in the last 90 days</td></tr><tr><th>Last match</th><td class="sp-mono sp-small">2026-09-21 13:30</td></tr></tbody></table></div></div></div></div></td><td class="sp-nowrap">18</td><td class="sp-actions"><form method="post" action="#"><button type="submit" class="has-text-danger">Remove</button></form></td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
@@ -64,10 +64,10 @@
|
||||
<div class="sp-box-head"><span class="sp-no">04</span><h2>Where the lists go</h2><div class="sp-end"><span class="tag is-success is-light">applied</span></div></div>
|
||||
<div class="sp-box-body">
|
||||
<dl class="sp-facts">
|
||||
<div><dt>Applied by</dt><dd>rspamd, from map files</dd></div>
|
||||
<div><dt>Files</dt><dd class="sp-mono sp-small">/data/antispam/allow.map<br>/data/antispam/deny.map</dd></div>
|
||||
<div><dt>Applied by</dt><dd>rspamd, from map files it rereads within seconds of a change</dd></div>
|
||||
<div><dt>Files</dt><dd><span class="sp-mono">/data/antispam/</span> — six, one for each list and kind of entry</dd></div>
|
||||
<div><dt>Last written</dt><dd>2026-09-11 08:40 UTC</dd></div>
|
||||
<div><dt>When the filter is off</dt><dd class="sp-muted">The lists are kept and editable, but nothing reads them until <code>INBOUND_ANTISPAM_MILTER</code> is set.</dd></div>
|
||||
<div><dt>When both lists match</dt><dd>Deny wins, whichever entries they are.</dd></div>
|
||||
</dl>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<div class="sp-grow">
|
||||
<p class="sp-kicker">Inbound</p>
|
||||
<h1 class="title is-3">Inbound log</h1>
|
||||
<p class="sp-lead">What port 25 decided about each message, one row per recipient. Accepts come from the inbound journal, rejects from <code>mail.log</code>; the filter's reasons appear when the spam filter is on.</p>
|
||||
<p class="sp-lead">What port 25 decided about each message, one row per recipient. Accepted mail is recorded as it passes, refused mail is read from <code>mail.log</code>; the filter's verdict appears when the spam filter is on. Rows older than 90 days are deleted — change that in <a href="settings.html">Settings</a>.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -17,17 +17,18 @@
|
||||
<input class="input" type="date" name="to" value="2026-09-21" aria-label="To date">
|
||||
<button type="submit" class="button"><i class="ti ti-filter"></i>Filter</button>
|
||||
</form>
|
||||
<a class="sp-help" href="help.html#inbound-log" title="What the decisions mean"><i class="ti ti-help-circle"></i></a>
|
||||
</div>
|
||||
<div class="table-container"><table class="table is-fullwidth is-hoverable">
|
||||
<thead><tr><th>Time, UTC</th><th>From · client</th><th>To</th><th>Subject</th><th>Decision</th><th>Why</th><th></th></tr></thead>
|
||||
<thead><tr><th>Time, UTC</th><th>From</th><th>To</th><th>Subject</th><th>Decision</th><th></th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 14:00:18</td><td>t.berg@<wbr>uni.example<br><span class="sp-mono sp-muted sp-small">192.0.2.77</span></td><td>dev@<wbr>lists.example.org</td><td class="sp-clip">Re: [dev] patch series v3</td><td><span class="tag is-success is-light">accept</span></td><td class="sp-muted sp-small">rspamd · score 0.4</td><td class="sp-actions"><a href="in-filter-lists.html">Deny sender</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:59:02</td><td>invoice@<wbr>partner-billing.example<br><span class="sp-mono sp-muted sp-small">203.0.113.140</span></td><td>owner@<wbr>lists.example.org</td><td class="sp-clip">Outstanding invoice — action required</td><td><span class="tag is-primary is-light">quarantine</span></td><td class="sp-muted sp-small">rspamd · score 9.8 · PHISHING, FORGED_SENDER</td><td class="sp-actions"><a href="in-quarantine-message.html">Review</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:57:41</td><td>promo@<wbr>bulk.example<br><span class="sp-mono sp-muted sp-small">198.51.100.90</span></td><td>sales@<wbr>lists.example.org</td><td class="sp-clip sp-muted">—</td><td><span class="tag is-danger is-light">reject</span></td><td class="sp-muted sp-small">postfix · 550 5.1.1 unknown recipient</td><td class="sp-actions"><a href="in-filter-lists.html">Deny sender</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:51:30</td><td>billing@<wbr>partner.example<br><span class="sp-mono sp-muted sp-small">198.51.100.12</span></td><td>owner@<wbr>lists.example.org</td><td class="sp-clip">Invoice 2026-0917</td><td><span class="tag is-danger is-light">reject</span></td><td class="sp-muted sp-small">rspamd · score 16.2 · BAYES_SPAM</td><td class="sp-actions"><a href="in-filter-lists.html">Allow sender</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:44:02</td><td>hr@<wbr>partner.example<br><span class="sp-mono sp-muted sp-small">192.0.2.31</span></td><td>jobs@<wbr>acme.io</td><td class="sp-clip">Application: backend engineer</td><td><span class="tag is-warning is-light">tempfail</span></td><td class="sp-muted sp-small">selfpost · upstream 10.0.4.12 timed out</td><td class="sp-actions"></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:30:55</td><td>noreply@<wbr>github.example<br><span class="sp-mono sp-muted sp-small">192.0.2.200</span></td><td>security@<wbr>lists.example.org</td><td class="sp-clip">[advisory] new report</td><td><span class="tag is-success is-light">accept</span></td><td class="sp-muted sp-small">rspamd · allow list</td><td class="sp-actions"></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 14:00:18</td><td>t.berg@<wbr>uni.example<br><span class="sp-mono sp-muted sp-small">192.0.2.77</span></td><td>dev@<wbr>lists.example.org</td><td class="sp-clip">Re: [dev] patch series v3</td><td><button type="button" class="tag is-success is-light" popovertarget="why-1">accept</button><div class="sp-sheet" id="why-1" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-success is-light">accept</span><h2>Why</h2><button type="button" popovertarget="why-1" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Decided by</th><td>rspamd</td></tr><tr><th>Verdict</th><td>no action, score 0.40 / 15.00</td></tr><tr><th>Subject</th><td>Re: [dev] patch series v3</td></tr></tbody></table></div></div></div></div></td><td class="sp-actions"><a href="in-filter-lists.html">Deny sender</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:59:02</td><td>invoice@<wbr>partner-billing.example<br><span class="sp-mono sp-muted sp-small">203.0.113.140</span></td><td>owner@<wbr>lists.example.org</td><td class="sp-clip">Outstanding invoice — action required</td><td><button type="button" class="tag is-success is-light" popovertarget="why-2">accept</button><div class="sp-sheet" id="why-2" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-success is-light">accept</span><h2>Why</h2><button type="button" popovertarget="why-2" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Decided by</th><td>rspamd</td></tr><tr><th>Verdict</th><td>add header, score 9.80 / 15.00</td></tr><tr><th>Symbols</th><td class="sp-mono sp-small">PHISHING(5.00)<br>FORGED_SENDER(3.00)<br>R_SPF_FAIL(1.00)</td></tr><tr><th>Subject</th><td>Outstanding invoice — action required</td></tr></tbody></table></div></div></div></div></td><td class="sp-actions"><a href="in-filter-lists.html">Deny sender</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:57:41</td><td>promo@<wbr>bulk.example<br><span class="sp-mono sp-muted sp-small">198.51.100.90</span></td><td>sales@<wbr>lists.example.org</td><td class="sp-clip sp-muted">—</td><td><button type="button" class="tag is-danger is-light" popovertarget="why-3">reject</button><div class="sp-sheet" id="why-3" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-danger is-light">reject</span><h2>Why</h2><button type="button" popovertarget="why-3" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Decided by</th><td>postfix</td></tr><tr><th>Reply</th><td class="sp-mono sp-small">550 5.1.1 <sales@lists.example.org>: Recipient address rejected: User unknown in relay recipient table</td></tr></tbody></table></div></div></div></div></td><td class="sp-actions"></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:51:30</td><td>billing@<wbr>partner.example<br><span class="sp-mono sp-muted sp-small">198.51.100.12</span></td><td>owner@<wbr>lists.example.org</td><td class="sp-clip sp-muted">—</td><td><button type="button" class="tag is-danger is-light" popovertarget="why-4">reject</button><div class="sp-sheet" id="why-4" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-danger is-light">reject</span><h2>Why</h2><button type="button" popovertarget="why-4" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Decided by</th><td>rspamd</td></tr><tr><th>Reply</th><td class="sp-mono sp-small">5.7.1 Spam message rejected</td></tr></tbody></table></div></div></div></div></td><td class="sp-actions"><a href="in-filter-lists.html">Allow sender</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:44:02</td><td>hr@<wbr>partner.example<br><span class="sp-mono sp-muted sp-small">192.0.2.31</span></td><td>jobs@<wbr>acme.io</td><td class="sp-clip sp-muted">—</td><td><button type="button" class="tag is-warning is-light" popovertarget="why-5">tempfail</button><div class="sp-sheet" id="why-5" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-warning is-light">tempfail</span><h2>Why</h2><button type="button" popovertarget="why-5" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Decided by</th><td>rspamd</td></tr><tr><th>Reply</th><td class="sp-mono sp-small">4.7.1 Try again later</td></tr></tbody></table></div></div></div></div></td><td class="sp-actions"><a href="in-filter-lists.html">Allow sender</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:30:55</td><td>noreply@<wbr>github.example<br><span class="sp-mono sp-muted sp-small">192.0.2.200</span></td><td>security@<wbr>lists.example.org</td><td class="sp-clip">[advisory] new report</td><td><button type="button" class="tag is-success is-light" popovertarget="why-6">accept</button><div class="sp-sheet" id="why-6" popover><div class="sp-sheet-in"><div class="sp-sheet-head"><span class="tag is-success is-light">accept</span><h2>Why</h2><button type="button" popovertarget="why-6" popovertargetaction="hide" aria-label="Close"><i class="ti ti-x"></i></button></div><div class="sp-sheet-body"><div class="table-container"><table class="table is-fullwidth"><tbody><tr><th>Decided by</th><td>rspamd</td></tr><tr><th>Verdict</th><td>no action, score -98.10 / 15.00</td></tr><tr><th>Symbols</th><td class="sp-mono sp-small">SELFPOST_ALLOW_FROM(-100.00)<br>R_DKIM_ALLOW(-0.20)</td></tr><tr><th>Subject</th><td>[advisory] new report</td></tr></tbody></table></div></div></div></div></td><td class="sp-actions"><a href="in-filter-lists.html">Deny sender</a></td></tr>
|
||||
</tbody>
|
||||
</table></div>
|
||||
<div class="sp-box-foot"><span>Page 1 of 38</span><a href="#">Older →</a><span class="sp-end">Allow / Deny sender opens <a href="in-filter-lists.html">Filter lists</a> with the entry filled in · global administrator only</span></div>
|
||||
<div class="sp-box-foot"><span>Page 1 of 38</span><a href="#">Older →</a><span class="sp-end">A decision opens its reason · Allow / Deny sender opens <a href="in-filter-lists.html">Filter lists</a> with the entry filled in</span></div>
|
||||
</div>
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 14:01:50</td><td>shop@<wbr>shop.example.org</td><td>anna.lind@<wbr>outlook.com</td><td class="sp-clip">Your order #48212 has shipped</td><td><span class="tag is-success is-light">sent</span></td><td class="sp-actions"><a href="out-message.html">Details</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:58:07</td><td>alerts@<wbr>example.org</td><td>noc@<wbr>example.org</td><td class="sp-clip">[FIRING] disk usage above 90 % on db-2</td><td><span class="tag is-warning is-light">deferred</span></td><td class="sp-actions"><a href="out-message.html">Details</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:41:19</td><td>no-reply@<wbr>notify.acme.io</td><td>j.doe@<wbr>nonexistent.example</td><td class="sp-clip">Reset your password</td><td><span class="tag is-danger is-light">bounced</span></td><td class="sp-actions"><a href="out-message.html">Details</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:40:55</td><td>shop@<wbr>shop.example.org</td><td>k.berg@<wbr>web.de</td><td class="sp-clip">A message from the director</td><td><span class="tag is-danger is-light">wrong From</span></td><td class="sp-actions"><a href="out-message.html">Details</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:40:02</td><td>prod@<wbr>example.org</td><td>billing@<wbr>partner.example</td><td class="sp-clip">Invoice 2026-0917</td><td><span class="tag is-light">rate-limited</span></td><td class="sp-actions"><a href="out-message.html">Details</a></td></tr>
|
||||
<tr><td class="sp-mono sp-muted sp-nowrap sp-small">09-21 13:12:44</td><td>shop@<wbr>shop.example.org</td><td>p.novak@<wbr>seznam.cz</td><td class="sp-clip">We received your return request</td><td><span class="tag is-success is-light">sent</span></td><td class="sp-actions"><a href="out-message.html">Details</a></td></tr>
|
||||
</tbody>
|
||||
|
||||
@@ -16,11 +16,12 @@
|
||||
<div class="field"><label class="label" for="ret">Keep outbound log rows, days</label>
|
||||
<div class="control"><input class="input" id="ret" name="send_log_retention_days" type="number" min="7" max="365" value="30" required></div>
|
||||
<p class="help">The main driver of <code>/data</code> growth. Does not affect <code>mail.log</code> rotation (fourteen daily files).</p></div>
|
||||
<div class="field"><label class="label" for="inret">Keep inbound log rows, days</label>
|
||||
<div class="control"><input class="input" id="inret" name="inbound_spam_log_retention_days" type="number" min="7" max="365" value="90" required></div>
|
||||
<p class="help">The inbound log also never holds more than 10 000 rows: refused mail is logged at the rate strangers knock on port 25.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="column is-6">
|
||||
<div class="box" id="rate-limits">
|
||||
<div class="sp-box-head"><span class="sp-no">02</span><h2>Sending rate limits</h2><div class="sp-end">Read-only</div><a class="sp-help" href="help.html#limits" title="Levels 1 and 2"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-body">
|
||||
@@ -31,6 +32,43 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="column is-6">
|
||||
<div class="box" id="notifications">
|
||||
<div class="sp-box-head"><span class="sp-no">03</span><h2>Notifications</h2><a class="sp-help" href="help.html#notifications" title="What is sent and when"><i class="ti ti-help-circle"></i></a></div>
|
||||
<div class="sp-box-body">
|
||||
<div class="field"><label class="checkbox"><input type="checkbox" name="notify" checked> Send notification e-mails</label>
|
||||
<p class="help">From <span class="sp-mono">postmaster@mail.example.org</span>, to every user who has an e-mail and has not turned them off under <a href="account.html">Account</a>.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="box" id="notification-mail">
|
||||
<div class="sp-box-head"><span class="sp-no">04</span><h2>Notification mail</h2><div class="sp-end">DNS records of the server's own name</div><a class="sp-help" href="help.html#notifications" title="What the panel sends and what makes it arrive"><i class="ti ti-help-circle"></i></a></div>
|
||||
|
||||
<div class="sp-record">
|
||||
<div class="sp-record-head"><h3>DKIM</h3><span class="tag is-success is-light">published</span><small>Not a secret · selector <code>sp2026</code></small></div>
|
||||
<div class="sp-row">
|
||||
<div class="field"><label class="label">Host / name</label>
|
||||
<div class="field has-addons"><div class="control is-expanded"><input class="input" readonly value="sp2026._domainkey.mail.example.org"></div><div class="control"><button type="button" class="button"><i class="ti ti-copy"></i>Copy</button></div></div></div>
|
||||
<div class="field"><label class="label">Type</label><div class="control"><input class="input" readonly value="TXT"></div></div>
|
||||
</div>
|
||||
<div class="field"><label class="label">Value to publish</label>
|
||||
<div class="field has-addons"><div class="control is-expanded"><textarea class="textarea" rows="5" readonly>v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAq7n2Lw0c5vXeR1dYt8uJb3mKs6pHf9aZoWg4iNy2TqCx0lVjE7rUh5sBk1dMa8fPz3oGc6yXw9tIe4nQv2uL…IDAQAB</textarea></div><div class="control"><button type="button" class="button"><i class="ti ti-copy"></i>Copy</button></div></div></div>
|
||||
</div>
|
||||
|
||||
<div class="sp-record">
|
||||
<div class="sp-record-head"><h3>SPF</h3><span class="tag is-warning is-light">missing</span><small>For the server's own name, not for your domains</small></div>
|
||||
<div class="sp-row">
|
||||
<div class="field"><label class="label">Host / name</label>
|
||||
<div class="field has-addons"><div class="control is-expanded"><input class="input" readonly value="mail.example.org"></div><div class="control"><button type="button" class="button"><i class="ti ti-copy"></i>Copy</button></div></div></div>
|
||||
<div class="field"><label class="label">Type</label><div class="control"><input class="input" readonly value="TXT"></div></div>
|
||||
</div>
|
||||
<div class="field"><label class="label">Value to publish</label>
|
||||
<div class="field has-addons"><div class="control is-expanded"><input class="input" readonly value="v=spf1 a -all"></div><div class="control"><button type="button" class="button"><i class="ti ti-copy"></i>Copy</button></div></div></div>
|
||||
<p class="help">Notifications are sent without it, and a receiver may file them as spam.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="buttons"><button type="submit" class="button is-primary">Save settings</button></div>
|
||||
</form>
|
||||
|
||||
@@ -269,4 +269,44 @@ code { background: var(--sp-well); color: var(--sp-ink); border-radius: 3px; pad
|
||||
.sp-record-head { flex-wrap: wrap; }
|
||||
.sp-record-head small { margin-left: 0; text-align: left; flex-basis: 100%; }
|
||||
.sp-postmark { display: none; }
|
||||
.sp-cards { display: flex; flex-direction: column; padding-inline: .75rem; }
|
||||
.sp-cards > .column { display: contents; }
|
||||
.sp-cards .box { margin-bottom: 1.5rem; }
|
||||
.sp-cards > .column:first-child > .box:nth-child(1) { order: 1; }
|
||||
.sp-cards > .column:first-child > .box:nth-child(2) { order: 3; }
|
||||
.sp-cards > .column:first-child > .box:nth-child(3) { order: 5; }
|
||||
.sp-cards > .column:first-child > .box:nth-child(4) { order: 7; }
|
||||
.sp-cards > .column:last-child > .box:nth-child(1) { order: 2; }
|
||||
.sp-cards > .column:last-child > .box:nth-child(2) { order: 4; }
|
||||
.sp-cards > .column:last-child > .box:nth-child(3) { order: 6; }
|
||||
.sp-cards > .column:last-child > .box:nth-child(4) { order: 8; }
|
||||
}
|
||||
|
||||
/* cards: two stacks of boxes that are read in rows - the odd ones on the left,
|
||||
the even ones on the right, each stack closed up so a short box leaves no
|
||||
gap beside a tall one. On a phone the stacks dissolve and the boxes come in
|
||||
the order of their numbers (the rules above). */
|
||||
|
||||
/* sheet: what does not fit a table row, opened over the page from the row's
|
||||
tag. It is the browser's own popover (the tag is a button with
|
||||
popovertarget), so it needs no script: Esc, a click outside and the close
|
||||
button all shut it, and focus returns to the tag. It rises from the bottom
|
||||
edge across the whole window - the one component that is not laid out by the
|
||||
page, hence the widths - with its content, a table of name and value, in a
|
||||
column of reading width in the middle. */
|
||||
.sp-sheet {
|
||||
inset: auto 0 0 0; width: auto; margin: 0; padding: 0; border: 0; border-top: 3px solid var(--sp-brick);
|
||||
max-height: 75vh; overflow: auto; background: #fff; color: inherit; box-shadow: 0 -12px 40px rgba(40, 20, 15, .22);
|
||||
}
|
||||
.sp-sheet-in { max-width: 44rem; margin: 0 auto; padding: 0 1.25rem; }
|
||||
.sp-sheet::backdrop { background: rgba(40, 20, 15, .38); }
|
||||
.sp-sheet-head { display: flex; align-items: center; gap: .7rem; padding: 1rem 0 .8rem; border-bottom: 1px solid var(--sp-rule); }
|
||||
.sp-sheet-head h2 { font-size: 1.15rem; font-weight: 600; margin-right: auto; }
|
||||
.sp-sheet-head button { border: 0; background: none; cursor: pointer; font-size: 1.3rem; color: var(--sp-muted); }
|
||||
.sp-sheet-body { padding: .4rem 0 1.4rem; }
|
||||
/* The sheet's content is a table of name and value: the name is a label, set like the dt of facts. */
|
||||
.sp-sheet .table th { font-family: var(--sp-mono); font-size: var(--sp-fs-xs); font-weight: 500; letter-spacing: .08em; text-transform: uppercase; color: var(--sp-muted); white-space: nowrap; vertical-align: top; width: 1%; padding-right: 2rem; }
|
||||
.sp-sheet .table td { overflow-wrap: anywhere; }
|
||||
/* A tag that opens a sheet is a button: it keeps the tag's look and says it can
|
||||
be pressed with the dotted underline links to more detail carry. */
|
||||
button.tag { border: 0; cursor: pointer; text-decoration: underline dotted; text-underline-offset: 3px; }
|
||||
|
||||
@@ -0,0 +1,954 @@
|
||||
# Code audit — 2026-10-12
|
||||
|
||||
**What this is.** A full-tree audit of the code and infrastructure at `8f226bd`
|
||||
(release `2.4.0`): leftovers, unfinished work, defects, performance, doc drift
|
||||
and test gaps — explicit ones (there are no `TODO`/`FIXME` markers anywhere in
|
||||
the product code) and the implicit ones that only reading finds. The panel's
|
||||
visual design (mockups, `panel.css`, layout) is a separate audit and is not
|
||||
touched here. Nothing was changed in the tree; this file is the only output.
|
||||
|
||||
**Method.** Six parallel readers, one per zone (mail path; background roles;
|
||||
store/backup/composition root; HTTP surface; view and services; build/deploy/
|
||||
CI/e2e), each reading every file of its zone and its tests in full, each
|
||||
told to find where the code does not do what `architecture.md` says, and to
|
||||
prove it (a throwaway test, `EXPLAIN QUERY PLAN`, a probe) rather than guess.
|
||||
Machine checks: `go build`, `go vet`, `gofmt -l`, `go mod tidy`/`verify`,
|
||||
`staticcheck`, `govulncheck`, `go test ./...`. Five claims the readers could
|
||||
not settle from source were settled on the test server
|
||||
(`selfpost.mixfed.ru`, running a locally built `2.3.0-post.c817f08`) and
|
||||
against GHCR. Two hypotheses were refuted there and are **not** findings:
|
||||
`saslpasswd2 -d` of a missing login exits 0 (deletion is idempotent, as the
|
||||
code says), and a duplicate key in a `texthash:` map is a warning with the
|
||||
first entry winning, not a fatal error.
|
||||
|
||||
**Machine results.** Build, vet, gofmt, tidy, verify: clean. `staticcheck`:
|
||||
four items (an unused `domainIDSet` in `handlers/authz.go:63`, two unused
|
||||
assignments in `handlers_outbound_pages_test.go:174,194`, one `S1003` in
|
||||
`view/pages_test.go:170`). `govulncheck`: nine standard-library findings
|
||||
(`net/http` HTTP/2, `net/textproto`, `html/template`), all fixed in the
|
||||
current Go patch release; `golang.org/x/crypto` `0.54.0` has three published
|
||||
fixes in `0.56.0` that this code does not call. `modernc.org/sqlite` is seven
|
||||
minor versions behind (`1.53.0` → `1.60.1`). `go test ./...`: green on Linux
|
||||
semantics; on Windows two tests fail for OS reasons (`TestWriteLoadPrivateKeyRoundtrip`
|
||||
expects a POSIX mode; `TestFollowTailsAndRotates` renames an open file) — the
|
||||
tests, not the product, are platform-bound.
|
||||
|
||||
Severity: **critical** — mail or data lost or the security boundary crossed
|
||||
today; **high** — a documented guarantee is false in a way an operator will
|
||||
hit; **medium** — wrong under a realistic condition, or a stated guard that is
|
||||
not there; **low** — wrong under an unlikely condition, or cost without
|
||||
failure; **nit** — hygiene. Confidence is stated where a finding was not
|
||||
proven.
|
||||
|
||||
---
|
||||
|
||||
## Findings
|
||||
|
||||
### High
|
||||
|
||||
**H1. Every published release image is stamped `VERSION=e2e`, so the
|
||||
backup/restore version check never checks anything.**
|
||||
[release.yml:75-84](../.github/workflows/release.yml) builds the image with
|
||||
`VERSION=X.Y.Z` and tags it `selfpost:e2e`; the e2e step then runs
|
||||
`docker compose build` through
|
||||
[test/e2e/compose.override.yml:23-32](../test/e2e/compose.override.yml),
|
||||
which rebuilds **the same tag** with `args: VERSION: e2e`; `docker tag
|
||||
selfpost:e2e …X.Y.Z-amd64` pushes the rebuilt one. Confirmed: `docker run
|
||||
ghcr.io/mixeme/selfpost:1.3.0 panel -version` prints `e2e`. Consequences:
|
||||
the footer and `-version` lie; `backup.CheckRestore` compares a manifest's
|
||||
version to `e2e`, so a backup of any release restores into any other and the
|
||||
promise in development.md:270 and guide.md:191 ("the version is the single
|
||||
source … so they cannot drift apart") is false. Every image this workflow has
|
||||
ever published is affected; they stay as they are (owner, 2026-10-12: old tags
|
||||
and images are history and are not touched) — the fix is for the next
|
||||
publication.
|
||||
|
||||
**H2. Renewed TLS certificates are never picked up while the container
|
||||
runs.** [postfix-config.sh:75-89](../build/postfix-config.sh) copies the
|
||||
mounted cert and key into `/etc/postfix/tls-internal/` at start and points
|
||||
Postfix at the copy; [postfix-cert-reload.sh](../build/postfix-cert-reload.sh)
|
||||
runs `postfix reload` daily but nothing re-copies. The copy is the only
|
||||
`tls-internal` reference in the tree; on the test server the copy carries the
|
||||
container's start time. Scenario: Let's Encrypt renews at day 60, nobody
|
||||
restarts, from day 90 ports 465/587 serve an expired certificate.
|
||||
architecture.md:33,154-155 and supervisord.conf:92-95 say the opposite.
|
||||
|
||||
**H3. The queue-reconcile sweep marks delivered mail `bounced` whenever the
|
||||
tailer is blind.** Four mechanisms, one consequence (a row the log would have
|
||||
resolved as `sent` is closed as a failure, which the Outbound log shows and
|
||||
the notifier believes):
|
||||
- *Tailer stuck, sweep running.* [logtail.go:559-581](../internal/logtail/logtail.go):
|
||||
`caughtUp` closes after the first drain and never re-opens; `reconcileLoop`
|
||||
(:174-191) then runs every 5 min regardless. If `openAt` fails after a
|
||||
rotation (:573-576, e.g. postlogd recreated the file `0600` — entrypoint.sh:132
|
||||
says this happens) the loop logs and retries forever on the old inode while
|
||||
every new delivery line goes unread; two minutes later `reconcile()` finds the
|
||||
rows absent from `postqueue -p` (they were delivered) and bounces them.
|
||||
- *Truncated queue listing read as complete.*
|
||||
[postfix/queue.go:64-68,81-87](../internal/postfix/queue.go): `Queue()` caps
|
||||
`postqueue -p` at 4 MiB and appends a text note; `QueueIDs()` parses that
|
||||
text and returns the first 4 MiB of ids with no error. A deferred backlog of
|
||||
~30k messages — the outage the sweep exists for — makes every row past the
|
||||
cut `bounced` while Postfix still holds the mail. `queuejson.go:48-52` handles
|
||||
the same cap correctly for the notifier; the two drifted.
|
||||
- *Rotation drain window.* :567-578 drains the renamed inode once more; postlogd
|
||||
keeps writing to it until master's reload reaches it (seconds), so lines
|
||||
written in that window are never read. architecture.md:372-373 claims the
|
||||
drain covers it.
|
||||
- *No status guard on close.* `UpdateStatus` ([store/sendlog.go:86-98](../internal/store/sendlog.go))
|
||||
is `WHERE queue_id = ? AND to_addr = ?`; the reconcile path can overwrite a
|
||||
`sent` written an instant earlier, and a reused short queue id to the same
|
||||
recipient rewrites a months-old row.
|
||||
|
||||
**H4. Application logins are case-sensitive in the panel and case-insensitive
|
||||
in Postfix; a tenant can take another tenant's envelope sender.**
|
||||
[app/validate.go:35-48](../internal/app/validate.go) accepts `A-Z` and nothing
|
||||
lowercases a login; `applications.login` is `UNIQUE` under BINARY collation
|
||||
([0001_init.sql:91](../internal/store/migrations/0001_init.sql)), so `Ops` and
|
||||
`ops` are two applications. The sender map is written as stored
|
||||
([postfix/postfix.go:95-115](../internal/postfix/postfix.go)); Postfix's
|
||||
`reject_sender_login_mismatch` compares logins without regard to case (the
|
||||
code's own comment, [milter/sender.go:35-36](../internal/milter/sender.go),
|
||||
and architecture.md:239-240 say so). Scenario: domain A has `Ops` bound to
|
||||
`@a.com`; a *domain*-role administrator of B creates `ops` in B, authenticates
|
||||
as `ops`, sends `MAIL FROM:<x@a.com>` with `From: y@b.com`. Postfix accepts
|
||||
the envelope (`@a.com Ops` matches), the sender check judges only the header,
|
||||
which is B's — the message leaves with A as its SPF and bounce identity.
|
||||
Confidence: the panel side is confirmed; Postfix's compare is taken from the
|
||||
code's own statement. Fix shape: lower-case at validation, `COLLATE NOCASE`
|
||||
on the column (a migration — check the live registry for a pair first).
|
||||
|
||||
### Medium
|
||||
|
||||
Persistence, restore, backup
|
||||
|
||||
**M1. The restore manifest is consumed before anything that can fail.**
|
||||
[backup.go:436-446](../internal/backup/backup.go) deletes `manifest.json`
|
||||
inside `CheckRestore`; [main.go:282-300](../cmd/panel/main.go) runs
|
||||
`store.Open` and `resyncAfterRestore` afterwards. If either fails, supervisord
|
||||
restarts the panel with `restored == false` and the one-time drift-healing
|
||||
Resync never runs. Docs say "on the first *successful* boot"; the code says
|
||||
"on the first attempt". `restore_test.go` passes `testNoReload=true` and never
|
||||
fails Open/Resync, so nothing covers it.
|
||||
|
||||
**M2. Three queries on the single connection run without the index they
|
||||
need.** Measured with `EXPLAIN QUERY PLAN` against the real migrations:
|
||||
- `CountMessages` by domain ([ratelimits.go:186-190](../internal/store/ratelimits.go))
|
||||
and `sendStats` ([stats.go:77-94](../internal/store/stats.go)) use
|
||||
`idx_send_log_domain (domain)` and apply `created_at >=` by scanning every
|
||||
row of the domain over the retention window — on every `MAIL FROM` of an
|
||||
application under a domain-level limit, with the milter, tailer and HTTP
|
||||
queued behind it. The baseline's comment promises a composite index for
|
||||
"the journal milter … on every MAIL FROM"; only the `app_login` half exists.
|
||||
- `ListQueuedOlderThan` ([sendlog.go:117-122](../internal/store/sendlog.go))
|
||||
is a full `SCAN send_log` every 5 min (no index on `status`; the cutoff
|
||||
matches nearly every row). Mirror of what `0005` fixed for `rejected`: a
|
||||
partial index `(created_at) WHERE status = 'queued'` makes it a handful of
|
||||
rows.
|
||||
- Scoped `QuerySendLog` ([sendlog.go:214-219](../internal/store/sendlog.go))
|
||||
sorts a domain's whole scope with a temp b-tree for every page a domain user
|
||||
opens.
|
||||
|
||||
**M3. `selfpost-backup` with a wrong `SELFPOST_DB_PATH` creates an empty
|
||||
database and ships a "valid" backup of nothing.** [backup.go:394-405](../internal/backup/backup.go)
|
||||
opens the source without `mode=ro` or an existence check; `VACUUM INTO` on a
|
||||
non-existent path succeeds (measured), creates the file on disk and snapshots a
|
||||
4 KiB empty database; `Create` and `Check` pass. A restore of it opens the
|
||||
`/setup` link on a host that still carries live DKIM keys and `sasldb2`.
|
||||
|
||||
**M4. `secretfile`'s KDF cap does not do what its comment says.**
|
||||
[secretfile.go:108-113,184](../internal/secretfile/secretfile.go):
|
||||
`maxLogN=20, maxR=32, maxP=16` lets a 44-byte header demand
|
||||
`128·2²⁰·32` = 4 GiB and sixteen times the CPU, under a comment that says the
|
||||
cap exists "so a hostile header cannot make the reader allocate its way to an
|
||||
out-of-memory kill". The domain-import form calls `NewReader` on the upload.
|
||||
Operator foot-gun rather than attack surface (global role only), but the
|
||||
stated guarantee is false; the real writer never exceeds N=2¹⁵, r=8, p=1.
|
||||
|
||||
**M5. The CLI enforces no password floor for an encrypted backup.**
|
||||
[cmd/selfpost-backup/main.go:170-187](../cmd/selfpost-backup/main.go) takes
|
||||
any non-empty string; the panel enforces `validate.MinSecretFilePasswordLen`
|
||||
(12); security.md:291 says the minimum applies to these files. (The roadmap's
|
||||
"empty password accepted" item: an empty password is *not* accepted as an
|
||||
encryption password — empty means plain output by design; the gap is the
|
||||
floor.)
|
||||
|
||||
**M6. The compose pin, README and guide point at an image that does not
|
||||
exist.** `deploy/docker-compose.yml:24`, `README.md:115` and guide.md:187 pin
|
||||
`ghcr.io/mixeme/selfpost:2.4.0` (`2.4.1` since this cut); GHCR holds `1.0.0`
|
||||
and `1.3.0` only (checked
|
||||
through the Packages API and by pulling); no 2.x was ever published, and tag
|
||||
`v2.0.0` exists in git without a release. The quick start cannot work for
|
||||
anyone but the owner, who runs locally built `-post.<sha>` images. Decided
|
||||
(owner, 2026-10-12): the old tags and images are history and stay untouched;
|
||||
the pin points forward — it is made true by the next publication, after H1 is
|
||||
fixed, and until then the documents should not claim an image that is not
|
||||
there.
|
||||
|
||||
HTTP surface
|
||||
|
||||
**M7. The last-global-user guard is check-then-act without a transaction.**
|
||||
[store/users.go:315-341](../internal/store/users.go) (`DeleteUser`:
|
||||
`CountGlobalUsers` then `DELETE`), [handlers_users.go:277-287,317-322](../internal/web/handlers/handlers_users.go)
|
||||
(demotion), [users.go:80-97](../internal/store/users.go) (`CreateGlobalUser`:
|
||||
`UserExists` then `INSERT`). `MaxOpenConns(1)` serialises statements, not
|
||||
pairs. Two globals deleting or demoting each other at once leave no global;
|
||||
`/server/*` answers 404 to everyone and recovery is database surgery.
|
||||
security.md:155 promises the guard.
|
||||
|
||||
**M8. Login discloses whether a username exists by timing.**
|
||||
[auth/handlers.go:109-121](../internal/web/auth/handlers.go): unknown user
|
||||
returns at :114 without bcrypt; a known one costs ~250 ms (cost 12). The test
|
||||
compares bodies only. Rate limiting slows enumeration; a dummy compare stops
|
||||
it.
|
||||
|
||||
**M9. `submitUserUpdate` is five writes with no transaction.**
|
||||
[handlers_users.go:304-336](../internal/web/handlers/handlers_users.go):
|
||||
`UpdateUser` → `endSessionsAfterPasswordSet` → `SetUserRole` →
|
||||
`ClearUserReach` (error only logged) → `SetUserReach`. A failure after the
|
||||
first leaves a user renamed and signed out everywhere with the old role, or a
|
||||
`domain` user with empty reach — the state the "Assign at least one domain"
|
||||
check exists to prevent.
|
||||
|
||||
**M10. Domain import trusts the export file for everything but the domain
|
||||
name, selector and password.** [domain/transfer.go:145-190](../internal/domain/transfer.go):
|
||||
`DMARCRua` is not run through `validate.Email` (the form path is), client IPs
|
||||
not through `parseIPList`, rate-limit `Mode`/`WindowSeconds`/`MaxMessages`
|
||||
not through the form's parsers. `WindowSeconds: 0` yields a limit that counts
|
||||
nothing ([milter/ratelimit.go:61](../internal/milter/ratelimit.go)); garbage
|
||||
IPs make the imported application refuse every client with no word at import.
|
||||
[app/validate.go:50-55](../internal/app/validate.go) calls the file "untrusted
|
||||
input" — for the password only.
|
||||
|
||||
**M11. Polling stops for good after one failed refresh.**
|
||||
[panel.js:202-257](../internal/web/view/static/panel.js): the scheduled poll
|
||||
is `htmx.ajax(...)` without `source`, so htmx attributes it to `document.body`;
|
||||
`htmx:responseError` therefore reaches an element with no `data-poll` and
|
||||
nothing reschedules; `htmx:sendError` and `htmx:timeout` are not handled at
|
||||
all. A panel restart while Overview or Health is open freezes the page until
|
||||
the tab is hidden and shown. Related: an expired session answers an
|
||||
`HX-Request` with `303 /login` ([auth/middleware.go:14](../internal/web/auth/middleware.go));
|
||||
the XHR follows it and the sign-in document is swapped into the box — no
|
||||
`HX-Redirect` on either side.
|
||||
|
||||
Log tailer and checks
|
||||
|
||||
**M12. `deliveryRe` is unanchored; a stranger's HELO parses as a delivery
|
||||
result.** [logtail.go:85](../internal/logtail/logtail.go): a port-25 reject
|
||||
line ending `helo=<41E862C00D9E: to=<victim@…>, status=sent>` yields
|
||||
`(id, recipient, sent)` (probed), and `UpdateStatus` rewrites the matching row.
|
||||
The attacker must guess an id+recipient pair, so impact is bounded — but
|
||||
`lines.go:21-24` and `inbound.go:48-51` anchor at the head for exactly this
|
||||
reason, and this pattern, the one that writes status, does not.
|
||||
|
||||
**M13. The sender-check deferred counter misses the case it is documented to
|
||||
see.** [sendercheck.go:50](../internal/logtail/sendercheck.go) matches only
|
||||
`postfix/…smtpd[pid]`; header/EOH events run in cleanup(8), so the warning for
|
||||
a panel that accepts the socket but hangs at end-of-headers is logged by
|
||||
`postfix/cleanup`. The fixture at `sendercheck_test.go:64-68` puts those lines
|
||||
under `smtpd` and says it was "composed from Postfix's source rather than
|
||||
seen". architecture.md:286-287 claims the figure sees "a panel that is running
|
||||
and not answering". Confidence: plausible; one `kill -STOP` on the stand
|
||||
settles it.
|
||||
|
||||
**M14. Resolver fall-back is hollow when the first resolver drops packets.**
|
||||
[dnscheck/resolver.go:72-90](../internal/dnscheck/resolver.go) tries resolvers
|
||||
in turn under one 5 s context; a blackhole eats the budget and the second is
|
||||
asked with an expired context. The `DefaultResolvers` comment promises
|
||||
otherwise. **M15.** [dnscheck/server.go:26-33,42-46](../internal/dnscheck/server.go)
|
||||
turns a resolver timeout into "does not resolve" / "no PTR record" (every
|
||||
other check routes transient errors to Unknown), and empties `ServerIPs` so
|
||||
the SPF check goes Unknown in the same minute. architecture.md:339-340.
|
||||
|
||||
Infrastructure
|
||||
|
||||
**M16. `envdoc_test.go` compares three hand-written lists to each other and
|
||||
reads neither `main.go`, the scripts nor `guide.md`.**
|
||||
[envdoc_test.go:70-136](../cmd/panel/envdoc_test.go). Already drifted:
|
||||
`ANTISPAM_DIR`, `INBOUND_JOURNAL_MILTER_SOCKET`, `INBOUND_SPAM_LOG_RETENTION_DAYS`,
|
||||
`RATE_LIMIT_MESSAGES_PER_IP`, `RATE_LIMIT_WINDOW_SECONDS` are read by
|
||||
`loadConfig` and absent from `loadConfigKeys`; `postfix-config.sh` and
|
||||
`postfix-wrapper.sh` read `INBOUND_JOURNAL_MILTER_SOCKET`, `SELFPOST_DATA_DIR`,
|
||||
`SELFPOST_DB_PATH`, `INBOUND_RELAY_ENABLE`, absent from `buildScriptKeys`. The
|
||||
tests pass; development.md:353-357 and :503-504 describe a guard that does
|
||||
not exist.
|
||||
|
||||
**M17. The supervisord control socket is `0770 root:selfpost` and the
|
||||
`postfix` and `opendkim` users are in `selfpost`.**
|
||||
[supervisord.conf:26-29](../build/supervisord.conf) ("only root and the two
|
||||
SelfPost services"), [Dockerfile:68-71](../build/Dockerfile) (three users);
|
||||
confirmed on the server. A compromised smtpd can `supervisorctl stop panel`
|
||||
(all submission deferred), `stop opendkim` or `shutdown`. No escalation; a
|
||||
whole-stack denial from the least trusted process.
|
||||
|
||||
**M18. DMARC maps are never ownership-normalised.** entrypoint.sh:108-125
|
||||
normalises five maps; `dmarc_recipients`/`dmarc_transport`/`dmarc_relay_domains`
|
||||
are created by postfix-config.sh:315-317 as root only when missing. A tar
|
||||
restore as root (guide, methods 3/4) yields `root:root 0640`, which smtpd
|
||||
cannot open → `451 table lookup problem` on every report until the panel next
|
||||
rewrites them. Confidence: plausible, needs a run.
|
||||
|
||||
**M19. logrotate state lives in the container layer.** [logrotate-loop.sh:42](../build/logrotate-loop.sh)
|
||||
uses the default `/var/lib/logrotate/status`, not under `/data`. On the server
|
||||
the state file is dated the container's start; with no entry logrotate records
|
||||
today and does not rotate, so a host that recreates the container daily
|
||||
(`.env` change, image bump) never rotates `mail.log`. The e2e uses `-f` and
|
||||
cannot see it.
|
||||
|
||||
**M20. `test.yml` never compiles `test/e2e`.** A type error there surfaces in
|
||||
`release.yml` after the GitHub Release is published and the tag checked out.
|
||||
|
||||
**M21. `deploy/traefik/extract-cert.sh` writes empty PEM files and exits 0 on
|
||||
a `jq` miss** (:17-25, `sh` without pipefail; `jq` exits 0 on no match); the
|
||||
files are `0644` until :27. Postfix TLS breaks at the next restart.
|
||||
|
||||
### Low
|
||||
|
||||
- **L1** `-decrypt -o FILE` truncates the output before the password is
|
||||
verified ([main.go:132-148](../cmd/selfpost-backup/main.go)); a wrong
|
||||
password leaves an empty file where a good one may have been.
|
||||
- **L2** Backups skip symlinks silently and `Check` does not notice
|
||||
([backup.go:323-327](../internal/backup/backup.go)); a `certs/` of symlinks
|
||||
(the Let's Encrypt layout) yields an archive with no TLS material while the
|
||||
guide promises refusal.
|
||||
- **L3** Trailing bytes after the last `secretfile` chunk are ignored
|
||||
(:400-416): two envelopes concatenated open as the first.
|
||||
- **L4** `parseTrustedProxies` turns a bare `::ffff:10.0.0.1` into `::/32`
|
||||
silently ([main.go:247-253](../cmd/panel/main.go)).
|
||||
- **L5** `tls_policy` is keyed by next-hop; two inbound domains to one
|
||||
`[host]:port` with different TLS modes write duplicate keys and the first
|
||||
(by domain order) wins with a warning — confirmed on the server. The per-
|
||||
domain TLS mode in the form is not what Postfix does
|
||||
([postfix/inbound.go:69-72](../internal/postfix/inbound.go)).
|
||||
- **L6** Multi-file map rebuilds are not all-or-nothing
|
||||
([inbound.go:42-55](../internal/postfix/inbound.go), `dmarc.go:31-41`); a
|
||||
write failure mid-sequence leaves `relay_domains` ahead of `transport`.
|
||||
- **L7** `atomicfile.Write` never fsyncs the directory and the package has no
|
||||
tests ([atomicfile.go:40-44](../internal/atomicfile/atomicfile.go)); every map
|
||||
writer goes through it.
|
||||
- **L8** The refusal reply names the other sender's domain
|
||||
([sender.go:334](../internal/milter/sender.go)) — an oracle for the server's
|
||||
signing names, available to any authenticated application.
|
||||
- **L9** go-milter accumulates every message's headers for a connection's
|
||||
life (library `session.go:173-187`, cleared on ABORT only); the sender check
|
||||
is safe because it ignores `h`, and the test passes `nil` so would not catch
|
||||
a "simplification" to `h.Values("From")`.
|
||||
- **L10** `parsePage` is not clamped to `lastPage` for the Outbound log
|
||||
([handlers_monitor.go:357-371](../internal/web/handlers/handlers_monitor.go));
|
||||
the inbound log clamps. Known roadmap row, confirmed.
|
||||
- **L11** `parseAutoMultiplier` accepts `NaN` ([handlers_ratelimit.go:44-49](../internal/web/handlers/handlers_ratelimit.go));
|
||||
SQLite stores NULL, read-back is 0, recalc silently uses 2.5 (probed).
|
||||
- **L12** Outbound rows belong to a domain by **name** ([handlers_monitor.go:97,345](../internal/web/handlers/handlers_monitor.go)):
|
||||
a domain deleted and re-added shows its earlier rows to the new assignee.
|
||||
security.md:419 accepts this for the inbound log only.
|
||||
- **L13** `HandleReload` answers failures as plain-text `http.Error`
|
||||
(handlers_domains.go:173-184), and the flash "Configuration regenerated from
|
||||
the database" oversells: DMARC maps and filter-list files are skipped
|
||||
(architecture.md:856-859 says so for DMARC; `Lists.Sync` is documented
|
||||
nowhere).
|
||||
- **L14** `http.Server` has only `ReadHeaderTimeout` ([httpserver.go:145-149](../cmd/panel/httpserver.go)).
|
||||
- **L15** The Account password change shares the per-IP login bucket
|
||||
([handlers_account.go:142](../internal/web/handlers/handlers_account.go)):
|
||||
ten wrong current passwords lock sign-in for that IP 15 min.
|
||||
- **L16** Login issues a fresh token but does not destroy the session a
|
||||
presented cookie already holds ([auth/handlers.go:123-130](../internal/web/auth/handlers.go)).
|
||||
- **L17** `domain.Service.Delete` returns a bare error after the row is gone
|
||||
when `resync()` fails ([service.go:202-210](../internal/domain/service.go));
|
||||
`apps.Resync()` is not reached, so the sender map keeps the purged logins.
|
||||
- **L18** A `rate-limited` refusal is a grey tag in the Outbound log
|
||||
([outbound_log.go:213](../internal/web/view/outbound_log.go), no such status
|
||||
in `components.go:132-146`); its two siblings are red, and the test pins the
|
||||
grey.
|
||||
- **L19** *Recalculate rate limit* on the application form submits the whole
|
||||
form and discards every unsaved edit ([out_app.html:67](../internal/web/view/templates/out_app.html),
|
||||
handlers_ratelimit.go:178-189).
|
||||
- **L20** DMARC parse rewrites `<pct>0</pct>` to 100 and stores a missing
|
||||
`date_range` as 1970 ([parse.go:125-136](../internal/dmarc/parse.go)).
|
||||
- **L21** Several `mailStack` instances per process with no shared lock on
|
||||
table rebuilds (`run`, `newPanel`, `runNotifier`); the notifier's
|
||||
`EnsureServerSigned` at start can write a stale table over a domain added
|
||||
in the same second. **L22** `IncrDMARCParseFailures` is read-modify-write
|
||||
across concurrent `dmarc-ingest` processes ([dmarc.go:163-173](../internal/store/dmarc.go)).
|
||||
- **L23** The restore-boot Resync reloads Postfix before Postfix exists
|
||||
(panel priority 200, postfix 300) — an error line per restore boot; the
|
||||
DMARC bootstrap works around the same thing with a no-op hook and does not
|
||||
say why ([main.go:301-307](../cmd/panel/main.go)).
|
||||
- **L24** `health/processes.go:43`, `supervisor.go:17,30`: `exec.Command`
|
||||
without context; `/healthz` and the 5 s status poll hang on a wedged
|
||||
supervisord. **L25** `health/cert.go:70-83` assumes the first PEM block is
|
||||
the leaf and grades a not-yet-valid certificate OK; `machine.go:193-204`
|
||||
advances `prevAt` when only one of cpu/net succeeded.
|
||||
- **L26** Only successful sends count toward the notifier's 60/h ceiling
|
||||
([notify.go:196-207](../internal/notify/notify.go)); with sendmail broken,
|
||||
`CheckLimits` spawns a 20 s process per recipient per minute forever.
|
||||
- **L27** `v=dmarc1` matched case-insensitively ([records.go:74,121](../internal/dnscheck/records.go));
|
||||
RFC 7489 §6.4 says the version must match exactly, so a lower-case record is
|
||||
graded OK and ignored by receivers. **L28** The domain cache is keyed by
|
||||
name only ([dnscheck.go:155-173](../internal/dnscheck/dnscheck.go)) though
|
||||
the verdict depends on `ExpectedDKIM`/`ServerIPs`.
|
||||
- **L29** `lineTime` normalises a Feb 29 line read the following January to
|
||||
Mar 1 ([inbound.go:254-261](../internal/logtail/inbound.go)); `isoStampRe`
|
||||
accepts a zone-less stamp that `time.Parse` then rejects (dated `now`).
|
||||
- **L30** OpenDKIM: `Syslog no` with no `LogWhy`/`Logfile` — it logs nothing
|
||||
anywhere; no `Canonicalization relaxed/simple`; no `OversignHeaders From`
|
||||
(the header-append replay is the class 2.4.0 is about).
|
||||
- **L31** Postfix: `smtpd_tls_protocols` at the 3.7 default `>=TLSv1`,
|
||||
`disable_vrfy_command = no` (VRFY on 25 enumerates `relay_recipient_maps`),
|
||||
the stock `(Debian/GNU)` banner — all confirmed on the server.
|
||||
`RATE_LIMIT_*`, `*_SIZE_LIMIT`, `MILTER_*_TIMEOUT` reach `postconf -e`
|
||||
unvalidated; a non-integer passes `postfix check` and kills every smtpd at
|
||||
connect.
|
||||
- **L32** `crashexit` fires on FATAL only, which arises only from
|
||||
`startretries` exhausted within `startsecs` (1 s): a program that crashes
|
||||
after one second cycles forever and never exits the container.
|
||||
architecture.md:40-41 reads wider than that.
|
||||
- **L33** `.dockerignore` excludes `docs`, `dev`, `bin` but not `test/`; a
|
||||
developer build after an e2e run ships `test/e2e/.stage/data` (DKIM keys,
|
||||
`sasldb2`) into the build context. Base images pinned by tag, not digest.
|
||||
- **L34** e2e: `stage.go:77` pulls `alpine:3.20` from the internet;
|
||||
`hostname_gate_test.go:97-101` runs with the host's real network;
|
||||
`sender_check_test.go:141` has a fixed `time.Sleep(2s)` against
|
||||
development.md:409 "no fixed sleep".
|
||||
|
||||
### Nits
|
||||
|
||||
Dead code: `handlers/authz.go:63 domainIDSet`; `view.Facts`
|
||||
(`components.go:414`); `dmarc.Service.HostedSuggestion`, `ValidateHostedAddress`
|
||||
(`dmarc/service.go:98-109`); `app.Service.UpdateMode` (`app/service.go:132-145`,
|
||||
superseded by `SaveSettings`); `Settings.WithResult` (`view/account.go:137-145`)
|
||||
re-implements `resultFlash`; `KitPage()`'s hard-coded `User: "admin"`
|
||||
(overwritten by the handler); dead method checks in `handlers_dmarc.go:20-24`
|
||||
and the `default:` branches of `HandleUserNew`/`HandleUserEdit`/`HandleServerSettings`
|
||||
(mux patterns are method-bound); the `postfix set-permissions` branch in
|
||||
entrypoint.sh:170-172 (`postfix check` has already created the tree).
|
||||
Duplication: `canViewDMARCDomain` repeats `lookupDomain`'s check; `ParseForm`
|
||||
twice on the user-edit POST; `Lookup` then `Touch` per request in
|
||||
`auth/middleware.go:17,28`; `retry.go:58-67` repeats the six `postconf` keys as
|
||||
literals instead of `retryConfKeys`; three header-name canonicalisers with two
|
||||
behaviours (`milter.go:124`, `inbound.go:89`, `sender.go:248`). Behaviour:
|
||||
`submitUserCreate` does `CreateUser` then `UpdateUser` for the e-mail;
|
||||
self-demotion redirects to a page that is now 404; `redirectHome` gives a
|
||||
domain user with no outbound reach an empty 200 where the inbound list gives
|
||||
404; `sort.Slice` in `postfix/inbound.go:59` reorders the caller's slice;
|
||||
`strings.Split(raw, "\n")` in `sender.go:65,104` keeps `\r`; `lists.go:428`
|
||||
trims the filter key while the Lua side reads it verbatim; `InDomain.WithMX`
|
||||
labels a `warn` status "NOT CHECKED"; `senderSession` does not override `Body`
|
||||
(the library discards the session after every accepted message — harmless,
|
||||
inconsistent with `milter.go:130-134`). The handlers test package takes ~105 s
|
||||
on this machine (likely bcrypt cost 12 on the user-creation paths).
|
||||
|
||||
### Doc drift
|
||||
|
||||
architecture.md: :167 (§ OpenDKIM) still says the `From:` header is not held
|
||||
against the login — contradicted by 2.4.0 and by § Sender check in the same
|
||||
file; :272-274 credits `/run` being tmpfs for a fresh `.alive` mark (it is a
|
||||
plain image directory; a restarted container keeps the mark); :328-329 "a
|
||||
deleted domain's row goes with it" — only `dns:` rows, `limit:` lingers up to
|
||||
48 h; :493 and § Who may open what omit `/inbound/log` and
|
||||
`/inbound/filter-lists`; :800 names only migration `0002`; :832-836 omits
|
||||
`inbound_spam_log`, `inbound_spam_list`, `notify_state`, `sender_check_days`;
|
||||
:838 "the panel is the only writer" — `dmarc-ingest` is a separate process
|
||||
that opens the store, migrates and writes; :856-859 vs `backup.go:19-23` and
|
||||
`httpserver.go:52-53` ("the same Resync") — the button skips DMARC maps and
|
||||
filter lists; the Internal env list omits `ANTISPAM_DIR` and
|
||||
`INBOUND_JOURNAL_MILTER_SOCKET`; § View engine calls the kit page "fixtures"
|
||||
(it is production data). schema-migrations.md: :20 "chain head `user_version
|
||||
= 5`" (six files; `TestBaselineSchema` asserts 6); the "Schema at head (v2)"
|
||||
table omits `sender_check_days` and `send_log.reason`/`header_from`.
|
||||
guide.md: :214 `INBOUND_JOURNAL_MILTER_SOCKET` "Set in: compose" (it is not);
|
||||
:1260 lines "deleted together with the row" (at the next six-hourly prune, and
|
||||
not while a reused id has a row); :720 "first successful boot" (M1).
|
||||
development.md: :270 version/tag coupling (H1); :353-357, :503 the env guard
|
||||
(M16); :404 "milter fail-open" as a covered negative (the suite now proves
|
||||
deferral); :409 "no fixed sleep". roadmap.md: `template-data-typing` describes
|
||||
`renderDomainDetail` with forty map keys — gone since 2.0 (one typed-before-
|
||||
render map remains, the health sampler's); `structured-logging` says `logf`
|
||||
"in five packages" — it is in 21 files of `internal/web`;
|
||||
`review-2026-08-followups` row "empty password" is misdescribed (M5). Code
|
||||
comments: `milter.go:7-11` "it never rejects" (it tempfails at `MAIL FROM`);
|
||||
`store/sendlog.go:19-22,61-65` `StatusRejected`/`InsertRejected` describe the
|
||||
level-2 case only; `store.go:2-3` "domain-admins"; `schema_test.go:15`
|
||||
`TestBaselineIsTheOnlyMigration`; `health/machine.go:225` "see roadmap
|
||||
panel-docs … later" (shipped 1.8.0); `inboundspam.go:349-350` "does not read
|
||||
the whole journal" (it scans it); `logtail.go:466` mentions `copytruncate`;
|
||||
supervisord.conf:25 "two services", :107 "7-14 files"; entrypoint.sh:119-120
|
||||
"no secrets" in `/data/antispam` (`filter.key`); docker-compose.yml:85,88 cite
|
||||
a guide section by a title it does not have.
|
||||
|
||||
### Test gaps
|
||||
|
||||
No test for: the H1 stamp (a `panel -version` assertion in the e2e against the
|
||||
version the stand was told to build); cert re-copy (H2); `follow()` with an
|
||||
unreadable or recreated log, a truncated `postqueue -p`, lines written to the
|
||||
old inode after the switch, `UpdateStatus` from `sent` (H3); two logins
|
||||
differing in case (H4); M1's ordering (restore tests never fail Open/Resync or
|
||||
exercise the real reload hook); `CheckRestore` with malformed JSON; M3, M5,
|
||||
L1, L2; `secretfile` concatenation (L3); the sender-check counter with the
|
||||
real program name (M13); a `LookupAddr` timeout (M15); `TestLimitRefusalsSince`
|
||||
EXPLAINs a stand-in query, not the real one; `TestOpenMigratesA20Database`
|
||||
stops at 0004; `Import` with a bad rua / IPs / window 0 / mode (M10); panel.js
|
||||
has no test at all (M11); login timing (M8); concurrent last-global (M7);
|
||||
`TestOriginCheck` lacks the Host-with-port vs Origin case;
|
||||
`TestServerSubtreeIsGlobalOnly` skips any `/server/` route not in
|
||||
`routes.always`; `internal/atomicfile` and `internal/supervisor` have no
|
||||
tests; `TestRenderInboundMaps` has no two-routes-one-next-hop case (L5);
|
||||
`TestSenderCheckResetsBetweenMessages` passes `h = nil` (L9); the handler test
|
||||
rows of `review-2026-08-followups` (apps/domains/users/dmarc handlers
|
||||
directly, `internal/dmarc` service) are still open.
|
||||
|
||||
### Checked and sound
|
||||
|
||||
Worth writing down so the next review does not re-read it: the sender check's
|
||||
layered rule holds against every `From:` shape tried (quoted local parts with
|
||||
`@`, CFWS, encoded words decoding to addresses, groups, `<>`, address
|
||||
literals, obs-route) — the "one `@` outside quotes" rule is the real backstop;
|
||||
extension stripping and case folding match Postfix; every error path of the
|
||||
check tempfails; level-2 accounting reserves under one lock and cannot be
|
||||
inflated by multi-RCPT or bypassed by ABORT/reconnect; the inbound verdict is
|
||||
trusted only under the queue-id-bound HMAC; the port-25 `-o` overrides are
|
||||
complete and read back by the e2e through `postconf -Ph`; the generated
|
||||
milter chain is exactly sender(tempfail) → OpenDKIM(tempfail) →
|
||||
journal(accept); `/server/` is guarded at registration and per handler;
|
||||
scopes fail closed (`1 = 0`); sessions store the hash only, `__Host-` cookie,
|
||||
constant-time setup token; `SaveSettings` restores prior state on mid-way
|
||||
failure; `secretfile`'s AAD binds header, counter and last-flag, and
|
||||
truncation, reorder and tamper are tested; shutdown order (tailer offset
|
||||
saved, HTTP drained, DB closed last) is right; rspamd map names, deny message,
|
||||
symbol prefixes and key-file format match the Go side byte for byte.
|
||||
|
||||
---
|
||||
|
||||
## Plan
|
||||
|
||||
Seven stages (six, and stage 2b split off stage 1 on 2026-10-14), each a
|
||||
self-contained cut, each small enough for one sitting,
|
||||
with model routing per development.md (security/infra → Opus, UI/docs →
|
||||
Sonnet). "Done when" is the test that proves it, not the edit.
|
||||
|
||||
**Agreed with the owner on 2026-10-12** (roadmap row `audit-2026-10-12`):
|
||||
stage 0 first, then stages 1 and 2 as one cut, the rest after in any order
|
||||
(stage 2b after `2.5.0`, before the others).
|
||||
This is the **active plan** in the sense of development.md § Plan checklists:
|
||||
the `- [ ]` lines below are the implementation checklist, a line is ticked in
|
||||
the commit that finishes it, and the roadmap's Progress column counts them.
|
||||
After a context reset, start at the first unchecked line of stage 0. The
|
||||
owner's decisions on the contested items are written next to the steps they
|
||||
changed and under *Decided with the owner* at the end; they are not open.
|
||||
|
||||
Working rules that apply, from [development.md](development.md) and the
|
||||
project's memory: every step updates CHANGELOG `[Unreleased]` in its own
|
||||
commit; a version is cut only at the owner's word; `main` is pushed, tags
|
||||
are not; design changes (the only one in this plan would be a *Delivery log*
|
||||
card, and it is not planned) are mockup-first; a migration the test server
|
||||
has already run is never edited — the next file is added; the published
|
||||
`1.x` tags and images are history and are not touched.
|
||||
|
||||
### Stage 0 — release hygiene (no product code) → `2.4.2`
|
||||
|
||||
- [x] **Stamp the right version** (H1). Make the e2e build take the version from
|
||||
the environment (`VERSION: ${SELFPOST_E2E_VERSION:-e2e}` in
|
||||
`compose.override.yml`, set by `release.yml`), or have the e2e reuse the
|
||||
image already built instead of rebuilding. Done when the e2e asserts
|
||||
`panel -version` equals what it was told, and `release.yml` runs that
|
||||
assertion before the push. — **Opus**
|
||||
- [x] **Compile the e2e in `test.yml`** (M20): `cd test/e2e && go vet ./... && go
|
||||
test -c -o /dev/null .` (not `go test -run xxx`: `TestMain` would still
|
||||
run and bring up the stand). — **Haiku**
|
||||
- [x] **`.dockerignore` + `test/`** (L33). — **Haiku**
|
||||
- [x] **The pin** (M6): old tags and images are not touched (owner,
|
||||
2026-10-12). The next publication through the fixed `release.yml` is the
|
||||
one the pin names; until then README/guide say that the current line is
|
||||
not yet published rather than name an image that is not there. — **Sonnet**
|
||||
- [x] **Toolchain and deps**: `modernc.org/sqlite`, `x/crypto`, `x/sys`; note in
|
||||
development.md that the image is rebuilt after each Go patch release
|
||||
(`golang:1.26-bookworm` floats, so a rebuild suffices). — **Haiku**
|
||||
- [x] The four `staticcheck` items. — **Haiku**
|
||||
- [ ] **A negative e2e case passes only on the refusal it is about** (owner,
|
||||
2026-10-14; seen, not found by the audit). `negative_level1_ratelimit`
|
||||
sends up to sixty messages and waits for one to be refused; it once
|
||||
passed on "limit tripped on attempt 1/60" when the connection was reset
|
||||
because Postfix was still starting. The case counts as the limit only the
|
||||
reply code of the limit, and only after at least one message was
|
||||
accepted; a reset, a timeout or any other reply fails it and says what
|
||||
came. Then every other negative case of the suite is read for the same
|
||||
mistake - any failure taken for the expected refusal - and held to its
|
||||
own reply code. - **Opus**
|
||||
|
||||
### Stage 1 — mail that is shown wrong (H3, H2, M12, M13) → `2.5.0`
|
||||
|
||||
- [ ] **`QueueIDs` on a truncated listing returns an error**, as `postqueueJSON`
|
||||
does; `reconcile` already treats an error as no information. Test: a 5 MiB
|
||||
fake `postqueue -p`. — **Opus**
|
||||
- [ ] **Reconcile only closes `queued` rows** (`AND status = 'queued'` in the
|
||||
reconcile path's update) and marks what it closed: `reason = 'reconcile'`
|
||||
on the row (the 0006 column; a new value, no schema change). — **Opus**
|
||||
- [ ] **Measure before deciding the sweep's fate** (owner, 2026-10-12: the
|
||||
keep / remove / `lost` question needs real data, and today the only
|
||||
record of a sweep is a stdout line that dies with the container). A table
|
||||
`delivery_log_days` (migration `0007`, DDL only; a row per UTC day, kept a
|
||||
year, the shape of `sender_check_days`) with four counters:
|
||||
`reconcile_closed` (rows the sweep closed); `reconcile_contradicted` (a
|
||||
`reason = 'reconcile'` row the tailer later rewrote from a `status=` line —
|
||||
the log wins, and the rewrite is counted); `tailer_blind_seconds`
|
||||
(`follow()` unable to read: reopen failing, file gone — the sender check's
|
||||
downtime mechanics); `queued_over_lifetime` (rows still `queued` past
|
||||
`maximal_queue_lifetime` at the day's end — what removing the sweep would
|
||||
leave visible). **No screen for this** (owner, 2026-10-12: nothing is
|
||||
drawn that ordinary operation will not need): the table is read off a
|
||||
copy of the database when the month is up. Whether a durable *Delivery
|
||||
log* card (tailer blindness, stuck rows — the two counters that outlive
|
||||
the decision) earns a place on Health is a separate question, asked only
|
||||
if the data shows those numbers are ever non-zero.
|
||||
Decision rule, written now: `closed = 0` for a month → remove the sweep;
|
||||
`closed > 0, contradicted = 0` → keep; `contradicted > 0` → each case
|
||||
explained by `tailer_blind` means fix the tailer and keep the sweep,
|
||||
otherwise `lost` or remove. Decide after 30 days of data, or sooner if
|
||||
`contradicted` is ever non-zero. — **Opus**
|
||||
- *Deferred until the data is in* (not a step yet): pausing the sweep while the tailer is
|
||||
not reading (publish "healthy" from `follow`, skip the sweep otherwise),
|
||||
and the rotation window (keep draining the old inode until it has not
|
||||
grown for a tick and the new file has a line). Both are fixes to the
|
||||
tailer's blindness and stay on the list whatever the sweep's fate; their
|
||||
priority follows `tailer_blind_seconds`.
|
||||
- [ ] **Anchor `deliveryRe`** at the head of the line like its two siblings;
|
||||
test with the probed HELO and quoted-sender lines. — **Opus**
|
||||
- [ ] **TLS re-copy** (H2): `postfix-cert-reload.sh` compares the mounted files
|
||||
to the copy and re-copies before `postfix reload` (atomically, same modes);
|
||||
e2e replaces the cert in the stand and reads the new serial off 465.
|
||||
— **Opus**
|
||||
- [ ] **Deferred counter** (M13): on the stand, `kill -STOP` the panel mid-message
|
||||
and record which program logs the warning; widen the matcher accordingly
|
||||
and replace the composed fixture with the seen lines. — **Opus**
|
||||
- [ ] Docs: architecture § Log tailer, § Queue reconcile (the measurement and
|
||||
the rule), § Sender check "what it costs"; guide § Delivery log, § Health.
|
||||
— **Sonnet**
|
||||
|
||||
### Stage 2 — the tenant boundary and the registry (H4, M7, M9, M10) → in `2.5.0` (migration `0008`)
|
||||
|
||||
- [ ] **Logins lower-cased** at validation (owner, 2026-10-12: fold, do not
|
||||
refuse) and `applications.login COLLATE NOCASE` (migration `0008` rebuilds
|
||||
the unique index). Before the migration a check
|
||||
that no two existing logins differ only in case (there is one
|
||||
installation; look). Tests: `Ops`/`ops` refused; the map written lower-
|
||||
case. Then verify Postfix's compare on the stand once (`MAIL FROM` as the
|
||||
other case) so the doc's claim is seen, not assumed. — **Opus**
|
||||
- [ ] **Last global in one transaction**: `DeleteUser`/`SetUserRole` take a
|
||||
`tx`, count and act inside it; `CreateGlobalUser` likewise. A concurrency
|
||||
test with two goroutines. — **Opus**
|
||||
- [ ] **`submitUserUpdate` in one transaction** (store method that takes the
|
||||
whole change). The users pages get direct handler tests while they are
|
||||
rewired - a row of review-2026-08-followups, placed here 2026-10-14. — **Opus**
|
||||
- [ ] **Import validates like the form**: route `DMARCRua`, IPs, limit mode,
|
||||
window and count through the same parsers; refuse the file with the
|
||||
validator's text. Tests per field. — **Opus**
|
||||
- [ ] **Send-log rows belong to a domain by id** (L12; owner, 2026-10-12).
|
||||
`send_log.domain_id INTEGER REFERENCES domains(id) ON DELETE SET NULL` in
|
||||
migration `0008`, backfilled once by name against the registry as it is at
|
||||
migration time (rows of a domain already gone stay NULL); written by the
|
||||
journal milter and by the sender check's refusal rows at insert, looked up
|
||||
from the login's domain; the `domain` name column stays for display and
|
||||
for the global view. A domain user's log, stats, auto rate limit and the
|
||||
notifier's refusal count select by id; a NULL row is nobody's but the
|
||||
global role's. A domain deleted and re-added under its old name therefore
|
||||
starts with an empty log. The inbound journal (`inbound_spam_log`, the
|
||||
same accepted risk in security.md:419) gets the same treatment in the same
|
||||
migration — my proposal, so that the two logs do not follow two rules.
|
||||
Tests: delete-and-re-add shows nothing to the new assignee; the old rows
|
||||
are still in the global log. — **Opus**
|
||||
- [ ] Docs: security.md § Roles (the guard) and the struck accepted risk, guide
|
||||
§ Domain export, schema-migrations.md. — **Sonnet**
|
||||
- [ ] **Which key OpenDKIM signs with, seen and not reasoned** (owner,
|
||||
2026-10-13, from the Not verified list of from-header-binding). The check
|
||||
and OpenDKIM read `From:` with two parsers; that they agree was argued
|
||||
from the check's two rules, never run. An e2e case sends, as one
|
||||
application, the `From:` lines the check passes - quoted strings,
|
||||
comments, folded lines, groups, escapes, encoded words, an address
|
||||
extension - and reads the `d=` of the signature on what arrives: any
|
||||
domain but the application's fails the test. The lines live in one table
|
||||
shared with the check's unit test, so a line added to one is run by the
|
||||
other. A disagreement found is fixed in the check before the cut, and
|
||||
security.md § Mail path says the claim is tested. - **Opus**
|
||||
- [ ] **The 550 reply names no other domain** (L8): `sender.go:334` stops at
|
||||
"it names another sender of this server"; the refused message's page
|
||||
still shows the `From:` line as written (`header_from`), so the operator
|
||||
loses nothing. Test pins the reply text; e2e's sender-check negatives
|
||||
assert the reply does not contain the other tenant's name. — **Opus**
|
||||
|
||||
### Stage 2b — the log tailer, and what the panel shows of a refusal → `2.5.1`
|
||||
|
||||
Split off stage 1 on 2026-10-14 at the owner's word: the six steps below were
|
||||
agreed on 2026-10-13/14, after the plan, and had made stages 1 and 2 a cut of
|
||||
twenty-one steps. Stages 1 and 2 ship as `2.5.0` in their first size; this
|
||||
stage follows as a cut of its own, with its own reviews. Five of the six
|
||||
change one thing, the tailer and the state it saves, and are done in this
|
||||
order - each stands on the one before.
|
||||
|
||||
- [ ] **A log rotated while the panel was down is read to its end** (owner,
|
||||
2026-10-13; the Not done list of from-header-binding had it as "deferrals
|
||||
are lost", and it is wider). At start, a saved offset whose fingerprint
|
||||
is not the current file's means "start on the new file"
|
||||
([offset.go](../internal/logtail/offset.go)); what Postfix wrote between
|
||||
the panel's stop and the rotation is never read - delivery statuses,
|
||||
stored delivery lines, refusals, inbound journal rows, the deferred
|
||||
counter, and the recovery of unwritten refusals above. The tailer first
|
||||
opens the rotated file (`mail.log.1`), checks by fingerprint that it is
|
||||
the one the offset belongs to, reads it from the offset to its end, then
|
||||
goes on to the new file. The fingerprint gains the inode, which closes
|
||||
the roadmap's review-2026-08-followups row on collisions. Not found, or
|
||||
rotated twice: logged, with from when to when nothing was read. A test
|
||||
that stops the tailer, appends, rotates, appends, starts it, and finds
|
||||
both halves once. - **Opus**
|
||||
- [ ] **The deferred counter remembers a place in the log, not a second**
|
||||
(owner, 2026-10-14, from the Not done list of from-header-binding; he
|
||||
chose it over leaving the case written down). After an unclean stop the
|
||||
tailer re-reads lines it had read, and the counter passes over every
|
||||
warning no later than the last one it counted
|
||||
([logtail/sendercheck.go](../internal/logtail/sendercheck.go), `last`);
|
||||
`mail.log` stamps whole seconds, so a warning of that very second, later
|
||||
in the file, is lost. The count and the place in the log it has reached
|
||||
are saved in one transaction, and what to pass over is decided by the
|
||||
place. Done after the rotation step above, on the same saved state - the
|
||||
place must stay true across a rotated file. A test that stops the tailer
|
||||
between two warnings of one second and finds both counted, once each.
|
||||
- **Opus**
|
||||
- [ ] **A refused message's page shows what Postfix wrote** (owner,
|
||||
2026-10-13: "the log box is required" - the box is the reason
|
||||
delivery-log-storage was built, and its absence from the picture he
|
||||
approved for 2.4.0 was an oversight, not a decision). Today every
|
||||
`rejected` row - `wrong From`, `wrong client`, `rate-limited` - has no
|
||||
queue id, and its page shows the box with one sentence,
|
||||
"This message never reached the queue, so Postfix wrote no delivery lines
|
||||
for it" ([handlers_monitor.go](../internal/web/handlers/handlers_monitor.go)).
|
||||
The sentence is untrue: Postfix wrote a line for each (`milter-reject:
|
||||
END-OF-MESSAGE ... 5.7.1 From: ...` from cleanup, `NOQUEUE: reject: ...`
|
||||
from smtpd). The tailer attaches that line to the row it belongs to, by
|
||||
client address, time and the addresses in the line, and stores it in
|
||||
`send_log_lines` like any other; the page shows it. First on the stand:
|
||||
read `mail.log` for the three kinds of refusal and write down what
|
||||
identifies each line. Then the mockup: the refused variant of `out-message`
|
||||
gets the box with the line in it (`design:` commit), then the code. The
|
||||
sentence stays only for a row whose line was truly never written or is
|
||||
older than this change, and says that. Done when the e2e shows the stored
|
||||
line on the page of each of the three refusals. - **Opus**
|
||||
- [ ] **A refusal the panel did not get to write is written from
|
||||
`mail.log`** (owner, 2026-10-13, from the Not done list of
|
||||
from-header-binding; built on the step above). The check answers 550 and
|
||||
writes the row a moment later, through a buffer; a panel that stops in
|
||||
between - a rollout, a crash - leaves an application that was refused
|
||||
and a log that does not know it. When the tailer meets a refusal line
|
||||
that has no row, it makes the row from the line (client, sender,
|
||||
recipient, reason), marked as recovered. The tailer resumes from its
|
||||
saved offset, so this covers a kill as well as a clean stop - which a
|
||||
flush of the buffer at shutdown would not. Done when the e2e refuses a
|
||||
message, kills the panel before the row is written, and finds the row
|
||||
after the restart, once. - **Opus**
|
||||
- [ ] **Downtime is counted when supervisord does not answer at start**
|
||||
(owner, 2026-10-13, from the Not done list of from-header-binding). The
|
||||
panel asks supervisord once since when Postfix runs
|
||||
([sendercheck.go](../cmd/panel/sendercheck.go), `postfixStarted`); an
|
||||
error is logged and that start's downtime is not counted at all - the
|
||||
number the owner will decide by reads low. The question is repeated for
|
||||
about thirty seconds. Still no answer: the downtime is taken from the
|
||||
alive mark alone - an upper bound, Postfix may not have been running -
|
||||
and the day is marked as estimated (a column in `sender_check_days`, the
|
||||
next migration), which the card shows. The mark on the card is drawn in
|
||||
the mockup with the renamed column above (one `design:` commit). - **Opus**
|
||||
- [ ] **The Sender check card names what it counts** (owner, 2026-10-13, from
|
||||
the Not done list of from-header-binding): the column *Messages deferred*
|
||||
becomes *Connections turned away*. Postfix refuses from the connection on
|
||||
while the check does not answer, before any message is offered, so a
|
||||
client that connects three times with one message is three, and a monitor
|
||||
that only connects is one; messages cannot be counted. The mockup `health`
|
||||
and help topic `sender-check` first (`design:` commit), then the page and
|
||||
`view/help.go`, word for word. - **Sonnet**
|
||||
- [ ] Docs: architecture § Log tailer (the rotated file, the saved place, a
|
||||
refusal's line and a recovered row), guide § Outbound log and § Health,
|
||||
schema-migrations.md for the mark on `sender_check_days`. — **Sonnet**
|
||||
|
||||
### Stage 3 — persistence and restore (M1–M5, L1–L4, L7) → `2.5.2`
|
||||
|
||||
- [ ] **Consume the manifest after the resync succeeds** (`CheckRestore` reports;
|
||||
`run` deletes after `resyncAfterRestore` returns nil). Test: Resync fails →
|
||||
manifest kept → second boot resyncs. And a corrupted or truncated archive
|
||||
is refused on restore and on import with its message, nothing half
|
||||
written (a row of review-2026-08-followups, placed here 2026-10-14). — **Opus**
|
||||
- [ ] **Indexes**: `idx_send_log_domain_created_at (domain_id, created_at)` —
|
||||
on the id column stage 2 adds, replacing `idx_send_log_domain`;
|
||||
partial `idx_send_log_queued (created_at) WHERE status = 'queued'`;
|
||||
consider `(domain, id)` for the scoped page. Migration `0009` DDL only;
|
||||
`TestLimitRefusalsSince`-style EXPLAIN tests against the **real** query
|
||||
strings. — **Opus**
|
||||
- [ ] **CLI backup**: open the source `mode=ro`, refuse a missing file; password
|
||||
floor of 12 shared with the panel (`validate.MinSecretFilePasswordLen`);
|
||||
`-decrypt -o` verifies the password (first chunk) before opening the
|
||||
output; `OnWarn` for skipped symlinks and `Check` fails when `certs/` is
|
||||
named but empty. — **Opus**
|
||||
- [ ] **`secretfile` caps**: bound `128·N·r` to a stated budget (e.g. 128 MiB)
|
||||
and `p` to 1-2; refuse the rest with `ErrHeader`; reject trailing bytes
|
||||
after the last chunk. Tests. — **Opus**
|
||||
- [ ] **`atomicfile`**: fsync the directory after rename; a test file for the
|
||||
package. — **Opus**
|
||||
- [ ] Docs: security.md § Backup; guide § Backup. — **Sonnet**
|
||||
|
||||
### Stage 4 — panel behaviour (M8, M11, L10–L19) → `2.5.3`
|
||||
|
||||
- [ ] Login: dummy bcrypt on unknown user; a test that measures both paths within
|
||||
a tolerance. Account password change gets its own limiter bucket. Login
|
||||
destroys the presented session. — **Opus**
|
||||
- [ ] `http.Server`: `ReadTimeout`, `WriteTimeout`, `IdleTimeout`. — **Opus**
|
||||
- [ ] **Polling recovery**: schedule from `htmx:afterRequest` on the box (or pass
|
||||
`source`), handle `sendError`/`timeout` with back-off; `RequireAuth`
|
||||
answers an `HX-Request` with `HX-Redirect: /login` instead of a 303. A
|
||||
minimal JS test (node + jsdom) or at least an e2e that restarts the panel
|
||||
with Overview open. — **Sonnet**
|
||||
- [ ] `parsePage` clamp; `NaN` refused; `rate-limited` tag red (add the status to
|
||||
`components.go`, flip the test); the recalc button posts only what it
|
||||
needs or warns; `HandleReload` either covers DMARC maps and filter lists or
|
||||
the flash and the two comments say what it covers. The log's paging is
|
||||
tested at its edges: first page, last, past the last, an empty log (a row
|
||||
of review-2026-08-followups, placed here 2026-10-14). — **Sonnet**
|
||||
- [ ] `domain.Service.Delete`: say the domain is deleted when a later step fails,
|
||||
and run `apps.Resync()` anyway. — **Opus**
|
||||
- [ ] **A status filter and a day filter in the Outbound log** (owner, 2026-10-13, from the
|
||||
Not done list of from-header-binding). The mockup `out-log` has drawn a
|
||||
third list since the redesign and the built log never had it; the Inbound
|
||||
log has its counterpart built (domain, decision, day). One list: *Any
|
||||
status*, `queued`, `sent`, `deferred`, `bounced`, `rejected` - the statuses
|
||||
of a row, and `rejected` shows all three kinds of refusal together. He was
|
||||
shown the list with the refusals split by reason and chose it without the
|
||||
split. The mockup's own list is wrong (it offers `rate-limited`, which is
|
||||
a reason, and lacks `queued` and `rejected`): correct it first (`design:`
|
||||
commit). And a filter by day, as the Inbound log has it (owner, the same
|
||||
day): the same control, drawn in the mockup first. Both filters survive
|
||||
paging and the rows' polling. - **Sonnet**
|
||||
- [ ] **Milter sockets: "Listening" is asked of the socket** (owner,
|
||||
2026-10-13, from the Not done list of from-header-binding).
|
||||
`health.CheckSocket` ([socket.go](../internal/health/socket.go)) stats the
|
||||
file and writes "Listening"; a process that died and left its socket
|
||||
reads ok while mail is deferred or goes out unsigned. It connects and
|
||||
closes instead, and a refused connection is an error that says nobody
|
||||
answers. Find out first, on the stand: whether the panel's user may
|
||||
connect to OpenDKIM's socket, and whether a connection closed without a
|
||||
byte ends the sender check (security.md § Accepted risks: the milter
|
||||
library panics on an empty packet) - if it does, that is fixed first, in
|
||||
this step. A test with a socket file nobody listens on. - **Opus**
|
||||
- [ ] **The `cards` layout takes its phone order from the box's number**
|
||||
(owner, 2026-10-13, from the Not done list of from-header-binding). The
|
||||
order on a phone is written in `panel.css` for four boxes to a stack
|
||||
(`.sp-cards > .column:first-child > .box:nth-child(1..4)`); a fifth box
|
||||
gets no rule and lands out of place with nothing failing. The box partial
|
||||
sets an order class from its own number (a class, not a style: the CSP
|
||||
allows no inline styles), the stylesheet orders by that class, and a test
|
||||
checks that the numbers of a rendered `cards` page run in order. A change
|
||||
to a component: the mockup's components page and `theme.css` first
|
||||
(`design:` commit). - **Sonnet**
|
||||
- [ ] **A notification is owed to each recipient, not to the subject**
|
||||
(owner, 2026-10-14; from the reviews of panel-notifications, not found by
|
||||
the audit). `Notifier.send` ([notify.go](../internal/notify/notify.go))
|
||||
counts a subject as said when one recipient's copy was handed to Postfix;
|
||||
a recipient whose copy failed, or was "held back" by the hourly ceiling,
|
||||
is never told. Unseen while the owner is the only recipient; the first
|
||||
domain administrator makes it "the global administrator knows the
|
||||
domain's DNS broke, its administrator does not". The state of a subject
|
||||
is kept per recipient (the next migration), and a later check sends to
|
||||
those not yet told - so the ceiling holds a copy back instead of dropping
|
||||
it. A user who gains the right to hear of a subject while it is already
|
||||
open is not mailed about it. "Cleared" goes to those who were told of the
|
||||
problem. Tests: one copy fails and goes out at the next check, once; the
|
||||
ceiling; a user added mid-subject. - **Opus**
|
||||
- [ ] **A DNS outage is "unknown", not "wrong"** (M14, M15; owner,
|
||||
2026-10-14 - the two findings had no step). Each resolver gets a time
|
||||
budget of its own, so a silent first resolver no longer leaves the second
|
||||
an expired context ([dnscheck/resolver.go](../internal/dnscheck/resolver.go));
|
||||
a timeout is Unknown, not "does not resolve" or "no PTR record"
|
||||
([dnscheck/server.go](../internal/dnscheck/server.go)); a failed lookup
|
||||
keeps the server's addresses it had instead of emptying them, so SPF does
|
||||
not go Unknown with it. Notifications: a domain's DNS is reported broken
|
||||
only on a result that is wrong, never on Unknown - checked in `notify`,
|
||||
and tested. Tests: a first resolver that drops packets, a PTR timeout.
|
||||
architecture.md:339-340 and the `DefaultResolvers` comment say what is
|
||||
true. — **Opus**
|
||||
- [ ] **DMARC reports tested where they come in** (owner, 2026-10-14; rows of
|
||||
review-2026-08-followups). The panel parses reports that arrive by mail
|
||||
from anyone, and the `internal/dmarc` service (`service.go`,
|
||||
`addresses.go`) and the DMARC handlers have no tests; no other step
|
||||
touches them. Tests of the service, and of a report that is malformed,
|
||||
oversized, compressed past its limit, or names a domain the instance does
|
||||
not have: refused or skipped with a reason, nothing stored, the panel
|
||||
alive. A finding is fixed in this step. — **Opus**
|
||||
- [ ] **The last untyped page data** (owner, 2026-10-13): `healthChecks`
|
||||
([handlers_status.go](../internal/web/handlers/handlers_status.go)) returns
|
||||
a struct instead of `map[string]any`, so a mistyped key in `healthPage` /
|
||||
`healthCards` is a compile error and not an empty card on Server › Health.
|
||||
No template change. This closes what is left of the roadmap's former
|
||||
`template-data-typing`. — **Sonnet**
|
||||
- [ ] **Upstream TLS mode validated at save** (L5): saving an inbound domain
|
||||
whose `[host]:port` is already used by another domain with a different
|
||||
TLS mode is refused with the validator's text naming that domain and its
|
||||
mode; `TestRenderInboundMaps` gets the two-routes-one-next-hop case and the
|
||||
map is asserted to hold one key. Guide § Inbound domains says a next hop
|
||||
has one TLS mode. — **Opus**
|
||||
|
||||
### Stage 5 — image and Postfix hardening (M17–M19, M21, L30–L32) → `2.6.0`
|
||||
|
||||
- [ ] **Supervisor socket**: a group of its own (`supervisor`), panel only;
|
||||
`postfix`/`opendkim` keep `selfpost` for the file paths. — **Opus**
|
||||
- [ ] **OpenDKIM**: `Syslog yes` (supervisord captures it) or `Logfile`;
|
||||
`Canonicalization relaxed/simple`; `OversignHeaders From`. e2e: DKIM still
|
||||
verifies; a message with two `From:` is refused before signing anyway.
|
||||
— **Opus**
|
||||
- [ ] **Postfix**: `smtpd_tls_protocols = >=TLSv1.2`, `smtpd_tls_mandatory_protocols`
|
||||
likewise, `disable_vrfy_command = yes`, `smtpd_banner = $myhostname ESMTP`;
|
||||
integer validation of `RATE_LIMIT_*`, `*_SIZE_LIMIT`, `MILTER_*_TIMEOUT` at
|
||||
the top of `postfix-config.sh` with a clear exit. — **Opus**
|
||||
- [ ] **logrotate state under `/data/log/.logrotate.status`** (`-s`), group-
|
||||
readable as the directory is. — **Opus**
|
||||
- [ ] **entrypoint**: normalise the three DMARC maps with the other five;
|
||||
remove the dead `set-permissions` branch. — **Opus**
|
||||
- [ ] **`extract-cert.sh`**: `set -o pipefail` under `bash`, check both outputs
|
||||
are non-empty PEM before moving them into place, create `0600` first.
|
||||
— **Opus**
|
||||
- [ ] **`crashexit`**: subscribe to `PROCESS_STATE_EXITED` with a restart-count
|
||||
threshold, or raise `startsecs` so a crash after 1 s still reaches FATAL;
|
||||
say in architecture.md what actually exits the container. — **Opus**
|
||||
- [ ] **`envdoc_test` made real**: parse `envDefault|envInt|os.Getenv("…")` out of
|
||||
`cmd/panel/*.go` and `${VAR` out of `build/*.sh`, parse the key tables out
|
||||
of guide.md and architecture.md, compare. — **Opus**
|
||||
- [ ] Docs: architecture § Image and processes, § Configuration; security.md
|
||||
§ Process. — **Sonnet**
|
||||
|
||||
### Stage 6 — documentation and roadmap sweep (no product code, no cut)
|
||||
|
||||
- [ ] Every item under *Doc drift* above, file by file. — **Sonnet**
|
||||
- [ ] roadmap.md: done ahead on 2026-10-13/14 with the owner -
|
||||
`template-data-typing` struck, `structured-logging` dropped,
|
||||
`review-2026-08-followups` closed with each row placed in a stage or
|
||||
dropped with its reason. What is left: check that every *Test gaps* entry
|
||||
above is named by the step that touches its code, and add it to that step
|
||||
where it is not. — **Sonnet**
|
||||
- [ ] Comments named above (`milter.go`, `sendlog.go`, `store.go`, `schema_test.go`,
|
||||
`machine.go`, `inboundspam.go`, `logtail.go`, supervisord.conf,
|
||||
entrypoint.sh, docker-compose.yml). — **Haiku**
|
||||
- [ ] Dead code removal from § Nits, with `staticcheck` added to `test.yml` so it
|
||||
stays gone. — **Haiku**
|
||||
|
||||
### Decided with the owner on 2026-10-12
|
||||
|
||||
Four items I had first put below the line; three of them are in the plan
|
||||
above after discussion, which is why they are recorded here with the
|
||||
reasoning rather than struck.
|
||||
|
||||
- L5 (`tls_policy` per next-hop): the full fix is a model change (TLS mode per
|
||||
upstream, not per domain) — a design item. Decided (owner, 2026-10-12):
|
||||
validate instead — stage 4, item 6.
|
||||
- L12 (rows by domain name): decided (owner, 2026-10-12): the send log gets
|
||||
the domain's id — stage 2, item 5.
|
||||
- L8 (the signing-name oracle): decided (owner, 2026-10-12): do it — stage 2,
|
||||
item 7. (The names are not public: rDNS gives the hostname only, so the
|
||||
reply told one tenant which other domains are hosted here.)
|
||||
- L9 (go-milter header accumulation): decided (owner, 2026-10-12): locally
|
||||
only — a comment on `senderSession.Headers` saying why `h` is ignored, and
|
||||
`TestSenderCheckResetsBetweenMessages` passing an `h` that still holds the
|
||||
first message's `From:` (stage 2, with item 7). The library itself stays
|
||||
as vendored; an upstream patch is a separate, unhurried matter.
|
||||
- The Windows-bound tests: not worth platform guards; CI is Linux.
|
||||
+26
-8
@@ -107,7 +107,7 @@ than the default «meaningful step» rule in [§ Commits and release build](#com
|
||||
2. Git tag `vX.Y.Z` and publish the GitHub Release only on explicit request (see
|
||||
[§ Release image](#release-image)).
|
||||
|
||||
**Planned version cuts** (from pin `2.0.0`; adjust if semver changes mid-track):
|
||||
**Planned version cuts** (from pin `2.4.1`; adjust if semver changes mid-track):
|
||||
|
||||
| Stage | ID | Cut |
|
||||
|---|---|---|
|
||||
@@ -124,7 +124,12 @@ than the default «meaningful step» rule in [§ Commits and release build](#com
|
||||
| 11 | panel-redesign plan: component kit, stages, reorder (docs only, owner-requested cut) | `1.9.4` (PATCH) |
|
||||
| 12 | GHCR release fix: keep per-arch images, sweep orphaned versions (CI only) | `1.9.5` (PATCH) |
|
||||
| 13 | panel-redesign | `2.0.0` (MAJOR, no compatibility) |
|
||||
| 14 | inbound-antispam-panel | `2.1.0` (planned; was `1.10.0`, reordered 2026-10-09) |
|
||||
| 14 | DMARC ingest drops and bounds, full backup from the panel, no queue in a backup | `2.0.1` (PATCH) |
|
||||
| 15 | inbound-antispam-panel | `2.1.0` (was `1.10.0`, reordered 2026-10-09) |
|
||||
| 16 | delivery-log-storage | `2.2.0` |
|
||||
| 17 | panel-notifications | `2.3.0` |
|
||||
| 18 | from-header-binding | `2.4.0` |
|
||||
| 19 | code audit 2026-10-12 and its plan (docs only, owner-requested cut) | `2.4.1` (PATCH) |
|
||||
|
||||
Docs-only prep (checklists in plans, no product code) uses the same per-step
|
||||
commit rule but **no** version cut until the next product stage ships.
|
||||
@@ -139,7 +144,7 @@ commit rule but **no** version cut until the next product stage ships.
|
||||
| **SQLite** | `modernc.org/sqlite` (pure Go, no cgo) |
|
||||
| **Build** | [Makefile](../Makefile): primary targets `vet`, `test`, `build`, `e2e` (also `all`, `clean`) |
|
||||
| **Container** | Docker + Compose v2 on the dev host and in CI |
|
||||
| **Image (build stage)** | `golang:1.26-bookworm` — [build/Dockerfile](../build/Dockerfile) |
|
||||
| **Image (build stage)** | `golang:1.26-bookworm` — [build/Dockerfile](../build/Dockerfile). The tag floats to the newest 1.26 patch release, and most standard-library security fixes (`net/http`, `html/template`, …) arrive only that way: rebuild the image after each Go patch release, no edit needed. Dependencies are raised by hand (`go get …@latest`, `go mod tidy`), at least at each audit |
|
||||
| **Image (runtime)** | `debian:bookworm-slim` + Postfix, OpenDKIM, supervisord, SASL, logrotate |
|
||||
| **CI** | GitHub Actions — [.github/workflows/](../.github/workflows/) |
|
||||
| **Image registry** | `ghcr.io/mixeme/selfpost` |
|
||||
@@ -264,13 +269,22 @@ The release image is published **only** for a SemVer version `X.Y.Z`: a
|
||||
that supplies that version. Pushing a git tag alone does not publish. Ordinary
|
||||
commits, and a dispatch from `main` without a version input, do not publish.
|
||||
The version is the single source that drives the image tag and `-ldflags` in
|
||||
the binaries so they cannot drift apart.
|
||||
the binaries so they cannot drift apart. The e2e gate rebuilds the image it
|
||||
then pushes, so `release.yml` hands it the version as `SELFPOST_E2E_VERSION`;
|
||||
the suite's `image_stamped_with_version` and a check right before the push
|
||||
both compare `panel -version` and `selfpost-backup -version` with it. (Until
|
||||
`2.4.2` the rebuild stamped every published image `e2e`; those images stay as
|
||||
they are.)
|
||||
|
||||
**Steps (on explicit request):**
|
||||
|
||||
1. Close `[Unreleased]` in [CHANGELOG.md](../CHANGELOG.md) and bump the pinned
|
||||
tag in [deploy/docker-compose.yml](../deploy/docker-compose.yml) (and any
|
||||
local-trial image references) in the **same** release commit.
|
||||
local-trial image references) in the **same** release commit. While no
|
||||
image of the current line is published, README, guide § Fixed image tag
|
||||
and the compose file's header say so and show the local build under the
|
||||
pinned name; bump the version there too. The first publication of the
|
||||
line removes those notes.
|
||||
2. Create and push git tag `vX.Y.Z` on that commit.
|
||||
3. Publish the GitHub Release for `vX.Y.Z` (not a draft).
|
||||
4. Workflow [release.yml](../.github/workflows/release.yml) builds, e2e-gates,
|
||||
@@ -390,7 +404,8 @@ make e2e
|
||||
[test/e2e/compose.override.yml](../test/e2e/compose.override.yml) — same
|
||||
`cap_drop`/`cap_add`/`no-new-privileges` as production. Override: high ports
|
||||
(`20465`/`20587`/`20080`), test hostname, self-signed TLS,
|
||||
`PANEL_COOKIE_SECURE=false`, isolated compose project. Mail is hermetic:
|
||||
`PANEL_COOKIE_SECURE=false`, isolated compose project, and the image's
|
||||
`VERSION` from `SELFPOST_E2E_VERSION` (unset locally: `e2e`). Mail is hermetic:
|
||||
CoreDNS (fake zone) + Postfix `smtp-sink` as sink-MX; DKIM TXT is scraped from
|
||||
the panel and published into the zone — the test verifies the records an
|
||||
operator would actually use.
|
||||
@@ -417,7 +432,9 @@ Workflows in [.github/workflows/](../.github/workflows/). What each job runs —
|
||||
|
||||
`design-first` (needs the full history) → `gofmt -l` → `go vet ./...` →
|
||||
`go test ./...` (main module, no e2e; includes the panel guard tests, see
|
||||
[§ Panel pages](#panel-pages-the-component-kit-and-its-guards)).
|
||||
[§ Panel pages](#panel-pages-the-component-kit-and-its-guards)) → the e2e
|
||||
module compiled, not run (`go vet ./...` and `go test -c -o /dev/null .` in
|
||||
`test/e2e`).
|
||||
|
||||
### `release.yml` — published GitHub Release, or `workflow_dispatch` with SemVer
|
||||
|
||||
@@ -435,7 +452,8 @@ release: published
|
||||
prepare (version from release tag or workflow_dispatch input; checkout vX.Y.Z)
|
||||
→ build [matrix: ubuntu-latest / ubuntu-24.04-arm]
|
||||
→ docker build --load (VERSION from prepare)
|
||||
→ e2e (test/e2e)
|
||||
→ e2e (test/e2e; rebuilds selfpost:e2e with SELFPOST_E2E_VERSION=X.Y.Z)
|
||||
→ check panel / selfpost-backup -version = X.Y.Z
|
||||
→ push ghcr.io/...:X.Y.Z-amd64 | X.Y.Z-arm64
|
||||
→ merge
|
||||
→ docker buildx imagetools create → unified manifest X.Y.Z
|
||||
|
||||
+240
-26
@@ -33,6 +33,7 @@ changed (see the CHANGELOG, *Removed*).
|
||||
- [Queue and System log](#queue-and-system-log)
|
||||
- [Account](#account)
|
||||
- [Settings](#settings)
|
||||
- [Notifications](#notifications)
|
||||
- [Users](#users)
|
||||
- [Sessions](#sessions)
|
||||
- [Upgrading](#upgrading)
|
||||
@@ -183,7 +184,18 @@ cat ./data/setup-token
|
||||
#### Fixed image tag
|
||||
|
||||
`deploy/docker-compose.yml` pins an explicit version (`ghcr.io/mixeme/selfpost:X.Y.Z`),
|
||||
deliberately never `:latest`. The current pin is `2.0.0`. Intermediate
|
||||
deliberately never `:latest`. The current pin is `2.4.1`. **No 2.x image is
|
||||
published on ghcr.io yet** (the newest there is `1.3.0`); until one is, build
|
||||
it from the checkout under the pinned name, and Compose uses the local image
|
||||
instead of pulling:
|
||||
|
||||
```sh
|
||||
docker build -f build/Dockerfile --build-arg VERSION=2.4.1 \
|
||||
-t ghcr.io/mixeme/selfpost:2.4.1 .
|
||||
```
|
||||
|
||||
`VERSION` must equal the tag: it is the version the backup check compares
|
||||
(see below). Intermediate
|
||||
CHANGELOG sections (`0.2.0`…`0.6.0`) record development cuts from before that
|
||||
image was published. Pinning matters because of the backup version check (see
|
||||
[Full backup and restore](#full-backup-and-restore)): the panel binary's
|
||||
@@ -209,11 +221,13 @@ expected to set; defaults match the code exactly.
|
||||
| `DMARC_MESSAGE_SIZE_LIMIT` | Maximum report message size in bytes when DMARC ingest is on. | `5242880` (5 MiB) | `.env` |
|
||||
| `INBOUND_ANTISPAM_MILTER` | Optional milter on the inbound relay listener only (requires `INBOUND_RELAY_ENABLE=true`; not 465/587 or DMARC-only port 25). Empty = off. Format `inet:host:port` or `unix:/path`. Example with [deploy/antispam/docker-compose.antispam.yml](../deploy/antispam/docker-compose.antispam.yml): `inet:antispam:11332`. The panel reads it to report on the *Spam filter* box of the inbound pages and to check that the filter answers (Server › Health). | *(empty)* | `.env` |
|
||||
| `INBOUND_ANTISPAM_MILTER_ACTION` | What Postfix does if that milter is down: `accept` (fail-open) or `tempfail` (defer). The *Spam filter* box states which. | `accept` | `.env` |
|
||||
| `INBOUND_SPAM_LOG_RETENTION_DAYS` | How long the inbound journal keeps a row, 7-365 days; it also never holds more than 10 000 rows. Seeds the setting on the first start with the inbound relay on. | `90` | `.env` |
|
||||
| `INBOUND_JOURNAL_MILTER_SOCKET` | Unix socket of the panel's inbound journal milter, the last milter on port 25 while the inbound relay is on. It records accepted inbound mail and never refuses any; there is no reason to change it. | `/run/selfpost/inbound-journal.sock` | compose |
|
||||
| `INBOUND_RATE_LIMIT_MESSAGES_PER_IP` | Coarse per-client-IP cap on inbound smtpd (`smtpd_client_message_rate_limit`). Uses the same window as `RATE_LIMIT_WINDOW_SECONDS`. | `20` | `.env` |
|
||||
| `INBOUND_MESSAGE_SIZE_LIMIT` | Maximum message size in bytes on inbound smtpd (`message_size_limit`). When both inbound relay and DMARC ingest are on, the shared listener uses the **greater** of this and `DMARC_MESSAGE_SIZE_LIMIT`. | `26214400` (25 MiB) | `.env` |
|
||||
| `RATE_LIMIT_MESSAGES_PER_IP` | Level-1 backstop: maximum messages one client IP may submit per window (Postfix `smtpd_client_message_rate_limit`). See [Rate limiting — level 1](#rate-limiting--level-1-ip-backstop). | `100` | `.env` |
|
||||
| `RATE_LIMIT_WINDOW_SECONDS` | Level-1 window length in seconds (Postfix `anvil_rate_time_unit`). | `3600` | `.env` |
|
||||
| `SEND_LOG_RETENTION_DAYS` | Initial default for how many days of send-log history are kept before the background sweep deletes rows — the main driver of `/data` growth over time. After the first panel start, change retention under **Server › Settings** (global role); the env value is only used to seed SQLite when the setting has never been saved. | `90` | `.env` |
|
||||
| `SEND_LOG_RETENTION_DAYS` | Initial default for how many days of send-log history are kept before the background sweep deletes rows, and with them the `mail.log` lines kept for each message (1–2 KB a message) — the main driver of `/data` growth over time and of the size of a full backup. After the first panel start, change retention under **Server › Settings** (global role); the env value is only used to seed SQLite when the setting has never been saved. | `90` | `.env` |
|
||||
| `PANEL_SESSION_IDLE_DAYS` | Sliding idle timeout for the panel login session, in days. There is no absolute cap: a user who keeps coming back stays signed in indefinitely. | `7` | `.env` |
|
||||
| `SELFPOST_DNS_RESOLVERS` | Comma-separated recursive resolvers the panel's PTR/SPF/DKIM/DMARC checks query directly (so they report what the internet sees, not what this host's stub resolver synthesises). | `1.1.1.1:53`, `8.8.8.8:53`, `9.9.9.9:53` when unset | `.env` |
|
||||
| `TRUSTED_PROXY_CIDR` | Comma-separated CIDRs (bare IPs allowed) of reverse proxies allowed to supply `X-Forwarded-For` for login, setup, and account-change rate-limiting. **Leave unset unless you know the exact address of your reverse proxy.** A wrong value lets a client spoof its rate-limit key by sending a forged `X-Forwarded-For` header — the panel trusts the last hop only when the TCP peer matches one of these CIDRs. Behind the default Apache host-network setup this is typically the Docker bridge gateway, e.g. `172.18.0.1`. | *(empty — XFF ignored)* | `.env` |
|
||||
@@ -374,7 +388,7 @@ and reverse DNS. A check the panel could not run shows as a warning, never as
|
||||
fine. Below the cards are the outbound domain list and, with the inbound relay
|
||||
on, the inbound one. Each card leads to its table on **Server › Health**
|
||||
(`/server/health`) — the queue card to the Outbound queue — and Health has the
|
||||
readings:
|
||||
readings, in the order of what stops mail first:
|
||||
|
||||
- **Machine** — processor (core and thread counts), memory and swap, and
|
||||
per-interface network throughput and totals, read from the kernel's
|
||||
@@ -383,7 +397,19 @@ readings:
|
||||
memory is a warning, because both delay or kill the mail path; throughput is
|
||||
only reported.
|
||||
- **Processes**, **TLS certificate** (validity and expiry) and **Milter
|
||||
sockets** (OpenDKIM and the journal-milter).
|
||||
sockets** (OpenDKIM, the sender check and the journal-milter).
|
||||
- **Sender check** — what the strict check of the `From:` line has cost
|
||||
today, in the last 30 days and in the last 12 months: for how long the
|
||||
panel did not answer while Postfix was running, and how many messages
|
||||
Postfix deferred because of it (see [Who an application may send
|
||||
as](#who-an-application-may-send-as)). A restart of the whole container is
|
||||
not counted: mail was stopped for another reason then. What is counted as
|
||||
a message is a connection Postfix turned away - it refuses before a
|
||||
message is offered - so a monitor that only connects during an outage is
|
||||
counted too. Days are UTC days. The card warns when
|
||||
a message was deferred in the last 24 hours. If these numbers do not stay
|
||||
at nothing, the panel is in the way of your mail more than it should be -
|
||||
report it.
|
||||
- **Hostname and reverse DNS** — compares `SELFPOST_HOSTNAME` against the PTR
|
||||
record the internet publishes for this server's IP (forward-confirmed
|
||||
reverse DNS), see [Server-level DNS](#server-level-dns-ptrrdns); use
|
||||
@@ -417,10 +443,11 @@ readings:
|
||||
**Account** (`/account`, every role) is the signed-in user's own: **Profile**
|
||||
(username and e-mail) and **Password**. Saving the profile does not ask for the
|
||||
password; changing the password asks for the current one and is rate-limited
|
||||
like a sign-in. The e-mail is the user's own address. Nothing is sent to it
|
||||
yet; it is what a domain's *Use my e-mail* button fills in (see [Domain-level
|
||||
DNS](#domain-level-dns-spf-dkim-dmarc)). Application SASL logins are separate
|
||||
and are not changed here.
|
||||
like a sign-in. The e-mail is the user's own address: it is where the panel
|
||||
sends [notifications](#notifications) while *Send me notifications* is ticked
|
||||
beside it, and what a domain's *Use my e-mail* button fills in (see
|
||||
[Domain-level DNS](#domain-level-dns-spf-dkim-dmarc)). Application SASL logins
|
||||
are separate and are not changed here.
|
||||
|
||||
### Settings
|
||||
|
||||
@@ -429,7 +456,58 @@ the whole instance: how many days the **Outbound log** keeps its rows before the
|
||||
background sweep deletes them (7–365 days; takes effect on the next six-hour
|
||||
prune cycle without a container restart), and, read-only, the level-1 rate
|
||||
limit that comes from `.env` ([Rate limiting — level 1](#rate-limiting--level-1-ip-backstop)).
|
||||
There is no panel-wide DMARC report address: each domain has its own.
|
||||
There is no panel-wide DMARC report address: each domain has its own. The page
|
||||
also has the switch for [notifications](#notifications) and the two DNS records
|
||||
that make them arrive.
|
||||
|
||||
### Notifications
|
||||
|
||||
The panel writes to its users when something needs them. Four things are
|
||||
reported, each once:
|
||||
|
||||
| What | When a message is sent | A second message |
|
||||
|---|---|---|
|
||||
| A sending domain's DKIM, SPF or DMARC record stops matching | after two hourly checks in a row found it wrong. A resolver that did not answer is not a mismatch; a missing DMARC record is advice on the domain's page, not a notification | when two checks in a row find the records right again |
|
||||
| A level-2 rate limit refuses mail of a sending domain's applications | within a minute of the first refusal | at most once a day for the same domain, while it goes on |
|
||||
| The TLS certificate is about to expire | 14 days before | 3 days before, if it is still the same certificate |
|
||||
| A message has waited in the queue for more than three days | at the next hourly check | when no message that old is left |
|
||||
|
||||
A user with the **global** role is told all of it; a user with the **domain**
|
||||
role is told about the DNS records and rate limits of the outbound domains
|
||||
assigned to them. A message goes to the e-mail on the user's
|
||||
[Account](#account) page and only while *Send me notifications* is ticked
|
||||
there; a user without an e-mail gets none. **Server › Settings** has one switch
|
||||
that turns notifications off for everyone. With it off nothing is checked, so
|
||||
what is wrong when it is turned on again is reported then.
|
||||
|
||||
Notifications are the panel's own mail. They are sent from
|
||||
`postmaster@<SELFPOST_HOSTNAME>`, one message per recipient, are not rows of
|
||||
the Outbound log and count against no rate limit. They name the page of the
|
||||
panel to open and carry no link: the panel does not know the address its
|
||||
reverse proxy serves it under.
|
||||
|
||||
**Two DNS records make them arrive.** The server's own name is not one of your
|
||||
sending domains, so it needs records of its own, shown with the values to
|
||||
publish under **Server › Settings › Notification mail**:
|
||||
|
||||
- a **DKIM** key for the hostname, which the panel generates and signs
|
||||
notifications with (`<selector>._domainkey.<hostname>`, TXT);
|
||||
- an **SPF** record for the hostname: `v=spf1 a -all` (TXT).
|
||||
|
||||
**Server › Health** shows whether each is in place in the *Notification mail*
|
||||
card. Without them notifications are still sent, and a receiver may file them
|
||||
as spam or, if the parent domain publishes a strict DMARC policy, refuse them.
|
||||
Mail of your sending domains depends on neither. If the hostname is itself one
|
||||
of your sending domains, the DKIM record is that domain's and is already
|
||||
published - and whoever administers that domain can then send mail that
|
||||
looks like the panel's; a hostname that is nobody's sending domain is the
|
||||
better choice.
|
||||
|
||||
The e-mail on Account may be at a public mail host (Gmail and the like); only
|
||||
a domain's DMARC report address may not.
|
||||
|
||||
A relay that cannot send cannot say so: none of the four is "the relay is
|
||||
down". Watch that from outside ([Container health](#container-health)).
|
||||
|
||||
### Users
|
||||
|
||||
@@ -542,10 +620,14 @@ journal-milter (level 2) is down. There is no per-IP bypass.
|
||||
### Full backup and restore
|
||||
|
||||
**Full backup** is a self-contained project archive: `data/` (SQLite, all
|
||||
domains' DKIM keys, all applications' SASL credentials, the Postfix queue,
|
||||
`manifest.json` with the version that created it), plus `docker-compose.yml`,
|
||||
`.env`, and `certs/` from the operator directory next to `./data`. Delivery
|
||||
logs under `data/log/` are excluded. The base compose file mounts the project
|
||||
domains' DKIM keys, all applications' SASL credentials, `manifest.json` with
|
||||
the version that created it), plus `docker-compose.yml`, `.env`, and `certs/`
|
||||
from the operator directory next to `./data`. Delivery logs under `data/log/`
|
||||
are excluded, and so is the Postfix queue: the archive is taken while the
|
||||
instance runs, and a copy of a queue Postfix is working on can hold a message
|
||||
twice or not at all. Mail still waiting for delivery is therefore not carried
|
||||
over, and a restored instance starts with an empty queue; to keep the queue,
|
||||
stop the instance and use `tar` (methods 3 and 4 below). The base compose file mounts the project
|
||||
directory read-only at `/selfpost-deploy` so the panel and CLI can read those
|
||||
deploy files — without that mount, *Full backup* refuses with an error.
|
||||
|
||||
@@ -556,12 +638,25 @@ instance keeps running, what goes into the archive, and what happens on restore.
|
||||
|
||||
**1. Panel — while the container is running (no downtime).** *Server › Backup*
|
||||
→ *Full backup*. *Encrypt with a password* is ticked; untick it for a plain
|
||||
archive. Same archive as the CLI below.
|
||||
archive. Same archive as the CLI below. The panel runs as its own user, not as
|
||||
root, so **the private key in `certs/` has to be readable by it**: give the
|
||||
file to the container's `selfpost` group, which has the fixed id 998 and holds
|
||||
both the panel and Postfix —
|
||||
|
||||
```sh
|
||||
chgrp 998 certs/privkey.pem && chmod 640 certs/privkey.pem
|
||||
```
|
||||
|
||||
— and have whatever renews the certificate do the same (a certbot deploy hook,
|
||||
for one). A key readable by root alone is enough for Postfix, but then the
|
||||
panel refuses to make a backup and names the file: it does not hand over an
|
||||
archive without the key.
|
||||
|
||||
**2. `selfpost-backup` CLI — while the container is running (no downtime).**
|
||||
The panel button and this command call the same code path: a consistent SQLite
|
||||
snapshot via `VACUUM INTO`, then a gzip tar of `data/` (minus diagnostics) plus
|
||||
deploy files.
|
||||
deploy files. `docker exec` runs the command as root, so it reads the private
|
||||
key whatever its permissions.
|
||||
|
||||
```sh
|
||||
docker exec <container> selfpost-backup > selfpost-backup.tar.gz
|
||||
@@ -616,6 +711,8 @@ methods avoid it by stopping first.
|
||||
| Self-contained archive | Yes | Yes | No — deploy files separate |
|
||||
| SQLite consistency | `VACUUM INTO` snapshot | Safe when stopped | Safe when stopped |
|
||||
| `data/log/` | Excluded | Included (optional exclude) | Included (optional exclude) |
|
||||
| Postfix queue | No | Yes | Yes |
|
||||
| `certs/` private key | Yes (the panel needs it readable by group 998) | Yes | No — deploy files separate |
|
||||
| `data/manifest.json` | Written (version stamp) | Absent | Absent |
|
||||
| Restore version check | Yes — must match image tag | No | No |
|
||||
| Post-restore *Resync* | Yes, on first boot | No | No |
|
||||
@@ -691,6 +788,15 @@ needed) for the new hostname or proxy, then `docker compose up -d`. The archive
|
||||
carries `certs/` from the old host — re-issue certificates when the hostname or
|
||||
IP changes. Set up the reverse-proxy vhost separately (not in the backup).
|
||||
|
||||
The inbound log and the filter lists are in the database and so in the
|
||||
backup, and so are the list files under `data/antispam/`; the panel writes
|
||||
those files again from the database on the first start after a restore. What
|
||||
the backup does not hold is the spam filter's own side: the `antispam/`
|
||||
directory with the compose fragment and the rspamd configuration. Like the
|
||||
reverse-proxy fragments it is part of the installation, not of its state -
|
||||
copy it from [deploy/antispam/](../deploy/antispam/) again, or the restored
|
||||
instance runs with no filter.
|
||||
|
||||
**Restoring an encrypted (`.spbk`) backup** needs a running container to
|
||||
decrypt it first — any container with the `selfpost-backup` CLI works; decryption
|
||||
does not read `/data` and performs no version check. Start one normally
|
||||
@@ -822,6 +928,58 @@ only sees SelfPost. Default action is fail-open (`accept`) so a down sidecar
|
||||
does not block backup-MX; set `INBOUND_ANTISPAM_MILTER_ACTION=tempfail` to
|
||||
defer instead.
|
||||
|
||||
The fragment mounts one file into the sidecar,
|
||||
[deploy/antispam/rspamd/milter_headers.conf](../deploy/antispam/rspamd/milter_headers.conf):
|
||||
it makes rspamd write its verdict - the action and the score with its symbols
|
||||
- into two headers of every message it lets through. SelfPost records them
|
||||
with the message, and the upstream receives them with the mail. Copy the
|
||||
`antispam/rspamd/` directory next to the fragment; without the file accepted
|
||||
mail is still recorded, with no verdict.
|
||||
|
||||
The same file makes rspamd **sign** each message, in a third header,
|
||||
`X-SelfPost-Filter`. The filter is attached so that mail still flows when it
|
||||
is down, and then a sender could write a verdict of his own into those
|
||||
headers; SelfPost therefore records a verdict only next to a signature it can
|
||||
recompute. The key is `data/antispam/filter.key`: SelfPost creates it on its
|
||||
first start with the inbound relay on, and the sidecar reads it from the
|
||||
directory it already mounts - there is nothing to configure. If Inbound › Log
|
||||
shows *selfpost* for mail the filter did judge, the sidecar is running without
|
||||
this file: copy `antispam/rspamd/` again and restart the sidecar. Nothing of
|
||||
rspamd's needs clearing, then or after a change of the lists or of the key: it
|
||||
rereads the list files by itself and reads the key anew for every message. A
|
||||
restart is needed only for a change of its own configuration - the files of
|
||||
`antispam/rspamd/`.
|
||||
|
||||
The same directory holds
|
||||
[multimap.conf](../deploy/antispam/rspamd/multimap.conf), the rules for the
|
||||
**allow and deny lists** kept in the panel. SelfPost does not apply the lists
|
||||
itself: it writes them as six files under `data/antispam/` (`allow_ip.map`,
|
||||
`allow_from.map`, `allow_domain.map` and the three `deny_` ones), the fragment
|
||||
mounts that directory into the sidecar, and rspamd rereads a file within
|
||||
seconds of a change - no restart. An entry is a sender address, a sender
|
||||
domain (that domain, not its subdomains) or a client address or network. A
|
||||
denied sender is refused outright. An allowed one is given a weight that no
|
||||
spam score outweighs, so it is accepted - unless a deny entry matches the same
|
||||
message: **deny wins**, whichever entries they are. **A sender address or
|
||||
domain on the allow list is not proof of anything**: it is the envelope
|
||||
sender, which the sending server states and nobody checks, so whoever knows an
|
||||
allowed address can send as it. Allow a client address or network where you
|
||||
can. The files are readable by
|
||||
anyone on the host; they hold addresses, not secrets.
|
||||
|
||||
In the panel the lists are **Inbound › Filter lists** (global role), and what
|
||||
port 25 decided about each message is **Inbound › Log**: accepted mail with the
|
||||
filter's verdict, refused mail with the reply it was given. A row of the log
|
||||
leads to the lists with its sender filled in. Beside each entry of a list is
|
||||
how often it decided a message, counted from the log - so over the days the
|
||||
log keeps - and *never* marks an entry that may have outlived its use. The
|
||||
count depends on two things in
|
||||
[multimap.conf](../deploy/antispam/rspamd/multimap.conf): the message of the
|
||||
deny rules and the names of the allow symbols; change them and every entry
|
||||
reads *never*. A user with the *domain* role
|
||||
sees the log of the inbound domains assigned to them, and not the lists. How
|
||||
long the log keeps a row is set under **Server › Settings**.
|
||||
|
||||
The panel only reports this setting; it does not change it. Every inbound
|
||||
domain page has a **Spam filter** box — on or off, what happens to mail while
|
||||
the filter is down, and that the filter belongs to the server, not to the
|
||||
@@ -891,7 +1049,13 @@ sees the *Ingest* box — whether ingest works, the last report received, report
|
||||
kept this week and parse failures. Open a domain's roll-up from the list or
|
||||
from *DMARC reports* in the domain page's menu. A user with the *domain* role
|
||||
sees the reports of the domains assigned to them, without the *Ingest* box.
|
||||
Summaries are pruned (500 kept, 90 days max).
|
||||
Summaries are pruned (500 kept for each domain, 90 days max).
|
||||
|
||||
A message that is not a report the relay keeps — unreadable as an aggregate
|
||||
report, about a domain not sent from here, or delivered to another domain's
|
||||
report address — is dropped without a retry and without a bounce. It adds one
|
||||
to *parse failures*, and the reason is on the message's delivery line in the mail log
|
||||
(*Server › System log*, `dropped: …`).
|
||||
|
||||
**Not an open relay.** The inbound smtpd offers no SASL. With DMARC ingest
|
||||
alone, `check_recipient_access` permits only configured report addresses;
|
||||
@@ -935,6 +1099,48 @@ to a domain:
|
||||
level 2](#rate-limiting--level-2-domain-and-application)), *Export domain*
|
||||
and, for the global role, *Delete domain*, which asks once more.
|
||||
|
||||
#### Who an application may send as
|
||||
|
||||
An application sends as any address of its domain, or only as the addresses
|
||||
listed for it. Since `2.4.0` that rule covers both senders a message has:
|
||||
|
||||
- the **envelope sender** (`MAIL FROM`), which Postfix checks;
|
||||
- the **`From:` line** of the message - the one a mail program shows and the
|
||||
one the DKIM signature is chosen by - which the panel checks.
|
||||
|
||||
An address with an extension counts as the address: an application listed as
|
||||
`invoices@…` may also send as `invoices+march@…`, in the envelope and in
|
||||
`From:` alike.
|
||||
|
||||
A message whose `From:` is anything else is refused with `550 5.7.1` and a
|
||||
text naming the line: another of your domains, the server's own name, an
|
||||
address at a domain that is not on this server, a line with two addresses,
|
||||
or no `From:` at all. The line is read strictly, because two programs read
|
||||
it - the check and the signer - and must not be able to read it differently:
|
||||
outside a quoted display name it may hold one `@`, the address's own, and no
|
||||
other name this server signs for may follow an `@` anywhere in it, a quoted
|
||||
name included. A display name with a comma or brackets has to be quoted
|
||||
(`"Shop, Inc." <orders@…>`), as the mail standard asks and mail libraries
|
||||
do. The refusal
|
||||
is permanent, so the application does not send the message again; it is a
|
||||
row of the [Outbound log](#outbound-log) tagged `wrong From`, and the row's
|
||||
page shows what the line said.
|
||||
|
||||
What this stops working when you upgrade from an earlier version:
|
||||
|
||||
- an application that sends with the `From:` of another of your domains -
|
||||
give that domain an application of its own;
|
||||
- an application with a list whose envelope address and `From:` address
|
||||
differ - add the second address to the list;
|
||||
- a contact form that puts its visitor's address into `From:` - put it into
|
||||
`Reply-To:` and send from an address of your own, which is also what
|
||||
DMARC requires.
|
||||
|
||||
The check is strict: while the panel does not answer, Postfix answers with a
|
||||
temporary error (`451`, and `454` to the login) and applications send again
|
||||
later; nothing is accepted unchecked. **Server › Health** counts how often
|
||||
that happens.
|
||||
|
||||
*Export domain* writes a single-domain file; *Import a domain* on *Server ›
|
||||
Backup* reads one back in (**global role only**) — see [Exporting and importing
|
||||
a single domain](#exporting-and-importing-a-single-domain).
|
||||
@@ -1008,8 +1214,8 @@ rate limits, and it takes exact addresses only — CIDR ranges are not parsed.
|
||||
|
||||
It is **not an authentication boundary.** The SASL login itself still succeeds;
|
||||
the check runs in the journal-milter, on the same fail-open path as level 2
|
||||
(below), so if the milter is unavailable the restriction is skipped rather than
|
||||
enforced. Use it to narrow where an application may send from, not to contain a
|
||||
(below), so if that milter does not answer the restriction is skipped rather
|
||||
than enforced. Use it to narrow where an application may send from, not to contain a
|
||||
leaked password — for that, regenerate the password.
|
||||
|
||||
**Level 2 — application** — optional override of the domain limit for one
|
||||
@@ -1032,10 +1238,12 @@ hours and on demand via **Recalculate now**.
|
||||
journal-milter and is deliberately fail-open: if the rate-limit lookup hits
|
||||
a store error, or the connecting client's IP is not available to the
|
||||
milter, level 2 is skipped and the message is accepted rather than held up.
|
||||
The same applies if the milter itself is down — Postfix is configured to
|
||||
accept rather than defer when it cannot reach it. Level 1 is the backstop
|
||||
that keeps working even when level 2 cannot run, and it is the only one of
|
||||
the two that does not depend on the panel process.
|
||||
The same applies if the journal-milter itself does not answer — Postfix is
|
||||
configured to accept rather than defer when it cannot reach it. (Since
|
||||
`2.4.0` a panel that is down altogether defers mail anyway, through the
|
||||
[sender check](#who-an-application-may-send-as) of the same process.) Level 1
|
||||
is the backstop that keeps working even when level 2 cannot run, and it is
|
||||
the only one of the two that does not depend on the panel process.
|
||||
|
||||
### Outbound log
|
||||
|
||||
@@ -1045,8 +1253,12 @@ rows of their domains. It refreshes its rows without resetting a filter being
|
||||
chosen. A row identifies its message and nothing more — time,
|
||||
sender, recipient, subject and status `queued` (accepted, not yet
|
||||
delivered), `sent` (handed off successfully), `deferred` (Postfix is
|
||||
retrying), `bounced` (final failure), or `rejected` (refused — for example
|
||||
by a [level-2 rate limit](#rate-limiting--level-2-domain-and-application));
|
||||
retrying), `bounced` (final failure), or refused before it was queued. A
|
||||
refused row says why in its tag: `rate-limited` (a [level-2 rate
|
||||
limit](#rate-limiting--level-2-domain-and-application)), `wrong client` (the
|
||||
application's client IP list) or `wrong From` (see [Who an application may
|
||||
send as](#who-an-application-may-send-as)); a row refused before `2.4.0`
|
||||
says `rejected`;
|
||||
*Details* opens that row's own page (`/outbound/log/{id}`). That page carries
|
||||
the sending domain, the application it was submitted under, the Postfix
|
||||
queue id and the journal id, beside the message's history — when it was
|
||||
@@ -1056,8 +1268,10 @@ backoff cap, queue lifetime), the same numbers the Outbound queue shows; users
|
||||
with the *domain* role see them here because they cannot open the queue. Under
|
||||
both sit the `mail.log` lines for its queue id: the connection to the
|
||||
receiving server, the server's reply, and the status that reply was filed
|
||||
as. Rows outlive `mail.log`, so an older message's lines may have rotated
|
||||
away; the page says so. Retention is set under **Server › Settings** (global
|
||||
as. The lines are kept in the database with the row, so they stay on the
|
||||
page after `mail.log` has rotated and are deleted together with the row; a
|
||||
message accepted before `2.2.0` shows them only while they are still in the
|
||||
current file, and the page says so when there are none. Retention is set under **Server › Settings** (global
|
||||
role); `SEND_LOG_RETENTION_DAYS` in `.env` is only the initial default until it
|
||||
is changed there.
|
||||
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
# Plan: delivery-log-storage
|
||||
|
||||
**Status:** shipped in `2.2.0` (2026-10-12); agreed the same day (owner: option 1 of the roadmap item, "store the lines")
|
||||
**Date:** 2026-10-12
|
||||
**Version:** `2.2.0` MINOR (one DDL migration; a 2.1.0 data directory boots unchanged).
|
||||
|
||||
---
|
||||
|
||||
## Goal
|
||||
|
||||
A message's page (`/outbound/log/{id}`, mockup `out-message`) shows what
|
||||
Postfix wrote about the message for as long as its row is in the send log.
|
||||
Today the *Delivery log* box is filled by a search of the current `mail.log`
|
||||
on every page view, so it is empty for anything sent before the last daily
|
||||
rotation, while the row itself lives for ninety days.
|
||||
|
||||
## Context (as-built)
|
||||
|
||||
- The log tailer ([internal/logtail](../../internal/logtail/logtail.go))
|
||||
already reads every `mail.log` line once, from a persisted offset, to
|
||||
advance delivery statuses.
|
||||
- `logtail.QueueLines` searches the last 4 MB of the current file for one
|
||||
queue id; `Handlers.deliveryLog` calls it per page view.
|
||||
- A message with several recipients is several send-log rows under one queue
|
||||
id; the page of each shows every line of the message, as the search does
|
||||
today.
|
||||
|
||||
## Decisions
|
||||
|
||||
### What is stored
|
||||
|
||||
A `mail.log` line whose head is a queue id - after the timestamp, the host
|
||||
name and `program[pid]:` - goes into a new table with that id, the line's own
|
||||
time (`lineTime`, the same reading the inbound journal uses) and the line as
|
||||
written, cut to 4096 bytes on a character boundary. The head is matched from
|
||||
the start of the line: the tail is the far end's words. `NOQUEUE` and
|
||||
`warning` are not queue ids - the pattern takes Postfix's short form
|
||||
(hexadecimal) and its long form (no vowels) and nothing else.
|
||||
|
||||
The tailer does not ask whether the id belongs to a send-log row before it
|
||||
writes. The first lines of a message (`client=`, `message-id=`) are logged
|
||||
before the journal milter writes the row at end-of-message, so such a check
|
||||
would drop exactly them.
|
||||
|
||||
**Mail that came in on port 25 is left out** (security review, 2026-10-12). A
|
||||
queue id that `postfix/inbound/smtpd` opened is a stranger's message; it never
|
||||
gets a send-log row, and what a stranger can make Postfix log - his own
|
||||
`message-id`, up to the cut - is not stored to be pruned later. The tailer
|
||||
remembers such ids (4096 at most) until qmgr's `removed` or until another
|
||||
listener opens a message under the id. What is still stored without a row is
|
||||
little: the bounces Postfix generates, and port-25 mail met in mid-flight
|
||||
after a restart.
|
||||
|
||||
Lines are written in one transaction per pass over the log, 500 at most
|
||||
(technical review): a commit per line is a disk sync per line on the one
|
||||
connection the journal milter writes through.
|
||||
|
||||
### Which lines are a row's
|
||||
|
||||
A line is shown on a row's page when the queue id is the same **and** the
|
||||
line's time lies between one hour before the row was written and the queue
|
||||
lifetime (the larger of `maximal_queue_lifetime` and `bounce_queue_lifetime`,
|
||||
as `postconf` reports them) plus a day after it. Postfix reuses a short queue
|
||||
id once the message has left the queue - the id is the microsecond of arrival
|
||||
and the inode of the queue file - so a row that lives ninety days can meet a
|
||||
later message under its id. The window does not rule that out, it limits it
|
||||
to the days the row's own message could still be in the queue; this is the
|
||||
bound the roadmap item asked for. The search of the current file that the
|
||||
page did before had the same exposure for a day. `enable_long_queue_ids`
|
||||
would end it and is a change to the mail path, not part of this item.
|
||||
|
||||
### Retention
|
||||
|
||||
Lines live as long as a send-log row with their queue id does. One rule covers
|
||||
both ends: a line older than an hour whose queue id has no send-log row is
|
||||
deleted. That removes the lines of a row the retention sweep has just deleted,
|
||||
and the lines of messages that never had a row. It runs at start and after
|
||||
every send-log sweep, six-hourly. It asks the send log once per queue id; at a
|
||||
million lines that is still a scan of the table, of the order of a second,
|
||||
during which the one connection is taken.
|
||||
|
||||
There is no cap of its own: with port-25 mail left out, the table is bounded
|
||||
by what authenticated applications send, which level 1 and level 2 limit.
|
||||
|
||||
### After an unclean stop
|
||||
|
||||
The tailer saves its place every few seconds, so after a crash it reads some
|
||||
lines twice. While it is catching up at start a line is written only if the
|
||||
same line is not there already - the rule the inbound journal follows.
|
||||
|
||||
### What is not kept
|
||||
|
||||
- Lines the tailer never reads are not stored: the tail of a file that was
|
||||
rotated away while the panel was down, as with delivery statuses
|
||||
(architecture.md § Log tailer).
|
||||
- A message in the queue across the upgrade to 2.2.0 has only its later lines
|
||||
stored, and the page then shows those alone: the search of the file runs
|
||||
only for a row with nothing stored. The only installation is the owner's
|
||||
test server; no merge of the two sources is built for one moment of it.
|
||||
|
||||
### The page
|
||||
|
||||
The page reads the table: the latest 200 lines of the row, oldest first. When
|
||||
the table has none, it still searches the current `mail.log` as before - that
|
||||
is what shows a message accepted before 2.2.0 while its lines are in today's
|
||||
file. Lines in rotated files are not imported: the only installation is the
|
||||
owner's test server.
|
||||
|
||||
Nothing in the page's layout changes; the `out-message` mockup already draws
|
||||
the box as a `log_pane`. Two texts change, mockup first: the *Outbound log*
|
||||
help topic (it says an old message may show no lines) and the note shown when
|
||||
there are none.
|
||||
|
||||
### Size
|
||||
|
||||
Six to ten lines of about 200 bytes per message: 1-2 KB next to a row of a few
|
||||
hundred bytes. The table becomes the larger part of what the send log costs in
|
||||
`/data` and in a backup; the retention setting bounds both together.
|
||||
|
||||
## Schema (migration `0003_send_log_lines.sql`)
|
||||
|
||||
```sql
|
||||
CREATE TABLE send_log_lines (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
queue_id TEXT NOT NULL,
|
||||
logged_at TEXT NOT NULL,
|
||||
line TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX idx_send_log_lines_queue_id ON send_log_lines (queue_id, logged_at);
|
||||
```
|
||||
|
||||
No foreign key: a line is written before its row exists, and one queue id is
|
||||
several rows.
|
||||
|
||||
## Scope
|
||||
|
||||
In: the table, the tailer writing and pruning it, the page reading it, the two
|
||||
texts, docs.
|
||||
|
||||
Out: the inbound journal (it decides for itself, roadmap § delivery-log-storage);
|
||||
importing rotated files; `enable_long_queue_ids`; any change to `mail.log`
|
||||
rotation or to the Server › Mail log page.
|
||||
|
||||
## Security
|
||||
|
||||
- The lines carry the far end's text (replies, HELO names). They are shown
|
||||
today from the file through the same escaping; storing them changes where
|
||||
they are read from, not what is shown or to whom. A domain administrator
|
||||
sees the lines of messages of their own domains, as now.
|
||||
- They now enter the database and so a full backup, which `log/` does not:
|
||||
submitting clients' addresses and SASL logins, message ids, receivers'
|
||||
replies. [security.md](../security.md) says so.
|
||||
- A line is matched from its head, so a reply cannot make itself a line of
|
||||
another message; Postfix does not log raw line breaks, and `postlogd` is the
|
||||
file's only writer.
|
||||
- A stranger on port 25 does not write to the table (above).
|
||||
|
||||
## Done when
|
||||
|
||||
A message sent before the last rotation shows its lines on its page; the lines
|
||||
disappear with the row; the page's wording no longer blames rotation for an
|
||||
ordinary case.
|
||||
|
||||
## Model routing (this plan)
|
||||
|
||||
| Role | Model |
|
||||
|------|-------|
|
||||
| All implementation (code, tests, docs) | **Opus** |
|
||||
| Technical review (reviewer ≠ author: a separate read-only agent) | **Opus** |
|
||||
| Security review | **Fable** |
|
||||
|
||||
## Implementation checklist
|
||||
|
||||
Target version cut: **`2.2.0`** (MINOR). One commit per step;
|
||||
[development.md](../development.md) § Plan checklists.
|
||||
|
||||
- [x] Migration `0003_send_log_lines.sql`, store methods (insert, insert once, lines of a row, prune) and their tests — **Opus**
|
||||
- [x] Log tailer: store queue-id lines, write once while catching up, prune with the send-log sweep — **Opus** (hourly as first built; six-hourly after the reviews)
|
||||
- [x] The two texts, mockup first: the *Outbound log* help topic and the empty note (`design:` commit) — **Opus** (the help topic is in the mockups; the empty note is not drawn there - the `out-message` mockup shows a message with lines - and changes with the page)
|
||||
- [x] Message page reads the table, falls back to the current `mail.log`; help text in `help.go` — **Opus**
|
||||
- [x] [architecture.md](../architecture.md), [guide.md](../guide.md), [schema-migrations.md](../schema-migrations.md) — **Opus** (schema-migrations.md went in with the migration)
|
||||
- [x] `go vet`, `go test`, e2e: a message's page still lists its lines after a forced rotation — **Opus** (on the test server: vet ok, 22 packages ok, e2e ok with the new step `delivery_log_survives_rotation`)
|
||||
- [x] Technical review — **Opus** (2026-10-12; no Critical or High. Fixed: one transaction per pass instead of a commit per line, the prune asks the send log once per queue id and runs six-hourly, the cut keeps characters whole, tests for what the comments claimed. Left as found and written down under *What is not kept*: lines the tailer never read, and a message in the queue across the upgrade)
|
||||
- [x] Security review — **Fable** (2026-10-12; no Critical or High. Fixed: port-25 mail is not stored, so a stranger cannot grow the table; [security.md](../security.md) states what a backup now carries. The window against queue-id reuse is described as the limit it is)
|
||||
@@ -0,0 +1,260 @@
|
||||
# Plan: from-header-binding
|
||||
|
||||
**Status:** shipped in `2.4.0` (2026-10-12); agreed the same day - the design questions were closed with the owner one at a time and he took it into work ("Да")
|
||||
**Date:** 2026-10-12
|
||||
**Version:** `2.4.0` MINOR (one migration; it refuses mail that was accepted before).
|
||||
|
||||
---
|
||||
|
||||
## Goal
|
||||
|
||||
An application sends only as itself - in the `From:` line a mail program
|
||||
shows and OpenDKIM chooses the key by, not only in the envelope.
|
||||
|
||||
Until now Postfix held the envelope sender against the SASL login and nothing
|
||||
held the header. An application of one sending domain could write another
|
||||
registered domain, or the server's own name, into `From:` and have the message
|
||||
signed with that name's key (security review of panel-notifications,
|
||||
2026-10-12).
|
||||
|
||||
## Decisions (owner, 2026-10-12)
|
||||
|
||||
### The rule
|
||||
|
||||
**`From:` is held to what the envelope is held to.** An application set to
|
||||
any address of its domain may write any address of that domain; one with a
|
||||
list of addresses, only those. Anything else is refused: another registered
|
||||
domain, the server's name, an address at a domain this server knows nothing
|
||||
of, no `From:` at all.
|
||||
|
||||
I proposed the narrower rule - refuse only a `From:` that names a domain this
|
||||
server signs for, and let a stranger's address through unsigned as before -
|
||||
and, for an application with a list, to check the domain alone. The owner
|
||||
chose the wider one both times. What it stops working: a form that puts its
|
||||
visitor's address into `From:`, and an application with a list whose envelope
|
||||
address and header address differ, until the second is added to the list.
|
||||
|
||||
When the hostname is itself a sending domain its applications write
|
||||
`…@<hostname>` by right, as the guide already says.
|
||||
|
||||
### Where it is checked, and what happens when it cannot be
|
||||
|
||||
**The panel checks, and the check is strict:** while the panel does not
|
||||
answer, Postfix replies with a temporary error and accepts nothing unchecked.
|
||||
|
||||
I first proposed the opposite (fail-open, as the journal and the level-2 limit
|
||||
are) and then a small process of its own. The owner's objection to the first:
|
||||
a check that is absent while the panel is down does not close the hole, and
|
||||
the panel only has to be slow - 15 seconds - for Postfix to go on without it.
|
||||
His answer to the second: not yet; measure first.
|
||||
|
||||
So the check is a role of the panel process on a socket of its own, attached
|
||||
with `default_action=tempfail`. The journal milter keeps its socket and stays
|
||||
fail-open: a slow database still does not defer mail. The check reads no
|
||||
database - it reads the file of sender bindings Postfix itself reads for the
|
||||
envelope, so the two cannot disagree - and moving it into a process of its own
|
||||
later is a change of where it is started.
|
||||
|
||||
### Statistics, to decide whether the check needs its own process
|
||||
|
||||
Per day, kept a year:
|
||||
|
||||
- **for how long the panel did not answer while Postfix was running.** The
|
||||
panel marks itself alive every few seconds; at start the gap since the last
|
||||
mark is its downtime, counted when Postfix was not restarted within it (a
|
||||
rollout or a reboot stops mail for another reason);
|
||||
- **how many messages were deferred because the check did not answer**, read
|
||||
from `mail.log`. This one also sees a panel that is running and not
|
||||
answering, which the mark does not.
|
||||
|
||||
Shown on **Server › Health** in a card of its own, *Sender check*: today, the
|
||||
last 30 days, the last 12 months. The card warns when a message was deferred
|
||||
in the last 24 hours; downtime that deferred nothing warns of nothing.
|
||||
|
||||
### What a refusal looks like
|
||||
|
||||
- **To the application:** `550 5.7.1`, permanent, naming the address. A
|
||||
level-2 limit answers 4xx because the message will pass in an hour; a wrong
|
||||
`From:` will not, and a 4xx would have the application send it again for
|
||||
days.
|
||||
- **In the Outbound log:** a row with the reason in the status tag -
|
||||
`wrong From`, beside `rate-limited` and the tag of a refused client address.
|
||||
The built panel wrote `rejected` for all of them; it follows the mockup now.
|
||||
(The mockup's status filter is not built - the log filters by domain and
|
||||
application - so there is nothing to keep in step with the tags.)
|
||||
- **On the message's page:** a fact *From: line* with the address written in
|
||||
the header, and a History step that says which address was refused, for
|
||||
which application, what that application may write, and that it was told
|
||||
`550 5.7.1`.
|
||||
|
||||
### Health, re-laid
|
||||
|
||||
The owner on the first picture of the new card: too many holes on the screen,
|
||||
the numbers should follow importance, and the boxes should read left to right
|
||||
and top to bottom - with stacked columns the third box was below the fold and
|
||||
the fourth was not.
|
||||
|
||||
- Order: 01 Processes, 02 Milter sockets, 03 Sender check, 04 TLS
|
||||
certificate, 05 Hostname and reverse DNS, 06 Notification mail, 07 Machine,
|
||||
08 Configuration.
|
||||
- Layout `cards` (new, on the components page): two stacks, odd numbers on the
|
||||
left and even on the right, each closed up, so a short box leaves no gap
|
||||
beside a tall one and the eye still goes 01, 02, 03. On a phone the boxes
|
||||
come in the order of their numbers.
|
||||
|
||||
## Settled while building
|
||||
|
||||
- **The header is read twice.** One `From:` field holding one address, by
|
||||
Go's parser; then every `@name` of the raw value is held against OpenDKIM's
|
||||
own `SigningTable`. The first build took the names for the second look from
|
||||
the sender bindings, and the e2e stand let a line through that named a
|
||||
domain with a key and no application yet: a domain is a signing name from
|
||||
the moment it is added, not from its first application.
|
||||
- **What Postfix writes when the check does not answer** (seen on the stand):
|
||||
a warning naming the socket, then `milter-reject: CONNECT … 451 4.7.1
|
||||
Service unavailable` and the same for `EHLO`; the client is told `454 4.3.0
|
||||
Try again later` at `AUTH` and leaves. No `MAIL` is ever offered, so what
|
||||
is counted is the connection turned away, once. The journal milter's
|
||||
level-2 refusal is written in the same words, which is why a reject line
|
||||
counts only after the warning.
|
||||
- **When Postfix started** is asked of supervisord (`supervisorctl status`,
|
||||
the uptime of `postfix`).
|
||||
- **The reply** is a milter reply code, `550 5.7.1 From: <line> - <why>`; what
|
||||
the client wrote is reduced to printable ASCII and cut first.
|
||||
- **The Delivery log box stays on a refused message's page**, with the one
|
||||
sentence it already had for a message that never reached the queue; the
|
||||
picture shown to the owner had left the box out.
|
||||
|
||||
## To settle while building (as first written)
|
||||
|
||||
- How the header is read so that the check and OpenDKIM cannot disagree: one
|
||||
`From:` field, one address, and a cruder second look at the raw value.
|
||||
- What Postfix writes in `mail.log` when a strict milter does not answer, and
|
||||
how a line is told to be this milter's and not OpenDKIM's.
|
||||
- How the panel learns when Postfix started.
|
||||
- Schema: the reason of a refusal and the header address on a send-log row;
|
||||
the table of days.
|
||||
- That a refusal for `From:` is not taken for a refusal under a limit by the
|
||||
notifier.
|
||||
|
||||
## Implementation checklist
|
||||
|
||||
- [x] Mockups: Health card and layout, the log row, help topic; accepted by the owner 2026-10-12 (`design:` commit)
|
||||
- [x] Migration and store: refusal reason and header address, days of the sender check (`0006_sender_check.sql`)
|
||||
- [x] The check: a role of the panel on its own socket, bindings read from Postfix's file; Postfix attaches it strictly; start order
|
||||
- [x] Statistics: the alive mark and downtime, deferred mail from `mail.log`
|
||||
- [x] Panel: Health (card, sockets row, layout), Outbound log tags, the message's page, help
|
||||
- [x] guide.md, architecture.md, security.md (the accepted risk goes), schema-migrations.md
|
||||
- [x] `go vet`, `go test`, e2e: refused for another domain's `From:`, for the hostname's and for an address off the list; accepted for its own (on the test server: vet ok, 23 packages ok, e2e ok - `sender_check_holds_the_from_line`, `panel_down_defers_mail`)
|
||||
- [x] Technical review — **Opus** (2026-10-12; one Critical, fixed: a folded `From:` line - what a mail library makes of a long display name - was refused for good. Also fixed: a display name in a charset Go does not convert was refused; the statistics ran before the check served, and a hung `supervisorctl` would have kept it from serving; the count of deferred mail missed a check that stops answering in the middle of a message, and is now taken at Postfix's warning; a card whose figures could not be read said ok; a killed container's stale socket satisfied the wrapper's wait; outside quoted strings the line may now hold one `@` only. Three were first left as found and fixed the same evening at the owner's question: downtime is cut at the UTC midnights it crossed; the row of a message with the null envelope sender takes the domain the login is bound to; a log read again after an unclean stop does not count again what it counted before)
|
||||
- [x] Security review — **Fable** (2026-10-12; no Critical or High. The same folded line, as its one Medium. Fixed: another sender's name in an encoded display name passed the second look; a client could arm the deferred counter by writing the socket's name where Postfix quotes it in a warning; a refused message wrote a row per recipient, a thousand at most - now one. Recorded: what a display name looks like to a person is not promised. The review also found the check stricter than Postfix for an address with an extension (`invoices+x@`, allowed in the envelope and refused in the header); the owner asked for it to work, and the check now makes Postfix's second try, with `recipient_delimiter` set explicitly. Confirmed from the code: no path to the signer around the check, and every failure of the check defers)
|
||||
|
||||
## Not done
|
||||
|
||||
Written at the owner's word when he had the version cut ("Запиши все
|
||||
недоделки в план", 2026-10-12). Nothing here was promised and dropped in
|
||||
silence; each line says what is missing and what it would take.
|
||||
|
||||
### Decided to wait
|
||||
|
||||
- **A process of its own for the check.** The owner: measure first. Server ›
|
||||
Health now has the numbers; the decision is his when they have had time to
|
||||
say something. Moving the role out is a change of where it is started -
|
||||
a subcommand of the panel binary and a `[program:]` in supervisord - and
|
||||
the downtime figure then needs a new meaning.
|
||||
|
||||
### Shown to the owner one way and built another
|
||||
|
||||
- **The Delivery log box is on a refused message's page.** The picture he
|
||||
approved had left it out; the built page keeps it, with one sentence. And
|
||||
the sentence is less than the truth for a `From:` refusal: cleanup did
|
||||
write a line about it under a queue id (`milter-reject: END-OF-MESSAGE ...
|
||||
5.7.1 From: ...`), but the check answers before the queue id reaches it, so
|
||||
the row keeps none and the page cannot show that line. **Decided
|
||||
2026-10-13 (owner):** the box is required - he had overlooked its absence
|
||||
from the picture, it was never a decision to drop it. The tailer attaches
|
||||
the line to the row, for all three kinds of refusal; a step of
|
||||
[audit-2026-10-12](../audit-2026-10-12.md) stage 2b.
|
||||
- **The Outbound log has no status filter.** The mockup draws one; the built
|
||||
log filters by domain and application. With three kinds of refusal now
|
||||
told apart by tag, a filter by status and reason would be worth having.
|
||||
**Decided 2026-10-13 (owner):** a filter by status, without the split by
|
||||
reason, and a filter by day as the Inbound log has; a step of [audit-2026-10-12](../audit-2026-10-12.md) stage 4.
|
||||
- **"Messages deferred" counts connections Postfix turned away.** Postfix
|
||||
refuses from the connection on, before a message is offered, so a monitor
|
||||
that only connects during an outage is counted. The guide says so; the
|
||||
card's column head and the help topic still say "messages". **Decided
|
||||
2026-10-13 (owner):** the column is renamed *Connections turned away*; a
|
||||
step of [audit-2026-10-12](../audit-2026-10-12.md) stage 2b.
|
||||
|
||||
### Known and small
|
||||
|
||||
- **"Today" on the card is the UTC day**, and the card does not say so.
|
||||
**Decided 2026-10-13 (owner):** not a label - the local day; it became the
|
||||
roadmap item `local-time`, for the whole panel.
|
||||
- **A deferral in the very second of the last one counted before an unclean
|
||||
stop is passed over** at the restart: `mail.log` stamps whole seconds.
|
||||
Chosen over counting a stretch of the log twice. **Decided 2026-10-14
|
||||
(owner):** counted exactly - the counter keeps a place in the log, saved
|
||||
with the count; a step of [audit-2026-10-12](../audit-2026-10-12.md)
|
||||
stage 2b.
|
||||
- **Deferrals are lost when `mail.log` is rotated while the panel is down**:
|
||||
the tailer starts on the new file. **Decided 2026-10-13 (owner):** the
|
||||
tailer reads the rotated file to its end first - not the deferrals alone,
|
||||
everything the panel takes from the log; a step of
|
||||
[audit-2026-10-12](../audit-2026-10-12.md) stage 2b.
|
||||
- **A start at which `supervisorctl` fails counts no downtime**; it is
|
||||
logged. **Decided 2026-10-13 (owner):** the question is repeated, and
|
||||
without an answer the downtime is counted from the alive mark and the day
|
||||
marked as estimated; a step of
|
||||
[audit-2026-10-12](../audit-2026-10-12.md) stage 2b.
|
||||
- **Logins are compared as written**, Postfix compares them without regard
|
||||
to case. The panel writes logins one way, so nothing differs today.
|
||||
- **The phone order of the `cards` layout is written for four boxes to a
|
||||
stack** and takes every child of a stack for a box (said at the partial).
|
||||
The rule the technical review suggested instead - the order from the
|
||||
box's own number - changes the component and would be drawn first.
|
||||
**Decided 2026-10-13 (owner):** that rule is built; a step of
|
||||
[audit-2026-10-12](../audit-2026-10-12.md) stage 4.
|
||||
- **Health's socket check looks at the file, not at who listens**: a socket
|
||||
nobody answers on would read "ok". The start now removes stale sockets,
|
||||
which was the one way to get such a file. **Decided 2026-10-13 (owner):**
|
||||
the check connects to the socket; a step of
|
||||
[audit-2026-10-12](../audit-2026-10-12.md) stage 4.
|
||||
- **The kit page's table of status tags does not list the new ones**
|
||||
(`wrong From`, `wrong client`, `rate-limited`), and `rate-limited` is grey
|
||||
beside two red tags - as the mockup has it, not as a decision.
|
||||
- **Refusal rows still in the buffer when the panel stops are not written**;
|
||||
the refusals themselves were answered. **Decided 2026-10-13 (owner):**
|
||||
the tailer writes such a row from the refusal's line in `mail.log`; a step
|
||||
of [audit-2026-10-12](../audit-2026-10-12.md) stage 2b.
|
||||
|
||||
### Accepted, in security.md
|
||||
|
||||
- **The panel's downtime is mail's.** The price of the strict check.
|
||||
- **What a display name looks like to a person is not promised.** A name of
|
||||
another sender written without an `@` the scan can find passes, signed
|
||||
with the application's own key.
|
||||
- **A malformed packet on the check's socket ends the panel** (the milter
|
||||
library panics on an empty one). The socket is reachable only inside the
|
||||
container, by the group Postfix is in; the journal milter always had this,
|
||||
and what is new is that the price is deferred mail.
|
||||
|
||||
### Not verified
|
||||
|
||||
- **OpenDKIM's reading of a `From:` line** was reasoned about, not run: its
|
||||
parser is not in this repository. The two rules that do not depend on it -
|
||||
no other signing name after any `@`, one `@` outside quotes - are what the
|
||||
claim rests on. A test that feeds the stand's OpenDKIM the lines the check
|
||||
passes and reads which key it signs with would settle it. **Decided
|
||||
2026-10-13 (owner):** that test is built; a step of
|
||||
[audit-2026-10-12](../audit-2026-10-12.md) stage 2.
|
||||
- **The pages were not looked at in a browser on the deployed panel**: I
|
||||
cannot sign in there. They were rendered from the fixtures and
|
||||
screenshotted, the polling and the phone order read from the code and
|
||||
exercised by e2e only as far as the Health numbers.
|
||||
- **No refusal has been seen on the test server itself**, only on the e2e
|
||||
stand: there is no application there I can send as.
|
||||
- **A check that stops answering in the middle of a message** is counted
|
||||
from a warning composed from Postfix's source; the stand only produced the
|
||||
case of nobody listening.
|
||||
@@ -1,6 +1,6 @@
|
||||
# Plan: inbound-antispam-panel
|
||||
|
||||
**Status:** agreed
|
||||
**Status:** shipped in `2.1.0` (2026-10-12)
|
||||
**Date:** 2026-08-19
|
||||
**Version:** `2.1.0` MINOR (opt-in; no change to outbound-only or inbound-without-filter paths). Was `1.10.0`; on 2026-10-09 the owner put the panel redesign first, so this ships on the 2.0 baseline and its screens are built from the redesign's component kit ([panel-redesign.md](panel-redesign.md) § Component kit).
|
||||
|
||||
@@ -34,8 +34,8 @@ Lists let the operator correct false positives/negatives (e.g. always allow
|
||||
|
||||
## Quarantine
|
||||
|
||||
**Not in this plan.** Held-mail review/release is tracked separately as roadmap
|
||||
candidate [inbound-quarantine](inbound-quarantine.md). The journal may still
|
||||
**Not in this plan.** Held-mail review/release is an idea kept outside the roadmap,
|
||||
[inbound-quarantine](inbound-quarantine.md). The journal may still
|
||||
record a `quarantine` **decision label** when the external filter reports it,
|
||||
without SelfPost storing the message.
|
||||
|
||||
@@ -57,8 +57,14 @@ smtpd_milters = { antispam, inbound-journal }
|
||||
```
|
||||
|
||||
Antispam runs first; inbound-journal runs at end-of-message on the **accept**
|
||||
path and reads rspamd-added headers (`X-Rspamd-Action`, `X-Rspamd-Score`, symbol
|
||||
summary when present).
|
||||
path and reads rspamd-added headers (`X-Rspamd-Action`, and `X-Spamd-Result`
|
||||
for the score and symbols - rspamd has no `X-Rspamd-Score`; the sidecar adds
|
||||
both through `deploy/antispam/rspamd/milter_headers.conf`). As built they are
|
||||
read only next to rspamd's signature of the message, `X-SelfPost-Filter`
|
||||
(owner, 2026-10-12: "А rspamd не может дописывать какой-то заголовок, по
|
||||
которому мы будем знать, что это заголовки нашего спам фильтра?") - an HMAC
|
||||
of the queue id, since a fixed mark could be written by a sender as easily as
|
||||
the verdict itself.
|
||||
|
||||
**Antispam rejects** do not reach end-of-message on the journal milter. v1 also
|
||||
tails `mail.log` for inbound `postfix/smtpd` **`milter-reject`** / **`reject`**
|
||||
@@ -73,7 +79,9 @@ antispam hook). Without an external filter:
|
||||
|
||||
- **accept** rows — message relayed (engine `selfpost`, detail empty);
|
||||
- **reject** / **tempfail** rows — from the mail.log tailer (Postfix policy,
|
||||
size limit, unknown recipient, etc.).
|
||||
unknown recipient, a sender domain that does not exist. As built, a refusal
|
||||
whose log line names no recipient - the size limit answered at MAIL FROM -
|
||||
and the per-client rate limit, which Postfix logs as a warning, make no row).
|
||||
|
||||
Panel copy explains that rspamd symbols appear only when the antispam milter is
|
||||
configured.
|
||||
@@ -83,6 +91,25 @@ configured.
|
||||
**Global administrator only** for journal and lists — same as inbound relay. No
|
||||
domain-admin access in v1.
|
||||
|
||||
*As built (2026-10-11):* the lists are the global role's; the **log follows
|
||||
the delegation of Inbound** that 2.0 introduced after this was written - a
|
||||
user with the `domain` role sees the rows of the inbound domains assigned to
|
||||
them and nothing else, without the *Allow / Deny sender* links. That is what
|
||||
the accepted `in-log` mockup draws (no "global role" in its head). The owner
|
||||
was told and can turn it back into a global-only page.
|
||||
|
||||
**Inbound › Log accepted by the owner on 2026-10-12** ("Этот экран
|
||||
утверждаем"), from screenshots of the page built from its fixture, after his
|
||||
changes to it: the sender column is headed *From*; *Why* is not a column but
|
||||
a sheet over the page, opened by the decision tag, holding a centred table of
|
||||
who decided, the verdict and symbols or the reply, and the subject whole
|
||||
(kept on the sheet at his word: the table clips it); a message Postfix itself
|
||||
refused offers no *Deny sender*. Inbound › Filter lists was redrawn at his word
|
||||
the same day - kind in a column of its own, the note on the entry's sheet,
|
||||
matches counted (§ Filter list usage) - and **accepted by him on 2026-10-12**
|
||||
("Да"), after a deny entry he added in the panel refused a test message with
|
||||
the list's reply and the journal recorded it.
|
||||
|
||||
### Retention
|
||||
|
||||
Mirror send-log retention:
|
||||
@@ -101,13 +128,53 @@ Mirror send-log retention:
|
||||
| Entry types | sender address, sender domain (`@domain` suffix), client IP / CIDR |
|
||||
| Precedence | **deny overrides allow** |
|
||||
| Scope | instance-wide (not per inbound domain in v1) |
|
||||
| Where applied | **rspamd map files** under `/data/antispam/` (allow.map, deny.map); documented compose fragment mounts them into the sidecar |
|
||||
| Where applied | **rspamd map files** under `/data/antispam/`, one per list and entry type (`allow_ip.map`, `allow_from.map`, `allow_domain.map`, `deny_…`) - an IP map and a sender map are different map types to rspamd; documented compose fragment mounts them into the sidecar. Deny is a prefilter reject, allow a weight of -100 (as built: two prefilters do not give "deny overrides allow") |
|
||||
| SelfPost pre-check | **out of v1** — lists live in rspamd only |
|
||||
| Edit access | global administrator only |
|
||||
| When inactive | lists editable in the panel but UI warns that sync applies only when `INBOUND_ANTISPAM_MILTER` is set; `postfix reload` / rspamd reload documented in guide |
|
||||
|
||||
List change → atomic map rewrite (same pattern as Postfix maps) → rspamd reload
|
||||
via documented operator step or sidecar `SIGHUP` in the compose fragment.
|
||||
List change → atomic map rewrite (same pattern as Postfix maps). No reload:
|
||||
rspamd rereads a changed map file by itself (about ten seconds on 3.11).
|
||||
|
||||
### Filter list usage (owner, 2026-10-12)
|
||||
|
||||
Each entry shows how often it decided a message (*Matched*, or *never*) and,
|
||||
on its sheet, when it last did - so that entries nothing matches any more can
|
||||
be removed. **The count is read off the journal on every view of the page;
|
||||
nothing is stored for it.** A journal row counts for an entry when the list
|
||||
decided the message - a rejected row whose reply is the deny rule's message,
|
||||
an accepted row whose verdict carries the allow rule's symbol
|
||||
(`SELFPOST_ALLOW_IP` / `_FROM` / `_DOMAIN`, which the journal milter always
|
||||
keeps) - and the entry fits the row's sender or client address the way rspamd
|
||||
matches it (`inbound.Lists.Usage`).
|
||||
|
||||
What follows from counting this way, and is accepted:
|
||||
|
||||
- the count reaches back as far as the journal does: its retention (90 days by
|
||||
default), which is what the sheet says;
|
||||
- a row counts for every fitting entry of the deciding list, since the journal
|
||||
does not say which of two the filter met first;
|
||||
- a message refused at end-of-message is one row whatever its recipient count,
|
||||
so a count is of messages, roughly.
|
||||
|
||||
**When to move to a counter kept with each entry** (a migration adding hits
|
||||
and last-hit to `inbound_spam_list`, bumped where the journal row is written;
|
||||
the owner asked for the condition to be written down). Any one of:
|
||||
|
||||
1. **The journal is cut by its 10 000-row cap and not by age.** Then it holds
|
||||
fewer days than its retention, the sheet's "in the last 90 days" is no
|
||||
longer true, and *never* stops being evidence that an entry is stale. The
|
||||
sign: the log tailer's prune line keeps reporting rows removed while the
|
||||
oldest row of Inbound › Log is younger than the retention.
|
||||
2. **The retention is set short** (towards its minimum of 7 days) for reasons
|
||||
of its own, privacy or disk. A week without a match does not make an entry
|
||||
stale.
|
||||
3. **Counts are wanted that outlive the journal** - since the entry was added.
|
||||
4. **The page is slow.** A view reads every journal row a list decided and
|
||||
tries it against every entry. With the cap that is at most 10 000 rows
|
||||
times the number of entries; past a few hundred entries, measure it.
|
||||
|
||||
Until then no migration is spent on it.
|
||||
|
||||
### Activation
|
||||
|
||||
@@ -200,7 +267,9 @@ flowchart LR
|
||||
## Security
|
||||
|
||||
- Same validation whitelists as inbound relay ([security.md](../security.md)).
|
||||
- Journal rows contain PII — retention and global-admin-only access.
|
||||
- Journal rows contain PII — retention, and access by reach: the global role
|
||||
reads all of it, a `domain` user the rows of assigned inbound domains (as
|
||||
built; see § RBAC).
|
||||
- Inbound journal milter fail-open must not block mail (backup-MX role).
|
||||
- List map writes injection-safe; deny wins over allow.
|
||||
|
||||
@@ -243,14 +312,14 @@ Target version cut: **`2.1.0`** (MINOR), after `2.0.0`. One commit per step;
|
||||
[development.md](../development.md) § Plan checklists.
|
||||
|
||||
- [x] Agree journal fields, retention, and RBAC (this plan § Decisions) — **Composer**
|
||||
- [ ] Migration `0002_inbound_spam_log.sql` on the 2.0 baseline — **Composer**
|
||||
- [ ] Inbound journal-milter + Postfix wiring — **Composer**
|
||||
- [ ] mail.log tailer: inbound reject rows — **Composer**
|
||||
- [ ] List CRUD + validation + atomic rspamd map sync — **Composer**
|
||||
- [ ] Panel: Inbound › Log and Inbound › Filter lists from the kit partials, mockups `in-log` / `in-filter-lists` (add client IP and engine columns mockup-first), evidence pairs — **Composer**
|
||||
- [ ] Backup includes `/data/antispam/` — **Composer**
|
||||
- [ ] Unit + handler tests — **Composer**
|
||||
- [ ] [guide.md](../guide.md), [architecture.md](../architecture.md) — **Composer**
|
||||
- [ ] `go vet`, `go test`, e2e if applicable — **Composer**
|
||||
- [ ] Technical review of inbound antispam changes — **Opus**
|
||||
- [ ] Security review inbound antispam path — **Fable**
|
||||
- [x] Migration `0002_inbound_spam_log.sql` on the 2.0 baseline — **Composer**
|
||||
- [x] Inbound journal-milter + Postfix wiring — **Composer**
|
||||
- [x] mail.log tailer: inbound reject rows — **Composer**
|
||||
- [x] List CRUD + validation + atomic rspamd map sync — **Composer**
|
||||
- [x] Panel: Inbound › Log and Inbound › Filter lists from the kit partials, mockups `in-log` / `in-filter-lists` (add client IP and engine columns mockup-first), evidence pairs — **Composer**
|
||||
- [x] Backup includes `/data/antispam/` — **Composer** (it always did: a backup walks all of `/data`; now held by a test, and the guide says the `antispam/` fragment directory is not in it)
|
||||
- [x] Unit + handler tests — **Composer** (written with each step: store, milter, tailer, lists and their usage, view fixtures held to the outlines, handlers for both roles)
|
||||
- [x] [guide.md](../guide.md), [architecture.md](../architecture.md) — **Composer** (kept up with each step; [security.md](../security.md) follows the security review)
|
||||
- [x] `go vet`, `go test`, e2e if applicable — **Composer** (on the test server for every step; last on `7192c38`: vet ok, 22 packages ok, e2e ok, which relays, refuses, reads the log page and edits a list through the panel)
|
||||
- [x] Technical review of inbound antispam changes — **Opus** (2026-10-12; findings fixed: client address of a connection's later messages, the lists' writer under a lock, the tailer's queue-id memory, hold rows written twice, the cap by time; and, outside the feature, the level-2 limit and client-address list skipped after a connection's first message)
|
||||
- [x] Security review inbound antispam path — **Fable** (2026-10-12; no Critical or High; HELO-forged journal rows and the six-hourly row cap fixed; a sender's own verdict while the filter is down closed by the filter's signature, the owner's idea; accepted risks recorded in [security.md](../security.md))
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Plan: inbound-quarantine
|
||||
|
||||
**Status:** candidate — design TBD; needs explicit agreement before any code
|
||||
**Status:** an idea to think about, not on the roadmap (owner, 2026-10-12) — design TBD; needs explicit agreement before any code
|
||||
**Version:** TBD (`1.x` MINOR if opt-in and backward-compatible; may warrant `2.x`
|
||||
if it changes inbound storage or backup contract).
|
||||
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
# Plan: panel-notifications
|
||||
|
||||
**Status:** shipped in `2.3.0` (2026-10-12); agreed the same day - the design questions were closed with the owner and he took it into work ("Делаем")
|
||||
**Date:** 2026-10-12
|
||||
**Version:** `2.3.0` MINOR (one migration; opt-out by a switch).
|
||||
|
||||
---
|
||||
|
||||
## Goal
|
||||
|
||||
The panel writes to its administrators when something needs them, instead of
|
||||
waiting to be opened.
|
||||
|
||||
## Decisions (owner, 2026-10-12)
|
||||
|
||||
### Events of the first version
|
||||
|
||||
Four, each of them something the panel can already tell and a person has to
|
||||
act on:
|
||||
|
||||
| Event | Who is told |
|
||||
|---|---|
|
||||
| A sending domain's DNS record stopped matching (SPF, DKIM, DMARC) | global users, and `domain` users the domain is assigned to |
|
||||
| A level-2 rate limit refused mail | the same, for the domain of the limit |
|
||||
| The TLS certificate is about to expire | global users |
|
||||
| A message has been in the queue for more than **three days** | global users |
|
||||
|
||||
Three days, not one (owner: "one day is not that much"): Postfix keeps a
|
||||
message five, so two are left to act before it bounces. The threshold is a
|
||||
constant, not a setting.
|
||||
|
||||
Left for later: a process or milter socket down (a relay in that state often
|
||||
cannot send), an inbound upstream unreachable, DMARC failures appearing
|
||||
(noisy). Mail held in quarantine is not an event: quarantine is an idea
|
||||
outside the roadmap.
|
||||
|
||||
### Sender and path
|
||||
|
||||
From `postmaster@<SELFPOST_HOSTNAME>`, submitted locally. Not from one of the
|
||||
sending domains (owner): with several domains the choice is arbitrary, and a
|
||||
message about the certificate has no business coming from a domain.
|
||||
|
||||
These messages have to arrive (owner: "we need the best deliverability for
|
||||
them"), so the server's own name gets what a sending domain has:
|
||||
|
||||
- a **DKIM key for the hostname**, generated by the panel, with which
|
||||
notifications are signed - no domain has to be chosen for it. An SPF record
|
||||
alone would pass DMARC, but not after the user's mailbox forwards the
|
||||
message on, and large receivers treat unsigned mail more strictly;
|
||||
- an **SPF record** for the hostname, `v=spf1 a -all`.
|
||||
|
||||
The records themselves are under **Server › Settings** (owner: "the DNS
|
||||
records go to Settings, Health has only the status"), in a box of their own
|
||||
drawn with the record component of a domain's DNS page: host, value to
|
||||
publish, copy buttons, state. **Server › Health** has a card of its own,
|
||||
*Notification mail*, with the state of each record and a link to Settings.
|
||||
A DMARC record for the hostname is not asked for: the name inherits its
|
||||
parent domain's policy, and with SPF and DKIM aligned it passes any.
|
||||
Without the records notifications are still sent, and the card warns.
|
||||
|
||||
Notifications are the panel's own mail: they are not rows of the Outbound log
|
||||
and count against no level-2 limit.
|
||||
|
||||
A relay that cannot send cannot say so; none of the four events is "the relay
|
||||
is down", and that is why.
|
||||
|
||||
### Who chooses what
|
||||
|
||||
- One switch for the instance under **Server › Settings**.
|
||||
- One checkbox per user under **Account**, "send me notifications" - on when
|
||||
the profile has an e-mail. No choice per event: four events, none of them
|
||||
noise. Per-event choice comes when there are more and some are.
|
||||
- A user without an e-mail gets nothing.
|
||||
|
||||
### When a message is sent
|
||||
|
||||
A background check once an hour for DNS, certificate and queue.
|
||||
|
||||
| Event | Sent | "Cleared" message |
|
||||
|---|---|---|
|
||||
| DNS mismatch | after two checks in a row found it (it has held for an hour). A resolver that did not answer is not a mismatch | after two good checks in a row |
|
||||
| Certificate | 14 days before expiry, and again 3 days before if it is still the same certificate | none |
|
||||
| Level-2 limit refused mail | at the first refusal of a sending domain's applications, then at most once a day for that domain (one subject per domain, as built: what the refused row says of its domain is the client's own word, so the registry is asked) | none |
|
||||
| Queue older than three days | when such a message appears | one, when none is left |
|
||||
|
||||
Event state is kept in the database, so a restart announces nothing a second
|
||||
time.
|
||||
|
||||
## To settle while building
|
||||
|
||||
- How the panel submits: `sendmail` as the unprivileged panel user, and what
|
||||
that needs of the image.
|
||||
- Whether a notification carries a link to the panel - there is no setting
|
||||
for the panel's own address.
|
||||
- Where the refusal of a level-2 limit is noticed: in the journal milter as
|
||||
it refuses, or from the `rejected` rows of the send log on the hourly check.
|
||||
- Schema: the user's checkbox, the instance switch, the event state table.
|
||||
|
||||
## Screens
|
||||
|
||||
Not drawn. The Notifications box was taken out of the `account` and
|
||||
`settings` mockups on 2026-10-10 (owner: 2.0 ships no box for a feature that
|
||||
is not built). Drawing them again - one checkbox on Account, one switch on
|
||||
Server › Settings, the SPF check on Server › Health - is the first step, shown
|
||||
to the owner before any code ([panel-redesign.md](panel-redesign.md)
|
||||
§ Enforcement).
|
||||
|
||||
## Implementation checklist
|
||||
|
||||
- [x] Mockups: Account checkbox, Settings switch and records, Health status card, help topic; accepted by the owner 2026-10-12 (`design:` commit)
|
||||
- [x] Migration and store: user flag, instance switch, event state
|
||||
- [x] Sending: local submission from `postmaster@<hostname>`, message text (`sendmail` as the panel user; signed through `non_smtpd_milters` with a key of the server's own name; no link in the text - the panel does not know its own address)
|
||||
- [x] Background checker: DNS, certificate, queue; the rules of *When a message is sent*
|
||||
- [x] Level-2 refusal event (from the `rejected` rows of the send log, looked at every minute)
|
||||
- [x] Panel: Account, Settings, Health card, help
|
||||
- [x] guide.md, architecture.md, security.md, schema-migrations.md
|
||||
- [x] `go vet`, `go test`, e2e: a level-2 refusal reaches a mailbox as a signed notification (on the test server: vet ok, 23 packages ok, e2e ok; "once" and "clears" are held by the unit tests of `internal/notify`)
|
||||
- [x] Technical review — **Opus** (2026-10-12; no Critical. Fixed: refusals counted by the registered domain and only under a limit, with an index of their own; account e-mails may be at public mail hosts; "said" means sent; DNS checks must be 45 minutes apart; the notifier's own mail is not a queue subject and a cut or unreadable listing is no listing; the tables must name the selector the panel shows; an unanswered `a`/`mx` lookup is not an SPF mismatch; a pass stops between domains at shutdown; days are rounded as a person counts. Left as found: a recipient whose copy failed while another's went out is not retried; a certificate that lives under 14 days is reported at each renewal)
|
||||
- [x] Security review — **Fable** (2026-10-12; no Critical. Fixed: the limit notification was built on the domain a client writes in `MAIL FROM` - a mail cannon and a way to write to another tenant's users; a ceiling of 60 messages an hour; [security.md](../security.md) corrected on who can submit locally and on what the account e-mail now is. Recorded as accepted, with a roadmap candidate: an application can have a message signed as the server by writing its name into `From:` - the header is not held against the login, for sending domains either)
|
||||
@@ -235,9 +235,9 @@ the first step of that feature's plan — not invented during implementation.
|
||||
| Roadmap item | Where it lands | Built from | Drawn |
|
||||
|---|---|---|---|
|
||||
| inbound-antispam-panel (agreed, `2.1.0`) | journal = **Inbound › Log** (add client IP and engine to the mockup's columns; `quarantine` joins the decisions); lists = new **Inbound › Filter lists**, global-only because the lists are instance-wide | list layout: add form + two tables, `status_tag`, filter form in `box_head`; the log links *Allow / Deny sender* to the lists | `in-log`, `in-filter-lists` |
|
||||
| inbound-quarantine (candidate) | new **Inbound › Quarantine**, list + detail with *Release* / *Discard* | list layout; detail like `out-message` with `facts`, `log_pane` for headers only, Release beside a `sp-danger-zone` Discard | `in-quarantine`, `in-quarantine-message` — the open questions of its plan (where mail lives, what release means, retention, RBAC) stay open; the mockup assumes SelfPost-held mail, release to upstream, 14 days, delegated like the rest of Inbound |
|
||||
| inbound-quarantine (an idea, off the roadmap since 2026-10-12) | new **Inbound › Quarantine**, list + detail with *Release* / *Discard* | list layout; detail like `out-message` with `facts`, `log_pane` for headers only, Release beside a `sp-danger-zone` Discard | `in-quarantine`, `in-quarantine-message` — the open questions of its plan (where mail lives, what release means, retention, RBAC) stay open; the mockup assumes SelfPost-held mail, release to upstream, 14 days, delegated like the rest of Inbound |
|
||||
| preflight (candidate) | new **Server › Preflight**, next to Health: Health is what runs now, Preflight is the deeper on-demand installation check with a test e-mail form | `postmark` verdict, a table of checks with `status_tag` and a *What to do* column, one form box for the test e-mail | `preflight` |
|
||||
| panel-notifications (candidate) | per-user event choice on **Account**; one switch under **Server › Settings** | checkboxes in a box, `help` | **no** — drawn once, taken out of `account` and `settings` on 2026-10-10 (owner): 2.0 ships no box for a feature that is not built; redrawn as the first step of its plan |
|
||||
| panel-notifications (candidate) | one checkbox per user on **Account** (no choice per event, owner 2026-10-12); one switch under **Server › Settings** | a checkbox in a box, `help` | **no** — drawn once, taken out of `account` and `settings` on 2026-10-10 (owner): 2.0 ships no box for a feature that is not built; redrawn as the first step of its plan |
|
||||
| password-reset (candidate) | a link on sign-in, two signed-out screens (request, set new password) | the signed-out layout | **no** |
|
||||
| delivery-log-storage (candidate) | the *Delivery log* box of **Outbound › Log › message**, filled from a table instead of a grep of today's `mail.log` | `log_pane`, unchanged | `out-message` |
|
||||
| csrf-tokens | no screen; every form gains a hidden field — forms are rendered through one helper so the token cannot be forgotten | — | n/a |
|
||||
@@ -539,5 +539,5 @@ only when both are done. Every stage-2 step ends with the evidence of
|
||||
order suits development, preflight, delivery-log-storage, panel-notifications,
|
||||
password-reset (two screens to draw first), queue as a table with the inbound
|
||||
queue, csrf-tokens
|
||||
(one hidden field in the form partial once security.md is settled),
|
||||
inbound-quarantine after its open questions.
|
||||
(one hidden field in the form partial once security.md is settled).
|
||||
inbound-quarantine is an idea outside the roadmap.
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
# Plan: preflight (installation check page)
|
||||
|
||||
**Status:** candidate
|
||||
**Date:** 2026-08-19
|
||||
**Version:** `1.x` MINOR; `candidate` until explicitly agreed.
|
||||
|
||||
---
|
||||
|
||||
## Goal
|
||||
|
||||
A dedicated `/server/preflight` page (global-admin only) that runs infrastructure-level
|
||||
checks and helps the administrator confirm the instance is fully configured and
|
||||
ready to deliver mail. Each failed check shows an actionable recommendation
|
||||
(what to do or where to look).
|
||||
|
||||
## Mockup
|
||||
|
||||
[preflight](../assets/panel-redesign/panel/preflight.html), under the design
|
||||
contract of [panel-redesign.md](panel-redesign.md): **Server › Preflight**,
|
||||
path `/server/preflight`.
|
||||
|
||||
## Scope
|
||||
|
||||
**In:**
|
||||
|
||||
- rDNS / PTR verification (reuse `dnscheck.Checker.checkServer`)
|
||||
- TLS certificate validity + hostname match (CN/SAN contains `SELFPOST_HOSTNAME`)
|
||||
- Port connectivity: 25, 465, and 587 (if `SUBMISSION_ENABLE`) — dial own
|
||||
public IP from inside the container, expect a 220 banner
|
||||
- HELO/EHLO banner — connect to localhost:25, verify Postfix announces
|
||||
`SELFPOST_HOSTNAME` (not `localhost` or a container ID)
|
||||
- Trusted proxy headers:
|
||||
- If `TRUSTED_PROXY_CIDR` set: verify peer IP is in range and
|
||||
`X-Forwarded-For` is present
|
||||
- If `PANEL_COOKIE_SECURE=true`: verify `X-Forwarded-Proto: https`
|
||||
- If `TRUSTED_PROXY_CIDR` empty: informational warning (panel exposed
|
||||
directly)
|
||||
- DKIM signing — OpenDKIM milter socket reachable (reuse `health.CheckSocket`)
|
||||
- Send test email — admin enters recipient address, system sends via
|
||||
localhost:25, shows queued/error result
|
||||
|
||||
**Out:**
|
||||
|
||||
- Per-domain DNS checks (SPF, DKIM TXT, DMARC) — those live on domain detail
|
||||
pages.
|
||||
- Delivery tracking of test email (only confirms local MTA accepted).
|
||||
|
||||
## Done when
|
||||
|
||||
An administrator can open `/server/preflight`, see a traffic-light summary of all
|
||||
infrastructure prerequisites, read specific fix instructions for any failures,
|
||||
and send a test email to verify end-to-end flow.
|
||||
|
||||
## Risks
|
||||
|
||||
- Port-connectivity check from inside the container may not reflect external
|
||||
reachability if NAT hairpin is unsupported (document limitation).
|
||||
- Test email requires at least one configured domain (or falls back to
|
||||
`postmaster@SELFPOST_HOSTNAME`).
|
||||
|
||||
## Implementation checklist
|
||||
|
||||
Target version cut: TBD (MINOR). One commit per step; code only after roadmap
|
||||
status is **agreed**. See [development.md](../development.md) § Plan checklists.
|
||||
|
||||
- [ ] Port-connectivity helper (`internal/health/port.go`) — dial own IP, read
|
||||
SMTP banner — **Sonnet**
|
||||
- [ ] HELO banner check (connect localhost:25, parse 220 line) — **Sonnet**
|
||||
- [ ] Hostname-in-certificate check (extend or wrap `health.CheckCertificate`)
|
||||
— **Sonnet**
|
||||
- [ ] Handler `handlers_preflight.go` (assemble all checks, actionable
|
||||
recommendations) — **Sonnet**
|
||||
- [ ] Test-email handler (`POST /server/preflight/test-email`, `net/smtp` to
|
||||
localhost:25) — **Sonnet**
|
||||
- [ ] Template `preflight.html` (traffic-light cards + test-email form) —
|
||||
**Sonnet**
|
||||
- [ ] Routes in `web.go`, nav link — **Sonnet**
|
||||
+374
-162
@@ -28,34 +28,44 @@ in `git log` and [CHANGELOG.md](../CHANGELOG.md).
|
||||
|
||||
| ID | Topic | Status | Progress | Plan |
|
||||
|---|---|---|---|---|
|
||||
| preflight | Installation check page (`/preflight`) | candidate | — | [plans/preflight.md](plans/preflight.md) |
|
||||
| contributing | `CONTRIBUTING.md` | candidate | — | — |
|
||||
| inbound-antispam-panel | Inbound antispam journal + allow/deny lists | agreed | 1/12 | [plans/inbound-antispam-panel.md](plans/inbound-antispam-panel.md) |
|
||||
| inbound-quarantine | Inbound spam quarantine (hold / review / release) | candidate | — | [plans/inbound-quarantine.md](plans/inbound-quarantine.md) |
|
||||
| csrf-tokens | Per-session CSRF tokens on state-changing forms | candidate | — | — |
|
||||
| template-data-typing | Typed structs for template data instead of `map[string]any` | candidate | — | — |
|
||||
| structured-logging | `log/slog` with levels and fields | candidate | — | — |
|
||||
| review-2026-08-followups | Remaining findings of the 2026-08-19 code review | candidate | — | — |
|
||||
| panel-notifications | E-mail notifications about important events | candidate | — | — |
|
||||
| delivery-log-storage | Keep a message's `mail.log` lines with its send-log row | candidate | — | — |
|
||||
| password-reset | Password reset by e-mail | candidate | — | — |
|
||||
| health-setup-checks | Server › Health also checks the certificate's name, Postfix's greeting and the trusted proxy (was preflight) | agreed (2026-10-13) | — | — |
|
||||
| test-email | Send a test message from the panel (was preflight) | agreed (2026-10-13) | — | — |
|
||||
| inbound-antispam-panel | Inbound antispam journal + allow/deny lists | shipped `2.1.0` | 12/12 | [plans/inbound-antispam-panel.md](plans/inbound-antispam-panel.md) |
|
||||
| origin-strict | Refuse a state-changing request that carries neither `Sec-Fetch-Site` nor `Origin` (was csrf-tokens) | agreed (2026-10-13) | — | — |
|
||||
| backup-reauth | Ask for the account password before a backup or a domain export is downloaded | agreed (2026-10-13) | — | — |
|
||||
| review-2026-08-followups | Remaining findings of the 2026-08-19 code review | closed 2026-10-14 (rows placed in [audit-2026-10-12](audit-2026-10-12.md)) | — | — |
|
||||
| panel-notifications | E-mail notifications about important events | shipped `2.3.0` | 10/10 | [plans/panel-notifications.md](plans/panel-notifications.md) |
|
||||
| delivery-log-storage | Keep a message's `mail.log` lines with its send-log row | shipped `2.2.0` | 8/8 | [plans/delivery-log-storage.md](plans/delivery-log-storage.md) |
|
||||
| password-reset | Password reset by e-mail | agreed (2026-10-13) | — | — |
|
||||
| local-time | The panel shows time, and counts its days, in a time zone other than UTC | agreed (2026-10-13; design questions open) | — | — |
|
||||
| phone-desktop-mode | A phone browser in "desktop site" mode gets the phone navigation | agreed (2026-10-13: after stages 1 and 2 of audit-2026-10-12) | — | — |
|
||||
| from-header-binding | Hold a message's `From:` header against the application that sends it | shipped `2.4.0` | 9/9 | [plans/from-header-binding.md](plans/from-header-binding.md) |
|
||||
| panel-redesign | Panel redesign (accepted mockups, design contract) | shipped `2.0.0` | 27/27 | [plans/panel-redesign.md](plans/panel-redesign.md) |
|
||||
| schema-squash | Squash SQLite migrations into a 2.x baseline | built in panel-redesign (stage 1) | — | [plans/panel-redesign.md](plans/panel-redesign.md) |
|
||||
| audit-2026-10-12 | Follow-ups of the 2026-10-12 full-tree code audit, seven stages | agreed (2026-10-12: stage 0 first, then 1+2; the rest after) | 6/61 | [audit-2026-10-12.md](audit-2026-10-12.md) |
|
||||
|
||||
**Recommended order** (not binding; owner, 2026-10-09: feature order follows
|
||||
development convenience): **panel-redesign** shipped as `2.0.0` (2026-10-11); the next stage is
|
||||
**inbound-antispam-panel** (`2.1.0`), built in the new component kit —
|
||||
styling its two screens once instead of twice. panel-docs shipped in
|
||||
development convenience): **panel-redesign** shipped as `2.0.0` (2026-10-11) and
|
||||
**inbound-antispam-panel** as `2.1.0` (2026-10-12), built in the new component
|
||||
kit — styling its two screens once instead of twice. **delivery-log-storage**
|
||||
shipped as `2.2.0` the same day, **panel-notifications** as `2.3.0` and
|
||||
**from-header-binding** as `2.4.0`.
|
||||
Nothing is agreed after it: every remaining
|
||||
item is a candidate. panel-docs shipped in
|
||||
[CHANGELOG.md](../CHANGELOG.md) `[1.8.0]`; dmarc-reports in `[1.7.0]`
|
||||
(security review of the ingest path pending); domain-stats-auto-ratelimit in
|
||||
(its ingest path first ran in `2.0.0` and was reviewed on 2026-10-11, see
|
||||
[security.md](security.md)); domain-stats-auto-ratelimit in
|
||||
`[1.6.0]`; send-log-retention in `[1.5.0]`; inbound-relay in `[1.4.0]`;
|
||||
queue-retries in `[1.3.1]`; the 2026-08-13 full-tree review follow-ups are in
|
||||
`[1.3.0]`. Candidates need explicit agreement before they join the queue.
|
||||
|
||||
The 2026-08-19 full-tree code review is closed except for four items, all
|
||||
listed above and none of them urgent: [csrf-tokens](#csrf-tokens),
|
||||
[template-data-typing](#template-data-typing),
|
||||
[structured-logging](#structured-logging) and
|
||||
listed above and none of them urgent: [origin-strict](#origin-strict) (its
|
||||
"add CSRF tokens", decided otherwise on 2026-10-13),
|
||||
template-data-typing (done by `2.0.0`; its last map is a step of
|
||||
[audit-2026-10-12](audit-2026-10-12.md) stage 4),
|
||||
structured-logging ([dropped](#dropped-with-the-condition-for-return) on
|
||||
2026-10-13) and
|
||||
[review-2026-08-followups](#review-2026-08-followups). What that review
|
||||
produced is in [CHANGELOG.md](../CHANGELOG.md) `[1.9.2]`; its own document
|
||||
was removed once these four items carried everything still open, and stays in
|
||||
@@ -68,24 +78,6 @@ is `done/total` checklist steps in that plan ([development.md](development.md)
|
||||
|
||||
---
|
||||
|
||||
## contributing
|
||||
|
||||
**Goal:** `CONTRIBUTING.md` in the root — the dev loop, the checks to run
|
||||
before a PR, the commit protocol; [development.md](development.md) links to it
|
||||
rather than repeating it.
|
||||
|
||||
**Boundary:** process documentation; worth writing once there is an external
|
||||
flow of PRs.
|
||||
|
||||
**Done when:** the file is in the root and development.md does not duplicate
|
||||
it.
|
||||
|
||||
**Dependencies / risks:** with a single developer and no PRs, this is low
|
||||
priority.
|
||||
**Version:** no bearing on semver.
|
||||
|
||||
---
|
||||
|
||||
## panel-docs
|
||||
|
||||
**Status:** done — shipped in `[1.8.0]` (2026-08-18). The implementation plan
|
||||
@@ -121,8 +113,8 @@ action) plus editable allow/deny lists that tune the external filter — without
|
||||
shipping rspamd or another engine inside the image.
|
||||
|
||||
**Boundary:** metadata journal and list sync only; SelfPost does not run the
|
||||
filter ([product.md](product.md)). Quarantine:
|
||||
[inbound-quarantine](#inbound-quarantine). Design closed in
|
||||
filter ([product.md](product.md)). Quarantine is not on the roadmap
|
||||
([Ideas](#ideas-not-on-the-roadmap)). Design closed in
|
||||
[plans/inbound-antispam-panel.md](plans/inbound-antispam-panel.md) § Decisions.
|
||||
|
||||
**Done when:** with inbound relay enabled, the operator sees journal rows
|
||||
@@ -133,130 +125,126 @@ hook is set.
|
||||
**Dependencies / risks:** inbound relay `[1.4.0]`; rspamd map/header coupling;
|
||||
journal PII; global-admin-only RBAC.
|
||||
|
||||
**Version:** `2.1.0` MINOR (was `1.10.0`; reordered after the redesign on 2026-10-09); **agreed**.
|
||||
**Version:** shipped `2.1.0` (2026-10-12), MINOR (was `1.10.0`; reordered after the redesign on 2026-10-09).
|
||||
|
||||
---
|
||||
|
||||
## inbound-quarantine
|
||||
## health-setup-checks
|
||||
|
||||
**Goal:** optional hold for suspicious inbound mail — review in the panel, then
|
||||
release to upstream, discard, or tune lists. Design not settled.
|
||||
**Decided by the owner on 2026-10-13**, in place of the former candidate
|
||||
*preflight* (a page of its own, Server › Preflight, planned in August before
|
||||
Server › Health existed). Of that plan's eight checks Health already makes
|
||||
three (reverse DNS, certificate validity, the DKIM socket); five were missing.
|
||||
Three of the five go into Health, one becomes [test-email](#test-email), one
|
||||
is not built ([Dropped](#dropped-with-the-condition-for-return)). No second
|
||||
page of checks.
|
||||
|
||||
**Boundary:** expands inbound from pure relay toward short-lived message
|
||||
storage; may conflict with "no mailboxes" in [product.md](product.md) until
|
||||
product explicitly agrees. Alternatives (rspamd-only quarantine vs SelfPost
|
||||
store) are listed in [plans/inbound-quarantine.md](plans/inbound-quarantine.md).
|
||||
**Goal:** three more lines on Server › Health, in the cards that are there:
|
||||
|
||||
**Done when:** TBD after open questions in the plan are answered.
|
||||
- **The certificate is for this server**: its names include
|
||||
`SELFPOST_HOSTNAME` (TLS certificate card). Today a certificate for another
|
||||
name is "valid".
|
||||
- **Postfix greets with the hostname**: the 220 line on port 25 names
|
||||
`SELFPOST_HOSTNAME`, not `localhost` or a container id (Hostname card).
|
||||
- **The trusted proxy**: the address the panel sees this request from, and
|
||||
whether it took it from `X-Forwarded-For`. With `TRUSTED_PROXY_CIDR` wrong
|
||||
or empty behind a proxy the header is ignored
|
||||
([internal/web/auth/auth.go](../internal/web/auth/auth.go)), every client
|
||||
has the proxy's address and the sign-in limiter is shared by all of them -
|
||||
silently.
|
||||
|
||||
**Dependencies / risks:** inbound relay `[1.4.0]`; storage/retention/backup
|
||||
size; malware in held MIME; overlap with
|
||||
[inbound-antispam-panel](#inbound-antispam-panel). No implementation checklist
|
||||
yet.
|
||||
**Boundary:** lines in existing cards, each with what to do when it fails;
|
||||
no new card unless the mockup shows the lines do not fit. Mockup-first
|
||||
([plans/panel-redesign.md](plans/panel-redesign.md) § Enforcement). The
|
||||
mockup `preflight` stays drawn and unbuilt.
|
||||
|
||||
**Version:** TBD; `candidate` until explicitly agreed.
|
||||
**Done when:** each line is shown wrong on the e2e stand or in a test with a
|
||||
wrong certificate name, a wrong greeting and an untrusted proxy, and right on
|
||||
the test server.
|
||||
|
||||
**Version:** `2.x` MINOR; `agreed`.
|
||||
|
||||
---
|
||||
|
||||
## preflight
|
||||
## test-email
|
||||
|
||||
**Goal:** a `/preflight` page (global-admin only) that verifies instance-level
|
||||
infrastructure: rDNS/PTR, TLS certificate + hostname match, port 25/465/587
|
||||
reachability, HELO banner correctness, reverse-proxy headers, DKIM milter
|
||||
socket, and a test-email sender. Each check shows an actionable fix
|
||||
recommendation on failure.
|
||||
**Decided by the owner on 2026-10-13**, taken out of the former *preflight*.
|
||||
|
||||
**Boundary:** instance infrastructure only; per-domain DNS checks remain on
|
||||
domain detail pages. Does not track test-email delivery beyond local MTA
|
||||
acceptance.
|
||||
**Goal:** a global administrator sends a test message from the panel to an
|
||||
address they type, and sees whether Postfix took it; the message then appears
|
||||
in Outbound › Log like any other. Today nobody can send through the server
|
||||
without setting up an application first.
|
||||
|
||||
**Done when:** the page loads, all checks run with traffic-light results and
|
||||
recommendations, and the test-email form successfully submits a message through
|
||||
Postfix.
|
||||
**Boundary:** global role only; from `postmaster@SELFPOST_HOSTNAME`, the
|
||||
sender notifications use, or from a sending domain the administrator picks -
|
||||
settled at the mockup. It does not follow the message further than the log
|
||||
does. Where the form lives (Health, or a sending domain's page) is settled at
|
||||
the mockup; mockup-first.
|
||||
|
||||
**Dependencies / risks:** port-connectivity check may not reflect external
|
||||
reachability behind NAT without hairpin; test email needs at least one domain or
|
||||
falls back to `postmaster@SELFPOST_HOSTNAME`.
|
||||
**Done when:** the form sends, the row appears in the log, the form is
|
||||
rate-limited and refuses a recipient list, and it passes a security review -
|
||||
a form that sends mail is a new surface.
|
||||
|
||||
**Version:** `1.x` MINOR; `candidate`.
|
||||
**Version:** `2.x` MINOR; `agreed`.
|
||||
|
||||
---
|
||||
|
||||
## csrf-tokens
|
||||
## origin-strict
|
||||
|
||||
**Goal:** a per-session CSRF token on every state-changing form, checked
|
||||
server-side in addition to the `Sec-Fetch-Site` / `Origin` check the panel
|
||||
relies on today (`originAllowed` in
|
||||
[internal/web/security.go](../internal/web/security.go)).
|
||||
**Decided by the owner on 2026-10-13**, in place of the former candidate
|
||||
*csrf-tokens* (per-session tokens on every form, recorded by the 2026-08-19
|
||||
review): no tokens; the origin check is made strict instead.
|
||||
|
||||
**Boundary:** defence in depth, not a replacement — the header check stays and
|
||||
keeps covering requests that carry no form. The token is bound to the existing
|
||||
session record; no new session model, no cookie changes.
|
||||
**Goal:** `originAllowed`
|
||||
([internal/web/security.go](../internal/web/security.go)) refuses a
|
||||
state-changing request that carries neither `Sec-Fetch-Site` nor `Origin`.
|
||||
Today such a request is let through, which leaves a client that sends neither
|
||||
header open to a request forged by any site
|
||||
([security.md](security.md) § Accepted risks).
|
||||
|
||||
**Done when:** every POST form and htmx fragment that changes state carries a
|
||||
token, a missing or mismatched token is refused with a test that proves it, and
|
||||
[security.md](security.md) describes the layered model in one place instead of
|
||||
two.
|
||||
**Boundary:** one branch of one function, its test, and the documents. No
|
||||
template changes, no hidden fields. Before the change, find what posts to the
|
||||
panel without a browser (the e2e stand, scripts) and make it send `Origin`.
|
||||
|
||||
**Dependencies / risks:** [security.md](security.md) currently records the
|
||||
opposite decision — origin checking without tokens, with the accepted risk and
|
||||
its revisit triggers written out in the ADR § *CSRF via origin checking*. The
|
||||
2026-08-19 review recorded "add tokens" without turning it into a task, which is
|
||||
why this item exists. Settle which decision stands before writing code; both
|
||||
outcomes are honest, and "keep headers" means deleting this item rather than
|
||||
leaving the contradiction standing. The work itself is a hidden field in roughly
|
||||
two dozen forms plus the htmx swap paths.
|
||||
**Done when:** a `POST` with neither header is refused with a test that proves
|
||||
it, the e2e stand passes, the accepted risk about header-less requests is
|
||||
struck from [security.md](security.md) and its ADR states the strict rule.
|
||||
|
||||
**Version:** `1.x` MINOR; `candidate`.
|
||||
**Dependencies / risks:** a browser that sends neither header can no longer
|
||||
save anything in the panel - accepted, the panel's users pick their own
|
||||
browsers. Tokens stay a recorded alternative, not a task: the conditions under
|
||||
which they are built are in [security.md](security.md), ADR § *CSRF via origin
|
||||
checking* (**Revisit if**).
|
||||
|
||||
**Version:** `2.x` PATCH; `agreed`.
|
||||
|
||||
---
|
||||
|
||||
## template-data-typing
|
||||
## backup-reauth
|
||||
|
||||
**Goal:** replace the `map[string]any` template payloads in
|
||||
`internal/web/handlers` with typed structs (`DomainDetailData`, `SettingsData`,
|
||||
...), so a renamed or mistyped key is a compile error instead of a silently
|
||||
empty page section.
|
||||
**Agreed with the owner on 2026-10-13**, out of
|
||||
[review-2026-08-followups](#review-2026-08-followups).
|
||||
|
||||
**Boundary:** handler-to-template plumbing only; no template redesign, no route
|
||||
or behaviour change. `renderDomainDetail`
|
||||
([handlers_apps.go](../internal/web/handlers/handlers_apps.go)) sets around
|
||||
forty keys on one map and is the case that motivates the item; roughly ten
|
||||
render sites still pass maps.
|
||||
**Problem:** Server › Backup hands the full backup and a domain export to any
|
||||
live session of a global administrator
|
||||
([handlers_backup.go](../internal/web/handlers/handlers_backup.go)). The
|
||||
archive holds the whole database - panel password hashes, application (SASL)
|
||||
passwords, DKIM keys - so a stolen session cookie, or a panel left open, is a
|
||||
quiet copy of the server. *Encrypt with a password* does not help: the thief
|
||||
chooses that password.
|
||||
|
||||
**Done when:** the page handlers pass structs, the shared footer/base fields
|
||||
(`Version`, `Copyright`, `SourceURL`) still arrive on every page, and the
|
||||
template guard tests pass unchanged.
|
||||
**Goal:** both downloads ask for the current account password, checked the
|
||||
way a password change checks it and under the same limiter.
|
||||
|
||||
**Dependencies / risks:** mechanical but wide — it touches every
|
||||
`handlers_*.go` and every template that reads those keys, and a missed key is
|
||||
invisible unless a test renders that page. Worth pairing with the handler tests
|
||||
in [review-2026-08-followups](#review-2026-08-followups) so the pages are
|
||||
covered before they are rewired. Reviewer's estimate: 2–3 hours.
|
||||
**Boundary:** one field in each of the two forms (mockup-first); import is
|
||||
not touched; the `selfpost-backup` CLI is not touched - it runs inside the
|
||||
container.
|
||||
|
||||
**Version:** no bearing on semver; `candidate`.
|
||||
**Done when:** a download without the password or with a wrong one is refused
|
||||
with a test that proves it, attempts are limited, and
|
||||
[security.md](security.md) § Backup says so.
|
||||
|
||||
---
|
||||
|
||||
## structured-logging
|
||||
|
||||
**Goal:** move from `log.Printf` to `log/slog` — levels, structured fields, and
|
||||
one logger injected through constructors, instead of the package-level
|
||||
`logf = log.Printf` alias repeated in five packages.
|
||||
|
||||
**Boundary:** internal logging only. The destination stays stdout/stderr under
|
||||
supervisord; no log shipping, no new dependency. Lines the panel's own Status
|
||||
page and log tailer depend on must keep parsing.
|
||||
|
||||
**Done when:** packages take a `*slog.Logger`, the per-package `logf` aliases
|
||||
are gone, and levels are used deliberately (an error is not an info).
|
||||
|
||||
**Dependencies / risks:** the log tailer reads Postfix's log rather than the
|
||||
panel's, so the blast radius is small — but anything that greps the panel's
|
||||
output (e2e assertions, operator habits) sees a new format. Reviewer's
|
||||
estimate: 3–4 hours.
|
||||
|
||||
**Version:** no bearing on semver; `candidate`.
|
||||
**Version:** `2.x` MINOR; `agreed`.
|
||||
|
||||
---
|
||||
|
||||
@@ -269,14 +257,14 @@ is in the table below; the rest of it shipped in
|
||||
|
||||
| Area | Item |
|
||||
|---|---|
|
||||
| Security | Re-authenticate (ask for the current password) before a backup download — [handlers_backup.go](../internal/web/handlers/handlers_backup.go) hands the archive to any signed-in session |
|
||||
| Tests | The `internal/dmarc` service layer (`service.go`, `addresses.go`) has no tests |
|
||||
| Tests | No direct tests for the apps / domains / users / dmarc handlers; `authz_test` covers them only indirectly |
|
||||
| Tests | Nothing covers error page templates, a corrupted backup archive, malformed or oversized DMARC XML, delivery-log pagination edges, concurrent store operations, or zero-traffic `store` stats |
|
||||
| Edge case | `internal/secretfile` accepts an empty password at library level — guard it, or state that validation is the caller's job |
|
||||
| Edge case | The log fingerprint in [logtail/offset.go](../internal/logtail/offset.go) can collide across a rotation — include the inode |
|
||||
| Edge case | `parsePage` in [handlers_monitor.go](../internal/web/handlers/handlers_monitor.go) does not clamp to `lastPage` |
|
||||
| Edge case | [dnscheck/resolver.go](../internal/dnscheck/resolver.go) fans out four parallel queries where a fallback chain would do — change it or write down why not |
|
||||
| Security | ~~Re-authenticate (ask for the current password) before a backup download~~ — struck: agreed on 2026-10-13 as an item of its own, [backup-reauth](#backup-reauth) |
|
||||
| Tests | ~~The `internal/dmarc` service layer (`service.go`, `addresses.go`) has no tests~~ — struck: a step of audit-2026-10-12 stage 4 (DMARC reports tested where they come in) |
|
||||
| Tests | ~~No direct tests for the apps / domains / users / dmarc handlers; `authz_test` covers them only indirectly~~ — struck: users with stage 2 (`submitUserUpdate`), DMARC with stage 4; apps and domains get theirs when a step changes them - a test without a change finds nothing the e2e does not |
|
||||
| Tests | ~~Nothing covers error page templates, a corrupted backup archive, malformed or oversized DMARC XML, delivery-log pagination edges, concurrent store operations, or zero-traffic `store` stats~~ — struck: the corrupted archive goes to stage 3, DMARC XML to stage 4, paging edges to stage 4 (`parsePage`), concurrency of the last global to stage 2; error templates, other concurrency and zero-traffic stats are dropped (owner, 2026-10-14) - code no step changes, and a test kept for its own sake finds nothing |
|
||||
| Edge case | ~~`internal/secretfile` accepts an empty password at library level~~ — struck: audit-2026-10-12 stage 3 (M5) |
|
||||
| Edge case | ~~The log fingerprint in `logtail/offset.go` can collide across a rotation — include the inode~~ — struck: agreed on 2026-10-13 as part of a step of [audit-2026-10-12](audit-2026-10-12.md) stage 2b (a log rotated while the panel was down is read to its end) |
|
||||
| Edge case | ~~`parsePage` does not clamp to `lastPage`~~ — struck: audit-2026-10-12 stage 4 |
|
||||
| Edge case | ~~`dnscheck/resolver.go` fans out four parallel queries where a fallback chain would do~~ — struck: finding M14 of [audit-2026-10-12](audit-2026-10-12.md) |
|
||||
| UI | ~~`.pair` (DMARC templates) and `.split` (every other page) are two names for one layout~~ — struck: retired by [panel-redesign](#panel-redesign); both stylesheets are gone and every page is laid out by the component kit |
|
||||
| UI | ~~Two `back_link` elements in a row in `dmarc_domain.html`~~ — struck: retired by [panel-redesign](#panel-redesign); the page is rebuilt on the kit with one breadcrumb in its head |
|
||||
| UI | ~~`initShowWhen` is not re-run after an htmx swap~~ — struck: the show/hide rules listen on the document and are applied again on `htmx:load` |
|
||||
@@ -286,10 +274,10 @@ apart from the backup re-auth prompt. Anything here that grows past an hour
|
||||
becomes its own roadmap item instead.
|
||||
|
||||
**Done when:** each row is either implemented or struck with a written reason.
|
||||
**Closed 2026-10-14** with the owner: every row is struck above, placed in a
|
||||
stage of [audit-2026-10-12](audit-2026-10-12.md) or dropped with its reason.
|
||||
|
||||
**Dependencies / risks:** the test rows overlap
|
||||
[template-data-typing](#template-data-typing) — write the handler tests first if
|
||||
both are taken up. One finding of that review was **declined** and is
|
||||
**Dependencies / risks:** One finding of that review was **declined** and is
|
||||
deliberately absent: `//go:build linux` for the `/proc` reader in
|
||||
[internal/health/machine.go](../internal/health/machine.go). That code is
|
||||
parameterised by root directory, calls nothing platform-specific, already
|
||||
@@ -348,38 +336,43 @@ time to final status plus the queue lifetime), not to the id alone; the table
|
||||
grows with the send log (small — a few hundred bytes per message) and is
|
||||
covered by the same retention.
|
||||
|
||||
**Version:** `2.x` MINOR; `candidate`. Fits after the redesign, built against
|
||||
the kit's `log_pane`.
|
||||
**Version:** shipped `2.2.0` (2026-10-12), MINOR; option 1. Decisions and
|
||||
checklist: [plans/delivery-log-storage.md](plans/delivery-log-storage.md).
|
||||
|
||||
---
|
||||
|
||||
## panel-notifications
|
||||
|
||||
**Goal:** the panel tells its administrators when something needs them instead
|
||||
of waiting to be opened: a DNS record stops matching, a domain or application
|
||||
hits its rate limit and mail is refused, an inbound upstream is unreachable,
|
||||
DMARC failures appear, mail is held in quarantine; for the global
|
||||
administrator also the TLS certificate nearing expiry, a process or milter
|
||||
socket down, a queue older than a day.
|
||||
of waiting to be opened. First version, four events: a sending domain's DNS
|
||||
record stops matching, a level-2 rate limit refuses mail, the TLS certificate
|
||||
is about to expire, a message has been in the queue for more than three days.
|
||||
|
||||
**Boundary:** e-mail only, sent through the relay itself from
|
||||
`postmaster@<hostname>`. Each user picks events under **Account**; a domain
|
||||
administrator hears only about assigned domains. One message when a problem
|
||||
starts and one when it clears — no streams, no digests in v1. One instance
|
||||
switch under **Server › Settings**. No webhooks, no chat integrations.
|
||||
**Boundary:** e-mail only, from `postmaster@<hostname>`, submitted locally -
|
||||
not from one of the sending domains. One checkbox per user under **Account**
|
||||
(no choice per event), one instance switch under **Server › Settings**; a
|
||||
domain administrator hears only about assigned domains. No streams, no
|
||||
digests, no webhooks, no chat integrations. Not in the first version: a
|
||||
process or milter down, an inbound upstream unreachable, DMARC failures.
|
||||
|
||||
**Done when:** an event from the list reaches the account e-mail of every user
|
||||
who chose it and may see its subject, once on start and once on clear; a user
|
||||
without an e-mail gets nothing; the switch turns it all off.
|
||||
**Done when:** each of the four events reaches the account e-mail of every
|
||||
user who has notifications on and may see its subject, by the rules of the
|
||||
plan (a "cleared" message for DNS and the queue only); a user without an
|
||||
e-mail gets nothing; the switch turns it all off.
|
||||
|
||||
**Dependencies / risks:** the account e-mail (2.0 baseline, see
|
||||
[plans/panel-redesign.md](plans/panel-redesign.md) § Account e-mail); a relay
|
||||
that cannot send cannot report that it cannot send — server events need the
|
||||
direct-to-MX path or stay panel-only; event state must survive restarts so a
|
||||
reboot does not re-announce everything. Screens are drawn: `account`,
|
||||
`settings`.
|
||||
that cannot send cannot report that it cannot send - none of the four events
|
||||
is "the relay is down"; mail from the bare hostname needs an SPF record for
|
||||
it to be believed, which Server › Health will check; event state must survive
|
||||
restarts so a reboot does not re-announce everything. The screens are not
|
||||
drawn: the Notifications box was taken out of `account` and `settings` on
|
||||
2026-10-10 and is redrawn as the first step.
|
||||
|
||||
**Version:** `2.x` MINOR; `candidate`.
|
||||
**Version:** shipped `2.3.0` (2026-10-12), MINOR - the design questions were closed
|
||||
with the owner on 2026-10-12
|
||||
([plans/panel-notifications.md](plans/panel-notifications.md)) and he took it into
|
||||
work the same day.
|
||||
|
||||
---
|
||||
|
||||
@@ -392,8 +385,22 @@ account e-mail instead of asking the global administrator to set a new one.
|
||||
same answer whether or not the username exists. The only global administrator
|
||||
without a working e-mail still recovers from the container (today's path).
|
||||
|
||||
**The e-mail must be confirmed (owner, 2026-10-13).** A reset link goes only
|
||||
to an address its owner has confirmed; an account whose e-mail is not
|
||||
confirmed has no reset, and recovers as today (the global administrator sets
|
||||
a password, or the container). The panel does not confirm addresses today -
|
||||
`users.email` is whatever was typed - so the item builds that first: saving
|
||||
an e-mail under Account mails a one-time link to it, opening the link marks
|
||||
the address confirmed (a migration: the mark on `users`), changing the
|
||||
address clears the mark, and Account shows which it is. An address typed for
|
||||
a user by the global administrator is unconfirmed until that user opens the
|
||||
link. To settle at the mockup: whether notifications (2.3.0) keep going to an
|
||||
unconfirmed address.
|
||||
|
||||
**Done when:** the flow works end to end, tokens expire and cannot be reused,
|
||||
requests are rate-limited, and it passes a security review.
|
||||
requests are rate-limited, an account with an unconfirmed e-mail gets no
|
||||
link and the sign-in page answers the same for it, and it passes a security
|
||||
review.
|
||||
|
||||
**Dependencies / risks:** the account e-mail; an account-takeover surface where
|
||||
there was none — security review is part of the item, not an afterthought.
|
||||
@@ -401,7 +408,145 @@ Not drawn yet. Changing the account e-mail must then require the current
|
||||
password (2.0 security review), because the e-mail becomes where a reset link
|
||||
goes and today Account saves it without the password.
|
||||
|
||||
**Version:** `2.x` MINOR; `candidate`.
|
||||
**Version:** `2.x` MINOR; `agreed` (owner, 2026-10-13: domain administrators
|
||||
to come need it - they are not the owner and should not have to ask him for
|
||||
a new password). Mockup-first; changing the account e-mail asks for the
|
||||
current password in the same cut.
|
||||
|
||||
---
|
||||
|
||||
## local-time
|
||||
|
||||
**Agreed with the owner on 2026-10-13.** It came from one cell: *Today* on
|
||||
Health's Sender check card is the UTC day and does not say so. Offered a
|
||||
label ("Today, UTC") or the local day, he chose the local day - which is a
|
||||
decision about the whole panel, not one cell.
|
||||
|
||||
**Today:** every time the panel shows is UTC and says so (*Time, UTC* in the
|
||||
logs, "UTC" after a date), and everything counted by day - the sender
|
||||
check's days, the delivery-log measurement, the statistics behind the
|
||||
charts, retention - is counted by the UTC day.
|
||||
|
||||
**Goal:** the panel shows time in a chosen time zone, and "today" is that
|
||||
zone's day.
|
||||
|
||||
**Where the zone comes from (owner, 2026-10-13):** one setting of the
|
||||
instance, *Time zone* under Server › Settings, UTC by default; it holds for
|
||||
every user and for notification mail. Not the browser's zone: only a script
|
||||
could read it, and the panel works without JavaScript. Not a setting of each
|
||||
user yet - **that is added (the same list under Account, "as the server" by
|
||||
default) when** a user of the panel lives in another zone than the one set
|
||||
for the instance, which the first domain administrator who is not the owner
|
||||
may bring.
|
||||
|
||||
**What is written beside a time (owner, 2026-10-13):** nothing. The "UTC"
|
||||
labels go - the column is *Time*, a date is a date - and the zone is stated
|
||||
in one place, the setting. His reason: there is nobody to send a line of the
|
||||
panel to yet. **The offset is written where "UTC" is today ("Time, UTC+3",
|
||||
"2026-09-21 16:40 UTC+3") when** times of the panel start to be read by
|
||||
someone who does not know the instance's zone: a second user of the panel,
|
||||
or lines copied out of it to another person.
|
||||
|
||||
**A trap this leaves, known in advance (owner asked for it to be written
|
||||
down, 2026-10-13):** Server › System log, and the *Delivery log* box on a
|
||||
message's page, show the lines of `mail.log` as Postfix wrote them, each with
|
||||
its own stamp ("Oct 13 14:01:52"). Those stamps are the container's time -
|
||||
UTC unless the container is given a `TZ` - and they are not rewritten. Once
|
||||
the rest of the panel is in the instance's zone and no time carries a label,
|
||||
a message that the Outbound log puts at 17:01 has lines stamped 14:01 on its
|
||||
own page, with nothing on the screen to say why. The plan must close this
|
||||
before the item ships, one way or another: rewrite the stamp when a line is
|
||||
shown, say on those two screens which clock the lines use, or run the
|
||||
container in the instance's zone (which moves `mail.log` itself and what the
|
||||
tailer reads). The guide's section on time must say the same thing as the
|
||||
screens.
|
||||
|
||||
**How a day is counted (owner, 2026-10-13):** the tables that keep a row
|
||||
per day - `sender_check_days`, and `delivery_log_days` of
|
||||
[audit-2026-10-12](audit-2026-10-12.md) stage 1 - key the row by the date in
|
||||
the instance's zone at the moment of counting, from the release on. Rows
|
||||
counted before it stay UTC days. Changing the zone later leaves the rows
|
||||
already written as days of the old zone, a shift of a few hours in a
|
||||
statistic; accepted - the zone is set once, and rows per hour (which would
|
||||
survive a change) cost a migration for it. The charts and statistics group
|
||||
the log by hour already and need nothing.
|
||||
|
||||
**Open, to settle at the mockups:** the trap above; which of the screens
|
||||
that show a date show the year.
|
||||
|
||||
**Boundary:** what the panel shows and how it groups by day. `mail.log`,
|
||||
the database's stored instants and the e2e stay as they are.
|
||||
|
||||
**Version:** `2.x` MINOR; `agreed`; the plan is written when it is taken up.
|
||||
|
||||
---
|
||||
|
||||
## phone-desktop-mode
|
||||
|
||||
**Problem (owner, 2026-10-11):** the panel opened on a phone with the
|
||||
browser's "desktop site" switched on does not become the desktop panel.
|
||||
Seen on Server › Health at `selfpost.mixfed.ru`: the page body is laid out in
|
||||
columns (Machine and Processes side by side, three cards in the next row),
|
||||
while the navigation is the phone one - not a bar but a stack with every
|
||||
section open (Outbound, Inbound, Server, the user menu), filling the first
|
||||
screen before the page starts. The content also sits flush against the left
|
||||
edge of the screen.
|
||||
|
||||
**Goal:** find out why, then decide what the panel should be in that mode -
|
||||
the desktop panel, or a phone panel that is at least consistent.
|
||||
|
||||
**Where to look first (a guess, not checked):** "desktop site" on Android
|
||||
lays a page out about 980 px wide. Bulma turns the navigation into a bar at
|
||||
1024 px and puts columns side by side from 769 px, so 980 px falls between
|
||||
the two: desktop columns under a touch navigation. `panel.css` has its own
|
||||
breakpoints at 768, 1023 and 1407 px. Check the width the phone really
|
||||
reports, why every dropdown is open rather than folded behind the burger,
|
||||
and where the left padding went.
|
||||
|
||||
**Boundary:** the navigation and page gutter between 769 and 1023 px, which
|
||||
is also a tablet in portrait and a narrow desktop window - the same defect is
|
||||
expected there. A design change is mockup-first
|
||||
([plans/panel-redesign.md](plans/panel-redesign.md) § Enforcement).
|
||||
|
||||
**Done when:** the cause is written down here, and the panel at that width
|
||||
has one coherent layout, checked on a phone in "desktop site" mode and in a
|
||||
980 px window.
|
||||
|
||||
**Version:** `2.x` PATCH or MINOR; `agreed` (owner, 2026-10-13), taken up
|
||||
after stages 1 and 2 of [audit-2026-10-12](audit-2026-10-12.md) - mail shown
|
||||
wrong and the tenant boundary come before layout.
|
||||
|
||||
---
|
||||
|
||||
## from-header-binding
|
||||
|
||||
**Problem (security review of panel-notifications, 2026-10-12):** Postfix
|
||||
holds the envelope sender against the SASL login
|
||||
(`reject_sender_login_mismatch`), and OpenDKIM chooses the signing key by the
|
||||
`From:` header. Nothing holds the header against the login. An application
|
||||
of one sending domain can therefore write another registered domain - or,
|
||||
since 2.3.0, the server's own name - into `From:` and have the message signed
|
||||
with that name's key: mail that passes DMARC as a domain the application has
|
||||
no right to, or that looks like a notification of the panel.
|
||||
|
||||
**Agreed with the owner on 2026-10-12**, built and reviewed the same day
|
||||
([plans/from-header-binding.md](plans/from-header-binding.md); what is not
|
||||
done is listed there under *Not done*). The rule he
|
||||
chose is wider than the one first written here: `From:` is held to exactly
|
||||
what the envelope is held to - any address of the application's domain, or
|
||||
the addresses of its list - and anything else is refused, a stranger's domain
|
||||
included. The check is a role of the panel on a strict socket of its own:
|
||||
while the panel does not answer, mail is deferred, not accepted unchecked.
|
||||
Server › Health counts what that costs, to decide later whether the check
|
||||
needs a process of its own.
|
||||
|
||||
**Done when:** the e2e stand shows an application refused for another
|
||||
tenant's `From:`, for the hostname's and for an address off its list, and
|
||||
accepted for its own.
|
||||
|
||||
**Version:** shipped `2.4.0` (2026-10-12), MINOR. What it costs - the
|
||||
panel's downtime is now mail's - is in [security.md](security.md) § Accepted
|
||||
risks.
|
||||
|
||||
---
|
||||
|
||||
@@ -482,3 +627,70 @@ explicit major). Squashing the current short chain is not a reason to cut 2.x on
|
||||
its own. A missed gate leaves a mid-chain 1.x database silently stuck.
|
||||
**Version:** `2.x` major only; not a 1.x item.
|
||||
|
||||
---
|
||||
|
||||
## Dropped, with the condition for return
|
||||
|
||||
Items the owner took off the roadmap. Each names what would bring it back;
|
||||
without that it is not offered again.
|
||||
|
||||
- **structured-logging** (dropped 2026-10-13) - move the panel from
|
||||
`log.Printf` to `log/slog` with levels and fields, one logger passed through
|
||||
constructors. Dropped because the panel's log has one reader, the owner, by
|
||||
eye in `docker logs`; the change is wide (some 225 call sites), changes no
|
||||
behaviour, and gives a new format to everything that reads the output.
|
||||
**Returns if:** the log goes to an outside collector, or it has to be
|
||||
filtered by level.
|
||||
- **Certificate warnings measured by the certificate's lifetime** (left as
|
||||
it is, owner, 2026-10-14). The panel warns 14 and 3 days before a
|
||||
certificate expires, whatever its lifetime
|
||||
([notify.go](../internal/notify/notify.go), `certFirstDays`,
|
||||
`certLastDays`). A certificate issued for less than 14 days is inside the
|
||||
first warning from its first hour, and each renewal of it is a new
|
||||
certificate and a new message. Not reachable with the 90-day certificate
|
||||
the instance has. **Returns if:** the instance takes a certificate that
|
||||
lives less than 30 days. The rule then offered: the first warning at a
|
||||
quarter of the lifetime left, the last at a tenth, and never earlier than
|
||||
the 14 and 3 days of today - so a longer certificate, a yearly one
|
||||
included, is warned about exactly as now.
|
||||
- **contributing** (dropped 2026-10-13) - a `CONTRIBUTING.md` in the root:
|
||||
the dev loop, the checks before a PR, the commit protocol. Dropped because
|
||||
there is one developer and no PRs, and everything the file would say is in
|
||||
[development.md](development.md); a second file is a second copy to keep
|
||||
in step. **Returns if:** the first outside PR arrives, or the owner decides
|
||||
to invite contributors.
|
||||
- **A check that ports 25, 465 and 587 are reachable** (dropped 2026-10-13,
|
||||
from the former *preflight*). The panel can only dial its own public
|
||||
address from inside the container: behind NAT without hairpin that shows an
|
||||
open port as closed, and with hairpin it shows as open a port the provider
|
||||
blocks from outside. A card that is wrong both ways is worse than none, and
|
||||
an honest check needs a probe outside the server, which is outside the
|
||||
product ([product.md](product.md)). The administrator's browser cannot do
|
||||
it either: 25, 465 and 587 are on the Fetch standard's list of blocked
|
||||
ports. Two substitutes were offered and declined the same day: a Health
|
||||
line "last connection from outside on each port", read from `mail.log`, and
|
||||
a command in the help to run from one's own machine. **Returns if:** the
|
||||
owner accepts an outside probe, or asks for one of the two substitutes.
|
||||
- **preflight as a page of its own** (dropped 2026-10-13): see
|
||||
[health-setup-checks](#health-setup-checks). Its plan,
|
||||
`plans/preflight.md`, was removed and stays in git history.
|
||||
- **csrf-tokens** (decided 2026-10-13) - session-bound tokens on every form.
|
||||
Replaced by [origin-strict](#origin-strict); the conditions for building
|
||||
tokens after all are in [security.md](security.md), ADR § *CSRF via origin
|
||||
checking*.
|
||||
|
||||
---
|
||||
|
||||
## Ideas, not on the roadmap
|
||||
|
||||
Thoughts kept so they are not lost. They are not candidates: nothing here is
|
||||
waiting for an OK, has a version or takes a place in the order above. An idea
|
||||
joins the index only when the owner says so.
|
||||
|
||||
- **Inbound quarantine** (moved here by the owner on 2026-10-12: "an idea to
|
||||
think about") - hold suspicious inbound mail, review it in the panel, then
|
||||
release it to the upstream or discard it. It would take the relay from
|
||||
passing mail on to storing it for a while, against "no mailboxes" in
|
||||
[product.md](product.md). Notes and open questions:
|
||||
[plans/inbound-quarantine.md](plans/inbound-quarantine.md); the screens
|
||||
`in-quarantine` and `in-quarantine-message` are drawn and stay unbuilt.
|
||||
|
||||
+22
-10
@@ -17,8 +17,8 @@ When you add or review a migration, update this file in the same change.
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| Chain head | `user_version = 1` |
|
||||
| Files | `0001_init.sql` (the 2.0 baseline) |
|
||||
| Chain head | `user_version = 5` |
|
||||
| Files | `0001_init.sql` (the 2.0 baseline), `0002_inbound_spam_log.sql`, `0003_send_log_lines.sql`, `0004_notifications.sql`, `0005_send_log_rejected_index.sql`, `0006_sender_check.sql` |
|
||||
| File mark | `PRAGMA application_id = 0x53503230` ("SP20") |
|
||||
| Database file | `/data/selfpost.db` (bind mount) |
|
||||
| Compatibility | 2.x MINOR releases must boot a `2.0.0` data directory. **Nothing is carried over from 1.x.** |
|
||||
@@ -60,10 +60,12 @@ What that means in practice:
|
||||
|
||||
**Implication:** once `2.0.0` is released, `0001_init.sql` is never edited
|
||||
again and no file is deleted, renamed or reordered — every 2.x database has
|
||||
already run it. Until that release the baseline is still being written:
|
||||
later steps of the redesign and the features queued behind it (the next one
|
||||
is `0002_inbound_spam_log.sql` of inbound-antispam-panel, `2.1.0`) add files
|
||||
after it.
|
||||
already run it. `2.0.0` and `2.0.1` shipped with the baseline alone; the
|
||||
features after them add files, the first being `0002_inbound_spam_log.sql` of
|
||||
inbound-antispam-panel (`2.1.0`), then `0003_send_log_lines.sql` of
|
||||
delivery-log-storage (`2.2.0`), `0004_notifications.sql` and
|
||||
`0005_send_log_rejected_index.sql` of panel-notifications (`2.3.0`), and
|
||||
`0006_sender_check.sql` of from-header-binding (`2.4.0`).
|
||||
|
||||
---
|
||||
|
||||
@@ -72,18 +74,23 @@ after it.
|
||||
| Ver | File | Shipped | Kind | Summary |
|
||||
|-----|------|---------|------|---------|
|
||||
| 1 | `0001_init.sql` | 2.0.0 | DDL | The whole 2.0 schema, below |
|
||||
| 2 | `0002_inbound_spam_log.sql` | 2.1.0 | DDL | `inbound_spam_log`, `inbound_spam_list` and two indexes; nothing existing changes |
|
||||
| 3 | `0003_send_log_lines.sql` | 2.2.0 | DDL | `send_log_lines` and one index; nothing existing changes |
|
||||
| 4 | `0004_notifications.sql` | 2.3.0 | DDL | `users.notify` (default 1: every user starts with notifications on) and `notify_state` |
|
||||
| 5 | `0005_send_log_rejected_index.sql` | 2.3.0 | DDL | `idx_send_log_rejected`, a partial index of the refused rows of `send_log` by time; a file of its own because the test installation had run 0004 before the review asked for it |
|
||||
| 6 | `0006_sender_check.sql` | 2.4.0 | DDL | `send_log.reason` and `send_log.header_from` (both default `''`: a row refused earlier keeps no reason), and `sender_check_days` |
|
||||
|
||||
**Kind:** *DDL* — schema only; *data* — `INSERT`/`UPDATE` that must stay correct
|
||||
for operators upgrading from older 2.x images.
|
||||
|
||||
---
|
||||
|
||||
## Schema at head (v1)
|
||||
## Schema at head (v2)
|
||||
|
||||
| Table | Role |
|
||||
|-------|------|
|
||||
| `settings` | Key/value settings of the instance — what is true whoever is signed in (send-log retention). Nothing that belongs to a user |
|
||||
| `users` | Panel logins. `role` is `global` or `domain`; `email` is the user's own address; `all_domains` / `all_inbound_domains` widen a `domain` user to every domain of that direction |
|
||||
| `users` | Panel logins. `role` is `global` or `domain`; `email` is the user's own address; `notify` is the user's own switch for notification e-mails (v4); `all_domains` / `all_inbound_domains` widen a `domain` user to every domain of that direction |
|
||||
| `sessions` | Panel login sessions (token hash, owning user by id and deleted with them, sliding idle expiry) |
|
||||
| `domains` | Sending (outbound) domains. `dmarc_rua` is where aggregate reports go (`''` = no reports) |
|
||||
| `user_domains` | Outbound domains assigned to a `domain` user |
|
||||
@@ -98,9 +105,14 @@ for operators upgrading from older 2.x images.
|
||||
| `user_inbound_domains` | Inbound domains assigned to a `domain` user — granted separately from the outbound ones |
|
||||
| `dmarc_reports` | Parsed aggregate report summaries |
|
||||
| `dmarc_report_records` | Per-source rows inside a report |
|
||||
| `inbound_spam_log` | Inbound antispam journal: one row per recipient of a message offered on port 25, with the decision (`accept`, `reject`, `tempfail`, `quarantine`) and who made it (`rspamd`, `postfix`, `selfpost`). Metadata only (v2) |
|
||||
| `inbound_spam_list` | Allow / deny entries for the external filter (`address`, `domain`, `ip`), instance-wide; unique per list (v2) |
|
||||
| `notify_state` | What the panel has already written to its users about, one row per subject (`dns:<domain>`, `cert`, `queue`, `limit:<domain>`), so that each thing is said once and a restart says nothing again (v4) |
|
||||
| `send_log_lines` | The `mail.log` lines Postfix wrote under a queue id, with the line's own time; what a message's page shows once the log has rotated. No reference to `send_log`: a line belongs to a row by queue id and time, and goes when no row carries its id (v3) |
|
||||
|
||||
`TestBaselineSchema` (`internal/store/schema_test.go`) pins every table, column
|
||||
and index of this list.
|
||||
and index of this list; `TestOpenMigratesA20Database` opens a database that
|
||||
holds the baseline alone.
|
||||
|
||||
### What changed against the last 1.x schema
|
||||
|
||||
@@ -129,7 +141,7 @@ for the domain. Nothing refers to a user: deleting one changes no domain.
|
||||
|
||||
1. Add `internal/store/migrations/00NN_short_name.sql` (next number).
|
||||
2. Update the table in [Migration chain](#migration-chain), the snapshot at the
|
||||
top, and the affected rows of [Schema at head](#schema-at-head-v1).
|
||||
top, and the affected rows of [Schema at head](#schema-at-head-v2).
|
||||
3. Update `TestBaselineSchema` if tables, columns or indexes change, and add a
|
||||
test that opens a database at the previous version and migrates it.
|
||||
4. A *data* migration states in a comment which 2.x versions it is written for.
|
||||
|
||||
+229
-14
@@ -26,7 +26,31 @@ the user form did not end that user's sessions, internal errors reached forms a
|
||||
(see [Authentication and sessions](#authentication-and-sessions) and [Errors
|
||||
shown to users](#errors-shown-to-users)); two deliberate behaviours are
|
||||
recorded under Accepted risks (upstream of a delegated inbound domain, profile
|
||||
saved without the password).
|
||||
saved without the password). The 2026-10-11 review (Fable) covered the DMARC
|
||||
report path from port 25 to the report pages, which ran for the first time in
|
||||
2.0.0 - the 2026-08-19 pass read code that Postfix could not yet start: no
|
||||
Critical or High finding; a report's size was bounded only as bytes (now 5 000
|
||||
records, 255 bytes a field), the retention cap was one for the whole server
|
||||
(now per domain), the delivery command could run a hundred at once (now one),
|
||||
and the reason logged for a dropped message carried the sender's bytes as they
|
||||
came (now cut and reduced to printable ASCII). The 2026-10-12 reviews of
|
||||
inbound-antispam-panel (`2.1.0`: the inbound journal milter, refusals read from
|
||||
`mail.log`, the filter lists and their pages) were two, by different models
|
||||
over the same diff. The technical one (Opus) found, outside the feature and
|
||||
there since `1.9.0`, that **the level-2 rate limit and an application's
|
||||
client-address list held only for the first message of a connection** - see
|
||||
[Mail path](#mail-path-security-relevant) - which was reproduced on the test
|
||||
stand and fixed. The security one (Fable) found no Critical or High: a client's
|
||||
HELO name could supply the sender, recipient and reply of a journal row and so
|
||||
make a refusal look like the deny list's (the reply now ends at Postfix's own
|
||||
`; from=<`); the journal's row cap was applied only every six hours, between
|
||||
which refusals were written at whatever rate a stranger sent recipients (the
|
||||
tailer now cuts to the cap every 500 rows); and two behaviours are recorded
|
||||
under Accepted risks (the allow list keyed on an address anyone can claim, the
|
||||
list files readable by anyone). A third it named - with the filter down, a
|
||||
sender's own verdict headers were recorded as the filter's - was closed the
|
||||
same day at the owner's suggestion: the filter signs each message and a
|
||||
verdict is read only next to its signature.
|
||||
(2) **Accepted risks** —
|
||||
deliberate departures beyond the mandatory, recorded so the decision is not
|
||||
lost.
|
||||
@@ -113,6 +137,8 @@ assigned outbound domains and inbound domains **separately**, each list with
|
||||
| Outbound: domain page, applications, Domain settings (report address, rate limit, export), log | every domain | assigned outbound domains |
|
||||
| Outbound › DMARC reports | every report, with the server's ingest statistics | reports of assigned domains, without the ingest box |
|
||||
| Inbound: domain page, upstream, recipients | every inbound domain | assigned inbound domains |
|
||||
| Inbound › Log | every row, including those of a domain deleted since | rows of assigned inbound domains (by name), without the links into the filter lists |
|
||||
| Inbound › Filter lists; the inbound log's retention | yes | no — 404 |
|
||||
| Add or delete a domain, either direction; import a domain | yes | no |
|
||||
| Account, Help | yes | yes |
|
||||
|
||||
@@ -139,7 +165,7 @@ assigned outbound domains and inbound domains **separately**, each list with
|
||||
(`originAllowed`, [internal/web/security.go](../internal/web/security.go)):
|
||||
`Sec-Fetch-Site` must be `same-origin`, or `Origin` must name the panel's own
|
||||
host; the session cookie is `SameSite` as well.
|
||||
- There are **no CSRF tokens yet**. The decision, its price and the conditions
|
||||
- There are **no CSRF tokens**. The decision, its price and the conditions
|
||||
for revisiting it are in the accepted risks and the ADR below.
|
||||
|
||||
### Errors shown to users
|
||||
@@ -162,6 +188,30 @@ operator can act on it.
|
||||
|
||||
- **Not an open relay** — SASL only on 465/587; `reject_unauth_destination`;
|
||||
`smtpd_sender_login_maps` + `reject_sender_login_mismatch`.
|
||||
- **An application signs only as itself (`2.4.0`).** OpenDKIM chooses the
|
||||
key by the `From:` header and Postfix holds only the envelope sender
|
||||
against the login, so until `2.4.0` an application of one sending domain
|
||||
could write another registered domain - or, since `2.3.0`, the server's
|
||||
own name - into `From:` and have the message signed with that name's key
|
||||
(found by the security review of panel-notifications, 2026-10-12). The
|
||||
panel's sender check now holds the header to the table the envelope is
|
||||
held to and refuses anything else with `550 5.7.1`, a domain the server
|
||||
does not sign for included (the owner's rule: the header may say what the
|
||||
envelope may, nothing more). It is the first milter of the chain and
|
||||
strict: a message it could not judge is deferred, never passed on to the
|
||||
signer. It reads the header as one field holding one address and then
|
||||
looks at the raw value once more for any other name of OpenDKIM's
|
||||
`SigningTable`, so that a line its parser and OpenDKIM's could read
|
||||
differently is refused rather than argued about; outside quoted strings
|
||||
the line may hold one `@`. What this does not promise is what a display
|
||||
name looks like to a person: a name of another sender written without an
|
||||
`@` the scan can find (a full-width dot, a space inside the name) passes,
|
||||
signed with the application's own key - a forged display name is as old
|
||||
as mail, and no key of another's is involved. What the client wrote
|
||||
reaches the reply, the log and the send log only as printable ASCII, cut,
|
||||
and a refused message is one row of the log whatever its recipient count.
|
||||
Mail the panel submits itself does not pass the check: only root,
|
||||
`postfix` and the panel can submit locally. Architecture § Sender check.
|
||||
- **Inbound relay (optional, `INBOUND_RELAY_ENABLE`)** — port 25 is not an
|
||||
open relay either: SASL is off; `smtpd_relay_restrictions` /
|
||||
`smtpd_recipient_restrictions` are `reject_unauth_destination` and
|
||||
@@ -171,6 +221,41 @@ operator can act on it.
|
||||
refuse unknown addresses at RCPT (no backscatter). OpenDKIM is not attached
|
||||
on inbound. An optional antispam milter is inbound-only; default action is
|
||||
fail-open (`accept`).
|
||||
- **The panel's milters judge every message of a connection.** Postfix
|
||||
introduces a client to a milter once per connection, and the milter library
|
||||
replaces the milter's session after any reply that is not "continue". Until
|
||||
`2.1.0` the journal milter knew the client's address only from that
|
||||
introduction, and both of its checks passed a message that came without one:
|
||||
a second `MAIL FROM` after a refused one, or any message after the first on
|
||||
one connection, got past the level-2 limit and past an application's
|
||||
client-address list. Now the address comes with every `MAIL FROM`
|
||||
(`milter_mail_macros` carries `{client_addr}`), the level-2 limit does not
|
||||
ask for it at all, an application with a client-address list is refused when
|
||||
no address is known, and neither milter replies anything but "continue" to a
|
||||
message it lets through. `test/e2e` sends two `MAIL FROM` on one connection.
|
||||
- **The inbound journal is written from what strangers send.** The inbound
|
||||
journal milter is the last on port 25, read-only and attached fail-open; it
|
||||
keeps of the filter's verdict only what its patterns recognise. **A verdict
|
||||
is read only next to the filter's signature.** The filter is attached so
|
||||
that mail flows when it is down, and then nothing removes verdict headers a
|
||||
sender wrote himself. So rspamd adds a third header, `X-SelfPost-Filter`:
|
||||
the HMAC-SHA256 of the message's queue id under a key SelfPost generates
|
||||
(`data/antispam/filter.key`) and the sidecar reads from the directory it
|
||||
mounts. The journal milter recomputes it at end-of-message; with no
|
||||
signature, or one that does not fit, the row is that of mail nobody judged.
|
||||
The signature goes on to the upstream with the mail and is of no use there:
|
||||
it fits one queue id. Refusals are
|
||||
parsed from `mail.log` from the head of the line, with the reply ending at
|
||||
Postfix's own `; from=<` so that the HELO name after it - the client's text,
|
||||
logged as written - cannot stand in for the sender, the recipient or the
|
||||
reply; an address with a quote, a bracket or a space in it is not kept.
|
||||
Only a recipient at an inbound domain makes a row. The journal is cut to
|
||||
10 000 rows every 500 rows the tailer writes, and by age every six hours.
|
||||
- **Filter lists** reach another container as files: values are whitelisted on
|
||||
the way in (address, domain, canonical IP or network; no `/0`, no
|
||||
IPv4-mapped network), the writer refuses a stored value with whitespace, a
|
||||
newline or `#`, and a change of the table and the rewrite of the six files
|
||||
are one step under a lock.
|
||||
- TLS is mandatory before credentials are transmitted (465 wrapper / 587
|
||||
`encrypt`).
|
||||
- `TRUSTED_PROXY_CIDR` — only explicitly trusted proxies may supply
|
||||
@@ -183,6 +268,11 @@ operator can act on it.
|
||||
private key from `certs/` when present; a domain export carries the DKIM key
|
||||
and **working** application passwords in the clear (otherwise a transfer
|
||||
without recreating credentials would be impossible).
|
||||
- Since 2.2.0 a full backup also carries, in the database, what Postfix logged
|
||||
about each message still in the send log (`send_log_lines`): the submitting
|
||||
client's address and SASL login, the message id, the receiving server and
|
||||
its reply. `mail.log` itself stays out of a backup; these lines are state
|
||||
the panel shows, kept for the send log's retention and no longer.
|
||||
- Both downloads are encrypted with a password unless the box on the form is
|
||||
unticked (it is ticked by default):
|
||||
scrypt (N=2¹⁵, r=8, p=1) → AES-256-GCM, streamed in 64 KiB chunks, each
|
||||
@@ -216,7 +306,9 @@ deferred item from the roadmap.
|
||||
against, and a strict mode would not "protect" such a client, it would
|
||||
simply break the panel in it. Tightening is one line in `originAllowed`
|
||||
([internal/web/security.go](../internal/web/security.go)): return `false`
|
||||
instead of `true` in the "neither header present" branch.
|
||||
instead of `true` in the "neither header present" branch. That tightening
|
||||
is agreed (owner, 2026-10-13) as
|
||||
[origin-strict](roadmap.md#origin-strict); this bullet goes when it ships.
|
||||
- **Session-bound CSRF tokens are not implemented.** The origin check closes the
|
||||
neighbouring-subdomain case but depends on browser behaviour; a token does
|
||||
not. The price is a hidden field in roughly two dozen forms. The trigger to
|
||||
@@ -225,10 +317,12 @@ deferred item from the roadmap.
|
||||
(see the ADR below). A token would not save the panel from XSS inside it
|
||||
either: code executing in the panel's origin sends the request itself —
|
||||
against that, `html/template` auto-escaping and CSP do the work, which is why
|
||||
templates must contain no inline scripts and no inline styles. Tracked as
|
||||
[csrf-tokens](roadmap.md#csrf-tokens): implementing that item replaces this
|
||||
bullet and the ADR below, and dropping it leaves them as the single record of
|
||||
the decision. The two must not disagree.
|
||||
templates must contain no inline scripts and no inline styles. Decided by
|
||||
the owner on 2026-10-13: no tokens; the roadmap item that asked for them is
|
||||
dropped and the origin check is made strict instead
|
||||
([origin-strict](roadmap.md#origin-strict)). This bullet and the ADR below
|
||||
are the single record of the decision, and the ADR's **Revisit if** says
|
||||
when tokens are built after all.
|
||||
- **Destructive-action confirmation (`data-confirm`) is JavaScript-only.**
|
||||
Delete, regenerate-password, and clear-rate-limit forms carry a
|
||||
`data-confirm` prompt handled entirely in
|
||||
@@ -274,20 +368,63 @@ deferred item from the roadmap.
|
||||
reporters are arbitrary third parties and the mailbox address is published in
|
||||
DNS. The ingest already constrains the meaningful part — a report cannot be
|
||||
filed under a domain this instance does not host, and the claimed domain is
|
||||
cross-checked against the `+tag` of the delivery address — and the data is
|
||||
advisory statistics only; nothing enforces off it. The trigger to revisit is
|
||||
cross-checked against the `+tag` of the delivery address (effective since
|
||||
2.0.0: before it the recipient never reached the command) — and the data is
|
||||
advisory statistics only; nothing enforces off it. What a stranger can do is
|
||||
limited to a domain that chose a SelfPost-hosted address, through that
|
||||
address: file reports under it, replace one whose reporter and id they know,
|
||||
and fill that domain's 500 — not another domain's. A message that is not a
|
||||
report the relay keeps is dropped, not bounced (the envelope sender is the
|
||||
author's to write); a genuine report the parser cannot read is lost the same
|
||||
way, with a count and a line in the mail log. The trigger to revisit is
|
||||
any feature that lets report contents drive an automatic action (policy
|
||||
changes, rate limiting, alerting an operator into a config change).
|
||||
- **The application auth-IP allow-list is enforced fail-open.** The check runs
|
||||
in the journal milter (`internal/milter/authip.go`) with
|
||||
`default_action=accept` and skips on store errors, so if the milter is down
|
||||
the restriction is not applied; the SASL login itself always succeeds. This
|
||||
`default_action=accept` and skips on store errors, so if the journal milter
|
||||
does not answer the restriction is not applied; the SASL login itself
|
||||
always succeeds. (While the milter is up the list holds for every message,
|
||||
and an unknown client address is refused - see Mail path; what is accepted
|
||||
here is the milter not answering. Since `2.4.0` a panel that is down
|
||||
altogether defers mail through the sender check, which shares its process;
|
||||
what is left of this window is a journal milter that hangs or a database
|
||||
that errs while the check beside it still answers.) This
|
||||
is documented in [guide.md](guide.md) as **not an authentication boundary** —
|
||||
it is a blast-radius limiter for a leaked application password, on the same
|
||||
fail-open path as level-2 rate limiting, and a milter outage already stops
|
||||
the journal from recording sends at all. The trigger to revisit is a
|
||||
requirement for the allow-list to hold as a security boundary — at which
|
||||
point it must move to the SASL/smtpd layer and fail closed.
|
||||
- **The key the spam filter signs with is readable by anyone on the host.**
|
||||
The inbound journal reads the filter's verdict only next to the filter's
|
||||
signature (see Mail path), and the key for it, `data/antispam/filter.key`,
|
||||
is 0644 like the list files beside it: the filter is another user in another
|
||||
container. Someone with a shell on the host could therefore sign a verdict
|
||||
of his own - into a monitoring page, on a host where he could do far more.
|
||||
Senders on the internet, whom the signature is for, cannot read it. Trigger
|
||||
to revisit: the same as for the list files - a shared host.
|
||||
- **An allow entry for a sender address or domain trusts what the sending
|
||||
server says.** The filter matches the envelope sender, which is not
|
||||
authenticated: anyone who knows or guesses an allowed address can send as it
|
||||
and is given the entry's weight, which no spam score outweighs. The page's
|
||||
Help and the guide say so and recommend an entry for the client's address or
|
||||
network instead. Deny entries have no such problem - claiming to be a denied
|
||||
sender only gets mail refused - and deny wins over allow. Trigger to
|
||||
revisit: allow entries being used as the main way to receive mail from a
|
||||
sender, at which point the rule should also require the sender's SPF or
|
||||
DKIM to pass.
|
||||
- **The filter lists are files readable by anyone on the host.**
|
||||
`/data/antispam` is 0755 and its six files 0644, unlike the rest of `/data`:
|
||||
the filter runs under another user in another container. They hold sender
|
||||
addresses, domains and networks - who the operator blocks and whom he lets
|
||||
through - but no message data. The inbound journal itself is in the
|
||||
database, 0640. Trigger to revisit: a shared host, where the sidecar should
|
||||
instead be given the files through a group it shares with the panel.
|
||||
- **A row of the inbound log belongs to a domain by name.** A `domain` user
|
||||
sees the rows whose inbound domain has the name of one assigned to them, so
|
||||
an inbound domain that is deleted and added again shows its earlier rows to
|
||||
whoever it is assigned to next. Accepted: the journal is of a mail domain,
|
||||
and the rows age out with the retention.
|
||||
- **Level-2 in-flight reservations can outlive a dropped SMTP session until TTL.**
|
||||
The journal milter cannot observe connection-close directly through
|
||||
go-milter, so if a client disconnects without an explicit `Abort`, its
|
||||
@@ -303,7 +440,70 @@ deferred item from the roadmap.
|
||||
the file is inaccessible to others. The log holds envelope addresses and
|
||||
client IPs, but neither message bodies nor headers; it is excluded from
|
||||
backups (`log/` is skipped) so that a dump stays state rather than
|
||||
diagnostics.
|
||||
diagnostics. Since 2.2.0 the tailer copies the lines of queued outbound
|
||||
messages into the database for the message's page, and those are in a
|
||||
backup (§ Backup and domain export); mail offered on port 25 is not copied.
|
||||
- **The panel submits mail of its own.** Notifications leave through
|
||||
`sendmail(1)` run by the unprivileged panel user: a fixed argument list, the
|
||||
two addresses after `--`, no shell. The recipient is a panel user's account
|
||||
e-mail, validated when it was saved and checked again before it reaches a
|
||||
header or the command line; the subject and body are composed by the panel
|
||||
from its own readings. Two of those readings can carry text from outside - a
|
||||
DNS check's finding, and a queued message's sender address - and both are
|
||||
reduced to printable ASCII and cut before they are printed, so a
|
||||
notification's body can contain an address a stranger chose, as a few plain
|
||||
characters and nothing else. A refused-mail notification names the domain
|
||||
and the applications from the registry, never the domain a client wrote in
|
||||
`MAIL FROM`: that string is not yet held against the login when the milter
|
||||
refuses, and the first build took it for the subject - an application over
|
||||
its limit could have made the panel write a message per made-up domain, to
|
||||
another tenant's users among others (security review, 2026-10-12; fixed
|
||||
before release). The panel's membership in `postdrop` is what lets it
|
||||
submit: under `no-new-privileges` the setgid bit of `postdrop` does not
|
||||
take, so only root, `postfix` and the panel can. Notifications are signed
|
||||
with a key of the server's own name and bypass the level-1 and level-2
|
||||
limits. What bounds them: the subjects are the registered domains, the
|
||||
certificate and the queue; a domain's DNS can be reported and cleared about
|
||||
a dozen times a day at most by someone who flaps the zone; a domain's
|
||||
limits once a day; and no more than 60 messages leave in any hour whatever
|
||||
the rules arrive at.
|
||||
- **The sender check stops mail while the panel does not answer.** It is a
|
||||
role of the panel process, attached with `default_action=tempfail`, so a
|
||||
panel that is down or slower than the milter timeouts (15 s to connect
|
||||
and for a command, 30 s for the verdict at the end of the headers) defers
|
||||
every submission: availability of sending now depends on the panel,
|
||||
which it did not before `2.4.0`. Chosen by the owner over the alternative,
|
||||
a check that is absent whenever the panel is - that one could be switched
|
||||
off by whoever can make the panel slow. Supervisord restarts the panel at
|
||||
once and takes the container down after three failed starts, so the
|
||||
outage is seconds or the whole instance; Server › Health counts the
|
||||
downtime and the mail deferred for it. Trigger to revisit: those numbers
|
||||
not staying at nothing - then the check moves into a process of its own
|
||||
([plans/from-header-binding.md](plans/from-header-binding.md)).
|
||||
- **A malformed packet on a milter socket of the panel ends the panel.** The
|
||||
milter library indexes into an empty packet and nobody recovers the panic,
|
||||
so the process exits and supervisord starts it again. The sockets are
|
||||
reachable only inside the container, by the group Postfix is in (mode
|
||||
0660); the journal milter always had this, and since `2.4.0` the seconds
|
||||
of the restart defer mail instead of passing it unjournalled (technical
|
||||
review, 2026-10-12). Trigger to revisit: anything untrusted able to open
|
||||
a Unix socket in the container.
|
||||
- **When the hostname is itself a sending domain,** the server's mail is
|
||||
signed with that domain's key. Whoever may export the domain has the key,
|
||||
and its applications may send as `postmaster@<hostname>` by right. The
|
||||
guide says so; a hostname that is nobody's sending domain avoids it.
|
||||
- **A message's page can show another message's log lines after a queue-id
|
||||
collision.** Postfix's short queue ids repeat: the id is the microsecond of
|
||||
arrival and the queue file's inode, and inodes come round within seconds.
|
||||
The page takes the stored lines of the row's queue id dated within the
|
||||
row's own queue lifetime (about six days), so a second message has to get
|
||||
the same microsecond on the same inode in that span - about one in a
|
||||
million per reuse of the inode, and not something a sender can aim. When it
|
||||
happens, a `domain` user could read the log lines of a message that is not
|
||||
theirs. The search of the current `mail.log` that the page did before
|
||||
2.2.0 had the same exposure over a day. Trigger to revisit: a report of
|
||||
foreign lines on a page, or a reason to turn on `enable_long_queue_ids`,
|
||||
which ends it.
|
||||
- **A `domain` user may point an inbound domain's upstream at any host and
|
||||
port.** The upstream form checks that the host is a well-formed name or IP
|
||||
and the port is 1–65535; loopback, private and link-local addresses are
|
||||
@@ -319,8 +519,13 @@ deferred item from the roadmap.
|
||||
field to the global role, is the fix.
|
||||
- **Account's profile (username, e-mail) is saved without the password.**
|
||||
Saving the profile does not ask for the current password, as changing the
|
||||
password does. Today the e-mail is only the address the *Use my e-mail*
|
||||
button fills in, so a stolen session gains nothing it did not have. It
|
||||
password does. Since 2.3.0 the e-mail is where notifications go, so a stolen
|
||||
session can turn them to another address: the thief then learns what they
|
||||
say - which domain's DNS is wrong, which application is refused, when the
|
||||
certificate expires, sender addresses of old queued mail - and the user
|
||||
stops hearing it. Nothing in a notification gives access. A `domain` user
|
||||
can likewise point signed mail of this server at a third party's address,
|
||||
a dozen messages a day per assigned domain at most. It
|
||||
becomes a requirement to ask for the current password when changing
|
||||
`users.email` as soon as [password-reset](roadmap.md#password-reset) mails a
|
||||
link to it — from then on whoever can change the address could take the
|
||||
@@ -372,6 +577,16 @@ whose browsers/devices are not vetted) — at that point cross-role request
|
||||
forgery inside the panel would need its own analysis, separate from the
|
||||
external-site case this ADR covers.
|
||||
|
||||
**Decision of 2026-10-13 (owner).** Asked to choose between this ADR, tokens,
|
||||
and a strict origin check, the owner chose the ADR with the strict check
|
||||
([origin-strict](roadmap.md#origin-strict)) and no tokens. Session-bound tokens
|
||||
are the answer to return to - a hidden field in every state-changing form and
|
||||
htmx path, checked against the session, on top of the origin check - when
|
||||
either condition above comes true, and also when: a browser the panel must
|
||||
support is found to send neither header or to send them wrongly; the panel
|
||||
gains a state-changing request that a browser sends without these headers; or
|
||||
an outside review requires a defence that the server alone holds.
|
||||
|
||||
## How this list grows
|
||||
|
||||
The pre-release vulnerability review (history — CHANGELOG `[0.5.0]` Security)
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
module github.com/mixeme/selfpost
|
||||
|
||||
go 1.26
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/emersion/go-milter v0.4.1
|
||||
golang.org/x/crypto v0.54.0
|
||||
modernc.org/sqlite v1.53.0
|
||||
golang.org/x/crypto v0.58.0
|
||||
modernc.org/sqlite v1.60.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/emersion/go-message v0.18.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
modernc.org/libc v1.73.4 // indirect
|
||||
golang.org/x/sys v0.49.0 // indirect
|
||||
modernc.org/libc v1.77.1 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/memory v1.12.1 // indirect
|
||||
)
|
||||
|
||||
@@ -4,14 +4,14 @@ github.com/emersion/go-message v0.18.1 h1:tfTxIoXFSFRwWaZsgnqS1DSZuGpYGzSmCZD8SK
|
||||
github.com/emersion/go-message v0.18.1/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
|
||||
github.com/emersion/go-milter v0.4.1 h1:gLs9QD0zEHF8omgEw8M+aGz6iwBNpWLAcwgSur0ra4M=
|
||||
github.com/emersion/go-milter v0.4.1/go.mod h1:erCQVl0mH4SX9jEvwe+wyndit0rQtmvMLH86V6NGtkI=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
@@ -19,12 +19,12 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
|
||||
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
|
||||
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
|
||||
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
@@ -32,17 +32,16 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
@@ -55,33 +54,33 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
|
||||
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
|
||||
golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
|
||||
golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c=
|
||||
modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||
modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws=
|
||||
modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE=
|
||||
modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg=
|
||||
modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||
modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE=
|
||||
modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA=
|
||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc=
|
||||
modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
|
||||
modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||
modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA=
|
||||
modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8=
|
||||
modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs=
|
||||
modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
|
||||
modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
|
||||
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M=
|
||||
modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s=
|
||||
modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw=
|
||||
modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0=
|
||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
|
||||
+84
-10
@@ -1,16 +1,16 @@
|
||||
// Package backup implements SelfPost's full-server backup and the restore
|
||||
// version guard (architecture.md § Persistence). A full backup is a
|
||||
// gzip-compressed tar of the consolidated persistent state under /data — the
|
||||
// SQLite database (as a consistent snapshot), the per-domain DKIM keys, the
|
||||
// SASL database, and the Postfix queue — plus docker-compose.yml, .env, and
|
||||
// certs/ from the operator's deploy directory, and a manifest recording the
|
||||
// SelfPost version that produced it. Postfix delivery logs under log/ are
|
||||
// excluded (diagnostics, not state).
|
||||
// SQLite database (as a consistent snapshot), the per-domain DKIM keys and the
|
||||
// SASL database — plus docker-compose.yml, .env, and certs/ from the
|
||||
// operator's deploy directory, and a manifest recording the SelfPost version
|
||||
// that produced it. Postfix delivery logs under log/ are excluded
|
||||
// (diagnostics, not state), and so is the Postfix queue (see QueueDir).
|
||||
//
|
||||
// Restore is not a separate code path in the panel: a backup is extracted into
|
||||
// the operator's project directory (data/, docker-compose.yml, .env, certs/)
|
||||
// before first start. The archive carries everything needed to bring the
|
||||
// instance back — DKIM keys, sasldb2, sender map, queue, and deploy files —
|
||||
// instance back — DKIM keys, sasldb2, sender map, and deploy files —
|
||||
// so the operator only adjusts hostname or proxy settings on a new host. The
|
||||
// restore-specific steps the panel runs are CheckRestore, which refuses to
|
||||
// boot if the manifest's version does not match the running binary so
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
"compress/gzip"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
@@ -81,6 +82,18 @@ type Params struct {
|
||||
OnWarn func(string)
|
||||
}
|
||||
|
||||
// QueueDir is the Postfix queue, relative to the data directory. No backup
|
||||
// holds it (owner, 2026-10-11). An archive is made while the instance runs,
|
||||
// and Postfix moves messages between the queue's directories all the time: a
|
||||
// copy taken then can hold a message twice or not at all, so what it would
|
||||
// bring back is not the queue. Nor could the panel read it - Postfix keeps
|
||||
// every queue file readable by its own user alone, which is why the panel's
|
||||
// backup failed outright from 1.3.0 to 2.0.0. What is not carried over is mail
|
||||
// still waiting for delivery; a start without a queue directory builds an
|
||||
// empty one. Whoever needs the queue stops the instance and archives the
|
||||
// project directory (guide § Taking a full backup).
|
||||
const QueueDir = "postfix/queue"
|
||||
|
||||
// excludedFromArchive lists the data-directory entries a backup never carries.
|
||||
// The live database files are replaced by a consistent VACUUM INTO snapshot
|
||||
// written under the canonical name; the setup token is transient bootstrap
|
||||
@@ -145,7 +158,7 @@ func Create(w io.Writer, p Params) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := addTree(tw, p.DataDir, DataArchivePrefix, excludedFromArchive); err != nil {
|
||||
if err := addTree(tw, p.DataDir, DataArchivePrefix, excludedFromArchive, skipQueue); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -181,6 +194,52 @@ func validateDeployRoot(deployRoot string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// skipFunc says whether the entry at path (rel to the root being walked) is
|
||||
// left out of the archive. A directory left out is not descended into.
|
||||
type skipFunc func(path, rel string, d fs.DirEntry) bool
|
||||
|
||||
// skipQueue leaves the Postfix queue out of every archive.
|
||||
func skipQueue(_, rel string, _ fs.DirEntry) bool {
|
||||
return filepath.ToSlash(rel) == QueueDir
|
||||
}
|
||||
|
||||
// Check reads through everything Create would put into the archive, without
|
||||
// writing one, and returns the first thing that cannot be read. The panel
|
||||
// streams its archive to the browser, so once Create has begun a failure can
|
||||
// only cut the download short; this is run first, while a refusal can still be
|
||||
// shown as a page.
|
||||
//
|
||||
// The panel is not root, and nothing is left out on that account: its archive
|
||||
// is the one selfpost-backup makes, or it is not made. What it most likely
|
||||
// cannot read is the private key under certs/ - see UnreadableCert.
|
||||
func Check(p Params) error {
|
||||
if p.DataDir == "" || p.DBPath == "" {
|
||||
return fmt.Errorf("backup: DataDir and DBPath are required")
|
||||
}
|
||||
if err := ValidateDeployRoot(p.DeployRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := addTree(nil, p.DataDir, DataArchivePrefix, excludedFromArchive, skipQueue); err != nil {
|
||||
return err
|
||||
}
|
||||
return addDeployFiles(nil, p.DeployRoot, nil)
|
||||
}
|
||||
|
||||
// UnreadableCert reports whether err, from Check or Create, is a file under
|
||||
// the deploy root's certs/ that the caller is not allowed to read, and names
|
||||
// it relative to the deploy root ("certs/privkey.pem").
|
||||
func UnreadableCert(err error, deployRoot string) (name string, ok bool) {
|
||||
var pathErr *fs.PathError
|
||||
if !errors.As(err, &pathErr) || !errors.Is(err, fs.ErrPermission) {
|
||||
return "", false
|
||||
}
|
||||
rel, relErr := filepath.Rel(filepath.Join(deployRoot, CertsDirName), pathErr.Path)
|
||||
if relErr != nil || rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../") {
|
||||
return "", false
|
||||
}
|
||||
return CertsDirName + "/" + filepath.ToSlash(rel), true
|
||||
}
|
||||
|
||||
func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string)) error {
|
||||
for _, name := range []string{ComposeFileName, EnvFileName} {
|
||||
src := filepath.Join(deployRoot, name)
|
||||
@@ -206,14 +265,16 @@ func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string)) erro
|
||||
}
|
||||
return fmt.Errorf("backup: stat %s: %w", CertsDirName, err)
|
||||
}
|
||||
return addTree(tw, certsDir, CertsDirName+"/", nil)
|
||||
return addTree(tw, certsDir, CertsDirName+"/", nil, nil)
|
||||
}
|
||||
|
||||
// addTree walks root and adds every regular file (and directory, to preserve
|
||||
// empty ones and modes) to tw under archivePrefix + path relative to root.
|
||||
// When exclude is non-nil, top-level names relative to root are skipped.
|
||||
// When exclude is non-nil, top-level names relative to root are skipped; skip,
|
||||
// when non-nil, leaves out whatever it says. With a nil tw nothing is written:
|
||||
// every file is opened and closed, which is how Check finds what cannot be read.
|
||||
// Non-regular, non-directory entries (symlinks, sockets) are skipped.
|
||||
func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool) error {
|
||||
func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool, skip skipFunc) error {
|
||||
return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -236,12 +297,22 @@ func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool
|
||||
}
|
||||
}
|
||||
|
||||
if skip != nil && skip(path, rel, d) {
|
||||
if d.IsDir() {
|
||||
return fs.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case d.IsDir():
|
||||
if tw == nil {
|
||||
return nil
|
||||
}
|
||||
hdr := &tar.Header{
|
||||
Typeflag: tar.TypeDir,
|
||||
Name: name + "/",
|
||||
@@ -282,6 +353,9 @@ func writeTarFile(tw *tar.Writer, name string, mode fs.FileMode, srcPath string)
|
||||
return fmt.Errorf("backup: open %s: %w", srcPath, err)
|
||||
}
|
||||
defer f.Close()
|
||||
if tw == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -40,6 +42,7 @@ func seedDataDir(t *testing.T) (dataDir, dbPath, deployRoot string) {
|
||||
writeFile(t, filepath.Join(dataDir, "opendkim", "keys", "example.com", "selfpost.private"), "PRIVATE KEY")
|
||||
writeFile(t, filepath.Join(dataDir, "sasl", "sasldb2"), "SASLDB")
|
||||
writeFile(t, filepath.Join(dataDir, "postfix", "sender_login_maps"), "@example.com login")
|
||||
writeFile(t, filepath.Join(dataDir, "antispam", "deny_domain.map"), "evil.example\n")
|
||||
writeFile(t, filepath.Join(dataDir, "postfix", "queue", "deferred", "sample"), "queue-file")
|
||||
// Transient files that must NOT be archived.
|
||||
writeFile(t, filepath.Join(dataDir, "setup-token"), "secret-token")
|
||||
@@ -112,7 +115,10 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
||||
DataArchivePrefix + "opendkim/keys/example.com/selfpost.private",
|
||||
DataArchivePrefix + "sasl/sasldb2",
|
||||
DataArchivePrefix + "postfix/sender_login_maps",
|
||||
DataArchivePrefix + "postfix/queue/deferred/sample",
|
||||
// The filter lists as the spam filter reads them. They are rewritten
|
||||
// from the database on the first start after a restore; carrying them
|
||||
// means the filter has its lists before the panel is up.
|
||||
DataArchivePrefix + "antispam/deny_domain.map",
|
||||
ComposeFileName,
|
||||
EnvFileName,
|
||||
CertsDirName + "/fullchain.pem",
|
||||
@@ -129,6 +135,7 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
||||
DataArchivePrefix + "selfpost.db-shm",
|
||||
DataArchivePrefix + "log/mail.log",
|
||||
DataArchivePrefix + "log/mail.log.1",
|
||||
DataArchivePrefix + "postfix/queue/deferred/sample",
|
||||
} {
|
||||
if _, ok := files[name]; ok {
|
||||
t.Errorf("archive should not contain %s", name)
|
||||
@@ -235,3 +242,70 @@ func TestCheckRestoreWrongFormatRejected(t *testing.T) {
|
||||
t.Error("CheckRestore wrong format reported a restore")
|
||||
}
|
||||
}
|
||||
|
||||
// No archive holds the Postfix queue; the rest is there, the private key
|
||||
// included, and Check agrees that it can be made.
|
||||
func TestCreateLeavesTheQueueOut(t *testing.T) {
|
||||
dataDir, dbPath, deployRoot := seedDataDir(t)
|
||||
p := Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3", DeployRoot: deployRoot}
|
||||
|
||||
if err := Check(p); err != nil {
|
||||
t.Fatalf("Check: %v", err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := Create(&buf, p); err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
files := readArchive(t, buf.Bytes())
|
||||
for name := range files {
|
||||
if strings.HasPrefix(name, DataArchivePrefix+QueueDir) {
|
||||
t.Errorf("the panel's archive holds %s", name)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{
|
||||
DataArchivePrefix + "selfpost.db",
|
||||
DataArchivePrefix + "opendkim/keys/example.com/selfpost.private",
|
||||
DataArchivePrefix + "sasl/sasldb2",
|
||||
DataArchivePrefix + "postfix/sender_login_maps",
|
||||
ComposeFileName, EnvFileName,
|
||||
CertsDirName + "/fullchain.pem",
|
||||
CertsDirName + "/privkey.pem",
|
||||
} {
|
||||
if _, ok := files[name]; !ok {
|
||||
t.Errorf("archive missing %s", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A file under certs/ that may not be read is told apart from any other
|
||||
// failure, so the panel can say what to do about it.
|
||||
func TestUnreadableCert(t *testing.T) {
|
||||
root := filepath.Join("srv", "deploy")
|
||||
denied := func(path string) error {
|
||||
return fmt.Errorf("backup: open %s: %w", path, &fs.PathError{Op: "open", Path: path, Err: fs.ErrPermission})
|
||||
}
|
||||
if name, ok := UnreadableCert(denied(filepath.Join(root, "certs", "privkey.pem")), root); !ok || name != "certs/privkey.pem" {
|
||||
t.Errorf("certs/privkey.pem: got %q, %v", name, ok)
|
||||
}
|
||||
if name, ok := UnreadableCert(denied(filepath.Join(root, ".env")), root); ok {
|
||||
t.Errorf(".env was taken for a certificate file: %q", name)
|
||||
}
|
||||
if name, ok := UnreadableCert(denied(filepath.Join("data", "postfix", "x")), root); ok {
|
||||
t.Errorf("a data file was taken for a certificate file: %q", name)
|
||||
}
|
||||
missing := &fs.PathError{Op: "open", Path: filepath.Join(root, "certs", "privkey.pem"), Err: fs.ErrNotExist}
|
||||
if _, ok := UnreadableCert(missing, root); ok {
|
||||
t.Error("a missing file was taken for one that may not be read")
|
||||
}
|
||||
}
|
||||
|
||||
// Check finds what Create would stumble over, before anything is written.
|
||||
func TestCheckReportsWhatCannotBeRead(t *testing.T) {
|
||||
dataDir, dbPath, deployRoot := seedDataDir(t)
|
||||
if err := os.Remove(filepath.Join(deployRoot, EnvFileName)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := Check(Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3", DeployRoot: deployRoot}); err == nil {
|
||||
t.Fatal("Check passed with .env missing from the deploy root")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
package dmarc
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
|
||||
func reportXML(orgName, domain, records string) string {
|
||||
return `<?xml version="1.0"?><feedback><report_metadata><org_name>` + orgName +
|
||||
`</org_name><email>a@example.net</email><report_id>1</report_id>` +
|
||||
`<date_range><begin>1723593600</begin><end>1723680000</end></date_range></report_metadata>` +
|
||||
`<policy_published><domain>` + domain + `</domain><p>none</p></policy_published>` + records + `</feedback>`
|
||||
}
|
||||
|
||||
const oneRecord = `<record><row><source_ip>203.0.113.10</source_ip><count>3</count>` +
|
||||
`<policy_evaluated><disposition>none</disposition><dkim>pass</dkim><spf>pass</spf></policy_evaluated></row>` +
|
||||
`<identifiers><header_from>example.com</header_from></identifiers></record>`
|
||||
|
||||
// The XML is a stranger's: what it may make the relay store is bounded.
|
||||
func TestParseAggregateBoundsWhatAReportHolds(t *testing.T) {
|
||||
long := strings.Repeat("é", 4000)
|
||||
got, err := ParseAggregate([]byte(reportXML(long, "example.com", oneRecord)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Reporter) > maxFieldLen || !strings.HasPrefix(long, got.Reporter) || got.Reporter == "" {
|
||||
t.Fatalf("reporter name is %d bytes, want the start of the name in at most %d", len(got.Reporter), maxFieldLen)
|
||||
}
|
||||
if strings.ContainsRune(got.Reporter, '�') {
|
||||
t.Fatal("the name was cut in the middle of a character")
|
||||
}
|
||||
|
||||
huge := `<record><row><source_ip>203.0.113.10</source_ip><count>9223372036854775807</count>` +
|
||||
`<policy_evaluated><dkim>pass</dkim></policy_evaluated></row></record>`
|
||||
got, err = ParseAggregate([]byte(reportXML("r", "example.com", huge+huge)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.PassCount != 2*maxRecordCount {
|
||||
t.Fatalf("pass count = %d, want two records clamped to %d each", got.PassCount, maxRecordCount)
|
||||
}
|
||||
|
||||
if _, err := ParseAggregate([]byte(reportXML("r", "example.com", strings.Repeat(oneRecord, maxReportRecords)))); err != nil {
|
||||
t.Fatalf("a report of %d records was refused: %v", maxReportRecords, err)
|
||||
}
|
||||
if _, err := ParseAggregate([]byte(reportXML("r", "example.com", strings.Repeat(oneRecord, maxReportRecords+1)))); err == nil {
|
||||
t.Fatalf("a report of %d records was accepted", maxReportRecords+1)
|
||||
}
|
||||
}
|
||||
|
||||
// A report over the bound is one the relay does not keep: dropped, not retried.
|
||||
func TestIngestMessageRejectsAReportWithTooManyRecords(t *testing.T) {
|
||||
st, err := store.Open(t.TempDir() + "/test.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer st.Close()
|
||||
if _, err := st.AddDomain("example.com", "selfpost"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xml := reportXML("r", "example.com", strings.Repeat(oneRecord, maxReportRecords+1))
|
||||
err = IngestMessage(st, strings.NewReader(gzipMessage(t, xml)), "dmarc-reports+example.com@mail.example.com", time.Now().UTC())
|
||||
if !errors.Is(err, ErrRejected) {
|
||||
t.Fatalf("err = %v, want ErrRejected", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Reports filed under one domain do not push another domain's reports out.
|
||||
func TestIngestKeepsEachDomainsReportsApart(t *testing.T) {
|
||||
st, err := store.Open(t.TempDir() + "/test.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer st.Close()
|
||||
now := time.Now().UTC()
|
||||
if _, err := st.InsertDMARCReport(store.DMARCReport{
|
||||
Domain: "quiet.example", Reporter: "genuine", ReportID: "1", PeriodBegin: now, PeriodEnd: now, ReceivedAt: now.Add(-time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < store.DMARCReportsMaxKeep+20; i++ {
|
||||
if _, err := st.InsertDMARCReport(store.DMARCReport{
|
||||
Domain: "flooded.example", Reporter: "stranger", ReportID: strings.Repeat("x", 1+i), PeriodBegin: now, PeriodEnd: now, ReceivedAt: now,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := st.PruneDMARCReports("flooded.example"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
quiet, err := st.ListDMARCReports(store.DMARCReportScope{Domains: []string{"quiet.example"}}, 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(quiet) != 1 {
|
||||
t.Fatalf("the other domain has %d reports after the flood, want its 1", len(quiet))
|
||||
}
|
||||
flooded, err := st.ListDMARCReports(store.DMARCReportScope{Domains: []string{"flooded.example"}}, 10000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(flooded) > store.DMARCReportsMaxKeep {
|
||||
t.Fatalf("the flooded domain keeps %d reports, want at most %d", len(flooded), store.DMARCReportsMaxKeep)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package dmarc
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
@@ -16,6 +17,15 @@ import (
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
|
||||
// ErrRejected marks a message that is not a report this relay keeps and never
|
||||
// will be: it cannot be read as an aggregate report, or it reports on a domain
|
||||
// that is not sent from here, or it came to another domain's address. Sending
|
||||
// it again changes nothing, so the caller drops it. Any other error is the
|
||||
// store's and worth another try.
|
||||
var ErrRejected = errors.New("dmarc report rejected")
|
||||
|
||||
func rejected(err error) error { return fmt.Errorf("%w: %w", ErrRejected, err) }
|
||||
|
||||
// IngestMessage parses a raw RFC 5322 message from r and stores any DMARC
|
||||
// aggregate attachment it finds.
|
||||
//
|
||||
@@ -34,11 +44,11 @@ func IngestMessage(st *store.Store, r io.Reader, recipient string, receivedAt ti
|
||||
}
|
||||
payload, err := extractAggregatePayload(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
return rejected(err)
|
||||
}
|
||||
parsed, err := ParseAggregate(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
return rejected(err)
|
||||
}
|
||||
recipient = strings.ToLower(strings.TrimSpace(recipient))
|
||||
if err := checkReportDomain(st, parsed.Domain, recipient); err != nil {
|
||||
@@ -74,7 +84,7 @@ func IngestMessage(st *store.Store, r io.Reader, recipient string, receivedAt ti
|
||||
if _, err := st.InsertDMARCReport(rep); err != nil {
|
||||
return err
|
||||
}
|
||||
return st.PruneDMARCReports()
|
||||
return st.PruneDMARCReports(rep.Domain)
|
||||
}
|
||||
|
||||
// checkReportDomain refuses a report whose policy domain is not one of this
|
||||
@@ -82,17 +92,17 @@ func IngestMessage(st *store.Store, r io.Reader, recipient string, receivedAt ti
|
||||
// delivered to.
|
||||
func checkReportDomain(st *store.Store, domain, recipient string) error {
|
||||
if domain == "" {
|
||||
return fmt.Errorf("dmarc report: no policy domain")
|
||||
return rejected(fmt.Errorf("no policy domain"))
|
||||
}
|
||||
if tagged := TaggedDomain(recipient); tagged != "" && tagged != domain {
|
||||
return fmt.Errorf("dmarc report: domain %q does not match the address it was sent to (%s)", domain, recipient)
|
||||
return rejected(fmt.Errorf("domain %q does not match the address it was sent to (%s)", domain, recipient))
|
||||
}
|
||||
ok, err := st.DomainExists(domain)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("dmarc report: %q is not a domain configured on this relay", domain)
|
||||
return rejected(fmt.Errorf("%q is not a domain configured on this relay", domain))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
+50
-15
@@ -9,6 +9,7 @@ import (
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// ParsedReport is the structured form of a DMARC aggregate XML payload.
|
||||
@@ -74,6 +75,31 @@ type feedbackXML struct {
|
||||
} `xml:"record"`
|
||||
}
|
||||
|
||||
// What a report may hold. The XML comes from anyone who knows a report address,
|
||||
// and eight megabytes of it (maxAggregateXML) fit some 160 000 empty records or
|
||||
// one name of eight megabytes - every one of them a row or a table cell later.
|
||||
// A real report has tens of records, a few thousand from the largest
|
||||
// receivers, and no field longer than a line.
|
||||
const (
|
||||
maxReportRecords = 5000
|
||||
maxFieldLen = 255
|
||||
maxRecordCount = 1_000_000_000
|
||||
)
|
||||
|
||||
// field trims a text value of the report and cuts it to maxFieldLen bytes, on
|
||||
// a character boundary.
|
||||
func field(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) <= maxFieldLen {
|
||||
return s
|
||||
}
|
||||
s = s[:maxFieldLen]
|
||||
for len(s) > 0 && !utf8.ValidString(s) {
|
||||
s = s[:len(s)-1]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// ParseAggregate decodes gzip- or zip-compressed, or raw, DMARC aggregate XML.
|
||||
func ParseAggregate(raw []byte) (ParsedReport, error) {
|
||||
data, err := decompress(raw)
|
||||
@@ -87,34 +113,43 @@ func ParseAggregate(raw []byte) (ParsedReport, error) {
|
||||
if doc.Metadata.ReportID == "" || doc.Policy.Domain == "" {
|
||||
return ParsedReport{}, fmt.Errorf("dmarc xml: missing report_id or domain")
|
||||
}
|
||||
if len(doc.Records) > maxReportRecords {
|
||||
return ParsedReport{}, fmt.Errorf("dmarc xml: %d records, more than the %d a report may hold", len(doc.Records), maxReportRecords)
|
||||
}
|
||||
// Cut, not refused: a domain name is at most 253 bytes, so a longer one
|
||||
// names nothing this relay sends for and is turned away as such.
|
||||
out := ParsedReport{
|
||||
Reporter: strings.TrimSpace(doc.Metadata.OrgName),
|
||||
ReportID: strings.TrimSpace(doc.Metadata.ReportID),
|
||||
ContactEmail: strings.TrimSpace(doc.Metadata.Email),
|
||||
Reporter: field(doc.Metadata.OrgName),
|
||||
ReportID: field(doc.Metadata.ReportID),
|
||||
ContactEmail: field(doc.Metadata.Email),
|
||||
PeriodBegin: time.Unix(doc.Metadata.DateRange.Begin, 0).UTC(),
|
||||
PeriodEnd: time.Unix(doc.Metadata.DateRange.End, 0).UTC(),
|
||||
Domain: strings.ToLower(strings.TrimSpace(doc.Policy.Domain)),
|
||||
PolicyP: strings.TrimSpace(doc.Policy.P),
|
||||
PolicySP: strings.TrimSpace(doc.Policy.SP),
|
||||
Domain: strings.ToLower(field(doc.Policy.Domain)),
|
||||
PolicyP: field(doc.Policy.P),
|
||||
PolicySP: field(doc.Policy.SP),
|
||||
PolicyPct: doc.Policy.Pct,
|
||||
PolicyADKIM: strings.TrimSpace(doc.Policy.ADKIM),
|
||||
PolicyASPF: strings.TrimSpace(doc.Policy.ASPF),
|
||||
PolicyADKIM: field(doc.Policy.ADKIM),
|
||||
PolicyASPF: field(doc.Policy.ASPF),
|
||||
}
|
||||
if out.PolicyPct == 0 {
|
||||
out.PolicyPct = 100
|
||||
}
|
||||
for _, rec := range doc.Records {
|
||||
row := ParsedRecord{
|
||||
SourceIP: strings.TrimSpace(rec.Row.SourceIP),
|
||||
SourceIP: field(rec.Row.SourceIP),
|
||||
Count: rec.Row.Count,
|
||||
Disposition: strings.TrimSpace(rec.Row.Policy.Disposition),
|
||||
SPFResult: strings.TrimSpace(rec.Row.Policy.SPF),
|
||||
DKIMResult: strings.TrimSpace(rec.Row.Policy.DKIM),
|
||||
HeaderFrom: strings.TrimSpace(rec.Identifiers.HeaderFrom),
|
||||
Disposition: field(rec.Row.Policy.Disposition),
|
||||
SPFResult: field(rec.Row.Policy.SPF),
|
||||
DKIMResult: field(rec.Row.Policy.DKIM),
|
||||
HeaderFrom: field(rec.Identifiers.HeaderFrom),
|
||||
}
|
||||
if row.Count <= 0 {
|
||||
row.Count = 1
|
||||
}
|
||||
// Bounded so that the sums of a report cannot wrap around.
|
||||
if row.Count > maxRecordCount {
|
||||
row.Count = maxRecordCount
|
||||
}
|
||||
if dmarcRecordPasses(row) {
|
||||
out.PassCount += row.Count
|
||||
} else {
|
||||
@@ -172,12 +207,12 @@ func unzip(raw []byte) ([]byte, error) {
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dmarc zip open %s: %w", f.Name, err)
|
||||
return nil, fmt.Errorf("dmarc zip open %q: %w", field(f.Name), err)
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(rc, maxAggregateXML))
|
||||
rc.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dmarc zip read %s: %w", f.Name, err)
|
||||
return nil, fmt.Errorf("dmarc zip read %q: %w", field(f.Name), err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -126,8 +127,9 @@ func TestIngestMessageRejectsUnknownDomain(t *testing.T) {
|
||||
}
|
||||
defer st.Close()
|
||||
|
||||
if err := IngestMessage(st, bytes.NewBufferString(gzipMessage(t, sampleXML)), "dmarc-reports@mail.example.com", time.Now().UTC()); err == nil {
|
||||
t.Fatal("report for an unconfigured domain was accepted")
|
||||
err = IngestMessage(st, bytes.NewBufferString(gzipMessage(t, sampleXML)), "dmarc-reports@mail.example.com", time.Now().UTC())
|
||||
if !errors.Is(err, ErrRejected) {
|
||||
t.Fatalf("report for an unconfigured domain: err = %v, want ErrRejected", err)
|
||||
}
|
||||
list, err := st.ListDMARCReports(store.DMARCReportScope{AllDomains: true}, 10)
|
||||
if err != nil {
|
||||
@@ -155,8 +157,47 @@ func TestIngestMessageRejectsTagMismatch(t *testing.T) {
|
||||
|
||||
err = IngestMessage(st, bytes.NewBufferString(gzipMessage(t, sampleXML)),
|
||||
"dmarc-reports+other.example@mail.example.com", time.Now().UTC())
|
||||
if err == nil {
|
||||
t.Fatal("report was accepted at an address tagged for another domain")
|
||||
if !errors.Is(err, ErrRejected) {
|
||||
t.Fatalf("report at an address tagged for another domain: err = %v, want ErrRejected", err)
|
||||
}
|
||||
}
|
||||
|
||||
// What cannot be read as a report is rejected, not put off: Postfix would
|
||||
// offer the same bytes again for days.
|
||||
func TestIngestMessageRejectsWhatIsNotAReport(t *testing.T) {
|
||||
st, err := store.Open(t.TempDir() + "/test.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer st.Close()
|
||||
if _, err := st.AddDomain("example.com", "selfpost"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, msg := range map[string]string{
|
||||
"plain text": "From: someone@example.net\r\nSubject: hello\r\n\r\nnot a report\r\n",
|
||||
"no headers": "garbage",
|
||||
"broken gzip": "From: a@example.net\r\nContent-Type: application/gzip\r\n\r\nnot gzip at all",
|
||||
} {
|
||||
err := IngestMessage(st, bytes.NewBufferString(msg), "dmarc-reports+example.com@mail.example.com", time.Now().UTC())
|
||||
if !errors.Is(err, ErrRejected) {
|
||||
t.Errorf("%s: err = %v, want ErrRejected", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A failing store is not the report's fault: the message is to be tried again.
|
||||
func TestIngestMessageStoreFailureIsNotARejection(t *testing.T) {
|
||||
st, err := store.Open(t.TempDir() + "/test.db")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.AddDomain("example.com", "selfpost"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st.Close()
|
||||
err = IngestMessage(st, bytes.NewBufferString(gzipMessage(t, sampleXML)), "dmarc-reports+example.com@mail.example.com", time.Now().UTC())
|
||||
if err == nil || errors.Is(err, ErrRejected) {
|
||||
t.Fatalf("closed store: err = %v, want an error that is not ErrRejected", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -100,6 +100,7 @@ type Checker struct {
|
||||
servers map[string]cached[Server]
|
||||
domains map[string]cached[Domain]
|
||||
inbound map[string]cached[Result]
|
||||
mail map[string]cached[Mail]
|
||||
}
|
||||
|
||||
type cached[T any] struct {
|
||||
@@ -123,6 +124,7 @@ func newChecker(r resolver, timeout, srvTTL, domTTL time.Duration) *Checker {
|
||||
servers: make(map[string]cached[Server]),
|
||||
domains: make(map[string]cached[Domain]),
|
||||
inbound: make(map[string]cached[Result]),
|
||||
mail: make(map[string]cached[Mail]),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,6 +172,56 @@ func (c *Checker) Domain(q Query, force bool) Domain {
|
||||
return d
|
||||
}
|
||||
|
||||
// Mail is the published-DNS state of the server's own name as a sender: the
|
||||
// panel's notifications go out from postmaster@ the hostname, signed with a
|
||||
// key of that name.
|
||||
type Mail struct {
|
||||
Name string
|
||||
DKIM Result
|
||||
SPF Result
|
||||
Overall health.Status
|
||||
CheckedAt time.Time
|
||||
}
|
||||
|
||||
// ServerMailSPF is the SPF record to publish for the server's own name: its
|
||||
// address record is the one sender.
|
||||
const ServerMailSPF = "v=spf1 a -all"
|
||||
|
||||
// ServerMail checks the DKIM and SPF records of the server's own name. q is as
|
||||
// for a domain, with Name the hostname. Cached like a domain; force skips the
|
||||
// cache.
|
||||
func (c *Checker) ServerMail(q Query, force bool) Mail {
|
||||
key := "\x00mail\x00" + q.Name
|
||||
if !force {
|
||||
c.mu.Lock()
|
||||
entry, ok := c.mail[key]
|
||||
c.mu.Unlock()
|
||||
if ok && time.Now().Before(entry.expires) {
|
||||
return entry.value
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
|
||||
defer cancel()
|
||||
m := Mail{Name: q.Name, CheckedAt: time.Now()}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); m.DKIM = c.checkDKIM(ctx, q) }()
|
||||
go func() { defer wg.Done(); m.SPF = c.checkSPF(ctx, q) }()
|
||||
wg.Wait()
|
||||
// checkSPF words a missing record for a sending domain; for the server's
|
||||
// own name the record to publish is a different one.
|
||||
if m.SPF.Status == health.StatusError && len(m.SPF.Records) == 0 {
|
||||
m.SPF.Detail = fmt.Sprintf("No SPF record is published for %s. Publish the TXT record %q - without it a receiver has nothing authorising this server to send as its own name.", q.Name, ServerMailSPF)
|
||||
}
|
||||
m.Overall = health.Worst(m.DKIM.Status, m.SPF.Status)
|
||||
|
||||
c.mu.Lock()
|
||||
c.mail[key] = cached[Mail]{value: m, expires: m.CheckedAt.Add(c.domainTTL)}
|
||||
c.mu.Unlock()
|
||||
return m
|
||||
}
|
||||
|
||||
// Forget drops a domain's cached result, so the next page view re-checks it.
|
||||
// Used when a domain is removed or re-imported.
|
||||
func (c *Checker) Forget(domainName string) {
|
||||
|
||||
@@ -42,6 +42,9 @@ func (f *fakeResolver) LookupTXT(_ context.Context, name string) ([]string, erro
|
||||
|
||||
func (f *fakeResolver) LookupIPAddr(_ context.Context, host string) ([]net.IPAddr, error) {
|
||||
f.lookups++
|
||||
if f.fail[host] {
|
||||
return nil, &net.DNSError{Err: "server misbehaving", Name: host, IsTemporary: true}
|
||||
}
|
||||
if v, ok := f.addr[host]; ok {
|
||||
return v, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
package dnscheck
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/health"
|
||||
)
|
||||
|
||||
const serverDKIM = "v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAserverkey"
|
||||
|
||||
func serverMailQuery() Query {
|
||||
return Query{
|
||||
Name: "mail.example.org",
|
||||
Selector: "sp",
|
||||
ExpectedDKIM: serverDKIM,
|
||||
Hostname: "mail.example.org",
|
||||
ServerIPs: []string{"203.0.113.25"},
|
||||
}
|
||||
}
|
||||
|
||||
// The server's own name passes with its key published and "v=spf1 a -all":
|
||||
// the name's address record is this server.
|
||||
func TestServerMailPublished(t *testing.T) {
|
||||
f := &fakeResolver{
|
||||
txt: map[string][]string{
|
||||
"sp._domainkey.mail.example.org": {serverDKIM},
|
||||
"mail.example.org": {ServerMailSPF},
|
||||
},
|
||||
addr: map[string][]net.IPAddr{"mail.example.org": ipAddrs("203.0.113.25")},
|
||||
}
|
||||
c := newTestChecker(f)
|
||||
|
||||
m := c.ServerMail(serverMailQuery(), false)
|
||||
if m.DKIM.Status != health.StatusOK || m.SPF.Status != health.StatusOK || m.Overall != health.StatusOK {
|
||||
t.Fatalf("DKIM %s (%s), SPF %s (%s), overall %s", m.DKIM.Status, m.DKIM.Detail, m.SPF.Status, m.SPF.Detail, m.Overall)
|
||||
}
|
||||
|
||||
// Cached like a domain, and re-read when forced.
|
||||
before := f.lookups
|
||||
c.ServerMail(serverMailQuery(), false)
|
||||
if f.lookups != before {
|
||||
t.Errorf("a second reading looked DNS up again (%d lookups)", f.lookups-before)
|
||||
}
|
||||
c.ServerMail(serverMailQuery(), true)
|
||||
if f.lookups == before {
|
||||
t.Error("a forced reading used the cache")
|
||||
}
|
||||
}
|
||||
|
||||
// With nothing published both records are errors, and the SPF advice names the
|
||||
// record for a server's own name, not the one for a sending domain.
|
||||
func TestServerMailMissing(t *testing.T) {
|
||||
c := newTestChecker(&fakeResolver{})
|
||||
|
||||
m := c.ServerMail(serverMailQuery(), false)
|
||||
if m.DKIM.Status != health.StatusError || m.SPF.Status != health.StatusError || m.Overall != health.StatusError {
|
||||
t.Fatalf("DKIM %s, SPF %s, overall %s; want errors", m.DKIM.Status, m.SPF.Status, m.Overall)
|
||||
}
|
||||
if !strings.Contains(m.SPF.Detail, ServerMailSPF) || strings.Contains(m.SPF.Detail, "a:mail.example.org") {
|
||||
t.Errorf("SPF advice: %s", m.SPF.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
// A sending domain of the same name does not share the cache entry.
|
||||
func TestServerMailCacheIsItsOwn(t *testing.T) {
|
||||
f := &fakeResolver{
|
||||
txt: map[string][]string{
|
||||
"sp._domainkey.mail.example.org": {serverDKIM},
|
||||
"mail.example.org": {ServerMailSPF},
|
||||
},
|
||||
addr: map[string][]net.IPAddr{"mail.example.org": ipAddrs("203.0.113.25")},
|
||||
}
|
||||
c := newTestChecker(f)
|
||||
c.Domain(serverMailQuery(), false)
|
||||
before := f.lookups
|
||||
c.ServerMail(serverMailQuery(), false)
|
||||
if f.lookups == before {
|
||||
t.Error("the server's mail check was answered from the domain's cache entry")
|
||||
}
|
||||
}
|
||||
|
||||
// The address lookup behind an "a" mechanism did not answer: nothing was
|
||||
// learned, and the record is not called wrong for it. A name that answers
|
||||
// "no such host" is a finding, and the record is wrong.
|
||||
func TestSPFLookupThatDidNotAnswerIsNotAMismatch(t *testing.T) {
|
||||
// The fake fails by name, so the mechanism names a host of its own: the
|
||||
// TXT lookup of the server's name answers, the address lookup does not.
|
||||
f := &fakeResolver{
|
||||
txt: map[string][]string{"mail.example.org": {"v=spf1 a:addr.example.org -all"}},
|
||||
fail: map[string]bool{"addr.example.org": true},
|
||||
}
|
||||
c := newTestChecker(f)
|
||||
q := serverMailQuery()
|
||||
if got := c.ServerMail(q, true).SPF; got.Status != health.StatusWarn {
|
||||
t.Errorf("the lookup did not answer: SPF is %s (%s), want a warning", got.Status, got.Detail)
|
||||
}
|
||||
|
||||
f.fail = nil
|
||||
if got := c.ServerMail(q, true).SPF; got.Status != health.StatusError {
|
||||
t.Errorf("the name does not exist: SPF is %s (%s), want an error", got.Status, got.Detail)
|
||||
}
|
||||
}
|
||||
+35
-12
@@ -2,6 +2,7 @@ package dnscheck
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
@@ -133,7 +134,14 @@ func (c *Checker) evaluateSPF(ctx context.Context, record, domainName string, ip
|
||||
continue
|
||||
}
|
||||
budget--
|
||||
if qualifier == '+' && c.resolvesTo(ctx, target, ips) {
|
||||
match, failed := c.resolvesTo(ctx, target, ips)
|
||||
if failed {
|
||||
// Nothing was learned about this mechanism: "cannot tell",
|
||||
// not "does not authorise".
|
||||
note("a (the lookup did not answer)")
|
||||
continue
|
||||
}
|
||||
if qualifier == '+' && match {
|
||||
return term, unfollowed
|
||||
}
|
||||
|
||||
@@ -151,7 +159,12 @@ func (c *Checker) evaluateSPF(ctx context.Context, record, domainName string, ip
|
||||
continue
|
||||
}
|
||||
budget--
|
||||
if qualifier == '+' && c.mxResolvesTo(ctx, target, ips, &budget) {
|
||||
match, failed := c.mxResolvesTo(ctx, target, ips, &budget)
|
||||
if failed {
|
||||
note("mx (the lookup did not answer)")
|
||||
continue
|
||||
}
|
||||
if qualifier == '+' && match {
|
||||
return term, unfollowed
|
||||
}
|
||||
|
||||
@@ -176,39 +189,49 @@ func (c *Checker) evaluateSPF(ctx context.Context, record, domainName string, ip
|
||||
return "", unfollowed
|
||||
}
|
||||
|
||||
// lookupUnanswered tells a lookup that failed from one that answered "no such
|
||||
// name": only the second is a finding.
|
||||
func lookupUnanswered(err error) bool {
|
||||
var dnsErr *net.DNSError
|
||||
return err != nil && !(errors.As(err, &dnsErr) && dnsErr.IsNotFound)
|
||||
}
|
||||
|
||||
// resolvesTo reports whether name resolves to one of the server's addresses.
|
||||
func (c *Checker) resolvesTo(ctx context.Context, name string, ips []net.IP) bool {
|
||||
// failed is set when the lookup did not answer, which says nothing either way.
|
||||
func (c *Checker) resolvesTo(ctx context.Context, name string, ips []net.IP) (match, failed bool) {
|
||||
addrs, err := c.resolver.LookupIPAddr(ctx, name)
|
||||
if err != nil {
|
||||
return false
|
||||
return false, lookupUnanswered(err)
|
||||
}
|
||||
for _, a := range addrs {
|
||||
for _, ip := range ips {
|
||||
if a.IP.Equal(ip) {
|
||||
return true
|
||||
return true, false
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
return false, false
|
||||
}
|
||||
|
||||
// mxResolvesTo reports whether any of name's MX hosts resolves to one of the
|
||||
// server's addresses, spending at most the remaining lookup budget.
|
||||
func (c *Checker) mxResolvesTo(ctx context.Context, name string, ips []net.IP, budget *int) bool {
|
||||
func (c *Checker) mxResolvesTo(ctx context.Context, name string, ips []net.IP, budget *int) (match, failed bool) {
|
||||
mxs, err := c.resolver.LookupMX(ctx, name)
|
||||
if err != nil {
|
||||
return false
|
||||
return false, lookupUnanswered(err)
|
||||
}
|
||||
for _, mx := range mxs {
|
||||
if *budget <= 0 {
|
||||
return false
|
||||
return false, failed
|
||||
}
|
||||
*budget--
|
||||
if c.resolvesTo(ctx, strings.TrimSuffix(mx.Host, "."), ips) {
|
||||
return true
|
||||
m, f := c.resolvesTo(ctx, strings.TrimSuffix(mx.Host, "."), ips)
|
||||
if m {
|
||||
return true, false
|
||||
}
|
||||
failed = failed || f
|
||||
}
|
||||
return false
|
||||
return false, failed
|
||||
}
|
||||
|
||||
// coversAny reports whether an ip4:/ip6: value — a bare address or a CIDR —
|
||||
|
||||
@@ -144,6 +144,12 @@ func (o *OpenDKIM) Record(domainName, selector string) (DKIMRecord, error) {
|
||||
return dkimRecord(selector, domainName, &key.PublicKey)
|
||||
}
|
||||
|
||||
// SigningTablePath is the SigningTable's location: the names this server
|
||||
// signs for, which the sender check reads (internal/milter/sender.go).
|
||||
func (o *OpenDKIM) SigningTablePath() string {
|
||||
return o.signingTablePath
|
||||
}
|
||||
|
||||
// Rebuild regenerates KeyTable and SigningTable from the full domain set and
|
||||
// reloads OpenDKIM (architecture.md § OpenDKIM). Full regeneration (rather
|
||||
// than incremental edits) keeps the files a pure function of the registry, so
|
||||
@@ -163,6 +169,35 @@ func (o *OpenDKIM) Rebuild(domains []SigningDomain) error {
|
||||
return o.reload()
|
||||
}
|
||||
|
||||
// signs reports whether the tables on disk sign name with the key of
|
||||
// selector: a line for the name in the SigningTable, and in the KeyTable the
|
||||
// line that gives it that selector. Tables that do not exist yet sign nothing.
|
||||
// The selector is part of the question because the panel shows and checks the
|
||||
// record of one key; tables that name another would have OpenDKIM sign with a
|
||||
// key whose record nobody is told to publish.
|
||||
func (o *OpenDKIM) signs(name, selector string) (bool, error) {
|
||||
has := func(path, prefix string) (bool, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read %s: %w", filepath.Base(path), err)
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
if strings.HasPrefix(line, prefix) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
signed, err := has(o.signingTablePath, "*@"+name+" ")
|
||||
if err != nil || !signed {
|
||||
return false, err
|
||||
}
|
||||
return has(o.keyTablePath, name+" "+name+":"+selector+":")
|
||||
}
|
||||
|
||||
// Reload asks OpenDKIM to re-read its tables without regenerating them. It
|
||||
// backs the panel's manual reload button (architecture.md § Panel HTTP
|
||||
// surface).
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func readTables(t *testing.T, odk *OpenDKIM) (keyTable, signingTable string) {
|
||||
t.Helper()
|
||||
kt, err := os.ReadFile(odk.keyTablePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read KeyTable: %v", err)
|
||||
}
|
||||
st, err := os.ReadFile(odk.signingTablePath)
|
||||
if err != nil {
|
||||
t.Fatalf("read SigningTable: %v", err)
|
||||
}
|
||||
return string(kt), string(st)
|
||||
}
|
||||
|
||||
// The server's own name is signed like a sending domain - a key, a line in
|
||||
// each table - without being one: no row in the registry, so no application
|
||||
// can send as it.
|
||||
func TestServerNameIsSignedWithoutBeingADomain(t *testing.T) {
|
||||
svc, odk := newTestService(t, &fakeApps{})
|
||||
svc.SetServerName("mail.example.org")
|
||||
|
||||
if _, err := svc.Add("example.com"); err != nil {
|
||||
t.Fatalf("add domain: %v", err)
|
||||
}
|
||||
kt, st := readTables(t, odk)
|
||||
for _, want := range []string{"mail.example.org mail.example.org:selfpost:", "example.com example.com:selfpost:"} {
|
||||
if !strings.Contains(kt, want) {
|
||||
t.Errorf("KeyTable lacks %q:\n%s", want, kt)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(st, "*@mail.example.org mail.example.org\n") {
|
||||
t.Errorf("SigningTable does not sign the server's own name:\n%s", st)
|
||||
}
|
||||
domains, err := svc.List()
|
||||
if err != nil || len(domains) != 1 {
|
||||
t.Fatalf("registry holds %d domains, %v; want the one that was added", len(domains), err)
|
||||
}
|
||||
|
||||
mail, err := svc.ServerMail()
|
||||
if err != nil {
|
||||
t.Fatalf("ServerMail: %v", err)
|
||||
}
|
||||
if mail.Name != "mail.example.org" || mail.Selector != "selfpost" ||
|
||||
mail.Record.Name != "selfpost._domainkey.mail.example.org" || !strings.HasPrefix(mail.Record.Value, "v=DKIM1;") {
|
||||
t.Errorf("server mail = %+v", mail)
|
||||
}
|
||||
// Asked again, it is the same key: the record published in DNS stays valid.
|
||||
again, err := svc.ServerMail()
|
||||
if err != nil || again.Record.Value != mail.Record.Value {
|
||||
t.Errorf("the server's key changed between two readings (%v)", err)
|
||||
}
|
||||
}
|
||||
|
||||
// When the hostname is itself a sending domain, there is one key and one line,
|
||||
// the domain's; deleting the domain leaves the server's mail signed with it.
|
||||
func TestServerNameThatIsAlsoADomain(t *testing.T) {
|
||||
svc, odk := newTestService(t, &fakeApps{})
|
||||
svc.SetServerName("example.com")
|
||||
|
||||
d, err := svc.Add("example.com")
|
||||
if err != nil {
|
||||
t.Fatalf("add domain: %v", err)
|
||||
}
|
||||
_, st := readTables(t, odk)
|
||||
if strings.Count(st, "*@example.com ") != 1 {
|
||||
t.Errorf("the name is in the SigningTable more than once:\n%s", st)
|
||||
}
|
||||
asDomain, err := svc.DKIMRecord(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mail, err := svc.ServerMail()
|
||||
if err != nil || mail.Record.Value != asDomain.Value {
|
||||
t.Fatalf("the server's record is not the domain's (%v)", err)
|
||||
}
|
||||
|
||||
if err := svc.Delete(d.ID); err != nil {
|
||||
t.Fatalf("delete domain: %v", err)
|
||||
}
|
||||
_, st = readTables(t, odk)
|
||||
if !strings.Contains(st, "*@example.com example.com\n") {
|
||||
t.Errorf("deleting the domain stopped the server's own mail being signed:\n%s", st)
|
||||
}
|
||||
after, err := svc.ServerMail()
|
||||
if err != nil || after.Record.Value != asDomain.Value {
|
||||
t.Errorf("deleting the domain changed the server's key (%v)", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An installation upgraded to a version that signs the server's own name has
|
||||
// tables without its line until something rebuilds them. The start does, once.
|
||||
func TestEnsureServerSignedRebuildsOnce(t *testing.T) {
|
||||
svc, odk := newTestService(t, &fakeApps{})
|
||||
if _, err := svc.Add("example.com"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reloads := 0
|
||||
odk.reload = func() error { reloads++; return nil }
|
||||
|
||||
// The tables as the earlier version left them; now the name is known.
|
||||
svc.SetServerName("mail.example.org")
|
||||
if rebuilt, err := svc.EnsureServerSigned(); err != nil || !rebuilt {
|
||||
t.Fatalf("first start: rebuilt = %v, %v; want a rebuild", rebuilt, err)
|
||||
}
|
||||
if _, st := readTables(t, odk); !strings.Contains(st, "*@mail.example.org mail.example.org\n") || !strings.Contains(st, "*@example.com ") {
|
||||
t.Errorf("SigningTable after the rebuild:\n%s", st)
|
||||
}
|
||||
if rebuilt, err := svc.EnsureServerSigned(); err != nil || rebuilt {
|
||||
t.Errorf("second start: rebuilt = %v, %v; want nothing to do", rebuilt, err)
|
||||
}
|
||||
if reloads != 1 {
|
||||
t.Errorf("OpenDKIM was reloaded %d times, want once", reloads)
|
||||
}
|
||||
|
||||
// A name that only begins like the server's is not it.
|
||||
other, otherOdk := newTestService(t, &fakeApps{})
|
||||
other.SetServerName("mail.example.org.evil.example")
|
||||
if _, err := other.EnsureServerSigned(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other.SetServerName("mail.example.org")
|
||||
if signed, err := otherOdk.signs("mail.example.org", "selfpost"); err != nil || signed {
|
||||
t.Errorf("a longer name was taken for the server's: %v, %v", signed, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tables that sign the server's name with the key of another selector - the
|
||||
// default selector was changed - are rebuilt: OpenDKIM must sign with the key
|
||||
// whose record the panel shows.
|
||||
func TestEnsureServerSignedFollowsTheSelector(t *testing.T) {
|
||||
svc, odk := newTestService(t, &fakeApps{})
|
||||
svc.SetServerName("mail.example.org")
|
||||
if _, err := svc.EnsureServerSigned(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
svc.selector = "sp2027"
|
||||
if rebuilt, err := svc.EnsureServerSigned(); err != nil || !rebuilt {
|
||||
t.Fatalf("after the selector changed: rebuilt = %v, %v; want a rebuild", rebuilt, err)
|
||||
}
|
||||
kt, _ := readTables(t, odk)
|
||||
if !strings.Contains(kt, "mail.example.org mail.example.org:sp2027:") || strings.Contains(kt, ":selfpost:") {
|
||||
t.Errorf("KeyTable after the rebuild:\n%s", kt)
|
||||
}
|
||||
mail, err := svc.ServerMail()
|
||||
if err != nil || mail.Selector != "sp2027" {
|
||||
t.Errorf("the panel shows selector %q (%v), the tables sign with sp2027", mail.Selector, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Without a hostname nothing is added and nothing is signed for it.
|
||||
func TestNoServerName(t *testing.T) {
|
||||
svc, odk := newTestService(t, &fakeApps{})
|
||||
if _, err := svc.Add("example.com"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, st := readTables(t, odk); strings.Count(st, "\n") != 1 {
|
||||
t.Errorf("SigningTable has more than the one domain:\n%s", st)
|
||||
}
|
||||
if _, err := svc.ServerMail(); err == nil {
|
||||
t.Error("ServerMail without a hostname returned no error")
|
||||
}
|
||||
}
|
||||
+104
-1
@@ -38,6 +38,89 @@ type Service struct {
|
||||
odk *OpenDKIM
|
||||
apps Applications
|
||||
selector string
|
||||
// serverName is the server's own hostname, which the panel's notifications
|
||||
// are sent from and signed as (SetServerName). Empty: nothing is signed
|
||||
// for it.
|
||||
serverName string
|
||||
}
|
||||
|
||||
// SetServerName gives the service the server's own hostname. The panel sends
|
||||
// its notifications from postmaster@ that name, and for a receiver to believe
|
||||
// them the name is signed like a sending domain: it gets a DKIM key and a line
|
||||
// in OpenDKIM's tables, without being a row of the registry - no application
|
||||
// can send as it. The name comes from SELFPOST_HOSTNAME, which the entrypoint
|
||||
// has validated; it is not user input.
|
||||
func (s *Service) SetServerName(hostname string) {
|
||||
s.serverName = hostname
|
||||
}
|
||||
|
||||
// ServerMail is how the server's own name is signed.
|
||||
type ServerMail struct {
|
||||
Name string
|
||||
Selector string
|
||||
Record DKIMRecord
|
||||
}
|
||||
|
||||
// ServerMail returns the name notifications are sent as, its DKIM selector and
|
||||
// the record to publish for it, creating the key the first time. When the
|
||||
// hostname is itself a sending domain, it is that domain's key and selector:
|
||||
// one name has one key.
|
||||
func (s *Service) ServerMail() (ServerMail, error) {
|
||||
if s.serverName == "" {
|
||||
return ServerMail{}, fmt.Errorf("server mail: no hostname")
|
||||
}
|
||||
selector, err := s.serverSelector()
|
||||
if err != nil {
|
||||
return ServerMail{}, err
|
||||
}
|
||||
if _, err := s.odk.EnsureKey(s.serverName, selector); err != nil {
|
||||
return ServerMail{}, err
|
||||
}
|
||||
rec, err := s.odk.Record(s.serverName, selector)
|
||||
if err != nil {
|
||||
return ServerMail{}, err
|
||||
}
|
||||
return ServerMail{Name: s.serverName, Selector: selector, Record: rec}, nil
|
||||
}
|
||||
|
||||
// EnsureServerSigned makes sure OpenDKIM's tables have the line for the
|
||||
// server's own name, rebuilding them if they do not. The tables are otherwise
|
||||
// rebuilt only when a domain is added or deleted, so an installation that was
|
||||
// upgraded to a version with notifications would go on sending them unsigned
|
||||
// until then, and one whose DKIM_SELECTOR_DEFAULT was changed would go on
|
||||
// signing with the key of the old selector while the panel shows the record
|
||||
// of the new. It reports whether it had to rebuild.
|
||||
func (s *Service) EnsureServerSigned() (bool, error) {
|
||||
if s.serverName == "" {
|
||||
return false, nil
|
||||
}
|
||||
selector, err := s.serverSelector()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
signed, err := s.odk.signs(s.serverName, selector)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if signed {
|
||||
return false, nil
|
||||
}
|
||||
return true, s.resync()
|
||||
}
|
||||
|
||||
// serverSelector is the selector the server's own name is signed with: that of
|
||||
// the sending domain of the same name if there is one, the default otherwise.
|
||||
func (s *Service) serverSelector() (string, error) {
|
||||
domains, err := s.store.ListDomains()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, d := range domains {
|
||||
if d.Name == s.serverName {
|
||||
return d.DKIMSelector, nil
|
||||
}
|
||||
}
|
||||
return s.selector, nil
|
||||
}
|
||||
|
||||
// NewService builds the domain service. selectorDefault is the DKIM selector
|
||||
@@ -125,6 +208,14 @@ func (s *Service) Delete(id int64) error {
|
||||
if err := s.apps.Resync(); err != nil {
|
||||
return fmt.Errorf("rebuild sender map after deleting %s: %w", d.Name, err)
|
||||
}
|
||||
// A sending domain named like the server shared its key with the server's
|
||||
// own mail. The resync above kept the name in the tables under the default
|
||||
// selector: the same key if that was the domain's selector, a new one -
|
||||
// with a new record to publish, which Settings shows - if it was not.
|
||||
// Either way the directory holds the key now in use.
|
||||
if d.Name == s.serverName {
|
||||
return nil
|
||||
}
|
||||
if err := s.odk.RemoveKey(d.Name); err != nil {
|
||||
// The domain is gone from the registry and tables; a leftover key
|
||||
// directory is harmless. Surface it so it is not silently ignored.
|
||||
@@ -172,9 +263,21 @@ func (s *Service) resync() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
signing := make([]SigningDomain, 0, len(domains))
|
||||
signing := make([]SigningDomain, 0, len(domains)+1)
|
||||
registered := false
|
||||
for _, d := range domains {
|
||||
signing = append(signing, SigningDomain{Name: d.Name, Selector: d.DKIMSelector})
|
||||
if d.Name == s.serverName {
|
||||
registered = true
|
||||
}
|
||||
}
|
||||
// The server's own name, for the panel's notifications - unless it is a
|
||||
// sending domain already and so in the tables above.
|
||||
if s.serverName != "" && !registered {
|
||||
if _, err := s.odk.EnsureKey(s.serverName, s.selector); err != nil {
|
||||
return fmt.Errorf("dkim key for %s: %w", s.serverName, err)
|
||||
}
|
||||
signing = append(signing, SigningDomain{Name: s.serverName, Selector: s.selector})
|
||||
}
|
||||
return s.odk.Rebuild(signing)
|
||||
}
|
||||
|
||||
@@ -188,3 +188,21 @@ postfix FATAL Exited too quickly
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUptime(t *testing.T) {
|
||||
for detail, want := range map[string]time.Duration{
|
||||
"pid 41, uptime 0:03:12": 3*time.Minute + 12*time.Second,
|
||||
"pid 41, uptime 18 days, 3:02:11": 18*24*time.Hour + 3*time.Hour + 2*time.Minute + 11*time.Second,
|
||||
"pid 41, uptime 1 day, 0:00:00": 24 * time.Hour,
|
||||
"pid 7, uptime 127:00:01": 127*time.Hour + time.Second,
|
||||
} {
|
||||
if got, ok := Uptime(detail); !ok || got != want {
|
||||
t.Errorf("Uptime(%q) = %s, %v; want %s", detail, got, ok, want)
|
||||
}
|
||||
}
|
||||
for _, detail := range []string{"", "Not started", "Oct 12 04:00 PM", "pid 41, uptime soon", "pid 41, uptime x days, 0:00:01", "pid 41, uptime 0:00"} {
|
||||
if got, ok := Uptime(detail); ok {
|
||||
t.Errorf("Uptime(%q) = %s, want none", detail, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,9 @@ package health
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// supervisorConf is the supervisord configuration the panel's other control
|
||||
@@ -85,6 +87,39 @@ func parseProcesses(out string) []Process {
|
||||
return procs
|
||||
}
|
||||
|
||||
// Uptime reads for how long a running program has been up from the detail
|
||||
// supervisord prints for it: "pid 41, uptime 0:03:12", or with days in front,
|
||||
// "pid 41, uptime 18 days, 3:02:11".
|
||||
func Uptime(detail string) (time.Duration, bool) {
|
||||
_, rest, found := strings.Cut(detail, "uptime ")
|
||||
if !found {
|
||||
return 0, false
|
||||
}
|
||||
var days int
|
||||
if d, clock, hasDays := strings.Cut(rest, ", "); hasDays {
|
||||
n, _, _ := strings.Cut(d, " ")
|
||||
v, err := strconv.Atoi(n)
|
||||
if err != nil || v < 0 {
|
||||
return 0, false
|
||||
}
|
||||
days, rest = v, clock
|
||||
}
|
||||
parts := strings.Split(strings.TrimSpace(rest), ":")
|
||||
if len(parts) != 3 {
|
||||
return 0, false
|
||||
}
|
||||
var hms [3]int
|
||||
for i, p := range parts {
|
||||
v, err := strconv.Atoi(p)
|
||||
if err != nil || v < 0 {
|
||||
return 0, false
|
||||
}
|
||||
hms[i] = v
|
||||
}
|
||||
return time.Duration(days)*24*time.Hour + time.Duration(hms[0])*time.Hour +
|
||||
time.Duration(hms[1])*time.Minute + time.Duration(hms[2])*time.Second, true
|
||||
}
|
||||
|
||||
// processStatus grades a supervisord state. A one-shot program that is not
|
||||
// running is healthy; anything else that is not RUNNING means a component of
|
||||
// the mail path is down or flapping.
|
||||
|
||||
@@ -0,0 +1,447 @@
|
||||
package inbound
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/atomicfile"
|
||||
"github.com/mixeme/selfpost/internal/configsafe"
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
|
||||
// ListStore is the slice of the store the filter lists need. *store.Store
|
||||
// satisfies it.
|
||||
type ListStore interface {
|
||||
AddSpamListEntry(list, kind, value, note string) (store.SpamListEntry, error)
|
||||
ListSpamListEntries() ([]store.SpamListEntry, error)
|
||||
DeleteSpamListEntry(id int64) error
|
||||
ListSpamListHits(marks ...string) ([]store.SpamListHit, error)
|
||||
}
|
||||
|
||||
// Lists owns the allow and deny lists of the external spam filter: the rows in
|
||||
// SQLite and the map files the filter reads. SelfPost does not consult the
|
||||
// lists itself - it writes them out, six files in one directory, one entry a
|
||||
// line, and the filter (rspamd, deploy/antispam/rspamd/multimap.conf) rereads
|
||||
// a file when it changes. Which list wins when both match is the filter's
|
||||
// configuration: deny.
|
||||
type Lists struct {
|
||||
store ListStore
|
||||
dir string
|
||||
// mu makes a change of the table and the rewrite of the files one step.
|
||||
// Two requests at once could otherwise write the files in the other order
|
||||
// than they changed the table - the first to read the entries being the
|
||||
// last to write them - and leave the filter with an entry the page no
|
||||
// longer lists. One Lists serves the panel, so a mutex is enough.
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewLists builds the filter lists over the store and the map directory
|
||||
// (/data/antispam).
|
||||
func NewLists(st ListStore, dir string) *Lists {
|
||||
return &Lists{store: st, dir: dir}
|
||||
}
|
||||
|
||||
// noteMaxRunes caps the operator's comment on an entry.
|
||||
const noteMaxRunes = 200
|
||||
|
||||
// mapFiles names the file of every (list, kind). An IP map and a sender map
|
||||
// are different things to the filter, and a domain entry is matched against
|
||||
// another part of the sender than an address, so each has its own file.
|
||||
var mapFiles = map[[2]string]string{
|
||||
{store.SpamListAllow, store.SpamEntryIP}: "allow_ip.map",
|
||||
{store.SpamListAllow, store.SpamEntryAddress}: "allow_from.map",
|
||||
{store.SpamListAllow, store.SpamEntryDomain}: "allow_domain.map",
|
||||
{store.SpamListDeny, store.SpamEntryIP}: "deny_ip.map",
|
||||
{store.SpamListDeny, store.SpamEntryAddress}: "deny_from.map",
|
||||
{store.SpamListDeny, store.SpamEntryDomain}: "deny_domain.map",
|
||||
}
|
||||
|
||||
// Entries returns every entry of both lists.
|
||||
func (l *Lists) Entries() ([]store.SpamListEntry, error) {
|
||||
return l.store.ListSpamListEntries()
|
||||
}
|
||||
|
||||
// Add validates an entry, stores it normalised and rewrites the maps. A
|
||||
// refusal is a *ValidationError; an entry already on that list is
|
||||
// store.ErrSpamListEntryExists.
|
||||
func (l *Lists) Add(list, kind, rawValue, rawNote string) (store.SpamListEntry, error) {
|
||||
if list != store.SpamListAllow && list != store.SpamListDeny {
|
||||
return store.SpamListEntry{}, invalid("choose the allow or the deny list")
|
||||
}
|
||||
value, err := normalizeListValue(kind, rawValue)
|
||||
if err != nil {
|
||||
return store.SpamListEntry{}, err
|
||||
}
|
||||
note, err := normalizeNote(rawNote)
|
||||
if err != nil {
|
||||
return store.SpamListEntry{}, err
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
e, err := l.store.AddSpamListEntry(list, kind, value, note)
|
||||
if err != nil {
|
||||
return store.SpamListEntry{}, err
|
||||
}
|
||||
if err := l.sync(); err != nil {
|
||||
// An entry the filter never got is worse than no entry: the page
|
||||
// would show a rule that does nothing. The files are written again
|
||||
// after the row is taken back, since some of the six may already
|
||||
// hold it; if that fails too, the next change or start puts it right.
|
||||
_ = l.store.DeleteSpamListEntry(e.ID)
|
||||
_ = l.sync()
|
||||
return store.SpamListEntry{}, err
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// Delete removes an entry and rewrites the maps. If the maps cannot be
|
||||
// written the entry is put back: an entry that is gone from the page and
|
||||
// still in the filter's file could not even be removed a second time.
|
||||
func (l *Lists) Delete(id int64) error {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
entries, err := l.store.ListSpamListEntries()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var gone *store.SpamListEntry
|
||||
for i := range entries {
|
||||
if entries[i].ID == id {
|
||||
gone = &entries[i]
|
||||
}
|
||||
}
|
||||
if err := l.store.DeleteSpamListEntry(id); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := l.sync(); err != nil {
|
||||
if gone != nil {
|
||||
_, _ = l.store.AddSpamListEntry(gone.List, gone.Kind, gone.Value, gone.Note)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sync writes all six map files from SQLite, each atomically, so the filter
|
||||
// reads a whole old file or a whole new one. Every file is written even when
|
||||
// empty: the filter's configuration names all six.
|
||||
func (l *Lists) Sync() error {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.sync()
|
||||
}
|
||||
|
||||
// sync is Sync for a caller that holds mu.
|
||||
func (l *Lists) sync() error {
|
||||
entries, err := l.store.ListSpamListEntries()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lines := make(map[string][]string, len(mapFiles))
|
||||
for _, name := range mapFiles {
|
||||
lines[name] = nil
|
||||
}
|
||||
for _, e := range entries {
|
||||
name, ok := mapFiles[[2]string{e.List, e.Kind}]
|
||||
if !ok {
|
||||
return fmt.Errorf("filter list entry %d: unknown list %q or kind %q", e.ID, e.List, e.Kind)
|
||||
}
|
||||
// Values were validated on the way in; this is the writer's own guard
|
||||
// against a row that got into the table some other way. In a map file a
|
||||
// newline ends the entry, whitespace starts its value and # a comment.
|
||||
if err := configsafe.Token("filter list entry", e.Value, " \t\r\n#"); err != nil {
|
||||
return err
|
||||
}
|
||||
lines[name] = append(lines[name], e.Value)
|
||||
}
|
||||
// The filter runs under another user in another container: the directory
|
||||
// and the files are readable by anyone. They hold addresses, not secrets.
|
||||
if err := os.MkdirAll(l.dir, 0o755); err != nil {
|
||||
return fmt.Errorf("create %s: %w", l.dir, err)
|
||||
}
|
||||
for name, vals := range lines {
|
||||
sort.Strings(vals)
|
||||
var b strings.Builder
|
||||
b.WriteString("# Written by SelfPost (Inbound > Filter lists). Do not edit: the next change in the panel replaces this file.\n")
|
||||
for _, v := range vals {
|
||||
b.WriteString(v)
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
if err := atomicfile.Write(filepath.Join(l.dir, name), []byte(b.String()), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LastWritten is when the maps were last written: the newest modification
|
||||
// time of the six files, the zero time while none exists (the filter lists
|
||||
// have never been touched, or the directory is gone).
|
||||
func (l *Lists) LastWritten() time.Time {
|
||||
var newest time.Time
|
||||
for _, name := range mapFiles {
|
||||
if fi, err := os.Stat(filepath.Join(l.dir, name)); err == nil && fi.ModTime().After(newest) {
|
||||
newest = fi.ModTime()
|
||||
}
|
||||
}
|
||||
return newest
|
||||
}
|
||||
|
||||
// Listable reports whether value could be added to a list as an entry of kind.
|
||||
// The Inbound log asks before it offers a row's sender to the lists: an
|
||||
// envelope sender may be spelt in ways the lists do not take.
|
||||
func Listable(kind, value string) bool {
|
||||
_, err := normalizeListValue(kind, value)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// normalizeListValue validates one entry and returns the form it is stored and
|
||||
// written in: lower case for an address or a domain, the canonical text of an
|
||||
// address or a network for an IP.
|
||||
func normalizeListValue(kind, raw string) (string, error) {
|
||||
v := strings.ToLower(strings.TrimSpace(raw))
|
||||
switch kind {
|
||||
case store.SpamEntryAddress:
|
||||
if v == "" {
|
||||
return "", invalid("sender address is required")
|
||||
}
|
||||
if len(v) > 254 {
|
||||
return "", invalid("sender address must be at most 254 characters")
|
||||
}
|
||||
at := strings.LastIndexByte(v, '@')
|
||||
if at <= 0 || at >= len(v)-1 {
|
||||
return "", invalid("%q is not a valid email address", v)
|
||||
}
|
||||
local, domain := v[:at], v[at+1:]
|
||||
if local[0] == '.' || local[len(local)-1] == '.' || len(local) > 64 {
|
||||
return "", invalid("%q: invalid local part", v)
|
||||
}
|
||||
for i := 0; i < len(local); i++ {
|
||||
c := local[i]
|
||||
lower := c >= 'a' && c <= 'z'
|
||||
digit := c >= '0' && c <= '9'
|
||||
// '=' for the rewritten senders of forwarders (SRS0=…).
|
||||
if !lower && !digit && !strings.ContainsRune("._+-=", rune(c)) {
|
||||
return "", invalid("%q: local part contains invalid characters", v)
|
||||
}
|
||||
}
|
||||
if err := checkDomain(domain); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return v, nil
|
||||
case store.SpamEntryDomain:
|
||||
// "@example.com" is how a domain is often written in such a list.
|
||||
v = strings.TrimPrefix(v, "@")
|
||||
if err := checkDomain(v); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return v, nil
|
||||
case store.SpamEntryIP:
|
||||
if v == "" {
|
||||
return "", invalid("address or network is required")
|
||||
}
|
||||
if strings.Contains(v, "/") {
|
||||
p, err := netip.ParsePrefix(v)
|
||||
if err != nil {
|
||||
return "", invalid("%q is not a network in CIDR form (e.g. 192.0.2.0/24)", v)
|
||||
}
|
||||
if p.Addr().Is4In6() {
|
||||
return "", invalid("%q: write an IPv4 network as IPv4 (e.g. 192.0.2.0/24)", v)
|
||||
}
|
||||
if p.Bits() == 0 {
|
||||
return "", invalid("%q is every address there is", v)
|
||||
}
|
||||
// A network of one address is that address: 192.0.2.7/32 and
|
||||
// 192.0.2.7 are one entry, not two that look different.
|
||||
if p.Bits() == p.Addr().BitLen() {
|
||||
return p.Addr().String(), nil
|
||||
}
|
||||
return p.Masked().String(), nil
|
||||
}
|
||||
a, err := netip.ParseAddr(v)
|
||||
if err != nil || a.Zone() != "" {
|
||||
return "", invalid("%q is not an IP address or a network in CIDR form", v)
|
||||
}
|
||||
return a.Unmap().String(), nil
|
||||
default:
|
||||
return "", invalid("choose what the entry is: a sender address, a sender domain or a client address")
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeNote trims the operator's comment and refuses what a single line of
|
||||
// text cannot be.
|
||||
func normalizeNote(raw string) (string, error) {
|
||||
note := strings.TrimSpace(raw)
|
||||
if utf8.RuneCountInString(note) > noteMaxRunes {
|
||||
return "", invalid("note must be at most %d characters", noteMaxRunes)
|
||||
}
|
||||
for _, r := range note {
|
||||
if r < 0x20 || r == 0x7f || r == utf8.RuneError {
|
||||
return "", invalid("note must be one line of plain text")
|
||||
}
|
||||
}
|
||||
return note, nil
|
||||
}
|
||||
|
||||
// What a list leaves in the journal when it decides a message - the two ends
|
||||
// of deploy/antispam/rspamd/multimap.conf this package depends on. A deny rule
|
||||
// refuses with a message, which the log tailer files as the reply of a
|
||||
// rejected row. An allow rule adds a symbol named after the kind of entry,
|
||||
// which the journal milter keeps in the verdict of an accepted row.
|
||||
const (
|
||||
denyReplyMark = "this server's deny list"
|
||||
allowMark = "SELFPOST_ALLOW_"
|
||||
)
|
||||
|
||||
var allowSymbol = map[string]string{
|
||||
store.SpamEntryIP: "SELFPOST_ALLOW_IP(",
|
||||
store.SpamEntryAddress: "SELFPOST_ALLOW_FROM(",
|
||||
store.SpamEntryDomain: "SELFPOST_ALLOW_DOMAIN(",
|
||||
}
|
||||
|
||||
// Usage is how often an entry decided a message, as far back as the journal
|
||||
// reaches, and when it last did. The zero value is an entry that never did.
|
||||
type Usage struct {
|
||||
Hits int
|
||||
Last time.Time
|
||||
}
|
||||
|
||||
// Usage counts, for every entry, the journal rows it decided. Nothing is
|
||||
// stored for this: the journal already says which list decided a message and
|
||||
// who the message was from, and an entry is matched against that the way the
|
||||
// filter matches it - an address whole, a domain as the sender's domain, an
|
||||
// address or network against the client's. So the count reaches back as far
|
||||
// as the journal does (its retention, or less when it is full), and a row
|
||||
// counts for every entry of the deciding list that fits it: the journal does
|
||||
// not say which of two fitting entries the filter met first.
|
||||
//
|
||||
// A counter kept with each entry would not depend on the journal; the plan
|
||||
// (docs/plans/inbound-antispam-panel.md, Filter list usage) says when that is
|
||||
// worth a migration.
|
||||
func (l *Lists) Usage() (map[int64]Usage, error) {
|
||||
entries, err := l.store.ListSpamListEntries()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[int64]Usage, len(entries))
|
||||
if len(entries) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
hits, err := l.store.ListSpamListHits(denyReplyMark, allowMark)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type matcher struct {
|
||||
id int64
|
||||
list string
|
||||
kind string
|
||||
value string
|
||||
prefix netip.Prefix
|
||||
}
|
||||
matchers := make([]matcher, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
m := matcher{id: e.ID, list: e.List, kind: e.Kind, value: e.Value}
|
||||
if e.Kind == store.SpamEntryIP {
|
||||
if p, err := netip.ParsePrefix(e.Value); err == nil {
|
||||
m.prefix = p
|
||||
} else if a, err := netip.ParseAddr(e.Value); err == nil {
|
||||
m.prefix = netip.PrefixFrom(a, a.BitLen())
|
||||
} else {
|
||||
continue // not an entry this package wrote
|
||||
}
|
||||
}
|
||||
matchers = append(matchers, m)
|
||||
}
|
||||
for _, h := range hits {
|
||||
from := strings.ToLower(h.From)
|
||||
domain := ""
|
||||
if i := strings.LastIndexByte(from, '@'); i >= 0 {
|
||||
domain = from[i+1:]
|
||||
}
|
||||
client, clientErr := netip.ParseAddr(h.ClientIP)
|
||||
client = client.Unmap()
|
||||
denied := h.Decision != store.SpamDecisionAccept && strings.Contains(h.Detail, denyReplyMark)
|
||||
for _, m := range matchers {
|
||||
switch m.list {
|
||||
case store.SpamListDeny:
|
||||
if !denied {
|
||||
continue
|
||||
}
|
||||
case store.SpamListAllow:
|
||||
if h.Decision != store.SpamDecisionAccept || !strings.Contains(h.Detail, allowSymbol[m.kind]) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
fits := false
|
||||
switch m.kind {
|
||||
case store.SpamEntryAddress:
|
||||
fits = from == m.value
|
||||
case store.SpamEntryDomain:
|
||||
fits = domain == m.value
|
||||
case store.SpamEntryIP:
|
||||
fits = clientErr == nil && m.prefix.Contains(client)
|
||||
}
|
||||
if !fits {
|
||||
continue
|
||||
}
|
||||
u := out[m.id]
|
||||
u.Hits++
|
||||
if h.CreatedAt.After(u.Last) {
|
||||
u.Last = h.CreatedAt
|
||||
}
|
||||
out[m.id] = u
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// FilterKeyFile is the file, beside the list files, that holds the key the
|
||||
// spam filter signs messages with.
|
||||
const FilterKeyFile = "filter.key"
|
||||
|
||||
// FilterKey returns the key the spam filter signs each message with, creating
|
||||
// it on first use: 32 random bytes as 64 hex characters on one line, in the
|
||||
// directory the filter's container mounts.
|
||||
//
|
||||
// The filter is attached so that mail flows when it is down. Then nothing
|
||||
// removes verdict headers a sender wrote himself, and the inbound journal
|
||||
// would record his verdict as the filter's. So the filter signs each message
|
||||
// with this key and the journal reads a verdict only next to a signature it
|
||||
// can recompute (internal/milter, FilterSignature).
|
||||
//
|
||||
// The file is readable by anyone on the host, like the list files and for the
|
||||
// same reason: the filter is another user in another container. What the key
|
||||
// protects is the truth of a monitoring page, against senders on the internet;
|
||||
// someone with a shell on the host is not who it is kept from.
|
||||
func FilterKey(dir string) (string, error) {
|
||||
path := filepath.Join(dir, FilterKeyFile)
|
||||
if raw, err := os.ReadFile(path); err == nil {
|
||||
if key := strings.TrimSpace(string(raw)); len(key) >= 32 && !strings.ContainsAny(key, " \t\r\n") {
|
||||
return key, nil
|
||||
}
|
||||
// Empty or cut short: not a key anyone should sign with. Replaced below.
|
||||
} else if !os.IsNotExist(err) {
|
||||
return "", fmt.Errorf("read %s: %w", path, err)
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return "", fmt.Errorf("create %s: %w", dir, err)
|
||||
}
|
||||
buf := make([]byte, 32)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
return "", fmt.Errorf("generate the filter key: %w", err)
|
||||
}
|
||||
key := hex.EncodeToString(buf)
|
||||
if err := atomicfile.Write(path, []byte(key+"\n"), 0o644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
@@ -0,0 +1,440 @@
|
||||
package inbound
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
|
||||
func testLists(t *testing.T) (*Lists, *store.Store, string) {
|
||||
t.Helper()
|
||||
st, err := store.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { st.Close() })
|
||||
dir := filepath.Join(t.TempDir(), "antispam")
|
||||
return NewLists(st, dir), st, dir
|
||||
}
|
||||
|
||||
// mapLines reads a map file without its comment line.
|
||||
func mapLines(t *testing.T, dir, name string) []string {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", name, err)
|
||||
}
|
||||
var out []string
|
||||
for _, line := range strings.Split(strings.TrimRight(string(raw), "\n"), "\n") {
|
||||
if !strings.HasPrefix(line, "#") {
|
||||
out = append(out, line)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestNormalizeListValue(t *testing.T) {
|
||||
for _, c := range []struct{ kind, raw, want string }{
|
||||
{store.SpamEntryAddress, " Billing@Vendor.Example ", "billing@vendor.example"},
|
||||
{store.SpamEntryAddress, "srs0=ab12=cd=example.org=user@forwarder.example", "srs0=ab12=cd=example.org=user@forwarder.example"},
|
||||
{store.SpamEntryDomain, "@Evil.Example", "evil.example"},
|
||||
{store.SpamEntryDomain, "sub.evil.example", "sub.evil.example"},
|
||||
{store.SpamEntryIP, "192.0.2.7", "192.0.2.7"},
|
||||
{store.SpamEntryIP, "192.0.2.77/24", "192.0.2.0/24"},
|
||||
{store.SpamEntryIP, "192.0.2.7/32", "192.0.2.7"},
|
||||
{store.SpamEntryIP, "2001:db8::1/128", "2001:db8::1"},
|
||||
{store.SpamEntryIP, "::ffff:192.0.2.7", "192.0.2.7"},
|
||||
{store.SpamEntryIP, "2001:DB8:0:0::1", "2001:db8::1"},
|
||||
{store.SpamEntryIP, "2001:db8:1::5/48", "2001:db8:1::/48"},
|
||||
} {
|
||||
got, err := normalizeListValue(c.kind, c.raw)
|
||||
if err != nil || got != c.want {
|
||||
t.Errorf("normalizeListValue(%s, %q) = %q, %v; want %q", c.kind, c.raw, got, err, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
for _, c := range []struct{ kind, raw string }{
|
||||
{store.SpamEntryAddress, ""},
|
||||
{store.SpamEntryAddress, "no-at-sign"},
|
||||
{store.SpamEntryAddress, "@example.com"},
|
||||
{store.SpamEntryAddress, "a@localhost"},
|
||||
{store.SpamEntryAddress, "a b@example.com"},
|
||||
{store.SpamEntryAddress, "a#b@example.com"},
|
||||
{store.SpamEntryAddress, "a\nb@example.com"},
|
||||
{store.SpamEntryAddress, ".a@example.com"},
|
||||
{store.SpamEntryDomain, ""},
|
||||
{store.SpamEntryDomain, "example"},
|
||||
{store.SpamEntryDomain, "exa mple.com"},
|
||||
{store.SpamEntryDomain, "example.com # all of it"},
|
||||
{store.SpamEntryDomain, "user@example.com"},
|
||||
{store.SpamEntryIP, ""},
|
||||
{store.SpamEntryIP, "example.com"},
|
||||
{store.SpamEntryIP, "192.0.2.300"},
|
||||
{store.SpamEntryIP, "192.0.2.0/33"},
|
||||
{store.SpamEntryIP, "0.0.0.0/0"},
|
||||
{store.SpamEntryIP, "::/0"},
|
||||
{store.SpamEntryIP, "fe80::1%eth0"},
|
||||
{store.SpamEntryIP, "::ffff:192.0.2.0/120"},
|
||||
{store.SpamEntryIP, "192.0.2.7 192.0.2.8"},
|
||||
{"subject", "pills"},
|
||||
{"", "x"},
|
||||
} {
|
||||
got, err := normalizeListValue(c.kind, c.raw)
|
||||
var ve *ValidationError
|
||||
if err == nil || !errors.As(err, &ve) {
|
||||
t.Errorf("normalizeListValue(%q, %q) = %q, %v; want a ValidationError", c.kind, c.raw, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListsAddWritesTheMaps(t *testing.T) {
|
||||
l, _, dir := testLists(t)
|
||||
for _, e := range []struct{ list, kind, value string }{
|
||||
{store.SpamListAllow, store.SpamEntryAddress, "Billing@Vendor.Example"},
|
||||
{store.SpamListAllow, store.SpamEntryAddress, "accounts@vendor.example"},
|
||||
{store.SpamListDeny, store.SpamEntryDomain, "@evil.example"},
|
||||
{store.SpamListDeny, store.SpamEntryIP, "198.51.100.9/24"},
|
||||
// The same value on both lists is two entries: deny wins in the filter.
|
||||
{store.SpamListDeny, store.SpamEntryAddress, "billing@vendor.example"},
|
||||
} {
|
||||
if _, err := l.Add(e.list, e.kind, e.value, " why "); err != nil {
|
||||
t.Fatalf("Add(%s, %s, %q): %v", e.list, e.kind, e.value, err)
|
||||
}
|
||||
}
|
||||
|
||||
want := map[string]string{
|
||||
"allow_from.map": "accounts@vendor.example billing@vendor.example",
|
||||
"allow_domain.map": "",
|
||||
"allow_ip.map": "",
|
||||
"deny_from.map": "billing@vendor.example",
|
||||
"deny_domain.map": "evil.example",
|
||||
"deny_ip.map": "198.51.100.0/24",
|
||||
}
|
||||
for name, lines := range want {
|
||||
if got := strings.Join(mapLines(t, dir, name), " "); got != lines {
|
||||
t.Errorf("%s = %q, want %q", name, got, lines)
|
||||
}
|
||||
// The filter is another user in another container.
|
||||
if info, err := os.Stat(filepath.Join(dir, name)); err != nil {
|
||||
t.Errorf("stat %s: %v", name, err)
|
||||
} else if info.Mode().Perm()&0o004 == 0 && os.PathSeparator == '/' {
|
||||
t.Errorf("%s is mode %o, not readable by the filter", name, info.Mode().Perm())
|
||||
}
|
||||
}
|
||||
if files, _ := os.ReadDir(dir); len(files) != len(want) {
|
||||
t.Errorf("the directory holds %d files, want %d (no temp file left behind)", len(files), len(want))
|
||||
}
|
||||
|
||||
entries, err := l.Entries()
|
||||
if err != nil || len(entries) != 5 {
|
||||
t.Fatalf("Entries = %d, %v", len(entries), err)
|
||||
}
|
||||
if entries[0].Note != "why" {
|
||||
t.Errorf("note = %q, want it trimmed", entries[0].Note)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListsRefusals(t *testing.T) {
|
||||
l, _, dir := testLists(t)
|
||||
if _, err := l.Add(store.SpamListAllow, store.SpamEntryIP, "192.0.2.7", ""); err != nil {
|
||||
t.Fatalf("Add: %v", err)
|
||||
}
|
||||
// Written differently, the same entry.
|
||||
if _, err := l.Add(store.SpamListAllow, store.SpamEntryIP, "::ffff:192.0.2.7", ""); !errors.Is(err, store.ErrSpamListEntryExists) {
|
||||
t.Errorf("the same address again: %v, want ErrSpamListEntryExists", err)
|
||||
}
|
||||
var ve *ValidationError
|
||||
for name, add := range map[string]func() error{
|
||||
"an unknown list": func() error { _, err := l.Add("block", store.SpamEntryIP, "192.0.2.8", ""); return err },
|
||||
"a bad value": func() error { _, err := l.Add(store.SpamListDeny, store.SpamEntryIP, "nope", ""); return err },
|
||||
"a long note": func() error {
|
||||
_, err := l.Add(store.SpamListDeny, store.SpamEntryIP, "192.0.2.8", strings.Repeat("x", noteMaxRunes+1))
|
||||
return err
|
||||
},
|
||||
"a note of two lines": func() error {
|
||||
_, err := l.Add(store.SpamListDeny, store.SpamEntryIP, "192.0.2.8", "one\ntwo")
|
||||
return err
|
||||
},
|
||||
} {
|
||||
if err := add(); !errors.As(err, &ve) {
|
||||
t.Errorf("%s: %v, want a ValidationError", name, err)
|
||||
}
|
||||
}
|
||||
if got := strings.Join(mapLines(t, dir, "deny_ip.map"), " "); got != "" {
|
||||
t.Errorf("deny_ip.map = %q after refusals only", got)
|
||||
}
|
||||
if entries, _ := l.Entries(); len(entries) != 1 {
|
||||
t.Errorf("entries = %d, want the first one alone", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestListsDelete(t *testing.T) {
|
||||
l, _, dir := testLists(t)
|
||||
e, err := l.Add(store.SpamListDeny, store.SpamEntryDomain, "evil.example", "")
|
||||
if err != nil {
|
||||
t.Fatalf("Add: %v", err)
|
||||
}
|
||||
if err := l.Delete(e.ID); err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
if got := mapLines(t, dir, "deny_domain.map"); len(got) != 0 {
|
||||
t.Errorf("deny_domain.map = %v after the delete", got)
|
||||
}
|
||||
if err := l.Delete(e.ID); !errors.Is(err, store.ErrSpamListEntryNotFound) {
|
||||
t.Errorf("a second delete: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An entry the filter never got must not stay on the page.
|
||||
func TestListsAddRollsBackWhenTheMapsCannotBeWritten(t *testing.T) {
|
||||
st, err := store.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer st.Close()
|
||||
// A file where the directory should be.
|
||||
blocked := filepath.Join(t.TempDir(), "antispam")
|
||||
if err := os.WriteFile(blocked, nil, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
l := NewLists(st, blocked)
|
||||
if _, err := l.Add(store.SpamListDeny, store.SpamEntryDomain, "evil.example", ""); err == nil {
|
||||
t.Fatal("Add succeeded with nowhere to write the maps")
|
||||
}
|
||||
if entries, _ := l.Entries(); len(entries) != 0 {
|
||||
t.Errorf("the entry stayed: %+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
// The writer's own guard: a row that got into the table some other way is not
|
||||
// written into a map.
|
||||
func TestListsSyncRefusesAnUnsafeRow(t *testing.T) {
|
||||
l, st, dir := testLists(t)
|
||||
if _, err := st.AddSpamListEntry(store.SpamListDeny, store.SpamEntryDomain, "evil.example\ngood.example", ""); err != nil {
|
||||
t.Fatalf("AddSpamListEntry: %v", err)
|
||||
}
|
||||
if err := l.Sync(); err == nil {
|
||||
t.Fatal("Sync wrote a value with a newline in it")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "deny_domain.map")); err == nil {
|
||||
t.Error("a map file was written")
|
||||
}
|
||||
}
|
||||
|
||||
// With nothing on the lists all six files are there, empty.
|
||||
func TestListsSyncWritesEmptyMaps(t *testing.T) {
|
||||
l, _, dir := testLists(t)
|
||||
if err := l.Sync(); err != nil {
|
||||
t.Fatalf("Sync: %v", err)
|
||||
}
|
||||
for _, name := range mapFiles {
|
||||
if got := mapLines(t, dir, name); len(got) != 0 {
|
||||
t.Errorf("%s = %v", name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListsLastWritten(t *testing.T) {
|
||||
l, _, dir := testLists(t)
|
||||
if got := l.LastWritten(); !got.IsZero() {
|
||||
t.Fatalf("LastWritten before anything was written = %v", got)
|
||||
}
|
||||
if err := l.Sync(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := l.LastWritten(); got.IsZero() || time.Since(got) > time.Minute {
|
||||
t.Errorf("LastWritten after a sync = %v", got)
|
||||
}
|
||||
|
||||
// The newest of the six counts, whichever it is.
|
||||
old := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
for _, name := range mapFiles {
|
||||
if err := os.Chtimes(filepath.Join(dir, name), old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
newest := old.Add(48 * time.Hour)
|
||||
if err := os.Chtimes(filepath.Join(dir, "deny_domain.map"), newest, newest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := l.LastWritten(); !got.Equal(newest) {
|
||||
t.Errorf("LastWritten = %v, want %v", got, newest)
|
||||
}
|
||||
|
||||
// A file that is not one of the six does not count.
|
||||
other := newest.Add(time.Hour)
|
||||
if err := os.WriteFile(filepath.Join(dir, "notes.txt"), nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chtimes(filepath.Join(dir, "notes.txt"), other, other); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := l.LastWritten(); !got.Equal(newest) {
|
||||
t.Errorf("LastWritten counts a stray file: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// How often an entry decided a message is read off the journal: the list that
|
||||
// decided, and whether the entry fits who the message was from.
|
||||
func TestListsUsage(t *testing.T) {
|
||||
l, st, _ := testLists(t)
|
||||
add := func(list, kind, value string) int64 {
|
||||
t.Helper()
|
||||
e, err := l.Add(list, kind, value, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Add(%s, %s, %s): %v", list, kind, value, err)
|
||||
}
|
||||
return e.ID
|
||||
}
|
||||
denyDomain := add(store.SpamListDeny, store.SpamEntryDomain, "evil.example")
|
||||
denyAddr := add(store.SpamListDeny, store.SpamEntryAddress, "spammer@evil.example")
|
||||
denyNet := add(store.SpamListDeny, store.SpamEntryIP, "198.51.100.0/24")
|
||||
denyV6 := add(store.SpamListDeny, store.SpamEntryIP, "2001:db8::/32")
|
||||
allowAddr := add(store.SpamListAllow, store.SpamEntryAddress, "billing@vendor.example")
|
||||
allowDomain := add(store.SpamListAllow, store.SpamEntryDomain, "vendor.example")
|
||||
allowIP := add(store.SpamListAllow, store.SpamEntryIP, "203.0.113.5")
|
||||
unused := add(store.SpamListAllow, store.SpamEntryDomain, "quiet.example")
|
||||
|
||||
const denied = "5.7.1 Refused by this server's deny list"
|
||||
at := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
|
||||
hour := 0
|
||||
row := func(from, ip, decision, engine, detail string) {
|
||||
t.Helper()
|
||||
hour++
|
||||
err := st.InsertInboundSpam(store.InboundSpamEntry{
|
||||
InboundDomain: "in.example.com", To: "a@in.example.com", From: from, ClientIP: ip,
|
||||
Decision: decision, Engine: engine, Detail: detail, At: at.Add(time.Duration(hour) * time.Hour),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// hour 1: the address and its domain are both on the deny list, and both fit.
|
||||
row("Spammer@Evil.Example", "192.0.2.9", store.SpamDecisionReject, store.SpamEngineRspamd, denied)
|
||||
// hour 2: another address of the domain.
|
||||
row("other@evil.example", "192.0.2.9", store.SpamDecisionReject, store.SpamEngineRspamd, denied)
|
||||
// hour 3, 4: the networks, v4 (as Postfix may log it, mapped) and v6.
|
||||
row("x@elsewhere.example", "::ffff:198.51.100.77", store.SpamDecisionReject, store.SpamEngineRspamd, denied)
|
||||
row("x@elsewhere.example", "2001:db8::1", store.SpamDecisionReject, store.SpamEngineRspamd, denied)
|
||||
// hour 5: a subdomain is not the domain.
|
||||
row("x@sub.evil.example", "192.0.2.9", store.SpamDecisionReject, store.SpamEngineRspamd, denied)
|
||||
// hour 6: allowed by its address and its domain - two symbols, two entries.
|
||||
row("billing@vendor.example", "192.0.2.50", store.SpamDecisionAccept, store.SpamEngineRspamd,
|
||||
"no action, score -199.00 / 15.00; SELFPOST_ALLOW_DOMAIN(-100.00) SELFPOST_ALLOW_FROM(-100.00)")
|
||||
// hour 7: allowed by its domain alone.
|
||||
row("accounts@vendor.example", "192.0.2.50", store.SpamDecisionAccept, store.SpamEngineRspamd,
|
||||
"no action, score -99.00 / 15.00; SELFPOST_ALLOW_DOMAIN(-100.00)")
|
||||
// hour 8: allowed by the client address; the sender's domain is not why.
|
||||
row("x@quiet.example", "203.0.113.5", store.SpamDecisionAccept, store.SpamEngineRspamd,
|
||||
"no action, score -99.00 / 15.00; SELFPOST_ALLOW_IP(-100.00)")
|
||||
// Nobody's: the filter's own refusal, an accept with no list symbol, a
|
||||
// refusal by Postfix, and a sender who spells the mark in his own address.
|
||||
row("spammer@evil.example", "192.0.2.9", store.SpamDecisionReject, store.SpamEngineRspamd, "5.7.1 Spam message rejected")
|
||||
row("billing@vendor.example", "192.0.2.50", store.SpamDecisionAccept, store.SpamEngineRspamd, "no action, score 0.10 / 15.00")
|
||||
row("spammer@evil.example", "198.51.100.1", store.SpamDecisionReject, store.SpamEnginePostfix, "550 5.1.1 <nobody@in.example.com>: this server's deny list")
|
||||
row("billing@vendor.example", "203.0.113.5", store.SpamDecisionAccept, store.SpamEngineSelfPost, "")
|
||||
|
||||
usage, err := l.Usage()
|
||||
if err != nil {
|
||||
t.Fatalf("Usage: %v", err)
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
id int64
|
||||
hits int
|
||||
last int // hour
|
||||
}{
|
||||
"deny domain": {denyDomain, 2, 2}, "deny address": {denyAddr, 1, 1},
|
||||
"deny network": {denyNet, 1, 3}, "deny v6 network": {denyV6, 1, 4},
|
||||
"allow address": {allowAddr, 1, 6}, "allow domain": {allowDomain, 2, 7}, "allow ip": {allowIP, 1, 8},
|
||||
"unused": {unused, 0, 0},
|
||||
} {
|
||||
got := usage[c.id]
|
||||
wantLast := time.Time{}
|
||||
if c.hits > 0 {
|
||||
wantLast = at.Add(time.Duration(c.last) * time.Hour)
|
||||
}
|
||||
if got.Hits != c.hits || !got.Last.Equal(wantLast) {
|
||||
t.Errorf("%s: %d hits, last %v; want %d, %v", name, got.Hits, got.Last, c.hits, wantLast)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListsUsageWithNoEntries(t *testing.T) {
|
||||
l, _, _ := testLists(t)
|
||||
if usage, err := l.Usage(); err != nil || len(usage) != 0 {
|
||||
t.Errorf("Usage = %v, %v", usage, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The same address written as a network of one is the same entry.
|
||||
func TestListsOneAddressNetworkIsTheAddress(t *testing.T) {
|
||||
l, _, dir := testLists(t)
|
||||
if _, err := l.Add(store.SpamListDeny, store.SpamEntryIP, "192.0.2.7", ""); err != nil {
|
||||
t.Fatalf("Add: %v", err)
|
||||
}
|
||||
if _, err := l.Add(store.SpamListDeny, store.SpamEntryIP, "192.0.2.7/32", ""); !errors.Is(err, store.ErrSpamListEntryExists) {
|
||||
t.Errorf("192.0.2.7/32 after 192.0.2.7: %v, want ErrSpamListEntryExists", err)
|
||||
}
|
||||
if got := strings.Join(mapLines(t, dir, "deny_ip.map"), " "); got != "192.0.2.7" {
|
||||
t.Errorf("deny_ip.map = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListable(t *testing.T) {
|
||||
for value, want := range map[string]bool{
|
||||
"billing@vendor.example": true, "srs0=ab=cd=example.org=user@forwarder.example": true,
|
||||
"o'brien@example.org": false, "a#b@example.org": false, "": false, "not-an-address": false,
|
||||
} {
|
||||
if got := Listable(store.SpamEntryAddress, value); got != want {
|
||||
t.Errorf("Listable(address, %q) = %v, want %v", value, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The key the filter signs with is made once and then read, and is readable by
|
||||
// the filter, which is another user in another container.
|
||||
func TestFilterKey(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "antispam")
|
||||
key, err := FilterKey(dir)
|
||||
if err != nil || len(key) != 64 || strings.Trim(key, "0123456789abcdef") != "" {
|
||||
t.Fatalf("FilterKey = %q, %v", key, err)
|
||||
}
|
||||
again, err := FilterKey(dir)
|
||||
if err != nil || again != key {
|
||||
t.Errorf("a second call = %q, %v; want the same key", again, err)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, FilterKeyFile))
|
||||
if err != nil || string(raw) != key+"\n" {
|
||||
t.Errorf("file = %q, %v", raw, err)
|
||||
}
|
||||
if info, _ := os.Stat(filepath.Join(dir, FilterKeyFile)); os.PathSeparator == '/' && info.Mode().Perm() != 0o644 {
|
||||
t.Errorf("mode = %o", info.Mode().Perm())
|
||||
}
|
||||
|
||||
// A file that is not a key is replaced, not signed with.
|
||||
if err := os.WriteFile(filepath.Join(dir, FilterKeyFile), []byte("short\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fresh, err := FilterKey(dir)
|
||||
if err != nil || len(fresh) != 64 || fresh == key {
|
||||
t.Errorf("after a damaged file: %q, %v", fresh, err)
|
||||
}
|
||||
|
||||
// The lists leave it alone.
|
||||
st, err := store.Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer st.Close()
|
||||
if err := NewLists(st, dir).Sync(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kept, _ := FilterKey(dir); kept != fresh {
|
||||
t.Errorf("the key changed when the lists were written")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
package logtail
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
|
||||
// InboundStore is what the tailer needs of the inbound antispam journal:
|
||||
// writing the rows for refused mail, knowing which domains are inbound, and
|
||||
// pruning. *store.Store satisfies it.
|
||||
type InboundStore interface {
|
||||
InsertInboundSpam(e store.InboundSpamEntry) error
|
||||
InsertInboundSpamOnce(e store.InboundSpamEntry) (bool, error)
|
||||
IsInboundDomain(name string) (bool, error)
|
||||
PruneInboundSpam(cutoff time.Time, keep int) (int64, error)
|
||||
}
|
||||
|
||||
// Inbound turns on the inbound half of the tailer: journal rows for mail that
|
||||
// port 25 refused, and the journal's retention. Run takes nil when the inbound
|
||||
// relay is off.
|
||||
type Inbound struct {
|
||||
Store InboundStore
|
||||
Retention RetentionDays
|
||||
}
|
||||
|
||||
// The inbound journal milter writes a row for mail that was accepted; mail
|
||||
// that was refused never reaches it, and what is known about it is one line
|
||||
// of mail.log:
|
||||
//
|
||||
// postfix/inbound/smtpd[41]: NOQUEUE: reject: RCPT from mx.example.net[203.0.113.9]: 550 5.1.1 <nobody@in.example.com>: Recipient address rejected: User unknown in relay recipient table; from=<a@example.net> to=<nobody@in.example.com> proto=ESMTP helo=<mx.example.net>
|
||||
// postfix/cleanup[52]: 4A1B2C3D: milter-reject: END-OF-MESSAGE from mx.example.net[203.0.113.9]: 5.7.1 Spam message rejected; from=<a@example.net> to=<b@in.example.com> proto=ESMTP helo=<mx.example.net>
|
||||
//
|
||||
// A message the filter holds (milter-hold) is not read here: it goes on to the
|
||||
// inbound journal milter, which writes its row with the subject.
|
||||
//
|
||||
// The first is Postfix's own policy, written by the port-25 smtpd, which
|
||||
// build/postfix-config.sh names postfix/inbound so that its lines are told
|
||||
// from those of the submission ports. The second is the filter's verdict at
|
||||
// end-of-message, which cleanup(8) writes under one name for every listener -
|
||||
// so it counts only for a queue id the port-25 smtpd was seen to open
|
||||
// (inboundClientRe).
|
||||
//
|
||||
// Both are matched from the head of the line, after the timestamp and host
|
||||
// name: the tail of such a line is the sender's own words (HELO name,
|
||||
// addresses), and a pattern that could start anywhere would let him write
|
||||
// journal rows.
|
||||
//
|
||||
// For the same reason the reply is the shortest text before "; from=<", and
|
||||
// the HELO name - the last thing on the line, which Postfix logs as the client
|
||||
// wrote it, spaces and brackets included - is whatever is left. Taking the
|
||||
// longest reply instead would let a HELO name that itself spells
|
||||
// "; from=<…> to=<…> proto=…" supply the sender, the recipient and the reply
|
||||
// of the row (security review, 2026-10-12).
|
||||
var (
|
||||
inboundRejectRe = regexp.MustCompile(`^\S+ postfix/(inbound/smtpd|cleanup)\[\d+\]: ([0-9A-Za-z]+): (reject|milter-reject|milter-discard): \S+ from [^\[\s]*\[([0-9A-Fa-f:.]+)\]: (.*?); from=<([^>]*)> to=<([^>]*)> proto=\S+(?: helo=<.*>)?$`)
|
||||
inboundClientRe = regexp.MustCompile(`^\S+ postfix/inbound/smtpd\[\d+\]: ([0-9A-Za-z]+): client=`)
|
||||
// A queue id stops being port 25's when the message is queued (qmgr takes
|
||||
// it: the filter let it through) or when another listener opens a message
|
||||
// under the same id - Postfix reuses ids.
|
||||
inboundForgetRe = regexp.MustCompile(`^\S+ postfix/(?:qmgr\[\d+\]: ([0-9A-Za-z]+): from=|(?:[a-z0-9-]+/)?smtpd\[\d+\]: ([0-9A-Za-z]+): client=)`)
|
||||
)
|
||||
|
||||
// A row holds one line of reason; Postfix's reply text carries the far end's
|
||||
// bytes and is cut and reduced to printable ASCII like any of them.
|
||||
const inboundDetailMax = 255
|
||||
|
||||
// inboundQueueIDs is how many port-25 queue ids are remembered while waiting
|
||||
// for cleanup's verdict line on them. An id is forgotten when the verdict
|
||||
// comes, when the message is queued, or when another listener reuses it; the
|
||||
// cap bounds what is left - messages the client abandoned - and a flood.
|
||||
const inboundQueueIDs = 4096
|
||||
|
||||
// inboundPruneEvery is how many rows the tailer writes before it has the
|
||||
// journal cut back to its row cap, without waiting for the six-hourly prune.
|
||||
// Refusals are written at the rate strangers send RCPT TO, which no limit of
|
||||
// Postfix's counts, so between two prunes the journal would otherwise grow as
|
||||
// fast as someone cared to make it.
|
||||
const inboundPruneEvery = 500
|
||||
|
||||
// inboundJournal is the per-run state of the inbound half.
|
||||
type inboundJournal struct {
|
||||
st InboundStore
|
||||
// port25 holds the queue ids the port-25 smtpd opened and nobody has
|
||||
// accounted for yet, each with the number of its arrival; order is those
|
||||
// arrivals, for dropping the oldest. An id that was forgotten and opened
|
||||
// again is in order twice: the number tells the stale entry from the live
|
||||
// one when the stale one comes up for dropping.
|
||||
port25 map[string]uint64
|
||||
order []queued
|
||||
seq uint64
|
||||
// written counts the rows since the journal was last cut to its cap.
|
||||
written int
|
||||
}
|
||||
|
||||
type queued struct {
|
||||
id string
|
||||
seq uint64
|
||||
}
|
||||
|
||||
func newInboundJournal(st InboundStore) *inboundJournal {
|
||||
return &inboundJournal{st: st, port25: make(map[string]uint64)}
|
||||
}
|
||||
|
||||
// handle looks at one mail.log line and writes a journal row when it is a
|
||||
// refusal of mail for an inbound domain. Errors are logged; the tailer goes on.
|
||||
//
|
||||
// catchingUp says the line is one of those the tailer reads when it starts,
|
||||
// from where it last saved its place to the end of the file. It saves its
|
||||
// place every few seconds, so after an unclean stop some of those lines were
|
||||
// read before; a row is then written only if it is not there already. Lines
|
||||
// read as they are written are always new, and two identical refusals in one
|
||||
// second stay two rows.
|
||||
func (j *inboundJournal) handle(line string, now time.Time, catchingUp bool) {
|
||||
at, rest := lineTime(line, now)
|
||||
if m := inboundClientRe.FindStringSubmatch(rest); m != nil {
|
||||
j.remember(m[1])
|
||||
return
|
||||
}
|
||||
if m := inboundForgetRe.FindStringSubmatch(rest); m != nil {
|
||||
delete(j.port25, m[1]+m[2])
|
||||
return
|
||||
}
|
||||
m := inboundRejectRe.FindStringSubmatch(rest)
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
program, queueID, kind, clientIP, reason, from, to := m[1], m[2], m[3], m[4], m[5], m[6], m[7]
|
||||
// The sender's address is his own text too, and with a quoted local part it
|
||||
// may hold "> to=<…> proto=…" - which would end the sender early and supply
|
||||
// the recipient. Such a line shows it: the sender is left with an unclosed
|
||||
// quote. An address with a quote, a bracket or a space in it is not one the
|
||||
// journal keeps.
|
||||
if strings.ContainsAny(from, "\"<> ") || strings.ContainsAny(to, "\"<> ") {
|
||||
return
|
||||
}
|
||||
if program == "cleanup" {
|
||||
if _, ok := j.port25[queueID]; !ok {
|
||||
return // a submission-port message, or one opened before this run
|
||||
}
|
||||
delete(j.port25, queueID)
|
||||
}
|
||||
|
||||
// Only mail for an inbound domain is the relay's. "Relay access denied"
|
||||
// for a stranger's domain is every scanner on the internet, and a line
|
||||
// with no recipient (size limit at MAIL) belongs to no domain.
|
||||
domain := ""
|
||||
if i := strings.LastIndexByte(to, '@'); i >= 0 {
|
||||
domain = strings.ToLower(to[i+1:])
|
||||
}
|
||||
if domain == "" {
|
||||
return
|
||||
}
|
||||
ok, err := j.st.IsInboundDomain(domain)
|
||||
if err != nil {
|
||||
log.Printf("log-tailer: inbound journal: look up %s: %v", domain, err)
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
e := store.InboundSpamEntry{
|
||||
InboundDomain: domain,
|
||||
ClientIP: clientIP,
|
||||
From: from,
|
||||
To: to,
|
||||
Decision: store.SpamDecisionReject,
|
||||
Engine: store.SpamEngineRspamd,
|
||||
Detail: printable(reason, inboundDetailMax),
|
||||
At: at,
|
||||
}
|
||||
if kind == "reject" {
|
||||
e.Engine = store.SpamEnginePostfix
|
||||
}
|
||||
// "450 4.7.1 …" from Postfix, "4.7.1 …" from a milter: either way a reply
|
||||
// that starts with 4 asks the sender to come back.
|
||||
if e.Decision == store.SpamDecisionReject && strings.HasPrefix(reason, "4") {
|
||||
e.Decision = store.SpamDecisionTempfail
|
||||
}
|
||||
if catchingUp {
|
||||
if wrote, err := j.st.InsertInboundSpamOnce(e); err != nil {
|
||||
log.Printf("log-tailer: inbound journal: record %s -> %s: %v", from, to, err)
|
||||
return
|
||||
} else if !wrote {
|
||||
return // read before the last stop
|
||||
}
|
||||
} else if err := j.st.InsertInboundSpam(e); err != nil {
|
||||
log.Printf("log-tailer: inbound journal: record %s -> %s: %v", from, to, err)
|
||||
return
|
||||
}
|
||||
if j.written++; j.written >= inboundPruneEvery {
|
||||
j.written = 0
|
||||
// The zero time removes nothing by age: this is the cap alone.
|
||||
if _, err := j.st.PruneInboundSpam(time.Time{}, inboundSpamLogMaxRows); err != nil {
|
||||
log.Printf("log-tailer: inbound journal: cut to %d rows: %v", inboundSpamLogMaxRows, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (j *inboundJournal) remember(queueID string) {
|
||||
if _, ok := j.port25[queueID]; ok {
|
||||
return
|
||||
}
|
||||
for len(j.order) >= inboundQueueIDs {
|
||||
oldest := j.order[0]
|
||||
j.order = j.order[1:]
|
||||
if j.port25[oldest.id] == oldest.seq {
|
||||
delete(j.port25, oldest.id)
|
||||
}
|
||||
}
|
||||
j.seq++
|
||||
j.port25[queueID] = j.seq
|
||||
j.order = append(j.order, queued{queueID, j.seq})
|
||||
}
|
||||
|
||||
// printable cuts s to max bytes and replaces everything outside printable
|
||||
// ASCII.
|
||||
func printable(s string, max int) string {
|
||||
if len(s) > max {
|
||||
s = s[:max]
|
||||
}
|
||||
b := []byte(s)
|
||||
for i, c := range b {
|
||||
if c < 0x20 || c > 0x7e {
|
||||
b[i] = '?'
|
||||
}
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// lineTime reads the time at the head of a mail.log line and returns it with
|
||||
// the rest of the line. A refusal is dated by its line and not by when the
|
||||
// tailer got to it: after the panel was down the two are hours apart. The
|
||||
// syslog format carries neither year nor zone - the year is taken from now
|
||||
// (a stamp that would then lie in the future is last year's), the zone is the
|
||||
// container's, which is what postlogd wrote it in. A line with no readable
|
||||
// stamp is dated now.
|
||||
func lineTime(line string, now time.Time) (time.Time, string) {
|
||||
if m := isoStampRe.FindStringSubmatch(line); m != nil {
|
||||
rest := strings.TrimSpace(line[len(m[0]):])
|
||||
if t, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(m[0])); err == nil {
|
||||
return t.UTC(), rest
|
||||
}
|
||||
return now, rest
|
||||
}
|
||||
if m := syslogStampRe.FindStringSubmatch(line); m != nil {
|
||||
rest := strings.TrimSpace(line[len(m[0]):])
|
||||
stamp := strings.Join(strings.Fields(m[1]), " ")
|
||||
t, err := time.ParseInLocation("Jan 2 15:04:05", stamp, time.Local)
|
||||
if err != nil {
|
||||
return now, rest
|
||||
}
|
||||
t = t.AddDate(now.In(time.Local).Year(), 0, 0)
|
||||
if t.After(now.Add(24 * time.Hour)) {
|
||||
t = t.AddDate(-1, 0, 0)
|
||||
}
|
||||
return t.UTC(), rest
|
||||
}
|
||||
return now, line
|
||||
}
|
||||
|
||||
// The journal keeps at most this many rows whatever their age: refusals are
|
||||
// written at the rate strangers knock on port 25.
|
||||
const inboundSpamLogMaxRows = 10000
|
||||
|
||||
// inboundRetentionLoop prunes the inbound journal at once and then on the
|
||||
// cadence of the send log.
|
||||
func inboundRetentionLoop(ctx context.Context, in *Inbound) {
|
||||
prune := func() {
|
||||
days := in.Retention()
|
||||
if days <= 0 {
|
||||
days = store.InboundSpamLogRetentionDaysDefault
|
||||
}
|
||||
n, err := in.Store.PruneInboundSpam(time.Now().UTC().AddDate(0, 0, -days), inboundSpamLogMaxRows)
|
||||
if err != nil {
|
||||
log.Printf("log-tailer: inbound journal prune: %v", err)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("log-tailer: pruned %d inbound journal rows (older than %d days or beyond %d rows)", n, days, inboundSpamLogMaxRows)
|
||||
}
|
||||
}
|
||||
|
||||
prune()
|
||||
t := time.NewTicker(retentionInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
prune()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
package logtail
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
|
||||
type fakeInboundStore struct {
|
||||
domains map[string]bool
|
||||
rows []store.InboundSpamEntry
|
||||
lookupErr error
|
||||
pruned []int
|
||||
}
|
||||
|
||||
func (f *fakeInboundStore) InsertInboundSpam(e store.InboundSpamEntry) error {
|
||||
f.rows = append(f.rows, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
// InsertInboundSpamOnce writes unless an equal row is there, like the store's.
|
||||
func (f *fakeInboundStore) InsertInboundSpamOnce(e store.InboundSpamEntry) (bool, error) {
|
||||
for _, r := range f.rows {
|
||||
if r == e {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
f.rows = append(f.rows, e)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (f *fakeInboundStore) IsInboundDomain(name string) (bool, error) {
|
||||
if f.lookupErr != nil {
|
||||
return false, f.lookupErr
|
||||
}
|
||||
return f.domains[name], nil
|
||||
}
|
||||
|
||||
func (f *fakeInboundStore) PruneInboundSpam(_ time.Time, keep int) (int64, error) {
|
||||
f.pruned = append(f.pruned, keep)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func inboundFixture() (*inboundJournal, *fakeInboundStore) {
|
||||
st := &fakeInboundStore{domains: map[string]bool{"in.example.com": true}}
|
||||
return newInboundJournal(st), st
|
||||
}
|
||||
|
||||
var tailNow = time.Date(2026, 10, 11, 18, 0, 0, 0, time.UTC)
|
||||
|
||||
const (
|
||||
head = "2026-10-11T12:00:05.123456+00:00 mail "
|
||||
unknownRcp = head + "postfix/inbound/smtpd[41]: NOQUEUE: reject: RCPT from mx.example.net[203.0.113.9]: 550 5.1.1 <nobody@in.example.com>: Recipient address rejected: User unknown in relay recipient table; from=<a@example.net> to=<nobody@in.example.com> proto=ESMTP helo=<mx.example.net>"
|
||||
)
|
||||
|
||||
// Postfix's own refusal of a recipient at an inbound domain is a row, dated by
|
||||
// its line.
|
||||
func TestInboundRejectByPostfix(t *testing.T) {
|
||||
j, st := inboundFixture()
|
||||
j.handle(unknownRcp, tailNow, false)
|
||||
if len(st.rows) != 1 {
|
||||
t.Fatalf("rows = %+v", st.rows)
|
||||
}
|
||||
want := store.InboundSpamEntry{
|
||||
InboundDomain: "in.example.com", ClientIP: "203.0.113.9",
|
||||
From: "a@example.net", To: "nobody@in.example.com",
|
||||
Decision: store.SpamDecisionReject, Engine: store.SpamEnginePostfix,
|
||||
Detail: "550 5.1.1 <nobody@in.example.com>: Recipient address rejected: User unknown in relay recipient table",
|
||||
At: time.Date(2026, 10, 11, 12, 0, 5, 123456000, time.UTC),
|
||||
}
|
||||
if st.rows[0] != want {
|
||||
t.Errorf("row = %+v\nwant %+v", st.rows[0], want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInboundTempfailByPostfix(t *testing.T) {
|
||||
j, st := inboundFixture()
|
||||
j.handle(head+"postfix/inbound/smtpd[41]: NOQUEUE: reject: RCPT from unknown[2001:db8::9]: 450 4.1.8 <a@nx.example>: Sender address rejected: Domain not found; from=<a@nx.example> to=<B@IN.example.com> proto=ESMTP helo=<x>", tailNow, false)
|
||||
if len(st.rows) != 1 || st.rows[0].Decision != store.SpamDecisionTempfail || st.rows[0].ClientIP != "2001:db8::9" || st.rows[0].InboundDomain != "in.example.com" {
|
||||
t.Errorf("rows = %+v", st.rows)
|
||||
}
|
||||
}
|
||||
|
||||
// The filter's verdict is written by cleanup, under one name for every
|
||||
// listener: it is the journal's only for a queue id port 25 opened.
|
||||
func TestInboundMilterVerdictNeedsAPort25QueueID(t *testing.T) {
|
||||
const verdict = "postfix/cleanup[52]: 4A1B2C3D: milter-reject: END-OF-MESSAGE from mx.example.net[203.0.113.9]: 5.7.1 Spam message rejected; from=<a@example.net> to=<b@in.example.com> proto=ESMTP helo=<mx.example.net>"
|
||||
|
||||
j, st := inboundFixture()
|
||||
// Opened by a submission port: postfix/smtpd, not postfix/inbound/smtpd.
|
||||
j.handle(head+"postfix/smtpd[20]: 4A1B2C3D: client=app.example.com[203.0.113.4], sasl_method=PLAIN, sasl_username=app", tailNow, false)
|
||||
j.handle(head+verdict, tailNow, false)
|
||||
if len(st.rows) != 0 {
|
||||
t.Fatalf("a submission-port verdict was journalled: %+v", st.rows)
|
||||
}
|
||||
|
||||
j.handle(head+"postfix/inbound/smtpd[41]: 4A1B2C3D: client=mx.example.net[203.0.113.9]", tailNow, false)
|
||||
j.handle(head+verdict, tailNow, false)
|
||||
if len(st.rows) != 1 {
|
||||
t.Fatalf("rows = %+v", st.rows)
|
||||
}
|
||||
got := st.rows[0]
|
||||
if got.Decision != store.SpamDecisionReject || got.Engine != store.SpamEngineRspamd || got.Detail != "5.7.1 Spam message rejected" {
|
||||
t.Errorf("row = %+v", got)
|
||||
}
|
||||
// One verdict per message: the id is forgotten with it.
|
||||
j.handle(head+verdict, tailNow, false)
|
||||
if len(st.rows) != 1 {
|
||||
t.Errorf("the same queue id was journalled twice")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInboundMilterDecisions(t *testing.T) {
|
||||
for kind, c := range map[string]struct{ reply, decision string }{
|
||||
"milter-reject": {"4.7.1 Try again later", store.SpamDecisionTempfail},
|
||||
"milter-discard": {"milter triggers DISCARD action", store.SpamDecisionReject},
|
||||
} {
|
||||
j, st := inboundFixture()
|
||||
j.handle(head+"postfix/inbound/smtpd[41]: Q1: client=mx.example.net[203.0.113.9]", tailNow, false)
|
||||
j.handle(head+"postfix/cleanup[52]: Q1: "+kind+": END-OF-MESSAGE from mx.example.net[203.0.113.9]: "+c.reply+"; from=<a@example.net> to=<b@in.example.com> proto=ESMTP helo=<mx.example.net>", tailNow, false)
|
||||
if len(st.rows) != 1 || st.rows[0].Decision != c.decision || st.rows[0].Engine != store.SpamEngineRspamd {
|
||||
t.Errorf("%s: rows = %+v", kind, st.rows)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What is not the relay's mail, or is not a refusal at all, leaves no row.
|
||||
func TestInboundIgnores(t *testing.T) {
|
||||
for name, line := range map[string]string{
|
||||
"another domain (relay access denied)": head + "postfix/inbound/smtpd[41]: NOQUEUE: reject: RCPT from x[203.0.113.9]: 554 5.7.1 <c@elsewhere.example>: Relay access denied; from=<a@example.net> to=<c@elsewhere.example> proto=ESMTP helo=<x>",
|
||||
"no recipient (size limit at MAIL)": head + "postfix/inbound/smtpd[41]: NOQUEUE: reject: MAIL from x[203.0.113.9]: 552 5.3.4 Message size exceeds fixed limit; proto=ESMTP helo=<x>",
|
||||
"a submission port": head + "postfix/smtpd[20]: NOQUEUE: reject: RCPT from x[203.0.113.9]: 553 5.7.1 <b@in.example.com>: Sender address rejected: not owned by user app; from=<a@example.net> to=<b@in.example.com> proto=ESMTP helo=<x>",
|
||||
"a delivery line": head + "postfix/smtp[26]: 41E862C00D9E: to=<b@in.example.com>, relay=mx[203.0.113.9]:25, dsn=2.0.0, status=sent (250 OK)",
|
||||
"a connect line": head + "postfix/inbound/smtpd[41]: connect from mx.example.net[203.0.113.9]",
|
||||
// The HELO name is the sender's; a line cannot be started from inside one.
|
||||
"a refusal spelled inside another line": head + "postfix/inbound/smtpd[41]: warning: hostname x does not resolve: mail postfix/inbound/smtpd[1]: NOQUEUE: reject: RCPT from x[203.0.113.9]: 550 no; from=<a@example.net> to=<b@in.example.com> proto=ESMTP helo=<x>",
|
||||
"an address with a bracket in it": head + "postfix/inbound/smtpd[41]: NOQUEUE: reject: RCPT from x[203.0.113.9]: 554 5.7.1 <b@elsewhere.example>: Relay access denied; from=<\"a> to=<v@in.example.com> proto=X helo=<y\"@example.net> to=<b@elsewhere.example> proto=ESMTP helo=<x>",
|
||||
} {
|
||||
j, st := inboundFixture()
|
||||
j.handle(line, tailNow, false)
|
||||
if len(st.rows) != 0 {
|
||||
t.Errorf("%s: journalled %+v", name, st.rows)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInboundDetailIsCutAndPrintable(t *testing.T) {
|
||||
j, st := inboundFixture()
|
||||
j.handle(head+"postfix/inbound/smtpd[41]: NOQUEUE: reject: RCPT from x[203.0.113.9]: 550 5.7.1 caf\xc3\xa9 \x1b[31m"+strings.Repeat("x", 400)+"; from=<a@example.net> to=<b@in.example.com> proto=ESMTP helo=<x>", tailNow, false)
|
||||
if len(st.rows) != 1 {
|
||||
t.Fatalf("rows = %+v", st.rows)
|
||||
}
|
||||
d := st.rows[0].Detail
|
||||
if len(d) != inboundDetailMax || !strings.HasPrefix(d, "550 5.7.1 caf?? ?[31mxxx") {
|
||||
t.Errorf("detail (%d bytes) = %q", len(d), d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInboundLookupFailureLeavesNoRow(t *testing.T) {
|
||||
j, st := inboundFixture()
|
||||
st.lookupErr = errors.New("boom")
|
||||
j.handle(unknownRcp, tailNow, false)
|
||||
if len(st.rows) != 0 {
|
||||
t.Errorf("rows = %+v", st.rows)
|
||||
}
|
||||
}
|
||||
|
||||
// A message the filter holds goes on to the inbound journal milter, which
|
||||
// writes its row; a second one from the log would be the same message twice.
|
||||
func TestInboundHoldIsTheMiltersRow(t *testing.T) {
|
||||
j, st := inboundFixture()
|
||||
j.handle(head+"postfix/inbound/smtpd[41]: Q1: client=mx.example.net[203.0.113.9]", tailNow, false)
|
||||
j.handle(head+"postfix/cleanup[52]: Q1: milter-hold: END-OF-MESSAGE from mx.example.net[203.0.113.9]: milter triggers HOLD action; from=<a@example.net> to=<b@in.example.com> proto=ESMTP helo=<mx.example.net>", tailNow, false)
|
||||
if len(st.rows) != 0 {
|
||||
t.Errorf("rows = %+v", st.rows)
|
||||
}
|
||||
}
|
||||
|
||||
// The HELO name is the last thing on the line and is logged as the client
|
||||
// wrote it. It cannot supply the row: the reply ends at the first "; from=<",
|
||||
// which is Postfix's own.
|
||||
func TestInboundHeloCannotWriteTheRow(t *testing.T) {
|
||||
j, st := inboundFixture()
|
||||
j.handle(head+"postfix/inbound/smtpd[41]: NOQUEUE: reject: RCPT from x[203.0.113.9]: 550 5.1.1 <nobody@in.example.com>: Recipient address rejected: User unknown in relay recipient table; from=<real@example.net> to=<nobody@in.example.com> proto=ESMTP helo=<x 5.7.1 Refused by this server's deny list; from=<victim@bank.example> to=<ceo@in.example.com> proto=ESMTP helo=<y>", tailNow, false)
|
||||
if len(st.rows) != 1 {
|
||||
t.Fatalf("rows = %+v", st.rows)
|
||||
}
|
||||
got := st.rows[0]
|
||||
if got.From != "real@example.net" || got.To != "nobody@in.example.com" ||
|
||||
got.Detail != "550 5.1.1 <nobody@in.example.com>: Recipient address rejected: User unknown in relay recipient table" {
|
||||
t.Errorf("row = %+v", got)
|
||||
}
|
||||
|
||||
// A HELO name with a closing bracket in it does not lose the line either.
|
||||
j, st = inboundFixture()
|
||||
j.handle(head+"postfix/inbound/smtpd[41]: NOQUEUE: reject: RCPT from x[203.0.113.9]: 550 5.1.1 <nobody@in.example.com>: no; from=<real@example.net> to=<nobody@in.example.com> proto=ESMTP helo=<a>b>", tailNow, false)
|
||||
if len(st.rows) != 1 || st.rows[0].From != "real@example.net" {
|
||||
t.Errorf("rows = %+v", st.rows)
|
||||
}
|
||||
|
||||
// The same on a verdict line: the filter's reply is not extended by the
|
||||
// HELO name, so a stranger cannot make a refusal look like the deny list's.
|
||||
j, st = inboundFixture()
|
||||
j.handle(head+"postfix/inbound/smtpd[41]: Q1: client=x[203.0.113.9]", tailNow, false)
|
||||
j.handle(head+"postfix/cleanup[52]: Q1: milter-reject: END-OF-MESSAGE from x[203.0.113.9]: 5.7.1 Spam message rejected; from=<real@example.net> to=<b@in.example.com> proto=ESMTP helo=<x this server's deny list; from=<spoofed@example.org> to=<b@in.example.com> proto=ESMTP helo=<y>", tailNow, false)
|
||||
if len(st.rows) != 1 || st.rows[0].Detail != "5.7.1 Spam message rejected" || st.rows[0].From != "real@example.net" {
|
||||
t.Errorf("rows = %+v", st.rows)
|
||||
}
|
||||
}
|
||||
|
||||
// A queue id is port 25's only until the message is queued or the id is
|
||||
// opened again by another listener - Postfix reuses ids.
|
||||
func TestInboundQueueIDIsForgotten(t *testing.T) {
|
||||
const verdict = "postfix/cleanup[52]: Q1: milter-reject: END-OF-MESSAGE from app[203.0.113.4]: 4.7.1 Service unavailable - try again later; from=<a@example.net> to=<b@in.example.com> proto=ESMTP helo=<app>"
|
||||
for name, forget := range map[string]string{
|
||||
"queued": "postfix/qmgr[58]: Q1: from=<a@example.net>, size=600, nrcpt=1 (queue active)",
|
||||
"reused by the submission port": "postfix/smtpd[20]: Q1: client=app.example.com[203.0.113.4], sasl_method=PLAIN, sasl_username=app",
|
||||
} {
|
||||
j, st := inboundFixture()
|
||||
j.handle(head+"postfix/inbound/smtpd[41]: Q1: client=mx.example.net[203.0.113.9]", tailNow, false)
|
||||
j.handle(head+forget, tailNow, false)
|
||||
j.handle(head+verdict, tailNow, false)
|
||||
if len(st.rows) != 0 || len(j.port25) != 0 {
|
||||
t.Errorf("%s: rows %+v, remembered %v", name, st.rows, j.port25)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An id that was forgotten and opened again is not dropped by its own stale
|
||||
// place in the queue of arrivals.
|
||||
func TestInboundStaleArrivalDoesNotDropALiveID(t *testing.T) {
|
||||
j, _ := inboundFixture()
|
||||
j.remember("Q1")
|
||||
delete(j.port25, "Q1") // its verdict came
|
||||
for i := 0; i < inboundQueueIDs-2; i++ {
|
||||
j.remember("F" + time.Duration(i).String())
|
||||
}
|
||||
j.remember("Q1") // reused; order is now full, with the stale Q1 first
|
||||
j.remember("LAST")
|
||||
if _, ok := j.port25["Q1"]; !ok {
|
||||
t.Error("the live Q1 was dropped by its stale arrival")
|
||||
}
|
||||
}
|
||||
|
||||
// Between the six-hourly prunes the tailer has the journal cut to its cap
|
||||
// itself, every so many rows: refusals come at the rate strangers send them.
|
||||
func TestInboundCutsTheJournalAsItWrites(t *testing.T) {
|
||||
j, st := inboundFixture()
|
||||
for i := 0; i < inboundPruneEvery*2+10; i++ {
|
||||
j.handle(unknownRcp, tailNow, false)
|
||||
}
|
||||
if len(st.pruned) != 2 || st.pruned[0] != inboundSpamLogMaxRows {
|
||||
t.Errorf("cuts = %v, want two, to %d rows", st.pruned, inboundSpamLogMaxRows)
|
||||
}
|
||||
}
|
||||
|
||||
// The queue ids waiting for a verdict are bounded.
|
||||
func TestInboundQueueIDsAreBounded(t *testing.T) {
|
||||
j, _ := inboundFixture()
|
||||
for i := 0; i < inboundQueueIDs+10; i++ {
|
||||
j.remember("Q" + strings.Repeat("A", i%7) + string(rune('0'+i%10)) + time.Duration(i).String())
|
||||
}
|
||||
if len(j.port25) != inboundQueueIDs || len(j.order) != inboundQueueIDs {
|
||||
t.Errorf("remembered %d ids (%d in order), want %d", len(j.port25), len(j.order), inboundQueueIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLineTime(t *testing.T) {
|
||||
now := time.Date(2026, 1, 2, 10, 0, 0, 0, time.UTC)
|
||||
for _, c := range []struct {
|
||||
line string
|
||||
want time.Time
|
||||
rest string
|
||||
}{
|
||||
{"2026-01-02T09:00:00.5+03:00 mail postfix/x: y", time.Date(2026, 1, 2, 6, 0, 0, 500000000, time.UTC), "mail postfix/x: y"},
|
||||
{"Jan 2 09:30:00 mail postfix/x: y", time.Date(2026, 1, 2, 9, 30, 0, 0, time.Local).UTC(), "mail postfix/x: y"},
|
||||
// December read in January is last year's.
|
||||
{"Dec 31 23:59:00 mail postfix/x: y", time.Date(2025, 12, 31, 23, 59, 0, 0, time.Local).UTC(), "mail postfix/x: y"},
|
||||
{"no stamp here", now, "no stamp here"},
|
||||
} {
|
||||
got, rest := lineTime(c.line, now)
|
||||
if !got.Equal(c.want) || rest != c.rest {
|
||||
t.Errorf("lineTime(%q) = %v, %q; want %v, %q", c.line, got, rest, c.want, c.rest)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInboundRetentionPrunesAtStart(t *testing.T) {
|
||||
old := retentionInterval
|
||||
retentionInterval = time.Hour
|
||||
defer func() { retentionInterval = old }()
|
||||
|
||||
st := &fakeInboundStore{}
|
||||
ctx, cancel := contextWithCancel()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
inboundRetentionLoop(ctx, &Inbound{Store: st, Retention: func() int { return 0 }})
|
||||
close(done)
|
||||
}()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for len(st.pruned) == 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
if len(st.pruned) != 1 || st.pruned[0] != inboundSpamLogMaxRows {
|
||||
t.Errorf("prune calls = %v", st.pruned)
|
||||
}
|
||||
}
|
||||
|
||||
func contextWithCancel() (context.Context, context.CancelFunc) {
|
||||
return context.WithCancel(context.Background())
|
||||
}
|
||||
|
||||
// The lines the tailer reads when it starts may be ones it read before its
|
||||
// last stop: a refusal among them is one row, not two. Lines read as they are
|
||||
// written are always new.
|
||||
func TestInboundCatchUpDoesNotWriteARowTwice(t *testing.T) {
|
||||
j, st := inboundFixture()
|
||||
j.handle(unknownRcp, tailNow, false) // before the stop
|
||||
j.handle(unknownRcp, tailNow, true) // the same line, read again on start
|
||||
if len(st.rows) != 1 {
|
||||
t.Fatalf("rows = %d, want the one refusal", len(st.rows))
|
||||
}
|
||||
j.handle(unknownRcp, tailNow, false) // a second refusal in the same second
|
||||
if len(st.rows) != 2 {
|
||||
t.Errorf("rows = %d, want two: a line read as it is written is new", len(st.rows))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package logtail
|
||||
|
||||
import (
|
||||
"log"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
|
||||
// A message's page shows what Postfix wrote about it. mail.log is rotated
|
||||
// daily and fourteen files are kept, while a send-log row lives for the
|
||||
// retention window, so the lines are kept in the database as the tailer reads
|
||||
// them (architecture.md § Log tailer).
|
||||
//
|
||||
// queueLineRe finds the queue id at the head of a line, after the timestamp
|
||||
// (already taken off by lineTime), the host name and "program[pid]:":
|
||||
//
|
||||
// mail postfix/smtp[26]: 41E862C00D9E: to=<a@example.net>, relay=…, status=sent (250 OK)
|
||||
//
|
||||
// It is matched from the start of the line and nowhere else: the tail of a
|
||||
// line is the far end's words - a reply, a HELO name - and a pattern that
|
||||
// could start anywhere would let them file a line under a queue id of their
|
||||
// choosing.
|
||||
//
|
||||
// What stands in that place is not always a queue id: "NOQUEUE:" and
|
||||
// "warning:" stand there too. A queue id is hexadecimal in capitals, or, with
|
||||
// enable_long_queue_ids, digits and letters without vowels; neither word is
|
||||
// either.
|
||||
var queueLineRe = regexp.MustCompile(`^\S+ ([^\s\[]+)\[\d+\]: ([0-9A-F]{6,24}|[0-9B-DF-HJ-NP-TV-Zb-df-hj-np-tv-z]{10,24}): (.*)`)
|
||||
|
||||
// smtpdProgramRe tells an smtpd from the other programs, whatever listener it
|
||||
// serves: "postfix/smtpd", "postfix/submission/smtpd", "postfix/inbound/smtpd".
|
||||
var smtpdProgramRe = regexp.MustCompile(`^postfix/(?:[a-z0-9-]+/)?smtpd$`)
|
||||
|
||||
// port25Program is the smtpd of the inbound relay's listener, which
|
||||
// build/postfix-config.sh names so that its lines are told from those of the
|
||||
// submission ports.
|
||||
const port25Program = "postfix/inbound/smtpd"
|
||||
|
||||
const (
|
||||
// lineGrace is how old a line must be before the prune asks whether a
|
||||
// send-log row carries its queue id. The first lines of a message are
|
||||
// logged when it arrives and its row is written at end-of-message; an hour
|
||||
// is a message body sent slowly.
|
||||
lineGrace = time.Hour
|
||||
|
||||
// lineBatch is how many lines are held before they are written without
|
||||
// waiting for the end of the pass over the log. A pass is a second's worth
|
||||
// of lines while following, and the whole backlog when the tailer starts.
|
||||
lineBatch = 500
|
||||
|
||||
// port25IDs is how many queue ids opened on port 25 are remembered so that
|
||||
// their lines are not kept. Beyond it the oldest knowledge is given up, and
|
||||
// those messages' lines are kept until the prune finds them without a row.
|
||||
port25IDs = 4096
|
||||
)
|
||||
|
||||
// lineKeeper collects the lines headed by a queue id and writes them in
|
||||
// batches. It is used from the follow loop only, so it needs no locking.
|
||||
type lineKeeper struct {
|
||||
st StatusStore
|
||||
// buf holds the lines not yet written; once says they were read while
|
||||
// catching up and may have been read before the last stop.
|
||||
buf []store.SendLogLine
|
||||
once bool
|
||||
// port25 holds the queue ids the port-25 smtpd opened: mail offered by
|
||||
// strangers, which never gets a send-log row. Its lines are not kept - what
|
||||
// a stranger can make Postfix log is not something to store on his say-so
|
||||
// and prune later. An id leaves when qmgr removes the message or when
|
||||
// another listener opens a message under it (Postfix reuses ids).
|
||||
port25 map[string]struct{}
|
||||
}
|
||||
|
||||
func newLineKeeper(st StatusStore) *lineKeeper {
|
||||
return &lineKeeper{st: st, port25: make(map[string]struct{})}
|
||||
}
|
||||
|
||||
// add takes one mail.log line and keeps it if it is headed by the queue id of
|
||||
// a message that did not come in on port 25. It does not ask whether a
|
||||
// send-log row exists for the id: for a message's first lines none does yet,
|
||||
// and the lines of messages that never get one - the bounces Postfix
|
||||
// generates, port-25 mail met in mid-flight after a restart - are removed by
|
||||
// pruneLines.
|
||||
//
|
||||
// catchingUp is as in inboundJournal.handle: the line may have been read
|
||||
// before the last stop, and is then written only if it is not there already.
|
||||
// The line is dated by its own stamp - after the panel was down, that and now
|
||||
// are hours apart, and a line is matched to its row by time.
|
||||
func (k *lineKeeper) add(line string, now time.Time, catchingUp bool) {
|
||||
at, rest := lineTime(line, now)
|
||||
m := queueLineRe.FindStringSubmatch(rest)
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
program, queueID, text := m[1], m[2], m[3]
|
||||
|
||||
if smtpdProgramRe.MatchString(program) && len(text) >= 7 && text[:7] == "client=" {
|
||||
if program == port25Program {
|
||||
k.rememberPort25(queueID)
|
||||
} else {
|
||||
delete(k.port25, queueID)
|
||||
}
|
||||
}
|
||||
if _, inbound := k.port25[queueID]; inbound {
|
||||
if program == "postfix/qmgr" && text == "removed" {
|
||||
delete(k.port25, queueID)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if catchingUp != k.once {
|
||||
k.flush()
|
||||
k.once = catchingUp
|
||||
}
|
||||
k.buf = append(k.buf, store.SendLogLine{QueueID: queueID, At: at, Line: line})
|
||||
if len(k.buf) >= lineBatch {
|
||||
k.flush()
|
||||
}
|
||||
}
|
||||
|
||||
func (k *lineKeeper) rememberPort25(queueID string) {
|
||||
if len(k.port25) >= port25IDs {
|
||||
// Which one goes does not matter: the cost of forgetting an id is its
|
||||
// remaining lines kept until the prune.
|
||||
for id := range k.port25 {
|
||||
delete(k.port25, id)
|
||||
break
|
||||
}
|
||||
}
|
||||
k.port25[queueID] = struct{}{}
|
||||
}
|
||||
|
||||
// flush writes the lines held. The follow loop calls it after every pass over
|
||||
// the log and before it saves its place, so the place saved is never ahead of
|
||||
// what was written. Lines that cannot be written are dropped and said so: the
|
||||
// tailer goes on, and a page without them falls back to the log file.
|
||||
func (k *lineKeeper) flush() {
|
||||
if len(k.buf) == 0 {
|
||||
return
|
||||
}
|
||||
if _, err := k.st.InsertSendLogLines(k.buf, k.once); err != nil {
|
||||
log.Printf("log-tailer: keep %d log lines: %v", len(k.buf), err)
|
||||
}
|
||||
k.buf = k.buf[:0]
|
||||
}
|
||||
|
||||
// pruneLines removes the stored lines no send-log row carries the queue id of:
|
||||
// those of rows the retention sweep deleted, and those of messages that never
|
||||
// had a row.
|
||||
func pruneLines(st StatusStore) {
|
||||
n, err := st.PruneSendLogLines(time.Now().UTC().Add(-lineGrace))
|
||||
if err != nil {
|
||||
log.Printf("log-tailer: prune stored log lines: %v", err)
|
||||
return
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("log-tailer: pruned %d stored log lines with no send-log row", n)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package logtail
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Which lines are kept for a message's page, and under which queue id: the id
|
||||
// at the head of the line and nothing the far end wrote further along it.
|
||||
func TestQueueLineHead(t *testing.T) {
|
||||
cases := []struct {
|
||||
name, rest, want string
|
||||
}{
|
||||
{"delivery", "mail postfix/smtp[26]: 41E862C00D9E: to=<a@example.net>, relay=mx.example.net[203.0.113.9]:25, dsn=2.0.0, status=sent (250 OK)", "41E862C00D9E"},
|
||||
{"submission smtpd opens the id", "mail postfix/submission/smtpd[10]: 41E862C00D9E: client=unknown[203.0.113.7], sasl_method=PLAIN, sasl_username=app", "41E862C00D9E"},
|
||||
{"port-25 smtpd", "mail postfix/inbound/smtpd[41]: 4A1B2C3D: client=mx.example.net[203.0.113.9]", "4A1B2C3D"},
|
||||
{"cleanup", "mail postfix/cleanup[52]: 41E862C00D9E: message-id=<1@example.com>", "41E862C00D9E"},
|
||||
{"qmgr removes it", "mail postfix/qmgr[10]: 41E862C00D9E: removed", "41E862C00D9E"},
|
||||
{"a bounce is sent: the id at the head, not the new message's", "mail postfix/bounce[30]: 41E862C00D9E: sender non-delivery notification: 5F00AA12", "41E862C00D9E"},
|
||||
{"long queue id", "mail postfix/qmgr[10]: 3sJ8Xh2Cgqz4Xy1: removed", "3sJ8Xh2Cgqz4Xy1"},
|
||||
{"refused before queueing", "mail postfix/inbound/smtpd[41]: NOQUEUE: reject: RCPT from mx.example.net[203.0.113.9]: 550 5.1.1 <n@in.example.com>: Recipient address rejected; from=<a@example.net> to=<n@in.example.com> proto=ESMTP helo=<mx.example.net>", ""},
|
||||
{"a warning", "mail postfix/smtpd[10]: warning: hostname mx.example.net does not resolve to address 203.0.113.9", ""},
|
||||
{"a connection", "mail postfix/smtpd[10]: connect from unknown[203.0.113.7]", ""},
|
||||
{"no program", "preexisting line, ignored on start", ""},
|
||||
{
|
||||
// The reply is the receiving server's text. One that spells the head
|
||||
// of a line is still the tail of this one: the line is the message's
|
||||
// it was written about, and no other's.
|
||||
"a reply that quotes a line head",
|
||||
"mail postfix/smtp[26]: 9F1A2C00D9E: to=<a@example.net>, relay=mx.example.net[203.0.113.9]:25, dsn=5.1.1, status=bounced (host said: 550 mail postfix/smtp[1]: 41E862C00D9E: to=<b@example.net>, status=sent)",
|
||||
"9F1A2C00D9E",
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := ""
|
||||
if m := queueLineRe.FindStringSubmatch(c.rest); m != nil {
|
||||
got = m[2]
|
||||
}
|
||||
if got != c.want {
|
||||
t.Errorf("%s: queue id %q, want %q", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A line is dated by its own stamp, not by when the tailer got to it, and is
|
||||
// kept as the log has it. Nothing is written until the pass over the log ends.
|
||||
func TestLineKeeperDatesByTheLine(t *testing.T) {
|
||||
cs := &captureStore{}
|
||||
k := newLineKeeper(cs)
|
||||
now := time.Date(2026, 10, 12, 18, 0, 0, 0, time.UTC)
|
||||
line := "Oct 12 09:00:00 mail postfix/qmgr[7]: 41E862C00D9E: removed"
|
||||
|
||||
k.add(line, now, false)
|
||||
k.add("Oct 12 09:00:01 mail postfix/smtpd[7]: connect from unknown[203.0.113.7]", now, false)
|
||||
if len(cs.lines) != 0 {
|
||||
t.Fatalf("written before the pass ended: %v", cs.kept())
|
||||
}
|
||||
k.flush()
|
||||
|
||||
if len(cs.lines) != 1 {
|
||||
t.Fatalf("kept %d lines, want 1: %v", len(cs.lines), cs.kept())
|
||||
}
|
||||
got := cs.lines[0]
|
||||
if got.QueueID != "41E862C00D9E" || got.Line != line {
|
||||
t.Errorf("kept %q under %q", got.Line, got.QueueID)
|
||||
}
|
||||
if want := time.Date(2026, 10, 12, 9, 0, 0, 0, time.Local).UTC(); !got.At.Equal(want) {
|
||||
t.Errorf("dated %v, want %v", got.At, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A backlog is written in batches of lineBatch, not held whole, and a pass
|
||||
// that changes from catching up to following does not write the later lines
|
||||
// under the earlier rule.
|
||||
func TestLineKeeperWritesInBatches(t *testing.T) {
|
||||
cs := &captureStore{}
|
||||
k := newLineKeeper(cs)
|
||||
now := time.Date(2026, 10, 12, 18, 0, 0, 0, time.UTC)
|
||||
line := func(i int) string {
|
||||
return "host postfix/qmgr[7]: 41E862C00D9E: from=<a@example.com>, size=" + itoa(i) + ", nrcpt=1 (queue active)"
|
||||
}
|
||||
|
||||
for i := 0; i < lineBatch+10; i++ {
|
||||
k.add(line(i), now, true)
|
||||
}
|
||||
if len(cs.lines) != lineBatch || cs.batches != 1 {
|
||||
t.Fatalf("after %d lines: %d written in %d batches, want %d in 1", lineBatch+10, len(cs.lines), cs.batches, lineBatch)
|
||||
}
|
||||
|
||||
// The same line again, now as one read while following: it is a new line,
|
||||
// and the ten still held from the catch-up are written first, under theirs.
|
||||
k.add(line(0), now, false)
|
||||
k.flush()
|
||||
if got, want := len(cs.lines), lineBatch+11; got != want {
|
||||
t.Errorf("%d lines written, want %d", got, want)
|
||||
}
|
||||
if cs.batches != 3 {
|
||||
t.Errorf("%d batches, want 3", cs.batches)
|
||||
}
|
||||
}
|
||||
|
||||
// Mail that comes in on port 25 is a stranger's and never gets a send-log row:
|
||||
// none of its lines are kept, from the smtpd that opened the queue id to
|
||||
// qmgr's "removed". The id is then anybody's again.
|
||||
func TestLineKeeperLeavesOutPort25Mail(t *testing.T) {
|
||||
cs := &captureStore{}
|
||||
k := newLineKeeper(cs)
|
||||
now := time.Date(2026, 10, 12, 18, 0, 0, 0, time.UTC)
|
||||
|
||||
const (
|
||||
outClient = "host postfix/submission/smtpd[10]: 41E862C00D9E: client=app.example.com[203.0.113.7], sasl_method=PLAIN, sasl_username=app"
|
||||
outSent = "host postfix/smtp[26]: 41E862C00D9E: to=<a@example.net>, relay=mx.example.net[203.0.113.9]:25, dsn=2.0.0, status=sent (250 OK)"
|
||||
reused = "host postfix/submission/smtpd[10]: 4A1B2C3D: client=app.example.com[203.0.113.7], sasl_method=PLAIN, sasl_username=app"
|
||||
bounce = "host postfix/cleanup[52]: 5F00AA12: message-id=<20261012180000.5F00AA12@mail.example.com>"
|
||||
)
|
||||
for _, line := range []string{
|
||||
"host postfix/inbound/smtpd[41]: 4A1B2C3D: client=mx.example.net[198.51.100.9]",
|
||||
outClient,
|
||||
"host postfix/cleanup[52]: 4A1B2C3D: message-id=<whatever-the-stranger-wrote@example.net>",
|
||||
"host postfix/qmgr[10]: 4A1B2C3D: from=<a@example.net>, size=900, nrcpt=1 (queue active)",
|
||||
outSent,
|
||||
"host postfix/smtp[27]: 4A1B2C3D: to=<b@in.example.com>, relay=upstream.example.com[192.0.2.1]:25, dsn=2.0.0, status=sent (250 OK)",
|
||||
"host postfix/qmgr[10]: 4A1B2C3D: removed",
|
||||
// The id comes round again for a submitted message, and a bounce has no
|
||||
// smtpd at all: both are kept.
|
||||
reused,
|
||||
bounce,
|
||||
} {
|
||||
k.add(line, now, false)
|
||||
}
|
||||
k.flush()
|
||||
|
||||
want := []string{"41E862C00D9E|" + outClient, "41E862C00D9E|" + outSent, "4A1B2C3D|" + reused, "5F00AA12|" + bounce}
|
||||
if got := cs.kept(); strings.Join(got, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("kept:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
|
||||
// A port-25 id that a submission listener opens before qmgr's "removed" was
|
||||
// seen (the line was lost, the panel restarted) is the submitted message's.
|
||||
k.add("host postfix/inbound/smtpd[41]: 6C00BB34: client=mx.example.net[198.51.100.9]", now, false)
|
||||
again := "host postfix/smtps/smtpd[11]: 6C00BB34: client=app.example.com[203.0.113.7], sasl_method=PLAIN, sasl_username=app"
|
||||
k.add(again, now, false)
|
||||
k.flush()
|
||||
if got := cs.kept(); got[len(got)-1] != "6C00BB34|"+again {
|
||||
t.Errorf("a reused port-25 id was not kept for the submitted message: %v", got[len(got)-1])
|
||||
}
|
||||
|
||||
// What is remembered of port 25 has a ceiling.
|
||||
for i := 0; i < port25IDs+50; i++ {
|
||||
k.rememberPort25("7D" + itoa(100000+i))
|
||||
}
|
||||
if len(k.port25) > port25IDs {
|
||||
t.Errorf("%d port-25 ids remembered, ceiling is %d", len(k.port25), port25IDs)
|
||||
}
|
||||
}
|
||||
|
||||
// The tailer keeps queue-id lines as it follows the log, and after an unclean
|
||||
// stop - its saved place lies before lines it had already read - it does not
|
||||
// keep them a second time.
|
||||
func TestRunKeepsQueueLinesOnce(t *testing.T) {
|
||||
old := pollInterval
|
||||
pollInterval = 10 * time.Millisecond
|
||||
t.Cleanup(func() { pollInterval = old })
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "mail.log")
|
||||
// Longer than a fingerprint, so the file is recognised across the restart.
|
||||
seed := strings.Repeat("host postfix/smtpd[1]: connect from unknown[203.0.113.7]\n", 20)
|
||||
if err := os.WriteFile(path, []byte(seed), 0o644); err != nil {
|
||||
t.Fatalf("seed log: %v", err)
|
||||
}
|
||||
|
||||
const (
|
||||
client = "host postfix/submission/smtpd[1]: 41E862C00D9E: client=unknown[203.0.113.7]"
|
||||
sent = "host postfix/smtp[1]: 41E862C00D9E: to=<a@example.net>, dsn=2.0.0, status=sent (ok)"
|
||||
later = "host postfix/qmgr[1]: 41E862C00D9E: removed"
|
||||
)
|
||||
|
||||
cs := &captureStore{}
|
||||
stop := startRun(t, path, cs)
|
||||
appendLine(t, path, client)
|
||||
appendLine(t, path, "host postfix/smtpd[1]: disconnect from unknown[203.0.113.7]")
|
||||
appendLine(t, path, sent)
|
||||
waitFor(t, func() bool { return len(cs.kept()) == 2 })
|
||||
stop()
|
||||
|
||||
// The process died before it saved its place: what it has is the place it
|
||||
// was at before those lines.
|
||||
cs.mu.Lock()
|
||||
cs.state.Offset = int64(len(seed))
|
||||
cs.mu.Unlock()
|
||||
appendLine(t, path, later)
|
||||
|
||||
stop = startRun(t, path, cs)
|
||||
waitFor(t, func() bool { return len(cs.kept()) >= 3 })
|
||||
stop()
|
||||
|
||||
want := []string{"41E862C00D9E|" + client, "41E862C00D9E|" + sent, "41E862C00D9E|" + later}
|
||||
if got := cs.kept(); strings.Join(got, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("kept:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// Lines without a send-log row are removed at the start and after every
|
||||
// send-log sweep, and only once they are old enough for the row to have been
|
||||
// written.
|
||||
func TestRetentionLoopPrunesLines(t *testing.T) {
|
||||
old := retentionInterval
|
||||
retentionInterval = 20 * time.Millisecond
|
||||
t.Cleanup(func() { retentionInterval = old })
|
||||
|
||||
cs := &captureStore{}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go retentionLoop(ctx, cs, func() int { return 90 })
|
||||
|
||||
waitFor(t, func() bool { return cs.prunes() >= 3 })
|
||||
|
||||
cs.mu.Lock()
|
||||
defer cs.mu.Unlock()
|
||||
for _, cutoff := range cs.pruned {
|
||||
if age := time.Since(cutoff); age < lineGrace || age > lineGrace+time.Minute {
|
||||
t.Errorf("prune cutoff %v ago, want %v", age, lineGrace)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,8 @@
|
||||
// Postfix only decides sent / deferred / bounced later, per recipient, and
|
||||
// reports it in mail.log. This package parses those lines by queue-id +
|
||||
// recipient and advances the matching rows, and prunes rows past the retention
|
||||
// window.
|
||||
// window. It also keeps every line Postfix wrote under a queue id, so that a
|
||||
// message's page can show them after the log has rotated (lines.go).
|
||||
package logtail
|
||||
|
||||
import (
|
||||
@@ -25,12 +26,14 @@ import (
|
||||
|
||||
// StatusStore is the slice of the store the log-tailer needs: advancing
|
||||
// delivery statuses, finding the rows still waiting for one, pruning the
|
||||
// retention window, and remembering how far into mail.log it has read.
|
||||
// *store.Store satisfies it.
|
||||
// retention window, keeping the lines of queued messages, and remembering how
|
||||
// far into mail.log it has read. *store.Store satisfies it.
|
||||
type StatusStore interface {
|
||||
UpdateStatus(queueID, recipient, status string) (int64, error)
|
||||
ListQueuedOlderThan(cutoff time.Time) ([]store.QueuedDelivery, error)
|
||||
DeleteSendLogBefore(cutoff time.Time) (int64, error)
|
||||
InsertSendLogLines(lines []store.SendLogLine, once bool) (int, error)
|
||||
PruneSendLogLines(cutoff time.Time) (int64, error)
|
||||
LogtailState(path string) (store.LogtailState, bool, error)
|
||||
SaveLogtailState(path string, st store.LogtailState) error
|
||||
}
|
||||
@@ -107,9 +110,20 @@ func parseDelivery(line string) (queueID, recipient, status string, ok bool) {
|
||||
// Run follows path and updates send-log statuses until ctx is cancelled, while
|
||||
// a background sweep prunes rows older than retention(). Reading resumes at
|
||||
// the offset the previous run persisted, so a restart parses the delivery lines
|
||||
// written while the panel was down. It returns nil on a clean shutdown.
|
||||
func Run(ctx context.Context, path string, st StatusStore, retention RetentionDays) error {
|
||||
// written while the panel was down. The lines headed by a queue id are kept
|
||||
// for the message's page, except those of mail that came in on port 25
|
||||
// (lines.go). With inbound set it also writes the
|
||||
// inbound journal's rows for refused mail and prunes that journal
|
||||
// (inbound.go), and with sender set it counts the mail Postfix deferred
|
||||
// because the sender check did not answer (sendercheck.go). It returns nil on
|
||||
// a clean shutdown.
|
||||
func Run(ctx context.Context, path string, st StatusStore, retention RetentionDays, inbound *Inbound, sender *SenderCheck) error {
|
||||
go retentionLoop(ctx, st, retention)
|
||||
var journal *inboundJournal
|
||||
if inbound != nil {
|
||||
go inboundRetentionLoop(ctx, inbound)
|
||||
journal = newInboundJournal(inbound.Store)
|
||||
}
|
||||
|
||||
// The reconcile sweep must not run against a backlog the tailer has not
|
||||
// read yet: on a restart the log holds the very lines that resolve the rows
|
||||
@@ -118,7 +132,30 @@ func Run(ctx context.Context, path string, st StatusStore, retention RetentionDa
|
||||
caughtUp := make(chan struct{})
|
||||
go reconcileLoop(ctx, st, caughtUp)
|
||||
|
||||
return follow(ctx, path, &tracker{st: st, path: path}, caughtUp, func(line string) {
|
||||
var deferred *deferredCounter
|
||||
if sender != nil {
|
||||
deferred = newDeferredCounter(sender)
|
||||
}
|
||||
|
||||
keeper := newLineKeeper(st)
|
||||
return follow(ctx, path, &tracker{st: st, path: path}, caughtUp, keeper.flush, func(line string) {
|
||||
// Until the first read reaches end-of-file the lines are those written
|
||||
// while the panel was down - and, after an unclean stop, some it had
|
||||
// read already.
|
||||
catchingUp := true
|
||||
select {
|
||||
case <-caughtUp:
|
||||
catchingUp = false
|
||||
default:
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if journal != nil {
|
||||
journal.handle(line, now, catchingUp)
|
||||
}
|
||||
if deferred != nil {
|
||||
deferred.handle(line, now, catchingUp)
|
||||
}
|
||||
keeper.add(line, now, catchingUp)
|
||||
queueID, recipient, status, ok := parseDelivery(line)
|
||||
if !ok {
|
||||
return
|
||||
@@ -198,7 +235,10 @@ func reconcile(st StatusStore, cutoff time.Time) {
|
||||
}
|
||||
}
|
||||
|
||||
// retentionLoop prunes expired send-log rows immediately and then periodically.
|
||||
// retentionLoop prunes expired send-log rows immediately and then periodically,
|
||||
// and after each sweep the stored log lines no row carries the queue id of:
|
||||
// those the deleted rows leave behind, and those that never had a row
|
||||
// (lines.go).
|
||||
func retentionLoop(ctx context.Context, st StatusStore, retention RetentionDays) {
|
||||
prune := func() {
|
||||
retentionDays := retention()
|
||||
@@ -217,6 +257,7 @@ func retentionLoop(ctx context.Context, st StatusStore, retention RetentionDays)
|
||||
}
|
||||
|
||||
prune()
|
||||
pruneLines(st)
|
||||
t := time.NewTicker(retentionInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
@@ -225,6 +266,7 @@ func retentionLoop(ctx context.Context, st StatusStore, retention RetentionDays)
|
||||
return
|
||||
case <-t.C:
|
||||
prune()
|
||||
pruneLines(st)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -426,7 +468,11 @@ func SplitTimestamp(line string) (stamp, rest string) {
|
||||
// caughtUp is closed after the first read that reaches end-of-file, which is
|
||||
// the point where every line the panel missed while it was down has been
|
||||
// handled.
|
||||
func follow(ctx context.Context, path string, tr *tracker, caughtUp chan struct{}, handle func(string)) error {
|
||||
//
|
||||
// flush is called at the end of every pass over the file, before the read
|
||||
// position is saved: a handler that gathers lines to write them together
|
||||
// writes them there, so the saved position is never ahead of what was written.
|
||||
func follow(ctx context.Context, path string, tr *tracker, caughtUp chan struct{}, flush func(), handle func(string)) error {
|
||||
var (
|
||||
f *os.File
|
||||
r *bufio.Reader
|
||||
@@ -476,6 +522,7 @@ func follow(ctx context.Context, path string, tr *tracker, caughtUp chan struct{
|
||||
}()
|
||||
|
||||
drain := func() {
|
||||
defer flush()
|
||||
for {
|
||||
line, err := r.ReadString('\n')
|
||||
if err == io.EOF {
|
||||
|
||||
@@ -147,6 +147,58 @@ type captureStore struct {
|
||||
// queued is what ListQueuedOlderThan returns, for the reconcile sweep.
|
||||
queued []store.QueuedDelivery
|
||||
queuedErr error
|
||||
|
||||
// lines are the log lines kept for messages' pages; pruned is the cutoff of
|
||||
// each PruneSendLogLines call.
|
||||
lines []store.SendLogLine
|
||||
batches int
|
||||
pruned []time.Time
|
||||
}
|
||||
|
||||
// InsertSendLogLines keeps what the store would: with once, a line is not
|
||||
// kept a second time under its queue id. batches counts the calls.
|
||||
func (c *captureStore) InsertSendLogLines(lines []store.SendLogLine, once bool) (int, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.batches++
|
||||
wrote := 0
|
||||
next:
|
||||
for _, l := range lines {
|
||||
if once {
|
||||
for _, have := range c.lines {
|
||||
if have.QueueID == l.QueueID && have.Line == l.Line {
|
||||
continue next
|
||||
}
|
||||
}
|
||||
}
|
||||
c.lines = append(c.lines, l)
|
||||
wrote++
|
||||
}
|
||||
return wrote, nil
|
||||
}
|
||||
|
||||
func (c *captureStore) PruneSendLogLines(cutoff time.Time) (int64, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.pruned = append(c.pruned, cutoff)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// kept returns the stored lines as "queue-id|line".
|
||||
func (c *captureStore) kept() []string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
out := make([]string, len(c.lines))
|
||||
for i, l := range c.lines {
|
||||
out[i] = l.QueueID + "|" + l.Line
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (c *captureStore) prunes() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return len(c.pruned)
|
||||
}
|
||||
|
||||
func (c *captureStore) UpdateStatus(queueID, recipient, status string) (int64, error) {
|
||||
@@ -212,7 +264,7 @@ func TestFollowTailsAndRotates(t *testing.T) {
|
||||
cs := &captureStore{}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- Run(ctx, path, cs, func() int { return 90 }) }()
|
||||
go func() { done <- Run(ctx, path, cs, func() int { return 90 }, nil, nil) }()
|
||||
|
||||
// Give follow() time to open at EOF (it seeks to end immediately on start,
|
||||
// so the seed line above is ignored), then append a delivery line.
|
||||
@@ -295,7 +347,7 @@ func startRun(t *testing.T, path string, cs *captureStore) func() {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- Run(ctx, path, cs, func() int { return 90 }) }()
|
||||
go func() { done <- Run(ctx, path, cs, func() int { return 90 }, nil, nil) }()
|
||||
// follow() opens and seeks on start; give it a moment before the caller
|
||||
// appends, so the append is not raced by the initial open.
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
@@ -20,8 +20,10 @@ const (
|
||||
// which os.SameFile cannot answer across a restart.
|
||||
fingerprintSize = 512
|
||||
// persistInterval throttles the offset write. Losing up to this much
|
||||
// progress on a crash only means re-parsing a few lines (UpdateStatus is
|
||||
// idempotent), which is much cheaper than a database write per poll tick.
|
||||
// progress on a crash means re-parsing a few lines, which is much cheaper
|
||||
// than a database write per poll tick. For a delivery status that changes
|
||||
// nothing (UpdateStatus is idempotent), and a refusal among them is not
|
||||
// written to the inbound journal a second time (inboundJournal.handle).
|
||||
persistInterval = 5 * time.Second
|
||||
)
|
||||
|
||||
@@ -46,8 +48,9 @@ type tracker struct {
|
||||
// state whose fingerprint still matches means continue from it, parsing the
|
||||
// tail written while the panel was down; anything else means the file is not
|
||||
// the one the offset referred to (rotated, recreated or truncated in the
|
||||
// meantime), so read it from the start. Re-parsing lines already seen is
|
||||
// harmless: UpdateStatus writes the same status onto the same row.
|
||||
// meantime), so read it from the start. Re-parsing lines already seen writes
|
||||
// the same status onto the same send-log row, and no second inbound journal
|
||||
// row for a refusal among them.
|
||||
func (t *tracker) resume(f *os.File) int64 {
|
||||
size, err := fileSize(f)
|
||||
if err != nil {
|
||||
|
||||
@@ -30,6 +30,12 @@ func (s *retentionProbeStore) DeleteSendLogBefore(cutoff time.Time) (int64, erro
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (s *retentionProbeStore) InsertSendLogLines([]store.SendLogLine, bool) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (s *retentionProbeStore) PruneSendLogLines(time.Time) (int64, error) { return 0, nil }
|
||||
|
||||
func (s *retentionProbeStore) LogtailState(string) (store.LogtailState, bool, error) {
|
||||
return store.LogtailState{}, false, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package logtail
|
||||
|
||||
import (
|
||||
"log"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The sender check is attached to Postfix strictly: while the panel does not
|
||||
// answer on its socket, mail is deferred (docs/plans/from-header-binding.md).
|
||||
// How many submissions that happened to is read here, from what Postfix
|
||||
// writes about it, and shown on Server > Health. It is the figure that also
|
||||
// sees a panel which is running and not answering.
|
||||
//
|
||||
// Postfix says it in lines of one smtpd process - why, then what (seen on
|
||||
// the e2e stand, 2026-10-12):
|
||||
//
|
||||
// postfix/smtpd[1183]: warning: connect to Milter service unix:/run/selfpost/sender.sock: No such file or directory
|
||||
// postfix/smtpd[1183]: NOQUEUE: milter-reject: CONNECT from unknown[10.77.0.1]: 451 4.7.1 Service unavailable - try again later; proto=SMTP
|
||||
// postfix/smtpd[1183]: NOQUEUE: milter-reject: EHLO from unknown[10.77.0.1]: 451 4.7.1 Service unavailable - try again later; proto=SMTP helo=<app>
|
||||
//
|
||||
// and the client, told "454 4.3.0 Try again later" at AUTH, goes away without
|
||||
// a line of its own. So what is counted is the connection Postfix turned
|
||||
// away, once: a client brings a message when it connects, and it will bring
|
||||
// it again.
|
||||
//
|
||||
// It is counted at the warning, which is the one line that names the socket.
|
||||
// The reject lines do not: the same words are written when OpenDKIM does not
|
||||
// answer and when the journal milter refuses under a level-2 limit, and for a
|
||||
// check that stops answering in the middle of a message they are written by
|
||||
// cleanup, another process, under a queue id. The warning for that case is
|
||||
// "milter unix:<socket>: can't read … reply packet header" (Postfix's
|
||||
// milter8.c; not seen on the stand).
|
||||
|
||||
// SenderCheckStore is where the count goes, and where the time of the last
|
||||
// counted line is kept. *store.Store satisfies it.
|
||||
type SenderCheckStore interface {
|
||||
AddSenderCheckDeferred(at time.Time, n int64) error
|
||||
SenderCheckDeferredAt() (time.Time, error)
|
||||
}
|
||||
|
||||
// SenderCheck turns the counting on: the store and the socket Postfix was
|
||||
// given for the check (SENDER_CHECK_SOCKET), as a path.
|
||||
type SenderCheck struct {
|
||||
Store SenderCheckStore
|
||||
Socket string
|
||||
}
|
||||
|
||||
var smtpdLineRe = regexp.MustCompile(`^\S+ postfix/(?:[a-z0-9-]+/)?smtpd\[(\d+)\]: (.*)$`)
|
||||
|
||||
const (
|
||||
// senderCheckPids bounds the processes remembered; smtpd processes are a
|
||||
// few dozen, and a log that names more is not kept track of.
|
||||
senderCheckPids = 1024
|
||||
)
|
||||
|
||||
// deferredCounter is the per-run state of the count.
|
||||
type deferredCounter struct {
|
||||
st SenderCheckStore
|
||||
// The two warnings, up to and including the socket's name: nobody
|
||||
// listening, and a reply that did not come.
|
||||
connect, broken string
|
||||
failed map[string]bool // smtpd pid -> its current client was counted already
|
||||
// last is the time of the latest line counted, by this run or an earlier
|
||||
// one. While the tailer catches up after a start it may be reading lines
|
||||
// it had read before an unclean stop; a warning no later than this was
|
||||
// counted then. mail.log's stamps are whole seconds, so a warning of the
|
||||
// very second of the last one counted can be passed over - once, at a
|
||||
// restart - which is the cheaper mistake than counting a stretch twice.
|
||||
last time.Time
|
||||
}
|
||||
|
||||
func newDeferredCounter(sc *SenderCheck) *deferredCounter {
|
||||
last, err := sc.Store.SenderCheckDeferredAt()
|
||||
if err != nil {
|
||||
log.Printf("log-tailer: sender check statistics: %v", err)
|
||||
}
|
||||
return &deferredCounter{
|
||||
last: last,
|
||||
st: sc.Store,
|
||||
connect: "warning: connect to Milter service unix:" + sc.Socket + ":",
|
||||
broken: "warning: milter unix:" + sc.Socket + ":",
|
||||
failed: make(map[string]bool),
|
||||
}
|
||||
}
|
||||
|
||||
// handle looks at one mail.log line. catchingUp is true until the tailer has
|
||||
// read to the end of the log for the first time.
|
||||
func (c *deferredCounter) handle(line string, now time.Time, catchingUp bool) {
|
||||
at, rest := lineTime(line, now)
|
||||
m := smtpdLineRe.FindStringSubmatch(rest)
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
pid, text := m[1], m[2]
|
||||
switch {
|
||||
case strings.HasPrefix(text, "connect from "):
|
||||
delete(c.failed, pid)
|
||||
case strings.HasPrefix(text, c.connect) || strings.HasPrefix(text, c.broken):
|
||||
// Postfix's own two sentences, from their first word: a warning also
|
||||
// quotes what a client sent ("Illegal address syntax ... in MAIL
|
||||
// command: <text>"), and a client can send the socket's name.
|
||||
// Once for the connection, however many warnings it draws.
|
||||
if c.failed[pid] {
|
||||
return
|
||||
}
|
||||
if len(c.failed) >= senderCheckPids {
|
||||
c.failed = make(map[string]bool)
|
||||
}
|
||||
c.failed[pid] = true
|
||||
if catchingUp && !at.After(c.last) {
|
||||
return
|
||||
}
|
||||
if at.After(c.last) {
|
||||
c.last = at
|
||||
}
|
||||
if err := c.st.AddSenderCheckDeferred(at, 1); err != nil {
|
||||
log.Printf("log-tailer: sender check statistics: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package logtail
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type deferredStore struct {
|
||||
days map[string]int64
|
||||
last time.Time
|
||||
}
|
||||
|
||||
func (d *deferredStore) SenderCheckDeferredAt() (time.Time, error) { return d.last, nil }
|
||||
|
||||
func (d *deferredStore) AddSenderCheckDeferred(at time.Time, n int64) error {
|
||||
if d.days == nil {
|
||||
d.days = make(map[string]int64)
|
||||
}
|
||||
d.days[at.UTC().Format("2006-01-02")] += n
|
||||
if at.After(d.last) {
|
||||
d.last = at
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// The lines of process 1183 and of cleanup are as the e2e stand wrote them
|
||||
// with the panel stopped and for a refused From: (2026-10-12; the stamps are
|
||||
// put in ISO form). Around them are the lines of two other things that use
|
||||
// the same words - a level-2 refusal of the journal milter, OpenDKIM not
|
||||
// answering - and, composed from Postfix's source rather than seen, a check
|
||||
// that stops answering in the middle of a message.
|
||||
func TestDeferredCounter(t *testing.T) {
|
||||
st := &deferredStore{}
|
||||
c := newDeferredCounter(&SenderCheck{Store: st, Socket: "/run/selfpost/sender.sock"})
|
||||
now := time.Date(2026, 10, 12, 18, 0, 0, 0, time.UTC)
|
||||
for _, line := range []string{
|
||||
// The journal milter's own temporary refusal: no warning, not counted.
|
||||
`2026-10-12T16:22:13.100000+00:00 mail postfix/smtpd[1149]: connect from unknown[10.77.0.1]`,
|
||||
`2026-10-12T16:22:13.200000+00:00 mail postfix/smtpd[1149]: NOQUEUE: milter-reject: MAIL from unknown[10.77.0.1]: 451 4.7.1 Service unavailable - try again later; from=<billing@sender.e2e.test> proto=ESMTP helo=<e2e-client.e2e.test> sasl_username=<l2app>`,
|
||||
|
||||
// The check does not answer: one connection, counted once.
|
||||
`2026-10-12T16:22:57.000000+00:00 mail postfix/smtpd[1183]: connect from unknown[10.77.0.1]`,
|
||||
`2026-10-12T16:22:57.100000+00:00 mail postfix/smtpd[1183]: warning: connect to Milter service unix:/run/selfpost/sender.sock: No such file or directory`,
|
||||
`2026-10-12T16:22:57.200000+00:00 mail postfix/smtpd[1183]: NOQUEUE: milter-reject: CONNECT from unknown[10.77.0.1]: 451 4.7.1 Service unavailable - try again later; proto=SMTP`,
|
||||
`2026-10-12T16:22:57.300000+00:00 mail postfix/smtpd[1183]: NOQUEUE: milter-reject: EHLO from unknown[10.77.0.1]: 451 4.7.1 Service unavailable - try again later; proto=SMTP helo=<e2e-client.e2e.test>`,
|
||||
`2026-10-12T16:22:57.400000+00:00 mail postfix/smtpd[1183]: disconnect from unknown[10.77.0.1] ehlo=1 auth=0/1 quit=1 commands=2/3`,
|
||||
|
||||
// The same process takes another client, and the check answers again:
|
||||
// a level-2 refusal there is not the check's.
|
||||
`2026-10-12T16:23:30.000000+00:00 mail postfix/smtpd[1183]: connect from unknown[10.77.0.1]`,
|
||||
`2026-10-12T16:23:30.100000+00:00 mail postfix/smtpd[1183]: NOQUEUE: milter-reject: MAIL from unknown[10.77.0.1]: 451 4.7.1 Service unavailable - try again later; from=<a@sender.e2e.test> proto=ESMTP helo=<x>`,
|
||||
|
||||
// A client that writes the socket's name where Postfix will quote it
|
||||
// in a warning, and then runs into a level-2 refusal: not counted.
|
||||
`2026-10-12T16:23:40.000000+00:00 mail postfix/smtpd[1190]: connect from unknown[10.77.0.1]`,
|
||||
`2026-10-12T16:23:40.100000+00:00 mail postfix/smtpd[1190]: warning: Illegal address syntax from unknown[10.77.0.1] in MAIL command: <warning: connect to Milter service unix:/run/selfpost/sender.sock: milter>`,
|
||||
`2026-10-12T16:23:40.200000+00:00 mail postfix/smtpd[1190]: NOQUEUE: milter-reject: MAIL from unknown[10.77.0.1]: 451 4.7.1 Service unavailable - try again later; from=<a@sender.e2e.test> proto=ESMTP helo=<x>`,
|
||||
|
||||
// OpenDKIM does not answer: another socket, not counted.
|
||||
`2026-10-12T16:24:00.000000+00:00 mail postfix/smtpd[1201]: connect from unknown[10.77.0.1]`,
|
||||
`2026-10-12T16:24:00.100000+00:00 mail postfix/smtpd[1201]: warning: connect to Milter service unix:/run/opendkim/opendkim.sock: Connection refused`,
|
||||
`2026-10-12T16:24:00.200000+00:00 mail postfix/smtpd[1201]: NOQUEUE: milter-reject: CONNECT from unknown[10.77.0.1]: 451 4.7.1 Service unavailable - try again later; proto=SMTP`,
|
||||
|
||||
// The check stops answering in the middle of a message, the next day.
|
||||
`2026-10-13T00:00:05.000000+00:00 mail postfix/smtpd[1300]: connect from unknown[10.77.0.1]`,
|
||||
`2026-10-13T00:00:21.000000+00:00 mail postfix/smtpd[1300]: warning: milter unix:/run/selfpost/sender.sock: can't read SMFIC_EOH reply packet header: Connection timed out`,
|
||||
`2026-10-13T00:00:21.050000+00:00 mail postfix/smtpd[1300]: warning: milter unix:/run/selfpost/sender.sock: can't read SMFIC_BODYEOB reply packet header: Connection timed out`,
|
||||
`2026-10-13T00:00:21.100000+00:00 mail postfix/cleanup[1195]: 4XcB7k2Jm9z1: milter-reject: END-OF-MESSAGE from unknown[10.77.0.1]: 4.7.1 Service unavailable - try again later; from=<a@sender.e2e.test> to=<b@sink.e2e.test> proto=ESMTP helo=<x>`,
|
||||
|
||||
// The check's own verdict is a refusal, not a deferral, and it is
|
||||
// cleanup that writes it.
|
||||
`2026-10-13T00:01:00.000000+00:00 mail postfix/cleanup[1195]: B6B484831946: milter-reject: END-OF-MESSAGE from unknown[10.77.0.1]: 5.7.1 From: director@other.e2e.test - not an address this application sends as; from=<alerts@sender.e2e.test> to=<rcpt@sink.e2e.test> proto=ESMTP helo=<x>`,
|
||||
} {
|
||||
c.handle(line, now, false)
|
||||
}
|
||||
if len(st.days) != 2 || st.days["2026-10-12"] != 1 || st.days["2026-10-13"] != 1 {
|
||||
t.Fatalf("deferred per day = %v, want one on each of the two days", st.days)
|
||||
}
|
||||
}
|
||||
|
||||
// After an unclean stop the tailer reads again some lines it had read. What
|
||||
// was counted then is not counted now; what was written while the panel was
|
||||
// down - the very lines this count exists for - is.
|
||||
func TestDeferredCounterDoesNotCountALogReadAgain(t *testing.T) {
|
||||
st := &deferredStore{}
|
||||
now := time.Date(2026, 10, 12, 18, 0, 0, 0, time.UTC)
|
||||
outage := func(pid, at string) []string {
|
||||
return []string{
|
||||
at + `0+00:00 mail postfix/smtpd[` + pid + `]: connect from unknown[10.77.0.1]`,
|
||||
at + `1+00:00 mail postfix/smtpd[` + pid + `]: warning: connect to Milter service unix:/run/selfpost/sender.sock: Connection refused`,
|
||||
}
|
||||
}
|
||||
first := append(outage("11", "2026-10-12T16:00:00.00000"), outage("12", "2026-10-12T16:00:05.00000")...)
|
||||
|
||||
c := newDeferredCounter(&SenderCheck{Store: st, Socket: "/run/selfpost/sender.sock"})
|
||||
for _, line := range first {
|
||||
c.handle(line, now, false)
|
||||
}
|
||||
if st.days["2026-10-12"] != 2 {
|
||||
t.Fatalf("first run counted %d, want 2", st.days["2026-10-12"])
|
||||
}
|
||||
|
||||
// A new run: the same lines again, then what the outage added.
|
||||
c = newDeferredCounter(&SenderCheck{Store: st, Socket: "/run/selfpost/sender.sock"})
|
||||
for _, line := range append(first, outage("13", "2026-10-12T16:00:09.00000")...) {
|
||||
c.handle(line, now, true)
|
||||
}
|
||||
if st.days["2026-10-12"] != 3 {
|
||||
t.Fatalf("after the restart the day has %d, want 3", st.days["2026-10-12"])
|
||||
}
|
||||
}
|
||||
@@ -8,11 +8,12 @@ import (
|
||||
)
|
||||
|
||||
// authIPAllowed reports whether the authenticated application may submit from
|
||||
// the connecting client IP. When the application has no IP restriction, or the
|
||||
// client IP is not known, the check passes. Store errors are fail-open (see
|
||||
// overLimit).
|
||||
// the connecting client IP. When the application has no IP restriction the
|
||||
// check passes. When it has one and the client's address is not known, the
|
||||
// check fails: an address list that an unknown address satisfies is no list.
|
||||
// Store errors are fail-open (see overLimit).
|
||||
func (s *session) authIPAllowed() bool {
|
||||
if s.login == "" || s.clientIP == "" {
|
||||
if s.login == "" {
|
||||
return true
|
||||
}
|
||||
a, err := s.rec.ApplicationByLogin(s.login)
|
||||
@@ -26,6 +27,10 @@ func (s *session) authIPAllowed() bool {
|
||||
if a.AllowsAuthFromIP(s.clientIP) {
|
||||
return true
|
||||
}
|
||||
if s.clientIP == "" {
|
||||
log.Printf("journal-milter: application %q refused — its client IPs are restricted and Postfix gave no client address", s.login)
|
||||
return false
|
||||
}
|
||||
log.Printf("journal-milter: application %q refused from %s — client IP not allowed", s.login, s.clientIP)
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
package milter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"log"
|
||||
"net"
|
||||
"net/textproto"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/emersion/go-milter"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/mailhdr"
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
|
||||
// InboundStore is what the inbound journal milter needs: the journal itself
|
||||
// and the list of inbound domains, since port 25 also takes mail that is not
|
||||
// the relay's. *store.Store satisfies it; tests substitute a fake.
|
||||
type InboundStore interface {
|
||||
InsertInboundSpam(e store.InboundSpamEntry) error
|
||||
IsInboundDomain(name string) (bool, error)
|
||||
}
|
||||
|
||||
// inboundSession is the inbound journal milter: the last milter on port 25,
|
||||
// after the optional external filter. It writes one journal row per recipient
|
||||
// of a message that reached end-of-message - that is, one the filter did not
|
||||
// refuse; refusals never get this far and are read from mail.log instead.
|
||||
//
|
||||
// Like the send-log milter it only watches: every callback continues, and
|
||||
// Postfix attaches the socket with default_action=accept, so
|
||||
// neither an error here nor the panel being down holds up inbound mail.
|
||||
type inboundSession struct {
|
||||
milter.NoOpMilter
|
||||
rec InboundStore
|
||||
// key is what the filter signs each message with (FilterSignature), or
|
||||
// empty when no filter is attached. A verdict is read only next to a
|
||||
// signature that fits: the filter is attached so that mail flows when it is
|
||||
// down, and then the verdict headers of a message are whatever its sender
|
||||
// wrote.
|
||||
key string
|
||||
|
||||
// clientIP is taken at Connect and again from {client_addr} with every
|
||||
// MAIL FROM: a session the library made after a non-continue reply never
|
||||
// sees Connect (see session.MailFrom in milter.go).
|
||||
clientIP string
|
||||
|
||||
from string
|
||||
rcpts []string
|
||||
subject string
|
||||
action string // X-Rspamd-Action
|
||||
result string // X-Spamd-Result
|
||||
seen bool // any verdict header of the filter
|
||||
sig string // X-SelfPost-Filter
|
||||
}
|
||||
|
||||
func (s *inboundSession) Connect(host, family string, port uint16, addr net.IP, m *milter.Modifier) (milter.Response, error) {
|
||||
if addr != nil {
|
||||
s.clientIP = addr.String()
|
||||
}
|
||||
return milter.RespContinue, nil
|
||||
}
|
||||
|
||||
// MailFrom starts a message: a connection may carry several.
|
||||
func (s *inboundSession) MailFrom(from string, m *milter.Modifier) (milter.Response, error) {
|
||||
s.reset()
|
||||
if ip := clientAddr(m); ip != "" {
|
||||
s.clientIP = ip
|
||||
}
|
||||
s.from = cleanAddress(from)
|
||||
return milter.RespContinue, nil
|
||||
}
|
||||
|
||||
func (s *inboundSession) RcptTo(rcpt string, m *milter.Modifier) (milter.Response, error) {
|
||||
s.rcpts = append(s.rcpts, cleanAddress(rcpt))
|
||||
return milter.RespContinue, nil
|
||||
}
|
||||
|
||||
// Header keeps the first Subject, the filter's verdict and its signature.
|
||||
// Postfix hands the message to its milters one after another at
|
||||
// end-of-message, each seeing what the one before added, which is how the
|
||||
// filter's headers arrive here. The filter replaces headers of these names, so
|
||||
// the last one read is its own - when it is up. Whether it was is decided at
|
||||
// end-of-message, by the signature.
|
||||
func (s *inboundSession) Header(name, value string, m *milter.Modifier) (milter.Response, error) {
|
||||
switch textproto.CanonicalMIMEHeaderKey(name) {
|
||||
case "Subject":
|
||||
if s.subject == "" {
|
||||
s.subject = mailhdr.DecodeSubject(value)
|
||||
}
|
||||
case "X-Rspamd-Action":
|
||||
s.action, s.seen = value, true
|
||||
case "X-Spamd-Result":
|
||||
s.result, s.seen = value, true
|
||||
case signatureHeader:
|
||||
s.sig = strings.TrimSpace(value)
|
||||
}
|
||||
return milter.RespContinue, nil
|
||||
}
|
||||
|
||||
// SignatureHeader is the header the filter signs a message in
|
||||
// (deploy/antispam/rspamd/milter_headers.conf writes it).
|
||||
const SignatureHeader = "X-SelfPost-Filter"
|
||||
|
||||
var signatureHeader = textproto.CanonicalMIMEHeaderKey(SignatureHeader)
|
||||
|
||||
// FilterSignature is what the filter writes into SignatureHeader for a message:
|
||||
// the HMAC-SHA256 of its queue id under the key SelfPost shares with the
|
||||
// filter, in hex. The key is used as the text it is stored as. A signature
|
||||
// fits one queue id, so the header going on to the upstream with the mail
|
||||
// gives nothing to whoever reads it there.
|
||||
func FilterSignature(key, queueID string) string {
|
||||
mac := hmac.New(sha256.New, []byte(key))
|
||||
mac.Write([]byte(queueID))
|
||||
return hex.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
// signed reports whether the message carries the filter's signature for its
|
||||
// queue id. No key, no queue id or no signature is "no".
|
||||
func (s *inboundSession) signed(queueID string) bool {
|
||||
if s.key == "" || queueID == "" || s.sig == "" {
|
||||
return false
|
||||
}
|
||||
return hmac.Equal([]byte(strings.ToLower(s.sig)), []byte(FilterSignature(s.key, queueID)))
|
||||
}
|
||||
|
||||
// Body fires at end-of-message: the message is about to be queued for its
|
||||
// upstream. "Continue" lets it through as "accept" would, and keeps this
|
||||
// session for the connection's next message.
|
||||
func (s *inboundSession) Body(m *milter.Modifier) (milter.Response, error) {
|
||||
// The queue id is known only now, and with it whether the verdict headers
|
||||
// read on the way here are the filter's.
|
||||
if !s.signed(macro(m, "i")) {
|
||||
s.seen = false
|
||||
}
|
||||
s.record()
|
||||
s.reset()
|
||||
return milter.RespContinue, nil
|
||||
}
|
||||
|
||||
func (s *inboundSession) Abort(m *milter.Modifier) error {
|
||||
s.reset()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *inboundSession) reset() {
|
||||
s.from, s.rcpts, s.subject = "", nil, ""
|
||||
s.action, s.result, s.seen, s.sig = "", "", false, ""
|
||||
}
|
||||
|
||||
// record writes one row per recipient at an inbound domain. Other recipients
|
||||
// of port 25 (a DMARC report address at the server's own name) are not the
|
||||
// relay's mail and get no row. Failures are logged, never propagated.
|
||||
func (s *inboundSession) record() {
|
||||
e := store.InboundSpamEntry{
|
||||
ClientIP: s.clientIP,
|
||||
From: s.from,
|
||||
Subject: s.subject,
|
||||
Decision: store.SpamDecisionAccept,
|
||||
Engine: store.SpamEngineSelfPost,
|
||||
}
|
||||
if s.seen {
|
||||
action, detail := rspamdVerdict(s.action, s.result)
|
||||
e.Engine, e.Detail = store.SpamEngineRspamd, detail
|
||||
if action == "quarantine" {
|
||||
e.Decision = store.SpamDecisionQuarantine
|
||||
}
|
||||
}
|
||||
for _, to := range s.rcpts {
|
||||
domain := domainOf(to)
|
||||
ok, err := s.rec.IsInboundDomain(domain)
|
||||
if err != nil {
|
||||
log.Printf("inbound-journal: look up %s: %v", domain, err)
|
||||
continue
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
e.InboundDomain, e.To = domain, to
|
||||
if err := s.rec.InsertInboundSpam(e); err != nil {
|
||||
log.Printf("inbound-journal: record %s -> %s: %v", s.from, to, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the journal keeps of a verdict: enough to see why, on one line.
|
||||
const (
|
||||
verdictMaxSymbols = 6
|
||||
verdictMaxBytes = 255
|
||||
)
|
||||
|
||||
// ListSymbolPrefix starts the names of the symbols the filter lists' rules add
|
||||
// (deploy/antispam/rspamd/multimap.conf): SELFPOST_ALLOW_IP, _FROM, _DOMAIN.
|
||||
const ListSymbolPrefix = "SELFPOST_"
|
||||
|
||||
var (
|
||||
// "default: False [7.20 / 15.00]" - the score and the reject threshold.
|
||||
rspamdScore = regexp.MustCompile(`\[(-?[0-9]+(?:\.[0-9]+)?) / (-?[0-9]+(?:\.[0-9]+)?)\]`)
|
||||
// "BAYES_SPAM(5.10)[100.00%]" - a symbol and its weight.
|
||||
rspamdSymbol = regexp.MustCompile(`([A-Z][A-Z0-9_]{1,63})\((-?[0-9]+(?:\.[0-9]+)?)\)`)
|
||||
)
|
||||
|
||||
// rspamdVerdict reduces the filter's headers to an action and a one-line
|
||||
// summary: the action, the score against the threshold, and the first few
|
||||
// symbols that carried weight. It reads only what it recognises - the headers
|
||||
// are read only next to the filter's signature, and even so nothing of them is
|
||||
// kept verbatim except through these patterns.
|
||||
func rspamdVerdict(actionHeader, resultHeader string) (action, detail string) {
|
||||
action = strings.ToLower(strings.TrimSpace(actionHeader))
|
||||
for _, r := range action {
|
||||
if (r < 'a' || r > 'z') && r != ' ' {
|
||||
action = ""
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(action) > 32 {
|
||||
action = ""
|
||||
}
|
||||
|
||||
var parts []string
|
||||
if action != "" {
|
||||
parts = append(parts, action)
|
||||
}
|
||||
if m := rspamdScore.FindStringSubmatch(resultHeader); m != nil {
|
||||
parts = append(parts, "score "+m[1]+" / "+m[2])
|
||||
}
|
||||
detail = strings.Join(parts, ", ")
|
||||
|
||||
// The filter lists' own symbols come first and are kept whatever else there
|
||||
// is: Filter lists counts how often an entry decided by finding them here,
|
||||
// and rspamd writes symbols heaviest first, which puts an allow entry's
|
||||
// -100 last, behind any six others.
|
||||
var symbols, rest []string
|
||||
for _, m := range rspamdSymbol.FindAllStringSubmatch(resultHeader, -1) {
|
||||
if strings.Trim(m[2], "-0.") == "" { // weight zero: informational
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(m[1], ListSymbolPrefix) {
|
||||
if len(symbols) < verdictMaxSymbols {
|
||||
symbols = append(symbols, m[1]+"("+m[2]+")")
|
||||
}
|
||||
continue
|
||||
}
|
||||
rest = append(rest, m[1]+"("+m[2]+")")
|
||||
}
|
||||
for _, sym := range rest {
|
||||
if len(symbols) == verdictMaxSymbols {
|
||||
break
|
||||
}
|
||||
symbols = append(symbols, sym)
|
||||
}
|
||||
if len(symbols) > 0 {
|
||||
if detail != "" {
|
||||
detail += "; "
|
||||
}
|
||||
detail += strings.Join(symbols, " ")
|
||||
}
|
||||
if len(detail) > verdictMaxBytes {
|
||||
detail = detail[:verdictMaxBytes]
|
||||
}
|
||||
return action, detail
|
||||
}
|
||||
|
||||
// ServeInbound runs the inbound journal milter on ln until ctx is cancelled.
|
||||
// key is what the external filter in front of it signs messages with, or
|
||||
// empty when none is attached (INBOUND_ANTISPAM_MILTER): no verdict is then
|
||||
// read at all. It returns nil on a clean shutdown.
|
||||
func ServeInbound(ctx context.Context, ln net.Listener, rec InboundStore, key string) error {
|
||||
srv := &milter.Server{
|
||||
NewMilter: func() milter.Milter { return &inboundSession{rec: rec, key: key} },
|
||||
Actions: 0, // read-only
|
||||
Protocol: milter.OptNoBody, // headers and end-of-message, not the body
|
||||
}
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
_ = srv.Close()
|
||||
}()
|
||||
|
||||
if err := srv.Serve(ln); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
package milter
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/emersion/go-milter"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
|
||||
// fakeInbound is the journal and the inbound-domain list of one test.
|
||||
type fakeInbound struct {
|
||||
domains map[string]bool
|
||||
rows []store.InboundSpamEntry
|
||||
insertErr error
|
||||
lookupErr error
|
||||
}
|
||||
|
||||
func (f *fakeInbound) InsertInboundSpam(e store.InboundSpamEntry) error {
|
||||
if f.insertErr != nil {
|
||||
return f.insertErr
|
||||
}
|
||||
f.rows = append(f.rows, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeInbound) IsInboundDomain(name string) (bool, error) {
|
||||
if f.lookupErr != nil {
|
||||
return false, f.lookupErr
|
||||
}
|
||||
return f.domains[name], nil
|
||||
}
|
||||
|
||||
// A header of the kind the documented rspamd sidecar adds.
|
||||
const spamdResult = "default: False [7.20 / 15.00];\n\tBAYES_SPAM(5.10)[100.00%];\n\tMIME_GOOD(-0.10)[text/plain];\n\tARC_NA(0.00)[];\n\tRCVD_COUNT_ZERO(0.00)[0];\n\tR_DKIM_NA(2.20)[]"
|
||||
|
||||
// message replays one message through s: envelope, headers in order, end.
|
||||
func message(t *testing.T, s *inboundSession, from string, rcpts []string, headers [][2]string) milter.Response {
|
||||
t.Helper()
|
||||
if _, err := s.MailFrom(from, mods(nil)); err != nil {
|
||||
t.Fatalf("MailFrom: %v", err)
|
||||
}
|
||||
for _, r := range rcpts {
|
||||
if _, err := s.RcptTo(r, mods(nil)); err != nil {
|
||||
t.Fatalf("RcptTo: %v", err)
|
||||
}
|
||||
}
|
||||
for _, h := range headers {
|
||||
if _, err := s.Header(h[0], h[1], mods(nil)); err != nil {
|
||||
t.Fatalf("Header: %v", err)
|
||||
}
|
||||
}
|
||||
resp, err := s.Body(mods(map[string]string{"i": "ABC123"}))
|
||||
if err != nil {
|
||||
t.Fatalf("Body: %v", err)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
// testKey is the key the tests' filter signs with; the messages they replay
|
||||
// have the queue id ABC123.
|
||||
const testKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
|
||||
var testSig = [2]string{SignatureHeader, FilterSignature(testKey, "ABC123")}
|
||||
|
||||
func inboundSessionFor(t *testing.T, rec InboundStore, filter bool) *inboundSession {
|
||||
t.Helper()
|
||||
s := &inboundSession{rec: rec}
|
||||
if filter {
|
||||
s.key = testKey
|
||||
}
|
||||
if _, err := s.Connect("mx.example.net", "tcp4", 0, net.ParseIP("203.0.113.7"), mods(nil)); err != nil {
|
||||
t.Fatalf("Connect: %v", err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// With no filter attached an accepted message is a plain row, one per
|
||||
// recipient, decided by nobody.
|
||||
func TestInboundRecordsAcceptedMessage(t *testing.T) {
|
||||
rec := &fakeInbound{domains: map[string]bool{"in.example.com": true}}
|
||||
s := inboundSessionFor(t, rec, false)
|
||||
resp := message(t, s, "<sender@example.net> SIZE=1200",
|
||||
[]string{"<a@in.example.com>", "b@IN.example.com"},
|
||||
[][2]string{{"Subject", "=?utf-8?Q?Sch=C3=B6n?="}})
|
||||
if resp != milter.RespContinue {
|
||||
t.Errorf("end-of-message response = %v, want continue", resp)
|
||||
}
|
||||
if len(rec.rows) != 2 {
|
||||
t.Fatalf("rows = %+v, want one per recipient", rec.rows)
|
||||
}
|
||||
want := store.InboundSpamEntry{
|
||||
InboundDomain: "in.example.com", ClientIP: "203.0.113.7",
|
||||
From: "sender@example.net", To: "a@in.example.com", Subject: "Schön",
|
||||
Decision: store.SpamDecisionAccept, Engine: store.SpamEngineSelfPost,
|
||||
}
|
||||
if rec.rows[0] != want {
|
||||
t.Errorf("row = %+v\nwant %+v", rec.rows[0], want)
|
||||
}
|
||||
if rec.rows[1].To != "b@IN.example.com" || rec.rows[1].InboundDomain != "in.example.com" {
|
||||
t.Errorf("second row = %+v", rec.rows[1])
|
||||
}
|
||||
}
|
||||
|
||||
// Port 25 also takes DMARC reports for the server's own name. They are not
|
||||
// the relay's mail and the journal says nothing about them.
|
||||
func TestInboundSkipsRecipientsOutsideInboundDomains(t *testing.T) {
|
||||
rec := &fakeInbound{domains: map[string]bool{"in.example.com": true}}
|
||||
s := inboundSessionFor(t, rec, false)
|
||||
message(t, s, "noreply-dmarc@google.com",
|
||||
[]string{"dmarc-example.com@mail.example.org", "a@in.example.com"}, nil)
|
||||
if len(rec.rows) != 1 || rec.rows[0].To != "a@in.example.com" {
|
||||
t.Errorf("rows = %+v, want the inbound recipient alone", rec.rows)
|
||||
}
|
||||
}
|
||||
|
||||
// With the filter attached its verdict headers become the engine and detail.
|
||||
func TestInboundReadsTheFiltersVerdict(t *testing.T) {
|
||||
rec := &fakeInbound{domains: map[string]bool{"in.example.com": true}}
|
||||
s := inboundSessionFor(t, rec, true)
|
||||
message(t, s, "sender@example.net", []string{"a@in.example.com"}, [][2]string{
|
||||
{"Subject", "Cheap pills"},
|
||||
{"X-Rspamd-Action", "add header"},
|
||||
{"X-Spamd-Result", spamdResult},
|
||||
testSig,
|
||||
})
|
||||
if len(rec.rows) != 1 {
|
||||
t.Fatalf("rows = %+v", rec.rows)
|
||||
}
|
||||
got := rec.rows[0]
|
||||
if got.Engine != store.SpamEngineRspamd || got.Decision != store.SpamDecisionAccept {
|
||||
t.Errorf("engine %q, decision %q", got.Engine, got.Decision)
|
||||
}
|
||||
if want := "add header, score 7.20 / 15.00; BAYES_SPAM(5.10) MIME_GOOD(-0.10) R_DKIM_NA(2.20)"; got.Detail != want {
|
||||
t.Errorf("detail = %q\nwant %q", got.Detail, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInboundQuarantineAction(t *testing.T) {
|
||||
rec := &fakeInbound{domains: map[string]bool{"in.example.com": true}}
|
||||
s := inboundSessionFor(t, rec, true)
|
||||
message(t, s, "sender@example.net", []string{"a@in.example.com"},
|
||||
[][2]string{{"x-rspamd-action", "quarantine"}, {"x-selfpost-filter", strings.ToUpper(testSig[1])}})
|
||||
if len(rec.rows) != 1 || rec.rows[0].Decision != store.SpamDecisionQuarantine || rec.rows[0].Detail != "quarantine" {
|
||||
t.Errorf("rows = %+v", rec.rows)
|
||||
}
|
||||
}
|
||||
|
||||
// Without a filter in front, a verdict header is the sender's own writing.
|
||||
func TestInboundIgnoresVerdictHeadersWithoutAFilter(t *testing.T) {
|
||||
rec := &fakeInbound{domains: map[string]bool{"in.example.com": true}}
|
||||
s := inboundSessionFor(t, rec, false)
|
||||
message(t, s, "sender@example.net", []string{"a@in.example.com"}, [][2]string{
|
||||
{"X-Rspamd-Action", "no action"},
|
||||
{"X-Spamd-Result", spamdResult},
|
||||
})
|
||||
if len(rec.rows) != 1 || rec.rows[0].Engine != store.SpamEngineSelfPost || rec.rows[0].Detail != "" {
|
||||
t.Errorf("rows = %+v", rec.rows)
|
||||
}
|
||||
}
|
||||
|
||||
// The filter is attached fail-open, so with it down the headers are again the
|
||||
// sender's: nothing of them reaches the journal except through the patterns.
|
||||
func TestRspamdVerdictKeepsOnlyWhatItRecognises(t *testing.T) {
|
||||
action, detail := rspamdVerdict("<script>alert(1)</script>", "default: False [1 / 15]; <b>HTML</b> lower_case(9.9) \x1b[31m")
|
||||
if action != "" {
|
||||
t.Errorf("action = %q, want it dropped", action)
|
||||
}
|
||||
if detail != "score 1 / 15" {
|
||||
t.Errorf("detail = %q", detail)
|
||||
}
|
||||
|
||||
var many strings.Builder
|
||||
for i := 0; i < 400; i++ {
|
||||
many.WriteString("SYMBOL_WITH_A_LONG_NAME_" + strings.Repeat("X", 30) + "(1.00)[];")
|
||||
}
|
||||
if _, detail := rspamdVerdict("reject", many.String()); len(detail) > verdictMaxBytes || strings.Count(detail, "(") > verdictMaxSymbols {
|
||||
t.Errorf("detail of %d bytes with %d symbols", len(detail), strings.Count(detail, "("))
|
||||
}
|
||||
if action, _ := rspamdVerdict(strings.Repeat("a", 33), ""); action != "" {
|
||||
t.Errorf("an action of 33 letters was kept")
|
||||
}
|
||||
}
|
||||
|
||||
// A second message on the same connection starts clean.
|
||||
func TestInboundResetsBetweenMessages(t *testing.T) {
|
||||
rec := &fakeInbound{domains: map[string]bool{"in.example.com": true}}
|
||||
s := inboundSessionFor(t, rec, true)
|
||||
message(t, s, "one@example.net", []string{"a@in.example.com"}, [][2]string{
|
||||
{"Subject", "First"}, {"X-Rspamd-Action", "add header"}, testSig,
|
||||
})
|
||||
message(t, s, "two@example.net", []string{"b@in.example.com"}, nil)
|
||||
if len(rec.rows) != 2 {
|
||||
t.Fatalf("rows = %+v", rec.rows)
|
||||
}
|
||||
want := store.InboundSpamEntry{
|
||||
InboundDomain: "in.example.com", ClientIP: "203.0.113.7",
|
||||
From: "two@example.net", To: "b@in.example.com",
|
||||
Decision: store.SpamDecisionAccept, Engine: store.SpamEngineSelfPost,
|
||||
}
|
||||
if rec.rows[1] != want {
|
||||
t.Errorf("second row = %+v\nwant %+v", rec.rows[1], want)
|
||||
}
|
||||
|
||||
// An aborted transaction leaves nothing behind for the next one either.
|
||||
if _, err := s.MailFrom("three@example.net", mods(nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.RcptTo("c@in.example.com", mods(nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Abort(mods(nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rec.rows) != 2 || len(s.rcpts) != 0 {
|
||||
t.Errorf("after an abort: rows %d, pending recipients %v", len(rec.rows), s.rcpts)
|
||||
}
|
||||
}
|
||||
|
||||
// The journal failing is never the mail's problem.
|
||||
func TestInboundAcceptsWhenTheStoreFails(t *testing.T) {
|
||||
for name, rec := range map[string]*fakeInbound{
|
||||
"insert": {domains: map[string]bool{"in.example.com": true}, insertErr: errors.New("boom")},
|
||||
"lookup": {lookupErr: errors.New("boom")},
|
||||
} {
|
||||
s := inboundSessionFor(t, rec, false)
|
||||
if resp := message(t, s, "sender@example.net", []string{"a@in.example.com"}, nil); resp != milter.RespContinue {
|
||||
t.Errorf("%s failure: response = %v, want continue", name, resp)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rspamd writes symbols heaviest first, so an allow entry's -100 comes last.
|
||||
// Filter lists counts an entry's matches by finding that symbol in the journal:
|
||||
// it is kept whatever else there is.
|
||||
func TestRspamdVerdictKeepsTheListSymbols(t *testing.T) {
|
||||
result := "default: False [-91.00 / 15.00]; SYM_A(3.00)[]; SYM_B(2.00)[]; SYM_C(1.50)[]; SYM_D(1.00)[]; SYM_E(0.80)[]; SYM_F(0.50)[]; SYM_G(0.20)[]; ARC_NA(0.00)[]; SELFPOST_ALLOW_FROM(-100.00)[billing@vendor.example]"
|
||||
_, detail := rspamdVerdict("no action", result)
|
||||
want := "no action, score -91.00 / 15.00; SELFPOST_ALLOW_FROM(-100.00) SYM_A(3.00) SYM_B(2.00) SYM_C(1.50) SYM_D(1.00) SYM_E(0.80)"
|
||||
if detail != want {
|
||||
t.Errorf("detail = %q\nwant %q", detail, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The client address of a connection's later messages comes from the MAIL
|
||||
// macro: a session the library made after a non-continue reply saw no Connect.
|
||||
func TestInboundClientAddressFromTheMailMacro(t *testing.T) {
|
||||
rec := &fakeInbound{domains: map[string]bool{"in.example.com": true}}
|
||||
s := &inboundSession{rec: rec} // no Connect
|
||||
if _, err := s.MailFrom("sender@example.net", mods(map[string]string{"{client_addr}": "IPv6:2001:db8::7"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.RcptTo("a@in.example.com", mods(nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Body(mods(nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rec.rows) != 1 || rec.rows[0].ClientIP != "2001:db8::7" {
|
||||
t.Errorf("rows = %+v", rec.rows)
|
||||
}
|
||||
}
|
||||
|
||||
// No callback of the inbound milter answers anything but "continue": it must
|
||||
// never be the reason a message is held up, and a non-continue reply would
|
||||
// cost it its session.
|
||||
func TestInboundOnlyEverContinues(t *testing.T) {
|
||||
rec := &fakeInbound{domains: map[string]bool{"in.example.com": true}}
|
||||
s := &inboundSession{rec: rec, key: testKey}
|
||||
check := func(stage string, resp milter.Response, err error) {
|
||||
t.Helper()
|
||||
if err != nil || resp != milter.RespContinue {
|
||||
t.Errorf("%s: %v, %v", stage, resp, err)
|
||||
}
|
||||
}
|
||||
resp, err := s.Connect("mx", "tcp4", 0, net.ParseIP("203.0.113.7"), mods(nil))
|
||||
check("connect", resp, err)
|
||||
resp, err = s.MailFrom("a@example.net", mods(nil))
|
||||
check("mail", resp, err)
|
||||
resp, err = s.RcptTo("a@in.example.com", mods(nil))
|
||||
check("rcpt", resp, err)
|
||||
resp, err = s.Header("X-Rspamd-Action", "reject", mods(nil))
|
||||
check("header", resp, err)
|
||||
resp, err = s.Body(mods(nil))
|
||||
check("end of message", resp, err)
|
||||
}
|
||||
|
||||
// The filter is attached so that mail flows when it is down, and then the
|
||||
// verdict headers of a message are its sender's own. A verdict is recorded only
|
||||
// next to the filter's signature for this message's queue id.
|
||||
func TestInboundReadsAVerdictOnlyWithTheFiltersSignature(t *testing.T) {
|
||||
verdict := [][2]string{{"X-Rspamd-Action", "quarantine"}, {"X-Spamd-Result", spamdResult}}
|
||||
for name, c := range map[string]struct {
|
||||
extra [][2]string
|
||||
signed bool
|
||||
}{
|
||||
"signed": {[][2]string{testSig}, true},
|
||||
"no signature": {nil, false},
|
||||
"a made-up signature": {[][2]string{{SignatureHeader, strings.Repeat("ab", 32)}}, false},
|
||||
"another message's signature": {[][2]string{{SignatureHeader, FilterSignature(testKey, "OTHER1")}}, false},
|
||||
"a signature under another key": {[][2]string{{SignatureHeader, FilterSignature(strings.Repeat("f", 64), "ABC123")}}, false},
|
||||
"the sender's first, then the real": {[][2]string{{SignatureHeader, "forged"}, testSig}, true},
|
||||
} {
|
||||
rec := &fakeInbound{domains: map[string]bool{"in.example.com": true}}
|
||||
s := inboundSessionFor(t, rec, true)
|
||||
message(t, s, "sender@example.net", []string{"a@in.example.com"}, append(append([][2]string{}, verdict...), c.extra...))
|
||||
if len(rec.rows) != 1 {
|
||||
t.Fatalf("%s: rows = %+v", name, rec.rows)
|
||||
}
|
||||
got := rec.rows[0]
|
||||
if c.signed && (got.Engine != store.SpamEngineRspamd || got.Decision != store.SpamDecisionQuarantine) {
|
||||
t.Errorf("%s: a signed verdict was not read: %+v", name, got)
|
||||
}
|
||||
if !c.signed && (got.Engine != store.SpamEngineSelfPost || got.Decision != store.SpamDecisionAccept || got.Detail != "") {
|
||||
t.Errorf("%s: an unsigned verdict was recorded: %+v", name, got)
|
||||
}
|
||||
}
|
||||
|
||||
// A signature is of no use to the next message on the connection.
|
||||
rec := &fakeInbound{domains: map[string]bool{"in.example.com": true}}
|
||||
s := inboundSessionFor(t, rec, true)
|
||||
message(t, s, "one@example.net", []string{"a@in.example.com"}, append(append([][2]string{}, verdict...), testSig))
|
||||
message(t, s, "two@example.net", []string{"a@in.example.com"}, verdict)
|
||||
if len(rec.rows) != 2 || rec.rows[1].Engine != store.SpamEngineSelfPost {
|
||||
t.Errorf("rows = %+v", rec.rows)
|
||||
}
|
||||
|
||||
// With no queue id nothing can be checked, so nothing is trusted.
|
||||
rec = &fakeInbound{domains: map[string]bool{"in.example.com": true}}
|
||||
s = inboundSessionFor(t, rec, true)
|
||||
_, _ = s.MailFrom("sender@example.net", mods(nil))
|
||||
_, _ = s.RcptTo("a@in.example.com", mods(nil))
|
||||
_, _ = s.Header("X-Rspamd-Action", "reject", mods(nil))
|
||||
_, _ = s.Header(SignatureHeader, FilterSignature(testKey, ""), mods(nil))
|
||||
_, _ = s.Body(mods(nil))
|
||||
if len(rec.rows) != 1 || rec.rows[0].Engine != store.SpamEngineSelfPost {
|
||||
t.Errorf("no queue id: rows = %+v", rec.rows)
|
||||
}
|
||||
}
|
||||
|
||||
// What rspamd 3.11 wrote for this key and queue id in a throwaway container
|
||||
// (dev trial, 2026-10-12): the two ends must compute the same thing.
|
||||
func TestFilterSignatureIsWhatTheFilterWrites(t *testing.T) {
|
||||
if got := FilterSignature(testKey, "4A1B2C3D5E"); got != "2aa1ca6dd1545d3f16c8c0c29ba59bfc28c050224ac2c5c953bd2617f73b6de2" {
|
||||
t.Errorf("FilterSignature = %s", got)
|
||||
}
|
||||
}
|
||||
@@ -53,7 +53,11 @@ type session struct {
|
||||
// valid zero value (no in-flight accounting).
|
||||
flight *inflight
|
||||
|
||||
clientIP string // captured once per connection
|
||||
// clientIP is the client's address. Postfix gives it at Connect, once per
|
||||
// connection, and again with every MAIL FROM as the {client_addr} macro
|
||||
// (milter_mail_macros in build/postfix-config.sh). The second is the one
|
||||
// that can be relied on: see MailFrom.
|
||||
clientIP string
|
||||
|
||||
login string
|
||||
from string
|
||||
@@ -62,9 +66,8 @@ type session struct {
|
||||
reserved []*reservation // level-2 slots held by the current message
|
||||
}
|
||||
|
||||
// Connect captures the client IP, which comes from the addr parameter rather
|
||||
// than a macro (the {client_addr} macro was empty in testing). It is the
|
||||
// rate-limit key; here it is recorded for completeness.
|
||||
// Connect captures the client IP of a connection's first message. It is not
|
||||
// enough on its own - see MailFrom.
|
||||
func (s *session) Connect(host, family string, port uint16, addr net.IP, m *milter.Modifier) (milter.Response, error) {
|
||||
if addr != nil {
|
||||
s.clientIP = addr.String()
|
||||
@@ -79,18 +82,31 @@ func (s *session) Connect(host, family string, port uint16, addr net.IP, m *milt
|
||||
// limit is enforced here: over the limit, the message is refused with a 4xx
|
||||
// tempfail before recipients are even offered (guide § Rate limiting).
|
||||
// Enforcement is fail-open — see overLimit.
|
||||
//
|
||||
// The client address is read again here, from the {client_addr} macro. The
|
||||
// milter library replaces the session with a new one after every reply that is
|
||||
// not "continue" - a refusal, or the accept that used to end a message - and
|
||||
// Postfix does not send Connect again on the same connection. A session made
|
||||
// that way never saw the client, so every message of a connection after the
|
||||
// first, and every MAIL FROM after a refused one, used to be judged with no
|
||||
// client address: the level-2 limit and an application's client-address list
|
||||
// both let it through. (Found on 2026-10-12; test/e2e sends two MAIL FROM on
|
||||
// one connection.)
|
||||
func (s *session) MailFrom(from string, m *milter.Modifier) (milter.Response, error) {
|
||||
s.releaseReservations() // a previous transaction that ended without EOM/ABORT
|
||||
if ip := clientAddr(m); ip != "" {
|
||||
s.clientIP = ip
|
||||
}
|
||||
s.from = cleanAddress(from)
|
||||
s.login = macro(m, "auth_authen")
|
||||
s.rcpts = nil
|
||||
s.subject = ""
|
||||
if !s.authIPAllowed() {
|
||||
s.recordRejected()
|
||||
s.recordRejected(store.ReasonClient)
|
||||
return milter.RespTempFail, nil
|
||||
}
|
||||
if s.overLimit() {
|
||||
s.recordRejected()
|
||||
s.recordRejected(store.ReasonLimit)
|
||||
return milter.RespTempFail, nil
|
||||
}
|
||||
return milter.RespContinue, nil
|
||||
@@ -113,13 +129,15 @@ func (s *session) Header(name, value string, m *milter.Modifier) (milter.Respons
|
||||
|
||||
// Body fires at end-of-message, when the queue-id macro {i} is set and the
|
||||
// message is about to be committed to the queue. This is where the "queued"
|
||||
// rows are written. We accept (this milter is done) without ever rejecting.
|
||||
// rows are written. The reply is "continue", which at end-of-message lets the
|
||||
// message through just as "accept" does - and, unlike it, leaves this session
|
||||
// in place for the connection's next message (see MailFrom).
|
||||
func (s *session) Body(m *milter.Modifier) (milter.Response, error) {
|
||||
s.record(macro(m, "i"))
|
||||
// The rows are in the send log now, so the stored count sees this message
|
||||
// and its level-2 slots are no longer needed.
|
||||
s.releaseReservations()
|
||||
return milter.RespAccept, nil
|
||||
return milter.RespContinue, nil
|
||||
}
|
||||
|
||||
// Abort ends the current transaction without an end-of-message (client RSET, or
|
||||
@@ -143,6 +161,21 @@ func macro(m *milter.Modifier, name string) string {
|
||||
return m.Macros["{"+name+"}"]
|
||||
}
|
||||
|
||||
// clientAddr is the client's address from the {client_addr} macro, or "" when
|
||||
// Postfix did not send one that reads as an address. An IPv6 address may come
|
||||
// in Sendmail's form, with "IPv6:" in front.
|
||||
func clientAddr(m *milter.Modifier) string {
|
||||
raw := strings.TrimSpace(macro(m, "client_addr"))
|
||||
if len(raw) > 5 && strings.EqualFold(raw[:5], "IPv6:") {
|
||||
raw = raw[5:]
|
||||
}
|
||||
ip := net.ParseIP(raw)
|
||||
if ip == nil {
|
||||
return ""
|
||||
}
|
||||
return ip.String()
|
||||
}
|
||||
|
||||
// record writes one send-log row per recipient. Failures are logged, never
|
||||
// propagated: journalling must not affect mail acceptance (architecture.md §
|
||||
// Mail path).
|
||||
|
||||
@@ -166,8 +166,10 @@ func TestBodyAcceptsEvenWhenRecorderFails(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Body returned error, must fail open: %v", err)
|
||||
}
|
||||
if resp != milter.RespAccept {
|
||||
t.Fatalf("Body response = %v, want Accept", resp)
|
||||
// "Continue" at end-of-message lets the message through, and keeps the
|
||||
// session for the connection's next message.
|
||||
if resp != milter.RespContinue {
|
||||
t.Fatalf("Body response = %v, want Continue", resp)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -389,21 +391,77 @@ func TestRateLimitFailsOpenOnCountError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateLimitNoIPKeyDoesNotApply(t *testing.T) {
|
||||
// A ceiling is on a domain or an application, not on an address: a message that
|
||||
// arrives with no client address is counted like any other. It used to be let
|
||||
// through, which is how the second MAIL FROM of a connection got past a limit.
|
||||
func TestRateLimitAppliesWithoutAClientIP(t *testing.T) {
|
||||
rec := &fakeRecorder{
|
||||
limits: map[string]store.RateLimit{
|
||||
store.RateLimitScopeDomain + "|example.com": domainLimit(),
|
||||
},
|
||||
counts: map[string]int64{store.RateLimitScopeDomain + "|example.com": 999},
|
||||
}
|
||||
// A session with no client IP (e.g. local submission) cannot be keyed.
|
||||
s := &session{rec: rec}
|
||||
resp, err := s.MailFrom("a@example.com", mods(map[string]string{"auth_authen": "app1"}))
|
||||
if err != nil {
|
||||
t.Fatalf("MailFrom: %v", err)
|
||||
}
|
||||
if resp != milter.RespContinue {
|
||||
t.Fatalf("no-IP session = %v, want Continue", resp)
|
||||
if resp != milter.RespTempFail {
|
||||
t.Fatalf("over the limit with no client IP = %v, want TempFail", resp)
|
||||
}
|
||||
}
|
||||
|
||||
// The milter library hands the milter a fresh session after every reply that
|
||||
// is not "continue", and Postfix sends Connect once per connection. Such a
|
||||
// session knows the client only from the {client_addr} macro of MAIL FROM, and
|
||||
// that has to be enough for an application's client-address list.
|
||||
func TestAuthIPRestrictHoldsForASessionThatNeverSawConnect(t *testing.T) {
|
||||
rec := &fakeRecorder{apps: map[string]store.Application{
|
||||
"app1": {Login: "app1", AuthIPRestrict: true, AuthAllowedIPs: []string{"198.51.100.1"}},
|
||||
}}
|
||||
for name, c := range map[string]struct {
|
||||
macros map[string]string
|
||||
want milter.Response
|
||||
}{
|
||||
"an unlisted address in the macro": {map[string]string{"auth_authen": "app1", "{client_addr}": limitIP}, milter.RespTempFail},
|
||||
"the listed address in the macro": {map[string]string{"auth_authen": "app1", "{client_addr}": "198.51.100.1"}, milter.RespContinue},
|
||||
// No address at all is not a pass: a list an unknown address satisfies is no list.
|
||||
"no client address": {map[string]string{"auth_authen": "app1"}, milter.RespTempFail},
|
||||
"not an address": {map[string]string{"auth_authen": "app1", "{client_addr}": "unknown"}, milter.RespTempFail},
|
||||
} {
|
||||
s := &session{rec: rec} // no Connect
|
||||
resp, err := s.MailFrom("a@example.com", mods(c.macros))
|
||||
if err != nil || resp != c.want {
|
||||
t.Errorf("%s: %v, %v; want %v", name, resp, err, c.want)
|
||||
}
|
||||
}
|
||||
// An application with no list is not held up by a missing address.
|
||||
open := &fakeRecorder{apps: map[string]store.Application{"app1": {Login: "app1"}}}
|
||||
s := &session{rec: open}
|
||||
if resp, _ := s.MailFrom("a@example.com", mods(map[string]string{"auth_authen": "app1"})); resp != milter.RespContinue {
|
||||
t.Errorf("no list, no address: %v, want Continue", resp)
|
||||
}
|
||||
}
|
||||
|
||||
// The macro wins over what Connect said, and reads both spellings of IPv6.
|
||||
func TestClientAddrFromTheMailMacro(t *testing.T) {
|
||||
for raw, want := range map[string]string{
|
||||
"203.0.113.7": "203.0.113.7", "2001:db8::1": "2001:db8::1", "IPv6:2001:DB8::1": "2001:db8::1",
|
||||
"ipv6:::1": "::1", "": "", "unknown": "", "IPv6:": "",
|
||||
} {
|
||||
if got := clientAddr(mods(map[string]string{"{client_addr}": raw})); got != want {
|
||||
t.Errorf("clientAddr(%q) = %q, want %q", raw, got, want)
|
||||
}
|
||||
}
|
||||
s := &session{rec: &fakeRecorder{}}
|
||||
_, _ = s.Connect("h", "tcp4", 0, net.ParseIP("192.0.2.1"), mods(nil))
|
||||
_, _ = s.MailFrom("a@example.com", mods(map[string]string{"client_addr": "192.0.2.2"}))
|
||||
if s.clientIP != "192.0.2.2" {
|
||||
t.Errorf("clientIP = %q, want the macro's", s.clientIP)
|
||||
}
|
||||
_, _ = s.MailFrom("a@example.com", mods(nil))
|
||||
if s.clientIP != "192.0.2.2" {
|
||||
t.Errorf("clientIP = %q after a MAIL FROM with no macro, want it kept", s.clientIP)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -26,10 +26,9 @@ import (
|
||||
// reaches the send log (or is abandoned) — see inflight for why the stored
|
||||
// count alone is not enough.
|
||||
func (s *session) overLimit() bool {
|
||||
if s.clientIP == "" {
|
||||
return false // no client IP to key on; level-2 does not apply
|
||||
}
|
||||
|
||||
// A ceiling is on a domain or an application, whatever address the mail
|
||||
// comes from, so the client address is not asked for here. It used to be,
|
||||
// and a message that arrived without one was let through uncounted.
|
||||
if s.login != "" {
|
||||
rl, ok, err := s.rec.RateLimit(store.RateLimitScopeApp, s.login)
|
||||
if err != nil {
|
||||
@@ -86,14 +85,16 @@ func (s *session) releaseReservations() {
|
||||
}
|
||||
|
||||
// recordRejected writes a send-log row for a message refused by a level-2
|
||||
// limit (guide § Rate limiting — refusals are recorded too), so the rejection
|
||||
// shows up in the monitoring screen. Only MAIL-stage fields are known; the
|
||||
// write is best-effort and never affects the response.
|
||||
func (s *session) recordRejected() {
|
||||
// limit (guide § Rate limiting — refusals are recorded too) or for its client
|
||||
// address, so the rejection shows up in the monitoring screen with its reason
|
||||
// (store.Reason*). Only MAIL-stage fields are known; the write is best-effort
|
||||
// and never affects the response.
|
||||
func (s *session) recordRejected(reason string) {
|
||||
err := s.rec.InsertRejected(store.SendLogEntry{
|
||||
Domain: domainOf(s.from),
|
||||
AppLogin: s.login,
|
||||
From: s.from,
|
||||
Reason: reason,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("journal-milter: record rejected %s: %v", s.from, err)
|
||||
|
||||
@@ -0,0 +1,504 @@
|
||||
package milter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"mime"
|
||||
"net"
|
||||
"net/mail"
|
||||
"net/textproto"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/emersion/go-milter"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/mailhdr"
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
|
||||
// The sender check: a message's From: line is held against the application
|
||||
// that sends it, under the rule Postfix holds the envelope sender to
|
||||
// (docs/plans/from-header-binding.md). OpenDKIM chooses the signing key by
|
||||
// that line, so without the check an application could have mail signed as
|
||||
// another domain of the server, or as the server itself.
|
||||
//
|
||||
// Unlike the journal milter this one is attached strictly
|
||||
// (default_action=tempfail in build/postfix-config.sh): while it does not
|
||||
// answer, Postfix defers mail. It therefore depends on as little as it can.
|
||||
// It reads no database: what an application may send as comes from the file
|
||||
// of sender bindings Postfix itself reads for the envelope, so the two go by
|
||||
// one table, by one lookup: the address, the address without its
|
||||
// "+extension", the domain. (Postfix compares logins without regard to case;
|
||||
// the check compares them as written, which is how the panel writes them.) The send-log row of a refusal is written by another goroutine and
|
||||
// is not waited for.
|
||||
|
||||
// tableFile is a table of the mail path that the panel rewrites and the check
|
||||
// reads: it is read again when it has changed. An unreadable file is an error
|
||||
// and the check then defers mail - a table from before would let a deleted
|
||||
// application go on sending, and no table would refuse everyone for good.
|
||||
type tableFile struct {
|
||||
path string
|
||||
modTime time.Time
|
||||
size int64
|
||||
loaded bool
|
||||
}
|
||||
|
||||
// changed returns the file's lines when they have to be read (again), and
|
||||
// nil when what was read last still stands.
|
||||
func (f *tableFile) changed() (lines []string, again bool, err error) {
|
||||
fi, err := os.Stat(f.path)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if f.loaded && fi.ModTime().Equal(f.modTime) && fi.Size() == f.size {
|
||||
return nil, false, nil
|
||||
}
|
||||
raw, err := os.ReadFile(f.path)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
f.modTime, f.size, f.loaded = fi.ModTime(), fi.Size(), true
|
||||
return strings.Split(string(raw), "\n"), true, nil
|
||||
}
|
||||
|
||||
// Bindings is Postfix's sender_login_maps file as the check reads it: each
|
||||
// sender - an address, or "@domain" for any address of a domain - with the
|
||||
// logins that may use it.
|
||||
type Bindings struct {
|
||||
mu sync.Mutex
|
||||
file tableFile
|
||||
logins map[string]map[string]bool // sender -> logins
|
||||
domainOf map[string]string // login -> the domain it sends as
|
||||
}
|
||||
|
||||
// NewBindings reads the file at path on first use.
|
||||
func NewBindings(path string) *Bindings {
|
||||
return &Bindings{file: tableFile{path: path}}
|
||||
}
|
||||
|
||||
// fresh makes the table match the file.
|
||||
func (b *Bindings) fresh() error {
|
||||
lines, again, err := b.file.changed()
|
||||
if err != nil || !again {
|
||||
return err
|
||||
}
|
||||
logins := make(map[string]map[string]bool)
|
||||
domains := make(map[string]string)
|
||||
for _, line := range lines {
|
||||
// texthash: <key><whitespace><value>, the value a comma-separated
|
||||
// list of logins (internal/postfix renderSenderLoginMaps).
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 || strings.HasPrefix(fields[0], "#") {
|
||||
continue
|
||||
}
|
||||
sender := strings.ToLower(fields[0])
|
||||
set := logins[sender]
|
||||
if set == nil {
|
||||
set = make(map[string]bool)
|
||||
logins[sender] = set
|
||||
}
|
||||
for _, l := range strings.Split(strings.Join(fields[1:], ""), ",") {
|
||||
if l != "" {
|
||||
set[l] = true
|
||||
if _, known := domains[l]; !known {
|
||||
domains[l] = domainOf(sender)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
b.logins, b.domainOf = logins, domains
|
||||
return nil
|
||||
}
|
||||
|
||||
// Allowed reports whether login may send as addr: the address itself is bound
|
||||
// to it, or its whole domain is. This is the lookup Postfix makes for the
|
||||
// envelope sender.
|
||||
func (b *Bindings) Allowed(login, addr string) (bool, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if err := b.fresh(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
addr = strings.ToLower(addr)
|
||||
if b.logins[addr][login] {
|
||||
return true, nil
|
||||
}
|
||||
// Postfix's second try: the address without its extension, so that an
|
||||
// application listed as invoices@ may also send as invoices+march@
|
||||
// (recipient_delimiter in build/postfix-config.sh).
|
||||
if bare := withoutExtension(addr); bare != addr && b.logins[bare][login] {
|
||||
return true, nil
|
||||
}
|
||||
if d := domainOf(addr); d != "" && b.logins["@"+d][login] {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// extensionDelimiter is Postfix's recipient_delimiter, which
|
||||
// build/postfix-config.sh sets to this.
|
||||
const extensionDelimiter = "+"
|
||||
|
||||
// withoutExtension drops the extension of an address's local part: what
|
||||
// follows its first delimiter. "user+tag@example.org" is "user@example.org";
|
||||
// an address whose local part begins with the delimiter has no extension.
|
||||
func withoutExtension(addr string) string {
|
||||
at := strings.LastIndexByte(addr, '@')
|
||||
if at < 0 {
|
||||
return addr
|
||||
}
|
||||
if i := strings.Index(addr[:at], extensionDelimiter); i > 0 {
|
||||
return addr[:i] + addr[at:]
|
||||
}
|
||||
return addr
|
||||
}
|
||||
|
||||
// DomainOf returns the domain login sends as, from the first of its bindings
|
||||
// in the file's order; "" for a login that has none. An application belongs
|
||||
// to one domain, so all of its bindings name the same one.
|
||||
func (b *Bindings) DomainOf(login string) string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.fresh() != nil {
|
||||
return ""
|
||||
}
|
||||
return b.domainOf[login]
|
||||
}
|
||||
|
||||
// Signers is OpenDKIM's SigningTable as the check reads it: the names this
|
||||
// server signs for - every sending domain, with applications or without, and
|
||||
// the server's own name.
|
||||
type Signers struct {
|
||||
mu sync.Mutex
|
||||
file tableFile
|
||||
names map[string]bool
|
||||
}
|
||||
|
||||
// NewSigners reads the file at path on first use.
|
||||
func NewSigners(path string) *Signers {
|
||||
return &Signers{file: tableFile{path: path}}
|
||||
}
|
||||
|
||||
// Signs reports whether mail whose From: is at domain gets a signature.
|
||||
func (s *Signers) Signs(domain string) (bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
lines, again, err := s.file.changed()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if again {
|
||||
names := make(map[string]bool)
|
||||
for _, line := range lines {
|
||||
// "*@<name> <key name>" (internal/domain renderTables).
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 2 || strings.HasPrefix(fields[0], "#") {
|
||||
continue
|
||||
}
|
||||
if d := domainOf(fields[0]); d != "" {
|
||||
names[d] = true
|
||||
}
|
||||
}
|
||||
s.names = names
|
||||
}
|
||||
return s.names[strings.ToLower(domain)], nil
|
||||
}
|
||||
|
||||
// Refusals is where the check leaves the send-log rows of what it refused.
|
||||
// *store.Store satisfies it.
|
||||
type Refusals interface {
|
||||
InsertRejected(e store.SendLogEntry) error
|
||||
}
|
||||
|
||||
// senderSession is one connection of the sender check.
|
||||
type senderSession struct {
|
||||
milter.NoOpMilter
|
||||
bind *Bindings
|
||||
signers *Signers
|
||||
refused chan<- store.SendLogEntry
|
||||
|
||||
login string
|
||||
from string
|
||||
rcpts []string
|
||||
subject string
|
||||
froms []string // the value of every From: field of the message
|
||||
}
|
||||
|
||||
// MailFrom starts a message and forgets the one before it: a connection may
|
||||
// carry several, and after a temporary failure the library hands the
|
||||
// connection a new session.
|
||||
func (s *senderSession) MailFrom(from string, m *milter.Modifier) (milter.Response, error) {
|
||||
s.from = cleanAddress(from)
|
||||
s.login = macro(m, "auth_authen")
|
||||
s.rcpts, s.subject, s.froms = nil, "", nil
|
||||
return milter.RespContinue, nil
|
||||
}
|
||||
|
||||
func (s *senderSession) RcptTo(rcpt string, m *milter.Modifier) (milter.Response, error) {
|
||||
s.rcpts = append(s.rcpts, cleanAddress(rcpt))
|
||||
return milter.RespContinue, nil
|
||||
}
|
||||
|
||||
func (s *senderSession) Header(name, value string, m *milter.Modifier) (milter.Response, error) {
|
||||
// The name is trimmed: the obsolete syntax allows blanks before the colon.
|
||||
switch textproto.CanonicalMIMEHeaderKey(strings.TrimSpace(name)) {
|
||||
case "From":
|
||||
s.froms = append(s.froms, value)
|
||||
case "Subject":
|
||||
if s.subject == "" {
|
||||
s.subject = mailhdr.DecodeSubject(value)
|
||||
}
|
||||
}
|
||||
return milter.RespContinue, nil
|
||||
}
|
||||
|
||||
// Headers is the end of the headers, where the message is judged.
|
||||
func (s *senderSession) Headers(h textproto.MIMEHeader, m *milter.Modifier) (milter.Response, error) {
|
||||
shown, why, err := s.judge()
|
||||
if err != nil {
|
||||
// Strict: what cannot be checked is not let through.
|
||||
log.Printf("sender-check: %v - deferring mail of %q", err, s.login)
|
||||
return milter.RespTempFail, nil
|
||||
}
|
||||
if why == "" {
|
||||
return milter.RespContinue, nil
|
||||
}
|
||||
log.Printf("sender-check: application %q refused: %s (From: %s)", s.login, why, shown)
|
||||
s.record(shown)
|
||||
// A reply code of its own, so that the application is told what is wrong
|
||||
// and, the refusal being permanent, does not send the message again.
|
||||
return milter.NewResponseStr('y', "550 5.7.1 From: "+shown+" - "+why), nil
|
||||
}
|
||||
|
||||
func (s *senderSession) Abort(m *milter.Modifier) error {
|
||||
s.rcpts, s.subject, s.froms = nil, "", nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// judge decides about the message at hand. why is empty when it may pass;
|
||||
// otherwise it says what is wrong, and shown is the From: line in a form that
|
||||
// is safe to print - both go into the reply to the client.
|
||||
func (s *senderSession) judge() (shown, why string, err error) {
|
||||
if s.login == "" {
|
||||
// Not reachable while the submission ports demand authentication; a
|
||||
// message nobody answers for is not signed either way.
|
||||
return "(none)", "no authenticated application", nil
|
||||
}
|
||||
switch len(s.froms) {
|
||||
case 0:
|
||||
return "(none)", "the message has no From: line", nil
|
||||
case 1:
|
||||
default:
|
||||
return printable(strings.Join(s.froms, " | ")), "the message has more than one From: line", nil
|
||||
}
|
||||
// Postfix hands a milter a header as it came, folded lines and all; the
|
||||
// address parser reads one line.
|
||||
raw := unfold(s.froms[0])
|
||||
shown = printable(raw)
|
||||
list, perr := fromParser.ParseList(raw)
|
||||
if perr != nil || len(list) != 1 || list[0].Address == "" {
|
||||
return shown, "it must be one address", nil
|
||||
}
|
||||
addr := strings.ToLower(list[0].Address)
|
||||
shown = printable(addr)
|
||||
ok, err := s.bind.Allowed(s.login, addr)
|
||||
if err != nil {
|
||||
return shown, "", fmt.Errorf("sender bindings: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return shown, "not an address this application sends as", nil
|
||||
}
|
||||
// A cruder second look. OpenDKIM finds the address in this line with a
|
||||
// parser of its own, and the key it signs with follows what that parser
|
||||
// finds. Wherever the two could read the line differently - a display
|
||||
// name, a comment, a quoted string - the other address would have to be
|
||||
// spelled out in it, so no other name this server signs for may appear.
|
||||
//
|
||||
// The display name is looked at once more as a reader will see it: a mail
|
||||
// program decodes an encoded word, and "=?utf-8?q?director=40example.org?="
|
||||
// shows as an address of another sender though no "@" is written in it.
|
||||
own := domainOf(addr)
|
||||
for _, d := range append(mentionedDomains(raw), mentionedDomains(list[0].Name)...) {
|
||||
if d == own {
|
||||
continue
|
||||
}
|
||||
signs, err := s.signers.Signs(d)
|
||||
if err != nil {
|
||||
return shown, "", fmt.Errorf("signing table: %w", err)
|
||||
}
|
||||
if signs {
|
||||
return printable(raw), "it names another sender of this server, " + d, nil
|
||||
}
|
||||
}
|
||||
// And one that needs to know nothing of OpenDKIM's parser: outside quoted
|
||||
// strings the line has the one "@" of its address. A second one - in a
|
||||
// comment, in an unquoted name - is where two parsers could part ways.
|
||||
if strings.Count(unquoted(raw), "@") != 1 {
|
||||
return printable(raw), "outside quotes it must have one @, the address's own", nil
|
||||
}
|
||||
return shown, "", nil
|
||||
}
|
||||
|
||||
// record leaves a row for the Outbound log. It never waits: a database that
|
||||
// is slow must not make the check slow. One row for the message, not one for
|
||||
// each recipient as an accepted message has: nothing was sent to anybody, and
|
||||
// a refused message with a thousand recipients would otherwise be a thousand
|
||||
// rows at the price of one.
|
||||
func (s *senderSession) record(shown string) {
|
||||
if s.refused == nil {
|
||||
return
|
||||
}
|
||||
to := ""
|
||||
if len(s.rcpts) > 0 {
|
||||
to = s.rcpts[0]
|
||||
}
|
||||
if more := len(s.rcpts) - 1; more > 0 {
|
||||
to += fmt.Sprintf(" and %d more", more)
|
||||
}
|
||||
e := store.SendLogEntry{
|
||||
Domain: s.domain(),
|
||||
AppLogin: s.login,
|
||||
From: s.from,
|
||||
To: to,
|
||||
Subject: s.subject,
|
||||
Reason: store.ReasonFrom,
|
||||
HeaderFrom: shown,
|
||||
}
|
||||
select {
|
||||
case s.refused <- e:
|
||||
default:
|
||||
log.Printf("sender-check: the refusal of %s -> %s is not in the log: too many are waiting to be written", s.from, to)
|
||||
}
|
||||
}
|
||||
|
||||
// fromParser reads the From: line. A display name encoded in a charset Go
|
||||
// does not convert (windows-1251, koi8-r, ...) is still a display name: its
|
||||
// bytes are passed through unread, since only the address is judged.
|
||||
var fromParser = mail.AddressParser{WordDecoder: &mime.WordDecoder{
|
||||
CharsetReader: func(charset string, input io.Reader) (io.Reader, error) { return input, nil },
|
||||
}}
|
||||
|
||||
// unfold joins the lines of a folded header value (RFC 5322 2.2.3): a line
|
||||
// break is dropped, the white space that follows it stays.
|
||||
func unfold(v string) string {
|
||||
return strings.NewReplacer("\r\n", "", "\n", "", "\r", "").Replace(v)
|
||||
}
|
||||
|
||||
// unquoted returns v without its quoted strings, a backslash inside one
|
||||
// escaping the next character.
|
||||
func unquoted(v string) string {
|
||||
var sb strings.Builder
|
||||
in := false
|
||||
for i := 0; i < len(v); i++ {
|
||||
switch c := v[i]; {
|
||||
case in && c == '\\':
|
||||
i++
|
||||
case c == '"':
|
||||
in = !in
|
||||
case !in:
|
||||
sb.WriteByte(c)
|
||||
}
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// domain is the sending domain of the refused message for its row in the
|
||||
// log: the envelope sender's, which Postfix has held against the login - or,
|
||||
// for the null sender of a bounce an application writes itself, which Postfix
|
||||
// holds against nobody, the domain the login is bound to. Without it the row
|
||||
// would belong to no domain and its administrator would never see it.
|
||||
func (s *senderSession) domain() string {
|
||||
if d := domainOf(s.from); d != "" {
|
||||
return d
|
||||
}
|
||||
return s.bind.DomainOf(s.login)
|
||||
}
|
||||
|
||||
// mentionedDomains returns, lower-cased, what follows every "@" of v as far
|
||||
// as it reads as a host name.
|
||||
func mentionedDomains(v string) []string {
|
||||
var out []string
|
||||
for i := 0; i < len(v); i++ {
|
||||
if v[i] != '@' {
|
||||
continue
|
||||
}
|
||||
j := i + 1
|
||||
for j < len(v) && isHostByte(v[j]) {
|
||||
j++
|
||||
}
|
||||
if d := strings.Trim(strings.ToLower(v[i+1:j]), ".-"); d != "" {
|
||||
out = append(out, d)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isHostByte(c byte) bool {
|
||||
return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '.' || c == '-'
|
||||
}
|
||||
|
||||
// printable reduces what a client wrote to printable ASCII without the
|
||||
// characters that mean something in an SMTP reply or a log line, and cuts it:
|
||||
// it is said back to the client, logged and stored.
|
||||
func printable(v string) string {
|
||||
const max = 120
|
||||
var sb strings.Builder
|
||||
for _, r := range v {
|
||||
if sb.Len() >= max {
|
||||
sb.WriteString("...")
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case r == '%' || r < 0x20 || r > 0x7e:
|
||||
sb.WriteByte('?')
|
||||
default:
|
||||
sb.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// ServeSender runs the sender check on ln until ctx is cancelled. rec may be
|
||||
// nil: refusals are then not logged.
|
||||
func ServeSender(ctx context.Context, ln net.Listener, bind *Bindings, signers *Signers, rec Refusals) error {
|
||||
var refused chan store.SendLogEntry
|
||||
if rec != nil {
|
||||
refused = make(chan store.SendLogEntry, 256)
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e := <-refused:
|
||||
if err := rec.InsertRejected(e); err != nil {
|
||||
log.Printf("sender-check: record refused %s: %v", e.From, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
srv := &milter.Server{
|
||||
NewMilter: func() milter.Milter {
|
||||
return &senderSession{bind: bind, signers: signers, refused: refused}
|
||||
},
|
||||
Actions: 0, // read-only
|
||||
Protocol: milter.OptNoBody, // the headers decide
|
||||
}
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
_ = srv.Close()
|
||||
}()
|
||||
|
||||
if err := srv.Serve(ln); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
package milter
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/emersion/go-milter"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
|
||||
// senderMaps is a sender_login_maps file as internal/postfix writes it: shop
|
||||
// and news send as any address of their domains, billing only as two listed
|
||||
// addresses of example.org, and two applications share shop.example.org.
|
||||
const senderMaps = `@news.example.net news
|
||||
@shop.example.org shop,shop2
|
||||
invoices@example.org billing
|
||||
billing@example.org billing
|
||||
`
|
||||
|
||||
// signingTable is OpenDKIM's SigningTable for the same server: the three
|
||||
// sending domains, one more that has no application yet, and the server's own
|
||||
// name.
|
||||
const signingTable = `*@news.example.net selfpost._domainkey.news.example.net
|
||||
*@shop.example.org selfpost._domainkey.shop.example.org
|
||||
*@example.org selfpost._domainkey.example.org
|
||||
*@empty.example.com selfpost._domainkey.empty.example.com
|
||||
*@mail.example.org selfpost._domainkey.mail.example.org
|
||||
`
|
||||
|
||||
func signers(t *testing.T) *Signers {
|
||||
t.Helper()
|
||||
return NewSigners(writeMaps(t, signingTable))
|
||||
}
|
||||
|
||||
func writeMaps(t *testing.T, content string) string {
|
||||
t.Helper()
|
||||
p := filepath.Join(t.TempDir(), "table")
|
||||
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// offer plays one message to the check and returns the reply at the end of
|
||||
// the headers, with what was left for the log.
|
||||
func offer(t *testing.T, bind *Bindings, login, envelope string, headers ...[2]string) (milter.Response, []store.SendLogEntry) {
|
||||
t.Helper()
|
||||
refused := make(chan store.SendLogEntry, 8)
|
||||
s := &senderSession{bind: bind, signers: signers(t), refused: refused}
|
||||
if _, err := s.MailFrom(envelope, mods(map[string]string{"{auth_authen}": login})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.RcptTo("someone@elsewhere.example", mods(nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, h := range headers {
|
||||
if _, err := s.Header(h[0], h[1], mods(nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
resp, err := s.Headers(nil, mods(nil))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
close(refused)
|
||||
var rows []store.SendLogEntry
|
||||
for e := range refused {
|
||||
rows = append(rows, e)
|
||||
}
|
||||
return resp, rows
|
||||
}
|
||||
|
||||
func replyText(r milter.Response) string {
|
||||
m := r.Response()
|
||||
return string(rune(m.Code)) + " " + strings.TrimRight(string(m.Data), "\x00")
|
||||
}
|
||||
|
||||
func TestSenderCheck(t *testing.T) {
|
||||
bind := NewBindings(writeMaps(t, senderMaps))
|
||||
from := func(v string) [2]string { return [2]string{"From", v} }
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
login string
|
||||
headers [][2]string
|
||||
pass bool
|
||||
wantText string // a part of the refusal
|
||||
}{
|
||||
{"own address", "shop", [][2]string{from("shop@shop.example.org")}, true, ""},
|
||||
{"own domain, any address, with a name", "shop", [][2]string{from(`"The Shop" <Orders@Shop.Example.Org>`)}, true, ""},
|
||||
{"second application of the domain", "shop2", [][2]string{from("x@shop.example.org")}, true, ""},
|
||||
{"a listed address", "billing", [][2]string{from("Invoices <invoices@example.org>")}, true, ""},
|
||||
{"a listed address with an extension", "billing", [][2]string{from("invoices+2026-march@example.org")}, true, ""},
|
||||
{"an extension, any address of the domain", "shop", [][2]string{from("orders+48213@shop.example.org")}, true, ""},
|
||||
{"header name in another case", "shop", [][2]string{{"FROM", "a@shop.example.org"}}, true, ""},
|
||||
{"its own name inside the display name", "shop", [][2]string{from(`"shop@shop.example.org" <a@shop.example.org>`)}, true, ""},
|
||||
{"an encoded display name", "shop", [][2]string{from("=?utf-8?b?0JzQsNCz0LDQt9C40L0=?= <a@shop.example.org>")}, true, ""},
|
||||
{"a folded line", "shop", [][2]string{from("\"A very long display name of the shop\"\r\n <a@shop.example.org>")}, true, ""},
|
||||
{"an encoded name, folded", "shop", [][2]string{from("=?utf-8?b?0JzQsNCz0LDQt9C40L0=?=\n\t<a@shop.example.org>")}, true, ""},
|
||||
{"a name in windows-1251", "shop", [][2]string{from("=?windows-1251?B?zODj4Ofo7Q==?= <a@shop.example.org>")}, true, ""},
|
||||
{"a name in koi8-r, quoted-printable", "shop", [][2]string{from("=?koi8-r?Q?=ED=C1=C7=C1=DA=C9=CE?= <a@shop.example.org>")}, true, ""},
|
||||
{"a comment after the address", "shop", [][2]string{from("a@shop.example.org (the shop)")}, true, ""},
|
||||
|
||||
{"another registered domain", "shop", [][2]string{from("director@example.org")}, false, "not an address this application sends as"},
|
||||
{"the server's own name", "shop", [][2]string{from("postmaster@mail.example.org")}, false, "not an address this application sends as"},
|
||||
{"a stranger's domain", "shop", [][2]string{from("someone@gmail.example")}, false, "not an address this application sends as"},
|
||||
{"an address off the list", "billing", [][2]string{from("director@example.org")}, false, "not an address this application sends as"},
|
||||
{"an address off the list, with an extension", "billing", [][2]string{from("director+invoices@example.org")}, false, "not an address this application sends as"},
|
||||
{"a listed name as the extension of another", "billing", [][2]string{from("+invoices@example.org")}, false, "not an address this application sends as"},
|
||||
{"a subdomain of its own", "shop", [][2]string{from("a@sub.shop.example.org")}, false, "not an address this application sends as"},
|
||||
{"no From: line", "shop", [][2]string{{"Subject", "hello"}}, false, "no From: line"},
|
||||
{"two From: lines", "shop", [][2]string{from("a@shop.example.org"), from("director@example.org")}, false, "more than one From: line"},
|
||||
{"two addresses in one line", "shop", [][2]string{from("a@shop.example.org, director@example.org")}, false, "one address"},
|
||||
{"not an address", "shop", [][2]string{from("the shop")}, false, "one address"},
|
||||
{"a group", "shop", [][2]string{from("friends: a@shop.example.org, b@shop.example.org;")}, false, "one address"},
|
||||
{"another sender in the display name", "shop", [][2]string{from(`"director@example.org" <a@shop.example.org>`)}, false, "names another sender of this server"},
|
||||
{"the server's name in a comment", "shop", [][2]string{from("a@shop.example.org (postmaster@mail.example.org)")}, false, "names another sender of this server"},
|
||||
{"a domain with a key and no application, in the display name", "shop", [][2]string{from(`"x@empty.example.com" <a@shop.example.org>`)}, false, "names another sender of this server"},
|
||||
{"a stranger's address in the display name", "shop", [][2]string{from(`"someone@gmail.example" <a@shop.example.org>`)}, true, ""},
|
||||
{"another sender in an encoded display name", "shop", [][2]string{from("=?utf-8?q?director=40example=2Eorg?= <a@shop.example.org>")}, false, "names another sender of this server"},
|
||||
{"a stranger's address in a comment", "shop", [][2]string{from("a@shop.example.org (someone@gmail.example)")}, false, "one @"},
|
||||
{"another sender folded onto the next line", "shop", [][2]string{from("\"x\"\r\n <director@example.org>")}, false, "not an address this application sends as"},
|
||||
{"a quote left open before another address", "shop", [][2]string{from(`"a@shop.example.org <director@example.org>`)}, false, "one address"},
|
||||
{"no login", "", [][2]string{from("a@shop.example.org")}, false, "no authenticated application"},
|
||||
{"an unknown login", "ghost", [][2]string{from("a@shop.example.org")}, false, "not an address this application sends as"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
resp, rows := offer(t, bind, c.login, "shop@shop.example.org", c.headers...)
|
||||
if c.pass {
|
||||
if resp != milter.RespContinue {
|
||||
t.Fatalf("refused: %s", replyText(resp))
|
||||
}
|
||||
if len(rows) != 0 {
|
||||
t.Fatalf("a passed message left rows: %+v", rows)
|
||||
}
|
||||
return
|
||||
}
|
||||
text := replyText(resp)
|
||||
if !strings.HasPrefix(text, "y 550 5.7.1 From: ") || !strings.Contains(text, c.wantText) {
|
||||
t.Fatalf("reply %q, want a 550 5.7.1 saying %q", text, c.wantText)
|
||||
}
|
||||
if len(rows) != 1 || rows[0].Reason != store.ReasonFrom || rows[0].To != "someone@elsewhere.example" || rows[0].AppLogin != c.login {
|
||||
t.Fatalf("log rows: %+v", rows)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// What the client wrote is said back to it, logged and stored: it must not
|
||||
// be able to put a line break or a control character there.
|
||||
func TestSenderCheckPrintsNothingRaw(t *testing.T) {
|
||||
bind := NewBindings(writeMaps(t, senderMaps))
|
||||
evil := "x@gmail.example\r\n250 ok %s \x00\x1b[31m " + strings.Repeat("a", 400)
|
||||
resp, rows := offer(t, bind, "shop", "shop@shop.example.org", [2]string{"From", evil})
|
||||
text := replyText(resp)
|
||||
for _, bad := range []string{"\r", "\n", "%", "\x1b"} {
|
||||
if strings.Contains(text, bad) || strings.Contains(rows[0].HeaderFrom, bad) {
|
||||
t.Fatalf("%q got through: %q / %q", bad, text, rows[0].HeaderFrom)
|
||||
}
|
||||
}
|
||||
if len(rows[0].HeaderFrom) > 130 {
|
||||
t.Fatalf("stored %d bytes of the header", len(rows[0].HeaderFrom))
|
||||
}
|
||||
}
|
||||
|
||||
// Without its file the check knows nothing, and what cannot be checked is
|
||||
// deferred - neither passed nor refused for good.
|
||||
func TestSenderCheckDefersWithoutBindings(t *testing.T) {
|
||||
bind := NewBindings(filepath.Join(t.TempDir(), "absent"))
|
||||
resp, rows := offer(t, bind, "shop", "shop@shop.example.org", [2]string{"From", "a@shop.example.org"})
|
||||
if resp != milter.RespTempFail {
|
||||
t.Fatalf("reply %s, want a temporary failure", replyText(resp))
|
||||
}
|
||||
if len(rows) != 0 {
|
||||
t.Fatalf("a deferred message left rows: %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// The panel rewrites the file when an application is added or deleted; the
|
||||
// check has to follow without a restart.
|
||||
func TestBindingsFollowTheFile(t *testing.T) {
|
||||
p := writeMaps(t, senderMaps)
|
||||
bind := NewBindings(p)
|
||||
if ok, err := bind.Allowed("shop", "a@shop.example.org"); err != nil || !ok {
|
||||
t.Fatalf("before: %v %v", ok, err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte("@news.example.net news\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A rewrite within the clock's resolution that keeps the size would not be
|
||||
// seen; the panel replaces the file by rename, which gives it a new time.
|
||||
later := time.Now().Add(2 * time.Second)
|
||||
if err := os.Chtimes(p, later, later); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok, err := bind.Allowed("shop", "a@shop.example.org"); err != nil || ok {
|
||||
t.Fatalf("after the application was deleted: %v %v", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Without the signing table the second look cannot be taken, and a line it
|
||||
// would have to judge is deferred; a line that names nobody else is not held
|
||||
// up by it.
|
||||
func TestSenderCheckDefersWithoutSigningTable(t *testing.T) {
|
||||
bind := NewBindings(writeMaps(t, senderMaps))
|
||||
play := func(v string) milter.Response {
|
||||
s := &senderSession{bind: bind, signers: NewSigners(filepath.Join(t.TempDir(), "absent"))}
|
||||
_, _ = s.MailFrom("shop@shop.example.org", mods(map[string]string{"{auth_authen}": "shop"}))
|
||||
_, _ = s.Header("From", v, mods(nil))
|
||||
resp, _ := s.Headers(nil, mods(nil))
|
||||
return resp
|
||||
}
|
||||
if resp := play(`"x@example.org" <a@shop.example.org>`); resp != milter.RespTempFail {
|
||||
t.Fatalf("reply %s, want a temporary failure", replyText(resp))
|
||||
}
|
||||
if resp := play(`a@shop.example.org`); resp != milter.RespContinue {
|
||||
t.Fatalf("a plain own address: %s", replyText(resp))
|
||||
}
|
||||
}
|
||||
|
||||
// A second message on the connection is judged by its own headers.
|
||||
func TestSenderCheckResetsBetweenMessages(t *testing.T) {
|
||||
bind := NewBindings(writeMaps(t, senderMaps))
|
||||
s := &senderSession{bind: bind, signers: signers(t)}
|
||||
auth := mods(map[string]string{"{auth_authen}": "shop"})
|
||||
for i, v := range []string{"director@example.org", "a@shop.example.org", "director@example.org"} {
|
||||
_, _ = s.MailFrom("shop@shop.example.org", auth)
|
||||
_, _ = s.Header("From", v, mods(nil))
|
||||
resp, _ := s.Headers(nil, mods(nil))
|
||||
if passed := resp == milter.RespContinue; passed != (i == 1) {
|
||||
t.Fatalf("message %d (From: %s): passed=%v", i, v, passed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A refused message is one row of the log however many recipients it had.
|
||||
func TestSenderCheckRefusalIsOneRow(t *testing.T) {
|
||||
bind := NewBindings(writeMaps(t, senderMaps))
|
||||
refused := make(chan store.SendLogEntry, 8)
|
||||
s := &senderSession{bind: bind, signers: signers(t), refused: refused}
|
||||
_, _ = s.MailFrom("shop@shop.example.org", mods(map[string]string{"{auth_authen}": "shop"}))
|
||||
for _, r := range []string{"a@x.example", "b@x.example", "c@x.example"} {
|
||||
_, _ = s.RcptTo(r, mods(nil))
|
||||
}
|
||||
_, _ = s.Header("From", "director@example.org", mods(nil))
|
||||
_, _ = s.Headers(nil, mods(nil))
|
||||
close(refused)
|
||||
var rows []store.SendLogEntry
|
||||
for e := range refused {
|
||||
rows = append(rows, e)
|
||||
}
|
||||
if len(rows) != 1 || rows[0].To != "a@x.example and 2 more" {
|
||||
t.Fatalf("rows = %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// A message with the null envelope sender - a bounce an application writes
|
||||
// itself - is judged like any other, and its row belongs to the domain the
|
||||
// login is bound to.
|
||||
func TestSenderCheckRefusalWithNullSenderHasItsDomain(t *testing.T) {
|
||||
bind := NewBindings(writeMaps(t, senderMaps))
|
||||
for login, want := range map[string]string{"shop": "shop.example.org", "billing": "example.org"} {
|
||||
_, rows := offer(t, bind, login, "", [2]string{"From", "someone@gmail.example"})
|
||||
if len(rows) != 1 || rows[0].Domain != want || rows[0].From != "" {
|
||||
t.Errorf("%s: rows = %+v, want one of domain %s", login, rows, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"mime"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
sendmailPath = "/usr/sbin/sendmail"
|
||||
sendmailTimeout = 20 * time.Second
|
||||
)
|
||||
|
||||
// Mailer hands notifications to Postfix by local submission, as
|
||||
// postmaster@<hostname>. They are the panel's own mail: they pass no SASL
|
||||
// login, are not rows of the send log and count against no rate limit.
|
||||
// Postfix signs them through OpenDKIM with the key of the server's own name
|
||||
// (non_smtpd_milters in build/postfix-config.sh, domain.Service.SetServerName).
|
||||
type Mailer struct {
|
||||
hostname string
|
||||
// run submits a finished message; a field so tests can read it instead.
|
||||
run func(from, to string, msg []byte) error
|
||||
}
|
||||
|
||||
// NewMailer returns a mailer for the server named hostname.
|
||||
func NewMailer(hostname string) *Mailer {
|
||||
return &Mailer{hostname: hostname, run: sendmail}
|
||||
}
|
||||
|
||||
// From is the address notifications are sent from.
|
||||
func From(hostname string) string { return "postmaster@" + hostname }
|
||||
|
||||
// Send submits one message to one recipient.
|
||||
func (m *Mailer) Send(to, subject, body string) error {
|
||||
// The address is a panel user's, validated when it was saved. It goes into
|
||||
// a header and onto a command line all the same, so what cannot be part of
|
||||
// an address is refused here as well (security.md).
|
||||
if to == "" || strings.ContainsAny(to, " \t\r\n<>,;\"") || strings.HasPrefix(to, "-") || !strings.Contains(to, "@") {
|
||||
// (validate.AccountEmail refuses the same, so a user is told at the
|
||||
// form and not here, once a minute, in the log.)
|
||||
return fmt.Errorf("not an address a notification can go to: %q", to)
|
||||
}
|
||||
from := From(m.hostname)
|
||||
id, err := messageID(m.hostname)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var b bytes.Buffer
|
||||
fmt.Fprintf(&b, "From: SelfPost <%s>\r\n", from)
|
||||
fmt.Fprintf(&b, "To: <%s>\r\n", to)
|
||||
fmt.Fprintf(&b, "Subject: %s\r\n", mime.QEncoding.Encode("utf-8", oneLine(subject)))
|
||||
fmt.Fprintf(&b, "Date: %s\r\n", time.Now().UTC().Format(time.RFC1123Z))
|
||||
fmt.Fprintf(&b, "Message-ID: %s\r\n", id)
|
||||
// Tells a vacation responder and a ticket system that nobody wrote this.
|
||||
b.WriteString("Auto-Submitted: auto-generated\r\n")
|
||||
b.WriteString("MIME-Version: 1.0\r\n")
|
||||
b.WriteString("Content-Type: text/plain; charset=utf-8\r\n")
|
||||
b.WriteString("Content-Transfer-Encoding: 8bit\r\n")
|
||||
b.WriteString("\r\n")
|
||||
b.WriteString(strings.ReplaceAll(strings.ReplaceAll(body, "\r\n", "\n"), "\n", "\r\n"))
|
||||
return m.run(from, to, b.Bytes())
|
||||
}
|
||||
|
||||
// oneLine keeps a subject on its line whatever went into it.
|
||||
func oneLine(s string) string {
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
|
||||
func messageID(hostname string) (string, error) {
|
||||
var raw [12]byte
|
||||
if _, err := rand.Read(raw[:]); err != nil {
|
||||
return "", fmt.Errorf("message id: %w", err)
|
||||
}
|
||||
return fmt.Sprintf("<%s.%d@%s>", hex.EncodeToString(raw[:]), time.Now().Unix(), hostname), nil
|
||||
}
|
||||
|
||||
// sendmail submits msg through Postfix's sendmail command. The argument list is
|
||||
// fixed apart from the two addresses, which follow "--" so that neither can be
|
||||
// read as an option; nothing goes through a shell (security.md). -i keeps a
|
||||
// line holding a single dot from ending the message.
|
||||
func sendmail(from, to string, msg []byte) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), sendmailTimeout)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, sendmailPath, "-i", "-f", from, "--", to)
|
||||
cmd.Stdin = bytes.NewReader(msg)
|
||||
// sendmail hands the message to postdrop; if that child outlives the
|
||||
// timeout holding the output pipe, do not wait for it for ever.
|
||||
cmd.WaitDelay = 5 * time.Second
|
||||
out, err := cmd.CombinedOutput()
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
return fmt.Errorf("sendmail timed out after %s", sendmailTimeout)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("sendmail: %w: %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,501 @@
|
||||
// Package notify writes to the panel's users when something needs them
|
||||
// (architecture.md § Notifications): a sending domain's DNS record does not
|
||||
// match, a level-2 rate limit refuses mail, the TLS certificate is about to
|
||||
// expire, a message has waited in the queue for more than three days.
|
||||
//
|
||||
// Each subject has a row of state in the database, so a thing is said once and
|
||||
// a restart says nothing again. The rules of when a message goes out are the
|
||||
// owner's (docs/plans/panel-notifications.md § When a message is sent) and
|
||||
// live in this file; what the messages say is in text.go, how they leave in
|
||||
// mailer.go.
|
||||
package notify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/dnscheck"
|
||||
"github.com/mixeme/selfpost/internal/health"
|
||||
"github.com/mixeme/selfpost/internal/postfix"
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
// CheckInterval is how often DNS, the certificate and the queue are looked
|
||||
// at.
|
||||
CheckInterval = time.Hour
|
||||
// LimitInterval is how often refused mail is looked for. A refusal is
|
||||
// reported within the minute rather than on the hour: it is mail not being
|
||||
// sent now. The look reads the refused rows through an index of their own
|
||||
// (migration 0005).
|
||||
LimitInterval = time.Minute
|
||||
|
||||
// dnsChecks is how many checks in a row must agree before a DNS mismatch is
|
||||
// reported, and before it is reported gone; dnsApart is how far apart the
|
||||
// first and the last of them must be. One check is a resolver's bad minute;
|
||||
// two, an hour apart, is a record that is wrong. The time is asked for as
|
||||
// well as the count because the first check after a start is minutes after
|
||||
// the last one before it.
|
||||
dnsChecks = 2
|
||||
dnsApart = 45 * time.Minute
|
||||
|
||||
// QueueAge is how long a message waits before the queue is reported. Postfix
|
||||
// keeps a message for five days by default, so two are left to act in.
|
||||
QueueAge = 3 * 24 * time.Hour
|
||||
|
||||
// The certificate is reported this many days before it expires, and again at
|
||||
// the second number if it is still the same certificate.
|
||||
certFirstDays = 14
|
||||
certLastDays = 3
|
||||
|
||||
// limitRepeat is the shortest time between two messages about the limits of
|
||||
// the same domain.
|
||||
limitRepeat = 24 * time.Hour
|
||||
|
||||
// hourlyCeiling is the most messages handed to Postfix in an hour, whatever
|
||||
// the rules above arrive at. They bypass every rate limit, and the number of
|
||||
// subjects grows with the number of domains; nothing here should ever come
|
||||
// near it, and a fault that does is stopped by it.
|
||||
hourlyCeiling = 60
|
||||
|
||||
phasePending = "pending" // seen wrong, not yet said
|
||||
phaseOpen = "open" // said
|
||||
)
|
||||
|
||||
// Store is what the notifier needs of the database. *store.Store satisfies it.
|
||||
type Store interface {
|
||||
NotificationsEnabled() (bool, error)
|
||||
NotifyRecipients(domainName string) ([]string, error)
|
||||
GetNotifyState(key string) (store.NotifyState, bool, error)
|
||||
SetNotifyState(st store.NotifyState) error
|
||||
DeleteNotifyState(key string) error
|
||||
ListNotifyStateKeys(prefix string) ([]string, error)
|
||||
LimitRefusalsSince(since time.Time) ([]store.LimitRefusal, error)
|
||||
ListDomains() ([]store.Domain, error)
|
||||
}
|
||||
|
||||
// Sender hands one message to the mail system. *Mailer does; tests collect.
|
||||
type Sender interface {
|
||||
Send(to, subject, body string) error
|
||||
}
|
||||
|
||||
// Checks are the readings the notifier acts on, as functions so that tests
|
||||
// need neither DNS nor a certificate nor a queue.
|
||||
type Checks struct {
|
||||
// Domain is a fresh check of a sending domain's published records.
|
||||
Domain func(d store.Domain) (dnscheck.Domain, error)
|
||||
// Certificate is the state of the TLS certificate Postfix serves.
|
||||
Certificate func() health.Certificate
|
||||
// Queue lists the messages Postfix holds; an error says nothing about any.
|
||||
Queue func() ([]postfix.QueuedMessage, error)
|
||||
}
|
||||
|
||||
// Notifier applies the rules. Create one with New and call Run.
|
||||
type Notifier struct {
|
||||
st Store
|
||||
checks Checks
|
||||
sender Sender
|
||||
hostname string
|
||||
now func() time.Time
|
||||
// ctx ends a pass between two domains when the process is asked to stop.
|
||||
ctx context.Context
|
||||
// sentAt holds the times of the messages of the last hour, for the ceiling.
|
||||
sentAt []time.Time
|
||||
}
|
||||
|
||||
// New builds a notifier for the server named hostname.
|
||||
func New(st Store, checks Checks, sender Sender, hostname string) *Notifier {
|
||||
return &Notifier{
|
||||
st: st, checks: checks, sender: sender, hostname: hostname,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
ctx: context.Background(),
|
||||
}
|
||||
}
|
||||
|
||||
// Run checks on both cadences until ctx is cancelled. The first hourly check
|
||||
// runs a few minutes after the start, not at it: a container that has just
|
||||
// come up is still starting Postfix and may have no network yet.
|
||||
func (n *Notifier) Run(ctx context.Context) error {
|
||||
n.ctx = ctx
|
||||
first := time.NewTimer(3 * time.Minute)
|
||||
defer first.Stop()
|
||||
hourly := time.NewTicker(CheckInterval)
|
||||
defer hourly.Stop()
|
||||
limits := time.NewTicker(LimitInterval)
|
||||
defer limits.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-first.C:
|
||||
n.Check()
|
||||
case <-hourly.C:
|
||||
n.Check()
|
||||
case <-limits.C:
|
||||
n.CheckLimits()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// enabled reports whether anything is to be done at all. With the instance
|
||||
// switch off nothing is checked and no state moves, so that what is wrong
|
||||
// when it is turned on again is said then.
|
||||
func (n *Notifier) enabled() bool {
|
||||
on, err := n.st.NotificationsEnabled()
|
||||
if err != nil {
|
||||
log.Printf("notify: read the switch: %v", err)
|
||||
return false
|
||||
}
|
||||
return on
|
||||
}
|
||||
|
||||
// Check is the hourly pass: DNS of every sending domain, the certificate, the
|
||||
// queue.
|
||||
func (n *Notifier) Check() {
|
||||
if !n.enabled() {
|
||||
return
|
||||
}
|
||||
n.checkDNS()
|
||||
n.checkCertificate()
|
||||
n.checkQueue()
|
||||
}
|
||||
|
||||
// outcome is what became of a message.
|
||||
type outcome int
|
||||
|
||||
const (
|
||||
// sent: at least one recipient's copy was handed to Postfix.
|
||||
sent outcome = iota
|
||||
// nobody: no user is to be told - none with an e-mail and the switch on.
|
||||
nobody
|
||||
// failed: there were recipients and none could be handed a copy.
|
||||
failed
|
||||
)
|
||||
|
||||
// send writes one message to every recipient of a subject, each in a message
|
||||
// of their own - users are not shown each other's addresses.
|
||||
//
|
||||
// What the caller makes of the outcome is the rule that keeps a thing from
|
||||
// being said never: a problem counts as said only when it was sent, so one
|
||||
// that arose while nobody could be told, or while the mail system was down, is
|
||||
// told at a later check. A recipient whose copy failed while another's went
|
||||
// out is not tried again: the subject is said.
|
||||
func (n *Notifier) send(domainName string, m message) outcome {
|
||||
to, err := n.st.NotifyRecipients(domainName)
|
||||
if err != nil {
|
||||
log.Printf("notify: recipients: %v", err)
|
||||
return failed
|
||||
}
|
||||
if len(to) == 0 {
|
||||
return nobody
|
||||
}
|
||||
handed := 0
|
||||
for _, addr := range to {
|
||||
if !n.underCeiling() {
|
||||
log.Printf("notify: more than %d messages in an hour; %q to %s is held back", hourlyCeiling, m.subject, addr)
|
||||
continue
|
||||
}
|
||||
if err := n.sender.Send(addr, m.subject, m.body); err != nil {
|
||||
log.Printf("notify: send %q to %s: %v", m.subject, addr, err)
|
||||
continue
|
||||
}
|
||||
n.sentAt = append(n.sentAt, n.now())
|
||||
handed++
|
||||
}
|
||||
if handed == 0 {
|
||||
return failed
|
||||
}
|
||||
log.Printf("notify: %q sent to %d of %d", m.subject, handed, len(to))
|
||||
return sent
|
||||
}
|
||||
|
||||
// underCeiling reports whether one more message may go out this hour.
|
||||
func (n *Notifier) underCeiling() bool {
|
||||
cutoff := n.now().Add(-time.Hour)
|
||||
keep := n.sentAt[:0]
|
||||
for _, t := range n.sentAt {
|
||||
if t.After(cutoff) {
|
||||
keep = append(keep, t)
|
||||
}
|
||||
}
|
||||
n.sentAt = keep
|
||||
return len(n.sentAt) < hourlyCeiling
|
||||
}
|
||||
|
||||
func (n *Notifier) save(st store.NotifyState) {
|
||||
if err := n.st.SetNotifyState(st); err != nil {
|
||||
log.Printf("notify: save state %s: %v", st.Key, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (n *Notifier) forget(key string) {
|
||||
if err := n.st.DeleteNotifyState(key); err != nil {
|
||||
log.Printf("notify: forget state %s: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- DNS
|
||||
|
||||
// dnsFault is what is wrong with a domain's records: the checks that failed,
|
||||
// in the order the page shows them.
|
||||
type dnsFault struct {
|
||||
Record string // "DKIM", "SPF", "DMARC"
|
||||
Detail string
|
||||
}
|
||||
|
||||
// dnsReading sorts a check into wrong, right, or nothing learned. A record
|
||||
// that fails is wrong whatever the others say. A lookup that did not answer is
|
||||
// nothing learned - unless another record is plainly wrong. A warning (no DMARC
|
||||
// record, an SPF include this check does not follow) is advice the page gives
|
||||
// and not a thing to write about.
|
||||
func dnsReading(d dnscheck.Domain) (faults []dnsFault, known bool) {
|
||||
known = true
|
||||
for _, r := range []struct {
|
||||
name string
|
||||
res dnscheck.Result
|
||||
}{{"DKIM", d.DKIM}, {"SPF", d.SPF}, {"DMARC", d.DMARC}} {
|
||||
switch r.res.Status {
|
||||
case health.StatusError:
|
||||
faults = append(faults, dnsFault{Record: r.name, Detail: r.res.Detail})
|
||||
case health.StatusUnknown:
|
||||
known = false
|
||||
}
|
||||
}
|
||||
if len(faults) > 0 {
|
||||
known = true
|
||||
}
|
||||
return faults, known
|
||||
}
|
||||
|
||||
func (n *Notifier) checkDNS() {
|
||||
domains, err := n.st.ListDomains()
|
||||
if err != nil {
|
||||
log.Printf("notify: list domains: %v", err)
|
||||
return
|
||||
}
|
||||
names := make(map[string]bool, len(domains))
|
||||
for _, d := range domains {
|
||||
names[d.Name] = true
|
||||
}
|
||||
for _, d := range domains {
|
||||
// A dead resolver costs seconds per domain; a process asked to stop
|
||||
// does not wait for the rest of them.
|
||||
if n.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
res, err := n.checks.Domain(d)
|
||||
if err != nil {
|
||||
log.Printf("notify: check %s: %v", d.Name, err)
|
||||
continue
|
||||
}
|
||||
n.dnsStep(d.Name, res)
|
||||
}
|
||||
// A deleted domain has no records to be right or wrong.
|
||||
keys, err := n.st.ListNotifyStateKeys("dns:")
|
||||
if err != nil {
|
||||
log.Printf("notify: list dns state: %v", err)
|
||||
return
|
||||
}
|
||||
for _, key := range keys {
|
||||
if !names[strings.TrimPrefix(key, "dns:")] {
|
||||
n.forget(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dnsStep moves one domain's state by one check.
|
||||
//
|
||||
// A subject is "pending" from the first bad check, with Since the time of it,
|
||||
// and "open" from the message. While open, Count is the good checks in a row
|
||||
// and Mark the time of the first of them.
|
||||
func (n *Notifier) dnsStep(name string, res dnscheck.Domain) {
|
||||
faults, known := dnsReading(res)
|
||||
if !known {
|
||||
return
|
||||
}
|
||||
bad := len(faults) > 0
|
||||
key := "dns:" + name
|
||||
st, have, err := n.st.GetNotifyState(key)
|
||||
if err != nil {
|
||||
log.Printf("notify: %v", err)
|
||||
return
|
||||
}
|
||||
now := n.now()
|
||||
|
||||
switch {
|
||||
case !have && !bad:
|
||||
// Nothing wrong and nothing said.
|
||||
case !have && bad:
|
||||
n.save(store.NotifyState{Key: key, Phase: phasePending, Count: 1, Since: now})
|
||||
case st.Phase == phasePending && !bad:
|
||||
n.forget(key) // it was the resolver's bad minute
|
||||
case st.Phase == phasePending && bad:
|
||||
st.Count++
|
||||
if st.Count >= dnsChecks && now.Sub(st.Since) >= dnsApart {
|
||||
if n.send(name, dnsProblem(n.hostname, name, faults, st.Since)) == sent {
|
||||
st.Phase, st.Count, st.Mark, st.SentAt = phaseOpen, 0, "", now
|
||||
}
|
||||
}
|
||||
n.save(st)
|
||||
case st.Phase == phaseOpen && bad:
|
||||
if st.Count != 0 {
|
||||
// It went right for a check and wrong again: still the same trouble.
|
||||
st.Count, st.Mark = 0, ""
|
||||
n.save(st)
|
||||
}
|
||||
case st.Phase == phaseOpen && !bad:
|
||||
st.Count++
|
||||
if st.Count == 1 {
|
||||
st.Mark = now.Format(time.RFC3339)
|
||||
}
|
||||
firstGood, _ := time.Parse(time.RFC3339, st.Mark)
|
||||
if st.Count >= dnsChecks && now.Sub(firstGood) >= dnsApart {
|
||||
// With nobody to tell any more, the trouble is over all the same.
|
||||
if n.send(name, dnsCleared(n.hostname, name, st.Since)) != failed {
|
||||
n.forget(key)
|
||||
return
|
||||
}
|
||||
}
|
||||
n.save(st)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- certificate
|
||||
|
||||
func (n *Notifier) checkCertificate() {
|
||||
c := n.checks.Certificate()
|
||||
if c.NotAfter.IsZero() {
|
||||
return // no certificate configured, or one that cannot be read: Health says so
|
||||
}
|
||||
const key = "cert"
|
||||
mark := c.NotAfter.UTC().Format(time.RFC3339)
|
||||
st, have, err := n.st.GetNotifyState(key)
|
||||
if err != nil {
|
||||
log.Printf("notify: %v", err)
|
||||
return
|
||||
}
|
||||
if have && st.Mark != mark {
|
||||
// A different certificate: what was said about the old one is over.
|
||||
n.forget(key)
|
||||
have, st = false, store.NotifyState{}
|
||||
}
|
||||
|
||||
left := c.NotAfter.Sub(n.now())
|
||||
level := ""
|
||||
switch {
|
||||
case left <= certLastDays*24*time.Hour:
|
||||
level = "last"
|
||||
case left <= certFirstDays*24*time.Hour:
|
||||
level = "first"
|
||||
}
|
||||
if level == "" || (have && (st.Phase == level || st.Phase == "last")) {
|
||||
return
|
||||
}
|
||||
if n.send("", certExpiring(n.hostname, c.NotAfter, left)) != sent {
|
||||
return
|
||||
}
|
||||
n.save(store.NotifyState{Key: key, Phase: level, Mark: mark, Since: n.now(), SentAt: n.now()})
|
||||
}
|
||||
|
||||
// ---- queue
|
||||
|
||||
// ownMail reports whether a queued message is the notifier's: a notification
|
||||
// that could not be delivered yet, or the bounce of one coming back to
|
||||
// postmaster@ this server, which receives no mail. Neither is the mail the
|
||||
// queue is watched for, and reporting them would have one dead mailbox
|
||||
// announce itself every three days.
|
||||
func (n *Notifier) ownMail(m postfix.QueuedMessage) bool {
|
||||
own := From(n.hostname)
|
||||
if strings.EqualFold(m.Sender, own) {
|
||||
return true
|
||||
}
|
||||
for _, rcpt := range m.Recipients {
|
||||
if strings.EqualFold(rcpt, own) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (n *Notifier) checkQueue() {
|
||||
msgs, err := n.checks.Queue()
|
||||
if err != nil {
|
||||
log.Printf("notify: read the queue: %v", err)
|
||||
return // no listing is no information
|
||||
}
|
||||
now := n.now()
|
||||
var old []postfix.QueuedMessage
|
||||
for _, m := range msgs {
|
||||
if now.Sub(m.Arrival) > QueueAge && !n.ownMail(m) {
|
||||
old = append(old, m)
|
||||
}
|
||||
}
|
||||
sort.Slice(old, func(i, j int) bool { return old[i].Arrival.Before(old[j].Arrival) })
|
||||
|
||||
const key = "queue"
|
||||
st, have, err := n.st.GetNotifyState(key)
|
||||
if err != nil {
|
||||
log.Printf("notify: %v", err)
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case len(old) > 0 && !have:
|
||||
if n.send("", queueStuck(n.hostname, old, now)) == sent {
|
||||
n.save(store.NotifyState{Key: key, Phase: phaseOpen, Since: now, SentAt: now})
|
||||
}
|
||||
case len(old) == 0 && have:
|
||||
if n.send("", queueCleared(n.hostname, st.Since)) != failed {
|
||||
n.forget(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- level-2 limits
|
||||
|
||||
// CheckLimits looks for mail refused under a level-2 rate limit and reports
|
||||
// each sending domain whose applications were refused, at most once in
|
||||
// limitRepeat.
|
||||
func (n *Notifier) CheckLimits() {
|
||||
if !n.enabled() {
|
||||
return
|
||||
}
|
||||
now := n.now()
|
||||
refusals, err := n.st.LimitRefusalsSince(now.Add(-limitRepeat))
|
||||
if err != nil {
|
||||
log.Printf("notify: %v", err)
|
||||
return
|
||||
}
|
||||
for _, r := range refusals {
|
||||
key := "limit:" + r.Domain
|
||||
st, have, err := n.st.GetNotifyState(key)
|
||||
if err != nil {
|
||||
log.Printf("notify: %v", err)
|
||||
continue
|
||||
}
|
||||
// Told within the day: the refusals counted now are the ones it was
|
||||
// told about, or more of the same. Told longer ago: everything in the
|
||||
// window is new since then.
|
||||
if have && now.Sub(st.SentAt) < limitRepeat {
|
||||
continue
|
||||
}
|
||||
if n.send(r.Domain, limitRefused(n.hostname, r)) == sent {
|
||||
n.save(store.NotifyState{Key: key, Phase: phaseOpen, Since: r.First, SentAt: now})
|
||||
}
|
||||
}
|
||||
|
||||
// A domain nobody was refused under for two days is not a subject any more.
|
||||
keys, err := n.st.ListNotifyStateKeys("limit:")
|
||||
if err != nil {
|
||||
log.Printf("notify: list limit state: %v", err)
|
||||
return
|
||||
}
|
||||
for _, key := range keys {
|
||||
st, have, err := n.st.GetNotifyState(key)
|
||||
if err == nil && have && now.Sub(st.SentAt) > 2*limitRepeat {
|
||||
n.forget(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user