From 5e5d157496c1d277fb4f774facd8ed76e7451856 Mon Sep 17 00:00:00 2001 From: Mix Date: Tue, 22 Sep 2026 15:24:51 +0300 Subject: [PATCH] release: 0.5.0: Pre-1.0 security review, domain-admin role, doc consolidation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bounce could be recorded as a successful delivery. The log-tailer's Fixed: - A bounce could be recorded as a successful delivery. The log-tailer's - Log-tailer resumes where it stopped instead of jumping to end-of-file on - Level-2 rate limit no longer overshoots under concurrency: messages that Changed: - The project has a single public home: `github.com/mixeme/selfpost`. Codeberg - Code comments no longer cite the archived specification. References like - `docs/code-review.md` is gone. Its plan is finished — phases 0 (bar the - `docs/architecture.md` gained a *Code layers* section: a diagram of - Phase 1 doc/code hygiene (`docs/code-review.md`): removed ~30 stale … (1 more in CHANGELOG) Security: - Pre-release security review (plan § D, model Fable, 2026-08-06): full pass - `saslpasswd2` argv: the application login is now passed after a `--` Added: - Optional password encryption for the two secret-bearing downloads (plan - Domain import accepts an encrypted export: the envelope is detected by its - `selfpost-backup` writes encrypted archives and reads them back: - `TestParseDelivery` covers the exotic mail.log shapes the review asked for - docs: README *Encrypting a backup or export*; `docs/security.md` § … (14 more in CHANGELOG) Removed: - docs: `security.md` — accepted risk «restore old backup revives session rows» Changed: - docs: `implementation-plan.md` trimmed to the sole open v1.x gate — pre-release - docs: `architecture.md` — sessions (SQLite, idle renew, password change), - docs: `development.md` — expanded e2e stack and `release.yml` matrix workflow. - docs: `security.md` — accepted risk for send-log rows stuck at `queued` after - docs: `roadmap.md` — optional send-log / `mail.log` follow-ups under v1.x tail. … (10 more in CHANGELOG) Full detail: CHANGELOG.md (0.5.0). --- .github/workflows/test.yml | 9 + CHANGELOG.md | 180 +++++++ Makefile | 2 +- README.md | 197 +++++++- build/Dockerfile | 15 +- build/entrypoint.sh | 8 +- build/postfix-config.sh | 2 +- build/postfix-wrapper.sh | 2 +- cmd/panel/envdoc_test.go | 133 +++++ cmd/panel/httpserver.go | 16 +- cmd/panel/journal.go | 10 +- cmd/panel/main.go | 61 +-- cmd/selfpost-backup/main.go | 145 +++++- cmd/selfpost-backup/main_test.go | 139 ++++++ deploy/.env.example | 1 + deploy/docker-compose.yml | 9 +- docs/architecture.md | 273 +++++++++++ .../specification-v1.0.md} | 5 + docs/assets/selfpost-icon-16.svg | 2 +- docs/assets/selfpost-icon.svg | 2 +- docs/assets/selfpost-proof.html | 26 +- docs/assets/selfpost-stamp-compact.svg | 2 +- docs/assets/selfpost-stamp.svg | 2 +- docs/development.md | 146 ++++++ docs/documentation-plan.md | 274 +++-------- docs/implementation-plan.md | 188 +------ docs/product.md | 102 ++++ docs/progress.md | 27 +- docs/roadmap.md | 190 ++++++++ docs/security.md | 156 +++++- go.mod | 2 +- internal/app/password.go | 4 +- internal/app/sasl.go | 49 +- internal/app/sasl_test.go | 7 +- internal/app/service.go | 70 +-- internal/app/service_test.go | 4 +- internal/app/validate.go | 19 +- internal/app/validate_test.go | 2 +- internal/backup/backup.go | 57 +-- internal/backup/backup_test.go | 2 +- internal/buildinfo/buildinfo.go | 4 +- internal/dnscheck/dnscheck.go | 2 +- internal/dnscheck/dnscheck_test.go | 2 +- internal/dnscheck/examples_test.go | 2 +- internal/dnscheck/records.go | 2 +- internal/dnscheck/server.go | 2 +- internal/dnscheck/spf.go | 2 +- internal/domain/dkim.go | 10 +- internal/domain/opendkim.go | 63 +-- internal/domain/service.go | 47 +- internal/domain/transfer.go | 42 +- internal/domain/transfer_test.go | 2 +- internal/health/cert.go | 4 +- internal/health/health_test.go | 25 + internal/health/liveness.go | 46 ++ internal/health/processes.go | 7 +- internal/logtail/logtail.go | 71 ++- internal/logtail/logtail_test.go | 163 ++++++- internal/logtail/offset.go | 136 ++++++ internal/milter/inflight.go | 99 ++++ internal/milter/milter.go | 79 +-- internal/milter/milter_test.go | 126 ++++- internal/milter/ratelimit.go | 49 +- internal/postfix/postfix.go | 57 +-- internal/postfix/postfix_test.go | 2 +- internal/postfix/queue.go | 12 +- internal/postfix/write.go | 2 +- internal/secretfile/secretfile.go | 458 ++++++++++++++++++ internal/secretfile/secretfile_test.go | 162 +++++++ internal/store/admin.go | 4 +- internal/store/admin_test.go | 2 +- internal/store/applications.go | 45 +- internal/store/domains.go | 13 +- internal/store/domains_test.go | 4 +- internal/store/logtail.go | 54 +++ .../store/migrations/0003_logtail_state.sql | 12 + internal/store/ratelimits.go | 44 +- internal/store/sendlog.go | 44 +- internal/store/sendlog_test.go | 7 +- internal/store/store.go | 4 +- internal/web/handlers_account.go | 12 +- internal/web/handlers_apps.go | 37 +- internal/web/handlers_auth.go | 8 +- internal/web/handlers_backup.go | 223 +++++++-- internal/web/handlers_backup_test.go | 127 +++++ internal/web/handlers_domains.go | 19 +- internal/web/handlers_monitor.go | 25 +- internal/web/handlers_ratelimit.go | 35 +- internal/web/handlers_setup.go | 6 +- internal/web/handlers_status.go | 16 +- internal/web/ratelimit.go | 2 +- internal/web/security.go | 16 +- internal/web/security_test.go | 2 +- internal/web/session.go | 4 +- internal/web/session_test.go | 2 +- internal/web/setup.go | 14 +- internal/web/static/favicon.svg | 2 +- internal/web/static/logo-compact.svg | 2 +- internal/web/static/logo.svg | 2 +- internal/web/static/panel.css | 165 ++++--- internal/web/static/panel.js | 49 +- internal/web/templates.go | 25 +- internal/web/templates/backup.html | 19 +- internal/web/templates/domain_detail.html | 7 +- internal/web/templates/encrypt_fields.html | 23 + internal/web/templates/layout.html | 2 +- internal/web/templates_test.go | 10 +- internal/web/token.go | 2 +- internal/web/validate.go | 12 +- internal/web/web.go | 72 +-- test/e2e/go.mod | 2 +- test/e2e/negative_test.go | 24 +- test/e2e/panel_client.go | 7 +- 113 files changed, 4284 insertions(+), 1140 deletions(-) create mode 100644 cmd/panel/envdoc_test.go create mode 100644 cmd/selfpost-backup/main_test.go create mode 100644 docs/architecture.md rename docs/{specification.md => archive/specification-v1.0.md} (99%) create mode 100644 docs/development.md create mode 100644 docs/product.md create mode 100644 docs/roadmap.md create mode 100644 internal/health/liveness.go create mode 100644 internal/logtail/offset.go create mode 100644 internal/milter/inflight.go create mode 100644 internal/secretfile/secretfile.go create mode 100644 internal/secretfile/secretfile_test.go create mode 100644 internal/store/logtail.go create mode 100644 internal/store/migrations/0003_logtail_state.sql create mode 100644 internal/web/handlers_backup_test.go create mode 100644 internal/web/templates/encrypt_fields.html diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 859ed79..619b007 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,15 @@ jobs: go-version: "1.26" cache: true + - name: Format + run: | + fmt="$(gofmt -l .)" + if [ -n "$fmt" ]; then + echo "$fmt" + echo "gofmt needed on the files above; run 'gofmt -w .'" + exit 1 + fi + - name: Vet run: go vet ./... diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c05525..3d7b225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,186 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); version ## [Unreleased] +## [0.5.0] - 2026-08-06 + +### Fixed + +- A bounce could be recorded as a successful delivery. The log-tailer's + delivery-line pattern matched `status=` greedily, so it took the *last* + occurrence on the line — and Postfix appends the remote server's reply + verbatim, which the far end controls. A rejection whose reply text contained + `status=sent` was filed as `sent` in the send log. The pattern now takes the + first `status=` after the recipient, which is the real field + (`internal/logtail/logtail.go`); found while extending `TestParseDelivery`. +- Log-tailer resumes where it stopped instead of jumping to end-of-file on + every start (phase 3, `docs/code-review.md`): the read position and a + fingerprint of the log's head are persisted (`logtail_state`, migration + `0003`), so delivery lines written while the panel was down are parsed and + their send-log rows no longer stay `queued` forever. A log that changed + identity while the panel was down is read from the start; a first-ever start, + with nothing stored, still begins at the end. Container recreate remains a + gap — `mail.log` is not in `/data` (`docs/security.md`). +- Level-2 rate limit no longer overshoots under concurrency: messages that + passed the check at MAIL FROM but have not reached the send log yet are + counted alongside the stored rows (`internal/milter/inflight.go`), so + parallel SMTP sessions cannot each spend the same last slot. Slots are + released at end-of-message, on ABORT, and after a 10-minute TTL, so a client + that drops mid-transaction cannot hold one — the limiter stays fail-open. + +### Changed + +- The project has a single public home: `github.com/mixeme/selfpost`. Codeberg + is being retired, so the Go module path moved with it — `go.mod`, + `test/e2e/go.mod`, every import, the `Makefile` `MODULE` variable and the + `-ldflags` version stamp in `build/Dockerfile` and `docs/development.md`. An + import path pointing at a host that is going away would break `go get` and + `go install` outright, which is why this is not only a documentation change. + README no longer lists a primary/mirror pair. +- Code comments no longer cite the archived specification. References like + "spec 7.6.1" or "spec 5.1" pointed into `docs/archive/specification-v1.0.md`, + which is explicitly not a source of truth; each is now a reference to the + live document that owns the subject — `docs/architecture.md` (with section), + `docs/product.md`, `docs/security.md`, or the README. Comments only; no + behaviour is affected. +- `docs/code-review.md` is gone. Its plan is finished — phases 0 (bar the + release-commit steps), 1, 1.5, 2 and 3 are all closed — and the rest of the + document had become a second copy of what `architecture.md`, `security.md` + and the code comments already say. What was genuinely open moved to + `docs/roadmap.md`: splitting `internal/web` into subpackages, a consolidated + documentation index in the README, the adaptive polling interval for an idle + but visible tab, and `CONTRIBUTING.md`. The review text stays in git history + (`522425a`); the CHANGELOG entries below that cite it are left as written. +- `docs/architecture.md` gained a *Code layers* section: a diagram of + handlers → services → store plus the adapters, and the reason the services + layer exists (multi-store writes and their rollback) — closing item A2 of + `docs/code-review.md`. +- Phase 1 doc/code hygiene (`docs/code-review.md`): removed ~30 stale + "Phase N" / historical-staging references from code and shell-script + comments (`cmd/panel`, `internal/*`, `build/*`) now that v1.0 is done; + fixed a stale dashboard comment (`internal/web/handlers_domains.go`) + claiming applications/send-log were unimplemented; added a CSRF ADR to + `docs/security.md` (why Origin-check, not tokens); resolved `docs/logo` in + `docs/roadmap.md` (directory doesn't exist, criterion already met); added a + `gofmt -l` check to CI (`.github/workflows/test.yml`). +- Phase 2 GUI polish (`docs/code-review.md`): the monitoring pages stop + polling while their tab is hidden — the skip is done in an + `htmx:beforeRequest` listener (`internal/web/static/panel.js`) rather than + htmx's own trigger filter, which is evaluated with `new Function` and would + be blocked by the panel's CSP. Dark mode is now a single reassignment of CSS + custom properties under `prefers-color-scheme: dark` instead of a cascade of + `!important` overrides, and the duplicate `main { max-width }` rule is + consolidated into one base rule with documented per-page overrides + (`internal/web/static/panel.css`). + +### Security + +- Pre-release security review (plan § D, model Fable, 2026-08-06): full pass + over the diff from the v1.0 audit (Phase 11, `bd64e80`) to HEAD plus the + complete spec 7.6 checklist. No exploitable findings; one defence-in-depth + fix below. Accepted risks in `docs/security.md` unchanged. +- `saslpasswd2` argv: the application login is now passed after a `--` + end-of-options marker (`internal/app/sasl.go`), so a login starting with + `-` (legal under the whitelist) can never be parsed as a flag by getopt. + +### Added + +- Optional password encryption for the two secret-bearing downloads (plan + phase 1.5, `docs/code-review.md`): an *Encrypt with a password* checkbox on + the full-backup and domain-export forms writes a `.spbk` / `.spde` envelope + instead of the plain `.tar.gz` / `.json` — scrypt key derivation and + AES-256-GCM over 64 KiB chunks, each authenticated with the header, its + counter and an end-of-stream flag, so a truncated or altered file refuses to + open (`internal/secretfile`). Unticked, both downloads are byte-for-byte what + they were. +- Domain import accepts an encrypted export: the envelope is detected by its + magic bytes, and a password field appears next to the file picker + (`internal/web/handlers_backup.go`, `templates/encrypt_fields.html`). +- `selfpost-backup` writes encrypted archives and reads them back: + `-decrypt` (with `-i`/`-o`) turns a `.spbk` into the plain `.tar.gz` a + restore unpacks. The password comes from `SELFPOST_BACKUP_PASSWORD` or + `-password-file`, never from argv. +- `TestParseDelivery` covers the exotic mail.log shapes the review asked for + (`docs/code-review.md` § 3): a `status=` quoted inside the remote reply, the + null recipient of a double bounce, `orig_to=` alongside `to=`, an + unrecognised status word, a capitalised one, and a cleanup line. +- docs: README *Encrypting a backup or export*; `docs/security.md` § + *Резервная копия и экспорт домена* + accepted risk (encryption is opt-in); + `docs/architecture.md` persistence § envelope summary. +- docs: `docs/roadmap.md` v1.x tail — retire `implementation-plan.md` in the + release commit (move to `docs/archive/`, retarget its references in README, + docs, Makefile, release workflow and the e2e test comment). +- docs: `docs/code-review.md` — phase 1.5 plan for optional password encryption + of full backup (`.spbk`) and domain export (`.spde`); checkbox UI pattern; + remove session-resurrection-from-backup as accepted risk. +- docs: `docs/code-review.md` — full codebase review (architecture, code quality, + documentation, GUI, legacy, risks) with prioritized implementation plan and + model routing; cross-links in `implementation-plan.md` and `progress.md`. +- docs (D6): Docker `HEALTHCHECK` probes `/healthz`; endpoint returns 503 unless + opendkim, panel, and postfix are RUNNING (`internal/health.Liveness`). +- docs (D6): README *Container health* — scope of `/healthz` vs authenticated Status. +- docs (D7): `cmd/panel/envdoc_test.go` — regression test that every + `loadConfig` and build-script env key is listed in README documentation. +- docs (D8): `docs/architecture.md` — as-built processes, mail path, routes, + persistence (verified against code). +- docs (D8): `docs/development.md` — local Go workflow, `make e2e`, dev-server + loop, commit/CHANGELOG protocol, agent rules. +- docs (D9): `docs/product.md` — product purpose, assumptions, out-of-scope, + multi-domain model. +- docs (D9): `docs/security.md` — self-contained mandatory security checklist + (former spec §7.6). +- docs (D9): `specification.md` archived to + `docs/archive/specification-v1.0.md`; live docs updated (`progress.md`, + `implementation-plan.md`, `roadmap.md`, `documentation-plan.md`). +- docs (D1): README *Operations* — panel screens (`/status`, domains, + deliveries, mail queue, system log, backup, account), upgrade procedure, + session behaviour (sliding idle, monitoring polls do not extend, password + change signs out other sessions), and `mail.log` rotation cadence. +- docs (D1): README *Rate limiting* — level-1 anvil limits + (`RATE_LIMIT_MESSAGES_PER_IP`, `RATE_LIMIT_WINDOW_SECONDS`) and level-2 + per-domain/application limits from the panel; fixes the `.env.example` link + that pointed at a missing section. +- docs (D2): README environment-variable reference — public `.env` table with + code-accurate defaults, `TRUSTED_PROXY_CIDR` security note, explicit + internal-variable list; `TRUSTED_PROXY_CIDR` wired through + `deploy/docker-compose.yml`. + +### Removed + +- docs: `security.md` — accepted risk «restore old backup revives session rows» + (not a concern in operator deployment). + +### Changed + +- docs: `implementation-plan.md` trimmed to the sole open v1.x gate — pre-release + security review (§ D); closed B.1–C.4 material moved to as-built and ops docs. +- docs: `architecture.md` — sessions (SQLite, idle renew, password change), + `mail.log` rotation (rename + `postfix reload`), `SELFPOST_HOSTNAME` startup + gate, log-tailer known gaps. +- docs: `development.md` — expanded e2e stack and `release.yml` matrix workflow. +- docs: `security.md` — accepted risk for send-log rows stuck at `queued` after + panel restart or container recreate. +- docs: `roadmap.md` — optional send-log / `mail.log` follow-ups under v1.x tail. +- docs: `progress.md`, `documentation-plan.md` — cross-links updated for the new layout. +- docs: `documentation-plan.md` marked closed (D1–D9); trimmed to package + checklist, code-verification method, and ongoing maintenance rules. +- docs: `roadmap.md` — v1.x doc/deploy tail (Codeberg Quick start, compose + image tag at release, `docs/logo`); archived-spec references replaced with + `product.md` / `security.md` / `development.md`. +- docs: `progress.md` — documentation pass closed; deferred polish in roadmap. +- `/healthz` now checks supervisord mail-path processes, not HTTP alone. +- `build/Dockerfile`: `curl` for `HEALTHCHECK`; probe on port 8080. +- docs (D3): README backup — stopped-container `tar` of `./data` (with live-container + WAL warning), `manifest.json` consumed after a matching restore. +- docs (D4): README status banner (v1.0 implemented, links to open questions and + documentation pass); new *Published ports* note for 587; compose usage comment + corrected (TLS via `./certs` bind mount, not `.env`). +- docs (D5): `implementation-plan.md` B.1 — password change signs out other + sessions only (implementation diverged from original plan; README was already + correct). +- docs: documentation plan now targets retiring `specification.md` after D9 — + migration map to `product.md`, `architecture.md`, `development.md`, and + expanded `security.md`; D9 added to the release gate. + ## [0.4.0] - 2026-08-04 ### Added diff --git a/Makefile b/Makefile index 5ac55d0..491605c 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ # Go, so no cgo is required) — see spec 7.1. VERSION ?= dev -MODULE := codeberg.org/mix/selfpost +MODULE := github.com/mixeme/selfpost LDFLAGS := -X $(MODULE)/internal/buildinfo.Version=$(VERSION) GOFLAGS := -trimpath diff --git a/README.md b/README.md index d09ddd3..51c8ff8 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,11 @@ SelfPost sends mail straight to the internet from your own IP, with DKIM signing, and is configured once through the panel. It is **outbound only** — it does not receive mail, provide mailboxes, or offer webmail. -> **Status: under active development.** See [docs/specification.md](docs/specification.md) -> for the full requirements, [docs/implementation-plan.md](docs/implementation-plan.md) -> for the phased build plan, and [docs/security.md](docs/security.md) for the -> security trade-offs that were accepted knowingly. +> **Status: v1.0 implemented; pre-release polish in progress.** Open v1.x +> questions: [docs/implementation-plan.md](docs/implementation-plan.md). +> Documentation pass before the next tag: +> [docs/documentation-plan.md](docs/documentation-plan.md). Accepted security +> trade-offs: [docs/security.md](docs/security.md). ## Requirements (site checklist) @@ -96,6 +97,51 @@ and Traefik by default). A proxy that rewrites `Host` instead makes the panel reject every form submission as cross-origin — the log says so explicitly, printing the `Origin` and `Host` it compared. +## Environment variables + +Copy [deploy/.env.example](deploy/.env.example) to `.env` next to your +`docker-compose.yml`. The table below lists every variable an operator is +expected to set; defaults match the code exactly. + +| Variable | Purpose | Default | Set in | +|---|---|---|---| +| `SELFPOST_HOSTNAME` | Mail-server identity: Postfix HELO/EHLO, SASL realm, certificate CN/SAN, and the hostname the PTR check expects. Bare FQDN only — no scheme or port. | *(required)* | `.env` | +| `SUBMISSION_ENABLE` | When `true`, also listen on port 587 with STARTTLS (RFC 6409 submission) alongside the primary 465/smtps listener. | `false` | `.env` | +| `RATE_LIMIT_MESSAGES_PER_IP` | Level-1 backstop: maximum messages one client IP may submit per window (Postfix `smtpd_client_message_rate_limit`). See [Rate limiting](#rate-limiting). | `100` | `.env` | +| `RATE_LIMIT_WINDOW_SECONDS` | Level-1 window length in seconds (Postfix `anvil_rate_time_unit`). | `3600` | `.env` | +| `SEND_LOG_RETENTION_DAYS` | Days of send-log history kept before the background sweep deletes rows — the main driver of `/data` growth over time. | `90` | `.env` | +| `PANEL_SESSION_IDLE_DAYS` | Sliding idle timeout for the panel login session, in days. There is no absolute cap: an admin who keeps coming back stays signed in indefinitely. | `7` | `.env` | +| `SELFPOST_DNS_RESOLVERS` | Comma-separated recursive resolvers the panel's PTR/SPF/DKIM/DMARC checks query directly (so they report what the internet sees, not what this host's stub resolver synthesises). | `1.1.1.1:53`, `8.8.8.8:53`, `9.9.9.9:53` when unset | `.env` | +| `TRUSTED_PROXY_CIDR` | Comma-separated CIDRs (bare IPs allowed) of reverse proxies allowed to supply `X-Forwarded-For` for login, setup, and account-change rate-limiting. **Leave unset unless you know the exact address of your reverse proxy.** A wrong value lets a client spoof its rate-limit key by sending a forged `X-Forwarded-For` header — the panel trusts the last hop only when the TCP peer matches one of these CIDRs. Behind the default Apache host-network setup this is typically the Docker bridge gateway, e.g. `172.18.0.1`. | *(empty — XFF ignored)* | `.env` | + +TLS certificate paths (`TLS_CERT_FILE`, `TLS_KEY_FILE`) are fixed in +[deploy/docker-compose.yml](deploy/docker-compose.yml) to match the `./certs` +bind mount — configure the mount, not these variables. + +**Internal variables (not part of the operator interface).** The following are +read by the panel or startup scripts but are not meant to be changed in a +normal deployment; documenting them here avoids treating accidental overrides as +supported configuration: + +- **Panel paths and tuning:** `SELFPOST_DATA_DIR` (`/data`), `SELFPOST_DB_PATH` + (`/data/selfpost.db`), `SELFPOST_SETUP_TOKEN_FILE` + (`/data/setup-token`), `PANEL_HTTP_ADDR` (`:8080`), + `JOURNAL_MILTER_SOCKET` (`/run/selfpost/journal.sock`), `MAIL_LOG` + (`/var/log/mail.log`), `PANEL_COOKIE_SECURE` (`true`), `OPENDKIM_SOCKET` + (`/run/opendkim/opendkim.sock`), `OPENDKIM_DIR` (`/data/opendkim`), + `DKIM_SELECTOR_DEFAULT` (`selfpost`), `SASL_DB_PATH` + (`/data/sasl/sasldb2`), `SASL_REALM` (defaults to `SELFPOST_HOSTNAME`), + `POSTFIX_DIR` (`/data/postfix`), `POSTFIX_SENDER_LOGIN_MAPS` + (`/data/postfix/sender_login_maps`). +- **Milter and Postfix startup:** `MILTER_CONNECT_TIMEOUT` (`15s`), + `MILTER_COMMAND_TIMEOUT` (`15s`), `MILTER_CONTENT_TIMEOUT` (`30s`), + `MILTER_WAIT_TIMEOUT` (`30` seconds). +- **Background maintenance:** `TLS_RELOAD_INTERVAL_SECONDS` (`86400` — daily + `postfix reload` to pick up renewed certificates), + `LOGROTATE_INTERVAL_SECONDS` (`21600` — check `mail.log` rotation every six + hours; rotated logs are kept 14 days and each rotation triggers + `postfix reload`). + ## DNS setup Two different scopes — don't confuse them: @@ -137,6 +183,82 @@ day one, and check the IP against major blocklists (Spamhaus and similar) before and during warmup. This is inherent to how mail reputation works on the public internet, not something SelfPost's configuration can shortcut. +## Operations + +After sign-in the panel opens on **Status** — the place to answer "is the +service healthy and will mail be accepted?" + +- **Status** (`/status`) — supervised processes (Postfix, OpenDKIM, panel), + TLS certificate validity and expiry, milter socket presence, and a short + Postfix queue summary. The hostname block compares `SELFPOST_HOSTNAME` + against the PTR record the internet publishes for this server's IP + (forward-confirmed reverse DNS); use *Re-check* after changing DNS. The + **Reload configuration** button re-applies OpenDKIM tables and the Postfix + sender map from the database — use it if daemons drifted from what the panel + shows after manual edits under `/data`. +- **Domains** (`/domains`) — add sending domains, inspect each domain's DKIM + TXT value, SPF/DMARC checks, and SASL applications. Per-domain rate limits + (level 2) are configured here. *Export domain* writes a single-domain archive; + *Import a domain* on the Backup page reads one back in. +- **Deliveries** (`/deliveries`) — searchable send log with server-side filters + by domain and application. Each row shows status `queued` (accepted, not yet + delivered), `sent` (handed off successfully), or `rejected` (refused — for + example by a level-2 rate limit). Retention is controlled by + `SEND_LOG_RETENTION_DAYS`. +- **Mail queue** (`/mail-queue`) — live view of messages Postfix is still + trying to deliver or deferring. +- **System log** (`/system-log`) — tail of `/var/log/mail.log` (Postfix and + related daemon lines). The log rotates daily (14 files kept) with a + `postfix reload` after each rotation; a background loop checks every six + hours. +- **Backup** (`/backup`) — download a full-server backup or import a + single-domain export. See [Backup, restore, and moving a single domain](#backup-restore-and-moving-a-single-domain). +- **Account** (`/account`) — change the administrator username and/or password. + Application SASL logins are separate and are not changed here. + +**Sessions.** A login survives a container restart: sessions live in SQLite, not +in memory. Expiry is a sliding idle window (`PANEL_SESSION_IDLE_DAYS`, default +seven days) with no absolute lifetime cap — an admin who keeps using the panel +stays signed in indefinitely. HTMX polling on the monitoring screens +(Deliveries, Mail queue, System log, and the Status health fragment) does +**not** count as activity, so an auto-refreshing tab left open will not keep a +session alive forever. Changing the password signs out every other session but +leaves the current browser signed in. + +**Upgrading.** Bump the pinned image tag in `docker-compose.yml` to the target +release, then `docker compose up -d`. The backup version check requires the +running image to match the version that created a full backup — see [Fixed image +tag](#fixed-image-tag). + +**Container health.** The image declares a Docker `HEALTHCHECK` that probes +`GET /healthz` on port 8080 (unauthenticated). It returns `200 ok` when +OpenDKIM, the panel, and Postfix are all `RUNNING` under supervisord; +otherwise `503 unhealthy`. This catches a dead mail path that would still leave +the HTTP server up, but it does **not** verify TLS certificates, DNS records, +or end-to-end delivery — use the authenticated **Status** page for that. External +monitoring can use the same endpoint through the reverse proxy if you expose it, +or poll `docker inspect` health state on the host. + +## Rate limiting + +SelfPost applies two independent limits; both can refuse a submission, but only +level 2 writes a `rejected` row in the send log. + +**Level 1 (IP backstop)** — always on, configured via `.env`: + +- `RATE_LIMIT_MESSAGES_PER_IP` → Postfix `smtpd_client_message_rate_limit` +- `RATE_LIMIT_WINDOW_SECONDS` → Postfix `anvil_rate_time_unit` + +This is an anvil limit per connecting client IP. It keeps working even if the +journal-milter (level 2) is down. + +**Level 2 (per domain / per application)** — optional, configured in the panel +on each domain's page or on an individual application. You set a message +ceiling, a time window, and optionally restrict the limit to specific client +IPs; an empty IP list means the differentiated limit does not apply. When +exceeded, Postfix returns a 4xx and the refusal is recorded in Deliveries as +`rejected`. + ## Backup, restore, and moving a single domain Two related but distinct operations — spec 7.5: @@ -150,10 +272,20 @@ Two related but distinct operations — spec 7.5: ``` **Restore** means unpacking that archive into a fresh `/data` bind mount and starting a container of the **exact same image version** that created it — - SelfPost refuses to start otherwise and tells you which tag to use. This is - why the compose file below pins a fixed tag rather than `:latest`: without a - known version, there'd be no way to tell which image restoring a given - backup actually requires. + SelfPost refuses to start otherwise and tells you which tag to use. On the + first successful start after restore, `manifest.json` from the archive is + **deleted** — it guards only that one boot, so a later in-place upgrade is + not blocked. This is why the compose file below pins a fixed tag rather than + `:latest`: without a known version, there'd be no way to tell which image + restoring a given backup actually requires. + + **Alternative: archive `./data` while stopped.** If the service can be taken + offline, `docker compose down` then `tar czf selfpost-data.tar.gz ./data` on + the host is safe — nothing is writing to SQLite. Do **not** tar `./data` while + the container is running: the database uses WAL mode and a naive copy can + capture an inconsistent snapshot. The panel/CLI backup remains preferable when + you cannot afford downtime because it takes a consistent SQLite snapshot via + the Backup API on a live container. - **Export/import a single domain** (domain page → *Export domain* to write the file, *Backup* → *Import a domain* to read it back in): moves one domain — its DKIM key and its applications' **working** @@ -164,7 +296,50 @@ Two related but distinct operations — spec 7.5: Both files are **secrets** — they contain the admin password hash (full backup) or working application credentials (domain export) in the clear or in directly reversible form. Treat them like any other credential material: -encrypt at rest, restrict who can read them, don't email them around. +restrict who can read them, don't email them around — and encrypt them, which +SelfPost can do for you. + +### Encrypting a backup or export + +Both download forms carry an **Encrypt with a password** checkbox. Ticked, the +file that comes down is an encrypted envelope instead of the plain archive: + +| Artefact | Plain | Encrypted | +|----------|-------|-----------| +| Full backup | `.tar.gz` | `.spbk` | +| Domain export | `.json` | `.spde` | + +The key is derived from the password with scrypt and the contents are sealed +with AES-256-GCM, in chunks, so a truncated or altered file fails to open rather +than restoring quietly. **SelfPost does not store the password** — lose it and +the file is unrecoverable, which is the entire point. + +*Import a domain* takes an encrypted export directly: tick **The file is +encrypted** and give the password. + +A full backup has to be turned back into a plain archive before it can be +unpacked into `/data`, which the CLI does with the same password: + +```sh +docker exec -i selfpost-backup -decrypt < backup.spbk > backup.tar.gz +``` + +The CLI also *writes* encrypted backups for scripted/cron use. The password +comes from `SELFPOST_BACKUP_PASSWORD` or `-password-file ` (first line), +never from a command-line argument, which would be visible in the process list: + +```sh +docker exec -e SELFPOST_BACKUP_PASSWORD="$PW" selfpost-backup > backup.spbk +``` + +With no password set, the CLI keeps writing the plain `.tar.gz` it always has. + +## Published ports + +`deploy/docker-compose.yml` maps **465** and **587** to the host. Port 465 +(smtps) is always active. Port **587** is published even when +`SUBMISSION_ENABLE=false`; nothing listens until you set it to `true` — harmless, +but it can look like an open port in external scans. ## Fixed image tag @@ -189,8 +364,8 @@ swap file is cheap insurance against those occasional coincident spikes. ## Repository -- Primary: -- Mirror: + — the single home of the project: source, +issues, releases and the `ghcr.io/mixeme/selfpost` image all live there. ## License diff --git a/build/Dockerfile b/build/Dockerfile index d45394f..7aad6f9 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -13,8 +13,7 @@ WORKDIR /src # Version stamped into both binaries; MUST match the image tag (spec 7.5.A). ARG VERSION=dev -# Module metadata first for layer caching. go.sum arrived in Phase 2 with the -# SQLite driver and bcrypt. +# Module metadata first for layer caching. COPY go.mod go.sum ./ RUN go mod download @@ -24,10 +23,10 @@ COPY internal ./internal ENV CGO_ENABLED=0 RUN go vet ./... \ && go build -trimpath \ - -ldflags "-X codeberg.org/mix/selfpost/internal/buildinfo.Version=${VERSION}" \ + -ldflags "-X github.com/mixeme/selfpost/internal/buildinfo.Version=${VERSION}" \ -o /out/panel ./cmd/panel \ && go build -trimpath \ - -ldflags "-X codeberg.org/mix/selfpost/internal/buildinfo.Version=${VERSION}" \ + -ldflags "-X github.com/mixeme/selfpost/internal/buildinfo.Version=${VERSION}" \ -o /out/selfpost-backup ./cmd/selfpost-backup # ---- runtime stage ----------------------------------------------------------- @@ -36,7 +35,7 @@ FROM debian:bookworm-slim AS runtime ENV DEBIAN_FRONTEND=noninteractive # Preseed Postfix so its install is non-interactive and yields a working -# main.cf. The real relay configuration is generated by the panel in Phase 5. +# main.cf. The real relay configuration is generated by the panel. RUN echo "postfix postfix/mailname string localhost" | debconf-set-selections \ && echo "postfix postfix/main_mailer_type string Internet Site" | debconf-set-selections \ && apt-get update \ @@ -50,6 +49,7 @@ RUN echo "postfix postfix/mailname string localhost" | debconf-set-selections \ supervisor \ logrotate \ ca-certificates \ + curl \ && rm -rf /var/lib/apt/lists/* # Unprivileged user for the panel process (spec 7.6.8). @@ -104,6 +104,11 @@ RUN chmod +x /usr/local/bin/postfix-wrapper.sh /usr/local/bin/postfix-config.sh # which needs no inbound listener or EXPOSE. EXPOSE 8080 465 587 +# Liveness probe: panel HTTP plus mail-path processes (opendkim, panel, postfix). +# Does not verify TLS, DNS, or end-to-end delivery — see README Operations. +HEALTHCHECK --interval=30s --timeout=5s --start-period=90s --retries=3 \ + CMD curl -fsS http://127.0.0.1:8080/healthz || exit 1 + # The entrypoint fixes /data ownership (bind mount) as root, then execs # supervisord, which becomes PID 1 and owns process supervision (spec 4). ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/build/entrypoint.sh b/build/entrypoint.sh index bcfaa40..779f92e 100644 --- a/build/entrypoint.sh +++ b/build/entrypoint.sh @@ -36,7 +36,7 @@ find /data/opendkim/keys -type f -name '*.private' -exec chmod 0640 {} + # sasldb2 via saslpasswd2; Postfix (user `postfix`) reads it to authenticate SMTP # clients. Share it through the `selfpost` group the same way as the DKIM tree: # setgid directory so new files inherit the group, and the database itself -# group-readable (0640). Postfix wiring to actually consult it lands in Phase 5. +# group-readable (0640). mkdir -p /data/sasl chown -R panel:selfpost /data/sasl chmod 2750 /data/sasl @@ -51,9 +51,9 @@ chown -R panel:selfpost /data/postfix chmod 2750 /data/postfix chmod 0640 /data/postfix/sender_login_maps -# Milter socket directories (spec 5 p.3, 7.3). From Phase 5 Postfix (user -# `postfix`) must actually CONNECT to both milter sockets — OpenDKIM's and the -# panel's journal-milter — not just probe them at start-up. The sockets are +# Milter socket directories (spec 5 p.3, 7.3). Postfix (user `postfix`) must +# actually CONNECT to both milter sockets — OpenDKIM's and the panel's +# journal-milter — not just probe them at start-up. The sockets are # created by the opendkim and panel users respectively, so bridge them to # `postfix` through the shared `selfpost` group: group-owned + setgid dirs mean # each socket created inside inherits group `selfpost`, and group-traversable diff --git a/build/postfix-config.sh b/build/postfix-config.sh index 1ee6b83..61578f4 100644 --- a/build/postfix-config.sh +++ b/build/postfix-config.sh @@ -106,7 +106,7 @@ postconf -e \ "smtpd_sender_restrictions=reject_sender_login_mismatch, permit" # Level-1 rate limit by client IP (spec 5 p.5). Backstop that keeps working even -# if the journal-milter (level 2, Phase 8) is down. +# if the journal-milter (level 2) is down. postconf -e \ "smtpd_client_message_rate_limit=${RATE_MSGS}" \ "anvil_rate_time_unit=${RATE_WINDOW}s" diff --git a/build/postfix-wrapper.sh b/build/postfix-wrapper.sh index fb1d2b5..83e166b 100644 --- a/build/postfix-wrapper.sh +++ b/build/postfix-wrapper.sh @@ -8,7 +8,7 @@ # an explicit startup failure instead of a relay running blind. # # This handles cold start only. Runtime milter failures after a successful start -# are governed by fail-open (milter_default_action), configured in Phase 5. +# are governed by fail-open (milter_default_action). set -eu OPENDKIM_SOCK="${OPENDKIM_SOCKET:-/run/opendkim/opendkim.sock}" diff --git a/cmd/panel/envdoc_test.go b/cmd/panel/envdoc_test.go new file mode 100644 index 0000000..3bb8675 --- /dev/null +++ b/cmd/panel/envdoc_test.go @@ -0,0 +1,133 @@ +package main + +import ( + "slices" + "testing" +) + +// documentedPublic matches the README "Environment variables" table. +var documentedPublic = []string{ + "SELFPOST_HOSTNAME", + "SUBMISSION_ENABLE", + "RATE_LIMIT_MESSAGES_PER_IP", + "RATE_LIMIT_WINDOW_SECONDS", + "SEND_LOG_RETENTION_DAYS", + "PANEL_SESSION_IDLE_DAYS", + "SELFPOST_DNS_RESOLVERS", + "TRUSTED_PROXY_CIDR", +} + +// documentedInternal matches README "Internal variables (not part of the operator interface)". +var documentedInternal = []string{ + "SELFPOST_DATA_DIR", + "SELFPOST_DB_PATH", + "SELFPOST_SETUP_TOKEN_FILE", + "PANEL_HTTP_ADDR", + "JOURNAL_MILTER_SOCKET", + "MAIL_LOG", + "PANEL_COOKIE_SECURE", + "OPENDKIM_SOCKET", + "OPENDKIM_DIR", + "DKIM_SELECTOR_DEFAULT", + "SASL_DB_PATH", + "SASL_REALM", + "POSTFIX_DIR", + "POSTFIX_SENDER_LOGIN_MAPS", + "MILTER_CONNECT_TIMEOUT", + "MILTER_COMMAND_TIMEOUT", + "MILTER_CONTENT_TIMEOUT", + "MILTER_WAIT_TIMEOUT", + "TLS_RELOAD_INTERVAL_SECONDS", + "LOGROTATE_INTERVAL_SECONDS", +} + +// documentedComposeFixed are env keys documented outside the table (TLS paths fixed in compose). +var documentedComposeFixed = []string{ + "TLS_CERT_FILE", + "TLS_KEY_FILE", +} + +// loadConfigKeys is every environment variable read by loadConfig() in main.go. +// Update together with loadConfig when adding a new key. +var loadConfigKeys = []string{ + "SELFPOST_DATA_DIR", + "PANEL_HTTP_ADDR", + "JOURNAL_MILTER_SOCKET", + "MAIL_LOG", + "SEND_LOG_RETENTION_DAYS", + "SELFPOST_DB_PATH", + "SELFPOST_SETUP_TOKEN_FILE", + "SELFPOST_HOSTNAME", + "PANEL_COOKIE_SECURE", + "SUBMISSION_ENABLE", + "TRUSTED_PROXY_CIDR", + "PANEL_SESSION_IDLE_DAYS", + "SELFPOST_DNS_RESOLVERS", + "TLS_CERT_FILE", + "OPENDKIM_SOCKET", + "OPENDKIM_DIR", + "DKIM_SELECTOR_DEFAULT", + "SASL_DB_PATH", + "SASL_REALM", + "POSTFIX_DIR", +} + +// buildScriptKeys is every ${VAR:-…} / os.Getenv used in build/*.sh and entrypoint.sh +// but not necessarily in loadConfig. Update when startup scripts gain a new knob. +var buildScriptKeys = []string{ + "SELFPOST_HOSTNAME", + "TLS_CERT_FILE", + "TLS_KEY_FILE", + "RATE_LIMIT_MESSAGES_PER_IP", + "RATE_LIMIT_WINDOW_SECONDS", + "OPENDKIM_SOCKET", + "JOURNAL_MILTER_SOCKET", + "POSTFIX_SENDER_LOGIN_MAPS", + "SASL_DB_PATH", + "SUBMISSION_ENABLE", + "MILTER_CONNECT_TIMEOUT", + "MILTER_COMMAND_TIMEOUT", + "MILTER_CONTENT_TIMEOUT", + "MILTER_WAIT_TIMEOUT", + "TLS_RELOAD_INTERVAL_SECONDS", + "LOGROTATE_INTERVAL_SECONDS", +} + +func documentedKeys() []string { + keys := append([]string{}, documentedPublic...) + keys = append(keys, documentedInternal...) + keys = append(keys, documentedComposeFixed...) + slices.Sort(keys) + return slices.Compact(keys) +} + +func TestLoadConfigKeysDocumented(t *testing.T) { + doc := documentedKeys() + for _, key := range loadConfigKeys { + if !slices.Contains(doc, key) { + t.Errorf("loadConfig reads %s but it is not listed in README public, internal, or compose-fixed env docs", key) + } + } +} + +func TestBuildScriptKeysDocumented(t *testing.T) { + doc := documentedKeys() + for _, key := range buildScriptKeys { + if !slices.Contains(doc, key) { + t.Errorf("build scripts read %s but it is not listed in README env documentation", key) + } + } +} + +func TestDocumentedKeysAreRead(t *testing.T) { + read := append([]string{}, loadConfigKeys...) + read = append(read, buildScriptKeys...) + slices.Sort(read) + read = slices.Compact(read) + + for _, key := range documentedKeys() { + if !slices.Contains(read, key) { + t.Errorf("README documents %s but no code in loadConfig or build scripts reads it", key) + } + } +} diff --git a/cmd/panel/httpserver.go b/cmd/panel/httpserver.go index 8a1e6b8..39bc84e 100644 --- a/cmd/panel/httpserver.go +++ b/cmd/panel/httpserver.go @@ -7,17 +7,17 @@ import ( "net/http" "time" - "codeberg.org/mix/selfpost/internal/app" - "codeberg.org/mix/selfpost/internal/buildinfo" - "codeberg.org/mix/selfpost/internal/domain" - "codeberg.org/mix/selfpost/internal/postfix" - "codeberg.org/mix/selfpost/internal/store" - "codeberg.org/mix/selfpost/internal/web" + "github.com/mixeme/selfpost/internal/app" + "github.com/mixeme/selfpost/internal/buildinfo" + "github.com/mixeme/selfpost/internal/domain" + "github.com/mixeme/selfpost/internal/postfix" + "github.com/mixeme/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/web" ) // serveHTTP runs the control-panel HTTP server until ctx is cancelled, using -// the database handle shared by all roles. From Phase 2 this serves the real -// setup, login and authenticated panel surface (spec 7.6). +// the database handle shared by all roles: setup, login and the authenticated +// panel surface (security.md). func serveHTTP(ctx context.Context, cfg config, st *store.Store) error { // Applications own the SASL accounts and the Postfix sender map; the domain // service delegates to them when a domain (and its applications) is deleted. diff --git a/cmd/panel/journal.go b/cmd/panel/journal.go index e15dbdc..bc206b9 100644 --- a/cmd/panel/journal.go +++ b/cmd/panel/journal.go @@ -8,14 +8,14 @@ import ( "os" "path/filepath" - "codeberg.org/mix/selfpost/internal/milter" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/milter" + "github.com/mixeme/selfpost/internal/store" ) // serveJournal opens the journal-milter Unix socket and runs the real milter -// (spec 7.3), recording accepted messages into the send log. Socket lifecycle -// (creation, stale cleanup, group permissions) lives here; the protocol handler -// lives in internal/milter. +// (architecture.md § Mail path), recording accepted messages into the send +// log. Socket lifecycle (creation, stale cleanup, group permissions) lives +// here; the protocol handler lives in internal/milter. func serveJournal(ctx context.Context, cfg config, st *store.Store) error { socketPath := cfg.journalSocket if err := os.MkdirAll(filepath.Dir(socketPath), 0o755); err != nil { diff --git a/cmd/panel/main.go b/cmd/panel/main.go index 8c11635..a2ea695 100644 --- a/cmd/panel/main.go +++ b/cmd/panel/main.go @@ -1,10 +1,7 @@ -// Command panel is the SelfPost control panel. In the finished product this -// single binary combines several roles (spec 7.1): the HTTP panel server, -// the journal-milter, the mail.log tailer and the rate-limit checks. -// -// Phase 1 wires those roles up as a supervised process with a minimal HTTP -// stub, a journal-milter socket stub (so the Postfix start wrapper's readiness -// probe passes) and a log-tailer stub. Real behaviour lands in later phases. +// Command panel is the SelfPost control panel. This single binary combines +// several roles (architecture.md § Image and processes) as a supervised +// process: the HTTP panel server, the journal-milter, the mail.log tailer and +// the rate-limit checks. package main import ( @@ -21,11 +18,11 @@ import ( "sync" "syscall" - "codeberg.org/mix/selfpost/internal/backup" - "codeberg.org/mix/selfpost/internal/buildinfo" - "codeberg.org/mix/selfpost/internal/dnscheck" - "codeberg.org/mix/selfpost/internal/logtail" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/backup" + "github.com/mixeme/selfpost/internal/buildinfo" + "github.com/mixeme/selfpost/internal/dnscheck" + "github.com/mixeme/selfpost/internal/logtail" + "github.com/mixeme/selfpost/internal/store" ) func main() { @@ -85,8 +82,9 @@ func loadConfig() config { httpAddr: envDefault("PANEL_HTTP_ADDR", ":8080"), journalSocket: envDefault("JOURNAL_MILTER_SOCKET", "/run/selfpost/journal.sock"), mailLog: envDefault("MAIL_LOG", "/var/log/mail.log"), - // Send-log retention window (spec 7.3). Non-positive/invalid falls back - // to the 90-day default inside the log-tailer. + // Send-log retention window (architecture.md § Persistence). + // Non-positive/invalid falls back to the 90-day default inside the + // log-tailer. retentionDays: envInt("SEND_LOG_RETENTION_DAYS", 90), dataDir: dataDir, @@ -94,7 +92,7 @@ func loadConfig() config { manifestPath: filepath.Join(dataDir, backup.ManifestName), setupTokenPath: envDefault("SELFPOST_SETUP_TOKEN_FILE", filepath.Join(dataDir, "setup-token")), hostname: os.Getenv("SELFPOST_HOSTNAME"), - // Secure cookies by default (spec 7.6.6); PANEL_COOKIE_SECURE=false is a + // Secure cookies by default (security.md); PANEL_COOKIE_SECURE=false is a // development-only escape hatch for testing over plain HTTP. cookieSecure: envDefault("PANEL_COOKIE_SECURE", "true") != "false", // Whether this deployment also runs the 587 submission listener. The @@ -106,7 +104,7 @@ func loadConfig() config { // XFF header is trivially forgeable, so it's ignored unless the panel is // told which proxy to trust. trustedProxies: parseTrustedProxies(os.Getenv("TRUSTED_PROXY_CIDR")), - // Sliding session idle timeout (spec 7.6.6, plan B.1). Non-positive/invalid + // Sliding session idle timeout (security.md, plan B.1). Non-positive/invalid // falls back to the 7-day default inside internal/web. sessionIdleDays: envInt("PANEL_SESSION_IDLE_DAYS", 7), // Recursive resolvers the deliverability checks query directly. Empty @@ -116,15 +114,16 @@ func loadConfig() config { tlsCertFile: envDefault("TLS_CERT_FILE", "/etc/postfix/tls/fullchain.pem"), opendkimSocket: envDefault("OPENDKIM_SOCKET", "/run/opendkim/opendkim.sock"), - // Per-domain DKIM state (spec 6). The directory layout matches what - // entrypoint.sh prepares (setgid, shared `selfpost` group). + // Per-domain DKIM state (architecture.md § OpenDKIM). The directory layout + // matches what entrypoint.sh prepares (setgid, shared `selfpost` group). opendkimDir: envDefault("OPENDKIM_DIR", filepath.Join(dataDir, "opendkim")), dkimSelectorDef: envDefault("DKIM_SELECTOR_DEFAULT", "selfpost"), - // Application SASL accounts and the Postfix sender map (spec 5.1, 9), - // both under /data so they survive restarts. The SASL realm defaults to - // the server hostname so account identities line up with Postfix's SASL - // configuration; it falls back to localhost outside the container. + // Application SASL accounts and the Postfix sender map (architecture.md § + // Mail path), both under /data so they survive restarts. The SASL realm + // defaults to the server hostname so account identities line up with + // Postfix's SASL configuration; it falls back to localhost outside the + // container. saslDBPath: envDefault("SASL_DB_PATH", filepath.Join(dataDir, "sasl", "sasldb2")), saslRealm: saslRealm(), postfixDir: envDefault("POSTFIX_DIR", filepath.Join(dataDir, "postfix")), @@ -133,7 +132,7 @@ func loadConfig() config { // saslRealm chooses the realm new SASL accounts live under. It mirrors the // hostname Postfix's SASL layer uses so a client authenticating with a bare -// login resolves to the right account (finalised in Phase 5). +// login resolves to the right account. func saslRealm() string { if r := os.Getenv("SASL_REALM"); r != "" { return r @@ -195,9 +194,10 @@ func parseTrustedProxies(raw string) []*net.IPNet { } // run starts the panel's three roles and blocks until a shutdown signal or the -// first fatal error from any role. A signal triggers a clean stop of all roles; -// a role error cancels the others and is returned so the process exits non-zero -// (letting supervisord/Docker see the failure — spec 4). +// first fatal error from any role. A signal triggers a clean stop of all +// roles; a role error cancels the others and is returned so the process exits +// non-zero (letting supervisord/Docker see the failure — architecture.md § +// Image and processes). func run() error { cfg := loadConfig() @@ -206,10 +206,11 @@ func run() error { log.Printf("starting selfpost panel %s", buildinfo.Version) - // Restore version guard (spec 7.5.A): if a backup was extracted into /data, - // its manifest version must match this binary before we touch the database, - // so schema/format skew between versions cannot corrupt the restored state. - // A match consumes the manifest; its absence is the normal (non-restore) case. + // Restore version guard (architecture.md § Persistence): if a backup was + // extracted into /data, its manifest version must match this binary before we + // touch the database, so schema/format skew between versions cannot corrupt + // the restored state. A match consumes the manifest; its absence is the + // normal (non-restore) case. if err := backup.CheckRestore(cfg.manifestPath, buildinfo.Version); err != nil { return err } diff --git a/cmd/selfpost-backup/main.go b/cmd/selfpost-backup/main.go index a84434e..829bf9b 100644 --- a/cmd/selfpost-backup/main.go +++ b/cmd/selfpost-backup/main.go @@ -1,6 +1,6 @@ // Command selfpost-backup produces the full persistent-state archive from inside // the container, invoked via `docker exec` for scripted/cron backups — the CLI -// equivalent of the panel's backup button (spec 7.5.A, 11.6). +// equivalent of the panel's backup button (architecture.md § Persistence). // // By default the gzip-compressed tar is written to stdout, so the usual form is: // @@ -8,22 +8,40 @@ // // Use -o to write to a file instead. The resulting archive contains DKIM private // keys, the admin password hash and SASL credentials — treat it as a secret -// (spec 7.5.A). +// (architecture.md § Persistence). +// +// Given a password (SELFPOST_BACKUP_PASSWORD or -password-file, never an +// argument, which would show up in the process list) the archive is written as +// an encrypted .spbk envelope instead. Turn one back into a plain .tar.gz with +// the same password: +// +// docker exec -i selfpost-backup -decrypt < backup.spbk > backup.tar.gz package main import ( "flag" "fmt" + "io" "os" "path/filepath" + "strings" - "codeberg.org/mix/selfpost/internal/backup" - "codeberg.org/mix/selfpost/internal/buildinfo" + "github.com/mixeme/selfpost/internal/backup" + "github.com/mixeme/selfpost/internal/buildinfo" + "github.com/mixeme/selfpost/internal/secretfile" ) +// passwordEnv names the environment variable holding the encryption password. +// A password must never be a command-line argument: the process list is +// readable by every process in the container. +const passwordEnv = "SELFPOST_BACKUP_PASSWORD" + func main() { showVersion := flag.Bool("version", false, "print version and exit") - out := flag.String("o", "", "write the archive to this file instead of stdout") + out := flag.String("o", "", "write the output to this file instead of stdout") + in := flag.String("i", "", "read the encrypted archive from this file instead of stdin (-decrypt only)") + decrypt := flag.Bool("decrypt", false, "decrypt an encrypted backup (.spbk) back to a plain .tar.gz") + pwFile := flag.String("password-file", "", "read the encryption password from this file (first line); "+passwordEnv+" is used when unset") flag.Parse() if *showVersion { @@ -31,40 +49,137 @@ func main() { return } - if err := run(*out); err != nil { + password, err := readPassword(*pwFile) + if err == nil { + if *decrypt { + err = runDecrypt(*in, *out, password) + } else { + err = run(*out, password) + } + } + if err != nil { fmt.Fprintf(os.Stderr, "selfpost-backup: %v\n", err) os.Exit(1) } } -func run(outPath string) error { +// run writes a backup, encrypting it when a password was supplied. +func run(outPath, password string) error { dataDir := envDefault("SELFPOST_DATA_DIR", "/data") dbPath := envDefault("SELFPOST_DB_PATH", filepath.Join(dataDir, "selfpost.db")) - w := os.Stdout - if outPath != "" { - // Backups are secret; create them owner-only. - f, err := os.OpenFile(outPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + w, closeOut, err := openOutput(outPath) + if err != nil { + return err + } + defer closeOut() + + sink := w + var env *secretfile.Writer + if password != "" { + env, err = secretfile.NewWriter(w, secretfile.TypeFullBackup, password) if err != nil { return err } - defer f.Close() - w = f + sink = env } - if err := backup.Create(w, backup.Params{ + if err := backup.Create(sink, backup.Params{ DataDir: dataDir, DBPath: dbPath, Version: buildinfo.Version, }); err != nil { return err } + if env != nil { + if err := env.Close(); err != nil { + return err + } + } if outPath != "" { - fmt.Fprintf(os.Stderr, "selfpost-backup: wrote %s (SelfPost %s)\n", outPath, buildinfo.Version) + kind := "plain" + if password != "" { + kind = "encrypted" + } + fmt.Fprintf(os.Stderr, "selfpost-backup: wrote %s (%s, SelfPost %s)\n", outPath, kind, buildinfo.Version) } return nil } +// runDecrypt turns a .spbk envelope back into the plain gzip tar, so an +// encrypted backup can be extracted with ordinary tools during a restore. +func runDecrypt(inPath, outPath, password string) error { + if password == "" { + return fmt.Errorf("-decrypt needs the password (set %s or use -password-file)", passwordEnv) + } + + r := io.Reader(os.Stdin) + if inPath != "" { + f, err := os.Open(inPath) + if err != nil { + return err + } + defer f.Close() + r = f + } + + env, err := secretfile.NewReader(r, password) + if err != nil { + return err + } + if env.Type() != secretfile.TypeFullBackup { + return fmt.Errorf("that file is a %s, not a full backup", env.Type()) + } + + w, closeOut, err := openOutput(outPath) + if err != nil { + return err + } + defer closeOut() + + if _, err := io.Copy(w, env); err != nil { + return err + } + if outPath != "" { + fmt.Fprintf(os.Stderr, "selfpost-backup: wrote %s (decrypted)\n", outPath) + } + return nil +} + +// openOutput returns stdout, or a freshly created owner-only file: both plain +// and encrypted backups are secret enough to keep off other users' eyes. +func openOutput(outPath string) (io.Writer, func(), error) { + if outPath == "" { + return os.Stdout, func() {}, nil + } + f, err := os.OpenFile(outPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + return nil, nil, err + } + return f, func() { _ = f.Close() }, nil +} + +// readPassword takes the password from the given file (first line) or, when no +// file is named, from the environment. An empty result means "no encryption". +func readPassword(pwFile string) (string, error) { + if pwFile == "" { + return os.Getenv(passwordEnv), nil + } + data, err := os.ReadFile(pwFile) + if err != nil { + return "", fmt.Errorf("read password file: %w", err) + } + // A password file is usually written with a trailing newline; take the first + // line and strip the line ending, but keep any other whitespace, which may + // well be part of the password. + line, _, _ := strings.Cut(string(data), "\n") + line = strings.TrimSuffix(line, "\r") + if line == "" { + return "", fmt.Errorf("password file %s is empty", pwFile) + } + return line, nil +} + func envDefault(key, def string) string { if v := os.Getenv(key); v != "" { return v diff --git a/cmd/selfpost-backup/main_test.go b/cmd/selfpost-backup/main_test.go new file mode 100644 index 0000000..f6c2cf7 --- /dev/null +++ b/cmd/selfpost-backup/main_test.go @@ -0,0 +1,139 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mixeme/selfpost/internal/store" +) + +// seedDataDir builds the minimum /data tree a backup can be taken from. +func seedDataDir(t *testing.T) string { + t.Helper() + dataDir := t.TempDir() + st, err := store.Open(filepath.Join(dataDir, "selfpost.db")) + if err != nil { + t.Fatalf("open store: %v", err) + } + if _, err := st.AddDomain("example.com", "selfpost"); err != nil { + t.Fatalf("add domain: %v", err) + } + if err := st.Close(); err != nil { + t.Fatalf("close store: %v", err) + } + t.Setenv("SELFPOST_DATA_DIR", dataDir) + t.Setenv("SELFPOST_DB_PATH", filepath.Join(dataDir, "selfpost.db")) + return dataDir +} + +// An encrypted backup is only worth having if the container it came from can +// hand it back as an ordinary archive during a restore, so the two halves of +// the CLI are tested as the one round trip an operator actually performs. +func TestEncryptedBackupRoundTrip(t *testing.T) { + seedDataDir(t) + dir := t.TempDir() + encrypted := filepath.Join(dir, "backup.spbk") + plain := filepath.Join(dir, "backup.tar.gz") + const password = "a long enough password" + + if err := run(encrypted, password); err != nil { + t.Fatalf("create encrypted backup: %v", err) + } + head, err := os.ReadFile(encrypted) + if err != nil { + t.Fatalf("read backup: %v", err) + } + if !strings.HasPrefix(string(head), "SELFPOST") { + t.Fatalf("encrypted backup does not start with the envelope magic") + } + + if err := runDecrypt(encrypted, plain, "the wrong password"); err == nil { + t.Fatal("decryption with the wrong password succeeded") + } + if err := runDecrypt(encrypted, plain, password); err != nil { + t.Fatalf("decrypt: %v", err) + } + + // What comes out must be the same gzip tar the plain path produces. + f, err := os.Open(plain) + if err != nil { + t.Fatalf("open decrypted archive: %v", err) + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatalf("gzip: %v", err) + } + names := map[string]bool{} + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("tar: %v", err) + } + names[hdr.Name] = true + } + for _, want := range []string{"manifest.json", "selfpost.db"} { + if !names[want] { + t.Errorf("decrypted archive has no %s (entries: %v)", want, names) + } + } +} + +// Without a password the CLI keeps producing the plain archive that existing +// backup scripts consume. +func TestUnencryptedBackupStaysPlain(t *testing.T) { + seedDataDir(t) + out := filepath.Join(t.TempDir(), "backup.tar.gz") + if err := run(out, ""); err != nil { + t.Fatalf("create backup: %v", err) + } + f, err := os.Open(out) + if err != nil { + t.Fatalf("open archive: %v", err) + } + defer f.Close() + if _, err := gzip.NewReader(f); err != nil { + t.Fatalf("plain backup is not a gzip archive: %v", err) + } +} + +// Decrypting needs a password, and it must come from a file or the environment +// — never an argument, which the process list would expose. +func TestReadPassword(t *testing.T) { + dir := t.TempDir() + pwFile := filepath.Join(dir, "pw") + if err := os.WriteFile(pwFile, []byte("from the file\nignored second line\n"), 0o600); err != nil { + t.Fatalf("write password file: %v", err) + } + + t.Setenv(passwordEnv, "from the environment") + got, err := readPassword("") + if err != nil || got != "from the environment" { + t.Errorf("readPassword(\"\") = %q, %v", got, err) + } + got, err = readPassword(pwFile) + if err != nil || got != "from the file" { + t.Errorf("readPassword(file) = %q, %v", got, err) + } + + if err := os.WriteFile(pwFile, nil, 0o600); err != nil { + t.Fatalf("truncate password file: %v", err) + } + if _, err := readPassword(pwFile); err == nil { + t.Error("an empty password file was accepted") + } + + os.Unsetenv(passwordEnv) + if err := runDecrypt("", "", ""); err == nil { + t.Error("-decrypt without a password was accepted") + } +} diff --git a/deploy/.env.example b/deploy/.env.example index 4e045b8..b12d881 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -1,4 +1,5 @@ # Copy to .env next to docker-compose.yml and fill in. +# Full reference: README "Environment variables". # Required — the container refuses to start without it. Hostname used both for # the panel's TLS vhost and for Postfix's SASL realm (SASL_REALM defaults to diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 1f547a0..d673e0b 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -11,7 +11,8 @@ # Usage: # 1. Copy this file (and .env.example as .env) next to your own ./data and # ./certs directories, or adjust the paths below. -# 2. Fill in .env (hostname, at least one strong TLS_CERT/KEY path). +# 2. Fill in .env (at minimum SELFPOST_HOSTNAME). Put PEM files in ./certs — +# TLS paths are fixed in this file to match that bind mount, not .env. # 3. docker compose up -d # # The image tag below is FIXED on purpose (spec 10 p.10, 7.5.A): backup @@ -45,6 +46,10 @@ services: # report what the internet sees rather than what this host's own stub # resolver synthesises. Empty uses public defaults. SELFPOST_DNS_RESOLVERS: "${SELFPOST_DNS_RESOLVERS:-}" + # Reverse-proxy CIDRs allowed to supply X-Forwarded-For for login/setup + # rate-limiting. See README "Environment variables" — wrong value lets a + # client spoof its rate-limit key. + TRUSTED_PROXY_CIDR: "${TRUSTED_PROXY_CIDR:-}" volumes: # All persistent state lives under /data (spec 9): SQLite DB, DKIM keys, # sasldb2, sender map, setup token. Back this up (panel button or the @@ -62,6 +67,8 @@ services: # the host network at 127.0.0.1:8080 (see the vhost fragment), so the # panel is never directly reachable from the internet without TLS. - "465:465" + # 587 is mapped even when SUBMISSION_ENABLE=false; Postfix listens only + # when the variable is true — see README "Published ports". - "587:587" - "127.0.0.1:8080:8080" # Hardening (spec 10 p.6). SelfPost's entrypoint still needs to run as diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..750064f --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,273 @@ +# SelfPost — architecture (as-built) + +**Source of truth:** the code tree, not historical specs. Synchronise this file +when env keys, routes, or mail-path behaviour change. Verification method: +[documentation-plan.md](documentation-plan.md) §2. + +User install/operations: [README.md](../README.md). Product boundaries: +[product.md](product.md). + +--- + +## Image and processes + +Single Debian slim image. `entrypoint.sh` (root) fixes `/data` ownership and +milter socket directories, **requires** `SELFPOST_HOSTNAME` (FQDN with at least +one dot; no scheme, port, or spaces — invalid or empty value → `exit 1` before +Postfix config or supervisord), then execs `supervisord` as PID 1. + +The hostname is not a tunable default: it must match PTR/rDNS, TLS CN/SAN, and +SASL realm together. Soft fallbacks (`localhost` in the panel vs container ID in +Postfix) split realms and break SMTP AUTH with a silent `535` in clients while +the panel looks healthy. Shipped `docker-compose.yml` already requires the +variable; the entrypoint gate catches `docker run`, custom compose, and k8s. + +Managed programs ([build/supervisord.conf](../build/supervisord.conf)): + +| Program | User | Priority | Role | +|---|---|---|---| +| `opendkim` | opendkim | 100 | DKIM signing milter | +| `panel` | panel | 200 | HTTP UI + journal-milter + log-tailer goroutine | +| `postfix` | root (wrapper) | 300 | MTA — started only after both milter sockets exist | +| `postfix-reload` | root | — | On-demand `postfix reload` (autostart off) | +| `cert-reload` | root | 400 | Daily `postfix reload` for renewed TLS certs | +| `logrotate` | root | 400 | Periodic `mail.log` rotation | + +Start order: OpenDKIM → panel (opens journal-milter socket) → Postfix wrapper +polls unix sockets (timeout `MILTER_WAIT_TIMEOUT`, default 30s) then +`postfix start-fg`. + +`crashexit` event listener exits the container on any managed process FATAL so +Docker `restart: unless-stopped` recreates a broken instance. + +**Liveness:** `GET /healthz` (unauthenticated) returns 200 when opendkim, +panel, and postfix are RUNNING; Docker `HEALTHCHECK` uses the same probe. + +--- + +## Mail path + +``` +Client ──TLS+SASL──► Postfix (465 smtps, optional 587 submission) + │ + ├─► OpenDKIM milter (sign, tempfail on failure) + ├─► journal-milter (send log + L2 rate limits, fail-open) + └─► outbound MX delivery (port 25 client) +``` + +### Postfix ([build/postfix-config.sh](../build/postfix-config.sh)) + +- **465/smtps** — implicit TLS, SASL required; primary listener. +- **587/submission** — only when `SUBMISSION_ENABLE=true`; STARTTLS with + `smtpd_tls_security_level=encrypt`. +- **No open relay** — `permit_sasl_authenticated`, `reject_unauth_destination`; + `smtpd_sender_login_maps` + `reject_sender_login_mismatch`. +- **Level-1 rate limit** — `smtpd_client_message_rate_limit` / + `anvil_rate_time_unit` from `RATE_LIMIT_*` env vars; independent of milter. +- **Chroot disabled** for all services (DNS/TLS inside container). +- **TLS certs** — read-only mount at `TLS_CERT_FILE` / `TLS_KEY_FILE`; daily + reload via `cert-reload`. + +### OpenDKIM + +Per-domain keys under `/data/opendkim/keys`; `KeyTable` / `SigningTable` maintained +by the panel. Socket `/run/opendkim/opendkim.sock`. + +### Panel binary ([cmd/panel](../cmd/panel)) + +One process, three roles: + +1. **HTTP server** — `:8080` (`PANEL_HTTP_ADDR`); HTTPS terminated by reverse + proxy only. +2. **journal-milter** — unix socket `JOURNAL_MILTER_SOCKET`; records From/To/ + Subject/SASL user at DATA; enforces level-2 rate limits; **fail-open** + (`default_action=accept`) so milter failure does not stop mail. The level-2 + count is the stored send-log rows plus the messages this process has admitted + but not yet written (`internal/milter/inflight.go`), so concurrent sessions + cannot each spend the same last slot; a reservation is released at + end-of-message, on ABORT, or after a 10-minute TTL. +3. **log-tailer** — follows `MAIL_LOG`, updates send-log delivery status by + queue-id. Send-log `queued → sent` transitions depend on this goroutine alone + (`UpdateStatus` is only called from [internal/logtail](../internal/logtail/logtail.go)). + +Milter chain in Postfix: OpenDKIM (tempfail) then journal (accept on failure). + +### Log tailer and `mail.log` rotation + +`mail.log` lives under `/var/log` (not in `/data`). Rotation uses rename + +`postfix reload` ([build/logrotate-mail.conf](../build/logrotate-mail.conf)), not +`copytruncate` — the latter can drop `status=sent` lines and leave send-log rows +stuck at `queued`. After rename, logrotate runs `create 0644 root root` (Postfix +recreates the file lazily on first write as mode `0600`, which the unprivileged +panel user cannot read). `follow()` drains the old inode once more before +switching descriptors; the panel treats a missing log file as an empty tail, not +an error. + +**Read offset is persisted** (`logtail_state` table, migration `0003`): the +tailer stores its position plus a fingerprint of the log's first 512 bytes, and +on start resumes from it, parsing the tail written while the panel was down. If +the fingerprint no longer matches (rotated or recreated in the meantime) it reads +the current file from the start; re-parsing lines is harmless because +`UpdateStatus` writes the same status onto the same row. Only a first-ever start, +with nothing stored, begins at end-of-file, so installing the panel does not +replay a pre-existing log. + +**Remaining gap:** + +- **Container recreate** — `/var/log` is ephemeral; the log is lost with the + container, so the delivery lines for rows still `queued` are gone with it and + those rows stay `queued` forever. + +Possible follow-ups if this becomes painful: mount the mail log under `/data`, or +reconcile stuck rows via `postqueue`. + +--- + +## Panel HTTP surface + +Route table: [internal/web/web.go](../internal/web/web.go). Authenticated +unless noted. + +| Route | Purpose | +|---|---| +| `/healthz` | Liveness (no auth) | +| `/setup/*` | One-time admin bootstrap | +| `/login`, `/logout` | Session auth | +| `/status` | Process, cert, socket, PTR checks | +| `/domains`, `/domains/*` | Domain and application CRUD, DKIM, L2 limits | +| `/deliveries` | Send log with filters | +| `/mail-queue` | Postfix queue view | +| `/system-log` | `mail.log` tail | +| `/reload` | Reload OpenDKIM + Postfix maps | +| `/backup` | Full backup download, domain import | +| `/account` | Admin username/password | + +HTMX polling refreshes monitoring fragments; polling does not extend session +idle timeout (only non-`HX-Request` GET and mutating requests count as activity). + +### Sessions + +Stored in SQLite (`sessions` table, migration `0002`): cookie holds a random +token; the database stores **SHA-256 of the token**, not the token itself — a +stolen DB or backup archive does not alone grant login, but a browser that still +holds the cookie works after process restart, redeploy, or full backup restore. + +- **Idle timeout** — sliding window, `PANEL_SESSION_IDLE_DAYS` (default 7); no + absolute cap (regular use keeps the session alive indefinitely). +- **Renewal** — DB `last_seen` and cookie `Max-Age` update at most once per hour + (`renewThreshold` in [internal/web/session.go](../internal/web/session.go)). +- **Password change** — all other sessions are deleted; the current session stays + active ([internal/store/sessions.go](../internal/store/sessions.go), + [handlers_account.go](../internal/web/handlers_account.go)). + +Restoring an **older** backup also restores session rows: a session invalidated +after that backup was taken can become valid again if the browser still has the +cookie and idle timeout has not expired. + +--- + +## Code layers + +Handlers never touch SQLite or the filesystem directly; every write that has to +land in more than one place (SQLite row, `sasldb2` entry, Postfix map, OpenDKIM +table) goes through a service, which is also where the rollback of a partial +failure lives. The adapters below the services are the only code that knows +about Postfix, OpenDKIM, DNS or the log file, which is what makes them +substitutable in tests — `milter.Store`, `app.SenderMaps` and +`logtail.StatusStore` are the seams the unit tests replace with fakes. + +```mermaid +flowchart TB + subgraph cmd ["cmd — composition root"] + panel["panel: HTTP + journal-milter + log-tailer"] + backupcli["selfpost-backup CLI"] + end + subgraph web ["internal/web — HTTP surface"] + handlers["handlers_*.go, templates, session/security"] + end + subgraph services ["Services — multi-store operations + rollback"] + domainSvc["internal/domain"] + appSvc["internal/app"] + end + subgraph persistence ["Persistence"] + store["internal/store — SQLite, embedded migrations"] + end + subgraph adapters ["Adapters — the only infrastructure-aware code"] + postfix["internal/postfix"] + milterPkg["internal/milter"] + logtail["internal/logtail"] + dnscheck["internal/dnscheck"] + backupPkg["internal/backup"] + health["internal/health"] + secretfile["internal/secretfile"] + end + panel --> web + panel --> milterPkg + panel --> logtail + backupcli --> backupPkg + backupcli --> secretfile + web --> domainSvc + web --> appSvc + web --> backupPkg + web --> dnscheck + web --> health + web --> secretfile + domainSvc --> store + appSvc --> store + milterPkg --> store + logtail --> store + domainSvc --> postfix + appSvc --> postfix +``` + +The three roles inside the `panel` process (HTTP server, journal-milter, +log-tailer goroutine) share one binary and one SQLite handle on purpose — see +[Panel binary](#panel-binary-cmdpanel) for why, and *Persistence* below for the +single-connection trade-off that follows from it. + +--- + +## Persistence (`/data` bind mount) + +| Path | Contents | +|---|---| +| `selfpost.db` | SQLite: domains, apps, admin, sessions, send log, L2 limits, log-tailer offset | +| `setup-token` | First-run setup token file | +| `opendkim/` | DKIM keys + tables | +| `sasl/sasldb2` | Application SASL credentials | +| `postfix/sender_login_maps` | Login → From binding | +| `manifest.json` | Backup version stamp (consumed on restore) | + +Not in `/data`: TLS certificates (reverse-proxy mount), Postfix queue +(transit mail not migrated by design). + +**Rotation:** send-log retention `SEND_LOG_RETENTION_DAYS` (default 90); +`mail.log` via logrotate (14 rotated files, check every 6h, rename + +`postfix reload` in `postrotate` — see § Log tailer above). + +**Backup:** panel button or `selfpost-backup` CLI — SQLite snapshot + tar of +`/data` tree; version check on restore. Stopped-container `tar` of `./data` is +safe (see README). + +**Optional encryption** of the two secret-bearing downloads +([internal/secretfile](../internal/secretfile/secretfile.go)): password → +scrypt → AES-256-GCM over 64 KiB chunks, each authenticated with the header, +its counter and an end-of-stream flag (so truncation and reordering fail to +open). Full backup `.tar.gz` → `.spbk`, domain export `.json` → `.spde`; the +plain forms remain the default. Domain import detects the envelope by magic +bytes; an encrypted full backup is converted back with `selfpost-backup +-decrypt` before restore. + +--- + +## Security (summary) + +Mandatory checklist: [security.md](security.md). Accepted trade-offs (CSRF +origin check, no CSRF tokens) are documented there separately. + +--- + +## Configuration + +Public and internal env vars: [README § Environment variables](../README.md#environment-variables). +Regression test: [cmd/panel/envdoc_test.go](../cmd/panel/envdoc_test.go). diff --git a/docs/specification.md b/docs/archive/specification-v1.0.md similarity index 99% rename from docs/specification.md rename to docs/archive/specification-v1.0.md index ac77da3..a9ebcae 100644 --- a/docs/specification.md +++ b/docs/archive/specification-v1.0.md @@ -1,3 +1,8 @@ +> **Исторический снимок v1.0, не источник истины.** Актуальные документы: +> [product.md](../product.md), [architecture.md](../architecture.md), +> [development.md](../development.md), [security.md](../security.md), +> [README.md](../../README.md). + # Техническое задание: SelfPost **Версия:** 1.0 diff --git a/docs/assets/selfpost-icon-16.svg b/docs/assets/selfpost-icon-16.svg index 9e714db..f197cf3 100644 --- a/docs/assets/selfpost-icon-16.svg +++ b/docs/assets/selfpost-icon-16.svg @@ -1,5 +1,5 @@ - diff --git a/docs/assets/selfpost-icon.svg b/docs/assets/selfpost-icon.svg index 8b29d72..c724436 100644 --- a/docs/assets/selfpost-icon.svg +++ b/docs/assets/selfpost-icon.svg @@ -1,5 +1,5 @@ - diff --git a/docs/assets/selfpost-proof.html b/docs/assets/selfpost-proof.html index 757dd0e..e885a15 100644 --- a/docs/assets/selfpost-proof.html +++ b/docs/assets/selfpost-proof.html @@ -89,7 +89,7 @@

Эталон

- @@ -112,7 +112,7 @@
mixeme / selfpost
- @@ -126,7 +126,7 @@
mixeme / selfpost
- @@ -144,7 +144,7 @@

Шапка панели

- @@ -166,14 +166,14 @@
- SP SelfPost — Domains
- @@ -186,11 +186,11 @@
- - - + + +
-
80 / 48 / 32 px
Аватар организации на Codeberg и GitHub. Порог перехода на версию SP — 32 px.
+
80 / 48 / 32 px
Аватар организации на GitHub. Порог перехода на версию SP — 32 px.
@@ -199,7 +199,7 @@

Размеры и охранное поле

- @@ -208,14 +208,14 @@ SELF-HOSTED SMTP RELAY полная · от 280 px
- SelfPost компактная · от 100 px
- diff --git a/docs/assets/selfpost-stamp-compact.svg b/docs/assets/selfpost-stamp-compact.svg index 3c03928..9676b21 100644 --- a/docs/assets/selfpost-stamp-compact.svg +++ b/docs/assets/selfpost-stamp-compact.svg @@ -1,5 +1,5 @@ - diff --git a/docs/assets/selfpost-stamp.svg b/docs/assets/selfpost-stamp.svg index cb46ba3..3054766 100644 --- a/docs/assets/selfpost-stamp.svg +++ b/docs/assets/selfpost-stamp.svg @@ -1,5 +1,5 @@ - diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..26d26e6 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,146 @@ +# SelfPost — development + +**What this file is.** How to build, test, and ship changes. Current sprint +state lives in [progress.md](progress.md) — read that first after `/clear`. + +Product boundaries: [product.md](product.md). As-built layout: +[architecture.md](architecture.md). + +--- + +## Repository layout + +- `cmd/panel` — panel HTTP server + milter + log-tailer +- `cmd/selfpost-backup` — CLI backup (`docker exec … selfpost-backup`) +- `internal/` — domain logic, store, web handlers, health checks +- `build/` — Dockerfile, supervisord, Postfix/OpenDKIM wiring, entrypoint +- `deploy/` — `docker-compose.yml`, proxy examples, `.env.example` +- `test/e2e/` — **separate Go module**; container integration tests + +--- + +## Local Go workflow + +Requires Go 1.26+ and `CGO_ENABLED=0` (pure Go SQLite). + +```sh +make vet # go vet ./... +make test # go test ./... +make build # bin/panel, bin/selfpost-backup (VERSION=dev by default) +make build VERSION=1.0.0 +``` + +Or directly: + +```sh +go vet ./... +go test ./... +go build -trimpath -ldflags "-X github.com/mixeme/selfpost/internal/buildinfo.Version=dev" -o bin/panel ./cmd/panel +``` + +**Env documentation regression:** `go test ./cmd/panel -run TestLoadConfig` — +new `loadConfig` keys must appear in README env lists +([cmd/panel/envdoc_test.go](../cmd/panel/envdoc_test.go)). + +--- + +## End-to-end tests + +Hermetic container suite (implemented as `test/e2e/`, separate Go module): + +```sh +make e2e +# equivalent: cd test/e2e && go test -v -timeout 20m ./... +``` + +**Stack:** shipped [deploy/docker-compose.yml](../deploy/docker-compose.yml) + +[test/e2e/compose.override.yml](../test/e2e/compose.override.yml) — same +`cap_drop`/`cap_add`/`no-new-privileges` as production. Override uses high +ports (`20465`/`20587`/`20080`), test hostname, self-signed TLS, +`PANEL_COOKIE_SECURE=false`, isolated compose project. Mail is hermetic: CoreDNS +fake zone + Postfix `smtp-sink` as sink-MX; DKIM TXT is scraped from the panel +and published into the zone so the test verifies the record the operator would +use. + +**Coverage (summary):** full bootstrap → SMTP AUTH → delivery → DKIM verify → +send-log `queued → sent`; negatives (no AUTH, relay, sender/login mismatch, L1/L2 +limits, milter fail-open, bad `SELFPOST_HOSTNAME`, session survives +`docker restart`). Polling with timeouts only — no fixed `sleep`. + +Requires **Docker + Compose v2** on the test host. Not included in `go test ./...` +of the main module. + +**CI** ([release.yml](../.github/workflows/release.yml)): tag `vX.Y.Z` triggers +`prepare` (version from tag) → native matrix `[ubuntu-latest, ubuntu-24.04-arm]` +— build `--load`, e2e, push per-arch tag → `merge` publishes unified +`ghcr.io/...:X.Y.Z` via `docker buildx imagetools create`. Failed e2e blocks the +image. Ordinary pushes still run only `vet`/`test` in [test.yml](../.github/workflows/test.yml). + +--- + +## Dev server (full container) + +When unit tests are not enough — Postfix, OpenDKIM, supervisord, real SMTP: + +**Typical setup:** edit locally → sync tree to dev host → build/test there. + +Documented dev host: `selfpost.example.com` (Debian 12). Sync example from +[progress.md](progress.md): + +```sh +tar -czf - --exclude=.git . | ssh root@selfpost.example.com \ + 'rm -rf /root/selfpost-src && mkdir -p /root/selfpost-src && tar -xzf - -C /root/selfpost-src' +``` + +On the server (Go in `/usr/local/go/bin` if not in PATH): + +```sh +cd /root/selfpost-src +/usr/local/go/bin/go vet ./... +/usr/local/go/bin/go test ./... +docker build -f build/Dockerfile -t selfpost:dev --build-arg VERSION=dev . +``` + +Manual smoke on production-like host: panel at `https://selfpost.example.com`, +real LE cert, live deliverability — not replaceable by e2e alone (no outbound +25 on CI runners, no real PTR/reputation). + +--- + +## Commit and changelog protocol + +From [progress.md](progress.md): + +1. Meaningful step → entry under `[Unreleased]` in [CHANGELOG.md](../CHANGELOG.md) + (Keep a Changelog format). +2. Commit on `main` unless asked for a branch. **Do not commit unless the user + asks.** +3. Before `/clear` at end of a phase: update `progress.md`, verify acceptance + criteria, CHANGELOG, final commit. + +Documentation changes that add/rename env vars, panel routes, or observable +mail behaviour ship in the **same commit** as the code change. + +Release tagging and image push — only on explicit request. + +--- + +## Agent rules (formerly spec §12) + +1. **No git commits** without explicit instruction in the prompt. +2. After Go changes: `go build`, `go vet`; fix all issues. Run `go test` when + tests exist. +3. Before calling a container task done: image builds and container starts. +4. Iterate: minimal skeleton first, then features. +5. Security requirements in [security.md](security.md) — implement with the feature, + not deferred. +6. Do not implement out-of-scope items ([product.md](product.md)) or change + fixed assumptions without agreement. +7. For large tasks: propose a plan before coding unless the user already approved + one. +8. Check licence compatibility of new Go dependencies (permissive or + GPL-family for AGPL-3.0 project). + +Model routing (from progress): security/infra → Opus; UI/docs → Sonnet; +trivial mechanics → Haiku. Pre-release security **review** (not authorship) → +Fable. diff --git a/docs/documentation-plan.md b/docs/documentation-plan.md index 257714e..37b848e 100644 --- a/docs/documentation-plan.md +++ b/docs/documentation-plan.md @@ -1,248 +1,86 @@ # План документации SelfPost -**Зачем этот файл.** Документация — часть поставки (ТЗ раздел 11, пп. 5, 7, 9), -а не сопроводительный текст. Перед тегом релиза она обязана описывать **то, что -делает код**, а не то, что задумывалось: расхождение здесь — такой же дефект, -как несоответствие ТЗ, только обнаруживает его пользователь на своём проде. -План описывает, из чего состоит пакет, как он сверяется с кодом, что уже -разошлось (первый проход выполнен, результаты ниже) и что с этим делать. +**Статус: закрыт (D1–D9, август 2026).** Проход выполнен; история задач и +находок — в [CHANGELOG.md](../CHANGELOG.md) и `git log`. Этот файл дальше +держит **состав пакета**, **метод сверки с кодом** и **правила**, чтобы +документация не разошлась снова. -**Место в общем плане:** документационный проход идёт вместе с **D.5** -([implementation-plan.md](implementation-plan.md)) — до тега следующего релиза, -после B.1–B.3 и C.4, потому что именно они изменили поведение, которое README -описывает (сессии, ротация лога, обязательность `SELFPOST_HOSTNAME`). +**Живые документы (вместо архивного ТЗ):** -**Модель:** документация → Sonnet (правило [progress.md](progress.md)). -Исключение — **D6** (HEALTHCHECK/эндпоинт мониторинга) и формулировки про -`TRUSTED_PROXY_CIDR`: инфра/безопасность → Opus. +| Дом | Файл | +|---|---| +| Пользовательская поставка | [README.md](../README.md) | +| Границы продукта | [product.md](product.md) | +| As-built устройство | [architecture.md](architecture.md) | +| Процесс разработки | [development.md](development.md) | +| Безопасность | [security.md](security.md) | +| Исторический снимок v1.0 | [archive/specification-v1.0.md](archive/specification-v1.0.md) | + +Отложенная полировка v1.x (тег образа в compose) — [roadmap.md](roadmap.md) +§ «v1.x — хвост документации и деплоя». Пункты про Quick start и `docs/logo` +закрыты. --- ## 1. Состав пакета -### Поставляемое пользователю (обязательно по ТЗ) +### Поставляемое пользователю -| Артефакт | Требование | Состояние | -|---|---|---| -| [README.md](../README.md) | ТЗ 11 п. 7: установка, требования к площадке, per-domain DNS, прогрев IP, **эксплуатация**, бэкап/восстановление и экспорт/импорт домена (оба файла — секреты), репозиторий (Codeberg основной / GitHub зеркало), откуда образ (`ghcr.io`), фиксированный тег, минимальные требования к машине, лицензия в 2–3 предложениях | Есть всё, **кроме раздела «эксплуатация»**; см. находки 1–4, 10 | -| [LICENSE](../LICENSE) | ТЗ 11 п. 9: полный текст AGPL-3.0 | Полный текст на месте, правок не требует | -| [deploy/docker-compose.yml](../deploy/docker-compose.yml) + фрагменты прокси | ТЗ 11 п. 5, ТЗ 10 п. 3: Apache основной, nginx/Caddy/Traefik альтернативами; для каждого — что монтируется и в какие пути | Все четыре есть ([apache](../deploy/apache/), [nginx](../deploy/nginx/), [caddy](../deploy/caddy/), [traefik](../deploy/traefik/)), в README сведены таблицей; см. находки 7, 8 | -| [deploy/.env.example](../deploy/.env.example) | Пользовательские переменные с пояснениями | Есть 6 переменных; полного справочника нет — находка 3 | -| [CHANGELOG.md](../CHANGELOG.md) | Keep a Changelog, запись на каждом осмысленном шаге | Ведётся; `[Unreleased]` наполнен B.1–B.3, C.4 | +| Артефакт | Состояние | +|---|---| +| [README.md](../README.md) | Установка, площадка, DNS, прогрев IP, эксплуатация, rate limiting, env, бэкап, репозиторий, образ, лицензия | +| [LICENSE](../LICENSE) | AGPL-3.0, полный текст | +| [deploy/docker-compose.yml](../deploy/docker-compose.yml) + прокси | Apache + nginx/Caddy/Traefik в [deploy/](../deploy/) | +| [deploy/.env.example](../deploy/.env.example) | Публичные переменные; полный справочник в README | +| [CHANGELOG.md](../CHANGELOG.md) | Keep a Changelog | -### Рабочие документы проекта (не поставка, но обязаны быть верны) +### Рабочие документы -[specification.md](specification.md) — ТЗ v1.0, источник истины по требованиям. -[implementation-plan.md](implementation-plan.md) — открытые вопросы и линия 2.x. -[progress.md](progress.md) — живой трекер, читается первым после `/clear`. -[security.md](security.md) — принятые риски. Этот файл — план по документации. +[progress.md](progress.md), [implementation-plan.md](implementation-plan.md), +[roadmap.md](roadmap.md), этот файл. -**Границы:** отдельные `CONTRIBUTING.md`, `docs/architecture.md`, man-страницы и -сайт документации ТЗ **не требует** и в объём v1.x не входят. Dev-петля (сборка -и тесты на dev-сервере) остаётся в `progress.md` и в память проекта — выносить -её в поставляемую документацию не нужно. +**Вне объёма v1.x:** `CONTRIBUTING.md`, man-страницы, отдельный сайт документации. --- ## 2. Метод сверки с кодом -Правило: у каждого утверждения в документации есть **ровно один источник истины -в дереве**, и сверка идёт от кода к тексту (что код делает → сказано ли об -этом), а не наоборот — иначе не видно того, что забыли описать. +Правило: у каждого утверждения в документации есть **источник истины в дереве**; +сверка идёт от кода к тексту. | Класс утверждений | Источник истины | |---|---| -| Переменные окружения, значения по умолчанию | `loadConfig`/`envDefault`/`envInt` — [cmd/panel/main.go:80](../cmd/panel/main.go:80)–159; `${VAR:-default}` в [build/](../build/) (`postfix-config.sh`, `postfix-wrapper.sh`, `postfix-cert-reload.sh`, `logrotate-loop.sh`, `entrypoint.sh`) | -| Поведение почтового тракта (порты, TLS, SASL, анти-relay, лимиты, milter-цепочка) | [build/postfix-config.sh](../build/postfix-config.sh) | -| Экраны и действия панели (что вообще можно делать в эксплуатации) | таблица маршрутов [internal/web/web.go:133](../internal/web/web.go:133)–190 | -| Бэкап/восстановление, экспорт/импорт домена, проверка версии | [internal/backup/backup.go](../internal/backup/backup.go), [cmd/selfpost-backup/main.go](../cmd/selfpost-backup/main.go) | -| Сессии, вход, смена пароля | [internal/store/sessions.go](../internal/store/sessions.go), [internal/web/session.go](../internal/web/session.go), [internal/web/handlers_account.go](../internal/web/handlers_account.go) | -| Ротация лога, периодический reload, интервалы | [build/logrotate-mail.conf](../build/logrotate-mail.conf), [build/logrotate-loop.sh](../build/logrotate-loop.sh), [build/postfix-cert-reload.sh](../build/postfix-cert-reload.sh) | -| Деплой: тег образа, порты, монтирования, capabilities | [deploy/docker-compose.yml](../deploy/docker-compose.yml), [build/Dockerfile](../build/Dockerfile) | -| Требования, а не реализация (что вообще должно быть описано) | [specification.md](specification.md) разделы 9, 10, 11 | +| Env-переменные, дефолты | `loadConfig` — [cmd/panel/main.go](../cmd/panel/main.go); `${VAR:-…}` в [build/](../build/) | +| Почтовый тракт | [build/postfix-config.sh](../build/postfix-config.sh) | +| Маршруты панели | [internal/web/web.go](../internal/web/web.go) | +| Бэкап/restore, экспорт домена | [internal/backup/](../internal/backup/), [cmd/selfpost-backup/](../cmd/selfpost-backup/) | +| Сессии | [internal/store/sessions.go](../internal/store/sessions.go), [internal/web/session.go](../internal/web/session.go) | +| Ротация лога, reload | [build/logrotate-mail.conf](../build/logrotate-mail.conf), [build/logrotate-loop.sh](../build/logrotate-loop.sh), [build/postfix-cert-reload.sh](../build/postfix-cert-reload.sh) | +| Деплой | [deploy/docker-compose.yml](../deploy/docker-compose.yml), [build/Dockerfile](../build/Dockerfile) | +| Чеклист README | таблица «Поставляемое пользователю» выше | +| Продукт, out of scope | [product.md](product.md) | +| As-built | [architecture.md](architecture.md) | +| Обязательная безопасность | [security.md](security.md) | -Порядок прохода: сначала перечислить фактическое (env-ключи, маршруты, -`postconf`-настройки), потом искать каждый пункт в README — так находится и -неверное, и **отсутствующее**. +Порядок: перечислить фактическое в коде → найти в README / `architecture.md`. +Перед каждым тегом — короткий проход по этой таблице, не полная ревизия текста. --- -## 3. Результаты первого прохода (выполнен) +## 3. Правила поддержки -Сверено: env-переменные, маршруты панели, конфигурация Postfix, бэкап/CLI, -сессии, ротация, compose/Dockerfile. Ниже — всё найденное, по убыванию -приоритета. Нумерация сквозная, на неё ссылаются задачи в разделе 4. - -**Высокий (пробел против ТЗ или вводит в заблуждение)** - -1. **Нет раздела «эксплуатация»** — прямое требование ТЗ 11 п. 7 - ([specification.md:427](specification.md)). В README нет ни слова про - `/status` (проверки PTR/hostname), `/deliveries` (журнал отправки, фильтры), - `/mail-queue` (очередь Postfix), `/system-log` (хвост `mail.log`), `/reload`, - `/account`, `/backup` — при том что всё это реализовано - ([internal/web/web.go:152](../internal/web/web.go:152)–188). Не описана и - процедура апгрейда (бамп тега → `docker compose up -d`), хотя раздел - «Fixed image tag» на неё намекает. -2. **Битая ссылка + недокументированные лимиты.** - [deploy/.env.example:13](../deploy/.env.example:13) отсылает к разделу README - «Rate limiting», которого в README нет. Сам двухуровневый лимит не описан - нигде для пользователя: L1 — `smtpd_client_message_rate_limit` + - `anvil_rate_time_unit` ([build/postfix-config.sh:110](../build/postfix-config.sh:110)), - L2 — per-domain/per-app из панели - ([internal/web/web.go:165](../internal/web/web.go:165), - [:169](../internal/web/web.go:169)), с записью `rejected` в журнал. -3. **Нет справочника переменных окружения.** В `.env.example` шесть штук; код - читает заметно больше. Панель: `SELFPOST_DATA_DIR`, `SELFPOST_DB_PATH`, - `SELFPOST_SETUP_TOKEN_FILE`, `PANEL_HTTP_ADDR`, `JOURNAL_MILTER_SOCKET`, - `MAIL_LOG`, `PANEL_COOKIE_SECURE`, `TLS_CERT_FILE`, `OPENDKIM_SOCKET`, - `OPENDKIM_DIR`, `DKIM_SELECTOR_DEFAULT`, `SASL_DB_PATH`, `SASL_REALM`, - `POSTFIX_DIR` ([cmd/panel/main.go:80](../cmd/panel/main.go:80)–127). - Обвязка: `TLS_KEY_FILE`, `POSTFIX_SENDER_LOGIN_MAPS`, - `MILTER_CONNECT_TIMEOUT`/`COMMAND`/`CONTENT` (15s/15s/30s, - [build/postfix-config.sh:133](../build/postfix-config.sh:133)), - `MILTER_WAIT_TIMEOUT`, `TLS_RELOAD_INTERVAL_SECONDS` (86400, - [build/postfix-cert-reload.sh:14](../build/postfix-cert-reload.sh:14)), - `LOGROTATE_INTERVAL_SECONDS` (21600, - [build/logrotate-loop.sh:17](../build/logrotate-loop.sh:17)). - Отдельно: **`TRUSTED_PROXY_CIDR`** — переменная с последствиями для - безопасности (доверие `X-Forwarded-For` при rate-limit логина), описана - только в `.env.example`, в README её нет вообще. - *Решение, которое надо принять в задаче D2:* какие переменные публичные - (таблица в README), а какие внутренние (достаточно комментария в коде) — - документировать все 20+ вредно, они станут «поддерживаемым интерфейсом». -4. **Оговорка ТЗ 9 про прямой `tar` не отражена.** - [specification.md:375](specification.md) требует описать оба пути: «бэкап на - лету — панель/CLI» и «прямой `tar` директории безопасен, если контейнер - остановлен». В README ([Backup, restore…](../README.md)) есть только первый, - поэтому у читателя нет ответа на очевидный вопрос «а можно просто - заархивировать `./data`?». - -**Средний (стало неверным после последних фаз)** - -5. **Статус-баннер устарел.** [README.md:12](../README.md:12)–15: «under active - development» и ссылка на `implementation-plan.md` как на «phased build plan» - — фазы 0→14 закрыты, план теперь про открытые вопросы и линию 2.x. Перед - тегом релиза баннер переформулировать (или снять), ссылки уточнить. -6. **`implementation-plan.md` разошёлся с кодом.** Пункт B.1 говорит: «смена - пароля завершает **все** сессии, включая ту, из которой её делают → редирект - на `/login`». Код делает иначе — гасит все, **кроме** текущей - ([internal/store/sessions.go:95](../internal/store/sessions.go:95) - `DeleteOtherSessions`, [internal/web/session.go:131](../internal/web/session.go:131)), - и панель так и пишет («Any other signed-in sessions were signed out», - [internal/web/handlers_account.go:49](../internal/web/handlers_account.go:49)). - README здесь **верен**. Расхождение в плане — зафиксировать как осознанное - изменение при реализации, а не молча переписать. -7. **Комментарий в compose вводит в заблуждение.** - [deploy/docker-compose.yml:14](../deploy/docker-compose.yml:14) велит - «fill in .env (hostname, at least one strong TLS_CERT/KEY path)», но - `TLS_CERT_FILE`/`TLS_KEY_FILE` захардкожены в самом файле - ([:30](../deploy/docker-compose.yml:30)–31) и в `.env.example` отсутствуют — - настраивается на деле **bind mount** `./certs`, а не переменные. -8. **Порт 587 публикуется всегда** ([deploy/docker-compose.yml:61](../deploy/docker-compose.yml:61)), - хотя listener появляется только при `SUBMISSION_ENABLE=true` - ([build/postfix-config.sh:151](../build/postfix-config.sh:151)). Безвредно - (слушать некому), но выглядит как лишний открытый порт — одна строка - пояснения в README/compose снимает вопрос. - -**Низкий / требует решения, а не только текста** - -9. **`/healthz` есть, `HEALTHCHECK` нет.** Эндпоинт реализован - ([internal/web/web.go:133](../internal/web/web.go:133)), в - [build/Dockerfile](../build/Dockerfile) объявления `HEALTHCHECK` нет (пробел - отмечен и в плане, п. B.3). Решить в D6: либо задокументировать `/healthz` - как точку внешнего мониторинга, либо добавить `HEALTHCHECK` в образ (это уже - код + строка в CHANGELOG). Документировать «здоровье контейнера» до принятия - решения нельзя — получится обещание, которого образ не даёт. -10. **Из B.1–B.3/C.4 в README попала только обязательность `SELFPOST_HOSTNAME`.** - Не описаны: сессия переживает рестарт и живёт по скользящему сроку - бездействия (`PANEL_SESSION_IDLE_DAYS`, «вкладка с автообновлением вход не - продлевает» — контринтуитивно и заслуживает строки), ротация `mail.log` - (14 файлов, проверка каждые 6 ч, суточный `postfix reload`) — README - упоминает только «kept 14 days in-image» в требованиях к машине. -11. **Мелочи, без правки кода:** Quick start тянет файлы с - `raw.githubusercontent.com` (зеркало), хотя основной репозиторий — - Codeberg; тег образа `1.0.0` в compose ([:24](../deploy/docker-compose.yml:24)) - придётся бампить в тот же коммит, что и релиз; пустой каталог `docs/logo`. - -**Проверено и расхождений не найдено** (важно не переделывать): требования к -площадке; DNS-раздел (совпадает с тем, что реально проверяет `/status` и -DNS-карточка домена); прогрев IP; смысл фиксированного тега; проверка версии при -восстановлении — README описывает её точно так, как ведёт себя `CheckRestore` -([internal/backup/backup.go](../internal/backup/backup.go)); форма вызова -`docker exec … selfpost-backup > …` ([cmd/selfpost-backup/main.go:7](../cmd/selfpost-backup/main.go:7)); -таблица reverse-proxy и требование пропускать `Host`; секретность архива и -экспорта домена; лицензия; ретенция журнала отправки (90 дней) и то, что она — -основной драйвер роста `/data`. +1. **Правило шага:** новая/переименованная env-переменная, маршрут панели или + наблюдаемое поведение почтового тракта закрываются вместе с README / + `.env.example` и записью в CHANGELOG (протокол — [progress.md](progress.md)). +2. **Регресс env (D7):** [cmd/panel/envdoc_test.go](../cmd/panel/envdoc_test.go) — + падает на недокументированном ключе `loadConfig` или build-скриптов. +3. **Новые расхождения** дописываются в [roadmap.md](roadmap.md) или + [implementation-plan.md](implementation-plan.md), а не исправляются молча. --- -## 4. Задачи +## 4. Гейт релиза (документация) -Порядок — как перечислены; D1–D3 самые крупные. Каждая заканчивается записью в -`[Unreleased]` [CHANGELOG.md](../CHANGELOG.md) и коммитом. - -**D1. README: раздел «Operations» + «Rate limiting».** Закрывает находки 1, 2 и -часть 10. Что описать: экраны панели и зачем они нужны (`/status` и что именно -он проверяет; журнал отправки со статусами `queued/sent/rejected`; очередь; -хвост `mail.log`; `/reload`; смена учётных данных); двухуровневый лимит с -именами переменных L1 и указанием, что L2 задаётся в панели per-domain/app; -процедура апгрейда версии; поведение сессии (скользящий срок, опросы не -продлевают, смена пароля гасит остальные). Тон и объём — как в существующих -разделах: практика, а не пересказ ТЗ. -*Готово, когда:* каждый маршрут из [internal/web/web.go:152](../internal/web/web.go:152)–188, -кроме HTMX-фрагментов, либо описан, либо сознательно опущен; ссылка -«Rate limiting» из `.env.example` ведёт в существующий якорь. - -**D2. Справочник переменных окружения.** Закрывает находку 3. Сначала решение о -границе публичного набора, затем таблица (имя, назначение, дефолт, где -задаётся) в README + синхронизация `.env.example` и комментариев в compose. -`TRUSTED_PROXY_CIDR` описать с прямым предупреждением: неверное значение -позволяет подделать ключ rate-limit'а логина. -*Готово, когда:* каждый ключ из `loadConfig` и из `${VAR:-…}` в `build/*.sh` -попал либо в таблицу, либо в явный список «внутренние, не интерфейс»; дефолты в -таблице совпадают с кодом дословно. - -**D3. Бэкап: дописать путь «остановленный контейнер + `tar`».** Закрывает -находку 4 — по формулировке [specification.md:375](specification.md), с явным -«на живом контейнере так делать не надо, потому что SQLite в WAL». Заодно -упомянуть, что `manifest.json` после успешного восстановления потребляется. - -**D4. Точечные правки.** Находки 5, 7, 8, 11: баннер статуса и ссылки; комментарий -в шапке compose про `.env`; строка про 587; бамп тега образа (делается в -релизном коммите, не раньше). - -**D5. Синхронизация внутренних документов.** Находка 6 — поправить B.1 в -`implementation-plan.md` (с пометкой, что решение изменилось при реализации и -почему), заодно перечитать `progress.md` на предмет утверждений, которые код уже -опроверг. Не переписывать историю в CHANGELOG. - -**D6. Решение по `HEALTHCHECK`/`/healthz` (Opus).** Находка 9: либо строка в -`Dockerfile` + описание, либо только описание эндпоинта. Влияет на код образа — -поэтому отдельной задачей и не на Sonnet. - -**D7. Регресс-защита (см. раздел 5).** - ---- - -## 5. Чтобы не разошлось снова - -1. **Правило шага:** изменение, добавляющее/переименовывающее env-переменную, - маршрут панели или наблюдаемое поведение почтового тракта, закрывается - только вместе с правкой README/`.env.example` — в том же коммите, наравне с - записью в CHANGELOG (протокол закрытия шага в [progress.md](progress.md)). -2. **Дешёвая машинная проверка (D7):** тест или скрипт, сверяющий множество - ключей `loadConfig` со списком, объявленным в документации, и падающий на - новом недокументированном ключе. Ловит самый частый класс расхождений - (находка 3) без ручного прохода. Границу «внутренних» ключей задать явным - списком-исключением в самом тесте. -3. **Перед каждым тегом** — короткий проход по разделу 2 этого файла (семь - источников истины), а не полная ревизия текста. - ---- - -## 6. Гейт релиза - -Документационный проход — часть того же гейта, что e2e (C.4) и ревизия -безопасности (D.5): **D1–D6 закрыты до тега**. D7 — желателен, но тег не -блокирует. Находки, обнаруженные позже, дописываются сюда, а не исправляются -молча: этот файл — журнал состояния документации, а не одноразовый список дел. +Документационный проход **D1–D9 закрыт.** До тега релиза остаётся общий гейт: +e2e (готов, [development.md](development.md)) и ревизия безопасности +([implementation-plan.md](implementation-plan.md) § D) — см. [progress.md](progress.md). diff --git a/docs/implementation-plan.md b/docs/implementation-plan.md index 29c567c..eb8c3c2 100644 --- a/docs/implementation-plan.md +++ b/docs/implementation-plan.md @@ -1,174 +1,30 @@ # План реализации: SelfPost -**Статус:** выполненные фазы здесь не описываются — текущее состояние в -[progress.md](progress.md), история сделанного в [CHANGELOG.md](../CHANGELOG.md) -и `git log`. Ниже остаётся только то, что **ещё не сделано**: открытые вопросы -для согласования и опциональная линия 2.x.x. +**Статус:** линия v1.0/v1.x до тега релиза **закрыта** — предрелизная ревизия +безопасности (§ D) выполнена 2026-08-06, релизный гейт (e2e + ревизия) открыт. +B.1–B.3 и C.4 закрыты — as-built в [architecture.md](architecture.md), +e2e/CI в [development.md](development.md), принятые риски в +[security.md](security.md). Текущее состояние и следующий шаг: +[progress.md](progress.md). Объём 2.x.x — [roadmap.md](roadmap.md). -**Основа:** [specification.md](specification.md) v1.0. +**Основа:** [product.md](product.md) v1.0. --- -## Открытые вопросы — требует внимания и обсуждения (перед фиксацией v1.0) +## D. Предрелизная ревизия безопасности — ВЫПОЛНЕНО (2026-08-06) -Ниже — то, что **выходит за букву ТЗ**, но заслуживает решения перед тем, как -считать v1.0 «финальным». Ничего из этого **не является дефектом -соответствия**; это осознанные компромиссы и потенциальные улучшения. Каждый -пункт — решение «делаем в v1.x / откладываем в 2.x / оставляем как есть», -принимается пользователем. +**Проверка моделью Fable** по дифу от аудита v1.0 (Фаза 11, `bd64e80` — тега +`v1.0.0` в репозитории нет, это его фактический эквивалент) до HEAD, плюс +полный повторный проход по чек-листу безопасности (бывшее ТЗ 7.6, теперь +[security.md](security.md)). Приоритеты из плана покрыты: аутентификация и +сессии, валидация ввода, запись в конфиги/map-файлы, `os/exec`, права в +`/data`, секреты. -**Раздел A (принятые риски безопасности) переехал в -[security.md](security.md)** — риск не задача, а решение, и в плане несделанной -работы ему делать нечего. Буквы разделов и сквозная нумерация пунктов ниже -оставлены как были: на них ссылаются `progress.md`, коммиты и обсуждения. - -### B. Надёжность и эксплуатация - -1. **Сессии — решено: хранить в БД, скользящий срок бездействия.** Прежнее поведение (только в памяти, absolute TTL 12 ч) заменяется на: - - таблица `sessions` в SQLite (миграция `0002`) — вход переживает рестарт, редеплой и восстановление из полного бэкапа. В БД лежит SHA-256 от токена, а не сам токен: украденный файл БД или архив бэкапа во вход не превращается, зато браузер, у которого есть исходная cookie, работает и после восстановления; - - срок — **скользящий, 7 дней бездействия**, задаётся `PANEL_SESSION_IDLE_DAYS` (целое число дней, как `SEND_LOG_RETENTION_DAYS`). Абсолютного потолка нет **сознательно**: у админа, заходящего регулярно, сессия живёт неограниченно долго; - - **мониторинговые опросы сессию не продлевают.** Четыре фрагмента (`/status/fragment`, `/queue/body`, `/logtail/body`, `/sendlog/rows`) опрашивают сервер `every 5s`; продлевай их — и забытая открытая вкладка держала бы вход вечно, а «7 дней бездействия» означало бы «7 дней без открытой вкладки». Активностью считается переход по странице или действие, то есть всё, кроме GET-запросов с заголовком `HX-Request`; - - `Max-Age` cookie равен сроку и переставляется ровно тогда, когда продлевается строка в БД (запись в БД — не чаще раза в час, чтобы не писать на каждый клик); - - смена пароля завершает **все** сессии, включая ту, из которой её делают → редирект на `/login`. - - Известное свойство, вытекающее из хранения в БД: восстановление старого бэкапа возвращает и строки сессий, поэтому сессия, разлогиненная уже после снятия бэкапа, оживёт — если её браузер всё ещё хранит cookie и срок не истёк. -2. **Ротация `mail.log` — решено: отказаться от `copytruncate` в пользу «переименовать + `postfix reload`».** Прежняя формулировка («несколько строк мониторинга, приемлемо как известное свойство») занижала проблему: тот же тейлер, что рисует экран лога, сверяет и финальные статусы доставки — `UpdateStatus` вызывается **только** из [internal/logtail](../internal/logtail/logtail.go), больше ниоткуда. Значит потерянная строка `status=sent` — это строка журнала отправки, навсегда застрявшая в `queued`, то есть тихая порча данных, а не пробел в мониторинге. Окон потери при `copytruncate` два: - - **до одного интервала опроса (1 с) строк** — записанное после последнего `drain()` и до `truncate` физически сохранено в `mail.log.1`, но дескриптор тейлера смотрит на уже обрезанный inode и это пропускает. Это доминирующее окно; - - **миллисекунды между «`cp` дочитал до EOF» и `truncate`** — эти строки не попадают никуда; для `copytruncate` устранить нельзя. - - **Решение** — ротация переименованием, ровно та механика, которую применяет сам Postfix в `postfix logrotate` (`mv`, затем `HUP` мастеру): rename атомарен, postlogd продолжает писать в переименованный inode до перезапуска, а тейлер держит дескриптор на том же inode и дочитывает хвост перед переключением на новый файл. Не теряется ничего ни на стороне записи, ни на стороне чтения. Правки: - - [build/logrotate-mail.conf](../build/logrotate-mail.conf): убрать `copytruncate`, добавить `postrotate /usr/sbin/postfix reload endscript`. `rotate 14`/`compress`/`delaycompress` остаются: удержание N файлов (ТЗ 9) — за logrotate, поэтому берётся не сам `postfix logrotate` (у него нет retention, он лишь переименовывает с меткой времени и жмёт), а его механика. **`create 0644 root root`, не `nocreate`** — см. стендовую проверку ниже, предположение о `nocreate` не подтвердилось; - - `follow()` в [internal/logtail/logtail.go](../internal/logtail/logtail.go): при обнаружении смены inode дочитать старый дескриптор ещё раз перед закрытием — иначе остаётся микроокно между `drain()` и проверкой смены файла. Проверка `ni.Size() < pos` сохраняется как страховка от обрезания посторонней схемой ротации, но перестаёт быть основным механизмом; - - `readLogTail()` в [internal/web/handlers_monitor.go](../internal/web/handlers_monitor.go): `fs.ErrNotExist` — не ошибка, а пустой экран. После rename файла нет, пока Postfix не запишет в него первую строку (порядка секунды в сутки), и баннер ошибки в этот момент — шум. - - **Цена:** один `postfix reload` в сутки — ровно то, что уже делает [postfix-cert-reload.sh](../build/postfix-cert-reload.sh) ради сертификатов, никакой новой машинерии. - - **Проверено на живом 1.0.0 до принятия решения:** Postfix 3.7.11 (команда `postfix logrotate` есть начиная с 3.4, её реализация в `postfix-script` — это `mv` + `master -t || kill -HUP` + `sleep 1` + компрессор); postlogd работает под uid `postfix`, `/var/log` принадлежит root, `/var/log/mail.log` — `root:root 0644` и пользователю `postfix` на запись недоступен; в образе файла нет — значит создаёт его привилегированная сторона. - - **Стендовая проверка при реализации (обязательная по плану) обнаружила, что предположение о `nocreate` неверно.** На живом контейнере (`selfpost.example.com`, тестовый образ) после `mv` + `postfix reload` файл действительно появляется — но не сразу и не на `0644`: сам HUP лог не пересоздаёт, это происходит лениво при следующей фактической записи, и создаётся он с режимом `0600` — непривилегированная панель (свой uid) такой файл читать не может, то есть просмотр `mail.log` в панели остаётся сломан до следующего холодного старта контейнера (там `0644` берётся из другого, не связанного с этим, пути создания). Решение — то, что план заранее указал как запасной вариант: `create 0644 root root` вместо `nocreate`. logrotate создаёт пустой файл на 644 сразу после `mv`, ещё до запуска `postrotate`, и Postfix при следующей записи просто открывает и дозаписывает уже существующий файл, не трогая его режим. Проверено многократно на стенде: после ротации файл сразу (без окна) читаем непривилегированным uid панели, и остаётся на 644 после того, как в него попадает новый трафик. - - **Смежное, не решённое (тот же класс потерь, вариантом выше не лечится):** при рестарте панели `follow()` стартует с конца файла, поэтому строки, записанные пока она не читала, пропускаются; при редеплое `mail.log` исчезает вместе с контейнером — `/var/log` не в volume. В обоих случаях статусы писем, бывших в полёте, остаются `queued` навсегда — вероятно, чаще, чем при ротации. Кандидаты, если решим закрывать: переживать рестарт (запоминать позицию), вынести лог в `/data`, либо досверять зависшие строки по `postqueue`. -3. **Поведение при незаданном `SELFPOST_HOSTNAME` — решено: фатальная проверка в [entrypoint.sh](../build/entrypoint.sh) с развёрнутым текстом ошибки.** Прежнее предложение («предупреждать громко в лог, но не падать») отклонено: оба отказа мягкого fallback'а тихие и отложенные, а предупреждение в лог для таких отказов не работает — оно печатается при старте, а последствие проявляется через часы и в другом месте. - - **Что на самом деле ломает мягкий fallback** (формулировка «падают в `localhost`» занижала проблему — два fallback'а расходятся между собой): - - панель берёт realm как `SASL_REALM` → `SELFPOST_HOSTNAME` → **`localhost`** ([main.go:128](../cmd/panel/main.go:128)), а `postfix-config.sh` берёт `myhostname` как `SELFPOST_HOSTNAME` → **`hostname -f`**, то есть ID контейнера ([postfix-config.sh:21](../build/postfix-config.sh:21)). При пустом `smtpd_sasl_local_domain` Cyrus резолвит голый логин против realm = `myhostname` (механика проверена на стенде в Фазе 5), значит аккаунты пишутся в один realm, а ищутся в другом → **аутентификация не работает ни для одного приложения**, при полностью зелёной панели, а видно это только как `535` в чужом приложении. Само расхождение выведено из проверенной механики, отдельно не воспроизводилось; - - `EHLO` = ID контейнера: не FQDN, не совпадает с PTR, SPF-проверка HELO падает → почта, если она всё же уходит, попадает в спам. Отказ, не видимый вообще нигде; - - побочная ловушка того же класса: `SASL_REALM` читает только панель, `postfix-config.sh` про него не знает — задать один `SASL_REALM` без `SELFPOST_HOSTNAME` ломает так же. - - **Почему фатально, а не мягко.** Это не настройка с разумным умолчанием, а идентичность, обязанная одновременно совпасть с PTR/rDNS, с CN/SAN сертификата и с SASL-realm (ТЗ 5.2 п.3, 8) — значения, удовлетворяющего всем трём, угадать нельзя, поэтому **любой** fallback заведомо неверен. Ужесточением это не является: штатный деплой уже требует переменную (`${SELFPOST_HOSTNAME:?...}`, [docker-compose.yml:27](../deploy/docker-compose.yml:27)), CI контейнер не поднимает ([test.yml](../.github/workflows/test.yml) — только `vet`/`test`), а локальный запуск стоит одной явной переменной (`SELFPOST_HOSTNAME=localhost`). Меняется поведение только вне поставляемого compose (`docker run`, k8s, свой compose). - - **Правки:** - - `entrypoint.sh`: проверка до `postfix-config.sh` и до `supervisord`; при пустом значении — `exit 1`. Текст ошибки развёрнутый, а не `SELFPOST_HOSTNAME is required`: что это за имя, почему обязательно (PTR + CN/SAN + SASL realm), пример значения, где задаётся (`.env`). Это и есть замена «баннера в панели» — объяснительность там, где её реально прочитают (вывод `docker compose up`); - - там же — синтаксическая проверка значения: минимум одна точка, без схемы, порта и пробелов. Ловит типовые `https://mail.example.com` и `mail.example.com:465`, которые realm не ломают (обе стороны берут одну переменную), но ломают HELO и совпадение с сертификатом, то есть дают тот же тихий спам-отказ; - - `saslRealm()` ([main.go:128](../cmd/panel/main.go:128)) и fallback в `postfix-config.sh` **оставляем как есть**: после гейта в контейнере эти ветки мертвы, а вне контейнера (запуск бинаря локально, без Postfix) расходиться не с чем. Существующие «(SELFPOST_HOSTNAME is not set)» на статус-странице и `PTR: unknown` ([server.go:21](../internal/dnscheck/server.go:21)) тоже остаются — они как раз про этот случай. - - **Почему отклонён вариант «панель поднимается с баннером, фатально только для почтового тракта»** (обсуждался как более мягкий): - - он отменяет инвариант Фазы 4 — listener `crashexit` роняет весь контейнер, когда любой managed-процесс уходит в FATAL, именно чтобы не оставался «живой контейнер с мёртвым компонентом» ([crashexit.py](../build/crashexit.py)). В наивной реализации он в фатальный вариант и вырождается, только на минуту позже и с тремя циклами ретраев в логе; - - у него нет обычного оправдания degraded-режима — «починить на живую». Для TLS-сертификата degraded-режим выбран сознательно (файл можно доложить в mount, `cert-reload` подхватит без рестарта), а hostname вшивается в `myhostname` при генерации конфига, поэтому лечение всё равно = правка `.env` + пересоздание контейнера; - - главное: в таком состоянии панель остаётся полноценным писателем состояния, и состояние будет неверным. Созданные аккаунты лягут в realm `localhost`, а после задания hostname и рестарта `Secret()` и `Delete` ищут по паре (login, realm) ([sasl.go:97](../internal/app/sasl.go:97), [sasl.go:67](../internal/app/sasl.go:67)) → экспорт их не видит, удаление приложения оставляет сироту в `sasldb2` навсегда, а в панели они выглядят существующими. Чтобы это было безопасно, пришлось бы ещё блокировать записывающие действия — третий режим работы вместо одной проверки; - - канал доставки баннера испорчен ровно тем условием, о котором он предупреждает: адрес панели на первом запуске берётся из setup-ссылки, а она в этом сценарии печатается как `https://localhost/setup/` ([setup.go:124](../internal/web/setup.go:124)); - - `HEALTHCHECK` в образе не объявлен, поэтому «панель жива, почта мертва» для внешнего мониторинга выглядит здоровым контейнером, а crash-loop виден любой системе. - - **Цена:** контейнер без `SELFPOST_HOSTNAME` не стартует — это и есть цель. **Проверка на стенде при реализации:** контейнер без переменной падает с ожидаемым текстом и не уходит в бесконечный тихий retry; обычный деплой из [deploy/docker-compose.yml](../deploy/docker-compose.yml) не меняется; значение с портом/схемой отклоняется. Обязательность отразить в [README](../README.md) и `deploy/.env.example` (там переменная уже первая в списке). - -### C. CI и тесты - -4. **Интеграционный e2e — решено: герметичный контейнерный прогон отдельным Go-модулем, гейт перед публикацией образа.** Контейнерные e2e каждой фазы прогонялись вручную и задокументированы в git-истории; автоматизируется по сути тот же сценарий. - - **Что закрывается.** Не «интеграция вообще», а один класс отказов — **обвязка контейнера**, невидимая для `go test`: все 22 тестовых файла фейкуют границу процесса (`saslpasswd2` подменён хуком `s.run` — [sasl_test.go](../internal/app/sasl_test.go), milter гоняется против `fakeRecorder`, web — через `httptest`), реальные Postfix/OpenDKIM/supervisord/sasldb2 не стартуют нигде. Исторически ломалось ровно здесь: chroot ломал DNS ([postfix-config.sh:168](../build/postfix-config.sh:168)), Postfix не доставал до milter-сокетов (общая группа + setgid, [entrypoint.sh:62](../build/entrypoint.sh:62)), расходился SASL-realm (`535` для всех приложений), reload Postfix сигналом не работал (потребовалась one-shot program), а недостаточный `cap_add` в [docker-compose.yml](../deploy/docker-compose.yml) уронил контейнер **в проде**. Общее свойство всех пяти: панель зелёная, юнит-тесты зелёные, тракт мёртв. - - **Что автоматизировать нельзя и не пытаемся:** реальные PTR/rDNS, сертификат LE, репутация IP, доставка во внешний ящик (исходящий 25 на GitHub-раннерах закрыт). Это остаётся ручной проверкой на проде. - - **Форма:** - - `test/e2e/` — **отдельный Go-модуль со своим `go.mod`**: `go test ./...` не подхватывает его без build-тегов, а тестовые зависимости (проверка DKIM-подписи и прочее) не попадают в граф основного модуля, где сейчас три прямых зависимости; - - стенд — **поставляемый [deploy/docker-compose.yml](../deploy/docker-compose.yml) плюс override**, а не отдельный тестовый compose: иначе `cap_drop`/`cap_add`/`no-new-privileges` — ровно то, что сломалось в проде, — останутся непроверенными. Override задаёт самоподписанный сертификат, `PANEL_COOKIE_SECURE=false`, тестовый `SELFPOST_HOSTNAME`, заниженный `RATE_LIMIT_MESSAGES_PER_IP`, резолвер (`dns:`) и **высокие порты вместо 465/587/8080** — на dev-сервере они заняты продом, и без этого `make e2e` там падал бы на конфликте портов, работая при этом в CI; - - герметичная почта: фиктивная DNS-зона (CoreDNS/dnsmasq) + sink-MX на `smtp-sink` из пакета postfix — новых зависимостей ноль. DKIM-запись тест **берёт из панели и сам публикует в зону**, поэтому попутно проверяется, что запись, которую панель печатает пользователю, вообще рабочая; - - детерминизм обязателен: никаких `sleep N`, только опрос с таймаутом. Плавающий гейт перестают чинить, и тогда он хуже отсутствующего. - - **Объём проверок.** Позитив: старт контейнера (автостартующие `opendkim`/`panel`/`postfix`/`cert-reload`/`logrotate` в `RUNNING`, `postfix-reload` — `NOT STARTED`, он `autostart=false`) → setup по токену из `/data/setup-token` → login → домен → приложение → SMTP AUTH на 465 → письмо доставлено на sink → подпись проверяется против ключа из зоны → строка журнала переходит `queued → sent` (это же покрывает `logtail`, а после B.2 — и ротацию). Негативы: - 1. отправка без AUTH — отказ; - 2. чужой отправитель при валидном AUTH — отказ (`reject_sender_login_mismatch`, ТЗ 5.1 п.3); - 3. relay на чужой домен — отказ (`reject_unauth_destination`), прямая проверка «не open relay»; - 4. L1-лимит по IP (заниженный в override) — отказ после исчерпания; - 5. L2-лимит, выставленный **через панель**, с записью `rejected` — заодно путь панель→БД→milter; - 6. fail-open journal-milter'а: `supervisorctl stop panel` → письмо всё равно принято, контейнер жив. Это выполнимо, потому что `crashexit` подписан только на `PROCESS_STATE_FATAL` ([supervisord.conf:122](../build/supervisord.conf:122)), а штатный stop даёт `STOPPED`; - 7. пустой и синтаксически неверный `SELFPOST_HOSTNAME` — контейнер падает с ожидаемым текстом (проверка из B.3); - 8. сессия переживает `docker restart` (проверка из B.1). - - **Вне объёма:** «зависший», а не упавший milter — требует подставного сокета внутри контейнера, это уже chaos-тест ради одного таймаута. - - **Запуск.** Основной путь — `make e2e` на dev-сервере перед тегированием (там и так идёт вся сборка) плюс `workflow_dispatch`. В CI прогон вешается **на тег `vX.Y.Z` и блокирует публикацию образа**; на обычный push не вешается — `vet`/`test` в [test.yml](../.github/workflows/test.yml) остаются как есть. Уведомления специально не настраиваются: признак провала — отсутствие образа в `ghcr` после тега, смотрится вкладкой Actions. - - **Переработка [release.yml](../.github/workflows/release.yml)** — следствие требования покрыть обе архитектуры. Сейчас multi-arch собирается через qemu; гонять под эмуляцией полный стек Postfix мучительно долго, поэтому релиз переезжает на нативную сборку по архитектурам: job `prepare` (единственная точка деривации версии из тега — на ней держится инвариант ТЗ 7.5.А) → матрица `[ubuntu-latest, ubuntu-24.04-arm]`, в каждой сборка `--load` → e2e → push per-arch тега `X.Y.Z-amd64`/`X.Y.Z-arm64` → job `merge`: `docker buildx imagetools create -t …:X.Y.Z`. `setup-qemu-action` уходит, `provenance: false` сохраняется. Порядок «сначала тест, потом push» выбран ради того, чтобы публиковались **ровно те байты, которые прогонялись**; альтернатива (push по digest → тест → сборка манифеста) даёт ту же гарантию, но оставляет в registry мусорные untagged-манифесты после красного прогона и требует второго пути для `workflow_dispatch`. Per-arch теги остаются в registry побочным продуктом; неизменяемость версионного тега (ТЗ 10.1) это не нарушает. Бесплатные arm-раннеры доступны, потому что зеркало `github.com/mixeme/selfpost` публичное. - - **Цена:** ~10–15 минут на релиз; переработка релизного workflow, который сейчас работает; новый модуль и compose-override на сопровождении. **Проверка при реализации:** `make e2e` на dev-сервере проходит, не задевая прод-порты; красный e2e действительно не даёт опубликовать образ — проверяется одноразовым тегом на заведомо сломанном прогоне (тег и per-arch пакеты после проверки удалить). - - **Порядок работ:** сначала B.1–B.3 (они полностью специфицированы, иначе харнесс пришлось бы переписывать под них), затем харнесс — и стендовые проверки B.1/B.3 переезжают в него постоянными регрессиями (пункты 7–8 выше), а не выбрасываются после однократного прогона. - -### D. Ревизия безопасности - -5. **Проверка на уязвимости моделью Fable — решено: отдельный проход после B.1–B.3 и C.4, до тега релиза.** - - **Почему после всех четырёх, а не по ходу каждого.** Каждый пункт трогает ровно ту поверхность, которую аудит ТЗ 7.6 на v1.0 видел в другом виде: B.1 переписывает аутентификацию (сессии в SQLite, SHA-256 от токена, скользящее продление, разлогин всех при смене пароля), B.2 меняет обращение с дескриптором лога и вешает `postfix reload` на logrotate, B.3 добавляет разбор значения переменной в shell до старта supervisord, C.4 приносит переработанный релизный workflow и compose-override с **сознательно ослабленными** настройками (`PANEL_COOKIE_SECURE=false`, самоподписанный сертификат, заниженные лимиты), которому нельзя утечь в прод. Ревизия по пунктам дала бы четыре среза, а смотреть надо итоговое состояние — и заведомо один раз, а не четыре. - - **Объём.** Диф от тега `v1.0.0` до состояния перед следующим тегом целиком — то есть вместе с Фазами 12–14, которых в аудите v1.0 не было, — плюс повторный проход по чек-листу ТЗ 7.6, а не только по изменённым строкам: регресс в 7.6 возможен и в нетронутом коде, если рядом поменялся вызывающий. Приоритет задаёт то, что панель публично доступна (ТЗ 2.4): аутентификация и сессии, валидация ввода, запись в конфиги и map-файлы (injection), `os/exec` без shell, права на файлы в `/data`, обращение с секретами (пароли приложений, `sasldb2`, архив бэкапа). - - **Модель — Fable, и это сознательно не Opus:** B и C пишет Opus, а проверка собственной работы систематически слабее независимой. Правило progress.md «безопасность/инфра → Opus» этим не отменяется — оно про написание кода, здесь речь про ревизию. Форма прогона: `/security-review` по изменениям, пока они ещё в ветке (скилл смотрит диф), плюс отдельный ручной проход по 7.6 целиком. - - **Что с находками.** Каждая закрывается явно: правка до тега либо запись в [security.md](security.md) как принятый риск с обоснованием (там уже два таких). «Посмотрели и ладно» закрытием не считается. **Гейт:** вместе с e2e из C.4 — до тега; находка класса «эксплуатируется снаружи» тег откладывает. - -### E. Указатель на объём 2.x - -6. **Входящий релей и pluggable-антиспам** вынесены в опциональные фазы O1+ ниже (линия 2.x.x, вне v1.0, только по согласованию — ТЗ 12.6). Здесь перечислены лишь как напоминание, что это **сознательно отложенный объём**, а не забытый. -7. **Роль администратора домена** — кандидат на 2.x, вне объёма v1.x. Прежняя формулировка пункта («2FA и несколько администраторов») заменена: 2FA снята с рассмотрения, а «несколько администраторов» уточнено до одной конкретной роли, потому что нужна не вторая копия всевластного админа, а ограниченный доступ владельца отдельного домена. - - **Что это.** Сейчас в панели ровно один субъект: `requireAuth` — булев гейт, а не роль ([web.go:182](../internal/web/web.go:182)), сессия не несёт ничего, кроме факта входа. Роль выдаёт доступ к одному домену и только к нему: приложения этого домена (создание, режим отправителя, перегенерация пароля, удаление, свой L2-лимит), DKIM/DNS-статус домена и журнал отправки, отфильтрованный по домену — фильтр в журнале уже есть ([handlers_monitor.go:49](../internal/web/handlers_monitor.go:49)). Вне роли остаётся то, что глобально по своей природе: добавление и удаление доменов, `/reload`, полный бэкап (это весь `/data` вместе с `sasldb2`, то есть все домены сразу), очередь и хвост `mail.log` — они серверные и к домену не привязаны. - - **Почему 2.x, а не v1.x.** ТЗ 3 относит «несколько пользователей панели, роли» к явным не-целям (панель рассчитана на одного администратора), поэтому появление второго субъекта — расширение границ проекта, как и Фаза O1: сначала согласование (ТЗ 12.6) и правка ТЗ, только потом код. Цена — уровня фазы, а не патча: таблица пользователей и их привязка к доменам, роль в сессии, авторизация в каждом хендлере (а не только на маршруте — сейчас `{id}`/`{aid}` не сверяются ни с чем, кроме существования), пересмотр первичного setup'а и смены пароля под нескольких пользователей, учёт нового субъекта в бэкапе и экспорте домена. - ---- - -## Опциональные фазы — целевой релиз 2.x.x (вне базового объёма v1.0) - -Эти фазы **не входят** в линейный базис 0→11 и не являются частью поставки v1.0 (v1.x — только исходящий релей). Они отнесены к **релизной линии 2.x.x** и добавлены в дорожную карту как согласуемые расширения. **Реализация — только после явного согласования (ТЗ 12.6):** ТЗ v1.0 раздел 3 явно исключает приём входящей почты из объёма, поэтому включение этой функциональности — сознательное расширение границ проекта (major-релиз 2.0), а не доработка по своей инициативе. Внесение в план фиксирует намерение и дизайн; кодирование начинается отдельным решением. - -### Фаза O1 (→ 2.x.x) — Входящий релей (backup-MX / пересылка) — опция/плагин - -**Цель:** возможность принимать почту на порт 25 для явно настроенных доменов и пересылать её на заданный вышестоящий backend (роль backup-MX / relay-forwarder), **как выключаемый по умолчанию модуль**, не затрагивающий поведение и поверхность атаки базового исходящего релея. - -**Зачем это нужно (сценарии):** -- **Backup-MX** — принять почту, когда основной почтовый сервер домена временно недоступен, и передать её, когда он вернётся. -- **Фронт для сервера без внешнего IP** — у оператора есть свой почтовый сервер, который по каким-то причинам **сам не может принимать почту из интернета** (нет статического/внешнего IP, за NAT, серый адрес, закрытый порт 25 на входящую и т.п.). SelfPost с публичным IP и корректным PTR выступает публичным входным узлом для домена (MX указывает на него) и пересылает почту на этот внутренний/недоступный извне сервер. - -**Граница объёма (критично — что это НЕ):** -- **ЭТО:** приём на 25 для доменов из явного списка + пересылка (relay/forward) на upstream (`relay_domains` + `transport_maps` + `relay_recipient_maps`). Postfix здесь — чистый пересыльщик, без локальной доставки. -- **ЭТО НЕ (остаётся out of scope, ТЗ 3):** локальная доставка в почтовые ящики, IMAP/POP3, webmail, Dovecot. Никаких mailbox'ов. SelfPost также **не реализует и не тянет в свой образ** движок антиспама/антивируса (rspamd/ClamAV) — но, в отличие от прежней формулировки, и **не** перекладывает фильтрацию на backend (см. блок «Антиспам» ниже): предоставляет точку подключения внешнего фильтра. - -**Почему как опция/плагин:** -- Приём на порт 25 меняет модель угроз (open relay для входящей, backscatter, spam-ingress). Поэтому по умолчанию **выключено** флагом env `INBOUND_RELAY_ENABLE=false`; включение — осознанный шаг оператора. -- Изоляция: отдельные таблицы SQLite, отдельные хендлеры/страницы панели, отдельная ветка генерации конфига. При выключенном флаге входной listener, таблицы и UI отсутствуют — базовый исходящий тракт байт-в-байт неизменен. - -**Что делать:** -- Env-флаг `INBOUND_RELAY_ENABLE` (default false); при `true` — генерировать входной сервис и его конфиг из состояния панели тем же путём, что остальной конфиг (`postfix-config.sh`). -- **`master.cf`:** входной `smtp inet` на 25 для приёма из интернета (сейчас 25 используется только на исходящую доставку). Отдельный от 465/587: на 25 **не** предлагается SASL и **не** разрешается отправка наружу — только приём для `relay_domains`. -- **Анти-open-relay для входящей (обязательно):** `smtpd_relay_restrictions`/`smtpd_recipient_restrictions` входного smtpd принимают почту **только** для доменов из `relay_domains` и **только** для известных получателей (`relay_recipient_maps`); всё прочее — `reject_unauth_destination`/`reject_unlisted_recipient`. Открытый релей и приём «для кого угодно» невозможны. -- **Backscatter:** предпочтительно знать валидных получателей (reject unknown recipient на этапе RCPT), чтобы не порождать bounce на несуществующие адреса. -- **Панель управляет:** список входящих доменов; для каждого — upstream destination (`host:port`, транспорт), опциональный список валидных получателей, опциональный TLS к upstream. Строгая валидация домена/хоста/порта (whitelist), injection-safe запись map-файлов (как `sender_login_maps` в Фазе 4), `os/exec` без shell (ТЗ 7.6.2–4). -- **Милтеры:** OpenDKIM на входящем тракте не нужен (чужую входящую не подписываем). journal-milter опционально переиспользовать для журнала входящих (доп. работа) либо на первом этапе оставить входящий без него; поведение fail-open сохраняется. -- **Rate-limit/размер:** грубый лимит по client IP (`anvil`, как L1) и `message_size_limit` на входном smtpd. - -**Антиспам (важная, но опциональная возможность).** Это ценная опция, но она **не обязательна**: часть операторов вполне устроит **слепая пересылка без фильтрации** — например, когда backend сам умеет фильтровать по содержимому, стоит доверенный upstream, или объём/риск невелик. Поэтому антиспам-хук по умолчанию **выключен** (пустой `INBOUND_ANTISPAM_MILTER`), и входящий релей полностью работоспособен без него. Важно другое — где фильтрация возможна технически: при «слепом» relay целевой backend видит подключающимся IP адрес **SelfPost**, а не исходного отправителя, поэтому на backend'е ломается всё, что завязано на origin IP (DNSBL/репутация проверяются против IP SelfPost, SPF даёт fail — SelfPost не входит в SPF домена-отправителя). **Единственная точка, где ещё виден настоящий client IP — входной хоп на SelfPost**; поэтому тем, кому фильтрация нужна, она должна быть *подключаема именно здесь*, а не переложена на backend, который эту информацию уже потерял. Дизайн подключения: -- **Движок антиспама — отдельный опциональный контейнер** (rspamd и т.п.), который оператор запускает **только если нужна эта опция** (тот же принцип, что reverse-proxy — отдельный контейнер вне образа SelfPost). SelfPost его **не содержит и не запускает** — образ и принцип «один контейнер, три процесса» неизменны, ТЗ 3 не нарушается (SelfPost не реализует антиспам). -- **SelfPost предоставляет точку подключения:** milter-хук на входном smtpd. Адрес движка задаётся env (например, `INBOUND_ANTISPAM_MILTER=inet:antispam:11332`, пусто → хук выключен) и добавляется в `smtpd_milters` **только входного** тракта (не на 465/587). Postfix передаёт milter'у настоящий client IP/HELO/PTR — фильтр видит истинный origin. `milter_default_action` для этого milter'а — конфигурируемый (fail-open vs tempfail); дефолт определить при реализации. -- **Нативный backstop без зависимостей:** на том же входном хопе доступны средства Postfix по origin IP — `reject_rbl_client` (DNSBL), проверки HELO/PTR — работают даже без внешнего контейнера. Плюс сохранение аутентификации для downstream через ARC/`Received` там, где часть фильтрации всё же остаётся на backend. -- **docker-compose:** задокументировать опциональный фрагмент antispam-сайдкара (как альтернативные фрагменты reverse-proxy) — контейнер поднимается вместе со стеком только при включённой опции. -- **Персистентность:** новые таблицы и map-файлы под `/data` — попадают в полный бэкап автоматически (Фаза 9). Экспорт/импорт домена можно расширить входящей конфигурацией — опционально, пометить. -- **DNS-документация:** для входящего домена нужна `MX`-запись, указывающая на сервер (в отличие от исходящего, где MX не требуется) — отразить в разделе DNS README. - -**Безопасность (ТЗ 7.6 распространяется полностью):** валидация ввода на сервере, экранирование записи в конфиги, `exec` без интерполяции, никакого open relay, защита от backscatter. - -**Готово, когда:** при `INBOUND_RELAY_ENABLE=true` и настроенном домене письмо на порт 25 для этого домена пересылается на заданный upstream; почта для ненастроенных доменов/получателей отклоняется (не open relay, не backscatter); при заданном `INBOUND_ANTISPAM_MILTER` входящая проходит через внешний фильтр с настоящим origin IP (проверено сайдкар-контейнером), при пустом — хук не мешает; при `INBOUND_RELAY_ENABLE=false` — входной порт/таблицы/UI отсутствуют, базовый исходящий релей неизменён; `build`/`vet`/`test`/образ зелёные. - -**Риски:** open relay/backscatter (снимается `relay_domains` + `relay_recipient_maps` + `reject_unauth_destination`); потеря origin IP для фильтрации на backend'е при пересылке (снимается milter-хуком антиспама + нативным DNSBL на входном хопе, где origin IP ещё виден); порт 25 на приём расширяет поверхность атаки (по умолчанию выключено). **Модель:** Opus (инфра/безопасность, риск open relay). **Внешняя зависимость деплоя:** опциональный antispam-контейнер — вне образа SelfPost, поднимается оператором при включении опции. - -**Зависимости:** не является частью v1.0, зависит только от готового исходящего тракта (уже реализован) и требует отдельного согласования (ТЗ 12.6, расширение за пределы раздела 3) до кодирования. +**Результат.** Эксплуатируемых уязвимостей (high/medium) не найдено. Одна +находка defence-in-depth закрыта правкой до тега: логин приложения передаётся +в `saslpasswd2` после `--`, чтобы значение, начинающееся с `-` (допустимо +whitelist'ом), не могло быть разобрано getopt как флаг +([internal/app/sasl.go](../internal/app/sasl.go)). Принятые риски в +[security.md](security.md) не пополнились — существующие записи (origin-check +fallback, отсутствие CSRF-токенов, send-log gap) покрывают всё найденное. +Сводка ревизии — в записи `Security` CHANGELOG `[Unreleased]`. diff --git a/docs/product.md b/docs/product.md new file mode 100644 index 0000000..f889d43 --- /dev/null +++ b/docs/product.md @@ -0,0 +1,102 @@ +# SelfPost — product boundaries + +**What this file is.** Stable product definition for SelfPost v1.0: purpose, +deployment assumptions, explicit out-of-scope items, and the multi-domain +model. User-facing install and operations live in [README.md](../README.md); +as-built technical detail in [architecture.md](architecture.md). + +--- + +## Purpose + +SelfPost is a self-hosted **outbound SMTP relay** with a web control panel, +shipped as a single Docker image. It sends mail directly to the internet from +your own IP with per-domain DKIM signing. + +**Primary workflow:** configure the relay once (domains, DKIM, SASL +applications), then point scripts and apps at the SMTP endpoint. SelfPost +delivers as the configured sending domain. + +--- + +## Deployment context (fixed assumptions) + +These constraints are intentional; changing them requires an explicit product +decision: + +1. **VPS or home server** — one image for both. Raspberry Pi and similar boards + are not a current target but not ruled out forever. +2. **Send from your own IP (DIY)** — no intermediate relay provider. Requires + outbound port 25, static IP, and configurable PTR/rDNS from the operator. +3. **Single container** — Postfix, OpenDKIM, and the panel run under one + `supervisord` inside one image. +4. **Panel is internet-facing** — mandatory security requirements in + [security.md](security.md). + +**Infrastructure the operator provides (not SelfPost features):** unblocked +outbound TCP 25, static IP, PTR/rDNS, acceptable IP reputation. SelfPost does +not detect, bypass, or compensate for missing prerequisites; mail simply fails +to deliver when they are absent. + +--- + +## Out of scope + +Explicitly excluded to prevent scope creep: + +- Inbound mail (IMAP/POP3, mailboxes, delivery to user inboxes) +- Webmail +- Multi-user panel / organisations / roles — one administrator; managing + **multiple sending domains** is in scope (see below) +- Inbound antispam/antivirus (rspamd, ClamAV, etc.) +- A custom MTA — Postfix is used as-is +- Dovecot or a full mail stack for SASL — Cyrus SASL (`sasldb2`) only + +Future line **2.x.x** (optional inbound relay, domain-admin role) is tracked in +[roadmap.md](roadmap.md) and requires explicit approval before implementation. + +--- + +## Multi-domain model + +SelfPost is a **multi-domain outbound relay**. Two linked entities: + +### Sending domain + +Example: `example.com`. Has its own DKIM key and selector. + +### Application (account) + +A SASL login/password bound to **one** domain. A domain may have several +applications (e.g. newsletter vs alerts), each with its own credentials, but +each may send **only from its own domain**, never from another. + +### From-address mode (per application) + +Set when creating or editing an application: + +1. **Any address in the domain** — `*@example.com` (wildcard in + `smtpd_sender_login_maps`). +2. **Explicit address list** — only listed From addresses within the domain; + anything else is rejected even if it belongs to the same domain. + +In both modes the From address must belong to the application's domain. + +### What the panel manages per domain + +- DKIM key + selector (shared by all applications on that domain) +- One or more applications (SASL credentials + From mode + optional rate limits) +- DNS guidance (DKIM TXT; reminders for SPF/DMARC) + +Adding a domain does **not** create an application automatically. + +### Lifecycle + +- **Add domain** — record, DKIM key, DNS instructions; no application yet. +- **Add application** — SASL pair (password shown once), From mode, map entries, + `postfix reload`. +- **Delete domain** — removes DKIM key and **all** its applications. +- **Delete application** — removes only that app's SASL and map entries. + +This is not multi-tenancy (one admin); it is one owner operating several +sending domains with independent application credentials. diff --git a/docs/progress.md b/docs/progress.md index 3b3d3fc..1fece6e 100644 --- a/docs/progress.md +++ b/docs/progress.md @@ -1,23 +1,27 @@ # Прогресс реализации SelfPost Живой трекер состояния. **Переживает `/clear`** — читается первым при возобновлении работы. -План (открытые вопросы + опциональные фазы): [implementation-plan.md](implementation-plan.md). -ТЗ: [specification.md](specification.md). Принятые риски безопасности: [security.md](security.md). +План (открытые вопросы для v1.0/v1.x): [implementation-plan.md](implementation-plan.md). +Линия 2.x.x (входящий релей, роль администратора домена): [roadmap.md](roadmap.md). +Продукт: [product.md](product.md), устройство: [architecture.md](architecture.md). +Процесс разработки: [development.md](development.md). Принятые риски безопасности: [security.md](security.md). История релизов: [CHANGELOG.md](../CHANGELOG.md). История сделанного по фазам (0→13, все закрыты) — в `git log` и в CHANGELOG, здесь не дублируется. ## Как возобновить после сброса контекста 1. Прочитать этот файл (текущее состояние, что дальше). -2. Открыть `implementation-plan.md` — там нерешённые вопросы и опциональная линия 2.x.x (Фаза O1+); принятые риски безопасности — в `security.md`. -3. При необходимости — детали в `specification.md`. +2. Открыть `implementation-plan.md` — там остаётся предрелизная ревизия безопасности (§ D); линия 2.x.x — в `roadmap.md`; принятые риски — в `security.md`; as-built B.1–C.4 — в `architecture.md` и `development.md`. +3. При необходимости — [architecture.md](architecture.md) и [product.md](product.md). 4. Продолжить с пункта «Следующий шаг». ## Модель по типу работы Правило: безопасность / инфра / риск-критичное → **Opus**; UI / документация / бойлерплейт → **Sonnet**; тривиальная механика → **Haiku**. -Исключение — **ревизия** (не написание) кода: предрелизная проверка на уязвимости (пункт **D.5** плана) делается моделью **Fable**, чтобы проверял не тот, кто писал. +Исключение — **ревизия** (не написание) кода: предрелизная проверка на уязвимости +([implementation-plan.md](implementation-plan.md) § D) делается моделью **Fable**, +чтобы проверял не тот, кто писал. ## Коммиты @@ -42,9 +46,16 @@ - **B.2 реализован** (не выкачен на прод): ротация `mail.log` ушла с `copytruncate` на «переименовать + `postfix reload`» — `build/logrotate-mail.conf` (`nocreate` заменён на `create 0644 root root` **не по плану, а по стендовой проверке**: после reload Postfix пересоздаёт лог сам только в момент следующей фактической записи и с режимом `0600`, недоступным непривилегированной панели, — `create` в logrotate закрывает это, отдавая файл ей же на 644 сразу после переименования); `follow()` в `internal/logtail/logtail.go` при обнаружении смены inode дочитывает старый дескриптор ещё раз перед переключением; `readLogTail()` в `internal/web/handlers_monitor.go` считает отсутствующий файл пустым экраном, а не ошибкой. Проверено на стенде (`selfpost.example.com`, отдельный контейнер `selfpost:b2test2`): цикл трафик → принудительная ротация → файл пуст и сразу читаем непривилегированным uid панели (0 читает `mail.log` сразу после rename, без окна недоступности) → новый трафик после ротации уходит в новый файл на 644, ничего не потеряно по обе стороны rename. `go vet`/`go test ./...`/`gofmt -l .` чистые (на dev-сервере; локально на Windows `TestFollowTailsAndRotates` падает — rename открытого файла запрещён ОС, к делу не относится). - **B.3 реализован** (не выкачен на прод): `build/entrypoint.sh` проверяет `SELFPOST_HOSTNAME` до `postfix-config.sh` и до `supervisord` — при пустом значении `exit 1` с развёрнутым текстом ошибки (что это за имя, почему обязательно, пример, где задаётся); плюс синтаксическая проверка через `case`: минимум одна точка, без схемы/порта/пробелов (`*://*`, `*:*`, пробел/таб — тот же класс тихого спам-отказа, что и пустое значение). `saslRealm()` и fallback в `postfix-config.sh` не тронуты — после гейта эти ветки мертвы. Заодно отмечена обязательность переменной в `README.md` и `deploy/.env.example`. Проверено на стенде (`selfpost.example.com`, отдельный образ `selfpost:b3test`, cap-list как в поставляемом compose): без переменной — `exit 1` с ожидаемым текстом, без бесконечного тихого retry; `https://mail.example.com:465` и `localhost` отклонены с понятными сообщениями; валидный `mail.example.com` — обычный старт, все процессы supervisord поднимаются. `go vet`/`go test ./...` чистые. - **C.4 реализован** (не выкачен на прод — это CI/тестовая инфраструктура, а не образ): герметичный контейнерный e2e отдельным Go-модулем `test/e2e/` (свой `go.mod`, не подхватывается `go test ./...` основного модуля) поверх поставляемого `deploy/docker-compose.yml` плюс `test/e2e/compose.override.yml` (самоподписанный сертификат, `PANEL_COOKIE_SECURE=false`, `SELFPOST_HOSTNAME=mail.e2e.test`, высокие порты `20465/20587/20080`, изолированный compose-проект `selfpost-e2e`, свой `--project-directory` — прод на том же хосте не задет). Герметичная почта: CoreDNS (`test/e2e/dns/Corefile` — авторитетна только для `e2e.test`, `file`-плагин с саб-директивой `reload` перечитывает `db.zone` по mtime, без сигналов) плюс `smtp-sink` из пакета postfix (`test/e2e/sink/`) как sink-MX. Сценарий (`test/e2e/*_test.go`): старт контейнера → все supervisord-программы `RUNNING` (`postfix-reload` — `STOPPED`) → токен из `/data/setup-token` → setup → login → добавление домена → DKIM-запись **скраплена со страницы панели** и опубликована в фейковую зону → добавление приложения → SMTP AUTH на 465 → письмо на sink → DKIM-подпись проверена (`go-msgauth/dkim` с кастомным `LookupTXT` через CoreDNS) против ключа **из DNS**, не из панели напрямую → send-log `queued → sent`. Негативы: без AUTH, relay на чужой домен без AUTH, sender/login mismatch (`reject_sender_login_mismatch` репортится Postfix'ом на RCPT, не MAIL — `smtpd_delay_reject=yes` по умолчанию), L1-лимит (anvil, override `RATE_LIMIT_MESSAGES_PER_IP=50` — специально высокий, чтобы остальные под-тесты не расходовали общий бюджет по IP раньше времени; сам тест шлёт до 60 раз, ждёт отказа), L2-лимит через панель (домен/приложение → `rejected`-строка в send-log), fail-open journal-milter'а (`supervisorctl stop panel`, письмо всё равно принято, контейнер жив), пустой/синтаксически неверный `SELFPOST_HOSTNAME` (отдельный один-разовый контейнер, не общий стенд), сессия переживает `docker restart` (плюс явное ожидание готовности smtps-порта после рестарта — панель и Postfix поднимаются независимо). `make e2e` — локальный/dev-server прогон. Найдено и исправлено по ходу стендовой проверки: `reload` — саб-директива `file`-плагина CoreDNS, а не отдельный топ-левел плагин (топ-левел `reload` следит за самим Corefile, не за зоной); `docker compose build.context` резолвится относительно `--project-directory`, а не относительно файла, где объявлен; `smtp-sink` отказывается стартовать от root без `-u`; `html/template` эскейпит `+` в `+` даже в тексте — скрапер значений со страницы обязан `html.UnescapeString`; проверки состояния сразу после `up`/`restart` должны поллиться, а не разово опрашиваться (supervisord/postfix поднимаются не мгновенно). **Проверено на dev-сервере (`selfpost.example.com`)**: `make e2e` — зелёный (`go vet`/`gofmt -l` тоже чистые в обоих модулях). `release.yml` переработан: job `prepare` (версия из тега) → матрица `[ubuntu-latest, ubuntu-24.04-arm]` — каждая нативно собирает образ (`--load`), прогоняет e2e, пушит тег `X.Y.Z-amd64`/`X.Y.Z-arm64` → job `merge` — `docker buildx imagetools create` в единый тег `X.Y.Z`; `setup-qemu-action` убран. Не проверено вживую (нельзя без реального тега): сам workflow на GitHub Actions — синтаксис вычитан, логика идентична локальному `make e2e` пути. -- **Дальше — документация:** [documentation-plan.md](documentation-plan.md) — состав пакета (ТЗ 11), метод сверки с кодом, результаты первого прохода (11 находок: нет раздела «эксплуатация» по ТЗ 11 п.7, нет справочника env-переменных, битая ссылка на «Rate limiting» из `.env.example`, не отражена оговорка ТЗ 9 про прямой `tar`) и задачи D1–D7. Часть предрелизного гейта наравне с e2e и ревизией безопасности. -- **Дальше:** пункт **D.5** плана — предрелизная проверка на уязвимости моделью Fable по всему дифу от `v1.0.0` плюс повторный проход по ТЗ 7.6; вместе с e2e (C.4, готов) это гейт перед тегом релиза. -- **Дальше — то, что перечислено в `implementation-plan.md`:** открытые вопросы закрыты, раздел E теперь только указатель на объём 2.x (входящий релей O1+ и роль администратора домена; 2FA снята с рассмотрения); остаются принятые риски безопасности (переехали в [security.md](security.md): `POST` без `Sec-Fetch-Site`/`Origin` пропускается, CSRF-токенов нет) и опциональная **Фаза O1+** (входящий релей, линия 2.x.x, требует согласования). +- **Документация:** план D1–D9 закрыт ([documentation-plan.md](documentation-plan.md) — только метод и правила поддержки). Хвост v1.x — [roadmap.md](roadmap.md) § «v1.x — хвост документации и деплоя»; из него остался только бамп тега образа (Quick start и `docs/logo` закрыты). +- **Рецензирование кодовой базы** (2026-08-05, `522425a`): 10 разделов (архитектура, качество, docs, GUI, legacy, риски) плюс приоритизированный план доработок фазами 0–3. Критичных багов не найдено; единственным блокером релиза названа § D. **План выполнен целиком** (см. записи ниже), поэтому сам документ `docs/code-review.md` удалён — незакрытые пункты унесены в [roadmap.md](roadmap.md) (разбиение `internal/web`, индекс документации в README, адаптивный интервал опроса, `CONTRIBUTING.md`), остальное либо сделано, либо уже описано в architecture.md / security.md / комментариях кода. Текст ревизии — в git-истории. +- **§ D выполнен (2026-08-06):** предрелизная ревизия безопасности моделью Fable — диф от аудита v1.0 (Фаза 11, `bd64e80`) до HEAD + полный проход по чек-листу [security.md](security.md) (бывшее ТЗ 7.6). Эксплуатируемых находок нет; одна правка defence-in-depth (`--` перед логином в argv `saslpasswd2`, `internal/app/sasl.go` + тест). Принятые риски не пополнились. Детали — [implementation-plan.md](implementation-plan.md) § D и CHANGELOG `[Unreleased]/Security`. Локально `go vet`/`go test ./internal/app/...` чистые; падения `internal/domain` (`TestWriteLoadPrivateKeyRoundtrip`, `TestRenderTables`) и `internal/logtail` (`TestFollowTailsAndRotates`) — Windows-специфика (права файлов/`\` в путях/rename открытого файла), на Linux CI зелено. +- **Фаза 1 плана ревизии выполнена (2026-08-06)** (doc/code hygiene, P1): cleanup ~30 stale «Phase N» комментариев в коде и shell-скриптах; исправлен stale-комментарий в `handlers_domains.go`; ADR CSRF (Origin vs токены) добавлен в [security.md](security.md); known-limitations по log-tailer уже был в [architecture.md](architecture.md) § Log tailer — отдельного действия не потребовалось; `docs/logo` в [roadmap.md](roadmap.md) закрыт (каталога нет, критерию соответствует); `gofmt -l` добавлен в CI (`.github/workflows/test.yml`). `gofmt`/`go vet`/`go test ./...` чистые в обоих модулях (dev-server). +- **Фаза 1.5 плана ревизии выполнена (2026-08-06)** (шифрование резервных копий, P1): новый пакет `internal/secretfile` — конверт `magic SELFPOST1 | type | scrypt-параметры | salt | nonce-prefix` + поток 64 KiB чанков AES-256-GCM, каждый с AAD `header+counter+last`, поэтому обрезка, перестановка и подмена не открываются (стриминг в обе стороны — полный бэкап не держится в памяти). Панель: чекбокс «Encrypt with a password» в форме полного бэкапа и экспорта домена (общий партиал `templates/encrypt_fields.html`, показ/очистка полей — `panel.js`, без inline-скриптов), импорт домена принимает `.spde` (шифрование определяется по magic, не по расширению) с полем пароля. CLI `selfpost-backup`: пишет `.spbk` при заданном пароле и умеет `-decrypt` (иначе зашифрованный бэкап нечем распаковать при restore); пароль — только `SELFPOST_BACKUP_PASSWORD` / `-password-file`, никогда argv. Умолчание не изменилось: галочка снята — прежние `.tar.gz` / `.json` байт в байт. Тесты: round-trip по размерам (0, границы чанка, несколько чанков), неверный пароль, обрезка, перестановка чанков, порча байта, чужие KDF-параметры; валидация формы пароля; round-trip CLI create→decrypt→tar. Docs: README § *Encrypting a backup or export*, [security.md](security.md) § «Резервная копия и экспорт домена» + принятый риск (шифрование опционально), [architecture.md](architecture.md) § Persistence. `gofmt`/`go vet`/`go test ./...` чистые (кроме известных Windows-падений `internal/domain`, `internal/logtail`). E2E-сценарий не добавлялся: в `test/e2e/` бэкапа не было и раньше, а прогнать новый тест локально нечем (нет Docker) — кандидат при следующем прогоне на dev-сервере. +- **Фаза 2 плана ревизии выполнена (2026-08-06)** (GUI polish, P2): опрос мониторинговых страниц не уходит на сервер, пока вкладка скрыта — фильтр повешен на `htmx:beforeRequest` в `panel.js`, а не на встроенный в htmx фильтр триггера (тот вычисляется через `new Function`, что CSP панели `default-src 'self'` без `unsafe-eval` молча ломает); тёмная тема переписана с каскада `!important` на переопределение CSS-переменных в одном блоке `prefers-color-scheme: dark`; дублирующее правило `main { max-width }` сведено к одному базовому плюс задокументированные постраничные оверрайды. Только CSS/JS, поведения сервера не касается; вживую не проверялось (нет Docker локально) — кандидат на следующий прогон на dev-сервере. +- **Фаза 3 плана ревизии выполнена (2026-08-06)** (operational improvements, P2–P3): (1) log-tailer сохраняет позицию чтения — таблица `logtail_state` (миграция `0003`, `internal/store/logtail.go`) хранит offset + отпечаток первых 512 байт лога, `internal/logtail/offset.go` решает откуда стартовать: отпечаток совпал → продолжаем с offset (дочитывается хвост, написанный пока панель лежала); не совпал (лог сменился/пересоздан) → читаем файл с начала (повторный разбор безвреден, `UpdateStatus` идемпотентен); записи нет вовсе (первый запуск) → с конца, как раньше. Запись offset — не чаще раза в 5 с, плюс форс при ротации и на выключении; сохраняется позиция *потреблённых* байт (минус недочитанная частичная строка). (2) L2-лимит перестал промахиваться при параллельных сессиях: между проверкой на MAIL FROM и вставкой строки на end-of-message сообщение не видно в БД, поэтому N одновременных сессий пропускали друг друга — теперь к счёту из БД добавляются «в полёте» (`internal/milter/inflight.go`, общий на процесс реестр резерваций); резервация освобождается после записи в send-log, на ABORT и по TTL 10 минут (у go-milter нет колбэка на закрытие соединения, а вечная резервация — это fail-closed-дрейф, которого у лимитера быть не должно). Транзакция «count+insert», как предлагал review, невозможна буквально: эти два шага разнесены по разным стадиям SMTP-транзакции. Тесты: restart/rotation-resume для tailer'а, четыре сценария резерваций для лимита. `gofmt`/`go vet` чистые; `go test ./...` — падения только известные Windows-специфичные (`internal/domain`, `TestFollowTailsAndRotates`). Не проверено на стенде (нет Docker локально) — кандидат на следующий прогон на dev-сервере. +- **Добор по плану ревизии выполнен (2026-08-06):** (1) проект переехал на единственную площадку — GitHub (Codeberg уходит): вместе с URL, лицензионными шапками SVG/HTML и docs переехал путь Go-модуля на `github.com/mixeme/selfpost` (`go.mod`, `test/e2e/go.mod`, все импорты, `MODULE` в Makefile, `-ldflags` в Dockerfile и development.md) — оставлять импорты на исчезающем хосте нельзя, `go get`/`go install` сломались бы; (2) ссылки на архивную спецификацию убраны из кода целиком — не только «spec 7.x», как просило ревью, но и «spec 4/5/6/8/9», страдавшие тем же, каждая заменена на живой документ с секцией там, где документ большой; (3) [architecture.md](architecture.md) § Code layers — диаграмма слоёв (A2); (4) `TestParseDelivery` расширен экзотикой mail.log — и **вскрыл реальный баг**: шаблон брал `status=` жадно, то есть последнее вхождение в строке, а Postfix дописывает ответ удалённого сервера дословно, поэтому отказ с `status=sent` в тексте ответа попадал в журнал как доставленный (исправлено на ленивый разбор); (5) `CONTRIBUTING.md` перенесён в 2.x, бамп тега образа и git-тег оставлены в [roadmap.md](roadmap.md) § v1.x. `gofmt`/`go vet` чистые в обоих модулях, `go test ./...` — падения только известные Windows-специфичные (`internal/domain`, `TestFollowTailsAndRotates`). На стенде не проверялось (нет Docker локально). +- **Дальше:** релизный гейт (Фаза 0) закрыт по существу — e2e C.4 и ревизия § D пройдены; остаются только шаги, которые делаются в момент резки версии (бамп тега образа в compose, git tag) по явной команде пользователя. Все polish-фазы плана ревизии (1, 1.5, 2, 3) закрыты. +- **Принятые риски** — [security.md](security.md). **Опционально v1.x / 2.x** — [roadmap.md](roadmap.md) (хвост документации, send-log gaps, Фаза O1+, роль администратора домена). - **Прод:** `selfpost.example.com`, реальный Let's Encrypt сертификат, живой e2e (DKIM/SPF pass). Контейнер там всё ещё на образе v1.0 — Фаза 14 в него не выкатывалась. При апгрейде: админа один раз разлогинит (сменилось имя cookie), а от reverse-proxy требуется передача исходного `Host` (Apache-фрагмент из `deploy/` это делает). ## Рабочая петля (dev loop) — ВАЖНО diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..6c346c5 --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,190 @@ +# Дорожная карта: SelfPost 2.x.x + +**Статус:** здесь собран объём, отнесённый к релизной линии **2.x.x** — вне +базового объёма v1.0/v1.x (v1.x — только исходящий релей). Реализация — +только после явного согласования ([product.md](product.md), [development.md](development.md) § Agent rules): +[product.md](product.md) явно исключает часть этого объёма (приём входящей +почты; несколько пользователей/роли), поэтому включение — сознательное +расширение границ проекта, а не доработка по своей инициативе. Присутствие +пункта здесь фиксирует намерение и дизайн; кодирование начинается отдельным +решением. + +**Основа:** [product.md](product.md) v1.0. Несделанное для v1.0/v1.x +— в [implementation-plan.md](implementation-plan.md). Хвост закрытого +документационного прохода (D1–D9) — в секции ниже. + +--- + +## v1.x — хвост документации и деплоя + +**Статус:** не блокирует релизный тег; перенесено из закрытого +[documentation-plan.md](documentation-plan.md) (бывшая находка 11 и отложенный +пункт D4). Делать по желанию или в релизном коммите, где указано. + +**Тег образа в compose + git tag — один релизный коммит (R1).** В +[deploy/docker-compose.yml](../deploy/docker-compose.yml) поле `image:` бампить +до версии релиза **в том же коммите**, что и git-тег `vX.Y.Z` — не раньше. +Сейчас там `0.1.0`, то есть отстаёт от целевой версии; несовпадение мешает +только до первого выката по тегу. Сам тег — последний шаг релизного гейта: +содержательная часть (e2e C.4, ревизия § D) закрыта, режется по явной команде +оператора ([progress.md](progress.md)). После тега `release.yml` +собирает и публикует `ghcr.io/mixeme/selfpost:X.Y.Z`, поэтому compose с новым +тегом и сам тег обязаны появиться вместе — иначе compose неделю ссылается на +несуществующий образ. + +**Убрать `implementation-plan.md` — в релизном коммите.** Документ закрыт: +уникального содержания в нём нет, § D (предрелизная ревизия безопасности) +продублирован в [progress.md](progress.md), [security.md](security.md) и +CHANGELOG `[Unreleased]/Security`, а разделы B.1–B.3 и C.4 вырезаны ещё в +`22f86d1`. Держится до тега только потому, что описывает релизный гейт, пока тот +формально не закрыт. При резке версии: + +1. Переместить в `docs/archive/` (рядом со `specification-v1.0.md`) — история + § D сохраняется, из активной документации уходит. +2. Перецелить ссылки из кода и CI ([Makefile](../Makefile), + [.github/workflows/release.yml](../.github/workflows/release.yml), + [test/e2e/main_test.go](../test/e2e/main_test.go)) — они ссылаются на «план + C.4», секцию, которой в файле уже нет; актуальное описание e2e — в + [development.md](development.md). +3. Перецелить ссылки из документации: [README.md](../README.md) («Open v1.x + questions» — открытых вопросов там нет) → [progress.md](progress.md); + [security.md](security.md), [documentation-plan.md](documentation-plan.md), + [progress.md](progress.md) и шапку этого файла → на + `progress.md`/`security.md`. +4. В [progress.md](progress.md) убрать шаг «Открыть `implementation-plan.md`» — + он выполнен. + +**Готово, когда:** тег образа в compose совпадает с релизом и рядом стоит +git-тег `vX.Y.Z`; `implementation-plan.md` в `docs/archive/`, ссылок на него в +активных документах и в коде/CI не осталось. + +(Закрыто и действия не требует: `docs/logo` как каталога нет — критерию «либо +содержит файлы, либо отсутствует» удовлетворяет; Quick start в +[README.md](../README.md) тянет `docker-compose.yml` и `.env.example` с +`raw.githubusercontent.com` — это и есть единственная площадка проекта, зеркал +больше нет.) + +**Сводный индекс документации в README.** Ссылки на `docs/` разбросаны по +тексту README (блок в шапке плюс упоминания по месту), единого списка нет — +читателю, который ищет «а где вообще что», приходится вычитывать документ. +Стоит одного абзаца со списком всех файлов `docs/` и одной строкой на каждый. +Мелочь, но именно она делает набор документов набором, а не россыпью. + +**Опрос мониторинга у открытой, но незанятой вкладки.** Скрытая вкладка уже не +опрашивает сервер (фильтр на `htmx:beforeRequest` в +[panel.js](../internal/web/static/panel.js)). Остаток: вкладка на переднем +плане, с которой не работают, всё равно ходит раз в 5 с. Кандидат — адаптивный +интервал (5 с при активности, 30 с при простое) по `htmx:afterRequest` без +изменения `hx-trigger`. Ценность низкая: нагрузка — один SQL-запрос и рендер +фрагмента, так что это скорее гигиена, чем экономия. Допустимый исход — +осознанно не делать. + +**Send-log vs `mail.log` (частично закрыто).** Persist позиции чтения сделан +(таблица `logtail_state`, миграция `0003`): после +рестарта панели log-tailer дочитывает пропущенный хвост. Остаётся пересоздание +контейнера — `mail.log` не в `/data` и теряется вместе с ним, такие строки +навсегда останутся `queued`. Кандидаты, если станет больно: volume для лога, +сверка зависших строк через `postqueue`. As-built и принятый риск: +[architecture.md](architecture.md) § Log tailer, [security.md](security.md). + +--- + +## Фаза O1 (→ 2.x.x) — Входящий релей (backup-MX / пересылка) — опция/плагин + +**Цель:** возможность принимать почту на порт 25 для явно настроенных доменов и пересылать её на заданный вышестоящий backend (роль backup-MX / relay-forwarder), **как выключаемый по умолчанию модуль**, не затрагивающий поведение и поверхность атаки базового исходящего релея. + +**Зачем это нужно (сценарии):** +- **Backup-MX** — принять почту, когда основной почтовый сервер домена временно недоступен, и передать её, когда он вернётся. +- **Фронт для сервера без внешнего IP** — у оператора есть свой почтовый сервер, который по каким-то причинам **сам не может принимать почту из интернета** (нет статического/внешнего IP, за NAT, серый адрес, закрытый порт 25 на входящую и т.п.). SelfPost с публичным IP и корректным PTR выступает публичным входным узлом для домена (MX указывает на него) и пересылает почту на этот внутренний/недоступный извне сервер. + +**Граница объёма (критично — что это НЕ):** +- **ЭТО:** приём на 25 для доменов из явного списка + пересылка (relay/forward) на upstream (`relay_domains` + `transport_maps` + `relay_recipient_maps`). Postfix здесь — чистый пересыльщик, без локальной доставки. +- **ЭТО НЕ (out of scope, [product.md](product.md)):** локальная доставка в почтовые ящики, IMAP/POP3, webmail, Dovecot. Никаких mailbox'ов. SelfPost также **не реализует и не тянет в свой образ** движок антиспама/антивируса (rspamd/ClamAV) — но, в отличие от прежней формулировки, и **не** перекладывает фильтрацию на backend (см. блок «Антиспам» ниже): предоставляет точку подключения внешнего фильтра. + +**Почему как опция/плагин:** +- Приём на порт 25 меняет модель угроз (open relay для входящей, backscatter, spam-ingress). Поэтому по умолчанию **выключено** флагом env `INBOUND_RELAY_ENABLE=false`; включение — осознанный шаг оператора. +- Изоляция: отдельные таблицы SQLite, отдельные хендлеры/страницы панели, отдельная ветка генерации конфига. При выключенном флаге входной listener, таблицы и UI отсутствуют — базовый исходящий тракт байт-в-байт неизменен. + +**Что делать:** +- Env-флаг `INBOUND_RELAY_ENABLE` (default false); при `true` — генерировать входной сервис и его конфиг из состояния панели тем же путём, что остальной конфиг (`postfix-config.sh`). +- **`master.cf`:** входной `smtp inet` на 25 для приёма из интернета (сейчас 25 используется только на исходящую доставку). Отдельный от 465/587: на 25 **не** предлагается SASL и **не** разрешается отправка наружу — только приём для `relay_domains`. +- **Анти-open-relay для входящей (обязательно):** `smtpd_relay_restrictions`/`smtpd_recipient_restrictions` входного smtpd принимают почту **только** для доменов из `relay_domains` и **только** для известных получателей (`relay_recipient_maps`); всё прочее — `reject_unauth_destination`/`reject_unlisted_recipient`. Открытый релей и приём «для кого угодно» невозможны. +- **Backscatter:** предпочтительно знать валидных получателей (reject unknown recipient на этапе RCPT), чтобы не порождать bounce на несуществующие адреса. +- **Панель управляет:** список входящих доменов; для каждого — upstream destination (`host:port`, транспорт), опциональный список валидных получателей, опциональный TLS к upstream. Строгая валидация домена/хоста/порта (whitelist), injection-safe запись map-файлов (как `sender_login_maps` в Фазе 4), `os/exec` без shell ([security.md](security.md)). +- **Милтеры:** OpenDKIM на входящем тракте не нужен (чужую входящую не подписываем). journal-milter опционально переиспользовать для журнала входящих (доп. работа) либо на первом этапе оставить входящий без него; поведение fail-open сохраняется. +- **Rate-limit/размер:** грубый лимит по client IP (`anvil`, как L1) и `message_size_limit` на входном smtpd. + +**Антиспам (важная, но опциональная возможность).** Это ценная опция, но она **не обязательна**: часть операторов вполне устроит **слепая пересылка без фильтрации** — например, когда backend сам умеет фильтровать по содержимому, стоит доверенный upstream, или объём/риск невелик. Поэтому антиспам-хук по умолчанию **выключен** (пустой `INBOUND_ANTISPAM_MILTER`), и входящий релей полностью работоспособен без него. Важно другое — где фильтрация возможна технически: при «слепом» relay целевой backend видит подключающимся IP адрес **SelfPost**, а не исходного отправителя, поэтому на backend'е ломается всё, что завязано на origin IP (DNSBL/репутация проверяются против IP SelfPost, SPF даёт fail — SelfPost не входит в SPF домена-отправителя). **Единственная точка, где ещё виден настоящий client IP — входной хоп на SelfPost**; поэтому тем, кому фильтрация нужна, она должна быть *подключаема именно здесь*, а не переложена на backend, который эту информацию уже потерял. Дизайн подключения: +- **Движок антиспама — отдельный опциональный контейнер** (rspamd и т.п.), который оператор запускает **только если нужна эта опция** (тот же принцип, что reverse-proxy — отдельный контейнер вне образа SelfPost). SelfPost его **не содержит и не запускает** — образ и принцип «один контейнер, три процесса» неизменны, [product.md](product.md) out of scope не нарушается (SelfPost не реализует антиспам). +- **SelfPost предоставляет точку подключения:** milter-хук на входном smtpd. Адрес движка задаётся env (например, `INBOUND_ANTISPAM_MILTER=inet:antispam:11332`, пусто → хук выключен) и добавляется в `smtpd_milters` **только входного** тракта (не на 465/587). Postfix передаёт milter'у настоящий client IP/HELO/PTR — фильтр видит истинный origin. `milter_default_action` для этого milter'а — конфигурируемый (fail-open vs tempfail); дефолт определить при реализации. +- **Нативный backstop без зависимостей:** на том же входном хопе доступны средства Postfix по origin IP — `reject_rbl_client` (DNSBL), проверки HELO/PTR — работают даже без внешнего контейнера. Плюс сохранение аутентификации для downstream через ARC/`Received` там, где часть фильтрации всё же остаётся на backend. +- **docker-compose:** задокументировать опциональный фрагмент antispam-сайдкара (как альтернативные фрагменты reverse-proxy) — контейнер поднимается вместе со стеком только при включённой опции. +- **Персистентность:** новые таблицы и map-файлы под `/data` — попадают в полный бэкап автоматически (Фаза 9). Экспорт/импорт домена можно расширить входящей конфигурацией — опционально, пометить. +- **DNS-документация:** для входящего домена нужна `MX`-запись, указывающая на сервер (в отличие от исходящего, где MX не требуется) — отразить в разделе DNS README. + +**Безопасность ([security.md](security.md)):** валидация ввода на сервере, экранирование записи в конфиги, `exec` без интерполяции, никакого open relay, защита от backscatter. + +**Готово, когда:** при `INBOUND_RELAY_ENABLE=true` и настроенном домене письмо на порт 25 для этого домена пересылается на заданный upstream; почта для ненастроенных доменов/получателей отклоняется (не open relay, не backscatter); при заданном `INBOUND_ANTISPAM_MILTER` входящая проходит через внешний фильтр с настоящим origin IP (проверено сайдкар-контейнером), при пустом — хук не мешает; при `INBOUND_RELAY_ENABLE=false` — входной порт/таблицы/UI отсутствуют, базовый исходящий релей неизменён; `build`/`vet`/`test`/образ зелёные. + +**Риски:** open relay/backscatter (снимается `relay_domains` + `relay_recipient_maps` + `reject_unauth_destination`); потеря origin IP для фильтрации на backend'е при пересылке (снимается milter-хуком антиспама + нативным DNSBL на входном хопе, где origin IP ещё виден); порт 25 на приём расширяет поверхность атаки (по умолчанию выключено). **Модель:** Opus (инфра/безопасность, риск open relay). **Внешняя зависимость деплоя:** опциональный antispam-контейнер — вне образа SelfPost, поднимается оператором при включении опции. + +**Зависимости:** не является частью v1.0, зависит только от готового исходящего тракта (уже реализован) и требует отдельного согласования ([development.md](development.md)) до кодирования. + +--- + +## Роль администратора домена — кандидат на 2.x + +**Что это.** Сейчас в панели ровно один субъект: `requireAuth` — булев гейт, а не роль ([web.go:182](../internal/web/web.go:182)), сессия не несёт ничего, кроме факта входа. Роль выдаёт доступ к одному домену и только к нему: приложения этого домена (создание, режим отправителя, перегенерация пароля, удаление, свой L2-лимит), DKIM/DNS-статус домена и журнал отправки, отфильтрованный по домену — фильтр в журнале уже есть ([handlers_monitor.go:49](../internal/web/handlers_monitor.go:49)). Вне роли остаётся то, что глобально по своей природе: добавление и удаление доменов, `/reload`, полный бэкап (это весь `/data` вместе с `sasldb2`, то есть все домены сразу), очередь и хвост `mail.log` — они серверные и к домену не привязаны. + +**Почему 2.x, а не v1.x.** [product.md](product.md) относит «несколько пользователей +панели, роли» к out of scope (один администратор), поэтому появление второго +субъекта — расширение границ проекта, как и Фаза O1: сначала согласование +([development.md](development.md)), только потом код. Цена — уровня фазы, а не патча: таблица пользователей и их привязка к доменам, роль в сессии, авторизация в каждом хендлере (а не только на маршруте — сейчас `{id}`/`{aid}` не сверяются ни с чем, кроме существования), пересмотр первичного setup'а и смены пароля под нескольких пользователей, учёт нового субъекта в бэкапе и экспорте домена. + +*(Прежняя формулировка этого пункта — «2FA и несколько администраторов» — заменена: 2FA снята с рассмотрения, а «несколько администраторов» уточнено до одной конкретной роли, потому что нужна не вторая копия всевластного админа, а ограниченный доступ владельца отдельного домена.)* + +--- + +## `CONTRIBUTING.md` — кандидат на 2.x + +**Что это.** Точка входа для стороннего контрибьютора: dev loop (правка на +Windows → сборка и прогон на Debian-сервере, потому что Docker локально нет), +маршрутизация моделей по типу работы, протокол коммитов, требование +`gofmt`/`vet`/`test`/`make e2e` до PR. Сейчас всё это есть, но в +[development.md](development.md) и [progress.md](progress.md) — то есть на +русском и вперемешку с внутренним состоянием проекта. + +**Почему 2.x, а не v1.x.** Файл имеет смысл, когда есть кому его читать: у +проекта один разработчик и внешнего потока PR нет, поэтому сейчас +`CONTRIBUTING.md` был бы документом без аудитории и ещё одним местом, где +расходится правда о dev loop. Уместен вместе с тем, что реально открывает +проект вовне: английская документация процесса (сейчас процессные документы — +`progress.md`, `roadmap.md`, `development.md` — на русском, а README и +`architecture.md` на английском; для EN-only контрибьютора это барьер) и +первый внешний интерес после публикации релиза. + +**Готово, когда:** `CONTRIBUTING.md` в корне описывает dev loop, требования к +проверкам перед PR и протокол коммитов; [development.md](development.md) не +дублирует его, а ссылается. + +--- + +## Разбиение `internal/web` на подпакеты — кандидат на 2.x + +**Что это.** `internal/web` — самый крупный пакет проекта: 47 файлов, ~3030 +строк, в одной плоскости лежат хендлеры всех разделов панели, сессии, +security-заголовки, проверка Origin, валидация форм и рендер шаблонов. +Кандидаты на выделение — `web/handlers` и `web/auth`, либо разрез по доменам +панели. + +**Почему 2.x, а не сейчас.** На нынешнем размере плоский пакет читается: имена +файлов (`handlers_domains.go`, `handlers_apps.go`, `handlers_monitor.go`) +работают не хуже каталогов, а разбиение потянуло бы за собой экспорт того, что +сейчас пакетно-приватно, — то есть расширение внутреннего API ради +косметики. Смысл появляется ровно тогда, когда пакет начнёт расти: обе задачи +2.x выше добавляют в него код — роль администратора домена приносит +авторизацию в каждый хендлер, входящий релей — отдельные страницы и хендлеры +входящих доменов. Рефакторинг дешевле делать перед этим ростом, чем после. + +**Готово, когда:** решение принято осознанно в момент старта 2.x — либо пакет +разрезан, либо зафиксировано, что он остаётся плоским. diff --git a/docs/security.md b/docs/security.md index f9c2b83..3f5f587 100644 --- a/docs/security.md +++ b/docs/security.md @@ -1,19 +1,101 @@ -# Безопасность: принятые риски +# Безопасность -**Что здесь.** Обязательные требования к безопасности — [ТЗ 7.6](specification.md); -соответствие им проверено полным аудитом на v1.0 и здесь не пересказывается. -Hardening сверх обязательного 7.6 (security-заголовки, проверка origin, cookie -`__Host-` с обнаружением дублей — Фаза 14) тоже закрыт, история — в -[CHANGELOG.md](../CHANGELOG.md) и `git log`. Этот документ держит третью -категорию: то, что закрыто **сознательно не было**, чтобы решение не потерялось -и не переоткрывалось заново. +**Что здесь.** (1) **Обязательные требования** — чеклист, который v1.0 обязан +выполнять; полный аудит на v1.0 пройден. Предрелизная ревизия (план § D, +модель Fable, 2026-08-06) прошла по всему дифу от аудита v1.0 (Фаза 11) до +HEAD и по чек-листу целиком: эксплуатируемых находок нет; одна правка +defence-in-depth — `--` перед логином в argv `saslpasswd2` +([internal/app/sasl.go](../internal/app/sasl.go)). (2) **Принятые риски** — +сознательные отступления сверх обязательного, чтобы решение не потерялось. -Здесь, а не в [implementation-plan.md](implementation-plan.md), потому что план — -только про несделанную работу, а принятый риск — не работа, а решение: у него нет -состояния «в очереди», есть условие, при котором к нему возвращаются. +Hardening сверх обязательного (security-заголовки, проверка origin, cookie +`__Host-` с обнаружением дублей — Фаза 14) закрыт; история — в +[CHANGELOG.md](../CHANGELOG.md) и `git log`. + +Продуктовые границы: [product.md](product.md). Устройство as-built: +[architecture.md](architecture.md). + +--- + +## Обязательные требования + +Панель публична из интернета — пункты ниже **не опциональны**. + +### Первичная инициализация администратора + +- Одноразовая secret-ссылка `/setup/`, **не** env с готовым хэшем пароля. +- Токен ≥128 бит (`crypto/rand`); дублируется в `/data/setup-token`. +- Срок жизни токена — **10 минут**; после истечения или рестарта без завершённой + настройки — перегенерация и новый вывод в лог. +- Rate limiting на `/setup/` по IP, отдельно от логина. +- Сравнение токена — **константное по времени** (`subtle.ConstantTimeCompare`). +- Неудачные попытки **не** инвалидируют токен досрочно (защита от DoS настройки). +- После создания администратора — токен навсегда недействителен, `/setup/*` → 404. +- Пароль администратора — только bcrypt (или argon2) в SQLite; без plaintext/MD5. +- `PANEL_USERNAME` / `PANEL_PASSWORD_HASH` в env **не используются**. + +### SASL-пароли приложений + +- Панель **генерирует** пароль при создании/перевыпуске, показывает **один раз**. +- В `sasldb2` — в форме, требуемой SASL (не plaintext в панели); утерян — только + перевыпуск. + +### Ввод и конфигурация + +- Серверная валидация email/доменов (whitelist символов); клиентская не считается + защитой. +- Режим «список адресов» — каждый адрес принадлежит домену приложения до записи. +- `postfix reload` и любой `exec` — **без** shell-интерполяции пользовательского + ввода; аргументы отдельными элементами. +- Запись в конфиг-файлы — с экранированием (нет инъекции директив Postfix). + +### Аутентификация и сессии + +- Rate limiting на логин (по IP, с блокировкой/задержкой). +- Сессии: криптографически случайный токен; cookie `HttpOnly`, `Secure`, `SameSite`. +- Сессии в SQLite (SHA-256 токена, не сам токен); скользящий idle + (`PANEL_SESSION_IDLE_DAYS`). + +### Вывод и процесс + +- Рендер через `html/template` с автоэкранированием (очередь, лог, журнал, темы). +- Процесс панели **не root** (`user=panel` в supervisord); доступ к путям через + группу `selfpost` и минимальные права. + +### Почтовый тракт (связанное с безопасностью) + +- **Не open relay** — только SASL; `reject_unauth_destination`; + `smtpd_sender_login_maps` + `reject_sender_login_mismatch`. +- TLS обязателен до передачи кредов (465 wrapper / 587 `encrypt`). +- `TRUSTED_PROXY_CIDR` — только явно доверенные прокси для `X-Forwarded-For` + при rate-limit логина; пусто = XFF игнорируется. + +### Резервная копия и экспорт домена + +- Оба файла — секреты: полный бэкап несёт DKIM-ключи, `sasldb2` и хеш пароля + админа; экспорт домена — DKIM-ключ и **рабочие** пароли приложений открытым + текстом (иначе перенос без пересоздания кредов невозможен). +- Оба скачивания можно зашифровать паролем (чекбокс в форме): scrypt + (N=2¹⁵, r=8, p=1) → AES-256-GCM, поток из 64 KiB чанков, каждый + аутентифицирован заголовком, номером и флагом конца потока — обрезанный или + подменённый файл не открывается вместо тихого восстановления «хвоста». + Формат и обёртка: [internal/secretfile](../internal/secretfile/secretfile.go). +- Расширения: `.spbk` (полный бэкап), `.spde` (экспорт домена); незашифрованные + остаются `.tar.gz` / `.json`. Импорт домена определяет шифрование по magic + файла, а не по расширению. +- Пароль нигде не сохраняется: восстановить файл без него нельзя. Пароль в CLI — + только через `SELFPOST_BACKUP_PASSWORD` или `-password-file`, никогда + аргументом (список процессов читается любым процессом контейнера). +- Минимальная длина пароля — как у пароля администратора (12): файл лежит + offline и подбирается без ограничений по времени. + +--- ## Принятые риски +Здесь, а не в [implementation-plan.md](implementation-plan.md): план — про +несделанную работу, принятый риск — решение с условием возврата. + - **`POST` без `Sec-Fetch-Site` и без `Origin` пропускается.** Клиент, не посылающий ни одного из двух — по-настоящему старый браузер или webview с замороженным движком, — остаётся уязвим к CSRF с любого сайта. @@ -28,12 +110,54 @@ Hardening сверх обязательного 7.6 (security-заголовки вопросу считать появление требования «устойчиво независимо от браузера». От XSS внутри самой панели не спас бы и токен: код, исполняющийся в origin панели, отправит запрос сам — против этого работают автоэкранирование - `html/template` (7.6.7) и CSP, поэтому шаблоны не должны содержать + `html/template` и CSP, поэтому шаблоны не должны содержать inline-скриптов и inline-стилей. +- **Шифрование бэкапа и экспорта — опция, а не умолчание.** Галочка снята — + файл скачивается открытым, как в 1.0. Иначе оператор, у которого нет места + для хранения пароля, потерял бы возможность сделать бэкап вообще, а + безвозвратно нерасшифровываемый архив хуже незашифрованного: пароль SelfPost + не хранит. Триггером сделать шифрование обязательным считать появление + второго администратора (тогда «кто скачал» перестаёт быть одним человеком). +- **Send-log может навсегда остаться `queued` после пересоздания контейнера.** + Рестарт панели это больше не вызывает: log-tailer сохраняет позицию чтения + (`logtail_state`, миграция `0003`) и после старта дочитывает хвост, написанный + пока панель лежала. Но `mail.log` живёт не в `/data` и теряется вместе с + контейнером — строки, чьи delivery-строки были только в нём, останутся + `queued`. См. [architecture.md](architecture.md) § Log tailer. + +## ADR: CSRF через проверку Origin, без токенов + +**Контекст.** Панель — формы (`POST`) с cookie-сессией; классическая CSRF- +поверхность. Нужен способ отличить запрос со страницы панели от запроса, +инициированного сторонним сайтом в браузере залогиненного админа. + +**Решение.** `originAllowed` в +[internal/web/security.go](../internal/web/security.go) сверяет `Sec-Fetch-Site` +(если браузер его шлёт) либо `Origin` (fallback) с хостом панели; запрос без +обоих заголовков **пропускается**, а не отклоняется. Токенов, привязанных к +сессии и встроенных в формы, нет. + +**Почему не токены.** Панель однопользовательская (один администратор на +инстанс) — модель угроз не включает межпользовательский CSRF внутри самой +панели, только внешний сайт, заставляющий браузер админа отправить запрос. +Origin-проверка закрывает это без изменения ни одного шаблона: токен потребовал +бы скрытого поля примерно в двух десятках форм и синхронизации при каждой +новой форме, а от XSS внутри панели токен всё равно не защищает — код, +исполняющийся в origin панели, читает токен и отправляет запрос сам. От XSS +защищают автоэкранирование `html/template` и CSP, поэтому это отдельная линия +обороны, не CSRF-токен. + +**Компромисс.** Клиент, не посылающий ни `Sec-Fetch-Site`, ни `Origin` +(по-настоящему старый браузер или webview с замороженным движком), остаётся +уязвим — см. «Принятые риски» выше. Это осознанный выбор в пользу не ломать +панель в таком клиенте ценой узкой остаточной поверхности. + +**Пересмотр, если:** появится требование защиты, не зависящей от поведения +браузера, или панель станет многопользовательской. ## Как этот список пополняется -Предрелизная проверка на уязвимости (пункт **D.5** плана, модель Fable) закрывает -каждую находку одним из двух способов: правка до тега — либо запись сюда, с -обоснованием и условием возврата, как у двух пунктов выше. Третьего варианта -(«посмотрели и ладно») нет. +Предрелизная проверка на уязвимости ([implementation-plan.md](implementation-plan.md) +§ D, модель Fable) закрывает каждую находку одним из двух способов: правка до +тега — либо запись сюда, с обоснованием и условием возврата, как у пунктов выше. +Третьего варианта («посмотрели и ладно») нет. diff --git a/go.mod b/go.mod index 5d4e1e8..0b64846 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module codeberg.org/mix/selfpost +module github.com/mixeme/selfpost go 1.26 diff --git a/internal/app/password.go b/internal/app/password.go index 8289a82..9e27703 100644 --- a/internal/app/password.go +++ b/internal/app/password.go @@ -8,12 +8,12 @@ import ( // passwordBytes is the number of random bytes behind a generated application // password. 24 bytes = 192 bits of entropy, well above any brute-force concern -// for a SASL credential the panel shows exactly once (spec 7.6.1). +// for a SASL credential the panel shows exactly once (security.md). const passwordBytes = 24 // generatePassword returns a strong, URL-safe random password for an // application's SASL account. The panel generates it, shows it once and never -// stores the plaintext (spec 7.6.1); sasldb2 keeps only the hashed form. +// stores the plaintext (security.md); sasldb2 keeps only the hashed form. // // base64url output keeps the password to a safe ASCII alphabet with no shell or // SMTP-special characters, so it survives being typed into client configuration diff --git a/internal/app/sasl.go b/internal/app/sasl.go index 1865e12..84cc67a 100644 --- a/internal/app/sasl.go +++ b/internal/app/sasl.go @@ -10,11 +10,12 @@ import ( ) // SASLDB manages the Cyrus SASL account database (sasldb2) the panel maintains -// for application credentials (spec 5.1). The panel is the only writer; Postfix -// reads it to authenticate SMTP clients. Accounts are created and removed with -// the standard saslpasswd2 tool ("эквивалент saslpasswd2", per the plan). +// for application credentials (architecture.md § Mail path). The panel is the +// only writer; Postfix reads it to authenticate SMTP clients. Accounts are +// created and removed with the standard saslpasswd2 tool ("эквивалент +// saslpasswd2", per the plan). type SASLDB struct { - path string // sasldb2 file, under /data so it survives restarts (spec 9) + path string // sasldb2 file, under /data so it survives restarts realm string // SASL realm, so lookups match what Postfix's SASL uses // run executes saslpasswd2. It is a field so tests can substitute a fake; @@ -28,20 +29,20 @@ type SASLDB struct { // NewSASLDB builds a manager for the sasldb2 at path with the given realm. The // realm should match SELFPOST_HOSTNAME so the account identity lines up with -// Postfix's SASL configuration in Phase 5. +// Postfix's SASL configuration. func NewSASLDB(path, realm string) *SASLDB { return &SASLDB{path: path, realm: realm, run: runSaslpasswd2, dump: dumpSASLDB} } // Set creates or updates an application's SASL account with the given password -// (spec 5.1, 7.2.9). Used both at creation and when a password is regenerated; -// saslpasswd2 overwrites an existing entry in place. +// (architecture.md § Mail path). Used both at creation and when a password is +// regenerated; saslpasswd2 overwrites an existing entry in place. // // The password is passed to saslpasswd2 on stdin (never as an argument, so it // cannot leak through the process table or logs). The login is passed as a // separate argv element after being whitelisted by validateLogin — it never -// goes through a shell and is never interpolated into a command string (spec -// 7.6.3). +// goes through a shell and is never interpolated into a command string +// (security.md). func (s *SASLDB) Set(login, password string) error { if err := validateLogin(login); err != nil { return err @@ -50,21 +51,24 @@ func (s *SASLDB) Set(login, password string) error { // -c: create the account / set the password. // -f: operate on our sasldb2 rather than the system default path. // -u: the realm the account lives under. - args := []string{"-p", "-c", "-f", s.path, "-u", s.realm, login} + // --: end of options, so a login can never be parsed as a flag (the + // whitelist already forbids nothing that getopt would eat, but a login + // starting with '-' is legal there — this keeps it an operand). + args := []string{"-p", "-c", "-f", s.path, "-u", s.realm, "--", login} if err := s.run(args, []byte(password)); err != nil { return fmt.Errorf("saslpasswd2 set %q: %w", login, err) } return nil } -// Delete removes an application's SASL account (spec 7.2.8). A missing account +// Delete removes an application's SASL account (product.md). A missing account // is not treated as an error, so deletion is idempotent and safe to retry. func (s *SASLDB) Delete(login string) error { if err := validateLogin(login); err != nil { return err } - // -d: delete the account. - args := []string{"-d", "-f", s.path, "-u", s.realm, login} + // -d: delete the account. "--" as in Set: the login is always an operand. + args := []string{"-d", "-f", s.path, "-u", s.realm, "--", login} if err := s.run(args, nil); err != nil { return fmt.Errorf("saslpasswd2 delete %q: %w", login, err) } @@ -76,15 +80,16 @@ func (s *SASLDB) Delete(login string) error { var ErrSecretNotFound = fmt.Errorf("sasl secret not found") // Secret returns an application's stored password so it can be carried in a -// domain export and re-created verbatim on another instance (spec 7.5.B). This -// is possible because sasldb2 keeps the SASL secret in a password-equivalent -// form (the plaintext userPassword property, to serve challenge-response -// mechanisms) — unlike the admin's one-way bcrypt hash (spec 7.6). The value is -// realm-independent, so the importer can re-key it under its own realm. +// domain export and re-created verbatim on another instance (architecture.md § +// Persistence). This is possible because sasldb2 keeps the SASL secret in a +// password-equivalent form (the plaintext userPassword property, to serve +// challenge-response mechanisms) — unlike the admin's one-way bcrypt hash +// (security.md). The value is realm-independent, so the importer can re-key it +// under its own realm. // // It reads the database with db_dump (Berkeley DB), passing only our own file -// path as a fixed argument (no shell, no user input — spec 7.6.3), and returns -// ErrSecretNotFound if the login has no entry. +// path as a fixed argument (no shell, no user input — security.md), and +// returns ErrSecretNotFound if the login has no entry. func (s *SASLDB) Secret(login string) (string, error) { if err := validateLogin(login); err != nil { return "", err @@ -155,7 +160,7 @@ func parseSASLSecret(dump []byte, login, realm string) (string, bool, error) { // runSaslpasswd2 executes the real saslpasswd2 with the given arguments and // stdin. Arguments are passed as a fixed argv (no shell), so no user input is -// ever interpreted as a command (spec 7.6.3). +// ever interpreted as a command (security.md). func runSaslpasswd2(args []string, stdin []byte) error { cmd := exec.Command("saslpasswd2", args...) if stdin != nil { @@ -170,7 +175,7 @@ func runSaslpasswd2(args []string, stdin []byte) error { // dumpSASLDB runs db_dump to export the sasldb2 as key/value hex pairs. The path // is our own sasldb2 file (never user input) and is passed as a fixed argument -// with no shell (spec 7.6.3). +// with no shell (security.md). func dumpSASLDB(path string) ([]byte, error) { cmd := exec.Command("db_dump", path) out, err := cmd.CombinedOutput() diff --git a/internal/app/sasl_test.go b/internal/app/sasl_test.go index 78f7a5b..51172c0 100644 --- a/internal/app/sasl_test.go +++ b/internal/app/sasl_test.go @@ -39,8 +39,9 @@ func TestSASLSetPassesPasswordOnStdinNotArgv(t *testing.T) { if strings.Contains(joined, secret) { t.Errorf("password leaked into argv: %q", joined) } - // Expected fixed flags and the login as its own trailing argument. - want := []string{"-p", "-c", "-f", "/data/sasl/sasldb2", "-u", "mail.example.com", "alerts"} + // Expected fixed flags and the login as its own trailing argument, behind + // "--" so it can never be parsed as an option. + want := []string{"-p", "-c", "-f", "/data/sasl/sasldb2", "-u", "mail.example.com", "--", "alerts"} if len(fr.args) != len(want) { t.Fatalf("args = %v, want %v", fr.args, want) } @@ -56,7 +57,7 @@ func TestSASLDeleteArgs(t *testing.T) { if err := s.Delete("alerts"); err != nil { t.Fatalf("Delete: %v", err) } - want := []string{"-d", "-f", "/data/sasl/sasldb2", "-u", "mail.example.com", "alerts"} + want := []string{"-d", "-f", "/data/sasl/sasldb2", "-u", "mail.example.com", "--", "alerts"} if strings.Join(fr.args, " ") != strings.Join(want, " ") { t.Errorf("delete args = %v, want %v", fr.args, want) } diff --git a/internal/app/service.go b/internal/app/service.go index b088eb7..43beb3e 100644 --- a/internal/app/service.go +++ b/internal/app/service.go @@ -1,13 +1,13 @@ -// Package app owns application accounts (spec 4.1, 5.1): the SASL credentials in -// sasldb2, the per-application sender address mode, and the -// smtpd_sender_login_maps bindings that tie each login to the addresses it may -// send from. It keeps those three stores — the SQLite registry, sasldb2 and the -// Postfix map — in agreement and drives the Postfix reload. +// Package app owns application accounts (product.md § Multi-domain model): the +// SASL credentials in sasldb2, the per-application sender address mode, and +// the smtpd_sender_login_maps bindings that tie each login to the addresses it +// may send from. It keeps those three stores — the SQLite registry, sasldb2 +// and the Postfix map — in agreement and drives the Postfix reload. package app import ( - "codeberg.org/mix/selfpost/internal/postfix" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/postfix" + "github.com/mixeme/selfpost/internal/store" ) // SenderMaps is the slice of the Postfix manager the application service needs: @@ -19,7 +19,7 @@ type SenderMaps interface { // Service coordinates application state across SQLite, sasldb2 and the Postfix // sender_login_maps. Web handlers validate raw input first; the Service performs -// the domain-ownership checks that must not be skipped (spec 7.6.2) and keeps +// the domain-ownership checks that must not be skipped (security.md) and keeps // the stores consistent. type Service struct { store *store.Store @@ -33,7 +33,7 @@ func NewService(st *store.Store, sasl *SASLDB, pf SenderMaps) *Service { return &Service{store: st, sasl: sasl, pf: pf} } -// List returns a domain's applications with their address lists (spec 7.2.6). +// List returns a domain's applications with their address lists (product.md). func (s *Service) List(domainID int64) ([]store.Application, error) { return s.store.ListApplicationsByDomain(domainID) } @@ -44,10 +44,10 @@ func (s *Service) Get(id int64) (store.Application, error) { } // Create adds an application to a domain: it validates the login and (in list -// mode) that every address belongs to the domain (spec 7.6.2), generates a +// mode) that every address belongs to the domain (security.md), generates a // strong password, writes the SASL account and rebuilds the sender map (spec // 7.2.5). The generated password is returned so the caller can show it exactly -// once (spec 7.6.1) — it is never persisted in plaintext. +// once (security.md) — it is never persisted in plaintext. // // The registry row is written first so its UNIQUE constraint is the sole arbiter // of a duplicate login (avoiding a check-then-act race and, crucially, avoiding @@ -92,13 +92,14 @@ func (s *Service) rollbackCreate(id int64, login string) { _, _ = s.store.DeleteApplication(id) } -// ImportApplication re-creates an application from a domain-export file (spec -// 7.5.B): it validates the login and (in list mode) that every address belongs -// to the domain, inserts the registry row and writes the SASL account with the -// imported password verbatim, re-keyed under this instance's realm so the -// credential keeps working without regeneration. It deliberately does not -// rebuild the sender map — the caller (domain import) does that once after all -// applications are in — and returns store.ErrLoginExists if the login collides. +// ImportApplication re-creates an application from a domain-export file +// (architecture.md § Persistence): it validates the login and (in list mode) +// that every address belongs to the domain, inserts the registry row and +// writes the SASL account with the imported password verbatim, re-keyed under +// this instance's realm so the credential keeps working without regeneration. +// It deliberately does not rebuild the sender map — the caller (domain import) +// does that once after all applications are in — and returns +// store.ErrLoginExists if the login collides. func (s *Service) ImportApplication(domainID int64, login, mode string, rawAddresses []string, password string) error { addresses, err := s.validateForDomain(domainID, login, mode, rawAddresses) if err != nil { @@ -118,14 +119,15 @@ func (s *Service) ImportApplication(domainID int64, login, mode string, rawAddre return nil } -// Secret returns an application's stored password for a domain export (spec -// 7.5.B). See SASLDB.Secret for why this is possible and safe. +// Secret returns an application's stored password for a domain export +// (architecture.md § Persistence). See SASLDB.Secret for why this is possible +// and safe. func (s *Service) Secret(login string) (string, error) { return s.sasl.Secret(login) } // UpdateMode switches an application's address mode / list and rebuilds the -// sender map (spec 7.2.7). The login and password are untouched. Addresses are +// sender map (product.md). The login and password are untouched. Addresses are // re-validated against the application's domain. func (s *Service) UpdateMode(id int64, mode string, rawAddresses []string) error { a, err := s.store.GetApplication(id) @@ -172,22 +174,23 @@ func (s *Service) Delete(id int64) error { if err := s.sasl.Delete(a.Login); err != nil { return err } - // Drop the application's level-2 limit, if any (spec 7.4); rate_limits has no - // cascade of its own. + // Drop the application's level-2 limit, if any (README § Rate limiting); + // rate_limits has no cascade of its own. if err := s.store.DeleteRateLimit(store.RateLimitScopeApp, id); err != nil { return err } return s.Resync() } -// RateLimit returns the application-level differentiated rate limit (spec 7.4), -// and whether one is configured, for the application's edit form. +// RateLimit returns the application-level differentiated rate limit (README § +// Rate limiting), and whether one is configured, for the application's edit +// form. func (s *Service) RateLimit(appID int64) (store.RateLimit, bool, error) { return s.store.GetRateLimit(store.RateLimitScopeApp, appID) } // SaveRateLimit stores the application-level rate limit. The caller has validated -// the IPs and numbers (spec 7.6.2); the milter reads the row live, so no reload +// the IPs and numbers (security.md); the milter reads the row live, so no reload // is needed. func (s *Service) SaveRateLimit(appID int64, ips []string, maxMessages, windowSeconds int) error { return s.store.SetRateLimit(store.RateLimit{ @@ -199,14 +202,15 @@ func (s *Service) SaveRateLimit(appID int64, ips []string, maxMessages, windowSe }) } -// ClearRateLimit removes the application-level rate limit (spec 7.4). +// ClearRateLimit removes the application-level rate limit (README § Rate +// limiting). func (s *Service) ClearRateLimit(appID int64) error { return s.store.DeleteRateLimit(store.RateLimitScopeApp, appID) } // PurgeDomainSASL removes the SASL accounts of every application bound to a // domain. It must be called before the domain's registry rows are cascade- -// deleted, while the logins are still known (spec 7.2.4). The registry rows and +// deleted, while the logins are still known (product.md). The registry rows and // the sender map are handled by the domain deletion path; this only clears // sasldb2, which has no cascade of its own. func (s *Service) PurgeDomainSASL(domainID int64) error { @@ -223,10 +227,10 @@ func (s *Service) PurgeDomainSASL(domainID int64) error { } // Resync rebuilds smtpd_sender_login_maps from the full set of application -// bindings and reloads Postfix (spec 5.1). It is the single idempotent apply -// path shared by create/edit/delete and is also reachable from the manual -// reload button; it doubles as recovery if the map ever drifts from the -// database. +// bindings and reloads Postfix (architecture.md § Mail path). It is the single +// idempotent apply path shared by create/edit/delete and is also reachable +// from the manual reload button; it doubles as recovery if the map ever drifts +// from the database. func (s *Service) Resync() error { bindings, err := s.store.ListBindings() if err != nil { @@ -241,7 +245,7 @@ func (s *Service) Resync() error { // validateForDomain resolves the domain, validates the login and address mode, // and — in list mode — validates that every address belongs to the domain -// (spec 7.6.2). It returns the cleaned address list, which is empty in wildcard +// (security.md). It returns the cleaned address list, which is empty in wildcard // mode. Resolving the domain here also confirms it exists before any write. func (s *Service) validateForDomain(domainID int64, login, mode string, rawAddresses []string) ([]string, error) { d, err := s.store.GetDomain(domainID) diff --git a/internal/app/service_test.go b/internal/app/service_test.go index da9825d..bc99516 100644 --- a/internal/app/service_test.go +++ b/internal/app/service_test.go @@ -5,8 +5,8 @@ import ( "path/filepath" "testing" - "codeberg.org/mix/selfpost/internal/postfix" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/postfix" + "github.com/mixeme/selfpost/internal/store" ) // fakeMaps records the last set of bindings passed to a rebuild and can be told diff --git a/internal/app/validate.go b/internal/app/validate.go index 68f7d7b..31a42c2 100644 --- a/internal/app/validate.go +++ b/internal/app/validate.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/store" ) const ( @@ -13,12 +13,12 @@ const ( ) // validateLogin enforces a strict server-side whitelist for the SASL login -// (spec 7.6.2). It intentionally excludes '@': the login is stored in sasldb2, +// (security.md). It intentionally excludes '@': the login is stored in sasldb2, // where '@' separates the user from the realm, so allowing it would change the // account's identity. Client validation is never trusted. // // The login is the one piece of user input that is passed to saslpasswd2 as a -// command argument (never through a shell, spec 7.6.3); this whitelist is what +// command argument (never through a shell, security.md); this whitelist is what // makes that safe. func validateLogin(login string) error { if len(login) < minLoginLen || len(login) > maxLoginLen { @@ -36,10 +36,11 @@ func validateLogin(login string) error { } // validateImportedPassword guards a password taken from a domain-export file -// (spec 7.5.B) before it is written to sasldb2. Our own exports carry base64url -// passwords, but the file is untrusted input, so we reject an empty value or one -// containing control characters — saslpasswd2 reads the passphrase from stdin -// and a newline would silently truncate it (spec 7.6.2). +// (architecture.md § Persistence) before it is written to sasldb2. Our own +// exports carry base64url passwords, but the file is untrusted input, so we +// reject an empty value or one containing control characters — saslpasswd2 +// reads the passphrase from stdin and a newline would silently truncate it +// (security.md). func validateImportedPassword(password string) error { if password == "" { return fmt.Errorf("imported application password is empty") @@ -71,7 +72,7 @@ func normalizeAddress(addr string) string { } // validateSenderAddress enforces that a list-mode address is well-formed and, -// critically, belongs to the application's own domain (spec 7.6.2). The domain +// critically, belongs to the application's own domain (security.md). The domain // check is done here, before anything is written to a config file — not left to // smtpd_sender_login_maps to catch at delivery time. domain must already be a // validated, normalised domain name. @@ -92,7 +93,7 @@ func validateSenderAddress(addr, domain string) error { // validateLocalPart applies a conservative whitelist to the part before '@'. // This is deliberately stricter than RFC 5321 (no quoted local parts) so the -// value is always safe to write verbatim into the Postfix map (spec 7.6.4). +// value is always safe to write verbatim into the Postfix map (security.md). func validateLocalPart(local string) error { if local == "" { return fmt.Errorf("missing the part before '@'") diff --git a/internal/app/validate_test.go b/internal/app/validate_test.go index 288a62b..f50d148 100644 --- a/internal/app/validate_test.go +++ b/internal/app/validate_test.go @@ -25,7 +25,7 @@ func TestValidateLogin(t *testing.T) { } func TestValidateSenderAddressDomainOwnership(t *testing.T) { - // The critical check (spec 7.6.2): an address must belong to the app's domain. + // The critical check (security.md): an address must belong to the app's domain. if err := validateSenderAddress("alerts@example.com", "example.com"); err != nil { t.Errorf("same-domain address rejected: %v", err) } diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 439835d..d1bf4f8 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -1,17 +1,18 @@ // Package backup implements SelfPost's full-server backup and the restore -// version guard (spec 7.5.A). A full backup is a gzip-compressed tar of the -// consolidated persistent state under /data — the SQLite database (as a -// consistent snapshot), the per-domain DKIM keys and the SASL database — plus a -// manifest recording the SelfPost version that produced it. TLS certificates -// (the reverse proxy's responsibility) and the Postfix queue are deliberately -// excluded (spec 7.5.A). +// version guard (architecture.md § Persistence). A full backup is a +// gzip-compressed tar of the consolidated persistent state under /data — the +// SQLite database (as a consistent snapshot), the per-domain DKIM keys and the +// SASL database — plus a manifest recording the SelfPost version that produced +// it. TLS certificates (the reverse proxy's responsibility) and the Postfix +// queue are deliberately excluded (architecture.md § Persistence). // -// Restore is intentionally not a separate code path: a backup is extracted into -// the /data bind mount before first start, and the panel regenerates Postfix and -// OpenDKIM from the restored SQLite state exactly as on any normal start. The -// only restore-specific step is CheckRestore, which refuses to boot if the -// manifest's version does not match the running binary, so schema/format skew -// between versions cannot silently corrupt state (spec 7.5.A). +// Restore is intentionally not a separate code path: a backup is extracted +// into the /data bind mount before first start, and the panel regenerates +// Postfix and OpenDKIM from the restored SQLite state exactly as on any normal +// start. The only restore-specific step is CheckRestore, which refuses to boot +// if the manifest's version does not match the running binary, so +// schema/format skew between versions cannot silently corrupt state +// (architecture.md § Persistence). package backup import ( @@ -39,9 +40,9 @@ const FormatFull = "selfpost-full-backup" const ManifestName = "manifest.json" // Manifest is the small JSON document embedded in every backup archive. Its -// Version is the single fact that makes restore safe: the panel refuses to boot -// a data directory whose manifest version does not match its own binary (spec -// 7.5.A). +// Version is the single fact that makes restore safe: the panel refuses to +// boot a data directory whose manifest version does not match its own binary +// (architecture.md § Persistence). type Manifest struct { Format string `json:"format"` Version string `json:"version"` @@ -63,9 +64,9 @@ type Params struct { // written under the canonical name; the setup token is transient bootstrap // state; a stale manifest from a previous restore must not be re-captured (a // fresh one is written instead); and a "tls" directory holds the reverse -// proxy's certificates, which are explicitly out of scope for a SelfPost backup -// (spec 7.5.A) — excluding it keeps that guarantee even when an operator points -// TLS_CERT_FILE inside /data. +// proxy's certificates, which are explicitly out of scope for a SelfPost +// backup (architecture.md § Persistence) — excluding it keeps that guarantee +// even when an operator points TLS_CERT_FILE inside /data. var excludedFromArchive = map[string]bool{ "selfpost.db": true, "selfpost.db-wal": true, @@ -78,9 +79,10 @@ var excludedFromArchive = map[string]bool{ // Create writes a gzip-compressed tar backup to w. Archive entries are named // relative to DataDir, so extracting the archive into the /data bind mount -// reconstructs the state in place (spec 7.5.A). The SQLite database is added as -// a consistent snapshot under "selfpost.db"; everything else under DataDir is -// copied as-is except the entries in excludedFromArchive. +// reconstructs the state in place (architecture.md § Persistence). The SQLite +// database is added as a consistent snapshot under "selfpost.db"; everything +// else under DataDir is copied as-is except the entries in +// excludedFromArchive. func Create(w io.Writer, p Params) error { if p.DataDir == "" || p.DBPath == "" { return fmt.Errorf("backup: DataDir and DBPath are required") @@ -254,12 +256,13 @@ func snapshotDB(dbPath string) (path string, cleanup func(), err error) { return target, cleanup, nil } -// CheckRestore enforces the backup version guard (spec 7.5.A). If manifestPath -// exists (a backup was extracted into the data directory), its version must -// match binaryVersion or the panel refuses to start, telling the operator which -// image tag to use. On a match the manifest is consumed (deleted) so it guards -// only the first boot after a restore and never blocks a later in-place image -// upgrade. Absence of the manifest is the normal case and returns nil. +// CheckRestore enforces the backup version guard (architecture.md § +// Persistence). If manifestPath exists (a backup was extracted into the data +// directory), its version must match binaryVersion or the panel refuses to +// start, telling the operator which image tag to use. On a match the manifest +// is consumed (deleted) so it guards only the first boot after a restore and +// never blocks a later in-place image upgrade. Absence of the manifest is the +// normal case and returns nil. func CheckRestore(manifestPath, binaryVersion string) error { data, err := os.ReadFile(manifestPath) if os.IsNotExist(err) { diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index 0ba5da5..7545f72 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -11,7 +11,7 @@ import ( "strings" "testing" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/store" ) // seedDataDir builds a realistic /data tree: a migrated SQLite database plus the diff --git a/internal/buildinfo/buildinfo.go b/internal/buildinfo/buildinfo.go index 44682e2..e72117f 100644 --- a/internal/buildinfo/buildinfo.go +++ b/internal/buildinfo/buildinfo.go @@ -3,8 +3,8 @@ package buildinfo // Version is the SelfPost release. It is set at build time with // -// -ldflags "-X codeberg.org/mix/selfpost/internal/buildinfo.Version=" +// -ldflags "-X github.com/mixeme/selfpost/internal/buildinfo.Version=" // // and must match the Docker image tag; it is used for the backup/restore -// compatibility check (spec 7.5.A). Defaults to "dev" for local/unstamped builds. +// compatibility check (architecture.md § Persistence). Defaults to "dev" for local/unstamped builds. var Version = "dev" diff --git a/internal/dnscheck/dnscheck.go b/internal/dnscheck/dnscheck.go index 87e945c..ba33e4f 100644 --- a/internal/dnscheck/dnscheck.go +++ b/internal/dnscheck/dnscheck.go @@ -18,7 +18,7 @@ import ( "sync" "time" - "codeberg.org/mix/selfpost/internal/health" + "github.com/mixeme/selfpost/internal/health" ) const ( diff --git a/internal/dnscheck/dnscheck_test.go b/internal/dnscheck/dnscheck_test.go index 04eddb9..11eb75c 100644 --- a/internal/dnscheck/dnscheck_test.go +++ b/internal/dnscheck/dnscheck_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "codeberg.org/mix/selfpost/internal/health" + "github.com/mixeme/selfpost/internal/health" ) // fakeResolver serves a fixed zone, so the checks can be driven through every diff --git a/internal/dnscheck/examples_test.go b/internal/dnscheck/examples_test.go index 6cc03d4..2a542cf 100644 --- a/internal/dnscheck/examples_test.go +++ b/internal/dnscheck/examples_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "codeberg.org/mix/selfpost/internal/health" + "github.com/mixeme/selfpost/internal/health" ) func TestSPFExample(t *testing.T) { diff --git a/internal/dnscheck/records.go b/internal/dnscheck/records.go index b9f666b..6d1c9ae 100644 --- a/internal/dnscheck/records.go +++ b/internal/dnscheck/records.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "codeberg.org/mix/selfpost/internal/health" + "github.com/mixeme/selfpost/internal/health" ) // checkDKIM compares the TXT record published at ._domainkey. diff --git a/internal/dnscheck/server.go b/internal/dnscheck/server.go index b711d58..728f1ab 100644 --- a/internal/dnscheck/server.go +++ b/internal/dnscheck/server.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "codeberg.org/mix/selfpost/internal/health" + "github.com/mixeme/selfpost/internal/health" ) // checkServer resolves the panel's own hostname and confirms the reverse diff --git a/internal/dnscheck/spf.go b/internal/dnscheck/spf.go index 71412fd..e8ca01a 100644 --- a/internal/dnscheck/spf.go +++ b/internal/dnscheck/spf.go @@ -6,7 +6,7 @@ import ( "net" "strings" - "codeberg.org/mix/selfpost/internal/health" + "github.com/mixeme/selfpost/internal/health" ) // spfLookupBudget caps how many DNS lookups one SPF evaluation may spend on diff --git a/internal/domain/dkim.go b/internal/domain/dkim.go index 40fe459..aa086aa 100644 --- a/internal/domain/dkim.go +++ b/internal/domain/dkim.go @@ -1,8 +1,8 @@ // Package domain owns SelfPost's sending-domain model: per-domain DKIM key // generation, the OpenDKIM KeyTable/SigningTable that drive signing, and the // orchestration that keeps the SQLite registry, the on-disk keys and OpenDKIM -// in agreement (spec 4.1, 6). Key material lives under /data so it survives -// container restarts (spec 6.1, 9). +// in agreement (architecture.md § OpenDKIM). Key material lives under /data so +// it survives container restarts (architecture.md § OpenDKIM). package domain import ( @@ -46,7 +46,7 @@ func writePrivateKeyPEM(path string, key *rsa.PrivateKey) error { // loadPrivateKeyPEM reads and parses a PKCS#1 RSA private key written by // writePrivateKeyPEM. It is used to recompute the public DNS record on demand, -// keeping the private key file the single source of truth (spec 7.2.10). +// keeping the private key file the single source of truth (product.md). func loadPrivateKeyPEM(path string) (*rsa.PrivateKey, error) { data, err := os.ReadFile(path) if err != nil { @@ -63,7 +63,7 @@ func loadPrivateKeyPEM(path string) (*rsa.PrivateKey, error) { return key, nil } -// DKIMRecord is the DNS TXT record a user must publish for a domain (spec 7.2.10). +// DKIMRecord is the DNS TXT record a user must publish for a domain (product.md). type DKIMRecord struct { // Name is the record's host, e.g. "selfpost._domainkey.example.com". Name string @@ -89,7 +89,7 @@ func dkimRecord(selector, domainName string, pub *rsa.PublicKey) (DKIMRecord, er // writeFileAtomic writes data to path via a temp file in the same directory // followed by a rename, so readers only ever see the complete old or new file. // It is the single safe-write primitive for DKIM keys and OpenDKIM tables -// (spec 7.6.4). +// (security.md). func writeFileAtomic(path string, data []byte, perm os.FileMode) error { dir := filepath.Dir(path) tmp, err := os.CreateTemp(dir, ".tmp-*") diff --git a/internal/domain/opendkim.go b/internal/domain/opendkim.go index 92b3f92..434e30b 100644 --- a/internal/domain/opendkim.go +++ b/internal/domain/opendkim.go @@ -12,9 +12,9 @@ import ( ) // OpenDKIM manages the on-disk OpenDKIM state the panel is responsible for -// (spec 6): per-domain signing keys under keysDir and the KeyTable/SigningTable -// that map domains to those keys. After rewriting the tables it asks OpenDKIM to -// reload them. +// (architecture.md § OpenDKIM): per-domain signing keys under keysDir and the +// KeyTable/SigningTable that map domains to those keys. After rewriting the +// tables it asks OpenDKIM to reload them. type OpenDKIM struct { keysDir string keyTablePath string @@ -50,8 +50,8 @@ func (o *OpenDKIM) keyPath(domainName, selector string) string { // EnsureKey makes sure a signing key exists for the domain. An existing key is // reused untouched — critical because overwriting it would silently invalidate -// the DKIM record already published in DNS (spec 6.1). Returns whether a new key -// was generated. +// the DKIM record already published in DNS (architecture.md § OpenDKIM). +// Returns whether a new key was generated. func (o *OpenDKIM) EnsureKey(domainName, selector string) (bool, error) { if err := assertConfigSafe(domainName, selector); err != nil { return false, err @@ -77,8 +77,8 @@ func (o *OpenDKIM) EnsureKey(domainName, selector string) (bool, error) { return true, nil } -// RemoveKey deletes a domain's key directory (spec 6.5). A missing directory is -// not an error. +// RemoveKey deletes a domain's key directory (architecture.md § OpenDKIM). A +// missing directory is not an error. func (o *OpenDKIM) RemoveKey(domainName string) error { if err := assertConfigSafe(domainName, "x"); err != nil { return err @@ -89,10 +89,11 @@ func (o *OpenDKIM) RemoveKey(domainName string) error { return nil } -// ExportKey returns a domain's DKIM private key as PKCS#1 PEM, for carrying in a -// domain export so the receiving instance signs with the same key and the DNS -// TXT record never has to change (spec 7.5.B). It re-marshals the parsed key -// rather than returning the raw file, so a malformed on-disk key is caught here. +// ExportKey returns a domain's DKIM private key as PKCS#1 PEM, for carrying in +// a domain export so the receiving instance signs with the same key and the +// DNS TXT record never has to change (architecture.md § Persistence). It +// re-marshals the parsed key rather than returning the raw file, so a +// malformed on-disk key is caught here. func (o *OpenDKIM) ExportKey(domainName, selector string) ([]byte, error) { if err := assertConfigSafe(domainName, selector); err != nil { return nil, err @@ -105,11 +106,12 @@ func (o *OpenDKIM) ExportKey(domainName, selector string) ([]byte, error) { return pem.EncodeToMemory(block), nil } -// ImportKey writes an imported DKIM private key to disk for a domain (spec -// 7.5.B). The PEM is parsed and re-marshalled through the same writer used for -// generated keys, so only a well-formed PKCS#1 RSA key is ever stored. Unlike -// EnsureKey it overwrites: an import (re-)creates the domain with exactly this -// key, which is the whole point of keeping the published DNS record valid. +// ImportKey writes an imported DKIM private key to disk for a domain +// (architecture.md § Persistence). The PEM is parsed and re-marshalled through +// the same writer used for generated keys, so only a well-formed PKCS#1 RSA +// key is ever stored. Unlike EnsureKey it overwrites: an import (re-)creates +// the domain with exactly this key, which is the whole point of keeping the +// published DNS record valid. func (o *OpenDKIM) ImportKey(domainName, selector string, pemKey []byte) error { if err := assertConfigSafe(domainName, selector); err != nil { return err @@ -130,7 +132,7 @@ func (o *OpenDKIM) ImportKey(domainName, selector string, pemKey []byte) error { } // Record returns the published DKIM DNS record for a domain, recomputed from the -// private key on disk (spec 7.2.10). +// private key on disk (product.md). func (o *OpenDKIM) Record(domainName, selector string) (DKIMRecord, error) { key, err := loadPrivateKeyPEM(o.keyPath(domainName, selector)) if err != nil { @@ -140,10 +142,10 @@ func (o *OpenDKIM) Record(domainName, selector string) (DKIMRecord, error) { } // Rebuild regenerates KeyTable and SigningTable from the full domain set and -// reloads OpenDKIM (spec 6.2). Full regeneration (rather than incremental -// edits) keeps the files a pure function of the registry, so add and delete -// share one idempotent path. Both files are written atomically before the -// reload signal is sent. +// reloads OpenDKIM (architecture.md § OpenDKIM). Full regeneration (rather +// than incremental edits) keeps the files a pure function of the registry, so +// add and delete share one idempotent path. Both files are written atomically +// before the reload signal is sent. func (o *OpenDKIM) Rebuild(domains []SigningDomain) error { keyTable, signingTable, err := renderTables(o.keysDir, domains) if err != nil { @@ -158,17 +160,18 @@ func (o *OpenDKIM) Rebuild(domains []SigningDomain) error { return o.reload() } -// Reload asks OpenDKIM to re-read its tables without regenerating them. It backs -// the panel's manual reload button (spec 7.2.12). +// Reload asks OpenDKIM to re-read its tables without regenerating them. It +// backs the panel's manual reload button (architecture.md § Panel HTTP +// surface). func (o *OpenDKIM) Reload() error { return o.reload() } // renderTables builds the KeyTable and SigningTable byte contents for a domain // set, sorted by name so the output is deterministic. Every domain is -// re-checked for shell/config-injection safety before being written (spec -// 7.6.4) — validation upstream already guarantees this, but the table writer -// refuses to emit anything unsafe as a hard backstop. +// re-checked for shell/config-injection safety before being written +// (security.md) — validation upstream already guarantees this, but the table +// writer refuses to emit anything unsafe as a hard backstop. func renderTables(keysDir string, domains []SigningDomain) (keyTable, signingTable []byte, err error) { sorted := append([]SigningDomain(nil), domains...) sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name }) @@ -191,9 +194,9 @@ func renderTables(keysDir string, domains []SigningDomain) (keyTable, signingTab // assertConfigSafe rejects any domain/selector value that could break out of a // single table line. Domains are already whitelisted to [a-z0-9.-] and selectors -// to a similar set before they reach here (spec 7.6.2); this is defence in depth +// to a similar set before they reach here (security.md); this is defence in depth // against a validation gap ever letting whitespace, a newline or a field -// separator through into a config file (spec 7.6.4). +// separator through into a config file (security.md). func assertConfigSafe(domainName, selector string) error { for _, v := range []string{domainName, selector} { if v == "" { @@ -210,10 +213,10 @@ func assertConfigSafe(domainName, selector string) error { // OpenDKIM process SIGUSR1, which makes it re-read KeyTable/SigningTable // (opendkim's documented reload signal). The panel runs unprivileged and cannot // signal another user's process directly, so it goes through the supervisor -// control socket, reachable via the shared `selfpost` group (spec 7.6.3, 7.6.8). +// control socket, reachable via the shared `selfpost` group (security.md). // // Arguments are fixed literals — no user input is interpolated into the command, -// and it never goes through a shell (spec 7.6.3). +// and it never goes through a shell (security.md). func reloadViaSupervisor() error { cmd := exec.Command("supervisorctl", "-c", "/etc/supervisor/supervisord.conf", diff --git a/internal/domain/service.go b/internal/domain/service.go index 4a4ff51..44f77f9 100644 --- a/internal/domain/service.go +++ b/internal/domain/service.go @@ -3,7 +3,7 @@ package domain import ( "fmt" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/store" ) // Applications is the slice of the application service the domain service needs @@ -19,18 +19,20 @@ type Applications interface { // and reloads Postfix. Resync() error // Secret returns an application's stored password, for a domain export - // (spec 7.5.B). + // (architecture.md § Persistence). Secret(login string) (string, error) // ImportApplication re-creates an application (registry row + SASL account) - // from a domain-export file, without rebuilding the sender map (spec 7.5.B). + // from a domain-export file, without rebuilding the sender map + // (architecture.md § Persistence). ImportApplication(domainID int64, login, mode string, addresses []string, password string) error } // Service coordinates the places a sending domain lives: the SQLite registry, // the on-disk DKIM keys and OpenDKIM's tables, plus — on deletion — the SASL // database and Postfix sender map its applications touch. Callers (the web -// handlers) validate user input first; Service keeps the stores in agreement and -// drives the OpenDKIM/Postfix reloads (spec 6, 7.2.2-4, 7.2.10). +// handlers) validate user input first; Service keeps the stores in agreement +// and drives the OpenDKIM/Postfix reloads (architecture.md § OpenDKIM, +// product.md). type Service struct { store *store.Store odk *OpenDKIM @@ -39,14 +41,15 @@ type Service struct { } // NewService builds the domain service. selectorDefault is the DKIM selector -// assigned to new domains (spec 8: DKIM_SELECTOR_DEFAULT); it is -// operator-configured, not user input. apps is used only on deletion, to clear -// the SASL accounts and sender-map bindings of the domain's applications. +// assigned to new domains (README § Environment variables: +// DKIM_SELECTOR_DEFAULT); it is operator-configured, not user input. apps is +// used only on deletion, to clear the SASL accounts and sender-map bindings of +// the domain's applications. func NewService(st *store.Store, odk *OpenDKIM, apps Applications, selectorDefault string) *Service { return &Service{store: st, odk: odk, apps: apps, selector: selectorDefault} } -// List returns all domains with application counts (spec 7.2.2). +// List returns all domains with application counts (product.md). func (s *Service) List() ([]store.Domain, error) { return s.store.ListDomains() } @@ -57,7 +60,7 @@ func (s *Service) Get(id int64) (store.Domain, error) { } // Add registers a new sending domain: it records the row, ensures a DKIM key -// exists on disk, and regenerates + reloads the OpenDKIM tables (spec 7.2.3). +// exists on disk, and regenerates + reloads the OpenDKIM tables (product.md). // name must already be normalised and validated by the caller. A duplicate // returns store.ErrDomainExists. // @@ -91,7 +94,7 @@ func (s *Service) rollbackAdd(id int64) { _ = s.store.DeleteDomain(id) } -// Delete removes a domain and everything bound to it (spec 7.2.4, 6.5). The +// Delete removes a domain and everything bound to it (product.md). The // order matters: the applications' SASL accounts are cleared first, while their // logins are still in the registry; then the registry rows (applications and // their addresses) go via the DB cascade; then the OpenDKIM tables and the @@ -107,8 +110,9 @@ func (s *Service) Delete(id int64) error { return fmt.Errorf("clear SASL accounts for %s: %w", d.Name, err) } // Drop the domain's own level-2 limit and those of its applications while the - // application rows still exist (the cleanup query joins them). rate_limits has - // no cascade of its own (ref_id is a plain integer, spec 7.4/9). + // application rows still exist (the cleanup query joins them). rate_limits + // has no cascade of its own (ref_id is a plain integer, README § Rate + // limiting; architecture.md § Persistence). if err := s.store.DeleteRateLimitsForDomain(id); err != nil { return fmt.Errorf("clear rate limits for %s: %w", d.Name, err) } @@ -129,19 +133,19 @@ func (s *Service) Delete(id int64) error { return nil } -// DKIMRecord returns the DNS TXT record to publish for a domain (spec 7.2.10). +// DKIMRecord returns the DNS TXT record to publish for a domain (product.md). func (s *Service) DKIMRecord(d store.Domain) (DKIMRecord, error) { return s.odk.Record(d.Name, d.DKIMSelector) } -// RateLimit returns the domain-level differentiated rate limit (spec 7.4), and -// whether one is configured, for the domain's edit form. +// RateLimit returns the domain-level differentiated rate limit (README § Rate +// limiting), and whether one is configured, for the domain's edit form. func (s *Service) RateLimit(domainID int64) (store.RateLimit, bool, error) { return s.store.GetRateLimit(store.RateLimitScopeDomain, domainID) } // SaveRateLimit stores the domain-level rate limit. The caller has validated the -// IPs and numbers (spec 7.6.2); the milter reads the row live, so no reload is +// IPs and numbers (security.md); the milter reads the row live, so no reload is // needed. func (s *Service) SaveRateLimit(domainID int64, ips []string, maxMessages, windowSeconds int) error { return s.store.SetRateLimit(store.RateLimit{ @@ -154,14 +158,15 @@ func (s *Service) SaveRateLimit(domainID int64, ips []string, maxMessages, windo } // ClearRateLimit removes the domain-level rate limit, falling back to level 1 -// only (spec 7.4). +// only (README § Rate limiting). func (s *Service) ClearRateLimit(domainID int64) error { return s.store.DeleteRateLimit(store.RateLimitScopeDomain, domainID) } -// Resync regenerates the OpenDKIM tables from the registry and reloads OpenDKIM. -// It backs the manual reload button (spec 7.2.12) and doubles as a recovery path -// if the tables ever drift from the database. +// Resync regenerates the OpenDKIM tables from the registry and reloads +// OpenDKIM. It backs the manual reload button (architecture.md § Panel HTTP +// surface) and doubles as a recovery path if the tables ever drift from the +// database. func (s *Service) Resync() error { return s.resync() } diff --git a/internal/domain/transfer.go b/internal/domain/transfer.go index 36809c7..300ed55 100644 --- a/internal/domain/transfer.go +++ b/internal/domain/transfer.go @@ -3,19 +3,20 @@ package domain import ( "fmt" - "codeberg.org/mix/selfpost/internal/buildinfo" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/buildinfo" + "github.com/mixeme/selfpost/internal/store" ) -// FormatDomainExport identifies a single-domain export file (spec 7.5.B). +// FormatDomainExport identifies a single-domain export file (architecture.md § +// Persistence). const FormatDomainExport = "selfpost-domain-export" // DomainExport is the serialisable form of one sending domain, for moving it -// between two independently running SelfPost instances (spec 7.5.B). It carries -// the DKIM private key (so the published DNS record stays valid) and each -// application's working password (so credentials transfer without regeneration). -// The file is therefore as sensitive as a full backup and must be handled as a -// secret. +// between two independently running SelfPost instances (architecture.md § +// Persistence). It carries the DKIM private key (so the published DNS record +// stays valid) and each application's working password (so credentials +// transfer without regeneration). The file is therefore as sensitive as a full +// backup and must be handled as a secret. type DomainExport struct { Format string `json:"format"` Version string `json:"version"` @@ -35,8 +36,8 @@ type AppExport struct { // Export builds the transferable representation of a domain: its DKIM key, its // selector and every application with its address mode and working password -// (spec 7.5.B). The returned struct is marshalled to JSON by the caller and -// offered as a secret download. +// (architecture.md § Persistence). The returned struct is marshalled to JSON +// by the caller and offered as a secret download. func (s *Service) Export(id int64) (DomainExport, error) { d, err := s.store.GetDomain(id) if err != nil { @@ -73,17 +74,18 @@ func (s *Service) Export(id int64) (DomainExport, error) { return exp, nil } -// Import re-creates a domain from an export file on this instance (spec 7.5.B): -// it stores the imported DKIM key (so the published DNS record needs no change), -// registers the domain and rebuilds the OpenDKIM tables, then re-creates each -// application with its working password and rebuilds the Postfix sender map. +// Import re-creates a domain from an export file on this instance +// (architecture.md § Persistence): it stores the imported DKIM key (so the +// published DNS record needs no change), registers the domain and rebuilds the +// OpenDKIM tables, then re-creates each application with its working password +// and rebuilds the Postfix sender map. // -// exp.Domain must already be normalised and validated by the caller (spec -// 7.6.2); the selector is checked for config-injection safety here. A domain or -// login that already exists is rejected (store.ErrDomainExists / -// store.ErrLoginExists) rather than merged. If any step fails, everything the -// import created is rolled back, so a partial import never leaves the instance -// in an inconsistent state. +// exp.Domain must already be normalised and validated by the caller +// (security.md); the selector is checked for +// config-injection safety here. A domain or login that already exists is +// rejected (store.ErrDomainExists / store.ErrLoginExists) rather than merged. +// If any step fails, everything the import created is rolled back, so a +// partial import never leaves the instance in an inconsistent state. func (s *Service) Import(exp DomainExport) (store.Domain, error) { if exp.Format != FormatDomainExport { return store.Domain{}, fmt.Errorf("not a SelfPost domain export (format %q)", exp.Format) diff --git a/internal/domain/transfer_test.go b/internal/domain/transfer_test.go index 7c3d159..98c8987 100644 --- a/internal/domain/transfer_test.go +++ b/internal/domain/transfer_test.go @@ -5,7 +5,7 @@ import ( "fmt" "testing" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/store" ) // fakeApps stands in for *app.Service in the domain-transfer tests: it records diff --git a/internal/health/cert.go b/internal/health/cert.go index f529af5..221253a 100644 --- a/internal/health/cert.go +++ b/internal/health/cert.go @@ -14,8 +14,8 @@ import ( const certWarnDays = 14 // Certificate is the state of the TLS certificate Postfix serves on 465/587 -// (spec 8: TLS_CERT_FILE). The panel only reads it — the file is supplied by -// the reverse proxy through a read-only mount. +// (README § Environment variables: TLS_CERT_FILE). The panel only reads it — +// the file is supplied by the reverse proxy through a read-only mount. type Certificate struct { Path string Subject string diff --git a/internal/health/health_test.go b/internal/health/health_test.go index 37994ee..9d9b902 100644 --- a/internal/health/health_test.go +++ b/internal/health/health_test.go @@ -163,3 +163,28 @@ func writeCert(t *testing.T, path, cn string, validFor time.Duration) { t.Fatal(err) } } + +func TestLivenessFromParsedProcesses(t *testing.T) { + allRunning := `opendkim RUNNING pid 21, uptime 0:04:10 +panel RUNNING pid 22, uptime 0:04:09 +postfix RUNNING pid 23, uptime 0:04:08 +postfix-reload STOPPED Not started +` + procs := parseProcesses(allRunning) + for _, p := range procs { + if mailPathPrograms[p.Name] && p.Status != StatusOK { + t.Fatalf("%s should be ok for liveness, got %q", p.Name, p.Status) + } + } + + postfixDead := `opendkim RUNNING pid 21, uptime 0:04:10 +panel RUNNING pid 22, uptime 0:04:09 +postfix FATAL Exited too quickly +` + procs = parseProcesses(postfixDead) + for _, p := range procs { + if p.Name == "postfix" && p.Status == StatusOK { + t.Fatal("postfix FATAL should not grade as OK") + } + } +} diff --git a/internal/health/liveness.go b/internal/health/liveness.go new file mode 100644 index 0000000..bbb1b6d --- /dev/null +++ b/internal/health/liveness.go @@ -0,0 +1,46 @@ +package health + +import ( + "fmt" + "strings" +) + +// mailPathPrograms are the supervised processes whose absence means the +// container should not report healthy to orchestrators. +var mailPathPrograms = map[string]bool{ + "opendkim": true, + "panel": true, + "postfix": true, +} + +// Liveness reports whether the mail path is healthy enough for container +// probes. It requires opendkim, panel, and postfix to be RUNNING under +// supervisord. +func Liveness() error { + procs, err := Processes() + if err != nil { + return err + } + + seen := make(map[string]Status, len(mailPathPrograms)) + for _, p := range procs { + if mailPathPrograms[p.Name] { + seen[p.Name] = p.Status + } + } + + var unhealthy []string + for name := range mailPathPrograms { + switch seen[name] { + case StatusOK: + case StatusUnknown: + unhealthy = append(unhealthy, name+": missing") + default: + unhealthy = append(unhealthy, fmt.Sprintf("%s: %s", name, seen[name])) + } + } + if len(unhealthy) > 0 { + return fmt.Errorf("unhealthy: %s", strings.Join(unhealthy, ", ")) + } + return nil +} diff --git a/internal/health/processes.go b/internal/health/processes.go index bc8efd0..3bb4e74 100644 --- a/internal/health/processes.go +++ b/internal/health/processes.go @@ -25,11 +25,12 @@ type Process struct { Status Status } -// Processes returns the state of every supervised program (spec 4's three -// processes plus the reload/cert/logrotate helpers). +// Processes returns the state of every supervised program (architecture.md § +// Image and processes — the three processes plus the reload/cert/logrotate +// helpers). // // The command takes fixed arguments and no user input, so it never goes through -// a shell (spec 7.6.3). `supervisorctl status` deliberately exits non-zero when +// a shell (security.md). `supervisorctl status` deliberately exits non-zero when // some program is not running, so the output is parsed first and the exit status // only matters when nothing could be parsed from it. func Processes() ([]Process, error) { diff --git a/internal/logtail/logtail.go b/internal/logtail/logtail.go index 70c2984..77590e8 100644 --- a/internal/logtail/logtail.go +++ b/internal/logtail/logtail.go @@ -1,9 +1,10 @@ // Package logtail follows Postfix's mail.log and reconciles the send-log -// delivery statuses the journal-milter could not know at receive time (spec -// 7.3). A milter row starts life as "queued"; Postfix only decides sent / -// deferred / bounced later, per recipient, and reports it in mail.log. This -// package parses those lines by queue-id + recipient and advances the matching -// rows, and prunes rows past the retention window. +// delivery statuses the journal-milter could not know at receive time +// (architecture.md § Persistence). A milter row starts life as "queued"; +// Postfix only decides sent / deferred / bounced later, per recipient, and +// reports it in mail.log. This package parses those lines by queue-id + +// recipient and advances the matching rows, and prunes rows past the retention +// window. package logtail import ( @@ -17,14 +18,17 @@ import ( "strings" "time" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/store" ) -// StatusStore is the slice of the store the log-tailer needs. *store.Store -// satisfies it. +// StatusStore is the slice of the store the log-tailer needs: advancing +// delivery statuses, pruning the retention window, and remembering how far into +// mail.log it has read. *store.Store satisfies it. type StatusStore interface { UpdateStatus(queueID, recipient, status string) (int64, error) DeleteSendLogBefore(cutoff time.Time) (int64, error) + LogtailState(path string) (store.LogtailState, bool, error) + SaveLogtailState(path string, st store.LogtailState) error } // pollInterval is how often the tail loop checks for new bytes / rotation. It @@ -36,7 +40,7 @@ const ( // startup). The window itself is configurable; the cadence need not be. retentionInterval = 6 * time.Hour // defaultRetentionDays applies when the configured value is unset/invalid - // (spec 7.3). + // (README § Environment variables: SEND_LOG_RETENTION_DAYS). defaultRetentionDays = 90 ) @@ -47,7 +51,13 @@ const ( // // The ": to=, …, status=" shape is specific to the // delivery agents; qmgr/smtpd/cleanup lines do not match. -var deliveryRe = regexp.MustCompile(`\b([0-9A-Za-z]+): to=<([^>]*)>,.*\bstatus=(\w+)`) +// +// The run before status= is lazy on purpose. Postfix appends the remote +// server's reply verbatim, so a greedy match would take the *last* status= on +// the line — and that one can come from the reply text, which the far end +// controls. A bounce whose reply quoted "status=sent" would then be filed as a +// success. The real field is always the first one after to=<…>. +var deliveryRe = regexp.MustCompile(`\b([0-9A-Za-z]+): to=<([^>]*)>,.*?\bstatus=(\w+)`) // parseDelivery extracts (queue-id, recipient, status) from a mail.log line. // ok is false for lines that are not recognised delivery results. @@ -73,12 +83,13 @@ func parseDelivery(line string) (queueID, recipient, status string, ok bool) { } // Run follows path and updates send-log statuses until ctx is cancelled, while -// a background sweep prunes rows older than retentionDays. It returns nil on a -// clean shutdown. +// a background sweep prunes rows older than retentionDays. Reading resumes at +// the offset the previous run persisted, so a restart parses the delivery lines +// written while the panel was down. It returns nil on a clean shutdown. func Run(ctx context.Context, path string, st StatusStore, retentionDays int) error { go retentionLoop(ctx, st, retentionDays) - return follow(ctx, path, func(line string) { + return follow(ctx, path, &tracker{st: st, path: path}, func(line string) { queueID, recipient, status, ok := parseDelivery(line) if !ok { return @@ -120,10 +131,10 @@ func retentionLoop(ctx context.Context, st StatusStore, retentionDays int) { } // TailLines returns up to n of the most recent lines from path, for the -// panel's mail.log monitoring view (spec 7.2.13). It is a one-shot, -// point-in-time read on request — unrelated to the background follow loop -// above — that reads backwards in chunks so it stays cheap against a -// multi-megabyte log rather than reading the whole file every poll. +// panel's mail.log monitoring view (architecture.md § Panel HTTP surface). It +// is a one-shot, point-in-time read on request — unrelated to the background +// follow loop above — that reads backwards in chunks so it stays cheap against +// a multi-megabyte log rather than reading the whole file every poll. func TailLines(path string, n int) ([]string, error) { f, err := os.Open(path) if err != nil { @@ -166,10 +177,11 @@ func TailLines(path string, n int) ([]string, error) { } // follow tails path line by line, calling handle for each complete line, until -// ctx is cancelled. It starts at end-of-file (so a restart does not reprocess -// history) and reopens the file when it is rotated (inode change from +// ctx is cancelled. Where it starts is tr's decision (a persisted offset, the +// start of a file that changed while the panel was down, or end-of-file on a +// first ever run); it reopens the file when it is rotated (inode change from // logrotate's create, or truncation from copytruncate) so nothing is missed. -func follow(ctx context.Context, path string, handle func(string)) error { +func follow(ctx context.Context, path string, tr *tracker, handle func(string)) error { var ( f *os.File r *bufio.Reader @@ -199,7 +211,7 @@ func follow(ctx context.Context, path string, handle func(string)) error { // The container may start before Postfix has created mail.log; wait for it. for { - if err := openAt(0, io.SeekEnd); err == nil { + if err := openAt(0, io.SeekStart); err == nil { break } select { @@ -208,6 +220,10 @@ func follow(ctx context.Context, path string, handle func(string)) error { case <-time.After(pollInterval): } } + if _, err := f.Seek(tr.resume(f), io.SeekStart); err != nil { + log.Printf("log-tailer: seek %s: %v", path, err) + } + r.Reset(f) // the reader buffered from the pre-seek position defer func() { if f != nil { f.Close() @@ -231,11 +247,20 @@ func follow(ctx context.Context, path string, handle func(string)) error { } } + // read returns how many bytes of the open file have actually been consumed: + // the descriptor position less the partial line bufio handed back at EOF, + // which is re-read (and completed) on the next drain or the next start. + read := func() int64 { + pos, _ := f.Seek(0, io.SeekCurrent) + return pos - int64(len(pending)) + } + ticker := time.NewTicker(pollInterval) defer ticker.Stop() for { select { case <-ctx.Done(): + tr.record(f, read(), true) // shutdown: the next start resumes here return nil case <-ticker.C: drain() @@ -252,8 +277,12 @@ func follow(ctx context.Context, path string, handle func(string)) error { drain() if err := openAt(0, io.SeekStart); err != nil { log.Printf("log-tailer: reopen %s: %v", path, err) + continue } + tr.adopt(f) + continue } + tr.record(f, read(), false) } } } diff --git a/internal/logtail/logtail_test.go b/internal/logtail/logtail_test.go index 6abaf1f..5f6554b 100644 --- a/internal/logtail/logtail_test.go +++ b/internal/logtail/logtail_test.go @@ -4,11 +4,12 @@ import ( "context" "os" "path/filepath" + "strings" "sync" "testing" "time" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/store" ) func TestParseDelivery(t *testing.T) { @@ -60,6 +61,59 @@ func TestParseDelivery(t *testing.T) { line: "host postfix/smtpd[10]: 41E862C00D9E: client=unknown[203.0.113.7]", wantOK: false, }, + { + // The remote server's reply is quoted verbatim at the end of the + // line and is entirely attacker-influenced text. A "status=" that + // appears in there must not win over the real field, or a bounce + // would be filed as a success. + name: "status= quoted in the remote reply does not win", + line: "host postfix/smtp[26]: 9F1A2C00D9E: to=, relay=mx.example.net[203.0.113.9]:25, dsn=5.1.1, status=bounced (host mx.example.net said: 550 5.1.1 unknown status=sent (in reply to RCPT TO command))", + wantOK: true, + queueID: "9F1A2C00D9E", + recipient: "a@example.net", + status: store.StatusBounced, + }, + { + // Postfix logs the null sender's own delivery (double bounce) with + // an empty recipient. It parses, and the empty recipient simply + // matches no send-log row — the panel only ever records mail it + // accepted from an authenticated client. + name: "null recipient parses with an empty address", + line: "host postfix/smtp[26]: A1B2C3: to=<>, relay=none, delay=0.1, dsn=2.0.0, status=sent (250 OK)", + wantOK: true, + queueID: "A1B2C3", + recipient: "", + status: store.StatusSent, + }, + { + // An alias/virtual expansion carries orig_to= as well; the address + // the message was actually delivered to is the one in to=. + name: "orig_to is ignored in favour of to", + line: "host postfix/lmtp[26]: 4Xk9tS1abcz: to=, orig_to=, relay=x, dsn=2.0.0, status=sent (ok)", + wantOK: true, + queueID: "4Xk9tS1abcz", + recipient: "real@example.net", + status: store.StatusSent, + }, + { + // Postfix's own delivery agents write these two, but neither is a + // final result we model: "deliverable" comes from address + // verification probes, and anything unrecognised is dropped rather + // than guessed at, leaving the row in its previous state. + name: "unknown status word is not a delivery result", + line: "host postfix/smtp[26]: BEEF01: to=, relay=x, status=deliverable (ok)", + wantOK: false, + }, + { + name: "status matching is case-sensitive, as Postfix writes it", + line: "host postfix/smtp[26]: BEEF02: to=, relay=x, dsn=4.0.0, status=Deferred (connect timed out)", + wantOK: false, + }, + { + name: "cleanup message-id line ignored", + line: "host postfix/cleanup[12]: BEEF03: message-id=", + wantOK: false, + }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -77,10 +131,16 @@ func TestParseDelivery(t *testing.T) { } } -// captureStore records UpdateStatus calls for the follow integration test. +// captureStore records UpdateStatus calls for the follow integration test and +// keeps the persisted read offset in memory, so a "restart" in a test is a +// second Run against the same captureStore. type captureStore struct { mu sync.Mutex calls []string + + state store.LogtailState + haveState bool + stateErr error } func (c *captureStore) UpdateStatus(queueID, recipient, status string) (int64, error) { @@ -92,12 +152,37 @@ func (c *captureStore) UpdateStatus(queueID, recipient, status string) (int64, e func (c *captureStore) DeleteSendLogBefore(time.Time) (int64, error) { return 0, nil } +func (c *captureStore) LogtailState(string) (store.LogtailState, bool, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.stateErr != nil { + return store.LogtailState{}, false, c.stateErr + } + return c.state, c.haveState, nil +} + +func (c *captureStore) SaveLogtailState(_ string, st store.LogtailState) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.stateErr != nil { + return c.stateErr + } + c.state, c.haveState = st, true + return nil +} + func (c *captureStore) snapshot() []string { c.mu.Lock() defer c.mu.Unlock() return append([]string(nil), c.calls...) } +func (c *captureStore) reset() { + c.mu.Lock() + defer c.mu.Unlock() + c.calls = nil +} + // TestFollowTailsAndRotates writes delivery lines to a log file, then rotates // it (rename + fresh create, as logrotate does) and writes more, asserting the // tailer picks up lines from both the original and rotated file. @@ -142,6 +227,80 @@ func TestFollowTailsAndRotates(t *testing.T) { } } +// TestFollowResumesAfterRestart covers the persisted read offset: a restart +// must parse the delivery lines written while the tailer was down (rows that +// would otherwise stay "queued" forever), without re-parsing what it already +// read, and must fall back to reading the whole file when the log was rotated +// or recreated in the meantime. +func TestFollowResumesAfterRestart(t *testing.T) { + old := pollInterval + pollInterval = 10 * time.Millisecond + t.Cleanup(func() { pollInterval = old }) + + dir := t.TempDir() + path := filepath.Join(dir, "mail.log") + // A head longer than fingerprintSize, so the file stays identifiable across + // the restart; the lines themselves predate the first start and are ignored. + seed := strings.Repeat("host postfix/qmgr[1]: seed line, not a delivery\n", 20) + if err := os.WriteFile(path, []byte(seed), 0o644); err != nil { + t.Fatalf("seed log: %v", err) + } + + cs := &captureStore{} + stop := startRun(t, path, cs) + appendLine(t, path, "host postfix/smtp[1]: Q1: to=, dsn=2.0.0, status=sent (ok)") + waitFor(t, func() bool { return contains(cs.snapshot(), "Q1|a@example.net|sent") }) + stop() // persists the offset past Q1 + + // Down: Postfix keeps delivering. + appendLine(t, path, "host postfix/smtp[1]: Q2: to=, dsn=2.0.0, status=sent (ok)") + + cs.reset() + stop = startRun(t, path, cs) + waitFor(t, func() bool { return contains(cs.snapshot(), "Q2|b@example.net|sent") }) + if contains(cs.snapshot(), "Q1|a@example.net|sent") { + t.Fatal("resumed run re-parsed Q1: offset was not honoured") + } + stop() + + // Down again, and this time the log is replaced (logrotate + fresh create). + // The stored offset belongs to a file that no longer exists, so the new one + // must be read from the start. + if err := os.WriteFile(path, []byte(strings.Repeat("host postfix/qmgr[1]: fresh log after rotation\n", 20)+ + "host postfix/smtp[1]: Q3: to=, dsn=5.1.1, status=bounced (nope)\n"), 0o644); err != nil { + t.Fatalf("recreate log: %v", err) + } + + cs.reset() + stop = startRun(t, path, cs) + waitFor(t, func() bool { return contains(cs.snapshot(), "Q3|c@example.net|bounced") }) + stop() +} + +// startRun launches the tailer and returns a function that cancels it and waits +// for a clean return, the way a panel restart bookends a run. +func startRun(t *testing.T, path string, cs *captureStore) func() { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- Run(ctx, path, cs, 90) }() + // follow() opens and seeks on start; give it a moment before the caller + // appends, so the append is not raced by the initial open. + time.Sleep(50 * time.Millisecond) + return func() { + t.Helper() + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("Run: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Run did not return after cancel") + } + } +} + func appendLine(t *testing.T, path, line string) { t.Helper() f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) diff --git a/internal/logtail/offset.go b/internal/logtail/offset.go new file mode 100644 index 0000000..0b965cd --- /dev/null +++ b/internal/logtail/offset.go @@ -0,0 +1,136 @@ +package logtail + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "log" + "os" + "time" + + "github.com/mixeme/selfpost/internal/store" +) + +const ( + // fingerprintSize is how many bytes from the head of the log identify it. + // Postfix writes a timestamped line per event, so the first 512 bytes are + // effectively unique per log generation — enough to tell "the file we were + // reading" from "a fresh one created by logrotate while we were down", + // which os.SameFile cannot answer across a restart. + fingerprintSize = 512 + // persistInterval throttles the offset write. Losing up to this much + // progress on a crash only means re-parsing a few lines (UpdateStatus is + // idempotent), which is much cheaper than a database write per poll tick. + persistInterval = 5 * time.Second +) + +// tracker persists the tailer's read position so a restart resumes where the +// previous run stopped instead of jumping to end-of-file — the "send-log rows +// stay queued forever" gap (architecture.md § Log tailer). It is used from the +// follow loop only, so it needs no locking. +type tracker struct { + st StatusStore + path string + + fp string // fingerprint of the file currently open ("" if too short) + saved int64 // last offset written to the store + lastSave time.Time +} + +// resume returns the byte offset the tailer should start reading f at, having +// recorded f's fingerprint for later saves. +// +// The rules, in order: no stored state at all (first ever start) means start at +// the end, so installing the panel does not replay a pre-existing log; a stored +// state whose fingerprint still matches means continue from it, parsing the +// tail written while the panel was down; anything else means the file is not +// the one the offset referred to (rotated, recreated or truncated in the +// meantime), so read it from the start. Re-parsing lines already seen is +// harmless: UpdateStatus writes the same status onto the same row. +func (t *tracker) resume(f *os.File) int64 { + size, err := fileSize(f) + if err != nil { + log.Printf("log-tailer: stat %s: %v (reading from the start)", t.path, err) + return 0 + } + t.fp = fingerprintOf(f) + + prev, ok, err := t.st.LogtailState(t.path) + if err != nil { + log.Printf("log-tailer: read stored offset: %v (starting at end)", err) + return size + } + switch { + case !ok: + t.saved = size + return size + case prev.Fingerprint != "" && prev.Fingerprint == t.fp && prev.Offset <= size: + t.saved = prev.Offset + if prev.Offset < size { + log.Printf("log-tailer: resuming %s at offset %d (%d bytes to catch up)", + t.path, prev.Offset, size-prev.Offset) + } + return prev.Offset + default: + log.Printf("log-tailer: %s changed while the panel was down; reading from the start", t.path) + t.saved = 0 + return 0 + } +} + +// adopt re-fingerprints after the follow loop switched to a rotated-in file and +// persists the fresh start immediately, so a restart right after a rotation +// does not resume at the old file's offset. +func (t *tracker) adopt(f *os.File) { + t.fp = fingerprintOf(f) + t.saved = -1 // force the write below even if the old offset happened to be 0 + t.record(f, 0, true) +} + +// record persists offset, at most once per persistInterval unless force is set +// (rotation and shutdown, where the write must not be skipped). +func (t *tracker) record(f *os.File, offset int64, force bool) { + if offset == t.saved { + return + } + if !force && time.Since(t.lastSave) < persistInterval { + return + } + if t.fp == "" { + // The log was shorter than a fingerprint when we opened it; now that it + // has grown, an identifiable one may be available. + t.fp = fingerprintOf(f) + } + if err := t.st.SaveLogtailState(t.path, store.LogtailState{Fingerprint: t.fp, Offset: offset}); err != nil { + log.Printf("log-tailer: save offset: %v", err) + return + } + t.saved = offset + t.lastSave = time.Now() +} + +// fingerprintOf hashes the head of the file. It returns "" for a file too short +// to identify — the head would still change as Postfix appends, so such a +// fingerprint could not be compared meaningfully on the next start. +func fingerprintOf(f *os.File) string { + buf := make([]byte, fingerprintSize) + n, err := f.ReadAt(buf, 0) + if err != nil && !errors.Is(err, io.EOF) { + log.Printf("log-tailer: fingerprint read: %v", err) + return "" + } + if n < fingerprintSize { + return "" + } + sum := sha256.Sum256(buf) + return hex.EncodeToString(sum[:]) +} + +func fileSize(f *os.File) (int64, error) { + info, err := f.Stat() + if err != nil { + return 0, err + } + return info.Size(), nil +} diff --git a/internal/milter/inflight.go b/internal/milter/inflight.go new file mode 100644 index 0000000..78ca52b --- /dev/null +++ b/internal/milter/inflight.go @@ -0,0 +1,99 @@ +package milter + +import ( + "sync" + "time" +) + +// reservationTTL bounds how long a message may stay reserved. A reservation is +// released at end-of-message or on ABORT, but a client that simply drops the +// connection after MAIL FROM produces neither callback (go-milter has no +// connection-close hook), and a reservation that never expired would count +// against the limit forever — a fail-closed drift this milter must not have. +// The TTL is generously longer than any realistic DATA transfer, so a message +// still being received is never dropped from the count. +const reservationTTL = 10 * time.Minute + +// reservation is one message that passed the level-2 check and has not been +// written to the send log yet. +type reservation struct { + key string + at time.Time +} + +// inflight counts messages that are between the limit check (MAIL FROM) and the +// send-log insert (end-of-message). The stored count alone cannot see them, so +// without this several concurrent SMTP sessions each read the same pre-insert +// count, each conclude they are under the ceiling, and the limit is overshot by +// however many were in flight. Counting reservations closes that window without +// writing placeholder rows the operator would see in the UI. +// +// One instance is shared by every session of the process, hence the mutex. +// Methods tolerate a nil receiver so a session built without one (tests) simply +// behaves as it did before. +type inflight struct { + mu sync.Mutex + m map[string]map[*reservation]struct{} +} + +// count returns how many reservations for key were taken within the limit's +// window (at or after since), pruning any that outlived reservationTTL. +func (f *inflight) count(key string, since time.Time) int64 { + if f == nil { + return 0 + } + f.mu.Lock() + defer f.mu.Unlock() + + set := f.m[key] + cutoff := time.Now().Add(-reservationTTL) + var n int64 + for r := range set { + if r.at.Before(cutoff) { + delete(set, r) + continue + } + if !r.at.Before(since) { + n++ + } + } + if len(set) == 0 { + delete(f.m, key) + } + return n +} + +// reserve claims a slot for key until the message is recorded or released. +func (f *inflight) reserve(key string) *reservation { + if f == nil { + return nil + } + f.mu.Lock() + defer f.mu.Unlock() + + if f.m == nil { + f.m = make(map[string]map[*reservation]struct{}) + } + if f.m[key] == nil { + f.m[key] = make(map[*reservation]struct{}) + } + r := &reservation{key: key, at: time.Now()} + f.m[key][r] = struct{}{} + return r +} + +// release drops a reservation, either because the message reached the send log +// (where the stored count takes over) or because it never will. +func (f *inflight) release(r *reservation) { + if f == nil || r == nil { + return + } + f.mu.Lock() + defer f.mu.Unlock() + + set := f.m[r.key] + delete(set, r) + if len(set) == 0 { + delete(f.m, r.key) + } +} diff --git a/internal/milter/milter.go b/internal/milter/milter.go index cb58a48..6c0807c 100644 --- a/internal/milter/milter.go +++ b/internal/milter/milter.go @@ -1,13 +1,14 @@ // Package milter implements the SelfPost journal-milter: a lightweight milter -// (spec 7.3) attached to Postfix's smtpd_milters alongside OpenDKIM. On the -// receive path it reads the SASL login, From, recipients and Subject of each -// accepted message and records one send-log row per (queue-id, recipient), -// giving the panel a structured, filterable history that raw mail.log cannot. +// (architecture.md § Mail path) attached to Postfix's smtpd_milters alongside +// OpenDKIM. On the receive path it reads the SASL login, From, recipients and +// Subject of each accepted message and records one send-log row per (queue-id, +// recipient), giving the panel a structured, filterable history that raw +// mail.log cannot. // // It is monitoring only: it never rejects, and every callback returns Continue // or Accept so a failure of this milter can never block the relay. Postfix is // configured with default_action=accept for this milter's socket, so even a -// crash or hang fails open (spec 7.3). +// crash or hang fails open (architecture.md § Mail path). package milter import ( @@ -21,13 +22,14 @@ import ( "github.com/emersion/go-milter" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/store" ) // Store is the persistence the milter needs on the receive path: recording -// accepted messages (spec 7.3) and, for level-2 rate limiting (spec 7.4), -// looking up the configured limits and counting recent messages. *store.Store -// satisfies it; tests substitute a fake. +// accepted messages (architecture.md § Mail path) and, for level-2 rate +// limiting (README § Rate limiting), looking up the configured limits and +// counting recent messages. *store.Store satisfies it; tests substitute a +// fake. type Store interface { InsertQueued(e store.SendLogEntry) error InsertRejected(e store.SendLogEntry) error @@ -37,26 +39,31 @@ type Store interface { // session accumulates the fields of one message as the milter callbacks fire. // Milter macros arrive per-stage and do not accumulate, so each value is -// captured at the stage that carries it (spec 7.3 / Phase 0 spike): SASL login -// and From at MAIL, each recipient at RCPT, Subject in the headers, and the -// queue-id at end-of-message. go-milter creates one session per connection; a -// connection may carry several messages, so per-message fields are reset at -// MailFrom (the start of every transaction). +// captured at the stage that carries it (architecture.md § Mail path): SASL +// login and From at MAIL, each recipient at RCPT, Subject in the headers, and +// the queue-id at end-of-message. go-milter creates one session per +// connection; a connection may carry several messages, so per-message fields +// are reset at MailFrom (the start of every transaction). type session struct { milter.NoOpMilter rec Store + // flight is shared by every session of the process; it holds the messages + // that passed the level-2 check but are not in the send log yet. Nil is a + // valid zero value (no in-flight accounting). + flight *inflight clientIP string // captured once per connection - login string - from string - rcpts []string - subject string + login string + from string + rcpts []string + subject string + reserved []*reservation // level-2 slots held by the current message } // Connect captures the client IP, which comes from the addr parameter rather -// than a macro (the {client_addr} macro was empty in the spike). It is the -// rate-limit key for Phase 8; here it is recorded for completeness. +// than a macro (the {client_addr} macro was empty in testing). It is the +// rate-limit key; here it is recorded for completeness. func (s *session) Connect(host, family string, port uint16, addr net.IP, m *milter.Modifier) (milter.Response, error) { if addr != nil { s.clientIP = addr.String() @@ -69,9 +76,10 @@ func (s *session) Connect(host, family string, port uint16, addr net.IP, m *milt // macros). This is also the earliest stage where both the sending domain (from // the sender) and the application (the login) are known, so the level-2 rate // limit is enforced here: over the limit, the message is refused with a 4xx -// tempfail before recipients are even offered (spec 7.4). Enforcement is -// fail-open — see overLimit. +// tempfail before recipients are even offered (README § Rate limiting). +// Enforcement is fail-open — see overLimit. func (s *session) MailFrom(from string, m *milter.Modifier) (milter.Response, error) { + s.releaseReservations() // a previous transaction that ended without EOM/ABORT s.from = cleanAddress(from) s.login = macro(m, "auth_authen") s.rcpts = nil @@ -127,14 +135,26 @@ func decodeSubject(v string) string { // rows are written. We accept (this milter is done) without ever rejecting. func (s *session) Body(m *milter.Modifier) (milter.Response, error) { s.record(macro(m, "i")) + // The rows are in the send log now, so the stored count sees this message + // and its level-2 slots are no longer needed. + s.releaseReservations() return milter.RespAccept, nil } +// Abort ends the current transaction without an end-of-message (client RSET, or +// Postfix rejecting the message for its own reasons). No send-log row will be +// written, so the level-2 slots this message held must go back. +func (s *session) Abort(m *milter.Modifier) error { + s.releaseReservations() + s.rcpts = nil + s.subject = "" + return nil +} + // macro reads a milter macro, tolerating Postfix's convention of wrapping // multi-character macro names in curly braces (e.g. {auth_authen}) while // single-character names (e.g. i) arrive bare. go-milter stores whatever name -// Postfix sends verbatim, so a lookup must try both forms — this is exactly the -// distinction the SASL-less Phase 0 spike could not observe. +// Postfix sends verbatim, so a lookup must try both forms. func macro(m *milter.Modifier, name string) string { if v, ok := m.Macros[name]; ok { return v @@ -143,7 +163,8 @@ func macro(m *milter.Modifier, name string) string { } // record writes one send-log row per recipient. Failures are logged, never -// propagated: journalling must not affect mail acceptance (spec 7.3). +// propagated: journalling must not affect mail acceptance (architecture.md § +// Mail path). func (s *session) record(queueID string) { domain := domainOf(s.from) rcpts := s.rcpts @@ -180,8 +201,9 @@ func cleanAddress(a string) string { } // domainOf returns the lower-cased domain of an email address, or "" if there -// is no domain part. Sender binding (Phase 4) guarantees the From domain equals -// the application's domain, so this is the sending domain (spec 7.3). +// is no domain part. Sender binding guarantees the From domain equals the +// application's domain, so this is the sending domain (architecture.md § Mail +// path). func domainOf(addr string) string { if i := strings.LastIndexByte(addr, '@'); i >= 0 { return strings.ToLower(addr[i+1:]) @@ -192,8 +214,9 @@ func domainOf(addr string) string { // Serve runs the journal-milter on ln until ctx is cancelled. Each connection // gets a fresh session bound to rec. It returns nil on a clean shutdown. func Serve(ctx context.Context, ln net.Listener, rec Store) error { + flight := &inflight{} // shared: the level-2 window spans all connections srv := &milter.Server{ - NewMilter: func() milter.Milter { return &session{rec: rec} }, + NewMilter: func() milter.Milter { return &session{rec: rec, flight: flight} }, Actions: 0, // read-only: we make no message modifications Protocol: milter.OptNoBody, // the journal needs headers/EOM, not the body } diff --git a/internal/milter/milter_test.go b/internal/milter/milter_test.go index 999a43a..76f3cc6 100644 --- a/internal/milter/milter_test.go +++ b/internal/milter/milter_test.go @@ -9,7 +9,7 @@ import ( "github.com/emersion/go-milter" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/store" ) // fakeRecorder captures inserts and can be made to fail, to prove the milter @@ -331,6 +331,130 @@ func TestRateLimitNoIPKeyDoesNotApply(t *testing.T) { } } +// mailFromIn is mailFrom with an explicit shared in-flight registry, so a test +// can play several concurrent SMTP sessions of one process against each other. +func mailFromIn(t *testing.T, rec Store, fl *inflight, ip, from, login string) (*session, milter.Response) { + t.Helper() + s := &session{rec: rec, flight: fl} + if _, err := s.Connect("h", "tcp4", 0, net.ParseIP(ip), mods(nil)); err != nil { + t.Fatalf("Connect: %v", err) + } + resp, err := s.MailFrom(from, mods(map[string]string{"auth_authen": login})) + if err != nil { + t.Fatalf("MailFrom: %v", err) + } + return s, resp +} + +func limitedRecorder(count int64) *fakeRecorder { + return &fakeRecorder{ + limits: map[string]store.RateLimit{ + store.RateLimitScopeDomain + "|example.com": activeLimit(limitIP), + }, + counts: map[string]int64{store.RateLimitScopeDomain + "|example.com": count}, + } +} + +// Messages between MAIL FROM and end-of-message are not in the send log yet, so +// counting the stored rows alone lets concurrent sessions each pass the same +// check and overshoot the ceiling. The last free slot may only be taken once. +func TestRateLimitCountsInFlightMessages(t *testing.T) { + rec := limitedRecorder(4) // one below the ceiling of 5 + fl := &inflight{} + + if _, resp := mailFromIn(t, rec, fl, limitIP, "a@example.com", "app1"); resp != milter.RespContinue { + t.Fatalf("first message = %v, want Continue (4/5 stored)", resp) + } + // Same window, nothing written yet: the first message holds the fifth slot. + if _, resp := mailFromIn(t, rec, fl, limitIP, "b@example.com", "app1"); resp != milter.RespTempFail { + t.Fatalf("concurrent message = %v, want TempFail (would overshoot)", resp) + } + if len(rec.rejected) != 1 { + t.Fatalf("want one rejected send-log row, got %+v", rec.rejected) + } +} + +// Once the message is recorded the stored count sees it, so its reservation +// must be given back — otherwise it would be counted twice and the ceiling +// would drift closed. +func TestReservationReleasedAtEndOfMessage(t *testing.T) { + rec := limitedRecorder(4) + fl := &inflight{} + + s, resp := mailFromIn(t, rec, fl, limitIP, "a@example.com", "app1") + if resp != milter.RespContinue { + t.Fatalf("first message = %v, want Continue", resp) + } + if _, err := s.Body(mods(map[string]string{"i": "Q1"})); err != nil { + t.Fatalf("Body: %v", err) + } + if n := fl.count(store.RateLimitScopeDomain+"|example.com", time.Now().Add(-time.Hour)); n != 0 { + t.Fatalf("in-flight count after EOM = %d, want 0", n) + } +} + +// A transaction the client abandons (RSET, or a Postfix-side rejection) never +// reaches the send log, so its slot must not stay claimed. +func TestReservationReleasedOnAbort(t *testing.T) { + rec := limitedRecorder(4) + fl := &inflight{} + + s, resp := mailFromIn(t, rec, fl, limitIP, "a@example.com", "app1") + if resp != milter.RespContinue { + t.Fatalf("first message = %v, want Continue", resp) + } + if err := s.Abort(mods(nil)); err != nil { + t.Fatalf("Abort: %v", err) + } + if _, resp := mailFromIn(t, rec, fl, limitIP, "b@example.com", "app1"); resp != milter.RespContinue { + t.Fatalf("after abort = %v, want Continue (slot released)", resp) + } +} + +// A refused message must not leave the slots it claimed for the limits checked +// before the one that tripped, or every refusal would tighten the ceiling. +func TestRefusalReleasesEarlierReservation(t *testing.T) { + rec := &fakeRecorder{ + limits: map[string]store.RateLimit{ + store.RateLimitScopeDomain + "|example.com": activeLimit(limitIP), + store.RateLimitScopeApp + "|app1": activeLimit(limitIP), + }, + counts: map[string]int64{ + store.RateLimitScopeDomain + "|example.com": 0, // domain: plenty of room + store.RateLimitScopeApp + "|app1": 5, // app: at the ceiling + }, + } + fl := &inflight{} + if _, resp := mailFromIn(t, rec, fl, limitIP, "a@example.com", "app1"); resp != milter.RespTempFail { + t.Fatalf("app over limit = %v, want TempFail", resp) + } + if n := fl.count(store.RateLimitScopeDomain+"|example.com", time.Now().Add(-time.Hour)); n != 0 { + t.Fatalf("domain reservation left behind after refusal: %d", n) + } +} + +// The in-flight count only covers the limit's own window: a reservation older +// than it (a session stuck mid-DATA for longer than the window) must not be +// counted against a window it no longer belongs to. +func TestInflightIgnoresReservationsOutsideWindow(t *testing.T) { + fl := &inflight{} + r := fl.reserve("domain|example.com") + r.at = time.Now().Add(-time.Minute) + + if n := fl.count("domain|example.com", time.Now().Add(-time.Hour)); n != 1 { + t.Fatalf("count inside window = %d, want 1", n) + } + if n := fl.count("domain|example.com", time.Now().Add(-time.Second)); n != 0 { + t.Fatalf("count outside window = %d, want 0", n) + } + // Past the TTL the reservation is dropped even for a wide window, so a + // client that vanished after MAIL FROM cannot hold a slot forever. + r.at = time.Now().Add(-2 * reservationTTL) + if n := fl.count("domain|example.com", time.Now().Add(-3*reservationTTL)); n != 0 { + t.Fatalf("expired reservation still counted: %d", n) + } +} + func TestDomainOf(t *testing.T) { cases := map[string]string{ "user@Example.COM": "example.com", diff --git a/internal/milter/ratelimit.go b/internal/milter/ratelimit.go index a2ad2fd..418d4b8 100644 --- a/internal/milter/ratelimit.go +++ b/internal/milter/ratelimit.go @@ -4,19 +4,24 @@ import ( "log" "time" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/store" ) // overLimit reports whether the message currently being received should be -// refused under a level-2 differentiated limit (spec 7.4). It checks the -// domain-level and application-level limits in turn; either being exceeded is -// enough to refuse. +// refused under a level-2 differentiated limit (README § Rate limiting). It +// checks the domain-level and application-level limits in turn; either being +// exceeded is enough to refuse. // // It is deliberately fail-open: any store error, or the absence of a usable -// limit, is treated as "not over limit" so a malfunction of the level-2 limiter -// can never block mail — Postfix's level-1 anvil limit (spec 5) remains the -// backstop, and it does not depend on this milter at all. Only a clean count at -// or above a configured ceiling returns true. +// limit, is treated as "not over limit" so a malfunction of the level-2 +// limiter can never block mail — Postfix's level-1 anvil limit +// (architecture.md § Mail path) remains the backstop, and it does not depend +// on this milter at all. Only a clean count at or above a configured ceiling +// returns true. +// +// A message that passes reserves a slot per applicable limit, released once it +// reaches the send log (or is abandoned) — see inflight for why the stored +// count alone is not enough. func (s *session) overLimit() bool { if s.clientIP == "" { return false // no client IP to key on; level-2 does not apply @@ -25,6 +30,7 @@ func (s *session) overLimit() bool { {store.RateLimitScopeDomain, domainOf(s.from)}, {store.RateLimitScopeApp, s.login}, } + var taken []*reservation for _, c := range checks { if c.ref == "" { continue @@ -45,19 +51,38 @@ func (s *session) overLimit() bool { log.Printf("journal-milter: rate-limit count %s %q: %v (fail-open)", c.scope, c.ref, err) continue } + key := c.scope + "|" + c.ref + n += s.flight.count(key, since) if n >= int64(rl.MaxMessages) { log.Printf("journal-milter: %s %q over limit: %d/%d in %ds from %s — refusing 4xx", c.scope, c.ref, n, rl.MaxMessages, rl.WindowSeconds, s.clientIP) + // The message is refused, so the slots claimed for the limits + // checked before this one must not stay claimed. + for _, r := range taken { + s.flight.release(r) + } return true } + taken = append(taken, s.flight.reserve(key)) } + s.reserved = append(s.reserved, taken...) return false } -// recordRejected writes a send-log row for a message refused by a level-2 limit -// (spec 7.4, "опционально фиксирует ... для видимости в UI"), so the rejection -// shows up in the monitoring screen. Only MAIL-stage fields are known; the write -// is best-effort and never affects the response. +// releaseReservations gives back every slot this message holds. It runs once +// the message is in the send log (where the stored count sees it), and whenever +// the transaction ends without getting there. +func (s *session) releaseReservations() { + for _, r := range s.reserved { + s.flight.release(r) + } + s.reserved = nil +} + +// recordRejected writes a send-log row for a message refused by a level-2 +// limit (README § Rate limiting — refusals are recorded too), so the rejection +// shows up in the monitoring screen. Only MAIL-stage fields are known; the +// write is best-effort and never affects the response. func (s *session) recordRejected() { err := s.rec.InsertRejected(store.SendLogEntry{ Domain: domainOf(s.from), diff --git a/internal/postfix/postfix.go b/internal/postfix/postfix.go index 2db9e30..996952b 100644 --- a/internal/postfix/postfix.go +++ b/internal/postfix/postfix.go @@ -1,8 +1,8 @@ // Package postfix owns the Postfix configuration files the panel edits at -// runtime and the privileged reload that applies them (spec 5.1, 7.6.3-4). In -// Phase 4 that is the smtpd_sender_login_maps table binding each application's -// SASL login to the sender addresses it may use; the full relay configuration -// lands in Phase 5. +// runtime and the privileged reload that applies them (architecture.md § Mail +// path, security.md): the smtpd_sender_login_maps table binding each +// application's SASL login to the sender addresses it may use, plus the relay +// configuration in main.cf. package postfix import ( @@ -33,28 +33,30 @@ func New(dir string) *Postfix { } } -// SenderLoginMapsPath is the absolute path of the generated map, so the Postfix -// main.cf written in Phase 5 can point smtpd_sender_login_maps at it. +// SenderLoginMapsPath is the absolute path of the generated map, so main.cf +// can point smtpd_sender_login_maps at it. func (p *Postfix) SenderLoginMapsPath() string { return p.senderLoginMapsPath } -// Binding is one sender-address → login pair (spec 5.1). Address is either a -// domain wildcard "@example.com" or a specific address "alerts@example.com". +// Binding is one sender-address → login pair (architecture.md § Mail path). +// Address is either a domain wildcard "@example.com" or a specific address +// "alerts@example.com". type Binding struct { Address string Login string } // RebuildSenderLoginMaps regenerates the sender_login_maps file from the full -// set of bindings and reloads Postfix (spec 5.1). Full regeneration (rather than -// incremental edits) keeps the file a pure function of the registry, so add, -// edit and delete share one idempotent path. The file is written atomically -// before the reload. +// set of bindings and reloads Postfix (architecture.md § Mail path). Full +// regeneration (rather than incremental edits) keeps the file a pure function +// of the registry, so add, edit and delete share one idempotent path. The file +// is written atomically before the reload. // // Several applications may be authorised for the same address (many-to-one, -// spec 5.1 §4) — their logins are merged onto a single line as a comma-separated -// list, which is how Postfix expects multiple owners of one sender. +// architecture.md § Mail path) — their logins are merged onto a single line as +// a comma-separated list, which is how Postfix expects multiple owners of one +// sender. func (p *Postfix) RebuildSenderLoginMaps(bindings []Binding) error { content, err := renderSenderLoginMaps(bindings) if err != nil { @@ -67,7 +69,8 @@ func (p *Postfix) RebuildSenderLoginMaps(bindings []Binding) error { } // Reload asks Postfix to re-read its configuration without regenerating any -// file. It backs the panel's manual reload button (spec 7.2.12). +// file. It backs the panel's manual reload button (architecture.md § Panel +// HTTP surface). func (p *Postfix) Reload() error { return p.reload() } @@ -75,7 +78,7 @@ func (p *Postfix) Reload() error { // renderSenderLoginMaps builds the sender_login_maps file contents. Keys are // sorted for deterministic output and the logins under each key are sorted and // de-duplicated. Every address and login is re-checked for injection safety -// before being written (spec 7.6.4) — upstream validation already guarantees +// before being written (security.md) — upstream validation already guarantees // this, but the writer refuses to emit anything unsafe as a hard backstop. func renderSenderLoginMaps(bindings []Binding) ([]byte, error) { byAddr := make(map[string][]string) @@ -95,8 +98,8 @@ func renderSenderLoginMaps(bindings []Binding) ([]byte, error) { for _, addr := range order { logins := byAddr[addr] sort.Strings(logins) - // texthash format: . A comma-separated value - // lists every login permitted to use this sender (spec 5.1 §4). + // texthash format: . A comma-separated value lists + // every login permitted to use this sender (architecture.md § Mail path). fmt.Fprintf(&sb, "%s %s\n", addr, strings.Join(logins, ",")) } return []byte(sb.String()), nil @@ -114,9 +117,9 @@ func appendUnique(list []string, v string) []string { // assertMapSafe rejects any address/login value that could break out of a single // map line or inject a directive. Addresses are validated to a strict whitelist // (letters, digits, '@', '.', '-', '_', '+') and logins to an even stricter one -// upstream (spec 7.6.2); this is defence in depth against a validation gap ever +// upstream (security.md); this is defence in depth against a validation gap ever // letting whitespace, a newline or a comma (the value separator) through into -// the file (spec 7.6.4). +// the file (security.md). func assertMapSafe(address, login string) error { if address == "" || login == "" { return fmt.Errorf("postfix: empty address or login") @@ -131,17 +134,17 @@ func assertMapSafe(address, login string) error { } // reloadViaSupervisor asks supervisord (PID 1, running as root) to run the -// one-shot `postfix-reload` program, which executes the canonical -// `postfix reload` and re-reads main.cf/master.cf and the lookup tables they -// reference. The panel runs unprivileged: it cannot run `postfix reload` itself, -// and it cannot signal the Postfix master directly because `postfix start-fg` -// forks a separate master whose PID supervisord does not track (a SIGHUP to the +// one-shot `postfix-reload` program, which executes the canonical `postfix +// reload` and re-reads main.cf/master.cf and the lookup tables they reference. +// The panel runs unprivileged: it cannot run `postfix reload` itself, and it +// cannot signal the Postfix master directly because `postfix start-fg` forks a +// separate master whose PID supervisord does not track (a SIGHUP to the // supervised process would never reach it). Going through supervisord's // group-accessible control socket runs the reload as root without any panel -// privilege (spec 5.2, 7.2.12, 7.6.3, 7.6.8). +// privilege (architecture.md § Mail path, security.md). // // Arguments are fixed literals — no user input is interpolated into the command, -// and it never goes through a shell (spec 7.6.3). +// and it never goes through a shell (security.md). func reloadViaSupervisor() error { cmd := exec.Command("supervisorctl", "-c", "/etc/supervisor/supervisord.conf", diff --git a/internal/postfix/postfix_test.go b/internal/postfix/postfix_test.go index 4b70efc..04e5c88 100644 --- a/internal/postfix/postfix_test.go +++ b/internal/postfix/postfix_test.go @@ -7,7 +7,7 @@ import ( func TestRenderSenderLoginMaps(t *testing.T) { // Deliberately unsorted, with two logins sharing one wildcard key - // (many-to-one, spec 5.1 §4) to exercise merge + sort. + // (many-to-one, architecture.md § Mail path) to exercise merge + sort. bindings := []Binding{ {"@zeta.example", "z1"}, {"alerts@alpha.example", "a-listed"}, diff --git a/internal/postfix/queue.go b/internal/postfix/queue.go index d0df5a0..039b41a 100644 --- a/internal/postfix/queue.go +++ b/internal/postfix/queue.go @@ -6,12 +6,12 @@ import ( "strings" ) -// Queue returns Postfix's own human-readable mail-queue listing (spec 7.2.11): -// active, deferred and held messages, exactly as an administrator would see -// via the CLI. The command takes a single fixed flag and no user input, so it -// never goes through a shell (spec 7.6.3). The panel is responsible for -// escaping the output before display (spec 7.6.7); this function returns it -// as-is. +// Queue returns Postfix's own human-readable mail-queue listing +// (architecture.md § Panel HTTP surface): active, deferred and held messages, +// exactly as an administrator would see via the CLI. The command takes a +// single fixed flag and no user input, so it never goes through a shell +// (security.md). The panel is responsible for escaping the output before +// display (security.md); this function returns it as-is. func Queue() (string, error) { cmd := exec.Command("postqueue", "-p") out, err := cmd.CombinedOutput() diff --git a/internal/postfix/write.go b/internal/postfix/write.go index 6cc7ada..b79da1f 100644 --- a/internal/postfix/write.go +++ b/internal/postfix/write.go @@ -8,7 +8,7 @@ import ( // writeFileAtomic writes data to path via a temp file in the same directory // followed by a rename, so a concurrent Postfix reload only ever sees the -// complete old or new map, never a partial write (spec 7.6.4). It mirrors the +// complete old or new map, never a partial write (security.md). It mirrors the // same primitive used for the OpenDKIM tables. func writeFileAtomic(path string, data []byte, perm os.FileMode) error { dir := filepath.Dir(path) diff --git a/internal/secretfile/secretfile.go b/internal/secretfile/secretfile.go new file mode 100644 index 0000000..e8c46c6 --- /dev/null +++ b/internal/secretfile/secretfile.go @@ -0,0 +1,458 @@ +// Package secretfile wraps SelfPost's secret-bearing downloads — the full +// server backup and the single-domain export — in a password-encrypted +// envelope. Both artefacts carry DKIM private keys, SASL credentials and +// application passwords in the clear, so an operator who stores them outside +// the server (the whole point of a backup) has to protect them by hand. The +// envelope makes that optional-but-easy: tick a box, give a password, and the +// file that leaves the panel is useless without it. +// +// Layout — a fixed header followed by a sequence of independently +// authenticated chunks, so both writing and reading stream and a multi-megabyte +// backup never has to sit in memory: +// +// magic 9 "SELFPOST1" (the trailing digit is the envelope version) +// type 1 PayloadType: what the plaintext is +// kdf 1 KDF identifier (1 = scrypt) +// logN 1 scrypt cost, log2(N) +// r 4 scrypt block size (big-endian) +// p 4 scrypt parallelisation (big-endian) +// salt 16 scrypt salt +// prefix 8 nonce prefix +// chunks … repeated: length (4, big-endian) + AES-256-GCM ciphertext +// +// The KDF parameters travel in the header so a file encrypted today still opens +// after the cost is raised. Every chunk is sealed with the nonce prefix plus its +// own counter, and takes the whole header, that counter and an end-of-stream +// flag as additional data: a chunk cannot be reordered, swapped between files, +// or dropped from the end without decryption failing. +package secretfile + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "io" + + "golang.org/x/crypto/scrypt" +) + +// PayloadType identifies what a decrypted envelope contains, so opening a file +// can report "that is a domain export, not a backup" instead of handing an +// unexpected payload to a parser. +type PayloadType byte + +const ( + // TypeFullBackup is the gzip-compressed tar of a full server backup. + TypeFullBackup PayloadType = 1 + // TypeDomainExport is the JSON document of a single-domain export. + TypeDomainExport PayloadType = 2 +) + +// String names a payload type for operator-facing messages. +func (t PayloadType) String() string { + switch t { + case TypeFullBackup: + return "full backup" + case TypeDomainExport: + return "domain export" + default: + return fmt.Sprintf("unknown type %d", byte(t)) + } +} + +// File name suffixes for the two encrypted artefacts: SelfPost BacKup and +// SelfPost Domain Export. They exist so an operator can tell an encrypted file +// from a plain .tar.gz/.json at a glance; nothing reads them back — the magic +// bytes decide. +const ( + ExtBackup = ".spbk" + ExtDomainExport = ".spde" +) + +// magic prefixes every envelope. MagicLen bytes are enough to tell an encrypted +// file from a plain one, which is what an import handler needs to decide whether +// to ask for a password. +var magic = []byte("SELFPOST1") + +// MagicLen is the number of leading bytes HasMagic inspects. +const MagicLen = 9 + +const ( + kdfScrypt = 1 + + saltLen = 16 + noncePrefixLen = 8 + counterLen = 4 + headerLen = MagicLen + 1 + 1 + 1 + 4 + 4 + saltLen + noncePrefixLen + + // chunkSize is the plaintext carried by one sealed chunk. 64 KiB keeps the + // per-chunk overhead negligible while bounding the buffer a reader must + // allocate for a hostile length field. + chunkSize = 64 * 1024 + tagLen = 16 + keyLen = 32 // AES-256 +) + +// Default scrypt parameters: 32 MiB and roughly a tenth of a second per attempt +// on ordinary hardware. The panel derives a key at most once per download, so +// the cost is invisible to the operator and expensive for anyone brute-forcing a +// stolen backup. +const ( + defaultLogN = 15 + defaultR = 8 + defaultP = 1 + + // maxLogN caps what a file may ask for, so a hostile header cannot make the + // reader allocate its way to an out-of-memory kill before it ever fails + // authentication. + maxLogN = 20 + maxR = 32 + maxP = 16 +) + +var ( + // ErrNotEncrypted reports a file that is not a SelfPost envelope at all — + // most often a plain .json export handed to the encrypted path. + ErrNotEncrypted = errors.New("secretfile: not an encrypted SelfPost file") + // ErrWrongPassword reports a failed authentication: the password is wrong, + // or the file has been altered. The two are indistinguishable by design. + ErrWrongPassword = errors.New("secretfile: wrong password or corrupted file") + // ErrCorrupt reports structural damage — a truncated or malformed envelope. + ErrCorrupt = errors.New("secretfile: corrupted file") +) + +// HasMagic reports whether b begins with the envelope magic. b may be shorter +// than MagicLen (a short file simply is not an envelope). +func HasMagic(b []byte) bool { + if len(b) < MagicLen { + return false + } + for i, c := range magic { + if b[i] != c { + return false + } + } + return true +} + +// header is the parsed fixed prefix, kept alongside its raw bytes because those +// bytes are the additional data every chunk is authenticated with. +type header struct { + raw []byte + typ PayloadType + logN uint8 + r, p uint32 + salt []byte + noncePrefix []byte +} + +func (h *header) marshal() []byte { + buf := make([]byte, 0, headerLen) + buf = append(buf, magic...) + buf = append(buf, byte(h.typ), kdfScrypt, h.logN) + buf = binary.BigEndian.AppendUint32(buf, h.r) + buf = binary.BigEndian.AppendUint32(buf, h.p) + buf = append(buf, h.salt...) + buf = append(buf, h.noncePrefix...) + return buf +} + +func parseHeader(buf []byte) (*header, error) { + if !HasMagic(buf) { + return nil, ErrNotEncrypted + } + if len(buf) != headerLen { + return nil, ErrCorrupt + } + h := &header{raw: buf} + i := MagicLen + h.typ = PayloadType(buf[i]) + if buf[i+1] != kdfScrypt { + return nil, fmt.Errorf("%w: unsupported key derivation %d", ErrCorrupt, buf[i+1]) + } + h.logN = buf[i+2] + h.r = binary.BigEndian.Uint32(buf[i+3 : i+7]) + h.p = binary.BigEndian.Uint32(buf[i+7 : i+11]) + i += 11 + h.salt = buf[i : i+saltLen] + h.noncePrefix = buf[i+saltLen : i+saltLen+noncePrefixLen] + + // Reject absurd work factors before spending any memory on them. + if h.logN < 1 || h.logN > maxLogN || h.r < 1 || h.r > maxR || h.p < 1 || h.p > maxP { + return nil, fmt.Errorf("%w: unreasonable key-derivation parameters", ErrCorrupt) + } + return h, nil +} + +// deriveKey runs scrypt with the header's parameters. +func (h *header) deriveKey(password string) ([]byte, error) { + key, err := scrypt.Key([]byte(password), h.salt, 1< 0 { + n := chunkSize - len(e.buf) + if n > len(p) { + n = len(p) + } + e.buf = append(e.buf, p[:n]...) + p = p[n:] + written += n + if len(e.buf) == chunkSize { + if err := e.flush(false); err != nil { + return written, err + } + } + } + return written, nil +} + +// flush seals the buffered plaintext as one chunk. last marks the end of the +// stream inside the authenticated data. +func (e *Writer) flush(last bool) error { + e.sealed = e.aead.Seal(e.sealed[:0], nonce(e.hdr.noncePrefix, e.counter), e.buf, chunkAAD(e.hdr.raw, e.counter, last)) + var length [4]byte + binary.BigEndian.PutUint32(length[:], uint32(len(e.sealed))) + if _, err := e.w.Write(length[:]); err != nil { + e.err = fmt.Errorf("secretfile: write chunk: %w", err) + return e.err + } + if _, err := e.w.Write(e.sealed); err != nil { + e.err = fmt.Errorf("secretfile: write chunk: %w", err) + return e.err + } + e.buf = e.buf[:0] + e.counter++ + return nil +} + +// Close seals whatever is buffered as the final chunk. It does not close the +// underlying writer. +func (e *Writer) Close() error { + if e.err != nil { + return e.err + } + if e.closed { + return nil + } + e.closed = true + // Always emitted, even for empty input: the end-of-stream chunk is what + // proves the file was not truncated. + return e.flush(true) +} + +// Reader decrypts an envelope. Chunks are verified as they are read, so a +// truncated or tampered file surfaces as a read error rather than as short but +// plausible plaintext. +type Reader struct { + r io.Reader + aead cipher.AEAD + hdr *header + plain []byte // decrypted, not yet handed to the caller + sealed []byte // reusable ciphertext scratch + counter uint32 + done bool + err error +} + +// NewReader reads and verifies the envelope header, derives the key and returns +// a Reader over the plaintext. It fails with ErrNotEncrypted for a file that is +// not an envelope; a wrong password is only detected on the first Read, when +// the first chunk fails authentication. +func NewReader(r io.Reader, password string) (*Reader, error) { + raw := make([]byte, headerLen) + if _, err := io.ReadFull(r, raw); err != nil { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + // Too short to be an envelope: if what we did read is not even the + // magic, say so — that is the common "plain file" case. + if !HasMagic(raw) { + return nil, ErrNotEncrypted + } + return nil, ErrCorrupt + } + return nil, fmt.Errorf("secretfile: read header: %w", err) + } + h, err := parseHeader(raw) + if err != nil { + return nil, err + } + key, err := h.deriveKey(password) + if err != nil { + return nil, err + } + aead, err := newAEAD(key) + if err != nil { + return nil, err + } + return &Reader{r: r, aead: aead, hdr: h}, nil +} + +// Type reports what the envelope claims to contain. The claim is authenticated +// (the header is additional data for every chunk), so it is trustworthy as soon +// as the first Read succeeds. +func (d *Reader) Type() PayloadType { return d.hdr.typ } + +// Read returns decrypted plaintext, verifying one chunk at a time. +func (d *Reader) Read(p []byte) (int, error) { + for len(d.plain) == 0 { + if d.err != nil { + return 0, d.err + } + if d.done { + return 0, io.EOF + } + if err := d.next(); err != nil { + d.err = err + return 0, err + } + } + n := copy(p, d.plain) + d.plain = d.plain[n:] + return n, nil +} + +// next reads, verifies and decrypts the following chunk, setting done when the +// chunk carries the end-of-stream flag. +func (d *Reader) next() error { + var length [4]byte + if _, err := io.ReadFull(d.r, length[:]); err != nil { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + // The stream ended without a chunk marked last: truncated. + return ErrCorrupt + } + return fmt.Errorf("secretfile: read chunk: %w", err) + } + n := binary.BigEndian.Uint32(length[:]) + if n < tagLen || n > chunkSize+tagLen { + return ErrCorrupt + } + if cap(d.sealed) < int(n) { + d.sealed = make([]byte, n) + } + d.sealed = d.sealed[:n] + if _, err := io.ReadFull(d.r, d.sealed); err != nil { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return ErrCorrupt + } + return fmt.Errorf("secretfile: read chunk: %w", err) + } + + // A chunk authenticates under exactly one of the two end-of-stream flags; + // trying the ordinary one first keeps the common case to a single open. + nc := nonce(d.hdr.noncePrefix, d.counter) + plain, err := d.aead.Open(nil, nc, d.sealed, chunkAAD(d.hdr.raw, d.counter, false)) + if err != nil { + plain, err = d.aead.Open(nil, nc, d.sealed, chunkAAD(d.hdr.raw, d.counter, true)) + if err != nil { + return ErrWrongPassword + } + d.done = true + } + d.plain = plain + d.counter++ + return nil +} diff --git a/internal/secretfile/secretfile_test.go b/internal/secretfile/secretfile_test.go new file mode 100644 index 0000000..19720b6 --- /dev/null +++ b/internal/secretfile/secretfile_test.go @@ -0,0 +1,162 @@ +package secretfile + +import ( + "bytes" + "errors" + "io" + "testing" +) + +// encrypt is the round-trip helper: seal data and return the envelope bytes. +func encrypt(t *testing.T, typ PayloadType, password string, data []byte) []byte { + t.Helper() + var buf bytes.Buffer + w, err := NewWriter(&buf, typ, password) + if err != nil { + t.Fatalf("NewWriter: %v", err) + } + if _, err := w.Write(data); err != nil { + t.Fatalf("Write: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + return buf.Bytes() +} + +func decrypt(t *testing.T, enc []byte, password string) ([]byte, PayloadType, error) { + t.Helper() + r, err := NewReader(bytes.NewReader(enc), password) + if err != nil { + return nil, 0, err + } + out, err := io.ReadAll(r) + return out, r.Type(), err +} + +func TestRoundTripSizes(t *testing.T) { + // Empty, sub-chunk, exactly one chunk, and several chunks with a partial + // tail: the boundaries where chunk framing tends to break. + sizes := []int{0, 1, 1000, chunkSize - 1, chunkSize, chunkSize + 1, 3*chunkSize + 77} + for _, n := range sizes { + data := make([]byte, n) + for i := range data { + data[i] = byte(i * 7) + } + enc := encrypt(t, TypeFullBackup, "correct horse battery staple", data) + got, typ, err := decrypt(t, enc, "correct horse battery staple") + if err != nil { + t.Fatalf("size %d: decrypt: %v", n, err) + } + if typ != TypeFullBackup { + t.Errorf("size %d: type = %v, want full backup", n, typ) + } + if !bytes.Equal(got, data) { + t.Errorf("size %d: plaintext mismatch (%d bytes back)", n, len(got)) + } + } +} + +func TestCiphertextDoesNotLeakPlaintext(t *testing.T) { + secret := []byte("-----BEGIN RSA PRIVATE KEY-----\nMIIEow...\n") + enc := encrypt(t, TypeDomainExport, "a very long password", secret) + if bytes.Contains(enc, secret) { + t.Fatal("plaintext found verbatim in the envelope") + } + if !HasMagic(enc) { + t.Fatal("envelope does not start with the magic") + } +} + +func TestWrongPassword(t *testing.T) { + enc := encrypt(t, TypeDomainExport, "the right password", []byte("secret payload")) + _, _, err := decrypt(t, enc, "the wrong password") + if !errors.Is(err, ErrWrongPassword) { + t.Fatalf("err = %v, want ErrWrongPassword", err) + } +} + +func TestNotEncrypted(t *testing.T) { + for _, plain := range [][]byte{[]byte(`{"format":"selfpost-domain-export"}`), {}, []byte("SELF")} { + if _, err := NewReader(bytes.NewReader(plain), "pw"); !errors.Is(err, ErrNotEncrypted) { + t.Errorf("NewReader(%q) = %v, want ErrNotEncrypted", plain, err) + } + if HasMagic(plain) { + t.Errorf("HasMagic(%q) = true", plain) + } + } +} + +func TestTruncationDetected(t *testing.T) { + // A backup cut short mid-transfer must fail loudly rather than restore a + // plausible-looking prefix. + data := bytes.Repeat([]byte("payload"), 20000) // spans several chunks + enc := encrypt(t, TypeFullBackup, "password password", data) + + // Drop the final chunk entirely: what remains is a sequence of valid, + // correctly authenticated chunks, none of which is marked last. + var chunks []int + for off := headerLen; off < len(enc); { + n := int(enc[off])<<24 | int(enc[off+1])<<16 | int(enc[off+2])<<8 | int(enc[off+3]) + chunks = append(chunks, off) + off += 4 + n + } + if len(chunks) < 2 { + t.Fatalf("expected several chunks, got %d", len(chunks)) + } + truncated := enc[:chunks[len(chunks)-1]] + r, err := NewReader(bytes.NewReader(truncated), "password password") + if err != nil { + t.Fatalf("NewReader (truncated): %v", err) + } + if _, err := io.ReadAll(r); !errors.Is(err, ErrCorrupt) { + t.Fatalf("read truncated = %v, want ErrCorrupt", err) + } +} + +func TestTamperDetected(t *testing.T) { + enc := encrypt(t, TypeFullBackup, "password password", []byte("some archive bytes")) + for _, off := range []int{MagicLen /* type byte */, headerLen + 6 /* ciphertext */} { + bad := bytes.Clone(enc) + bad[off] ^= 0xff + if _, _, err := decrypt(t, bad, "password password"); err == nil { + t.Errorf("flipping byte %d was accepted", off) + } + } +} + +func TestChunkReorderRejected(t *testing.T) { + // Two full chunks plus a tail, so swapping the first two is possible without + // changing any length. + data := make([]byte, 2*chunkSize+10) + for i := range data { + data[i] = byte(i) + } + enc := encrypt(t, TypeFullBackup, "password password", data) + + const framed = 4 + chunkSize + tagLen + first := headerLen + second := headerLen + framed + swapped := bytes.Clone(enc) + copy(swapped[first:first+framed], enc[second:second+framed]) + copy(swapped[second:second+framed], enc[first:first+framed]) + + if _, _, err := decrypt(t, swapped, "password password"); !errors.Is(err, ErrWrongPassword) { + t.Fatalf("reordered chunks = %v, want authentication failure", err) + } +} + +func TestEmptyPasswordRejected(t *testing.T) { + if _, err := NewWriter(io.Discard, TypeFullBackup, ""); err == nil { + t.Fatal("empty password accepted") + } +} + +func TestUnreasonableKDFParamsRejected(t *testing.T) { + enc := encrypt(t, TypeFullBackup, "password password", []byte("x")) + bad := bytes.Clone(enc) + bad[MagicLen+2] = 40 // logN far beyond maxLogN + if _, err := NewReader(bytes.NewReader(bad), "password password"); !errors.Is(err, ErrCorrupt) { + t.Fatalf("err = %v, want ErrCorrupt", err) + } +} diff --git a/internal/store/admin.go b/internal/store/admin.go index 46d1565..31fcf77 100644 --- a/internal/store/admin.go +++ b/internal/store/admin.go @@ -10,7 +10,7 @@ import ( // ErrNoAdmin is returned by GetAdmin when primary setup has not happened yet. var ErrNoAdmin = errors.New("no administrator account") -// Admin is the single panel administrator (spec 7.6.1). +// Admin is the single panel administrator (security.md). type Admin struct { Username string PasswordHash string @@ -19,7 +19,7 @@ type Admin struct { // AdminExists reports whether the administrator account has been created. This // doubles as the "primary setup complete" flag: once true, the /setup route is -// permanently gone (spec 7.6.1). +// permanently gone (security.md). func (s *Store) AdminExists() (bool, error) { var n int if err := s.db.QueryRow("SELECT COUNT(*) FROM admin").Scan(&n); err != nil { diff --git a/internal/store/admin_test.go b/internal/store/admin_test.go index 6a3c485..c240f29 100644 --- a/internal/store/admin_test.go +++ b/internal/store/admin_test.go @@ -28,7 +28,7 @@ func TestUpdateAdmin(t *testing.T) { } // An update before setup must not create the account: only the one-time setup -// flow may do that (spec 7.6.1). +// flow may do that (security.md). func TestUpdateAdminWithoutAdmin(t *testing.T) { st := openTestStore(t) diff --git a/internal/store/applications.go b/internal/store/applications.go index a8efeed..dfe2577 100644 --- a/internal/store/applications.go +++ b/internal/store/applications.go @@ -8,23 +8,24 @@ import ( ) // ErrLoginExists is returned when an application login is already taken. The -// login is globally unique because it is the SASL identity Postfix authenticates -// (one sasldb2 across all domains, spec 5.1). +// login is globally unique because it is the SASL identity Postfix +// authenticates (one sasldb2 across all domains, architecture.md § Mail path). var ErrLoginExists = errors.New("application login already exists") // ErrApplicationNotFound is returned when an application id does not exist. var ErrApplicationNotFound = errors.New("application not found") -// Address modes (spec 4.1). Kept in sync with the CHECK constraint in the schema. +// Address modes (product.md § Multi-domain model). Kept in sync with the CHECK +// constraint in the schema. const ( AddressModeWildcard = "wildcard" // any address within the application's domain AddressModeList = "list" // only the explicitly listed addresses ) -// Application is a SASL account bound to a single domain (spec 4.1, 5.1). The -// password is never stored here — only in sasldb2, hashed — so it can be shown -// exactly once at creation/regeneration (spec 7.6.1). Addresses is populated only -// in 'list' mode. +// Application is a SASL account bound to a single domain (product.md § +// Multi-domain model). The password is never stored here — only in sasldb2, +// hashed — so it can be shown exactly once at creation/regeneration +// (security.md). Addresses is populated only in 'list' mode. type Application struct { ID int64 DomainID int64 @@ -35,9 +36,9 @@ type Application struct { } // Binding is one sender-address → login pair, as consumed by the -// smtpd_sender_login_maps generator (spec 5.1). For a wildcard application the -// Address is the domain wildcard "@example.com"; for a list application there is -// one Binding per listed address. +// smtpd_sender_login_maps generator (architecture.md § Mail path). For a +// wildcard application the Address is the domain wildcard "@example.com"; for +// a list application there is one Binding per listed address. type Binding struct { Address string Login string @@ -45,7 +46,7 @@ type Binding struct { // AddApplication inserts an application and, in list mode, its addresses, in a // single transaction. The caller must have validated login and every address -// (spec 7.6.2) beforehand; the query is parameterised regardless. A duplicate +// (security.md) beforehand; the query is parameterised regardless. A duplicate // login maps to ErrLoginExists. func (s *Store) AddApplication(domainID int64, login, mode string, addresses []string) (Application, error) { now := time.Now().UTC() @@ -82,7 +83,7 @@ func (s *Store) AddApplication(domainID int64, login, mode string, addresses []s } // UpdateApplicationMode switches an application's address mode and replaces its -// address list atomically (spec 7.2.7). The login and password are untouched. +// address list atomically (product.md). The login and password are untouched. // Returns ErrApplicationNotFound if the id does not exist. func (s *Store) UpdateApplicationMode(id int64, mode string, addresses []string) error { tx, err := s.db.Begin() @@ -163,7 +164,7 @@ func (s *Store) GetApplication(id int64) (Application, error) { } // ListApplicationsByDomain returns a domain's applications ordered by login, -// each with its address list populated (spec 7.2.6). +// each with its address list populated (product.md). func (s *Store) ListApplicationsByDomain(domainID int64) ([]Application, error) { rows, err := s.db.Query( "SELECT id, domain_id, login, address_mode, created_at FROM applications WHERE domain_id = ? ORDER BY login", @@ -196,8 +197,9 @@ func (s *Store) ListApplicationsByDomain(domainID int64) ([]Application, error) } // ListApplicationLogins returns every application login across all domains, -// ordered, for the send-log monitoring screen's filter dropdown (spec 7.2). -// Logins are globally unique (spec 5.1), so no domain qualifier is needed. +// ordered, for the send-log monitoring screen's filter dropdown (product.md). +// Logins are globally unique (architecture.md § Mail path), so no domain +// qualifier is needed. func (s *Store) ListApplicationLogins() ([]string, error) { rows, err := s.db.Query("SELECT login FROM applications ORDER BY login") if err != nil { @@ -218,7 +220,7 @@ func (s *Store) ListApplicationLogins() ([]string, error) { // ListLoginsByDomain returns the SASL logins of a domain's applications. Used to // purge sasldb2 entries before a domain (and its applications via cascade) is -// deleted, while the logins are still known (spec 7.2.4). +// deleted, while the logins are still known (product.md). func (s *Store) ListLoginsByDomain(domainID int64) ([]string, error) { rows, err := s.db.Query("SELECT login FROM applications WHERE domain_id = ? ORDER BY login", domainID) if err != nil { @@ -237,10 +239,11 @@ func (s *Store) ListLoginsByDomain(domainID int64) ([]string, error) { return out, rows.Err() } -// ListBindings returns every sender-address → login pair across all domains, the -// raw material for the smtpd_sender_login_maps file (spec 5.1). Wildcard -// applications yield a single "@domain" binding; list applications yield one -// binding per address. Ordered deterministically so the generated map is stable. +// ListBindings returns every sender-address → login pair across all domains, +// the raw material for the smtpd_sender_login_maps file (architecture.md § +// Mail path). Wildcard applications yield a single "@domain" binding; list +// applications yield one binding per address. Ordered deterministically so the +// generated map is stable. func (s *Store) ListBindings() ([]Binding, error) { rows, err := s.db.Query(` SELECT '@' || d.name, a.login @@ -271,7 +274,7 @@ func (s *Store) ListBindings() ([]Binding, error) { // DeleteApplication removes an application and its addresses (via cascade), // returning the deleted application so the caller can drop its sasldb2 entry -// (spec 7.2.8). Returns ErrApplicationNotFound if no such row existed. +// (product.md). Returns ErrApplicationNotFound if no such row existed. func (s *Store) DeleteApplication(id int64) (Application, error) { a, err := s.GetApplication(id) if err != nil { diff --git a/internal/store/domains.go b/internal/store/domains.go index fba6ce1..a370476 100644 --- a/internal/store/domains.go +++ b/internal/store/domains.go @@ -16,9 +16,10 @@ var ErrDomainExists = errors.New("domain already exists") // ErrDomainNotFound is returned when a domain id/name does not exist. var ErrDomainNotFound = errors.New("domain not found") -// Domain is a sending domain managed through the panel (spec 4.1). The DKIM key -// material itself lives on disk under /data; this row records the selector and -// metadata. AppCount is populated by the listing queries, not stored. +// Domain is a sending domain managed through the panel (product.md § +// Multi-domain model). The DKIM key material itself lives on disk under /data; +// this row records the selector and metadata. AppCount is populated by the +// listing queries, not stored. type Domain struct { ID int64 Name string @@ -28,7 +29,7 @@ type Domain struct { } // AddDomain inserts a new sending domain. The caller is responsible for having -// validated name (spec 7.6.2) before it reaches SQL; the query is parameterised +// validated name (security.md) before it reaches SQL; the query is parameterised // regardless. A duplicate name maps to ErrDomainExists. func (s *Store) AddDomain(name, selector string) (Domain, error) { now := time.Now().UTC() @@ -49,7 +50,7 @@ func (s *Store) AddDomain(name, selector string) (Domain, error) { return Domain{ID: id, Name: name, DKIMSelector: selector, CreatedAt: now}, nil } -// ListDomains returns every domain with its bound-application count (spec 7.2.2), +// ListDomains returns every domain with its bound-application count (product.md), // ordered by name. func (s *Store) ListDomains() ([]Domain, error) { rows, err := s.db.Query(` @@ -92,7 +93,7 @@ func (s *Store) GetDomain(id int64) (Domain, error) { } // DeleteDomain removes a domain. Its applications and their address/binding rows -// go with it via ON DELETE CASCADE (spec 7.2.4). Returns ErrDomainNotFound if no +// go with it via ON DELETE CASCADE (product.md). Returns ErrDomainNotFound if no // such row existed. func (s *Store) DeleteDomain(id int64) error { res, err := s.db.Exec("DELETE FROM domains WHERE id = ?", id) diff --git a/internal/store/domains_test.go b/internal/store/domains_test.go index 6d8e8ae..8fdab04 100644 --- a/internal/store/domains_test.go +++ b/internal/store/domains_test.go @@ -62,8 +62,8 @@ func TestDeleteDomainCascadesApplications(t *testing.T) { t.Fatal(err) } - // Insert an application + address directly (the AddApplication API lands in - // Phase 4); this verifies the ON DELETE CASCADE wiring now. + // Insert an application + address directly (bypassing the AddApplication + // API) to verify the ON DELETE CASCADE wiring. now := time.Now().UTC().Format(time.RFC3339) res, err := st.db.Exec( "INSERT INTO applications (domain_id, login, address_mode, created_at) VALUES (?, ?, 'wildcard', ?)", diff --git a/internal/store/logtail.go b/internal/store/logtail.go new file mode 100644 index 0000000..e9d1fc8 --- /dev/null +++ b/internal/store/logtail.go @@ -0,0 +1,54 @@ +package store + +import ( + "database/sql" + "errors" + "fmt" + "time" +) + +// LogtailState is the persisted read position of the mail.log tailer: how far +// into the log it had got, and a fingerprint of the file that offset refers to. +// It lets the tailer resume after a panel restart instead of jumping to +// end-of-file and losing the delivery lines written while it was down. +type LogtailState struct { + Fingerprint string + Offset int64 +} + +// LogtailState returns the stored read position for path. ok is false the first +// time a path is followed (nothing persisted yet), which the tailer treats as +// "start at the end". +func (s *Store) LogtailState(path string) (LogtailState, bool, error) { + var st LogtailState + err := s.db.QueryRow( + `SELECT fingerprint, read_offset FROM logtail_state WHERE path = ?`, + path, + ).Scan(&st.Fingerprint, &st.Offset) + if errors.Is(err, sql.ErrNoRows) { + return LogtailState{}, false, nil + } + if err != nil { + return LogtailState{}, false, fmt.Errorf("read logtail state for %q: %w", path, err) + } + return st, true, nil +} + +// SaveLogtailState records the tailer's read position for path, replacing any +// previous one. It is called on a timer while tailing, so it is a single small +// upsert rather than a transaction. +func (s *Store) SaveLogtailState(path string, st LogtailState) error { + _, err := s.db.Exec( + `INSERT INTO logtail_state (path, fingerprint, read_offset, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(path) DO UPDATE SET + fingerprint = excluded.fingerprint, + read_offset = excluded.read_offset, + updated_at = excluded.updated_at`, + path, st.Fingerprint, st.Offset, time.Now().UTC().Format(time.RFC3339), + ) + if err != nil { + return fmt.Errorf("save logtail state for %q: %w", path, err) + } + return nil +} diff --git a/internal/store/migrations/0003_logtail_state.sql b/internal/store/migrations/0003_logtail_state.sql new file mode 100644 index 0000000..c1ea93a --- /dev/null +++ b/internal/store/migrations/0003_logtail_state.sql @@ -0,0 +1,12 @@ +-- Log-tailer read position. Without it the tailer starts at end-of-file on +-- every start, so delivery lines written while the panel was down are never +-- parsed and their send-log rows stay "queued" forever. One row per followed +-- path; fingerprint identifies the file the offset belongs to (the head bytes +-- of the log), so a rotated or recreated mail.log is detected across a restart, +-- where os.SameFile cannot help. +CREATE TABLE logtail_state ( + path TEXT PRIMARY KEY, + fingerprint TEXT NOT NULL, + read_offset INTEGER NOT NULL, + updated_at TEXT NOT NULL +); diff --git a/internal/store/ratelimits.go b/internal/store/ratelimits.go index 37bb3fd..a2d6010 100644 --- a/internal/store/ratelimits.go +++ b/internal/store/ratelimits.go @@ -9,23 +9,25 @@ import ( "time" ) -// Rate-limit scopes (spec 7.4). A level-2 limit is attached either to a domain -// (counted across all its applications and IPs) or to a single application. +// Rate-limit scopes (README § Rate limiting). A level-2 limit is attached +// either to a domain (counted across all its applications and IPs) or to a +// single application. const ( RateLimitScopeDomain = "domain" RateLimitScopeApp = "application" ) -// RateLimit is a differentiated level-2 rate limit (spec 7.4): an optional set -// of expected client IPs plus a message ceiling over a sliding window, attached -// to a domain or an application. It is enforced in the journal-milter; level 1 -// (Postfix anvil, spec 5) is the IP backstop that always applies even when this -// is absent or the milter is down. +// RateLimit is a differentiated level-2 rate limit (README § Rate limiting): +// an optional set of expected client IPs plus a message ceiling over a sliding +// window, attached to a domain or an application. It is enforced in the +// journal-milter; level 1 (Postfix anvil, architecture.md § Mail path) is the +// IP backstop that always applies even when this is absent or the milter is +// down. // -// Both the IP binding and the ceiling are optional in the schema, but a limit is -// only enforced when it is Active(): the design deliberately allows an admin to -// leave the IP binding empty for apps that send from changing IPs, in which case -// only level 1 protects them (spec 7.4's caveat). +// Both the IP binding and the ceiling are optional in the schema, but a limit +// is only enforced when it is Active(): the design deliberately allows an +// admin to leave the IP binding empty for apps that send from changing IPs, in +// which case only level 1 protects them (README § Rate limiting). type RateLimit struct { Scope string RefID int64 @@ -35,8 +37,9 @@ type RateLimit struct { } // Active reports whether the limit is fully configured and should be enforced. -// A missing IP binding, ceiling or window leaves the differentiated limit inert -// (spec 7.4): the IP binding is what scopes the limit to a known sender. +// A missing IP binding, ceiling or window leaves the differentiated limit +// inert (README § Rate limiting): the IP binding is what scopes the limit to a +// known sender. func (r RateLimit) Active() bool { return len(r.AllowedIPs) > 0 && r.MaxMessages > 0 && r.WindowSeconds > 0 } @@ -76,7 +79,7 @@ func (s *Store) GetRateLimit(scope string, refID int64) (RateLimit, bool, error) } // SetRateLimit upserts the level-2 limit for a domain or application. The caller -// (panel) has already validated the IPs and numbers (spec 7.6.2); values are +// (panel) has already validated the IPs and numbers (security.md); values are // stored via bound parameters and read back live by the milter. func (s *Store) SetRateLimit(rl RateLimit) error { _, err := s.db.Exec( @@ -148,12 +151,13 @@ func (s *Store) RateLimit(scope, ref string) (RateLimit, bool, error) { return rl, true, nil } -// CountMessages returns how many distinct messages the reference (a domain name -// or an application login) has queued since t, for the level-2 sliding window -// (spec 7.4). It counts distinct queue-ids — one message with many recipients is -// one message, matching level 1's per-message semantics — and excludes rows that -// were themselves rejected by a limit (they were never sent). It reuses the send -// log the journal already writes (spec 7.4: "переиспользует данные журнала"). +// CountMessages returns how many distinct messages the reference (a domain +// name or an application login) has queued since t, for the level-2 sliding +// window (README § Rate limiting). It counts distinct queue-ids — one message +// with many recipients is one message, matching level 1's per-message +// semantics — and excludes rows that were themselves rejected by a limit (they +// were never sent). It reuses the send log the journal already writes (README +// § Rate limiting — the limiter reuses the send log). func (s *Store) CountMessages(scope, ref string, since time.Time) (int64, error) { var column string switch scope { diff --git a/internal/store/sendlog.go b/internal/store/sendlog.go index 0ef0faa..5abeda1 100644 --- a/internal/store/sendlog.go +++ b/internal/store/sendlog.go @@ -6,23 +6,25 @@ import ( "time" ) -// Send-log status values (spec 7.3). "queued" is written by the journal-milter -// when a message is accepted; the log-tailer advances it to one of the final -// states as Postfix reports delivery per recipient. +// Send-log status values (architecture.md § Persistence). "queued" is written +// by the journal-milter when a message is accepted; the log-tailer advances it +// to one of the final states as Postfix reports delivery per recipient. const ( StatusQueued = "queued" StatusSent = "sent" StatusDeferred = "deferred" StatusBounced = "bounced" // StatusRejected marks a message the journal-milter refused with a 4xx under - // a level-2 rate limit (spec 7.4). Such a row never gets a queue-id and is - // excluded from the level-2 message count (it was never sent). + // a level-2 rate limit (README § Rate limiting). Such a row never gets a + // queue-id and is excluded from the level-2 message count (it was never + // sent). StatusRejected = "rejected" ) // SendLogEntry is a single queued send-log row. The journal-milter creates one -// per (queue-id, recipient) pair at end-of-message (spec 7.3.3); every field -// except the status/timestamps comes from the accepted message. +// per (queue-id, recipient) pair at end-of-message (architecture.md § +// Persistence); every field except the status/timestamps comes from the +// accepted message. type SendLogEntry struct { QueueID string Domain string @@ -34,8 +36,8 @@ type SendLogEntry struct { // InsertQueued records an accepted message in the send log with status // "queued". It is called from the journal-milter hot path, so it returns any -// error for the caller to log rather than deciding policy here; the milter must -// stay fail-open regardless (spec 7.3). +// error for the caller to log rather than deciding policy here; the milter +// must stay fail-open regardless (architecture.md § Persistence). func (s *Store) InsertQueued(e SendLogEntry) error { now := time.Now().UTC().Format(time.RFC3339) _, err := s.db.Exec( @@ -51,9 +53,10 @@ func (s *Store) InsertQueued(e SendLogEntry) error { } // InsertRejected records a message the journal-milter refused under a level-2 -// rate limit (spec 7.4), so the rejection is visible in the send-log UI. Only -// the fields known at MAIL FROM are set (domain, sender, app login); there is no -// queue-id or recipient because the message was rejected before it was queued. +// rate limit (README § Rate limiting), so the rejection is visible in the +// send-log UI. Only the fields known at MAIL FROM are set (domain, sender, app +// login); there is no queue-id or recipient because the message was rejected +// before it was queued. func (s *Store) InsertRejected(e SendLogEntry) error { now := time.Now().UTC().Format(time.RFC3339) _, err := s.db.Exec( @@ -88,9 +91,9 @@ func (s *Store) UpdateStatus(queueID, recipient, status string) (int64, error) { return n, nil } -// SendLogRow is one row as returned to the monitoring UI (spec 7.2, 7.3.3): a -// SendLogEntry plus the fields that only exist once a row has been written -// (id, current status, timestamps). +// SendLogRow is one row as returned to the monitoring UI (architecture.md § +// Persistence): a SendLogEntry plus the fields that only exist once a row has +// been written (id, current status, timestamps). type SendLogRow struct { ID int64 QueueID string @@ -111,7 +114,7 @@ type SendLogFilter struct { } // QuerySendLog returns send-log rows matching filter, newest first, for the -// monitoring screen's server-side pagination (spec 7.2's send-log view). +// monitoring screen's server-side pagination (product.md's send-log view). func (s *Store) QuerySendLog(filter SendLogFilter, limit, offset int) ([]SendLogRow, error) { where, args := sendLogWhere(filter) args = append(args, limit, offset) @@ -171,10 +174,11 @@ func sendLogWhere(f SendLogFilter) (string, []any) { return " WHERE " + strings.Join(clauses, " AND "), args } -// DeleteSendLogBefore removes send-log rows created before cutoff, implementing -// the configurable retention window (spec 7.3, SEND_LOG_RETENTION_DAYS). It -// returns the number of rows pruned. created_at is stored as RFC3339 UTC, so a -// lexical comparison against the same format is chronologically correct. +// DeleteSendLogBefore removes send-log rows created before cutoff, +// implementing the configurable retention window (architecture.md § +// Persistence, SEND_LOG_RETENTION_DAYS). It returns the number of rows pruned. +// created_at is stored as RFC3339 UTC, so a lexical comparison against the +// same format is chronologically correct. func (s *Store) DeleteSendLogBefore(cutoff time.Time) (int64, error) { res, err := s.db.Exec( `DELETE FROM send_log WHERE created_at < ?`, diff --git a/internal/store/sendlog_test.go b/internal/store/sendlog_test.go index a042781..9947c80 100644 --- a/internal/store/sendlog_test.go +++ b/internal/store/sendlog_test.go @@ -5,8 +5,8 @@ import ( "time" ) -// readSendLog returns every send_log row ordered by id. Phase 6 has no read -// query yet (the monitoring UI is Phase 7), so tests read the table directly. +// readSendLog returns every send_log row ordered by id. The monitoring UI has +// no equivalent read query, so tests read the table directly. type sendLogRow struct { QueueID string Domain string @@ -40,7 +40,8 @@ func readSendLog(t *testing.T, s *Store) []sendLogRow { func TestInsertQueuedAndUpdateStatus(t *testing.T) { st := openTestStore(t) - // Two recipients on the same queue-id → two independent rows (spec 7.3.3). + // Two recipients on the same queue-id → two independent rows (architecture.md + // § Persistence). for _, to := range []string{"a@example.net", "b@example.net"} { if err := st.InsertQueued(SendLogEntry{ QueueID: "ABC123", diff --git a/internal/store/store.go b/internal/store/store.go index e9b22c0..4434eba 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -1,7 +1,7 @@ // Package store owns the SelfPost SQLite database: the single file under /data // that persists the administrator account, sending domains and applications, -// the send log and rate-limit settings (spec 9). It exposes typed queries so -// the rest of the panel never builds SQL by hand. +// the send log and rate-limit settings (architecture.md § Persistence). It +// exposes typed queries so the rest of the panel never builds SQL by hand. package store import ( diff --git a/internal/web/handlers_account.go b/internal/web/handlers_account.go index 845359d..8533d34 100644 --- a/internal/web/handlers_account.go +++ b/internal/web/handlers_account.go @@ -5,15 +5,15 @@ import ( "net/http" "strings" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/store" "golang.org/x/crypto/bcrypt" ) // handleAccount serves the administrator's own account settings: the username -// and password chosen during setup are the only panel credentials (spec 7.6.1), -// and until now they could be changed only by recreating the state. Changing -// them here never touches application SASL logins, which are a separate -// identity system (spec 5.1). +// and password chosen during setup are the only panel credentials +// (security.md), and until now they could be changed only by recreating the +// state. Changing them here never touches application SASL logins, which are a +// separate identity system (architecture.md § Mail path). func (s *Server) handleAccount(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodGet: @@ -58,7 +58,7 @@ func accountFlash(r *http.Request) string { // is always required, so a stolen session alone cannot lock the administrator // out of their own panel, and the attempt is throttled on the same limiter as // the login form so this route cannot be used to brute-force the password past -// that limit (spec 7.6.5). +// that limit (security.md). func (s *Server) submitAccount(w http.ResponseWriter, r *http.Request) { if !s.loginLimiter.Allow(clientIP(r, s.trustedProxies)) { s.renderAccount(w, r, http.StatusTooManyRequests, diff --git a/internal/web/handlers_apps.go b/internal/web/handlers_apps.go index c469aca..700280d 100644 --- a/internal/web/handlers_apps.go +++ b/internal/web/handlers_apps.go @@ -7,13 +7,13 @@ import ( "strconv" "strings" - "codeberg.org/mix/selfpost/internal/dnscheck" - "codeberg.org/mix/selfpost/internal/domain" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/dnscheck" + "github.com/mixeme/selfpost/internal/domain" + "github.com/mixeme/selfpost/internal/store" ) // newCred carries a freshly generated login/password to the template so it can -// be shown exactly once (spec 7.6.1). It is never read back from storage. +// be shown exactly once (security.md). It is never read back from storage. type newCred struct { Login string Password string @@ -29,9 +29,12 @@ type detailView struct { FormMode string FormAddrs string NewCred *newCred - // RateLimitErr surfaces a validation error from a domain- or application-level - // rate-limit form (spec 7.4) as a page banner. + // RateLimitErr surfaces a validation error from a domain- or + // application-level rate-limit form (README § Rate limiting) as a page + // banner. RateLimitErr string + // ExportErr surfaces a rejected encryption password from the export card. + ExportErr string } // appRateLimitView pairs an application with its differentiated rate-limit @@ -45,9 +48,9 @@ type appRateLimitView struct { WindowVal string // window seconds, defaulted when unset } -// handleDomainDetail shows a single domain: its DKIM DNS record (spec 7.2.10) +// handleDomainDetail shows a single domain: its DKIM DNS record (product.md) // and its applications with the controls to add, edit, delete and re-issue -// credentials (spec 7.2.5-9). +// credentials (product.md). func (s *Server) handleDomainDetail(w http.ResponseWriter, r *http.Request) { d, ok := s.lookupDomain(w, r) if !ok { @@ -97,8 +100,8 @@ func (s *Server) renderDomainDetail(w http.ResponseWriter, r *http.Request, stat } // What DNS actually publishes for the domain today, checked against the key - // this server signs with (phase 13.B). Cached by the checker, so - // re-rendering the page after a form post costs nothing. + // this server signs with. Cached by the checker, so re-rendering the page + // after a form post costs nothing. dns, srv := s.domainDNS(d, record, false) s.render(w, status, "domain_detail", map[string]any{ @@ -131,6 +134,8 @@ func (s *Server) renderDomainDetail(w http.ResponseWriter, r *http.Request, stat "Wildcard": store.AddressModeWildcard, "List": store.AddressModeList, "RateLimitErr": view.RateLimitErr, + "ExportErr": view.ExportErr, + "MinPwLen": minSecretFilePasswordLen, "DomainHasRL": domainRLok && domainRL.Active(), "DomainRLIPs": strings.Join(domainRL.AllowedIPs, "\n"), "DomainRLMax": intOrBlank(domainRL.MaxMessages), @@ -139,7 +144,7 @@ func (s *Server) renderDomainDetail(w http.ResponseWriter, r *http.Request, stat } // domainDNS resolves what the world sees for a domain: its DKIM, SPF and DMARC -// records (phase 13.B). The server's own address comes from the (separately +// records. The server's own address comes from the (separately // cached) hostname check, so the SPF heuristic knows which IP it is looking for // and no extra environment variable is needed. That server result is returned // alongside, because the page's suggested SPF record is built from the same @@ -210,7 +215,7 @@ func detailFlash(r *http.Request) string { } // handleAddApplication creates an application on a domain and renders the page -// back with the generated password shown once (spec 7.2.5, 7.6.1). Because the +// back with the generated password shown once (product.md, security.md). Because the // password cannot be recovered later, this deliberately renders inline rather // than redirecting. func (s *Server) handleAddApplication(w http.ResponseWriter, r *http.Request) { @@ -249,7 +254,7 @@ func (s *Server) handleAddApplication(w http.ResponseWriter, r *http.Request) { }) } -// handleUpdateAppMode switches an application's address mode / list (spec 7.2.7). +// handleUpdateAppMode switches an application's address mode / list (product.md). func (s *Server) handleUpdateAppMode(w http.ResponseWriter, r *http.Request) { a, ok := s.lookupApplication(w, r) if !ok { @@ -278,7 +283,7 @@ func (s *Server) handleUpdateAppMode(w http.ResponseWriter, r *http.Request) { } // handleRegenPassword issues a new password for an application and shows it once -// (spec 7.2.9, 7.6.1). Rendered inline, like creation, so the password is visible. +// (product.md, security.md). Rendered inline, like creation, so the password is visible. func (s *Server) handleRegenPassword(w http.ResponseWriter, r *http.Request) { a, ok := s.lookupApplication(w, r) if !ok { @@ -302,7 +307,7 @@ func (s *Server) handleRegenPassword(w http.ResponseWriter, r *http.Request) { } // handleDeleteApplication removes an application and returns to its domain page -// (spec 7.2.8). +// (product.md). func (s *Server) handleDeleteApplication(w http.ResponseWriter, r *http.Request) { a, ok := s.lookupApplication(w, r) if !ok { @@ -339,7 +344,7 @@ func (s *Server) lookupApplication(w http.ResponseWriter, r *http.Request) (stor // splitAddresses turns the textarea/field input (addresses separated by // newlines, commas or whitespace) into a raw slice. Normalisation and -// validation happen in the app service (spec 7.6.2). +// validation happen in the app service (security.md). func splitAddresses(s string) []string { return strings.FieldsFunc(s, func(r rune) bool { return r == '\n' || r == '\r' || r == ',' || r == ' ' || r == '\t' || r == ';' diff --git a/internal/web/handlers_auth.go b/internal/web/handlers_auth.go index 569c99a..72c546e 100644 --- a/internal/web/handlers_auth.go +++ b/internal/web/handlers_auth.go @@ -5,7 +5,7 @@ import ( "net/http" "strings" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/store" "golang.org/x/crypto/bcrypt" ) @@ -13,7 +13,7 @@ import ( // TLS in front, so CookieSecure — it carries the __Host- prefix, which turns // what the cookie's attributes merely promise into something the browser // enforces: Secure, Path=/ and, the point of the exercise, no Domain -// attribute, so no other host may set a cookie by this name (phase 14.B). +// attribute, so no other host may set a cookie by this name. // The prefix is only valid on a Secure cookie, so a development instance on // plain HTTP has to keep the bare name: with the prefix the browser would // discard the Set-Cookie outright and logging in would silently never stick. @@ -42,7 +42,7 @@ func (s *Server) sessionCookie() string { // is denial of service, not compromise; refusing the request and saying so in // the log is what makes it diagnosable instead of an endless login loop. The // __Host- prefix prevents this outright, but only where it applies — this -// check also covers the plain-HTTP development shape (phase 14.B). +// check also covers the plain-HTTP development shape. func (s *Server) sessionToken(r *http.Request) (string, bool) { name := s.sessionCookie() var token string @@ -127,7 +127,7 @@ func (s *Server) renderLogin(w http.ResponseWriter, status int, formErr string) } func (s *Server) submitLogin(w http.ResponseWriter, r *http.Request) { - // Brute-force throttle by client IP (spec 7.6.5). + // Brute-force throttle by client IP (security.md). if !s.loginLimiter.Allow(clientIP(r, s.trustedProxies)) { s.renderLogin(w, http.StatusTooManyRequests, "Too many attempts. Please wait and try again.") return diff --git a/internal/web/handlers_backup.go b/internal/web/handlers_backup.go index acb16fe..4b1d45f 100644 --- a/internal/web/handlers_backup.go +++ b/internal/web/handlers_backup.go @@ -1,20 +1,25 @@ package web import ( + "bytes" "encoding/json" "errors" "fmt" + "io" "net/http" "time" - "codeberg.org/mix/selfpost/internal/backup" - "codeberg.org/mix/selfpost/internal/domain" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/backup" + "github.com/mixeme/selfpost/internal/domain" + "github.com/mixeme/selfpost/internal/secretfile" + "github.com/mixeme/selfpost/internal/store" ) // maxImportBytes caps a domain-import upload. A domain export is a small JSON // document (a DKIM key and a handful of credentials); this leaves generous head -// room while refusing anything large enough to be an abuse attempt. +// room while refusing anything large enough to be an abuse attempt. An +// encrypted export adds only a header and per-chunk tags, so the same ceiling +// covers both forms. const maxImportBytes = 1 << 20 // 1 MiB // handleBackupPage renders the backup/migration screen: the full-server backup @@ -25,50 +30,108 @@ func (s *Server) handleBackupPage(w http.ResponseWriter, r *http.Request) { } // renderBackupPage draws the page; importErr surfaces a failed domain import -// (spec 7.5.B) next to the form that produced it. +// (architecture.md § Persistence) next to the form that produced it. func (s *Server) renderBackupPage(w http.ResponseWriter, r *http.Request, status int, importErr string) { + s.renderBackupPageWith(w, r, status, importErr, "") +} + +// renderBackupPageWith is renderBackupPage with the second of the page's two +// error slots: backupErr belongs to the full-backup card (a rejected encryption +// password), importErr to the import card, so neither message appears under the +// wrong form. +func (s *Server) renderBackupPageWith(w http.ResponseWriter, r *http.Request, status int, importErr, backupErr string) { s.render(w, status, "backup", map[string]any{ "Title": "SelfPost — backup", "User": currentUser(r), "Active": "backup", "ImportErr": importErr, + "BackupErr": backupErr, + "MinPwLen": minSecretFilePasswordLen, }) } -// handleBackup streams a full-server backup as a download (spec 7.5.A). It is an -// authenticated admin action (this handler sits behind the auth middleware). The -// archive carries DKIM private keys, the admin password hash and SASL -// credentials, so it is served with no-store and as an attachment to discourage -// caching of secret material. +// handleBackup streams a full-server backup as a download (architecture.md § +// Persistence). It is an authenticated admin action (this handler sits behind +// the auth middleware). The archive carries DKIM private keys, the admin +// password hash and SASL credentials, so it is served with no-store and as an +// attachment to discourage caching of secret material. When the operator ticks +// "encrypt with a password", the archive is wrapped in a .spbk envelope on the +// way out, so the file that lands on their disk — wherever it is copied +// afterwards — is useless without the password. func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { - filename := fmt.Sprintf("selfpost-backup-%s.tar.gz", time.Now().UTC().Format("20060102-150405")) + password, pwErr := secretFilePassword(r) + if pwErr != "" { + s.renderBackupPageWith(w, r, http.StatusBadRequest, "", pwErr) + return + } - w.Header().Set("Content-Type", "application/gzip") + stamp := time.Now().UTC().Format("20060102-150405") + filename := fmt.Sprintf("selfpost-backup-%s.tar.gz", stamp) + contentType := "application/gzip" + if password != "" { + filename = fmt.Sprintf("selfpost-backup-%s%s", stamp, secretfile.ExtBackup) + contentType = "application/octet-stream" + } + + w.Header().Set("Content-Type", contentType) w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filename)) w.Header().Set("Cache-Control", "no-store") - if err := backup.Create(w, backup.Params{ + // Everything below streams: past this point headers (and possibly some + // bytes) are already on the wire, so a failure cannot switch to a clean + // error page. Log it and let the truncated download fail loudly on the + // client side — for an encrypted archive that is a missing end-of-stream + // chunk, which decryption refuses outright. + sink := io.Writer(w) + var env *secretfile.Writer + if password != "" { + var err error + // The only failures here are key derivation (which happens before + // anything is written) and writing the envelope header, which fails only + // if the client is already gone. + env, err = secretfile.NewWriter(w, secretfile.TypeFullBackup, password) + if err != nil { + logf("panel: full backup: encrypt: %v", err) + http.Error(w, "backup failed", http.StatusInternalServerError) + return + } + sink = env + } + + if err := backup.Create(sink, backup.Params{ DataDir: s.cfg.DataDir, DBPath: s.cfg.DBPath, Version: s.cfg.Version, }); err != nil { - // Headers (and possibly some bytes) may already be on the wire, so we - // cannot switch to a clean error page; log it and let the truncated - // download fail loudly on the client side. logf("panel: full backup failed: %v", err) return } + if env != nil { + if err := env.Close(); err != nil { + logf("panel: full backup failed: %v", err) + } + } } -// handleExportDomain streams a single-domain export as a secret download (spec -// 7.5.B). Like the full backup it is POST-only (state is not changed, but the -// response contains the domain's DKIM private key and application passwords, so -// it must not be prefetchable or cached). +// handleExportDomain streams a single-domain export as a secret download +// (architecture.md § Persistence). Like the full backup it is POST-only (state +// is not changed, but the response contains the domain's DKIM private key and +// application passwords, so it must not be prefetchable or cached). Like the +// full backup it can be encrypted with a password, in which case the download +// is a .spde envelope instead of plain JSON. func (s *Server) handleExportDomain(w http.ResponseWriter, r *http.Request) { d, ok := s.lookupDomain(w, r) if !ok { return } + password, pwErr := secretFilePassword(r) + if pwErr != "" { + s.renderDomainDetail(w, r, http.StatusBadRequest, d, detailView{ + FormMode: store.AddressModeWildcard, + ExportErr: pwErr, + }) + return + } exp, err := s.domains.Export(d.ID) if err != nil { logf("panel: export domain %d: %v", d.ID, err) @@ -83,18 +146,41 @@ func (s *Server) handleExportDomain(w http.ResponseWriter, r *http.Request) { } filename := fmt.Sprintf("selfpost-domain-%s.json", d.Name) - w.Header().Set("Content-Type", "application/json") + contentType := "application/json" + if password != "" { + // An export is small, so it is sealed in memory: the response is only + // started once the ciphertext is complete and nothing can half-fail. + var buf bytes.Buffer + env, err := secretfile.NewWriter(&buf, secretfile.TypeDomainExport, password) + if err == nil { + _, err = env.Write(body) + } + if err == nil { + err = env.Close() + } + if err != nil { + logf("panel: export domain %d: encrypt: %v", d.ID, err) + http.Error(w, "export failed", http.StatusInternalServerError) + return + } + body = buf.Bytes() + filename = fmt.Sprintf("selfpost-domain-%s%s", d.Name, secretfile.ExtDomainExport) + contentType = "application/octet-stream" + } + + w.Header().Set("Content-Type", contentType) w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filename)) w.Header().Set("Cache-Control", "no-store") _, _ = w.Write(body) } // handleImportDomain accepts an uploaded domain-export file and re-creates the -// domain on this instance (spec 7.5.B). The domain name is normalised and -// validated here (spec 7.6.2); the domain service validates the selector, each -// login and address, and the DKIM key before writing anything. On success it -// redirects to the new domain's page; on failure it re-renders the backup page, -// where the import form lives, with a friendly message. +// domain on this instance (architecture.md § Persistence). The domain name is +// normalised and validated here (security.md); the domain service validates +// the selector, each login and address, and the DKIM key before writing +// anything. On success it redirects to the new domain's page; on failure it +// re-renders the backup page, where the import form lives, with a friendly +// message. func (s *Server) handleImportDomain(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, maxImportBytes) if err := r.ParseMultipartForm(maxImportBytes); err != nil { @@ -108,8 +194,49 @@ func (s *Server) handleImportDomain(w http.ResponseWriter, r *http.Request) { } defer file.Close() + // An encrypted export announces itself with the envelope magic, so the file + // decides which path it takes; the password field is only consulted when the + // file actually needs it, and a password typed for a plain file is a plain + // mistake worth reporting. + head := make([]byte, secretfile.MagicLen) + n, err := io.ReadFull(file, head) + if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { + s.renderBackupPage(w, r, http.StatusBadRequest, "Could not read the uploaded file.") + return + } + source := io.MultiReader(bytes.NewReader(head[:n]), file) + password := r.PostFormValue("import_password") + + if secretfile.HasMagic(head[:n]) { + if password == "" { + s.renderBackupPage(w, r, http.StatusBadRequest, "That file is encrypted — enter the password it was exported with.") + return + } + env, err := secretfile.NewReader(source, password) + if err != nil { + s.renderBackupPage(w, r, http.StatusBadRequest, decryptErrorMessage(err)) + return + } + if env.Type() != secretfile.TypeDomainExport { + s.renderBackupPage(w, r, http.StatusBadRequest, "That file is an encrypted "+env.Type().String()+", not a domain export.") + return + } + // Read the whole plaintext first: authentication of the last chunk is + // what proves the file is intact, and a streaming JSON decoder could + // accept a truncated document before ever reaching it. + plain, err := io.ReadAll(env) + if err != nil { + s.renderBackupPage(w, r, http.StatusBadRequest, decryptErrorMessage(err)) + return + } + source = bytes.NewReader(plain) + } else if password != "" { + s.renderBackupPage(w, r, http.StatusBadRequest, "That file is not encrypted — leave the password empty.") + return + } + var exp domain.DomainExport - dec := json.NewDecoder(file) + dec := json.NewDecoder(source) dec.DisallowUnknownFields() if err := dec.Decode(&exp); err != nil { s.renderBackupPage(w, r, http.StatusBadRequest, "That file is not a valid SelfPost domain export.") @@ -117,7 +244,7 @@ func (s *Server) handleImportDomain(w http.ResponseWriter, r *http.Request) { } // Normalise and validate the domain name before it reaches the service, the - // same gate the add-domain form uses (spec 7.6.2). + // same gate the add-domain form uses (security.md). exp.Domain = normalizeDomain(exp.Domain) if err := validateDomain(exp.Domain); err != nil { s.renderBackupPage(w, r, http.StatusBadRequest, "Invalid domain in export file: "+err.Error()) @@ -134,6 +261,46 @@ func (s *Server) handleImportDomain(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, fmt.Sprintf("/domains/%d?imported=1", d.ID), http.StatusSeeOther) } +// secretFilePassword reads the "encrypt this download with a password" controls +// shared by the full-backup and domain-export forms. It returns the password to +// encrypt with — empty when the box is not ticked, which keeps the plain +// .tar.gz/.json behaviour of earlier versions — or a message to show above the +// form. The confirmation field is checked here rather than in the browser +// because a typo in an encryption password is unrecoverable: the archive would +// be sealed with a secret the operator does not know. +func secretFilePassword(r *http.Request) (password, errMsg string) { + if err := r.ParseForm(); err != nil { + return "", "Invalid form submission." + } + if r.PostFormValue("encrypt") == "" { + return "", "" + } + password = r.PostFormValue("password") + if len([]rune(password)) < minSecretFilePasswordLen { + return "", fmt.Sprintf("The encryption password must be at least %d characters.", minSecretFilePasswordLen) + } + if password != r.PostFormValue("password_confirm") { + return "", "The two passwords do not match." + } + return password, "" +} + +// decryptErrorMessage phrases an envelope failure for the operator. A wrong +// password and a damaged file are deliberately indistinguishable to the code, +// so the message names both possibilities. +func decryptErrorMessage(err error) string { + switch { + case errors.Is(err, secretfile.ErrWrongPassword): + return "Wrong password, or the file has been altered since it was exported." + case errors.Is(err, secretfile.ErrCorrupt): + return "That file is damaged or incomplete." + case errors.Is(err, secretfile.ErrNotEncrypted): + return "That file is not a SelfPost export." + default: + return "Could not decrypt the file." + } +} + // importErrorMessage maps a domain-import failure (already logged by the caller) // to an HTTP status and a user-facing message. Duplicate domain/login are called // out specifically; other failures — validation errors describing what is wrong diff --git a/internal/web/handlers_backup_test.go b/internal/web/handlers_backup_test.go new file mode 100644 index 0000000..d9b3492 --- /dev/null +++ b/internal/web/handlers_backup_test.go @@ -0,0 +1,127 @@ +package web + +import ( + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/mixeme/selfpost/internal/secretfile" +) + +// postForm builds the kind of request the backup and export forms submit. +func postForm(values url.Values) *http.Request { + r := httptest.NewRequest(http.MethodPost, "/backup", strings.NewReader(values.Encode())) + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + return r +} + +// The encryption password is only ever typed once into a file nobody can +// recover without it, so every way of getting it wrong has to be caught before +// the archive is sealed — and leaving the box unticked has to keep producing +// the plain archive earlier versions produced. +func TestSecretFilePassword(t *testing.T) { + long := strings.Repeat("x", minSecretFilePasswordLen) + short := strings.Repeat("x", minSecretFilePasswordLen-1) + + tests := []struct { + name string + form url.Values + wantPass string + wantErr bool + }{ + { + name: "unticked box means no encryption", + form: url.Values{"password": {long}, "password_confirm": {long}}, + wantPass: "", + }, + { + name: "ticked with a matching password", + form: url.Values{"encrypt": {"1"}, "password": {long}, "password_confirm": {long}}, + wantPass: long, + }, + { + name: "mistyped confirmation", + form: url.Values{"encrypt": {"1"}, "password": {long}, "password_confirm": {long + "!"}}, + wantErr: true, + }, + { + name: "too short", + form: url.Values{"encrypt": {"1"}, "password": {short}, "password_confirm": {short}}, + wantErr: true, + }, + { + name: "ticked but empty", + form: url.Values{"encrypt": {"1"}}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pass, errMsg := secretFilePassword(postForm(tt.form)) + if tt.wantErr { + if errMsg == "" { + t.Fatalf("password %q accepted, want a rejection", pass) + } + if pass != "" { + t.Errorf("a rejected form still yielded password %q", pass) + } + return + } + if errMsg != "" { + t.Fatalf("unexpected rejection: %s", errMsg) + } + if pass != tt.wantPass { + t.Errorf("password = %q, want %q", pass, tt.wantPass) + } + }) + } +} + +// The messages the import form shows must distinguish the operator's likely +// mistakes; a wrong password and a tampered file stay deliberately merged. +func TestDecryptErrorMessage(t *testing.T) { + tests := []struct { + err error + want string + }{ + {secretfile.ErrWrongPassword, "Wrong password"}, + {fmt.Errorf("read: %w", secretfile.ErrCorrupt), "damaged"}, + {secretfile.ErrNotEncrypted, "not a SelfPost export"}, + {errors.New("something else"), "Could not decrypt"}, + } + for _, tt := range tests { + if got := decryptErrorMessage(tt.err); !strings.Contains(got, tt.want) { + t.Errorf("decryptErrorMessage(%v) = %q, want it to mention %q", tt.err, got, tt.want) + } + } +} + +// The encryption controls are shared markup pulled into two pages; a page that +// forgets to include the partial (or the data it needs) loses the option +// silently, since the plain download still works. +func TestBackupPageOffersEncryption(t *testing.T) { + tmpl, err := loadTemplates() + if err != nil { + t.Fatalf("loadTemplates: %v", err) + } + s := &Server{tmpl: tmpl, cfg: Config{Version: "test"}} + rec := httptest.NewRecorder() + s.renderBackupPageWith(rec, httptest.NewRequest(http.MethodGet, "/backup", nil), + http.StatusOK, "", "The two passwords do not match.") + + body := rec.Body.String() + for _, want := range []string{ + `name="encrypt"`, `name="password"`, `name="password_confirm"`, + `name="import_password"`, "data-encrypt-toggle", "data-encrypt-fields", + fmt.Sprintf("at least %d characters", minSecretFilePasswordLen), + "The two passwords do not match.", + } { + if !strings.Contains(body, want) { + t.Errorf("backup page is missing %q", want) + } + } +} diff --git a/internal/web/handlers_domains.go b/internal/web/handlers_domains.go index bdb7e91..88eae4c 100644 --- a/internal/web/handlers_domains.go +++ b/internal/web/handlers_domains.go @@ -6,12 +6,12 @@ import ( "net/http" "strconv" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/store" ) // handleDashboard is the authenticated landing page: the list of sending // domains with their DKIM/selector and application counts, plus the add-domain -// form (spec 7.2.2). Applications and the send log arrive in later phases. +// form (product.md). func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) { s.renderDashboard(w, r, http.StatusOK, "", "") } @@ -48,7 +48,7 @@ func dashboardFlash(r *http.Request) string { // handleAddDomain validates the submitted name, creates the domain (DKIM key + // OpenDKIM reload), and redirects to the domain's page so the DNS record to -// publish is shown (spec 7.2.3). +// publish is shown (product.md). func (s *Server) handleAddDomain(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { s.renderDashboard(w, r, http.StatusBadRequest, "Invalid form submission.", "") @@ -76,7 +76,7 @@ func (s *Server) handleAddDomain(w http.ResponseWriter, r *http.Request) { } // handleDeleteConfirm shows the cascade warning before a domain is removed: the -// panel must explicitly state that all bound applications go with it (spec 7.2.4). +// panel must explicitly state that all bound applications go with it (product.md). func (s *Server) handleDeleteConfirm(w http.ResponseWriter, r *http.Request) { d, ok := s.lookupDomain(w, r) if !ok { @@ -116,11 +116,12 @@ func (s *Server) handleDeleteDomain(w http.ResponseWriter, r *http.Request) { } // handleReload re-applies both the OpenDKIM configuration and the Postfix -// sender map on demand (spec 7.2.12). Each Resync regenerates its files from the -// database and reloads its daemon, so the button doubles as a drift-recovery. -// The button lives on the status page (phase 13.D): it is a "put the daemons -// back in the state the database describes" action, which belongs with the rest -// of the server-health screen rather than in the domain list's top bar. +// sender map on demand (architecture.md § Panel HTTP surface). Each Resync +// regenerates its files from the database and reloads its daemon, so the +// button doubles as a drift-recovery. The button lives on the status page: it +// is a "put the daemons back in the state the database describes" action, +// which belongs with the rest of the server-health screen rather than in the +// domain list's top bar. func (s *Server) handleReload(w http.ResponseWriter, r *http.Request) { if err := s.domains.Resync(); err != nil { logf("panel: manual reload (opendkim): %v", err) diff --git a/internal/web/handlers_monitor.go b/internal/web/handlers_monitor.go index 3708a0d..1d353a2 100644 --- a/internal/web/handlers_monitor.go +++ b/internal/web/handlers_monitor.go @@ -6,12 +6,12 @@ import ( "net/http" "strconv" - "codeberg.org/mix/selfpost/internal/logtail" - "codeberg.org/mix/selfpost/internal/postfix" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/logtail" + "github.com/mixeme/selfpost/internal/postfix" + "github.com/mixeme/selfpost/internal/store" ) -// sendLogPageSize bounds each send-log page (spec 7.2's monitoring screens +// sendLogPageSize bounds each send-log page (product.md's monitoring screens // call for pagination); logTailLines bounds how much of mail.log the log view // shows per refresh. const ( @@ -20,10 +20,10 @@ const ( ) // handleDeliveries renders the Deliveries page over the send log: server-side -// filters by domain/application and pagination (spec 7.3.3). The row table -// itself is the "deliveries_rows" fragment, shared verbatim with -// handleDeliveriesRows so the initial page and its HTMX-polled refreshes never -// diverge. +// filters by domain/application and pagination (architecture.md § +// Persistence). The row table itself is the "deliveries_rows" fragment, shared +// verbatim with handleDeliveriesRows so the initial page and its HTMX-polled +// refreshes never diverge. func (s *Server) handleDeliveries(w http.ResponseWriter, r *http.Request) { data, err := s.sendLogData(r) if err != nil { @@ -38,7 +38,8 @@ func (s *Server) handleDeliveries(w http.ResponseWriter, r *http.Request) { } // handleDeliveriesRows serves the HTMX polling fragment for the delivery table -// (spec 7.1: fragment endpoints return HTML, not JSON). +// (architecture.md § Panel HTTP surface: fragment endpoints return HTML, not +// JSON). func (s *Server) handleDeliveriesRows(w http.ResponseWriter, r *http.Request) { data, err := s.sendLogData(r) if err != nil { @@ -110,7 +111,8 @@ func parsePage(v string) int { return n } -// handleMailQueue renders the Mail queue page (spec 7.2.11). +// handleMailQueue renders the Mail queue page (architecture.md § Panel HTTP +// surface). func (s *Server) handleMailQueue(w http.ResponseWriter, r *http.Request) { out, errText := readQueue() s.render(w, http.StatusOK, "mail_queue", map[string]any{ @@ -143,7 +145,8 @@ func readQueue() (string, string) { return out, "" } -// handleSystemLog renders the System log page over mail.log (spec 7.2.13). +// handleSystemLog renders the System log page over mail.log (architecture.md § +// Panel HTTP surface). func (s *Server) handleSystemLog(w http.ResponseWriter, r *http.Request) { lines, errText := s.readLogTail() s.render(w, http.StatusOK, "system_log", map[string]any{ diff --git a/internal/web/handlers_ratelimit.go b/internal/web/handlers_ratelimit.go index a9cd9b0..021d875 100644 --- a/internal/web/handlers_ratelimit.go +++ b/internal/web/handlers_ratelimit.go @@ -7,17 +7,18 @@ import ( "strconv" "strings" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/store" ) -// defaultRateLimitWindowSeconds is the sliding-window length used when an admin -// sets a message ceiling but leaves the window blank (spec 7.4, matching the -// level-1 default hour, spec 8: RATE_LIMIT_WINDOW_SECONDS). +// defaultRateLimitWindowSeconds is the sliding-window length used when an +// admin sets a message ceiling but leaves the window blank (README § Rate +// limiting, matching the level-1 default hour; README § Environment variables: +// RATE_LIMIT_WINDOW_SECONDS). const defaultRateLimitWindowSeconds = 3600 -// rateLimitInput is the validated result of a rate-limit form submission. clear -// means "remove the differentiated limit" (spec 7.4: an empty IP binding leaves -// only level 1). +// rateLimitInput is the validated result of a rate-limit form submission. +// clear means "remove the differentiated limit" (README § Rate limiting: an +// empty IP binding leaves only level 1). type rateLimitInput struct { clear bool ips []string @@ -25,10 +26,10 @@ type rateLimitInput struct { windowSeconds int } -// parseRateLimitForm validates a rate-limit submission on the server (spec -// 7.6.2). It returns clear=true when the admin removes the limit or leaves the -// IP binding empty; otherwise it requires a positive ceiling and window. The -// returned error's message is safe to show to the admin. +// parseRateLimitForm validates a rate-limit submission on the server +// (security.md). It returns clear=true when the admin removes the limit or +// leaves the IP binding empty; otherwise it requires a positive ceiling and +// window. The returned error's message is safe to show to the admin. func parseRateLimitForm(r *http.Request) (rateLimitInput, error) { if err := r.ParseForm(); err != nil { return rateLimitInput{}, fmt.Errorf("invalid form submission") @@ -41,7 +42,8 @@ func parseRateLimitForm(r *http.Request) (rateLimitInput, error) { return rateLimitInput{}, err } if len(ips) == 0 { - // No IP binding: the differentiated limit does not apply (spec 7.4). + // No IP binding: the differentiated limit does not apply (README § Rate + // limiting). return rateLimitInput{clear: true}, nil } maxMessages, err := parsePositiveInt(r.PostFormValue("max_messages"), 0) @@ -57,7 +59,7 @@ func parseRateLimitForm(r *http.Request) (rateLimitInput, error) { // parseIPList parses the allowed-IP field (IPs separated by newlines, commas or // whitespace) into a deduplicated list of canonical addresses, rejecting any -// token that is not a valid IP (spec 7.6.2). The values are only ever stored as +// token that is not a valid IP (security.md). The values are only ever stored as // SQLite parameters and compared in the milter, never written to a config file. func parseIPList(raw string) ([]string, error) { fields := strings.FieldsFunc(raw, func(r rune) bool { @@ -89,8 +91,9 @@ func parsePositiveInt(raw string, def int) (int, error) { return strconv.Atoi(raw) } -// handleDomainRateLimit saves or clears a domain-level differentiated rate limit -// (spec 7.4). No reload is needed — the milter reads the row live. +// handleDomainRateLimit saves or clears a domain-level differentiated rate +// limit (README § Rate limiting). No reload is needed — the milter reads the +// row live. func (s *Server) handleDomainRateLimit(w http.ResponseWriter, r *http.Request) { d, ok := s.lookupDomain(w, r) if !ok { @@ -113,7 +116,7 @@ func (s *Server) handleDomainRateLimit(w http.ResponseWriter, r *http.Request) { } // handleAppRateLimit saves or clears an application-level differentiated rate -// limit (spec 7.4). +// limit (README § Rate limiting). func (s *Server) handleAppRateLimit(w http.ResponseWriter, r *http.Request) { a, ok := s.lookupApplication(w, r) if !ok { diff --git a/internal/web/handlers_setup.go b/internal/web/handlers_setup.go index 1505f75..06fcf2a 100644 --- a/internal/web/handlers_setup.go +++ b/internal/web/handlers_setup.go @@ -8,11 +8,11 @@ import ( ) // handleSetup serves the one-time administrator creation flow at -// /setup/ (spec 7.6.1). Once an administrator exists the whole route +// /setup/ (security.md). Once an administrator exists the whole route // returns 404; an invalid or expired token is indistinguishable from a missing // page, also 404. func (s *Server) handleSetup(w http.ResponseWriter, r *http.Request) { - // Route-specific rate limit, separate from login (spec 7.6.1). + // Route-specific rate limit, separate from login (security.md). if !s.setupLimiter.Allow(clientIP(r, s.trustedProxies)) { http.Error(w, "too many requests", http.StatusTooManyRequests) return @@ -92,7 +92,7 @@ func (s *Server) submitSetup(w http.ResponseWriter, r *http.Request, token strin return } - // Setup is now permanently complete: burn the token (spec 7.6.1). + // Setup is now permanently complete: burn the token (security.md). s.setup.complete() logf("panel: administrator %q created; setup link is now disabled", username) http.Redirect(w, r, "/login", http.StatusSeeOther) diff --git a/internal/web/handlers_status.go b/internal/web/handlers_status.go index fbe8c1d..031a1de 100644 --- a/internal/web/handlers_status.go +++ b/internal/web/handlers_status.go @@ -4,12 +4,12 @@ import ( "net/http" "strings" - "codeberg.org/mix/selfpost/internal/health" + "github.com/mixeme/selfpost/internal/health" ) // handleStatus renders the server status page: the panel's landing page and the -// one screen that answers "is the service healthy and will mail be accepted" -// (phase 13.A). The cheap local checks live in the polled "status_body" +// one screen that answers "is the service healthy and will mail be accepted". +// The cheap local checks live in the polled "status_body" // fragment; the hostname/PTR lookup and the configuration reload sit outside it, // because neither belongs on a five-second timer. func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { @@ -26,7 +26,8 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { } // handleStatusFragment serves the HTMX polling fragment for the local checks -// (spec 7.1: fragment endpoints return HTML, not JSON). +// (architecture.md § Panel HTTP surface: fragment endpoints return HTML, not +// JSON). func (s *Server) handleStatusFragment(w http.ResponseWriter, _ *http.Request) { s.renderFragment(w, http.StatusOK, "status_body", s.statusBody()) } @@ -90,9 +91,10 @@ func (s *Server) statusBody() map[string]any { } } -// queueSummary reduces postqueue's listing to the one line worth showing on the -// status page; the full listing has its own screen (spec 7.2.11). postqueue -// prints either "Mail queue is empty" or a trailing "-- N Kbytes in M Requests." +// queueSummary reduces postqueue's listing to the one line worth showing on +// the status page; the full listing has its own screen (architecture.md § +// Panel HTTP surface). postqueue prints either "Mail queue is empty" or a +// trailing "-- N Kbytes in M Requests." func queueSummary(out string) string { lines := strings.Split(strings.TrimSpace(out), "\n") for i := len(lines) - 1; i >= 0; i-- { diff --git a/internal/web/ratelimit.go b/internal/web/ratelimit.go index c0dde2b..664ad82 100644 --- a/internal/web/ratelimit.go +++ b/internal/web/ratelimit.go @@ -6,7 +6,7 @@ import ( ) // rateLimiter is a simple fixed-window per-key counter used to throttle the -// setup and login routes (spec 7.6.1, 7.6.5). Keys are client IPs. It is not a +// setup and login routes (security.md). Keys are client IPs. It is not a // precise sliding window — a coarse backstop against brute-force and log noise // is all these routes need. type rateLimiter struct { diff --git a/internal/web/security.go b/internal/web/security.go index b2eb1d4..d3b9120 100644 --- a/internal/web/security.go +++ b/internal/web/security.go @@ -8,7 +8,7 @@ import ( // contentSecurityPolicy is the panel's CSP. Everything the pages load — // stylesheet, HTMX, the panel's own script, the favicon — is served from // /static by this same origin, and no template carries an inline

The backup file is a secret (it contains private keys and credentials). Store and transfer it securely and delete it - once the restore succeeds.

-
+ once the restore succeeds. Encrypting it below is the simplest way to do that: + the download is then a .spbk file that only the password opens.

+ {{if .BackupErr}}

{{.BackupErr}}

{{end}} + + {{template "encryptfields" .}}
@@ -25,7 +28,17 @@ {{if .ImportErr}}

{{.ImportErr}}

{{end}}
- + +
+ +
+ + +
+
diff --git a/internal/web/templates/domain_detail.html b/internal/web/templates/domain_detail.html index 879102a..c1bb454 100644 --- a/internal/web/templates/domain_detail.html +++ b/internal/web/templates/domain_detail.html @@ -283,8 +283,11 @@ import the DNS record stays the same, so no DNS change is needed.

The export file is a secret — it contains the private DKIM key and application passwords. Transfer it securely and delete it - after the import.

-
+ after the import, or encrypt it below and move a .spde file + instead; the import form asks for the password.

+ {{if .ExportErr}}

{{.ExportErr}}

{{end}} + + {{template "encryptfields" .}}
diff --git a/internal/web/templates/encrypt_fields.html b/internal/web/templates/encrypt_fields.html new file mode 100644 index 0000000..3340201 --- /dev/null +++ b/internal/web/templates/encrypt_fields.html @@ -0,0 +1,23 @@ +{{/* Password fields shared by the full-backup and domain-export forms. The + checkbox is the switch: unticked, the download keeps its historic plain + form (.tar.gz / .json); ticked, the file is sealed in a password-encrypted + envelope (.spbk / .spde). The fields start hidden and are revealed by + panel.js — with JavaScript blocked they are simply always visible, and the + server still decides from the checkbox alone. */}} +{{define "encryptfields"}} +
+ +
+ + + + +

Keep this password: without it the file cannot be opened, + and SelfPost does not store it anywhere.

+
+
+{{end}} diff --git a/internal/web/templates/layout.html b/internal/web/templates/layout.html index 5b3249c..9cadf22 100644 --- a/internal/web/templates/layout.html +++ b/internal/web/templates/layout.html @@ -15,7 +15,7 @@ {{/* Unless told otherwise, htmx injects a stylesheet element of its own into the head for the request-indicator classes. The panel uses no hx-indicator, and that injected element would be the one thing the - Content-Security-Policy (phase 14.A) has to make an exception for, so it + Content-Security-Policy has to make an exception for, so it is switched off here. */}} diff --git a/internal/web/templates_test.go b/internal/web/templates_test.go index 0d1083e..63dfa89 100644 --- a/internal/web/templates_test.go +++ b/internal/web/templates_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "codeberg.org/mix/selfpost/internal/health" + "github.com/mixeme/selfpost/internal/health" ) // The navigation is rendered from the layout, not copied into each page, so @@ -142,9 +142,9 @@ func TestNavLeadsWithStatusAndPointsDomainsAtItsOwnPath(t *testing.T) { } } -// Since the panel root now redirects to the status page, a link left pointing at +// Since the panel root redirects to the status page, a link left pointing at // "/" silently lands on the wrong screen instead of failing — so no template may -// contain one (phase 13.C). +// contain one. func TestNoTemplateLinksToTheBareRoot(t *testing.T) { forEachTemplate(t, func(name, body string) { if strings.Contains(body, `href="/"`) { @@ -154,7 +154,7 @@ func TestNoTemplateLinksToTheBareRoot(t *testing.T) { } // The reload action is a server-health control and lives only on the status -// page (phase 13.D). +// page. func TestReloadFormLivesOnlyOnTheStatusPage(t *testing.T) { forEachTemplate(t, func(name, body string) { if strings.Contains(body, `action="/reload"`) && name != "status.html" { @@ -164,7 +164,7 @@ func TestReloadFormLivesOnlyOnTheStatusPage(t *testing.T) { } // The panel's Content-Security-Policy is a plain default-src 'self' with no -// inline exemption (phase 14.A), which makes inline script and inline style a +// inline exemption, which makes inline script and inline style a // failure mode rather than a style question: an onclick= handler or a // style="..." attribute added to a template does not error, it silently stops // working in the browser. Behaviour belongs in static/panel.js (triggered from diff --git a/internal/web/token.go b/internal/web/token.go index 6e9ba7e..56b578f 100644 --- a/internal/web/token.go +++ b/internal/web/token.go @@ -7,7 +7,7 @@ import ( // randomToken returns a URL-safe token with at least nBytes*8 bits of entropy // drawn from crypto/rand. Setup and session tokens both use this; the setup -// token needs >=128 bits (spec 7.6.1), so callers pass nBytes >= 16. +// token needs >=128 bits (security.md), so callers pass nBytes >= 16. // // It panics if the system RNG fails: that is unrecoverable and must never be // papered over with a weak fallback for a security token. diff --git a/internal/web/validate.go b/internal/web/validate.go index c720d93..a53483b 100644 --- a/internal/web/validate.go +++ b/internal/web/validate.go @@ -7,7 +7,7 @@ import ( ) // minAdminPasswordLen is the floor for the administrator password. The panel is -// public (spec 7.6), so this is deliberately not tiny. +// public (security.md), so this is deliberately not tiny. const minAdminPasswordLen = 12 const ( @@ -15,7 +15,13 @@ const ( maxUsernameLen = 64 ) -// validateUsername enforces a strict server-side whitelist (spec 7.6.2): +// minSecretFilePasswordLen is the floor for the password protecting an +// encrypted backup or domain export. Such a file is offline and can be attacked +// at leisure, so the floor matches the administrator password's rather than the +// weaker "any password is better than none". +const minSecretFilePasswordLen = minAdminPasswordLen + +// validateUsername enforces a strict server-side whitelist (security.md): // letters, digits, dot, dash, underscore. Client validation is never trusted. func validateUsername(u string) error { if len(u) < minUsernameLen || len(u) > maxUsernameLen { @@ -52,7 +58,7 @@ func normalizeDomain(name string) string { } // validateDomain enforces a strict server-side whitelist for sending-domain -// names (spec 7.6.2). The result is safe to write verbatim into the OpenDKIM +// names (security.md). The result is safe to write verbatim into the OpenDKIM // KeyTable/SigningTable and to use as a filesystem path segment: only // lower-case letters, digits, '.' and '-' are allowed, in valid DNS label // shape. Input must already be normalised with normalizeDomain. diff --git a/internal/web/web.go b/internal/web/web.go index ae0ae63..b23b3f7 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -1,6 +1,6 @@ // Package web implements the SelfPost control panel's HTTP surface: the -// one-time administrator setup flow (spec 7.6.1), login/session handling -// (spec 7.6.5-6) and the authenticated shell the later phases build on. +// one-time administrator setup flow (security.md), login/session handling +// (security.md) and the authenticated shell the later phases build on. package web import ( @@ -11,10 +11,11 @@ import ( "strings" "time" - "codeberg.org/mix/selfpost/internal/app" - "codeberg.org/mix/selfpost/internal/dnscheck" - "codeberg.org/mix/selfpost/internal/domain" - "codeberg.org/mix/selfpost/internal/store" + "github.com/mixeme/selfpost/internal/app" + "github.com/mixeme/selfpost/internal/dnscheck" + "github.com/mixeme/selfpost/internal/domain" + "github.com/mixeme/selfpost/internal/health" + "github.com/mixeme/selfpost/internal/store" ) //go:embed templates/*.html static/* @@ -23,24 +24,26 @@ var assetsFS embed.FS // Config holds the panel's HTTP-facing configuration. type Config struct { // Hostname is the server's external hostname, used to build the absolute - // setup link shown in the logs (spec 7.6.1, 8: SELFPOST_HOSTNAME). + // setup link shown in the logs (security.md; README § Environment + // variables for SELFPOST_HOSTNAME). Hostname string // CookieSecure sets the Secure attribute on the session cookie. It defaults - // to true (spec 7.6.6); it exists as a knob only so the panel can be tested + // to true (security.md); it exists as a knob only so the panel can be tested // over plain HTTP in development, never for production. CookieSecure bool // SubmissionEnabled mirrors SUBMISSION_ENABLE: whether this deployment also // runs the 587/STARTTLS submission listener next to the primary 465 one - // (spec 5). The panel only reports it on the domain page's connection - // settings; it is a deploy-time flag, not something the panel can verify. + // (architecture.md § Mail path). The panel only reports it on the domain + // page's connection settings; it is a deploy-time flag, not something the + // panel can verify. SubmissionEnabled bool // MailLogPath is where Postfix's delivery log lives, read by the mail.log - // monitoring view (spec 7.2.13). It is the same path the log-tailer role - // follows in cmd/panel. + // monitoring view (architecture.md § Panel HTTP surface). It is the same path + // the log-tailer role follows in cmd/panel. MailLogPath string // DataDir and DBPath locate the persistent state a full backup archives - // (spec 7.5.A); Version is stamped into the backup manifest. They mirror the - // panel's own configuration. + // (architecture.md § Persistence); Version is stamped into the backup + // manifest. They mirror the panel's own configuration. DataDir string DBPath string Version string @@ -50,8 +53,9 @@ type Config struct { // honoured, so the header can't be spoofed by anyone but a trusted proxy. // Empty (the default) keeps rate-limiting keyed on RemoteAddr only. TrustedProxyCIDRs []*net.IPNet - // TLSCertFile is the certificate Postfix serves on 465/587 (spec 8), read - // read-only by the status page to report how much validity is left. + // TLSCertFile is the certificate Postfix serves on 465/587 (README § + // Environment variables), read read-only by the status page to report how + // much validity is left. TLSCertFile string // OpenDKIMSocket and JournalSocket are the two milter sockets Postfix // connects to. The status page stats them: the first is required for mail @@ -88,9 +92,10 @@ type Server struct { } // New builds the panel server. setupTokenPath is where the current setup token -// is mirrored on disk (spec 7.6.1); domains is the sending-domain service that -// owns DKIM keys and the OpenDKIM tables (spec 6); apps owns application SASL -// accounts and the Postfix sender map (spec 5.1). +// is mirrored on disk (security.md); domains is the sending-domain service +// that owns DKIM keys and the OpenDKIM tables (architecture.md § OpenDKIM); +// apps owns application SASL accounts and the Postfix sender map +// (architecture.md § Mail path). func New(st *store.Store, domains *domain.Service, apps *app.Service, cfg Config, setupTokenPath string) (*Server, error) { tmpl, err := loadTemplates() if err != nil { @@ -109,12 +114,12 @@ func New(st *store.Store, domains *domain.Service, apps *app.Service, cfg Config sessions: newSessionStore(st, time.Duration(idleDays)*24*time.Hour), // Published-DNS checks for the status page and the domain pages. The // checker caches its own results, so page views do not each pay for a - // round of lookups (phase 13). + // round of lookups. dns: dnscheck.New(cfg.DNSResolvers), // Setup: a handful of attempts per minute per IP is plenty for a - // legitimate admin and blunts automated probing (spec 7.6.1). + // legitimate admin and blunts automated probing (security.md). setupLimiter: newRateLimiter(10, time.Minute), - // Login: throttle brute-force by IP (spec 7.6.5). + // Login: throttle brute-force by IP (security.md). loginLimiter: newRateLimiter(10, 15*time.Minute), trustedProxies: cfg.TrustedProxyCIDRs, @@ -124,7 +129,7 @@ func New(st *store.Store, domains *domain.Service, apps *app.Service, cfg Config } // Start performs first-run bootstrapping: if there is no administrator yet, it -// generates and announces the setup link (spec 7.6.1). Safe to call once at +// generates and announces the setup link (security.md). Safe to call once at // server startup. func (s *Server) Start() error { return s.setup.bootstrap() @@ -140,7 +145,7 @@ func (s *Server) Handler() http.Handler { // Vendored static assets (HTMX). Served from the embedded FS. mux.Handle("/static/", http.FileServer(http.FS(assetsFS))) - // One-time administrator setup (spec 7.6.1). + // One-time administrator setup (security.md). mux.HandleFunc("/setup/", s.handleSetup) // Authentication. @@ -151,7 +156,7 @@ func (s *Server) Handler() http.Handler { // above falls through to this sub-mux, wrapped once in the auth middleware. authed := http.NewServeMux() - // The landing page is the server status (phase 13.C): the first thing an + // The landing page is the server status: the first thing an // administrator should see after logging in is whether the service is // healthy, not the domain list. handleLogin still redirects to "/". authed.HandleFunc("GET /{$}", redirectToStatus) @@ -178,13 +183,14 @@ func (s *Server) Handler() http.Handler { // Administrator's own panel credentials. authed.HandleFunc("/account", s.handleAccount) - // Backup and migration: the page with both actions (spec 7.5.A-B), and the - // full-server backup download itself. + // Backup and migration: the page with both actions (architecture.md § + // Persistence-B), and the full-server backup download itself. authed.HandleFunc("GET /backup", s.handleBackupPage) authed.HandleFunc("POST /backup", s.handleBackup) - // Monitoring screens (spec 7.2.11-13): each page and its HTMX polling - // fragment (spec 7.1 — the /rows and /body endpoints return HTML, not JSON). + // Monitoring screens (architecture.md § Panel HTTP surface): each page and + // its HTMX polling fragment (architecture.md § Panel HTTP surface — the /rows + // and /body endpoints return HTML, not JSON). authed.HandleFunc("GET /deliveries", s.handleDeliveries) authed.HandleFunc("GET /deliveries/rows", s.handleDeliveriesRows) authed.HandleFunc("GET /mail-queue", s.handleMailQueue) @@ -195,18 +201,22 @@ func (s *Server) Handler() http.Handler { mux.Handle("/", s.requireAuth(authed)) // Security headers and the origin check wrap everything, including the - // unauthenticated login and setup routes (phase 14.A). + // unauthenticated login and setup routes. return s.secure(mux) } // redirectToStatus points the panel root at the status page, so there is one -// canonical URL for that content instead of two (phase 13.C). +// canonical URL for that content instead of two. func redirectToStatus(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/status", http.StatusSeeOther) } func handleHealth(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") + if err := health.Liveness(); err != nil { + http.Error(w, "unhealthy\n", http.StatusServiceUnavailable) + return + } w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("ok\n")) } diff --git a/test/e2e/go.mod b/test/e2e/go.mod index e88d05d..152a607 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -1,4 +1,4 @@ -module codeberg.org/mix/selfpost/test/e2e +module github.com/mixeme/selfpost/test/e2e go 1.26 diff --git a/test/e2e/negative_test.go b/test/e2e/negative_test.go index c828be9..e44a283 100644 --- a/test/e2e/negative_test.go +++ b/test/e2e/negative_test.go @@ -68,8 +68,8 @@ func testLevel2RateLimit(t *testing.T, sc *scenario) { } // testSenderLoginMismatch is plan C.4 negative check 2: an authenticated -// application cannot send as a sender it does not own (spec 5.1 p.3, -// reject_sender_login_mismatch) — the core anti-spoofing control. +// application cannot send as a sender it does not own (architecture.md § Mail +// path, reject_sender_login_mismatch) — the core anti-spoofing control. func testSenderLoginMismatch(t *testing.T, sc *scenario) { res := attemptSend(sendAttempt{ authLogin: sc.appLogin, authPassword: sc.appPassword, @@ -118,10 +118,10 @@ func testForeignRelayRejected(t *testing.T, sc *scenario) { } // testJournalMilterFailOpen is plan C.4 negative check 6: the journal-milter -// is monitoring-only and fails open (spec 7.3) — stopping the panel process -// (which owns the milter socket) must not block mail, and must not crash the -// container (crashexit only fires on PROCESS_STATE_FATAL, a clean supervisor -// stop is STOPPED, see build/crashexit.py). +// is monitoring-only and fails open (architecture.md § Mail path) — stopping +// the panel process (which owns the milter socket) must not block mail, and +// must not crash the container (crashexit only fires on PROCESS_STATE_FATAL, a +// clean supervisor stop is STOPPED, see build/crashexit.py). func testJournalMilterFailOpen(t *testing.T, sc *scenario) { if _, err := h.execIn("selfpost", "supervisorctl", "-c", "/etc/supervisor/supervisord.conf", "stop", "panel"); err != nil { t.Fatalf("stop panel: %v", err) @@ -196,12 +196,12 @@ func testSessionSurvivesRestart(t *testing.T, sc *scenario) { } // testLevel1RateLimit is plan C.4 negative check 4: the native Postfix anvil -// backstop (smtpd_client_message_rate_limit, spec 5 p.5), set by the override -// to RATE_LIMIT_MESSAGES_PER_IP=50, rejects once exceeded. It uses a dedicated -// application with no level-2 limit of its own, and retries well past that -// count, so the result is unambiguous regardless of how much of the shared -// per-IP budget earlier subtests already spent (they stay well under 50 -// between them). +// backstop (smtpd_client_message_rate_limit, README § Rate limiting), set by +// the override to RATE_LIMIT_MESSAGES_PER_IP=50, rejects once exceeded. It +// uses a dedicated application with no level-2 limit of its own, and retries +// well past that count, so the result is unambiguous regardless of how much of +// the shared per-IP budget earlier subtests already spent (they stay well +// under 50 between them). func testLevel1RateLimit(t *testing.T, sc *scenario) { login, password, err := sc.panel.addApplication(sc.domainID, "l1app", "wildcard", "") if err != nil { diff --git a/test/e2e/panel_client.go b/test/e2e/panel_client.go index f536678..066a46f 100644 --- a/test/e2e/panel_client.go +++ b/test/e2e/panel_client.go @@ -145,7 +145,7 @@ func (c *panelClient) dkimRecord(domainID string) (name, value string, err error } // addApplication submits the add-application form and returns the one-shot -// login/password the panel renders inline (spec 7.6.1 — never recoverable +// login/password the panel renders inline (security.md — never recoverable // later, so this is the only place to read it). func (c *panelClient) addApplication(domainID, login, mode, addresses string) (appLogin, password string, err error) { resp, body, err := c.postForm("/domains/"+domainID+"/applications", url.Values{ @@ -170,8 +170,9 @@ func (c *panelClient) addApplication(domainID, login, mode, addresses string) (a return appLogin, password, nil } -// setRateLimit saves a level-2 differentiated limit (spec 7.4) on either a -// domain (/domains/{id}/ratelimit) or an application (/applications/{id}/ratelimit). +// setRateLimit saves a level-2 differentiated limit (README § Rate limiting) +// on either a domain (/domains/{id}/ratelimit) or an application +// (/applications/{id}/ratelimit). func (c *panelClient) setRateLimit(path, allowedIP string, maxMessages, windowSeconds int) error { resp, body, err := c.postForm(path, url.Values{ "allowed_ips": {allowedIP},