From 41c3e6e896b1017b09e727c256aac5a5969c275f Mon Sep 17 00:00:00 2001 From: Mikhail Yenuchenko Date: Mon, 17 Aug 2026 14:36:55 +0300 Subject: [PATCH] Persist Postfix queue and ship self-contained full backups. Move the mail queue under /data so recreate no longer drops deferred mail, and archive data/, compose, .env, and certs/ together for restore on a fresh host. Co-authored-by: Cursor --- CHANGELOG.md | 14 ++ build/entrypoint.sh | 19 ++- build/postfix-config.sh | 6 +- cmd/panel/envdoc_test.go | 4 + cmd/panel/httpserver.go | 1 + cmd/panel/main.go | 2 + cmd/panel/restore_test.go | 86 ++++++++---- cmd/selfpost-backup/main.go | 16 ++- cmd/selfpost-backup/main_test.go | 39 +++++- deploy/caddy/docker-compose.caddy.yml | 1 + deploy/docker-compose.yml | 3 + deploy/nginx/docker-compose.nginx.yml | 1 + deploy/traefik/docker-compose.traefik.yml | 1 + docs/architecture.md | 27 ++-- docs/assets/panel-ui/backup.html | 2 +- docs/guide.md | 77 +++++----- docs/security.md | 9 +- internal/backup/backup.go | 162 ++++++++++++++++------ internal/backup/backup_test.go | 56 ++++++-- internal/web/handlers/handlers.go | 1 + internal/web/handlers/handlers_backup.go | 24 +++- internal/web/view/templates/backup.html | 14 +- internal/web/web.go | 13 +- 23 files changed, 408 insertions(+), 170 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8f112d..69a08f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); version ### Added +- Postfix queue under `/data/postfix/queue` — deferred and active mail survive + container recreate and are included in full backups. +- Full backup archives the whole operator project: `data/`, `docker-compose.yml`, + `.env`, and `certs/` (requires `.:/selfpost-deploy:ro` in compose). Restore + by unpacking into an empty project directory. + +### Changed + +- **Breaking:** full-backup archive layout — paths are prefixed with `data/`; + deploy files sit at the archive root. Old flat archives restore with + `tar xzf backup.tar.gz -C ./data` as before. +- docs: operator guide, architecture, security, and backup UI updated for + self-contained backups and persistent queue. + - docs: HTML mockups for a full panel UI refresh ([docs/assets/panel-ui/](docs/assets/panel-ui/index.html)) — current screens, agreed and candidate roadmap surfaces (queue-retries, inbound-relay, diff --git a/build/entrypoint.sh b/build/entrypoint.sh index c24179f..eb5674e 100644 --- a/build/entrypoint.sh +++ b/build/entrypoint.sh @@ -69,7 +69,7 @@ chmod 755 /data # later phase deliberately hands to another service. /data/log is exempt: it is # deliberately owned by postfix (postlogd writes the delivery log there) and is # normalised on its own below. -find /data -mindepth 1 -maxdepth 1 ! -user panel ! -name log -exec chown -R panel:panel {} + +find /data -mindepth 1 -maxdepth 1 ! -user panel ! -name log ! -name postfix -exec chown -R panel:panel {} + # DKIM key tree (spec 6, 9). The panel (user `panel`) generates keys and writes # the OpenDKIM tables; OpenDKIM (user `opendkim`) must read them. Normalise the @@ -99,13 +99,13 @@ chown -R panel:selfpost /data/sasl chmod 2750 /data/sasl [ -e /data/sasl/sasldb2 ] && chmod 0640 /data/sasl/sasldb2 -# Postfix sender_login_maps (spec 5.1). The panel writes it; Postfix reads it. -# Ensure the file exists (empty is fine) before Postfix starts so a reload that -# references it never fails on a missing file, and keep it group-readable. -mkdir -p /data/postfix +# Postfix state under /data (spec 5.1, architecture.md § Persistence). The panel +# writes sender_login_maps; Postfix owns the on-disk queue tree under queue/. +mkdir -p /data/postfix/queue [ -e /data/postfix/sender_login_maps ] || : > /data/postfix/sender_login_maps -chown -R panel:selfpost /data/postfix +chown panel:selfpost /data/postfix chmod 2750 /data/postfix +chown panel:selfpost /data/postfix/sender_login_maps chmod 0640 /data/postfix/sender_login_maps # Delivery log (architecture.md § Log tailer). postlogd writes it as user @@ -144,4 +144,11 @@ chmod 2750 /run/opendkim /run/selfpost # start the same way the /data normalisation above is. /usr/local/bin/postfix-config.sh +# Initialise the persistent queue tree on first start or after restore. postfix +# set-permissions reads queue_directory from main.cf (set by postfix-config.sh). +if [ ! -d /data/postfix/queue/active ]; then + postfix set-permissions +fi +chown -R postfix:postfix /data/postfix/queue + exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf diff --git a/build/postfix-config.sh b/build/postfix-config.sh index a595c96..d596015 100644 --- a/build/postfix-config.sh +++ b/build/postfix-config.sh @@ -57,6 +57,9 @@ JOURNAL_SOCK="${JOURNAL_MILTER_SOCKET:-/run/selfpost/journal.sock}" # Persistent panel-managed sender map (spec 5.1); texthash needs no postmap, so # the unprivileged panel can rewrite it and just ask for a reload. SENDER_LOGIN_MAPS="${POSTFIX_SENDER_LOGIN_MAPS:-/data/postfix/sender_login_maps}" +# Transit mail queue under /data so deferred/active messages survive container +# recreate (architecture.md § Persistence). Distinct from sender_login_maps. +QUEUE_DIR="${POSTFIX_QUEUE_DIR:-/data/postfix/queue}" SASLDB_PATH="${SASL_DB_PATH:-/data/sasl/sasldb2}" # Optional submission service on 587 (spec 5 p.1: off by default, enabled only @@ -128,7 +131,8 @@ postconf -e \ # resolves the full address first, then the "@domain" wildcard, so both address # modes work from the same map. postconf -e \ - "smtpd_sender_login_maps=texthash:${SENDER_LOGIN_MAPS}" + "smtpd_sender_login_maps=texthash:${SENDER_LOGIN_MAPS}" \ + "queue_directory=${QUEUE_DIR}" # Restrictions: authenticated clients only, no relay to foreign destinations, # and every authenticated sender address must be owned by its login. NO diff --git a/cmd/panel/envdoc_test.go b/cmd/panel/envdoc_test.go index 64368d2..edccb8c 100644 --- a/cmd/panel/envdoc_test.go +++ b/cmd/panel/envdoc_test.go @@ -33,6 +33,8 @@ var documentedInternal = []string{ "SASL_REALM", "POSTFIX_DIR", "POSTFIX_SENDER_LOGIN_MAPS", + "POSTFIX_QUEUE_DIR", + "SELFPOST_DEPLOY_ROOT", "MILTER_CONNECT_TIMEOUT", "MILTER_COMMAND_TIMEOUT", "MILTER_CONTENT_TIMEOUT", @@ -70,6 +72,7 @@ var loadConfigKeys = []string{ "SASL_DB_PATH", "SASL_REALM", "POSTFIX_DIR", + "SELFPOST_DEPLOY_ROOT", } // buildScriptKeys is every ${VAR:-…} / os.Getenv used in build/*.sh and entrypoint.sh @@ -84,6 +87,7 @@ var buildScriptKeys = []string{ "JOURNAL_MILTER_SOCKET", "MAIL_LOG", "POSTFIX_SENDER_LOGIN_MAPS", + "POSTFIX_QUEUE_DIR", "SASL_DB_PATH", "SUBMISSION_ENABLE", "MILTER_CONNECT_TIMEOUT", diff --git a/cmd/panel/httpserver.go b/cmd/panel/httpserver.go index 0449df9..6f9a18b 100644 --- a/cmd/panel/httpserver.go +++ b/cmd/panel/httpserver.go @@ -73,6 +73,7 @@ func newPanel(cfg config, st *store.Store) (*web.Server, error) { MailLogPath: cfg.mailLog, DataDir: cfg.dataDir, DBPath: cfg.dbPath, + DeployRoot: cfg.deployRoot, Version: buildinfo.Version, TrustedProxyCIDRs: cfg.trustedProxies, TLSCertFile: cfg.tlsCertFile, diff --git a/cmd/panel/main.go b/cmd/panel/main.go index 1e0e305..065670a 100644 --- a/cmd/panel/main.go +++ b/cmd/panel/main.go @@ -78,6 +78,7 @@ type config struct { saslDBPath string saslRealm string postfixDir string + deployRoot string } func loadConfig() config { @@ -138,6 +139,7 @@ func loadConfig() config { saslDBPath: envDefault("SASL_DB_PATH", filepath.Join(dataDir, "sasl", "sasldb2")), saslRealm: saslRealm(), postfixDir: envDefault("POSTFIX_DIR", filepath.Join(dataDir, "postfix")), + deployRoot: envDefault("SELFPOST_DEPLOY_ROOT", "/selfpost-deploy"), } } diff --git a/cmd/panel/restore_test.go b/cmd/panel/restore_test.go index 3097518..5575d1b 100644 --- a/cmd/panel/restore_test.go +++ b/cmd/panel/restore_test.go @@ -22,7 +22,7 @@ import ( ) // Restoring a SelfPost backup is not a code path in the panel: the operator -// extracts the archive into the /data bind mount and starts the image, and the +// extracts the archive into a project directory and starts the image, and the // panel is expected to come up on it (architecture.md § Persistence). Nothing // below stubs that story out — the archive is downloaded from a running panel // through /backup, unpacked the way `tar -xzf` unpacks it, and a second panel @@ -37,9 +37,10 @@ const ( // restored is the outcome of a full backup-and-restore round trip. type restored struct { - panel http.Handler // panel booted on the restored data directory - dataDir string // the restored /data - session *http.Cookie // a session opened before the backup was taken + panel http.Handler // panel booted on the restored data directory + deployRoot string // the restored project directory + dataDir string // the restored /data + session *http.Cookie // a session opened before the backup was taken } // restoreFromOwnBackup runs the operator's path end to end: seed a panel that @@ -50,7 +51,7 @@ type restored struct { func restoreFromOwnBackup(t *testing.T, password string) restored { t.Helper() - live := seedPanelData(t) + live := seedPanelProject(t) panel := bootPanel(t, live) session := signIn(t, panel) archive := downloadBackup(t, panel, session, password) @@ -58,7 +59,12 @@ func restoreFromOwnBackup(t *testing.T, password string) restored { target := t.TempDir() extract(t, archive, target) - return restored{panel: bootPanel(t, target), dataDir: target, session: session} + return restored{ + panel: bootPanel(t, target), + deployRoot: target, + dataDir: filepath.Join(target, "data"), + session: session, + } } // The panel has to come up on the restored directory and show the state that @@ -91,6 +97,12 @@ func TestPanelBootsOnADataDirectoryRestoredFromItsOwnBackup(t *testing.T) { t.Errorf("%s = %q, want %q", path, got, want) } } + + for _, name := range []string{backup.ComposeFileName, backup.EnvFileName} { + if _, err := os.Stat(filepath.Join(r.deployRoot, name)); err != nil { + t.Errorf("the restored project directory has no %s: %v", name, err) + } + } } // The one-time setup link is closed by the presence of a panel user, and the @@ -138,8 +150,9 @@ func TestAnEncryptedBackupRestoresTheSameWay(t *testing.T) { // A restore boot runs one Resync from SQLite. If the archive's Postfix map // drifted from the database, that step puts it back before mail flows. func TestResyncAfterRestoreHealsDriftedMaps(t *testing.T) { - dataDir := seedPanelData(t) - cfg := panelConfig(t, dataDir) + deployRoot := seedPanelProject(t) + dataDir := filepath.Join(deployRoot, "data") + cfg := panelConfig(t, deployRoot) mapPath := filepath.Join(dataDir, "postfix", "sender_login_maps") if err := os.WriteFile(mapPath, []byte("stale map\n"), 0o640); err != nil { @@ -191,12 +204,14 @@ func TestResyncAfterRestoreHealsDriftedMaps(t *testing.T) { // stays put on a mismatch: the operator's next move is to start the image the // backup names, and it has to be there when they do. func TestPanelRefusesADataDirectoryRestoredFromAnotherVersion(t *testing.T) { - live := seedPanelData(t) + deployRoot := seedPanelProject(t) + dataDir := filepath.Join(deployRoot, "data") var archive bytes.Buffer if err := backup.Create(&archive, backup.Params{ - DataDir: live, - DBPath: filepath.Join(live, "selfpost.db"), - Version: "9.9.9", + DataDir: dataDir, + DBPath: filepath.Join(dataDir, "selfpost.db"), + Version: "9.9.9", + DeployRoot: deployRoot, }); err != nil { t.Fatalf("create backup: %v", err) } @@ -218,13 +233,15 @@ func TestPanelRefusesADataDirectoryRestoredFromAnotherVersion(t *testing.T) { } } -// seedPanelData builds the /data tree of a panel that has been in use: an -// administrator, a sending domain with an application and one logged message, -// and the daemon state the mail path needs (a DKIM key, the SASL database and -// Postfix's sender map). -func seedPanelData(t *testing.T) string { +// seedPanelProject builds an operator project tree: data/ with a panel that has +// been in use, plus docker-compose.yml, .env, and certs/ for full backups. +func seedPanelProject(t *testing.T) string { t.Helper() - dataDir := t.TempDir() + deployRoot := t.TempDir() + dataDir := filepath.Join(deployRoot, "data") + if err := os.MkdirAll(dataDir, 0o750); err != nil { + t.Fatalf("mkdir data: %v", err) + } st, err := store.Open(filepath.Join(dataDir, "selfpost.db")) if err != nil { @@ -268,16 +285,33 @@ func seedPanelData(t *testing.T) string { t.Fatalf("write %s: %v", full, err) } } - return dataDir + + writeDeployFile(t, filepath.Join(deployRoot, backup.ComposeFileName), "services:\n selfpost:\n image: test\n") + writeDeployFile(t, filepath.Join(deployRoot, backup.EnvFileName), "SELFPOST_HOSTNAME=mail.example.ru\n") + writeDeployFile(t, filepath.Join(deployRoot, backup.CertsDirName, "fullchain.pem"), "CERT") + writeDeployFile(t, filepath.Join(deployRoot, backup.CertsDirName, "privkey.pem"), "KEY") + return deployRoot } -// panelConfig resolves the panel's own configuration for a data directory, so +func writeDeployFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } + if err := os.WriteFile(path, []byte(content), 0o640); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +// panelConfig resolves the panel's own configuration for a project directory, so // the test finds the files where the running binary would look for them rather // than where it put them. Cookies are marked insecure for the same reason the // e2e stand does it: the test client speaks plain HTTP. -func panelConfig(t *testing.T, dataDir string) config { +func panelConfig(t *testing.T, deployRoot string) config { t.Helper() + dataDir := filepath.Join(deployRoot, "data") t.Setenv("SELFPOST_DATA_DIR", dataDir) + t.Setenv("SELFPOST_DEPLOY_ROOT", deployRoot) t.Setenv("PANEL_COOKIE_SECURE", "false") t.Setenv("SELFPOST_HOSTNAME", "mail.example.ru") // MAIL_LOG's default is an absolute path, not one derived from the data @@ -288,13 +322,13 @@ func panelConfig(t *testing.T, dataDir string) config { // bootPanel performs the startup sequence run() performs, in the same order, // and returns the panel's HTTP handler. -func bootPanel(t *testing.T, dataDir string) http.Handler { +func bootPanel(t *testing.T, deployRoot string) http.Handler { t.Helper() - cfg := panelConfig(t, dataDir) + cfg := panelConfig(t, deployRoot) restored, err := backup.CheckRestore(cfg.manifestPath, buildinfo.Version) if err != nil { - t.Fatalf("the panel refused to start on %s: %v", dataDir, err) + t.Fatalf("the panel refused to start on %s: %v", deployRoot, err) } if restored { if _, err := os.Stat(cfg.manifestPath); err == nil { @@ -380,8 +414,8 @@ func downloadBackup(t *testing.T, h http.Handler, session *http.Cookie, password return plain } -// extract unpacks a backup archive into dir, as `tar -xzf` does onto the /data -// bind mount before the image is started. +// extract unpacks a backup archive into dir, as `tar -xzf` does onto the project +// directory before the image is started. func extract(t *testing.T, archive []byte, dir string) { t.Helper() gz, err := gzip.NewReader(bytes.NewReader(archive)) diff --git a/cmd/selfpost-backup/main.go b/cmd/selfpost-backup/main.go index b1631a0..6e81d3c 100644 --- a/cmd/selfpost-backup/main.go +++ b/cmd/selfpost-backup/main.go @@ -6,9 +6,9 @@ // // docker exec selfpost-backup > selfpost-backup.tar.gz // -// Use -o to write to a file instead. The resulting archive contains DKIM private -// keys, the admin password hash and SASL credentials — treat it as a secret -// (architecture.md § Persistence). +// Use -o to write to a file instead. The resulting archive is self-contained: +// data/, docker-compose.yml, .env, and certs/ when present — treat it as a +// secret (architecture.md § Persistence). // // Given a password (SELFPOST_BACKUP_PASSWORD or -password-file, never an // argument, which would show up in the process list) the archive is written as @@ -87,9 +87,13 @@ func run(outPath, password string) error { } if err := backup.Create(sink, backup.Params{ - DataDir: dataDir, - DBPath: dbPath, - Version: buildinfo.Version, + DataDir: dataDir, + DBPath: dbPath, + Version: buildinfo.Version, + DeployRoot: envDefault("SELFPOST_DEPLOY_ROOT", "/selfpost-deploy"), + OnWarn: func(msg string) { + fmt.Fprintf(os.Stderr, "selfpost-backup: %s\n", msg) + }, }); err != nil { return err } diff --git a/cmd/selfpost-backup/main_test.go b/cmd/selfpost-backup/main_test.go index f6c2cf7..6d9e56c 100644 --- a/cmd/selfpost-backup/main_test.go +++ b/cmd/selfpost-backup/main_test.go @@ -9,13 +9,18 @@ import ( "strings" "testing" + "github.com/mixeme/selfpost/internal/backup" "github.com/mixeme/selfpost/internal/store" ) -// seedDataDir builds the minimum /data tree a backup can be taken from. -func seedDataDir(t *testing.T) string { +// seedProject builds the minimum operator project tree a backup can be taken from. +func seedProject(t *testing.T) (deployRoot, dataDir string) { t.Helper() - dataDir := t.TempDir() + deployRoot = t.TempDir() + dataDir = filepath.Join(deployRoot, "data") + if err := os.MkdirAll(dataDir, 0o750); err != nil { + t.Fatalf("mkdir data: %v", err) + } st, err := store.Open(filepath.Join(dataDir, "selfpost.db")) if err != nil { t.Fatalf("open store: %v", err) @@ -26,16 +31,31 @@ func seedDataDir(t *testing.T) string { if err := st.Close(); err != nil { t.Fatalf("close store: %v", err) } + writeFile(t, filepath.Join(deployRoot, backup.ComposeFileName), "services:\n selfpost:\n image: test\n") + writeFile(t, filepath.Join(deployRoot, backup.EnvFileName), "SELFPOST_HOSTNAME=mail.example.com\n") + writeFile(t, filepath.Join(deployRoot, backup.CertsDirName, "fullchain.pem"), "CERT") + writeFile(t, filepath.Join(deployRoot, backup.CertsDirName, "privkey.pem"), "KEY") t.Setenv("SELFPOST_DATA_DIR", dataDir) t.Setenv("SELFPOST_DB_PATH", filepath.Join(dataDir, "selfpost.db")) - return dataDir + t.Setenv("SELFPOST_DEPLOY_ROOT", deployRoot) + return deployRoot, dataDir +} + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + t.Fatalf("mkdir %s: %v", path, err) + } + if err := os.WriteFile(path, []byte(content), 0o640); err != nil { + t.Fatalf("write %s: %v", path, err) + } } // An encrypted backup is only worth having if the container it came from can // hand it back as an ordinary archive during a restore, so the two halves of // the CLI are tested as the one round trip an operator actually performs. func TestEncryptedBackupRoundTrip(t *testing.T) { - seedDataDir(t) + seedProject(t) dir := t.TempDir() encrypted := filepath.Join(dir, "backup.spbk") plain := filepath.Join(dir, "backup.tar.gz") @@ -81,7 +101,12 @@ func TestEncryptedBackupRoundTrip(t *testing.T) { } names[hdr.Name] = true } - for _, want := range []string{"manifest.json", "selfpost.db"} { + for _, want := range []string{ + backup.DataArchivePrefix + backup.ManifestName, + backup.DataArchivePrefix + "selfpost.db", + backup.ComposeFileName, + backup.EnvFileName, + } { if !names[want] { t.Errorf("decrypted archive has no %s (entries: %v)", want, names) } @@ -91,7 +116,7 @@ func TestEncryptedBackupRoundTrip(t *testing.T) { // Without a password the CLI keeps producing the plain archive that existing // backup scripts consume. func TestUnencryptedBackupStaysPlain(t *testing.T) { - seedDataDir(t) + seedProject(t) out := filepath.Join(t.TempDir(), "backup.tar.gz") if err := run(out, ""); err != nil { t.Fatalf("create backup: %v", err) diff --git a/deploy/caddy/docker-compose.caddy.yml b/deploy/caddy/docker-compose.caddy.yml index 50c8088..fd0b5ed 100644 --- a/deploy/caddy/docker-compose.caddy.yml +++ b/deploy/caddy/docker-compose.caddy.yml @@ -31,6 +31,7 @@ services: TLS_KEY_FILE: /etc/postfix/tls/.key volumes: !override - ./data:/data + - .:/selfpost-deploy:ro # Replace with your actual mail/panel hostname, matching # the Caddyfile below and SELFPOST_HOSTNAME. - ./caddy/caddy-data/caddy/certificates/acme-v02.api.letsencrypt.org-directory/:/etc/postfix/tls:ro diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index b2285d1..e35271d 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -58,6 +58,9 @@ services: # kept) so the delivery lines that resolve a queued send-log row survive # a container recreate; it is diagnostics and is left out of backups. - ./data:/data + # Read-only project root for full backups: docker-compose.yml, .env, and + # certs/ are archived alongside /data (architecture.md § Persistence). + - .:/selfpost-deploy:ro # Read-only: SelfPost only ever reads certificates, never manages them # (spec 10 p.2). Point this at wherever your reverse-proxy/certbot # writes PEM files, e.g. /etc/letsencrypt on the host. diff --git a/deploy/nginx/docker-compose.nginx.yml b/deploy/nginx/docker-compose.nginx.yml index 83e7892..bb57d86 100644 --- a/deploy/nginx/docker-compose.nginx.yml +++ b/deploy/nginx/docker-compose.nginx.yml @@ -27,6 +27,7 @@ services: # network at selfpost:8080 instead (see nginx.conf.example). volumes: !override - ./data:/data + - .:/selfpost-deploy:ro # Same host directory certbot below writes into — plain bind mount, # no named volume, so the PEM files are as directly inspectable as in # the Apache scenario (spec 10.3). diff --git a/deploy/traefik/docker-compose.traefik.yml b/deploy/traefik/docker-compose.traefik.yml index 79edc9c..fa4de14 100644 --- a/deploy/traefik/docker-compose.traefik.yml +++ b/deploy/traefik/docker-compose.traefik.yml @@ -17,6 +17,7 @@ services: - "587:587" volumes: !override - ./data:/data + - .:/selfpost-deploy:ro # Populated by extract-cert.sh from traefik's acme.json — see below. - ./traefik/extracted-certs:/etc/postfix/tls:ro labels: diff --git a/docs/architecture.md b/docs/architecture.md index 018e2ca..fc141fc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -291,23 +291,27 @@ single-connection trade-off that follows from it. | `opendkim/` | DKIM keys + tables | | `sasl/sasldb2` | Application SASL credentials | | `postfix/sender_login_maps` | Login → From binding | +| `postfix/queue/` | Postfix transit mail (deferred/active); survives container recreate | | `log/mail.log` | Postfix delivery log + rotated copies (excluded from backups) | | `manifest.json` | Backup version stamp (consumed on restore) | -Not in `/data`: TLS certificates (reverse-proxy mount), Postfix queue -(transit mail not migrated by design). +Not in `/data`: TLS certificates for the panel (reverse-proxy mount) — though +full backups also archive the operator's `./certs` PEM files when present. **Rotation:** send-log retention `SEND_LOG_RETENTION_DAYS` (default 90); `mail.log` via logrotate (14 rotated files, check every 6h, rename + `postfix reload` in `postrotate` — see § Log tailer above). -**Restore:** panel button or `selfpost-backup` CLI — SQLite snapshot + tar of -`/data` tree, minus `log/`, the setup token and any `tls/`; version check on -restore. On the first successful boot after restore, the panel runs one -**Resync** — OpenDKIM's tables and Postfix's sender map are re-derived from -SQLite and both daemons are reloaded, so drift between the extracted archive -and the database is healed before mail flows (same step as `POST /reload` on -demand). Stopped-container `tar` of `./data` is safe (see guide). +**Restore:** panel button or `selfpost-backup` CLI — self-contained archive: +`data/` (SQLite snapshot + tree minus `log/`, the setup token and any `tls/` +under `/data`), `docker-compose.yml`, `.env`, and `certs/` when present; +version check on restore. Requires the project directory mounted read-only at +`SELFPOST_DEPLOY_ROOT` (`/selfpost-deploy` in the default compose file). On the +first successful boot after restore, the panel runs one **Resync** — OpenDKIM's +tables and Postfix's sender map are re-derived from SQLite and both daemons are +reloaded, so drift between the extracted archive and the database is healed +before mail flows (same step as `POST /reload` on demand). Stopped-container +`tar` of `./data` alone remains possible for state-only copies (see guide). **Optional encryption** of the two secret-bearing downloads ([internal/secretfile](../internal/secretfile/secretfile.go)): password → @@ -350,7 +354,10 @@ unsupported rather than as a missing doc: `POSTFIX_DIR` (`/data/postfix`), `POSTFIX_SENDER_LOGIN_MAPS` (`/data/postfix/sender_login_maps` — read by Postfix config only; the panel always writes `/sender_login_maps`, so overriding this env alone - desyncs the map Postfix reads from the file the panel maintains). + desyncs the map Postfix reads from the file the panel maintains), + `POSTFIX_QUEUE_DIR` (`/data/postfix/queue` — set in `build/postfix-config.sh`), + `SELFPOST_DEPLOY_ROOT` (`/selfpost-deploy` — operator project directory for + full backups; mount `.:/selfpost-deploy:ro` in compose). - **Milter and Postfix startup:** `MILTER_CONNECT_TIMEOUT` (`15s`), `MILTER_COMMAND_TIMEOUT` (`15s`), `MILTER_CONTENT_TIMEOUT` (`30s`), `MILTER_WAIT_TIMEOUT` (`30` seconds). diff --git a/docs/assets/panel-ui/backup.html b/docs/assets/panel-ui/backup.html index 13c12c5..f83e70b 100644 --- a/docs/assets/panel-ui/backup.html +++ b/docs/assets/panel-ui/backup.html @@ -14,7 +14,7 @@

Full backup

-

Download a full backup of all persistent state — the database, every domain’s DKIM key and the application credentials. Restore into a container of the same SelfPost version, with the same data mount, before first start. TLS certificates and the mail queue are not included.

+

Download a self-contained backup — data/ (database, DKIM keys, credentials, Postfix queue), docker-compose.yml, .env, and certs/. Extract into an empty project directory on a new machine, adjust hostname if needed, and start the same SelfPost version before first boot. The reverse-proxy vhost is not included.

The backup file is a secret. Encrypting it is the simplest way to store it: the download is then a .spbk that only the password opens.

diff --git a/docs/guide.md b/docs/guide.md index 9923a4f..f4d3761 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -417,25 +417,31 @@ journal-milter (level 2) is down. There is no per-IP bypass. ### Full backup and restore -**Full backup** (whole `/data` except `log/`: SQLite, all domains' DKIM keys, -all applications' SASL credentials, `manifest.json` with the version that -created it): panel button (*Backup* → *Full backup*), or from the host: +**Full backup** is a self-contained project archive: `data/` (SQLite, all +domains' DKIM keys, all applications' SASL credentials, the Postfix queue, +`manifest.json` with the version that created it), plus `docker-compose.yml`, +`.env`, and `certs/` from the operator directory next to `./data`. Delivery +logs under `data/log/` are excluded. The base compose file mounts the project +directory read-only at `/selfpost-deploy` so the panel and CLI can read those +deploy files — without that mount, *Full backup* refuses with an error. + +Take a backup from the panel (*Backup* → *Full backup*) or from the host: ```sh docker exec selfpost-backup > selfpost-backup.tar.gz ``` -**Restore** means unpacking that archive into a fresh `/data` bind mount and -starting a container of the **exact same image version** that created it — -SelfPost refuses to start otherwise and tells you which tag to use. On the -first successful start after restore, `manifest.json` from the archive is -**deleted** — it guards only that one boot, so a later in-place upgrade is -not blocked. On that same first boot the panel also runs one **Resync** — -OpenDKIM's tables and Postfix's sender map are re-derived from SQLite and both -daemons are reloaded, healing any drift between the extracted files and the -database (the Status page's *Reload configuration* button runs the same step -on demand). This is why the compose file pins a fixed tag rather than -`:latest`: without a known version, there'd be no way to tell which image +**Restore** means unpacking that archive into an **empty project directory** +(not into `./data` alone) and starting a container of the **exact same image +version** that created it — SelfPost refuses to start otherwise and tells you +which tag to use. On the first successful start after restore, `data/manifest.json` +from the archive is **deleted** — it guards only that one boot, so a later +in-place upgrade is not blocked. On that same first boot the panel also runs +one **Resync** — OpenDKIM's tables and Postfix's sender map are re-derived from +SQLite and both daemons are reloaded, healing any drift between the extracted +files and the database (the Status page's *Reload configuration* button runs +the same step on demand). This is why the compose file pins a fixed tag rather +than `:latest`: without a known version, there'd be no way to tell which image restoring a given backup actually requires (see [Fixed image tag](#fixed-image-tag)). @@ -446,16 +452,16 @@ back after a bad change): # 1. Stop the instance being replaced docker compose down -# 2. Move the current /data aside rather than deleting it, start from empty -mv ./data ./data.before-restore -mkdir ./data +# 2. Move the current project aside rather than deleting it +mv . ../selfpost.before-restore +mkdir selfpost && cd selfpost # 3. Unpack the backup into the fresh directory -tar xzf selfpost-backup.tar.gz -C ./data +tar xzf ../selfpost-backup.tar.gz -# 4. docker-compose.yml must already pin the exact tag the backup was made -# with — check the archive's manifest if unsure: -tar xzf selfpost-backup.tar.gz -O manifest.json +# 4. docker-compose.yml in the archive must pin the exact tag the backup was +# made with — check if unsure: +tar xzf ../selfpost-backup.tar.gz -O data/manifest.json # 5. Start it and watch the boot docker compose up -d @@ -473,28 +479,31 @@ Fix the tag in `docker-compose.yml`, `docker compose pull && docker compose up -d` again — the manifest is still there because the failed boot never got to delete it. -**Moving to a different host** is the same five steps, just starting cold: -bring the compose files and the correct pinned image tag to the new host -(step 1 of [Full deployment](#full-deployment)), put the backup archive in -place of step 3 above, then redo the reverse-proxy/TLS and DNS steps of a -[Full deployment](#full-deployment) — the PTR record and the certificate both -belong to the old IP/host and have to be reissued for the new one; nothing in -the backup carries them. +**Moving to a different host** is the same flow: create an empty project +directory, unpack the backup there, edit `.env` (and `docker-compose.yml` if +needed) for the new hostname or proxy, then `docker compose up -d`. The archive +carries `certs/` from the old host — re-issue certificates when the hostname or +IP changes. Set up the reverse-proxy vhost separately (not in the backup). **Restoring an encrypted (`.spbk`) backup** needs a running container to decrypt it first — any container with the `selfpost-backup` CLI works; decryption does not read `/data` and performs no version check. Start one normally -(step 5, but on an empty `/data` you haven't unpacked yet), then: +(step 5, but on an empty project you have not unpacked yet), then: ```sh docker exec -i selfpost-backup -decrypt < backup.spbk > selfpost-backup.tar.gz ``` -Stop it, wipe `/data` again, and continue from step 2 above with the -resulting `.tar.gz` — see [Encrypting a backup or +Stop it, wipe the project directory again, and continue from step 2 above with +the resulting `.tar.gz` — see [Encrypting a backup or export](#encrypting-a-backup-or-export) for the decrypt command's password options. +**Archives from older SelfPost versions** (flat layout: `manifest.json` and +`selfpost.db` at the archive root, no `data/` prefix, no deploy files) restore +with the previous procedure: `tar xzf backup.tar.gz -C ./data` into a project +that already has `docker-compose.yml` and `.env`. + Restoring an archive taken **before** a session row was removed can bring that session back: session rows travel with the backup, and a browser that still holds the matching cookie is signed in again on the next request if the @@ -520,9 +529,9 @@ domain](#exporting-and-importing-a-single-domain) — a different, domain-scoped operation that also lives on the *Backup* page (`/backup`). Both a full backup and a domain export are **secrets** — they contain the -admin password hash (full backup) or working application credentials (domain -export) in the clear or in directly reversible form. Treat them like any -other credential material: restrict who can read them, don't email them +admin password hash (full backup), TLS private keys and `.env` (full backup), +or working application credentials (domain export) in the clear or in +directly reversible form. Treat them like any other credential material: restrict who can read them, don't email them around — and encrypt them, which SelfPost can do for you. #### Encrypting a backup or export diff --git a/docs/security.md b/docs/security.md index 64c2305..32e8bbf 100644 --- a/docs/security.md +++ b/docs/security.md @@ -86,10 +86,11 @@ The panel is exposed to the internet — the items below are **not optional**. ### Backup and domain export -- Both files are secrets: a full backup carries DKIM keys, `sasldb2`, and the - administrator's password hash; a domain export carries the DKIM key and - **working** application passwords in the clear (otherwise a transfer without - recreating credentials would be impossible). +- Both files are secrets: a full backup carries DKIM keys, `sasldb2`, the + administrator's password hash, `docker-compose.yml`, `.env`, and the TLS + private key from `certs/` when present; a domain export carries the DKIM key + and **working** application passwords in the clear (otherwise a transfer + without recreating credentials would be impossible). - Both downloads can be encrypted with a password (a checkbox on the form): scrypt (N=2¹⁵, r=8, p=1) → AES-256-GCM, streamed in 64 KiB chunks, each authenticated with the header, the chunk number, and an end-of-stream flag — diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 10e44af..10e3862 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -1,15 +1,17 @@ // Package backup implements SelfPost's full-server backup and the restore // version guard (architecture.md § Persistence). A full backup is a // gzip-compressed tar of the consolidated persistent state under /data — the -// SQLite database (as a consistent snapshot), the per-domain DKIM keys and the -// SASL database — plus a manifest recording the SelfPost version that produced -// it. TLS certificates (the reverse proxy's responsibility) and the Postfix -// queue are deliberately excluded (architecture.md § Persistence). +// SQLite database (as a consistent snapshot), the per-domain DKIM keys, the +// SASL database, and the Postfix queue — plus docker-compose.yml, .env, and +// certs/ from the operator's deploy directory, and a manifest recording the +// SelfPost version that produced it. Postfix delivery logs under log/ are +// excluded (diagnostics, not state). // // Restore is not a separate code path in the panel: a backup is extracted into -// the /data bind mount before first start, and the archive already carries -// everything the mail path needs — DKIM keys, sasldb2, and Postfix's sender -// map — so the daemons can start on the extracted files alone. The +// the operator's project directory (data/, docker-compose.yml, .env, certs/) +// before first start. The archive carries everything needed to bring the +// instance back — DKIM keys, sasldb2, sender map, queue, and deploy files — +// so the operator only adjusts hostname or proxy settings on a new host. The // restore-specific steps the panel runs are CheckRestore, which refuses to // boot if the manifest's version does not match the running binary so // schema/format skew between versions cannot silently corrupt state @@ -40,11 +42,21 @@ import ( // FormatFull identifies a full-server backup manifest. const FormatFull = "selfpost-full-backup" -// ManifestName is the manifest's filename, both inside the archive and, after a -// restore extraction, at the root of the data directory where CheckRestore -// looks for it. +// ManifestName is the manifest's filename inside the data directory. After +// restore extraction it lives at data/manifest.json under the project root; +// CheckRestore reads it from the /data bind mount. const ManifestName = "manifest.json" +// DataArchivePrefix is the path prefix for every /data entry in the archive. +const DataArchivePrefix = "data/" + +// ComposeFileName and EnvFileName are required deploy files at the archive root. +const ( + ComposeFileName = "docker-compose.yml" + EnvFileName = ".env" + CertsDirName = "certs" +) + // Manifest is the small JSON document embedded in every backup archive. Its // Version is the single fact that makes restore safe: the panel refuses to // boot a data directory whose manifest version does not match its own binary @@ -58,21 +70,23 @@ type Manifest struct { // Params configures a backup. DataDir is the consolidated state root (/data); // DBPath is the live SQLite file within it, snapshotted consistently rather than // copied byte-for-byte while it may be mid-write; Version is stamped into the -// manifest. +// manifest; DeployRoot is the host project directory mounted read-only (holds +// docker-compose.yml, .env, and optionally certs/). OnWarn is called for +// non-fatal issues such as a missing certs/ directory. type Params struct { - DataDir string - DBPath string - Version string + DataDir string + DBPath string + Version string + DeployRoot string + OnWarn func(string) } // excludedFromArchive lists the data-directory entries a backup never carries. // The live database files are replaced by a consistent VACUUM INTO snapshot // written under the canonical name; the setup token is transient bootstrap // state; a stale manifest from a previous restore must not be re-captured (a -// fresh one is written instead); a "tls" directory holds the reverse proxy's -// certificates, which are explicitly out of scope for a SelfPost backup -// (architecture.md § Persistence) — excluding it keeps that guarantee even when -// an operator points TLS_CERT_FILE inside /data; and "log" is Postfix's raw +// fresh one is written instead); a "tls" directory under /data is skipped when +// an operator pointed TLS_CERT_FILE inside /data; and "log" is Postfix's raw // delivery log plus its fourteen rotated files, which is diagnostic output, not // state to restore, and by far the largest thing under /data. var excludedFromArchive = map[string]bool{ @@ -86,16 +100,22 @@ var excludedFromArchive = map[string]bool{ ManifestName: true, } -// Create writes a gzip-compressed tar backup to w. Archive entries are named -// relative to DataDir, so extracting the archive into the /data bind mount -// reconstructs the state in place (architecture.md § Persistence). The SQLite -// database is added as a consistent snapshot under "selfpost.db"; everything -// else under DataDir is copied as-is except the entries in -// excludedFromArchive. +// Create writes a gzip-compressed tar backup to w. Archive layout: +// +// data/manifest.json, data/selfpost.db, data/ +// docker-compose.yml, .env, certs/... +// +// Extract the archive into an empty project directory, then docker compose up. func Create(w io.Writer, p Params) error { if p.DataDir == "" || p.DBPath == "" { return fmt.Errorf("backup: DataDir and DBPath are required") } + if p.DeployRoot == "" { + return fmt.Errorf("backup: DeployRoot is required (mount the project directory at SELFPOST_DEPLOY_ROOT)") + } + if err := validateDeployRoot(p.DeployRoot); err != nil { + return err + } snapshot, cleanup, err := snapshotDB(p.DBPath) if err != nil { @@ -115,17 +135,21 @@ func Create(w io.Writer, p Params) error { if err != nil { return fmt.Errorf("backup: encode manifest: %w", err) } - if err := writeTarBytes(tw, ManifestName, 0o600, manifestJSON); err != nil { + if err := writeTarBytes(tw, DataArchivePrefix+ManifestName, 0o600, manifestJSON); err != nil { return err } // The consistent SQLite snapshot, under the canonical filename the panel // opens on start (the live file and its WAL/SHM are excluded from the walk). - if err := writeTarFile(tw, "selfpost.db", 0o640, snapshot); err != nil { + if err := writeTarFile(tw, DataArchivePrefix+"selfpost.db", 0o640, snapshot); err != nil { return err } - if err := addTree(tw, p.DataDir); err != nil { + if err := addTree(tw, p.DataDir, DataArchivePrefix, excludedFromArchive); err != nil { + return err + } + + if err := addDeployFiles(tw, p.DeployRoot, p.OnWarn); err != nil { return err } @@ -138,33 +162,79 @@ func Create(w io.Writer, p Params) error { return nil } -// addTree walks dataDir and adds every regular file (and directory, to preserve -// empty ones and modes) to tw under its path relative to dataDir, skipping the -// excluded entries. Non-regular, non-directory entries (symlinks, sockets) are -// skipped: /data holds none in normal operation, and copying them into a backup -// would be meaningless or unsafe. -func addTree(tw *tar.Writer, dataDir string) error { - return filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error { +// ValidateDeployRoot checks that the operator project directory is mounted and +// contains the files a full backup requires. Call before streaming a response. +func ValidateDeployRoot(deployRoot string) error { + if deployRoot == "" { + return fmt.Errorf("backup: DeployRoot is required (mount the project directory at SELFPOST_DEPLOY_ROOT)") + } + return validateDeployRoot(deployRoot) +} + +func validateDeployRoot(deployRoot string) error { + for _, name := range []string{ComposeFileName, EnvFileName} { + path := filepath.Join(deployRoot, name) + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("backup: deploy root %q is missing %s: %w", deployRoot, name, err) + } + } + return nil +} + +func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string)) error { + for _, name := range []string{ComposeFileName, EnvFileName} { + src := filepath.Join(deployRoot, name) + info, err := os.Stat(src) + if err != nil { + return fmt.Errorf("backup: stat deploy file %s: %w", name, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("backup: deploy file %s is not a regular file", name) + } + if err := writeTarFile(tw, name, info.Mode().Perm(), src); err != nil { + return err + } + } + + certsDir := filepath.Join(deployRoot, CertsDirName) + if _, err := os.Stat(certsDir); err != nil { + if os.IsNotExist(err) { + if onWarn != nil { + onWarn("certs/ not found in deploy root; backup will not include TLS material") + } + return nil + } + return fmt.Errorf("backup: stat %s: %w", CertsDirName, err) + } + return addTree(tw, certsDir, CertsDirName+"/", nil) +} + +// addTree walks root and adds every regular file (and directory, to preserve +// empty ones and modes) to tw under archivePrefix + path relative to root. +// When exclude is non-nil, top-level names relative to root are skipped. +// Non-regular, non-directory entries (symlinks, sockets) are skipped. +func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool) error { + return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } - rel, err := filepath.Rel(dataDir, path) + rel, err := filepath.Rel(root, path) if err != nil { return err } if rel == "." { - return nil // the data root itself is implicit - } - // filepath.Rel yields OS separators; tar names use forward slashes. - name := filepath.ToSlash(rel) - // Exclude by top-level name (the live DB, setup token and stale manifest - // all live at the data root). - if excludedFromArchive[name] { - if d.IsDir() { - return fs.SkipDir - } return nil } + name := archivePrefix + filepath.ToSlash(rel) + if exclude != nil { + top := strings.Split(filepath.ToSlash(rel), "/")[0] + if exclude[top] { + if d.IsDir() { + return fs.SkipDir + } + return nil + } + } info, err := d.Info() if err != nil { @@ -182,7 +252,7 @@ func addTree(tw *tar.Writer, dataDir string) error { case info.Mode().IsRegular(): return writeTarFile(tw, name, info.Mode().Perm(), path) default: - return nil // skip symlinks/sockets/devices + return nil } }) } diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index f340d0e..0960f76 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -15,10 +15,15 @@ import ( ) // seedDataDir builds a realistic /data tree: a migrated SQLite database plus the -// DKIM key, SASL and transient files a backup must include or exclude. -func seedDataDir(t *testing.T) (dataDir, dbPath string) { +// DKIM key, SASL and transient files a backup must include or exclude. It also +// seeds a deploy root beside data/ with compose, .env, and certs/. +func seedDataDir(t *testing.T) (dataDir, dbPath, deployRoot string) { t.Helper() - dataDir = t.TempDir() + deployRoot = t.TempDir() + dataDir = filepath.Join(deployRoot, "data") + if err := os.MkdirAll(dataDir, 0o750); err != nil { + t.Fatalf("mkdir data: %v", err) + } dbPath = filepath.Join(dataDir, "selfpost.db") st, err := store.Open(dbPath) @@ -35,6 +40,7 @@ func seedDataDir(t *testing.T) (dataDir, dbPath string) { writeFile(t, filepath.Join(dataDir, "opendkim", "keys", "example.com", "selfpost.private"), "PRIVATE KEY") writeFile(t, filepath.Join(dataDir, "sasl", "sasldb2"), "SASLDB") writeFile(t, filepath.Join(dataDir, "postfix", "sender_login_maps"), "@example.com login") + writeFile(t, filepath.Join(dataDir, "postfix", "queue", "deferred", "sample"), "queue-file") // Transient files that must NOT be archived. writeFile(t, filepath.Join(dataDir, "setup-token"), "secret-token") writeFile(t, filepath.Join(dataDir, "selfpost.db-wal"), "wal") @@ -43,7 +49,11 @@ func seedDataDir(t *testing.T) (dataDir, dbPath string) { // state, and the bulkiest thing under /data. writeFile(t, filepath.Join(dataDir, "log", "mail.log"), "Aug 8 07:26:41 mail postfix/smtp[1]: ABC: to=, status=sent (ok)") writeFile(t, filepath.Join(dataDir, "log", "mail.log.1"), "older") - return dataDir, dbPath + writeFile(t, filepath.Join(deployRoot, ComposeFileName), "services:\n selfpost:\n image: test\n") + writeFile(t, filepath.Join(deployRoot, EnvFileName), "SELFPOST_HOSTNAME=mail.example.com\n") + writeFile(t, filepath.Join(deployRoot, CertsDirName, "fullchain.pem"), "CERT") + writeFile(t, filepath.Join(deployRoot, CertsDirName, "privkey.pem"), "KEY") + return dataDir, dbPath, deployRoot } func writeFile(t *testing.T, path, content string) { @@ -87,21 +97,26 @@ func readArchive(t *testing.T, data []byte) map[string]string { } func TestCreateIncludesStateExcludesTransient(t *testing.T) { - dataDir, dbPath := seedDataDir(t) + dataDir, dbPath, deployRoot := seedDataDir(t) var buf bytes.Buffer - if err := Create(&buf, Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3"}); err != nil { + if err := Create(&buf, Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3", DeployRoot: deployRoot}); err != nil { t.Fatalf("Create: %v", err) } files := readArchive(t, buf.Bytes()) // Present. for _, name := range []string{ - ManifestName, - "selfpost.db", - "opendkim/keys/example.com/selfpost.private", - "sasl/sasldb2", - "postfix/sender_login_maps", + DataArchivePrefix + ManifestName, + DataArchivePrefix + "selfpost.db", + DataArchivePrefix + "opendkim/keys/example.com/selfpost.private", + DataArchivePrefix + "sasl/sasldb2", + DataArchivePrefix + "postfix/sender_login_maps", + DataArchivePrefix + "postfix/queue/deferred/sample", + ComposeFileName, + EnvFileName, + CertsDirName + "/fullchain.pem", + CertsDirName + "/privkey.pem", } { if _, ok := files[name]; !ok { t.Errorf("archive missing %s", name) @@ -109,8 +124,11 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) { } // Excluded. for _, name := range []string{ - "setup-token", "selfpost.db-wal", "selfpost.db-shm", - "log/mail.log", "log/mail.log.1", + DataArchivePrefix + "setup-token", + DataArchivePrefix + "selfpost.db-wal", + DataArchivePrefix + "selfpost.db-shm", + DataArchivePrefix + "log/mail.log", + DataArchivePrefix + "log/mail.log.1", } { if _, ok := files[name]; ok { t.Errorf("archive should not contain %s", name) @@ -119,7 +137,7 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) { // Manifest is well-formed and carries the version. var m Manifest - if err := json.Unmarshal([]byte(files[ManifestName]), &m); err != nil { + if err := json.Unmarshal([]byte(files[DataArchivePrefix+ManifestName]), &m); err != nil { t.Fatalf("manifest json: %v", err) } if m.Format != FormatFull || m.Version != "1.2.3" { @@ -128,7 +146,7 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) { // The archived selfpost.db is a real, openable SQLite snapshot with our data. snapPath := filepath.Join(t.TempDir(), "restored.db") - if err := os.WriteFile(snapPath, []byte(files["selfpost.db"]), 0o640); err != nil { + if err := os.WriteFile(snapPath, []byte(files[DataArchivePrefix+"selfpost.db"]), 0o640); err != nil { t.Fatalf("write snapshot: %v", err) } st, err := store.Open(snapPath) @@ -145,6 +163,14 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) { } } +func TestCreateRequiresDeployRoot(t *testing.T) { + dataDir, dbPath, _ := seedDataDir(t) + var buf bytes.Buffer + if err := Create(&buf, Params{DataDir: dataDir, DBPath: dbPath, Version: "1.0.0"}); err == nil { + t.Fatal("Create without DeployRoot succeeded") + } +} + func writeManifest(t *testing.T, dir, format, version string) string { t.Helper() path := filepath.Join(dir, ManifestName) diff --git a/internal/web/handlers/handlers.go b/internal/web/handlers/handlers.go index bb530d7..60305b8 100644 --- a/internal/web/handlers/handlers.go +++ b/internal/web/handlers/handlers.go @@ -20,6 +20,7 @@ type Config struct { MailLogPath string DataDir string DBPath string + DeployRoot string Version string TLSCertFile string OpenDKIMSocket string diff --git a/internal/web/handlers/handlers_backup.go b/internal/web/handlers/handlers_backup.go index b07b83e..e9d0759 100644 --- a/internal/web/handlers/handlers_backup.go +++ b/internal/web/handlers/handlers_backup.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "strings" "time" "github.com/mixeme/selfpost/internal/backup" @@ -70,6 +71,11 @@ func (h *Handlers) HandleBackup(w http.ResponseWriter, r *http.Request) { h.renderBackupPageWith(w, r, http.StatusBadRequest, "", pwErr) return } + if err := backup.ValidateDeployRoot(h.cfg.DeployRoot); err != nil { + logf("panel: full backup: %v", err) + h.renderBackupPageWith(w, r, http.StatusBadRequest, "", deployBackupErr(err)) + return + } stamp := time.Now().UTC().Format("20060102-150405") filename := fmt.Sprintf("selfpost-backup-%s.tar.gz", stamp) @@ -105,9 +111,13 @@ func (h *Handlers) HandleBackup(w http.ResponseWriter, r *http.Request) { } if err := backup.Create(sink, backup.Params{ - DataDir: h.cfg.DataDir, - DBPath: h.cfg.DBPath, - Version: h.cfg.Version, + DataDir: h.cfg.DataDir, + DBPath: h.cfg.DBPath, + Version: h.cfg.Version, + DeployRoot: h.cfg.DeployRoot, + OnWarn: func(msg string) { + logf("panel: full backup: %s", msg) + }, }); err != nil { logf("panel: full backup failed: %v", err) return @@ -310,6 +320,14 @@ func decryptErrorMessage(err error) string { } } +// deployBackupErr phrases a pre-flight backup failure for the operator. +func deployBackupErr(err error) string { + if strings.Contains(err.Error(), "DeployRoot") || strings.Contains(err.Error(), "deploy root") { + return "Full backup needs the project directory mounted read-only at /selfpost-deploy — add .:/selfpost-deploy:ro to docker-compose.yml and recreate the container." + } + return "Could not create the backup: " + err.Error() +} + // importErrorMessage maps a domain-import failure (already logged by the caller) // to an HTTP status and a user-facing message. Duplicate domain/login are called // out specifically; other failures — validation errors describing what is wrong diff --git a/internal/web/view/templates/backup.html b/internal/web/view/templates/backup.html index bf9352a..04645c2 100644 --- a/internal/web/view/templates/backup.html +++ b/internal/web/view/templates/backup.html @@ -3,13 +3,15 @@

Full backup

-

Download a full backup of all persistent state — the database, - every domain's DKIM key and the application credentials. Use it to move the - whole server to a new machine: restore it into a container of the - same SelfPost version, with the same data mount, before first - start. TLS certificates and the mail queue are not included.

+

Download a self-contained backup of the whole instance — + data/ (database, DKIM keys, application credentials, and the + Postfix queue), docker-compose.yml, .env, and + certs/. Extract it into an empty project directory on a new + machine, adjust hostname or proxy settings if needed, and start a container of + the same SelfPost version before first boot. The reverse-proxy + vhost is not included — set that up separately on the new host.

The backup file is a secret (it contains - private keys and credentials). Store and transfer it securely and delete it + private keys, TLS material, and credentials). Store and transfer it securely and delete it once the restore succeeds. Encrypting it below is the simplest way to do that: the download is then a .spbk file (SelfPost backup) that only the password opens.

diff --git a/internal/web/web.go b/internal/web/web.go index 39f7ea9..6088a22 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -40,11 +40,13 @@ type Config struct { // the log-tailer role follows in cmd/panel. MailLogPath string // DataDir and DBPath locate the persistent state a full backup archives - // (architecture.md § Persistence); Version is stamped into the backup - // manifest. They mirror the panel's own configuration. - DataDir string - DBPath string - Version string + // (architecture.md § Persistence); DeployRoot is the operator project + // directory (docker-compose.yml, .env, certs/); Version is stamped into + // the backup manifest. They mirror the panel's own configuration. + DataDir string + DBPath string + DeployRoot string + Version string // TrustedProxyCIDRs are the reverse-proxy addresses allowed to supply // X-Forwarded-For (env TRUSTED_PROXY_CIDR). A request whose // direct peer (RemoteAddr) is not in this list never has its XFF header @@ -106,6 +108,7 @@ func New(st *store.Store, domains *domain.Service, apps *app.Service, cfg Config MailLogPath: cfg.MailLogPath, DataDir: cfg.DataDir, DBPath: cfg.DBPath, + DeployRoot: cfg.DeployRoot, Version: cfg.Version, TLSCertFile: cfg.TLSCertFile, OpenDKIMSocket: cfg.OpenDKIMSocket,