From 8c9a71e6a04e8d2f0b23601991d3de8ef27ab4b9 Mon Sep 17 00:00:00 2001 From: Mix Date: Sun, 11 Oct 2026 19:55:18 +0300 Subject: [PATCH] backup: the panel's full backup works, without the queue Found by the e2e restore scenario of the next commit: the Full backup button answered 200 with ten bytes. The archive includes the Postfix queue, Postfix keeps every queue file and directory readable by its own user alone (not configurable), and the panel runs as its own user - the walk stopped at /data/postfix/queue/active with "permission denied" after the headers were out. So since the queue moved under /data in 1.3.0. Confirmed on the test server: selfpost-backup as root makes the whole archive, the same command as user panel fails the same way. Owner, 2026-10-11: the panel's archive comes without the queue. - backup.Params.Unprivileged: the queue is left out, and so is a file under certs/ that cannot be opened (the private key on the test server is root and Postfix's group only), named through OnWarn. The selfpost-backup command leaves it unset and takes both, as before. - backup.Check reads through everything Create would archive, writing nothing. The panel runs it before the first byte: a failure is a message on the page, not a download cut short. - The box, Help (design commit before this one) and the guide say what the panel's archive leaves out; the guide's comparison table has rows for the queue and the private key, and tells to put the key back before starting a restored instance. go vet ./internal/backup/ ./internal/web/handlers/: ok go test ./internal/backup/ ./cmd/... ./internal/web/...: ok make e2e on the test server, with the restore scenario: ok Co-Authored-By: Claude Opus 5.5 --- docs/guide.md | 17 +++- internal/backup/backup.go | 104 +++++++++++++++++++++-- internal/backup/backup_test.go | 44 ++++++++++ internal/web/handlers/handlers_backup.go | 32 +++++-- internal/web/view/help.go | 5 +- internal/web/view/templates/backup.html | 2 +- 6 files changed, 184 insertions(+), 20 deletions(-) diff --git a/docs/guide.md b/docs/guide.md index 46eb0be..6cda3d2 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -556,12 +556,23 @@ instance keeps running, what goes into the archive, and what happens on restore. **1. Panel — while the container is running (no downtime).** *Server › Backup* → *Full backup*. *Encrypt with a password* is ticked; untick it for a plain -archive. Same archive as the CLI below. +archive. The archive of the CLI below, less two things the panel may not read: +it runs as its own user, not as root. **The Postfix queue** is left out — +Postfix keeps it readable by its own user alone — so mail that was waiting for +delivery at that moment is not in the archive; the restored instance starts +with an empty queue. **A file in `certs/` the panel cannot open** is left out +too, which is usually the private key (the panel's log names it); put the key +back into `certs/` before starting a restored instance. If anything else +cannot be read, the panel says so and downloads nothing. **2. `selfpost-backup` CLI — while the container is running (no downtime).** The panel button and this command call the same code path: a consistent SQLite snapshot via `VACUUM INTO`, then a gzip tar of `data/` (minus diagnostics) plus -deploy files. +deploy files. `docker exec` runs the command as root, so this archive is the +complete one: it has the Postfix queue and everything in `certs/`. The queue is +copied while Postfix works on it, so a message on its way through may be in the +archive twice or not at all; stop the service first (method 3) if the queue +matters. ```sh docker exec selfpost-backup > selfpost-backup.tar.gz @@ -616,6 +627,8 @@ methods avoid it by stopping first. | Self-contained archive | Yes | Yes | No — deploy files separate | | SQLite consistency | `VACUUM INTO` snapshot | Safe when stopped | Safe when stopped | | `data/log/` | Excluded | Included (optional exclude) | Included (optional exclude) | +| Postfix queue | CLI: yes, copied live. Panel: no | Yes | Yes | +| `certs/` private key | CLI: yes. Panel: only if the panel may read it | Yes | No — deploy files separate | | `data/manifest.json` | Written (version stamp) | Absent | Absent | | Restore version check | Yes — must match image tag | No | No | | Post-restore *Resync* | Yes, on first boot | No | No | diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 10e3862..2be15d3 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -28,6 +28,7 @@ import ( "compress/gzip" "database/sql" "encoding/json" + "errors" "fmt" "io" "io/fs" @@ -79,8 +80,25 @@ type Params struct { Version string DeployRoot string OnWarn func(string) + // Unprivileged says the archive is made by the panel, which runs as its + // own user and not as root. Two things under the roots are then not its to + // read, and are left out instead of failing the whole archive: + // + // - the Postfix queue (QueueDir). Postfix keeps every queue file and + // directory readable by its own user alone, and that cannot be + // configured. What is lost is mail not yet delivered at that moment; + // a start without a queue directory builds an empty one. + // - a file under certs/ the panel may not open, which is usually the + // private key (root and Postfix's group only). OnWarn names it. + // + // The selfpost-backup command, run as root, leaves this unset and takes + // both. + Unprivileged bool } +// QueueDir is the Postfix queue, relative to the data directory. +const QueueDir = "postfix/queue" + // excludedFromArchive lists the data-directory entries a backup never carries. // The live database files are replaced by a consistent VACUUM INTO snapshot // written under the canonical name; the setup token is transient bootstrap @@ -145,11 +163,11 @@ func Create(w io.Writer, p Params) error { return err } - if err := addTree(tw, p.DataDir, DataArchivePrefix, excludedFromArchive); err != nil { + if err := addTree(tw, p.DataDir, DataArchivePrefix, excludedFromArchive, p.skipData()); err != nil { return err } - if err := addDeployFiles(tw, p.DeployRoot, p.OnWarn); err != nil { + if err := addDeployFiles(tw, p.DeployRoot, p.OnWarn, p.skipCert()); err != nil { return err } @@ -181,7 +199,66 @@ func validateDeployRoot(deployRoot string) error { return nil } -func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string)) error { +// skipFunc says whether the entry at path (rel to the root being walked) is +// left out of the archive. A directory left out is not descended into. +type skipFunc func(path, rel string, d fs.DirEntry) bool + +// skipData leaves the Postfix queue out of an unprivileged archive. +func (p Params) skipData() skipFunc { + if !p.Unprivileged { + return nil + } + return func(_, rel string, _ fs.DirEntry) bool { + return filepath.ToSlash(rel) == QueueDir + } +} + +// skipCert leaves a file under certs/ that cannot be opened out of an +// unprivileged archive, and says so through OnWarn. +func (p Params) skipCert() skipFunc { + if !p.Unprivileged { + return nil + } + return func(path, rel string, d fs.DirEntry) bool { + if d.IsDir() { + return false + } + f, err := os.Open(path) + if err == nil { + f.Close() + return false + } + if !errors.Is(err, fs.ErrPermission) { + return false + } + if p.OnWarn != nil { + p.OnWarn(CertsDirName + "/" + filepath.ToSlash(rel) + " is not readable by the panel and is left out of the backup") + } + return true + } +} + +// Check reads through everything Create would put into the archive, without +// writing one, and returns the first thing that cannot be read. The panel +// streams its archive to the browser, so once Create has begun a failure can +// only cut the download short; this is run first, while a refusal can still be +// shown as a page. +func Check(p Params) error { + if p.DataDir == "" || p.DBPath == "" { + return fmt.Errorf("backup: DataDir and DBPath are required") + } + if err := ValidateDeployRoot(p.DeployRoot); err != nil { + return err + } + quiet := p + quiet.OnWarn = nil + if err := addTree(nil, p.DataDir, DataArchivePrefix, excludedFromArchive, quiet.skipData()); err != nil { + return err + } + return addDeployFiles(nil, p.DeployRoot, nil, quiet.skipCert()) +} + +func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string), skip skipFunc) error { for _, name := range []string{ComposeFileName, EnvFileName} { src := filepath.Join(deployRoot, name) info, err := os.Stat(src) @@ -206,14 +283,16 @@ func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string)) erro } return fmt.Errorf("backup: stat %s: %w", CertsDirName, err) } - return addTree(tw, certsDir, CertsDirName+"/", nil) + return addTree(tw, certsDir, CertsDirName+"/", nil, skip) } // addTree walks root and adds every regular file (and directory, to preserve // empty ones and modes) to tw under archivePrefix + path relative to root. -// When exclude is non-nil, top-level names relative to root are skipped. +// When exclude is non-nil, top-level names relative to root are skipped; skip, +// when non-nil, leaves out whatever it says. With a nil tw nothing is written: +// every file is opened and closed, which is how Check finds what cannot be read. // Non-regular, non-directory entries (symlinks, sockets) are skipped. -func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool) error { +func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool, skip skipFunc) error { return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return err @@ -236,12 +315,22 @@ func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool } } + if skip != nil && skip(path, rel, d) { + if d.IsDir() { + return fs.SkipDir + } + return nil + } + info, err := d.Info() if err != nil { return err } switch { case d.IsDir(): + if tw == nil { + return nil + } hdr := &tar.Header{ Typeflag: tar.TypeDir, Name: name + "/", @@ -282,6 +371,9 @@ func writeTarFile(tw *tar.Writer, name string, mode fs.FileMode, srcPath string) return fmt.Errorf("backup: open %s: %w", srcPath, err) } defer f.Close() + if tw == nil { + return nil + } info, err := f.Stat() if err != nil { diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index 0960f76..75a3281 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -235,3 +235,47 @@ func TestCheckRestoreWrongFormatRejected(t *testing.T) { t.Error("CheckRestore wrong format reported a restore") } } + +// The panel's archive: no Postfix queue, everything else as before, and Check +// agrees that it can be made. +func TestCreateUnprivilegedLeavesTheQueueOut(t *testing.T) { + dataDir, dbPath, deployRoot := seedDataDir(t) + p := Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3", DeployRoot: deployRoot, Unprivileged: true} + + if err := Check(p); err != nil { + t.Fatalf("Check: %v", err) + } + var buf bytes.Buffer + if err := Create(&buf, p); err != nil { + t.Fatalf("Create: %v", err) + } + files := readArchive(t, buf.Bytes()) + for name := range files { + if strings.HasPrefix(name, DataArchivePrefix+QueueDir) { + t.Errorf("the panel's archive holds %s", name) + } + } + for _, name := range []string{ + DataArchivePrefix + "selfpost.db", + DataArchivePrefix + "opendkim/keys/example.com/selfpost.private", + DataArchivePrefix + "sasl/sasldb2", + DataArchivePrefix + "postfix/sender_login_maps", + ComposeFileName, EnvFileName, + CertsDirName + "/fullchain.pem", + } { + if _, ok := files[name]; !ok { + t.Errorf("archive missing %s", name) + } + } +} + +// Check finds what Create would stumble over, before anything is written. +func TestCheckReportsWhatCannotBeRead(t *testing.T) { + dataDir, dbPath, deployRoot := seedDataDir(t) + if err := os.Remove(filepath.Join(deployRoot, EnvFileName)); err != nil { + t.Fatal(err) + } + if err := Check(Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3", DeployRoot: deployRoot}); err == nil { + t.Fatal("Check passed with .env missing from the deploy root") + } +} diff --git a/internal/web/handlers/handlers_backup.go b/internal/web/handlers/handlers_backup.go index 98e15a4..588e761 100644 --- a/internal/web/handlers/handlers_backup.go +++ b/internal/web/handlers/handlers_backup.go @@ -70,11 +70,33 @@ func (h *Handlers) HandleBackup(w http.ResponseWriter, r *http.Request) { h.renderBackupPageWith(w, r, http.StatusBadRequest, "", pwErr) return } + // The panel is not root: its archive leaves out the Postfix queue and a + // certificate file it may not open (backup.Params.Unprivileged). + params := backup.Params{ + DataDir: h.cfg.DataDir, + DBPath: h.cfg.DBPath, + Version: h.cfg.Version, + DeployRoot: h.cfg.DeployRoot, + Unprivileged: true, + OnWarn: func(msg string) { + logf("panel: full backup: %s", msg) + }, + } if err := backup.ValidateDeployRoot(h.cfg.DeployRoot); err != nil { logf("panel: full backup: %v", err) h.renderBackupPageWith(w, r, http.StatusBadRequest, "", deployBackupErr(err)) return } + // Everything is read through once before the first byte goes out. Past the + // headers a failure can only cut the download short - which is what the + // button did from 1.3.0 to 2.0.0, with a 200 and ten bytes, because the + // queue was not the panel's to read. + if err := backup.Check(params); err != nil { + logf("panel: full backup: %v", err) + h.renderBackupPageWith(w, r, http.StatusInternalServerError, "", + "Could not read everything a backup holds. Nothing was downloaded; the reason is in the panel's log.") + return + } stamp := time.Now().UTC().Format("20060102-150405") filename := fmt.Sprintf("selfpost-backup-%s.tar.gz", stamp) @@ -109,15 +131,7 @@ func (h *Handlers) HandleBackup(w http.ResponseWriter, r *http.Request) { sink = env } - if err := backup.Create(sink, backup.Params{ - DataDir: h.cfg.DataDir, - DBPath: h.cfg.DBPath, - Version: h.cfg.Version, - DeployRoot: h.cfg.DeployRoot, - OnWarn: func(msg string) { - logf("panel: full backup: %s", msg) - }, - }); err != nil { + if err := backup.Create(sink, params); err != nil { logf("panel: full backup failed: %v", err) return } diff --git a/internal/web/view/help.go b/internal/web/view/help.go index 92608f5..13b7f05 100644 --- a/internal/web/view/help.go +++ b/internal/web/view/help.go @@ -188,9 +188,10 @@ var inboundTopics = []HelpTopic{ var serverTopics = []HelpTopic{ {ID: HelpBackup, Title: "Backup and moving a domain", Body: []Text{ - Rich("Full backup downloads one archive of the whole instance: ", Code("data/"), " (the database, DKIM keys, application credentials and the queue), ", + Rich("Full backup downloads one archive of the whole instance: ", Code("data/"), " (the database, DKIM keys and application credentials), ", Code("docker-compose.yml"), ", ", Code(".env"), " and ", Code("certs/"), - ". It holds secrets, so leave Encrypt with a password ticked: the file is sealed as ", Code(".spbk"), + ". The panel cannot read the Postfix queue, nor usually the private key in ", Code("certs/"), + ", and leaves them out; ", Code("selfpost-backup"), " run in the container takes both. It holds secrets, so leave Encrypt with a password ticked: the file is sealed as ", Code(".spbk"), ", the password is not stored, and without it nothing can be recovered. To restore, unpack it into an empty project directory (decrypt a ", Code(".spbk"), " first with ", Code("selfpost-backup -decrypt"), ") and start the same SelfPost version that made it; the panel refuses data from another version. The reverse-proxy configuration and ", diff --git a/internal/web/view/templates/backup.html b/internal/web/view/templates/backup.html index c26e675..71e6fca 100644 --- a/internal/web/view/templates/backup.html +++ b/internal/web/view/templates/backup.html @@ -13,7 +13,7 @@
{{template "box_open" .Full}} {{template "body_open"}} -

data/ (database, DKIM keys, credentials, the queue), docker-compose.yml, .env and certs/ in one archive. Restore into an empty project directory and start the same SelfPost version. The reverse-proxy vhost is not included.

+

data/ (database, DKIM keys, credentials), docker-compose.yml, .env and certs/ in one archive, without the Postfix queue and without a certificate file the panel may not read. Restore into an empty project directory and start the same SelfPost version. The reverse-proxy vhost is not included.