diff --git a/docs/guide.md b/docs/guide.md index 46eb0be..6cda3d2 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -556,12 +556,23 @@ instance keeps running, what goes into the archive, and what happens on restore. **1. Panel — while the container is running (no downtime).** *Server › Backup* → *Full backup*. *Encrypt with a password* is ticked; untick it for a plain -archive. Same archive as the CLI below. +archive. The archive of the CLI below, less two things the panel may not read: +it runs as its own user, not as root. **The Postfix queue** is left out — +Postfix keeps it readable by its own user alone — so mail that was waiting for +delivery at that moment is not in the archive; the restored instance starts +with an empty queue. **A file in `certs/` the panel cannot open** is left out +too, which is usually the private key (the panel's log names it); put the key +back into `certs/` before starting a restored instance. If anything else +cannot be read, the panel says so and downloads nothing. **2. `selfpost-backup` CLI — while the container is running (no downtime).** The panel button and this command call the same code path: a consistent SQLite snapshot via `VACUUM INTO`, then a gzip tar of `data/` (minus diagnostics) plus -deploy files. +deploy files. `docker exec` runs the command as root, so this archive is the +complete one: it has the Postfix queue and everything in `certs/`. The queue is +copied while Postfix works on it, so a message on its way through may be in the +archive twice or not at all; stop the service first (method 3) if the queue +matters. ```sh docker exec selfpost-backup > selfpost-backup.tar.gz @@ -616,6 +627,8 @@ methods avoid it by stopping first. | Self-contained archive | Yes | Yes | No — deploy files separate | | SQLite consistency | `VACUUM INTO` snapshot | Safe when stopped | Safe when stopped | | `data/log/` | Excluded | Included (optional exclude) | Included (optional exclude) | +| Postfix queue | CLI: yes, copied live. Panel: no | Yes | Yes | +| `certs/` private key | CLI: yes. Panel: only if the panel may read it | Yes | No — deploy files separate | | `data/manifest.json` | Written (version stamp) | Absent | Absent | | Restore version check | Yes — must match image tag | No | No | | Post-restore *Resync* | Yes, on first boot | No | No | diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 10e3862..2be15d3 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -28,6 +28,7 @@ import ( "compress/gzip" "database/sql" "encoding/json" + "errors" "fmt" "io" "io/fs" @@ -79,8 +80,25 @@ type Params struct { Version string DeployRoot string OnWarn func(string) + // Unprivileged says the archive is made by the panel, which runs as its + // own user and not as root. Two things under the roots are then not its to + // read, and are left out instead of failing the whole archive: + // + // - the Postfix queue (QueueDir). Postfix keeps every queue file and + // directory readable by its own user alone, and that cannot be + // configured. What is lost is mail not yet delivered at that moment; + // a start without a queue directory builds an empty one. + // - a file under certs/ the panel may not open, which is usually the + // private key (root and Postfix's group only). OnWarn names it. + // + // The selfpost-backup command, run as root, leaves this unset and takes + // both. + Unprivileged bool } +// QueueDir is the Postfix queue, relative to the data directory. +const QueueDir = "postfix/queue" + // excludedFromArchive lists the data-directory entries a backup never carries. // The live database files are replaced by a consistent VACUUM INTO snapshot // written under the canonical name; the setup token is transient bootstrap @@ -145,11 +163,11 @@ func Create(w io.Writer, p Params) error { return err } - if err := addTree(tw, p.DataDir, DataArchivePrefix, excludedFromArchive); err != nil { + if err := addTree(tw, p.DataDir, DataArchivePrefix, excludedFromArchive, p.skipData()); err != nil { return err } - if err := addDeployFiles(tw, p.DeployRoot, p.OnWarn); err != nil { + if err := addDeployFiles(tw, p.DeployRoot, p.OnWarn, p.skipCert()); err != nil { return err } @@ -181,7 +199,66 @@ func validateDeployRoot(deployRoot string) error { return nil } -func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string)) error { +// skipFunc says whether the entry at path (rel to the root being walked) is +// left out of the archive. A directory left out is not descended into. +type skipFunc func(path, rel string, d fs.DirEntry) bool + +// skipData leaves the Postfix queue out of an unprivileged archive. +func (p Params) skipData() skipFunc { + if !p.Unprivileged { + return nil + } + return func(_, rel string, _ fs.DirEntry) bool { + return filepath.ToSlash(rel) == QueueDir + } +} + +// skipCert leaves a file under certs/ that cannot be opened out of an +// unprivileged archive, and says so through OnWarn. +func (p Params) skipCert() skipFunc { + if !p.Unprivileged { + return nil + } + return func(path, rel string, d fs.DirEntry) bool { + if d.IsDir() { + return false + } + f, err := os.Open(path) + if err == nil { + f.Close() + return false + } + if !errors.Is(err, fs.ErrPermission) { + return false + } + if p.OnWarn != nil { + p.OnWarn(CertsDirName + "/" + filepath.ToSlash(rel) + " is not readable by the panel and is left out of the backup") + } + return true + } +} + +// Check reads through everything Create would put into the archive, without +// writing one, and returns the first thing that cannot be read. The panel +// streams its archive to the browser, so once Create has begun a failure can +// only cut the download short; this is run first, while a refusal can still be +// shown as a page. +func Check(p Params) error { + if p.DataDir == "" || p.DBPath == "" { + return fmt.Errorf("backup: DataDir and DBPath are required") + } + if err := ValidateDeployRoot(p.DeployRoot); err != nil { + return err + } + quiet := p + quiet.OnWarn = nil + if err := addTree(nil, p.DataDir, DataArchivePrefix, excludedFromArchive, quiet.skipData()); err != nil { + return err + } + return addDeployFiles(nil, p.DeployRoot, nil, quiet.skipCert()) +} + +func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string), skip skipFunc) error { for _, name := range []string{ComposeFileName, EnvFileName} { src := filepath.Join(deployRoot, name) info, err := os.Stat(src) @@ -206,14 +283,16 @@ func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string)) erro } return fmt.Errorf("backup: stat %s: %w", CertsDirName, err) } - return addTree(tw, certsDir, CertsDirName+"/", nil) + return addTree(tw, certsDir, CertsDirName+"/", nil, skip) } // addTree walks root and adds every regular file (and directory, to preserve // empty ones and modes) to tw under archivePrefix + path relative to root. -// When exclude is non-nil, top-level names relative to root are skipped. +// When exclude is non-nil, top-level names relative to root are skipped; skip, +// when non-nil, leaves out whatever it says. With a nil tw nothing is written: +// every file is opened and closed, which is how Check finds what cannot be read. // Non-regular, non-directory entries (symlinks, sockets) are skipped. -func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool) error { +func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool, skip skipFunc) error { return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { return err @@ -236,12 +315,22 @@ func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool } } + if skip != nil && skip(path, rel, d) { + if d.IsDir() { + return fs.SkipDir + } + return nil + } + info, err := d.Info() if err != nil { return err } switch { case d.IsDir(): + if tw == nil { + return nil + } hdr := &tar.Header{ Typeflag: tar.TypeDir, Name: name + "/", @@ -282,6 +371,9 @@ func writeTarFile(tw *tar.Writer, name string, mode fs.FileMode, srcPath string) return fmt.Errorf("backup: open %s: %w", srcPath, err) } defer f.Close() + if tw == nil { + return nil + } info, err := f.Stat() if err != nil { diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index 0960f76..75a3281 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -235,3 +235,47 @@ func TestCheckRestoreWrongFormatRejected(t *testing.T) { t.Error("CheckRestore wrong format reported a restore") } } + +// The panel's archive: no Postfix queue, everything else as before, and Check +// agrees that it can be made. +func TestCreateUnprivilegedLeavesTheQueueOut(t *testing.T) { + dataDir, dbPath, deployRoot := seedDataDir(t) + p := Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3", DeployRoot: deployRoot, Unprivileged: true} + + if err := Check(p); err != nil { + t.Fatalf("Check: %v", err) + } + var buf bytes.Buffer + if err := Create(&buf, p); err != nil { + t.Fatalf("Create: %v", err) + } + files := readArchive(t, buf.Bytes()) + for name := range files { + if strings.HasPrefix(name, DataArchivePrefix+QueueDir) { + t.Errorf("the panel's archive holds %s", name) + } + } + for _, name := range []string{ + DataArchivePrefix + "selfpost.db", + DataArchivePrefix + "opendkim/keys/example.com/selfpost.private", + DataArchivePrefix + "sasl/sasldb2", + DataArchivePrefix + "postfix/sender_login_maps", + ComposeFileName, EnvFileName, + CertsDirName + "/fullchain.pem", + } { + if _, ok := files[name]; !ok { + t.Errorf("archive missing %s", name) + } + } +} + +// Check finds what Create would stumble over, before anything is written. +func TestCheckReportsWhatCannotBeRead(t *testing.T) { + dataDir, dbPath, deployRoot := seedDataDir(t) + if err := os.Remove(filepath.Join(deployRoot, EnvFileName)); err != nil { + t.Fatal(err) + } + if err := Check(Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3", DeployRoot: deployRoot}); err == nil { + t.Fatal("Check passed with .env missing from the deploy root") + } +} diff --git a/internal/web/handlers/handlers_backup.go b/internal/web/handlers/handlers_backup.go index 98e15a4..588e761 100644 --- a/internal/web/handlers/handlers_backup.go +++ b/internal/web/handlers/handlers_backup.go @@ -70,11 +70,33 @@ func (h *Handlers) HandleBackup(w http.ResponseWriter, r *http.Request) { h.renderBackupPageWith(w, r, http.StatusBadRequest, "", pwErr) return } + // The panel is not root: its archive leaves out the Postfix queue and a + // certificate file it may not open (backup.Params.Unprivileged). + params := backup.Params{ + DataDir: h.cfg.DataDir, + DBPath: h.cfg.DBPath, + Version: h.cfg.Version, + DeployRoot: h.cfg.DeployRoot, + Unprivileged: true, + OnWarn: func(msg string) { + logf("panel: full backup: %s", msg) + }, + } if err := backup.ValidateDeployRoot(h.cfg.DeployRoot); err != nil { logf("panel: full backup: %v", err) h.renderBackupPageWith(w, r, http.StatusBadRequest, "", deployBackupErr(err)) return } + // Everything is read through once before the first byte goes out. Past the + // headers a failure can only cut the download short - which is what the + // button did from 1.3.0 to 2.0.0, with a 200 and ten bytes, because the + // queue was not the panel's to read. + if err := backup.Check(params); err != nil { + logf("panel: full backup: %v", err) + h.renderBackupPageWith(w, r, http.StatusInternalServerError, "", + "Could not read everything a backup holds. Nothing was downloaded; the reason is in the panel's log.") + return + } stamp := time.Now().UTC().Format("20060102-150405") filename := fmt.Sprintf("selfpost-backup-%s.tar.gz", stamp) @@ -109,15 +131,7 @@ func (h *Handlers) HandleBackup(w http.ResponseWriter, r *http.Request) { sink = env } - if err := backup.Create(sink, backup.Params{ - DataDir: h.cfg.DataDir, - DBPath: h.cfg.DBPath, - Version: h.cfg.Version, - DeployRoot: h.cfg.DeployRoot, - OnWarn: func(msg string) { - logf("panel: full backup: %s", msg) - }, - }); err != nil { + if err := backup.Create(sink, params); err != nil { logf("panel: full backup failed: %v", err) return } diff --git a/internal/web/view/help.go b/internal/web/view/help.go index 92608f5..13b7f05 100644 --- a/internal/web/view/help.go +++ b/internal/web/view/help.go @@ -188,9 +188,10 @@ var inboundTopics = []HelpTopic{ var serverTopics = []HelpTopic{ {ID: HelpBackup, Title: "Backup and moving a domain", Body: []Text{ - Rich("Full backup downloads one archive of the whole instance: ", Code("data/"), " (the database, DKIM keys, application credentials and the queue), ", + Rich("Full backup downloads one archive of the whole instance: ", Code("data/"), " (the database, DKIM keys and application credentials), ", Code("docker-compose.yml"), ", ", Code(".env"), " and ", Code("certs/"), - ". It holds secrets, so leave Encrypt with a password ticked: the file is sealed as ", Code(".spbk"), + ". The panel cannot read the Postfix queue, nor usually the private key in ", Code("certs/"), + ", and leaves them out; ", Code("selfpost-backup"), " run in the container takes both. It holds secrets, so leave Encrypt with a password ticked: the file is sealed as ", Code(".spbk"), ", the password is not stored, and without it nothing can be recovered. To restore, unpack it into an empty project directory (decrypt a ", Code(".spbk"), " first with ", Code("selfpost-backup -decrypt"), ") and start the same SelfPost version that made it; the panel refuses data from another version. The reverse-proxy configuration and ", diff --git a/internal/web/view/templates/backup.html b/internal/web/view/templates/backup.html index c26e675..71e6fca 100644 --- a/internal/web/view/templates/backup.html +++ b/internal/web/view/templates/backup.html @@ -13,7 +13,7 @@
{{template "box_open" .Full}} {{template "body_open"}} -

data/ (database, DKIM keys, credentials, the queue), docker-compose.yml, .env and certs/ in one archive. Restore into an empty project directory and start the same SelfPost version. The reverse-proxy vhost is not included.

+

data/ (database, DKIM keys, credentials), docker-compose.yml, .env and certs/ in one archive, without the Postfix queue and without a certificate file the panel may not read. Restore into an empty project directory and start the same SelfPost version. The reverse-proxy vhost is not included.