Persist Postfix queue and ship self-contained full backups.

Move the mail queue under /data so recreate no longer drops deferred mail, and archive data/, compose, .env, and certs/ together for restore on a fresh host.

Co-authored-by: Cursor <cursoragent@cursor.com>

internal/backup/backup.go

@@ -1,15 +1,17 @@
 // Package backup implements SelfPost's full-server backup and the restore
 // version guard (architecture.md § Persistence). A full backup is a
 // gzip-compressed tar of the consolidated persistent state under /data — the
-// SQLite database (as a consistent snapshot), the per-domain DKIM keys and the
-// SASL database — plus a manifest recording the SelfPost version that produced
-// it. TLS certificates (the reverse proxy's responsibility) and the Postfix
-// queue are deliberately excluded (architecture.md § Persistence).
+// SQLite database (as a consistent snapshot), the per-domain DKIM keys, the
+// SASL database, and the Postfix queue — plus docker-compose.yml, .env, and
+// certs/ from the operator's deploy directory, and a manifest recording the
+// SelfPost version that produced it. Postfix delivery logs under log/ are
+// excluded (diagnostics, not state).
 //
 // Restore is not a separate code path in the panel: a backup is extracted into
-// the /data bind mount before first start, and the archive already carries
-// everything the mail path needs — DKIM keys, sasldb2, and Postfix's sender
-// map — so the daemons can start on the extracted files alone. The
+// the operator's project directory (data/, docker-compose.yml, .env, certs/)
+// before first start. The archive carries everything needed to bring the
+// instance back — DKIM keys, sasldb2, sender map, queue, and deploy files —
+// so the operator only adjusts hostname or proxy settings on a new host. The
 // restore-specific steps the panel runs are CheckRestore, which refuses to
 // boot if the manifest's version does not match the running binary so
 // schema/format skew between versions cannot silently corrupt state
@@ -40,11 +42,21 @@ import (
 // FormatFull identifies a full-server backup manifest.
 const FormatFull = "selfpost-full-backup"
 
-// ManifestName is the manifest's filename, both inside the archive and, after a
-// restore extraction, at the root of the data directory where CheckRestore
-// looks for it.
+// ManifestName is the manifest's filename inside the data directory. After
+// restore extraction it lives at data/manifest.json under the project root;
+// CheckRestore reads it from the /data bind mount.
 const ManifestName = "manifest.json"
 
+// DataArchivePrefix is the path prefix for every /data entry in the archive.
+const DataArchivePrefix = "data/"
+
+// ComposeFileName and EnvFileName are required deploy files at the archive root.
+const (
+	ComposeFileName = "docker-compose.yml"
+	EnvFileName     = ".env"
+	CertsDirName    = "certs"
+)
+
 // Manifest is the small JSON document embedded in every backup archive. Its
 // Version is the single fact that makes restore safe: the panel refuses to
 // boot a data directory whose manifest version does not match its own binary
@@ -58,21 +70,23 @@ type Manifest struct {
 // Params configures a backup. DataDir is the consolidated state root (/data);
 // DBPath is the live SQLite file within it, snapshotted consistently rather than
 // copied byte-for-byte while it may be mid-write; Version is stamped into the
-// manifest.
+// manifest; DeployRoot is the host project directory mounted read-only (holds
+// docker-compose.yml, .env, and optionally certs/). OnWarn is called for
+// non-fatal issues such as a missing certs/ directory.
 type Params struct {
-	DataDir string
-	DBPath  string
-	Version string
+	DataDir    string
+	DBPath     string
+	Version    string
+	DeployRoot string
+	OnWarn     func(string)
 }
 
 // excludedFromArchive lists the data-directory entries a backup never carries.
 // The live database files are replaced by a consistent VACUUM INTO snapshot
 // written under the canonical name; the setup token is transient bootstrap
 // state; a stale manifest from a previous restore must not be re-captured (a
-// fresh one is written instead); a "tls" directory holds the reverse proxy's
-// certificates, which are explicitly out of scope for a SelfPost backup
-// (architecture.md § Persistence) — excluding it keeps that guarantee even when
-// an operator points TLS_CERT_FILE inside /data; and "log" is Postfix's raw
+// fresh one is written instead); a "tls" directory under /data is skipped when
+// an operator pointed TLS_CERT_FILE inside /data; and "log" is Postfix's raw
 // delivery log plus its fourteen rotated files, which is diagnostic output, not
 // state to restore, and by far the largest thing under /data.
 var excludedFromArchive = map[string]bool{
@@ -86,16 +100,22 @@ var excludedFromArchive = map[string]bool{
 	ManifestName:          true,
 }
 
-// Create writes a gzip-compressed tar backup to w. Archive entries are named
-// relative to DataDir, so extracting the archive into the /data bind mount
-// reconstructs the state in place (architecture.md § Persistence). The SQLite
-// database is added as a consistent snapshot under "selfpost.db"; everything
-// else under DataDir is copied as-is except the entries in
-// excludedFromArchive.
+// Create writes a gzip-compressed tar backup to w. Archive layout:
+//
+//	data/manifest.json, data/selfpost.db, data/<rest of /data>
+//	docker-compose.yml, .env, certs/...
+//
+// Extract the archive into an empty project directory, then docker compose up.
 func Create(w io.Writer, p Params) error {
 	if p.DataDir == "" || p.DBPath == "" {
 		return fmt.Errorf("backup: DataDir and DBPath are required")
 	}
+	if p.DeployRoot == "" {
+		return fmt.Errorf("backup: DeployRoot is required (mount the project directory at SELFPOST_DEPLOY_ROOT)")
+	}
+	if err := validateDeployRoot(p.DeployRoot); err != nil {
+		return err
+	}
 
 	snapshot, cleanup, err := snapshotDB(p.DBPath)
 	if err != nil {
@@ -115,17 +135,21 @@ func Create(w io.Writer, p Params) error {
 	if err != nil {
 		return fmt.Errorf("backup: encode manifest: %w", err)
 	}
-	if err := writeTarBytes(tw, ManifestName, 0o600, manifestJSON); err != nil {
+	if err := writeTarBytes(tw, DataArchivePrefix+ManifestName, 0o600, manifestJSON); err != nil {
 		return err
 	}
 
 	// The consistent SQLite snapshot, under the canonical filename the panel
 	// opens on start (the live file and its WAL/SHM are excluded from the walk).
-	if err := writeTarFile(tw, "selfpost.db", 0o640, snapshot); err != nil {
+	if err := writeTarFile(tw, DataArchivePrefix+"selfpost.db", 0o640, snapshot); err != nil {
 		return err
 	}
 
-	if err := addTree(tw, p.DataDir); err != nil {
+	if err := addTree(tw, p.DataDir, DataArchivePrefix, excludedFromArchive); err != nil {
+		return err
+	}
+
+	if err := addDeployFiles(tw, p.DeployRoot, p.OnWarn); err != nil {
 		return err
 	}
 
@@ -138,33 +162,79 @@ func Create(w io.Writer, p Params) error {
 	return nil
 }
 
-// addTree walks dataDir and adds every regular file (and directory, to preserve
-// empty ones and modes) to tw under its path relative to dataDir, skipping the
-// excluded entries. Non-regular, non-directory entries (symlinks, sockets) are
-// skipped: /data holds none in normal operation, and copying them into a backup
-// would be meaningless or unsafe.
-func addTree(tw *tar.Writer, dataDir string) error {
-	return filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error {
+// ValidateDeployRoot checks that the operator project directory is mounted and
+// contains the files a full backup requires. Call before streaming a response.
+func ValidateDeployRoot(deployRoot string) error {
+	if deployRoot == "" {
+		return fmt.Errorf("backup: DeployRoot is required (mount the project directory at SELFPOST_DEPLOY_ROOT)")
+	}
+	return validateDeployRoot(deployRoot)
+}
+
+func validateDeployRoot(deployRoot string) error {
+	for _, name := range []string{ComposeFileName, EnvFileName} {
+		path := filepath.Join(deployRoot, name)
+		if _, err := os.Stat(path); err != nil {
+			return fmt.Errorf("backup: deploy root %q is missing %s: %w", deployRoot, name, err)
+		}
+	}
+	return nil
+}
+
+func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string)) error {
+	for _, name := range []string{ComposeFileName, EnvFileName} {
+		src := filepath.Join(deployRoot, name)
+		info, err := os.Stat(src)
+		if err != nil {
+			return fmt.Errorf("backup: stat deploy file %s: %w", name, err)
+		}
+		if !info.Mode().IsRegular() {
+			return fmt.Errorf("backup: deploy file %s is not a regular file", name)
+		}
+		if err := writeTarFile(tw, name, info.Mode().Perm(), src); err != nil {
+			return err
+		}
+	}
+
+	certsDir := filepath.Join(deployRoot, CertsDirName)
+	if _, err := os.Stat(certsDir); err != nil {
+		if os.IsNotExist(err) {
+			if onWarn != nil {
+				onWarn("certs/ not found in deploy root; backup will not include TLS material")
+			}
+			return nil
+		}
+		return fmt.Errorf("backup: stat %s: %w", CertsDirName, err)
+	}
+	return addTree(tw, certsDir, CertsDirName+"/", nil)
+}
+
+// addTree walks root and adds every regular file (and directory, to preserve
+// empty ones and modes) to tw under archivePrefix + path relative to root.
+// When exclude is non-nil, top-level names relative to root are skipped.
+// Non-regular, non-directory entries (symlinks, sockets) are skipped.
+func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool) error {
+	return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
-		rel, err := filepath.Rel(dataDir, path)
+		rel, err := filepath.Rel(root, path)
 		if err != nil {
 			return err
 		}
 		if rel == "." {
-			return nil // the data root itself is implicit
-		}
-		// filepath.Rel yields OS separators; tar names use forward slashes.
-		name := filepath.ToSlash(rel)
-		// Exclude by top-level name (the live DB, setup token and stale manifest
-		// all live at the data root).
-		if excludedFromArchive[name] {
-			if d.IsDir() {
-				return fs.SkipDir
-			}
 			return nil
 		}
+		name := archivePrefix + filepath.ToSlash(rel)
+		if exclude != nil {
+			top := strings.Split(filepath.ToSlash(rel), "/")[0]
+			if exclude[top] {
+				if d.IsDir() {
+					return fs.SkipDir
+				}
+				return nil
+			}
+		}
 
 		info, err := d.Info()
 		if err != nil {
@@ -182,7 +252,7 @@ func addTree(tw *tar.Writer, dataDir string) error {
 		case info.Mode().IsRegular():
 			return writeTarFile(tw, name, info.Mode().Perm(), path)
 		default:
-			return nil // skip symlinks/sockets/devices
+			return nil
 		}
 	})
 }