mix/selfpost
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Persist Postfix queue and ship self-contained full backups.

Browse Source 
Move the mail queue under /data so recreate no longer drops deferred mail, and archive data/, compose, .env, and certs/ together for restore on a fresh host.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Микс
2026-08-17 14:36:55 +03:00
parent c6a75ce775
commit 41c3e6e896
23 changed files with 408 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/backup/backup.go
+116 -46
View File
@@ -1,15 +1,17 @@
// Package backup implements SelfPost's full-server backup and the restore
// version guard (architecture.md § Persistence). A full backup is a
// gzip-compressed tar of the consolidated persistent state under /data — the
// SQLite database (as a consistent snapshot), the per-domain DKIM keys and the
// SASL database — plus a manifest recording the SelfPost version that produced
// it. TLS certificates (the reverse proxy's responsibility) and the Postfix
// queue are deliberately excluded (architecture.md § Persistence).
// SQLite database (as a consistent snapshot), the per-domain DKIM keys, the
// SASL database, and the Postfix queue — plus docker-compose.yml, .env, and
// certs/ from the operator's deploy directory, and a manifest recording the
// SelfPost version that produced it. Postfix delivery logs under log/ are
// excluded (diagnostics, not state).
//
// Restore is not a separate code path in the panel: a backup is extracted into
// the /data bind mount before first start, and the archive already carries
// everything the mail path needs — DKIM keys, sasldb2, and Postfix's sender
// map — so the daemons can start on the extracted files alone. The
// the operator's project directory (data/, docker-compose.yml, .env, certs/)
// before first start. The archive carries everything needed to bring the
// instance back — DKIM keys, sasldb2, sender map, queue, and deploy files —
// so the operator only adjusts hostname or proxy settings on a new host. The
// restore-specific steps the panel runs are CheckRestore, which refuses to
// boot if the manifest's version does not match the running binary so
// schema/format skew between versions cannot silently corrupt state
@@ -40,11 +42,21 @@ import (
// FormatFull identifies a full-server backup manifest.
const FormatFull = "selfpost-full-backup"
// ManifestName is the manifest's filename, both inside the archive and, after a
// restore extraction, at the root of the data directory where CheckRestore
// looks for it.
// ManifestName is the manifest's filename inside the data directory. After
// restore extraction it lives at data/manifest.json under the project root;
// CheckRestore reads it from the /data bind mount.
const ManifestName = "manifest.json"
// DataArchivePrefix is the path prefix for every /data entry in the archive.
const DataArchivePrefix = "data/"
// ComposeFileName and EnvFileName are required deploy files at the archive root.
const (
ComposeFileName = "docker-compose.yml"
EnvFileName = ".env"
CertsDirName = "certs"
)
// Manifest is the small JSON document embedded in every backup archive. Its
// Version is the single fact that makes restore safe: the panel refuses to
// boot a data directory whose manifest version does not match its own binary
@@ -58,21 +70,23 @@ type Manifest struct {
// Params configures a backup. DataDir is the consolidated state root (/data);
// DBPath is the live SQLite file within it, snapshotted consistently rather than
// copied byte-for-byte while it may be mid-write; Version is stamped into the
// manifest.
// manifest; DeployRoot is the host project directory mounted read-only (holds
// docker-compose.yml, .env, and optionally certs/). OnWarn is called for
// non-fatal issues such as a missing certs/ directory.
type Params struct {
DataDir string
DBPath string
Version string
DataDir string
DBPath string
Version string
DeployRoot string
OnWarn func(string)
}
// excludedFromArchive lists the data-directory entries a backup never carries.
// The live database files are replaced by a consistent VACUUM INTO snapshot
// written under the canonical name; the setup token is transient bootstrap
// state; a stale manifest from a previous restore must not be re-captured (a
// fresh one is written instead); a "tls" directory holds the reverse proxy's
// certificates, which are explicitly out of scope for a SelfPost backup
// (architecture.md § Persistence) — excluding it keeps that guarantee even when
// an operator points TLS_CERT_FILE inside /data; and "log" is Postfix's raw
// fresh one is written instead); a "tls" directory under /data is skipped when
// an operator pointed TLS_CERT_FILE inside /data; and "log" is Postfix's raw
// delivery log plus its fourteen rotated files, which is diagnostic output, not
// state to restore, and by far the largest thing under /data.
var excludedFromArchive = map[string]bool{
@@ -86,16 +100,22 @@ var excludedFromArchive = map[string]bool{
ManifestName: true,
}
// Create writes a gzip-compressed tar backup to w. Archive entries are named
// relative to DataDir, so extracting the archive into the /data bind mount
// reconstructs the state in place (architecture.md § Persistence). The SQLite
// database is added as a consistent snapshot under "selfpost.db"; everything
// else under DataDir is copied as-is except the entries in
// excludedFromArchive.
// Create writes a gzip-compressed tar backup to w. Archive layout:
//
// data/manifest.json, data/selfpost.db, data/<rest of /data>
// docker-compose.yml, .env, certs/...
//
// Extract the archive into an empty project directory, then docker compose up.
func Create(w io.Writer, p Params) error {
if p.DataDir == "" || p.DBPath == "" {
return fmt.Errorf("backup: DataDir and DBPath are required")
}
if p.DeployRoot == "" {
return fmt.Errorf("backup: DeployRoot is required (mount the project directory at SELFPOST_DEPLOY_ROOT)")
}
if err := validateDeployRoot(p.DeployRoot); err != nil {
return err
}
snapshot, cleanup, err := snapshotDB(p.DBPath)
if err != nil {
@@ -115,17 +135,21 @@ func Create(w io.Writer, p Params) error {
if err != nil {
return fmt.Errorf("backup: encode manifest: %w", err)
}
if err := writeTarBytes(tw, ManifestName, 0o600, manifestJSON); err != nil {
if err := writeTarBytes(tw, DataArchivePrefix+ManifestName, 0o600, manifestJSON); err != nil {
return err
}
// The consistent SQLite snapshot, under the canonical filename the panel
// opens on start (the live file and its WAL/SHM are excluded from the walk).
if err := writeTarFile(tw, "selfpost.db", 0o640, snapshot); err != nil {
if err := writeTarFile(tw, DataArchivePrefix+"selfpost.db", 0o640, snapshot); err != nil {
return err
}
if err := addTree(tw, p.DataDir); err != nil {
if err := addTree(tw, p.DataDir, DataArchivePrefix, excludedFromArchive); err != nil {
return err
}
if err := addDeployFiles(tw, p.DeployRoot, p.OnWarn); err != nil {
return err
}
@@ -138,33 +162,79 @@ func Create(w io.Writer, p Params) error {
return nil
}
// addTree walks dataDir and adds every regular file (and directory, to preserve
// empty ones and modes) to tw under its path relative to dataDir, skipping the
// excluded entries. Non-regular, non-directory entries (symlinks, sockets) are
// skipped: /data holds none in normal operation, and copying them into a backup
// would be meaningless or unsafe.
func addTree(tw *tar.Writer, dataDir string) error {
return filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error {
// ValidateDeployRoot checks that the operator project directory is mounted and
// contains the files a full backup requires. Call before streaming a response.
func ValidateDeployRoot(deployRoot string) error {
if deployRoot == "" {
return fmt.Errorf("backup: DeployRoot is required (mount the project directory at SELFPOST_DEPLOY_ROOT)")
}
return validateDeployRoot(deployRoot)
}
func validateDeployRoot(deployRoot string) error {
for _, name := range []string{ComposeFileName, EnvFileName} {
path := filepath.Join(deployRoot, name)
if _, err := os.Stat(path); err != nil {
return fmt.Errorf("backup: deploy root %q is missing %s: %w", deployRoot, name, err)
}
}
return nil
}
func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string)) error {
for _, name := range []string{ComposeFileName, EnvFileName} {
src := filepath.Join(deployRoot, name)
info, err := os.Stat(src)
if err != nil {
return fmt.Errorf("backup: stat deploy file %s: %w", name, err)
}
if !info.Mode().IsRegular() {
return fmt.Errorf("backup: deploy file %s is not a regular file", name)
}
if err := writeTarFile(tw, name, info.Mode().Perm(), src); err != nil {
return err
}
}
certsDir := filepath.Join(deployRoot, CertsDirName)
if _, err := os.Stat(certsDir); err != nil {
if os.IsNotExist(err) {
if onWarn != nil {
onWarn("certs/ not found in deploy root; backup will not include TLS material")
}
return nil
}
return fmt.Errorf("backup: stat %s: %w", CertsDirName, err)
}
return addTree(tw, certsDir, CertsDirName+"/", nil)
}
// addTree walks root and adds every regular file (and directory, to preserve
// empty ones and modes) to tw under archivePrefix + path relative to root.
// When exclude is non-nil, top-level names relative to root are skipped.
// Non-regular, non-directory entries (symlinks, sockets) are skipped.
func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool) error {
return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
rel, err := filepath.Rel(dataDir, path)
rel, err := filepath.Rel(root, path)
if err != nil {
return err
}
if rel == "." {
return nil // the data root itself is implicit
}
// filepath.Rel yields OS separators; tar names use forward slashes.
name := filepath.ToSlash(rel)
// Exclude by top-level name (the live DB, setup token and stale manifest
// all live at the data root).
if excludedFromArchive[name] {
if d.IsDir() {
return fs.SkipDir
}
return nil
}
name := archivePrefix + filepath.ToSlash(rel)
if exclude != nil {
top := strings.Split(filepath.ToSlash(rel), "/")[0]
if exclude[top] {
if d.IsDir() {
return fs.SkipDir
}
return nil
}
}
info, err := d.Info()
if err != nil {
@@ -182,7 +252,7 @@ func addTree(tw *tar.Writer, dataDir string) error {
case info.Mode().IsRegular():
return writeTarFile(tw, name, info.Mode().Perm(), path)
default:
return nil // skip symlinks/sockets/devices
return nil
}
})
}
internal/backup/backup_test.go
+41 -15
View File
@@ -15,10 +15,15 @@ import (
)
// seedDataDir builds a realistic /data tree: a migrated SQLite database plus the
// DKIM key, SASL and transient files a backup must include or exclude.
func seedDataDir(t *testing.T) (dataDir, dbPath string) {
// DKIM key, SASL and transient files a backup must include or exclude. It also
// seeds a deploy root beside data/ with compose, .env, and certs/.
func seedDataDir(t *testing.T) (dataDir, dbPath, deployRoot string) {
t.Helper()
dataDir = t.TempDir()
deployRoot = t.TempDir()
dataDir = filepath.Join(deployRoot, "data")
if err := os.MkdirAll(dataDir, 0o750); err != nil {
t.Fatalf("mkdir data: %v", err)
}
dbPath = filepath.Join(dataDir, "selfpost.db")
st, err := store.Open(dbPath)
@@ -35,6 +40,7 @@ func seedDataDir(t *testing.T) (dataDir, dbPath string) {
writeFile(t, filepath.Join(dataDir, "opendkim", "keys", "example.com", "selfpost.private"), "PRIVATE KEY")
writeFile(t, filepath.Join(dataDir, "sasl", "sasldb2"), "SASLDB")
writeFile(t, filepath.Join(dataDir, "postfix", "sender_login_maps"), "@example.com login")
writeFile(t, filepath.Join(dataDir, "postfix", "queue", "deferred", "sample"), "queue-file")
// Transient files that must NOT be archived.
writeFile(t, filepath.Join(dataDir, "setup-token"), "secret-token")
writeFile(t, filepath.Join(dataDir, "selfpost.db-wal"), "wal")
@@ -43,7 +49,11 @@ func seedDataDir(t *testing.T) (dataDir, dbPath string) {
// state, and the bulkiest thing under /data.
writeFile(t, filepath.Join(dataDir, "log", "mail.log"), "Aug 8 07:26:41 mail postfix/smtp[1]: ABC: to=<a@example.net>, status=sent (ok)")
writeFile(t, filepath.Join(dataDir, "log", "mail.log.1"), "older")
return dataDir, dbPath
writeFile(t, filepath.Join(deployRoot, ComposeFileName), "services:\n selfpost:\n image: test\n")
writeFile(t, filepath.Join(deployRoot, EnvFileName), "SELFPOST_HOSTNAME=mail.example.com\n")
writeFile(t, filepath.Join(deployRoot, CertsDirName, "fullchain.pem"), "CERT")
writeFile(t, filepath.Join(deployRoot, CertsDirName, "privkey.pem"), "KEY")
return dataDir, dbPath, deployRoot
}
func writeFile(t *testing.T, path, content string) {
@@ -87,21 +97,26 @@ func readArchive(t *testing.T, data []byte) map[string]string {
}
func TestCreateIncludesStateExcludesTransient(t *testing.T) {
dataDir, dbPath := seedDataDir(t)
dataDir, dbPath, deployRoot := seedDataDir(t)
var buf bytes.Buffer
if err := Create(&buf, Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3"}); err != nil {
if err := Create(&buf, Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3", DeployRoot: deployRoot}); err != nil {
t.Fatalf("Create: %v", err)
}
files := readArchive(t, buf.Bytes())
// Present.
for _, name := range []string{
ManifestName,
"selfpost.db",
"opendkim/keys/example.com/selfpost.private",
"sasl/sasldb2",
"postfix/sender_login_maps",
DataArchivePrefix + ManifestName,
DataArchivePrefix + "selfpost.db",
DataArchivePrefix + "opendkim/keys/example.com/selfpost.private",
DataArchivePrefix + "sasl/sasldb2",
DataArchivePrefix + "postfix/sender_login_maps",
DataArchivePrefix + "postfix/queue/deferred/sample",
ComposeFileName,
EnvFileName,
CertsDirName + "/fullchain.pem",
CertsDirName + "/privkey.pem",
} {
if _, ok := files[name]; !ok {
t.Errorf("archive missing %s", name)
@@ -109,8 +124,11 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) {
}
// Excluded.
for _, name := range []string{
"setup-token", "selfpost.db-wal", "selfpost.db-shm",
"log/mail.log", "log/mail.log.1",
DataArchivePrefix + "setup-token",
DataArchivePrefix + "selfpost.db-wal",
DataArchivePrefix + "selfpost.db-shm",
DataArchivePrefix + "log/mail.log",
DataArchivePrefix + "log/mail.log.1",
} {
if _, ok := files[name]; ok {
t.Errorf("archive should not contain %s", name)
@@ -119,7 +137,7 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) {
// Manifest is well-formed and carries the version.
var m Manifest
if err := json.Unmarshal([]byte(files[ManifestName]), &m); err != nil {
if err := json.Unmarshal([]byte(files[DataArchivePrefix+ManifestName]), &m); err != nil {
t.Fatalf("manifest json: %v", err)
}
if m.Format != FormatFull || m.Version != "1.2.3" {
@@ -128,7 +146,7 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) {
// The archived selfpost.db is a real, openable SQLite snapshot with our data.
snapPath := filepath.Join(t.TempDir(), "restored.db")
if err := os.WriteFile(snapPath, []byte(files["selfpost.db"]), 0o640); err != nil {
if err := os.WriteFile(snapPath, []byte(files[DataArchivePrefix+"selfpost.db"]), 0o640); err != nil {
t.Fatalf("write snapshot: %v", err)
}
st, err := store.Open(snapPath)
@@ -145,6 +163,14 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) {
}
}
func TestCreateRequiresDeployRoot(t *testing.T) {
dataDir, dbPath, _ := seedDataDir(t)
var buf bytes.Buffer
if err := Create(&buf, Params{DataDir: dataDir, DBPath: dbPath, Version: "1.0.0"}); err == nil {
t.Fatal("Create without DeployRoot succeeded")
}
}
func writeManifest(t *testing.T, dir, format, version string) string {
t.Helper()
path := filepath.Join(dir, ManifestName)