Phase 9: full backup/restore + domain export/import (spec 7.5, 11.6)
Full server backup (spec 7.5.A): internal/backup produces a tar.gz of all of /data — a consistent SQLite snapshot via VACUUM INTO, DKIM keys, sasldb2 and a version manifest; TLS certs (tls/) and the Postfix queue are excluded. Two equal paths: the panel button (POST /backup, no-store) and the selfpost-backup CLI via docker exec (spec 11.6). CheckRestore runs before store.Open: a manifest version mismatch refuses to boot with the image tag to use; a match consumes the manifest so it only guards the first post-restore boot. Restore is not a separate branch — Postfix/OpenDKIM regenerate from the restored SQLite as on any start. Domain export/import (spec 7.5.B): DomainExport carries the DKIM private key and each application's working password. SASL secrets are read from sasldb2 via db_dump (the userPassword property is plaintext) and, on import, re-keyed under the local realm with saslpasswd2 — so credentials keep working on an instance with a different hostname, with no DKIM DNS change. Import validates and rolls back atomically on any failure. db-util (db_dump) is now an explicit image dep. Verified on the server (selfpost:p9): gofmt/vet/test green; container e2e for cross-realm domain export/import (SMTP AUTH 235 under the new realm), CLI and panel backups, same-version restore, and version-mismatch refusal. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,290 @@
|
||||
// Package backup implements SelfPost's full-server backup and the restore
|
||||
// version guard (spec 7.5.A). A full backup is a gzip-compressed tar of the
|
||||
// consolidated persistent state under /data — the SQLite database (as a
|
||||
// consistent snapshot), the per-domain DKIM keys and the SASL database — plus a
|
||||
// manifest recording the SelfPost version that produced it. TLS certificates
|
||||
// (the reverse proxy's responsibility) and the Postfix queue are deliberately
|
||||
// excluded (spec 7.5.A).
|
||||
//
|
||||
// Restore is intentionally not a separate code path: a backup is extracted into
|
||||
// the /data bind mount before first start, and the panel regenerates Postfix and
|
||||
// OpenDKIM from the restored SQLite state exactly as on any normal start. The
|
||||
// only restore-specific step is CheckRestore, which refuses to boot if the
|
||||
// manifest's version does not match the running binary, so schema/format skew
|
||||
// between versions cannot silently corrupt state (spec 7.5.A).
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite" // pure-Go SQLite driver, for the VACUUM INTO snapshot
|
||||
)
|
||||
|
||||
// FormatFull identifies a full-server backup manifest.
|
||||
const FormatFull = "selfpost-full-backup"
|
||||
|
||||
// ManifestName is the manifest's filename, both inside the archive and, after a
|
||||
// restore extraction, at the root of the data directory where CheckRestore
|
||||
// looks for it.
|
||||
const ManifestName = "manifest.json"
|
||||
|
||||
// Manifest is the small JSON document embedded in every backup archive. Its
|
||||
// Version is the single fact that makes restore safe: the panel refuses to boot
|
||||
// a data directory whose manifest version does not match its own binary (spec
|
||||
// 7.5.A).
|
||||
type Manifest struct {
|
||||
Format string `json:"format"`
|
||||
Version string `json:"version"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}
|
||||
|
||||
// Params configures a backup. DataDir is the consolidated state root (/data);
|
||||
// DBPath is the live SQLite file within it, snapshotted consistently rather than
|
||||
// copied byte-for-byte while it may be mid-write; Version is stamped into the
|
||||
// manifest.
|
||||
type Params struct {
|
||||
DataDir string
|
||||
DBPath string
|
||||
Version string
|
||||
}
|
||||
|
||||
// excludedFromArchive lists the data-directory entries a backup never carries.
|
||||
// The live database files are replaced by a consistent VACUUM INTO snapshot
|
||||
// written under the canonical name; the setup token is transient bootstrap
|
||||
// state; a stale manifest from a previous restore must not be re-captured (a
|
||||
// fresh one is written instead); and a "tls" directory holds the reverse
|
||||
// proxy's certificates, which are explicitly out of scope for a SelfPost backup
|
||||
// (spec 7.5.A) — excluding it keeps that guarantee even when an operator points
|
||||
// TLS_CERT_FILE inside /data.
|
||||
var excludedFromArchive = map[string]bool{
|
||||
"selfpost.db": true,
|
||||
"selfpost.db-wal": true,
|
||||
"selfpost.db-shm": true,
|
||||
"selfpost.db-journal": true,
|
||||
"setup-token": true,
|
||||
"tls": true,
|
||||
ManifestName: true,
|
||||
}
|
||||
|
||||
// Create writes a gzip-compressed tar backup to w. Archive entries are named
|
||||
// relative to DataDir, so extracting the archive into the /data bind mount
|
||||
// reconstructs the state in place (spec 7.5.A). The SQLite database is added as
|
||||
// a consistent snapshot under "selfpost.db"; everything else under DataDir is
|
||||
// copied as-is except the entries in excludedFromArchive.
|
||||
func Create(w io.Writer, p Params) error {
|
||||
if p.DataDir == "" || p.DBPath == "" {
|
||||
return fmt.Errorf("backup: DataDir and DBPath are required")
|
||||
}
|
||||
|
||||
snapshot, cleanup, err := snapshotDB(p.DBPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
gz := gzip.NewWriter(w)
|
||||
tw := tar.NewWriter(gz)
|
||||
|
||||
manifest := Manifest{
|
||||
Format: FormatFull,
|
||||
Version: p.Version,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
manifestJSON, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: encode manifest: %w", err)
|
||||
}
|
||||
if err := writeTarBytes(tw, ManifestName, 0o600, manifestJSON); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The consistent SQLite snapshot, under the canonical filename the panel
|
||||
// opens on start (the live file and its WAL/SHM are excluded from the walk).
|
||||
if err := writeTarFile(tw, "selfpost.db", 0o640, snapshot); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := addTree(tw, p.DataDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := tw.Close(); err != nil {
|
||||
return fmt.Errorf("backup: close tar: %w", err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
return fmt.Errorf("backup: close gzip: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// addTree walks dataDir and adds every regular file (and directory, to preserve
|
||||
// empty ones and modes) to tw under its path relative to dataDir, skipping the
|
||||
// excluded entries. Non-regular, non-directory entries (symlinks, sockets) are
|
||||
// skipped: /data holds none in normal operation, and copying them into a backup
|
||||
// would be meaningless or unsafe.
|
||||
func addTree(tw *tar.Writer, dataDir string) error {
|
||||
return filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, err := filepath.Rel(dataDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rel == "." {
|
||||
return nil // the data root itself is implicit
|
||||
}
|
||||
// filepath.Rel yields OS separators; tar names use forward slashes.
|
||||
name := filepath.ToSlash(rel)
|
||||
// Exclude by top-level name (the live DB, setup token and stale manifest
|
||||
// all live at the data root).
|
||||
if excludedFromArchive[name] {
|
||||
if d.IsDir() {
|
||||
return fs.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case d.IsDir():
|
||||
hdr := &tar.Header{
|
||||
Typeflag: tar.TypeDir,
|
||||
Name: name + "/",
|
||||
Mode: int64(info.Mode().Perm()),
|
||||
ModTime: info.ModTime(),
|
||||
}
|
||||
return tw.WriteHeader(hdr)
|
||||
case info.Mode().IsRegular():
|
||||
return writeTarFile(tw, name, info.Mode().Perm(), path)
|
||||
default:
|
||||
return nil // skip symlinks/sockets/devices
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// writeTarBytes writes an in-memory file entry.
|
||||
func writeTarBytes(tw *tar.Writer, name string, mode int64, data []byte) error {
|
||||
hdr := &tar.Header{
|
||||
Typeflag: tar.TypeReg,
|
||||
Name: name,
|
||||
Mode: mode,
|
||||
Size: int64(len(data)),
|
||||
ModTime: time.Now().UTC(),
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return fmt.Errorf("backup: write header %s: %w", name, err)
|
||||
}
|
||||
if _, err := tw.Write(data); err != nil {
|
||||
return fmt.Errorf("backup: write %s: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeTarFile streams a file from disk into the archive under name.
|
||||
func writeTarFile(tw *tar.Writer, name string, mode fs.FileMode, srcPath string) error {
|
||||
f, err := os.Open(srcPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: open %s: %w", srcPath, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: stat %s: %w", srcPath, err)
|
||||
}
|
||||
hdr := &tar.Header{
|
||||
Typeflag: tar.TypeReg,
|
||||
Name: name,
|
||||
Mode: int64(mode.Perm()),
|
||||
Size: info.Size(),
|
||||
ModTime: info.ModTime(),
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return fmt.Errorf("backup: write header %s: %w", name, err)
|
||||
}
|
||||
if _, err := io.Copy(tw, f); err != nil {
|
||||
return fmt.Errorf("backup: copy %s: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// snapshotDB produces a consistent copy of the SQLite database at dbPath using
|
||||
// VACUUM INTO, so the backup captures a coherent point-in-time image even while
|
||||
// the panel is writing to the live file under WAL. It returns the snapshot path
|
||||
// and a cleanup function the caller must defer.
|
||||
func snapshotDB(dbPath string) (path string, cleanup func(), err error) {
|
||||
dir, err := os.MkdirTemp("", "selfpost-backup-")
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("backup: temp dir: %w", err)
|
||||
}
|
||||
cleanup = func() { _ = os.RemoveAll(dir) }
|
||||
target := filepath.Join(dir, "selfpost.db")
|
||||
|
||||
// A short busy timeout lets VACUUM INTO wait out a brief writer rather than
|
||||
// failing immediately if the panel happens to be mid-write.
|
||||
db, err := sql.Open("sqlite", "file:"+dbPath+"?_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return "", nil, fmt.Errorf("backup: open database: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
// VACUUM INTO takes a string literal, not a bound parameter. target is a path
|
||||
// we generated (never user input); single quotes are doubled defensively.
|
||||
stmt := "VACUUM INTO '" + strings.ReplaceAll(target, "'", "''") + "'"
|
||||
if _, err := db.Exec(stmt); err != nil {
|
||||
cleanup()
|
||||
return "", nil, fmt.Errorf("backup: snapshot database: %w", err)
|
||||
}
|
||||
return target, cleanup, nil
|
||||
}
|
||||
|
||||
// CheckRestore enforces the backup version guard (spec 7.5.A). If manifestPath
|
||||
// exists (a backup was extracted into the data directory), its version must
|
||||
// match binaryVersion or the panel refuses to start, telling the operator which
|
||||
// image tag to use. On a match the manifest is consumed (deleted) so it guards
|
||||
// only the first boot after a restore and never blocks a later in-place image
|
||||
// upgrade. Absence of the manifest is the normal case and returns nil.
|
||||
func CheckRestore(manifestPath, binaryVersion string) error {
|
||||
data, err := os.ReadFile(manifestPath)
|
||||
if os.IsNotExist(err) {
|
||||
return nil // ordinary start, not a restore
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: read restore manifest: %w", err)
|
||||
}
|
||||
|
||||
var m Manifest
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
return fmt.Errorf("backup: restore manifest %s is not valid JSON: %w", manifestPath, err)
|
||||
}
|
||||
if m.Format != FormatFull {
|
||||
return fmt.Errorf("backup: %s is not a SelfPost full backup manifest (format %q)", manifestPath, m.Format)
|
||||
}
|
||||
if m.Version != binaryVersion {
|
||||
return fmt.Errorf(
|
||||
"backup: this backup was created by SelfPost %s but this image is %s — restore into the matching image (selfpost:%s)",
|
||||
m.Version, binaryVersion, m.Version)
|
||||
}
|
||||
// Version matches: consume the manifest so subsequent normal starts (and
|
||||
// in-place upgrades) are not gated by it.
|
||||
if err := os.Remove(manifestPath); err != nil {
|
||||
return fmt.Errorf("backup: consume restore manifest: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"codeberg.org/mix/selfpost/internal/store"
|
||||
)
|
||||
|
||||
// seedDataDir builds a realistic /data tree: a migrated SQLite database plus the
|
||||
// DKIM key, SASL and transient files a backup must include or exclude.
|
||||
func seedDataDir(t *testing.T) (dataDir, dbPath string) {
|
||||
t.Helper()
|
||||
dataDir = t.TempDir()
|
||||
dbPath = filepath.Join(dataDir, "selfpost.db")
|
||||
|
||||
st, err := store.Open(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
if _, err := st.AddDomain("example.com", "selfpost"); err != nil {
|
||||
t.Fatalf("add domain: %v", err)
|
||||
}
|
||||
if err := st.Close(); err != nil {
|
||||
t.Fatalf("close store: %v", err)
|
||||
}
|
||||
|
||||
writeFile(t, filepath.Join(dataDir, "opendkim", "keys", "example.com", "selfpost.private"), "PRIVATE KEY")
|
||||
writeFile(t, filepath.Join(dataDir, "sasl", "sasldb2"), "SASLDB")
|
||||
writeFile(t, filepath.Join(dataDir, "postfix", "sender_login_maps"), "@example.com login")
|
||||
// Transient files that must NOT be archived.
|
||||
writeFile(t, filepath.Join(dataDir, "setup-token"), "secret-token")
|
||||
writeFile(t, filepath.Join(dataDir, "selfpost.db-wal"), "wal")
|
||||
writeFile(t, filepath.Join(dataDir, "selfpost.db-shm"), "shm")
|
||||
return dataDir, dbPath
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, path, content string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o640); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// readArchive returns the set of regular-file entries (name -> content) in a
|
||||
// gzip tar produced by Create.
|
||||
func readArchive(t *testing.T, data []byte) map[string]string {
|
||||
t.Helper()
|
||||
gz, err := gzip.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("gzip: %v", err)
|
||||
}
|
||||
tr := tar.NewReader(gz)
|
||||
out := map[string]string{}
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("tar next: %v", err)
|
||||
}
|
||||
if hdr.Typeflag != tar.TypeReg {
|
||||
continue
|
||||
}
|
||||
b, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
t.Fatalf("tar read %s: %v", hdr.Name, err)
|
||||
}
|
||||
out[hdr.Name] = string(b)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
||||
dataDir, dbPath := seedDataDir(t)
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := Create(&buf, Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3"}); err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
files := readArchive(t, buf.Bytes())
|
||||
|
||||
// Present.
|
||||
for _, name := range []string{
|
||||
ManifestName,
|
||||
"selfpost.db",
|
||||
"opendkim/keys/example.com/selfpost.private",
|
||||
"sasl/sasldb2",
|
||||
"postfix/sender_login_maps",
|
||||
} {
|
||||
if _, ok := files[name]; !ok {
|
||||
t.Errorf("archive missing %s", name)
|
||||
}
|
||||
}
|
||||
// Excluded.
|
||||
for _, name := range []string{"setup-token", "selfpost.db-wal", "selfpost.db-shm"} {
|
||||
if _, ok := files[name]; ok {
|
||||
t.Errorf("archive should not contain %s", name)
|
||||
}
|
||||
}
|
||||
|
||||
// Manifest is well-formed and carries the version.
|
||||
var m Manifest
|
||||
if err := json.Unmarshal([]byte(files[ManifestName]), &m); err != nil {
|
||||
t.Fatalf("manifest json: %v", err)
|
||||
}
|
||||
if m.Format != FormatFull || m.Version != "1.2.3" {
|
||||
t.Errorf("manifest = %+v, want format=%s version=1.2.3", m, FormatFull)
|
||||
}
|
||||
|
||||
// The archived selfpost.db is a real, openable SQLite snapshot with our data.
|
||||
snapPath := filepath.Join(t.TempDir(), "restored.db")
|
||||
if err := os.WriteFile(snapPath, []byte(files["selfpost.db"]), 0o640); err != nil {
|
||||
t.Fatalf("write snapshot: %v", err)
|
||||
}
|
||||
st, err := store.Open(snapPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open snapshot: %v", err)
|
||||
}
|
||||
defer st.Close()
|
||||
domains, err := st.ListDomains()
|
||||
if err != nil {
|
||||
t.Fatalf("list domains from snapshot: %v", err)
|
||||
}
|
||||
if len(domains) != 1 || domains[0].Name != "example.com" {
|
||||
t.Errorf("snapshot domains = %+v, want one example.com", domains)
|
||||
}
|
||||
}
|
||||
|
||||
func writeManifest(t *testing.T, dir, format, version string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, ManifestName)
|
||||
b, _ := json.Marshal(Manifest{Format: format, Version: version, CreatedAt: "now"})
|
||||
if err := os.WriteFile(path, b, 0o600); err != nil {
|
||||
t.Fatalf("write manifest: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestCheckRestoreNoManifestIsNormalStart(t *testing.T) {
|
||||
if err := CheckRestore(filepath.Join(t.TempDir(), "manifest.json"), "1.0.0"); err != nil {
|
||||
t.Errorf("CheckRestore with no manifest = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckRestoreMatchConsumesManifest(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := writeManifest(t, dir, FormatFull, "1.0.0")
|
||||
if err := CheckRestore(path, "1.0.0"); err != nil {
|
||||
t.Fatalf("CheckRestore matching = %v, want nil", err)
|
||||
}
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Errorf("manifest should be consumed after a matching restore, stat err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckRestoreVersionMismatchRefusesAndKeeps(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := writeManifest(t, dir, FormatFull, "1.0.0")
|
||||
err := CheckRestore(path, "2.0.0")
|
||||
if err == nil {
|
||||
t.Fatal("CheckRestore mismatch = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "1.0.0") || !strings.Contains(err.Error(), "2.0.0") {
|
||||
t.Errorf("error should name both versions: %v", err)
|
||||
}
|
||||
if _, statErr := os.Stat(path); statErr != nil {
|
||||
t.Errorf("manifest must be kept on mismatch so the operator can switch images: %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckRestoreWrongFormatRejected(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := writeManifest(t, dir, "something-else", "1.0.0")
|
||||
if err := CheckRestore(path, "1.0.0"); err == nil {
|
||||
t.Error("CheckRestore accepted a non-backup manifest")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user