audit 3: a restore is ended by its Resync, and half an archive is refused
test / test (push) Canceled after 0s
test / test (push) Canceled after 0s
The restore manifest was consumed as soon as its version was checked, before store.Open and the Resync; a start that failed in between left a data directory the next start took for an ordinary one (M1). openData now runs CheckRestore, store.Open, the Resync, and only then ConsumeRestore. A truncated archive had nothing to refuse it: the manifest is the first entry, so a download cut short or a tar stopped by a damaged block was a restore on half a database. Every archive now ends with data/contents.json, the size and SHA-256 of each data/ file (deploy files are the operator's to edit), and CheckRestore compares /data with it before the database is opened, then renames it contents.checked.json so a start after a failed one does not compare again unless the manifest changed. Tests: a start whose Resync fails, then a start that resyncs and consumes; an archive cut in the database, before the deploy files and in its last entry (refused, database untouched, no -wal); a damaged key named; the record's names confined to data/; the domain import of a cut plain export, a cut and an altered .spde (refused, nothing created). e2e on the test server: full_backup_restore now also sees contents.json in the archive and gone after the first start, through the real entrypoint and daemons (all pass, 762 s); go vet + go test of 24 packages pass there. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+58
-44
@@ -15,7 +15,8 @@
|
||||
// restore-specific steps the panel runs are CheckRestore, which refuses to
|
||||
// boot if the manifest's version does not match the running binary so
|
||||
// schema/format skew between versions cannot silently corrupt state
|
||||
// (architecture.md § Persistence), and a one-time Resync of OpenDKIM's tables
|
||||
// (architecture.md § Persistence), or if the archive was unpacked in part
|
||||
// (ContentsName), and a one-time Resync of OpenDKIM's tables
|
||||
// and the Postfix sender map from SQLite on that first boot, so any drift
|
||||
// between the archive and the database is healed before mail flows. If
|
||||
// on-disk state drifts again later — for example after a manual edit under
|
||||
@@ -25,6 +26,7 @@ package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
@@ -113,12 +115,15 @@ var excludedFromArchive = map[string]bool{
|
||||
"tls": true,
|
||||
"log": true,
|
||||
ManifestName: true,
|
||||
ContentsName: true,
|
||||
contentsCheckedName: true,
|
||||
}
|
||||
|
||||
// Create writes a gzip-compressed tar backup to w. Archive layout:
|
||||
//
|
||||
// data/manifest.json, data/selfpost.db, data/<rest of /data>
|
||||
// docker-compose.yml, .env, certs/...
|
||||
// data/contents.json
|
||||
//
|
||||
// Extract the archive into an empty project directory, then docker compose up.
|
||||
func Create(w io.Writer, p Params) error {
|
||||
@@ -140,6 +145,7 @@ func Create(w io.Writer, p Params) error {
|
||||
|
||||
gz := gzip.NewWriter(w)
|
||||
tw := tar.NewWriter(gz)
|
||||
a := &archive{tw: tw}
|
||||
|
||||
manifest := Manifest{
|
||||
Format: FormatFull,
|
||||
@@ -150,21 +156,26 @@ func Create(w io.Writer, p Params) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: encode manifest: %w", err)
|
||||
}
|
||||
if err := writeTarBytes(tw, DataArchivePrefix+ManifestName, 0o600, manifestJSON); err != nil {
|
||||
if err := writeTarBytes(a, DataArchivePrefix+ManifestName, 0o600, manifestJSON); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The consistent SQLite snapshot, under the canonical filename the panel
|
||||
// opens on start (the live file and its WAL/SHM are excluded from the walk).
|
||||
if err := writeTarFile(tw, DataArchivePrefix+"selfpost.db", 0o640, snapshot); err != nil {
|
||||
if err := writeTarFile(a, DataArchivePrefix+"selfpost.db", 0o640, snapshot); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := addTree(tw, p.DataDir, DataArchivePrefix, excludedFromArchive, skipQueue, false); err != nil {
|
||||
if err := addTree(a, p.DataDir, DataArchivePrefix, excludedFromArchive, skipQueue, false); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := addDeployFiles(tw, p.DeployRoot, p.OnWarn); err != nil {
|
||||
if err := addDeployFiles(a, p.DeployRoot, p.OnWarn); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Last, so that only an archive unpacked to its end has it.
|
||||
if err := a.writeContents(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -249,7 +260,7 @@ func UnreadableCert(err error, deployRoot string) (name string, ok bool) {
|
||||
return CertsDirName + "/" + filepath.ToSlash(rel), true
|
||||
}
|
||||
|
||||
func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string)) error {
|
||||
func addDeployFiles(a *archive, deployRoot string, onWarn func(string)) error {
|
||||
for _, name := range []string{ComposeFileName, EnvFileName} {
|
||||
src := filepath.Join(deployRoot, name)
|
||||
info, err := os.Stat(src)
|
||||
@@ -259,7 +270,7 @@ func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string)) erro
|
||||
if !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("backup: deploy file %s is not a regular file", name)
|
||||
}
|
||||
if err := writeTarFile(tw, name, info.Mode().Perm(), src); err != nil {
|
||||
if err := writeTarFile(a, name, info.Mode().Perm(), src); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -286,19 +297,19 @@ func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string)) erro
|
||||
// are links again, relative, into archive/ (eff-certbot docs, "Where are
|
||||
// my certificates?"; assumptions C-Q4, C-A1). The archive carries the
|
||||
// files they lead to, under the links' names.
|
||||
return addTree(tw, certsDir, CertsDirName+"/", nil, nil, true)
|
||||
return addTree(a, certsDir, CertsDirName+"/", nil, nil, true)
|
||||
}
|
||||
|
||||
// addTree walks root and adds every regular file (and directory, to preserve
|
||||
// empty ones and modes) to tw under archivePrefix + path relative to root.
|
||||
// empty ones and modes) to a under archivePrefix + path relative to root.
|
||||
// When exclude is non-nil, top-level names relative to root are skipped; skip,
|
||||
// when non-nil, leaves out whatever it says. With a nil tw nothing is written:
|
||||
// when non-nil, leaves out whatever it says. With a nil a nothing is written:
|
||||
// every file is opened and closed, which is how Check finds what cannot be read.
|
||||
// Non-regular, non-directory entries (symlinks, sockets) are skipped - except,
|
||||
// with follow, a symbolic link to a regular file, which is archived as that
|
||||
// file under the link's name; a link that leads nowhere this process can
|
||||
// reach is an error, as an unreadable file is.
|
||||
func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool, skip skipFunc, follow bool) error {
|
||||
func addTree(a *archive, root, archivePrefix string, exclude map[string]bool, skip skipFunc, follow bool) error {
|
||||
return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -334,7 +345,7 @@ func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool
|
||||
}
|
||||
switch {
|
||||
case d.IsDir():
|
||||
if tw == nil {
|
||||
if a == nil {
|
||||
return nil
|
||||
}
|
||||
hdr := &tar.Header{
|
||||
@@ -343,9 +354,9 @@ func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool
|
||||
Mode: int64(info.Mode().Perm()),
|
||||
ModTime: info.ModTime(),
|
||||
}
|
||||
return tw.WriteHeader(hdr)
|
||||
return a.tw.WriteHeader(hdr)
|
||||
case info.Mode().IsRegular():
|
||||
err := writeTarFile(tw, name, info.Mode().Perm(), path)
|
||||
err := writeTarFile(a, name, info.Mode().Perm(), path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// A writer's temporary file, renamed into place between
|
||||
// the listing and the open: not part of /data's state.
|
||||
@@ -360,7 +371,7 @@ func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool
|
||||
if !target.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
return writeTarFile(tw, name, target.Mode().Perm(), path)
|
||||
return writeTarFile(a, name, target.Mode().Perm(), path)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
@@ -368,21 +379,14 @@ func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool
|
||||
}
|
||||
|
||||
// writeTarBytes writes an in-memory file entry.
|
||||
func writeTarBytes(tw *tar.Writer, name string, mode int64, data []byte) error {
|
||||
hdr := &tar.Header{
|
||||
func writeTarBytes(a *archive, name string, mode int64, data []byte) error {
|
||||
return a.writeFile(&tar.Header{
|
||||
Typeflag: tar.TypeReg,
|
||||
Name: name,
|
||||
Mode: mode,
|
||||
Size: int64(len(data)),
|
||||
ModTime: time.Now().UTC(),
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return fmt.Errorf("backup: write header %s: %w", name, err)
|
||||
}
|
||||
if _, err := tw.Write(data); err != nil {
|
||||
return fmt.Errorf("backup: write %s: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}, bytes.NewReader(data))
|
||||
}
|
||||
|
||||
// writeTarFile streams a file from disk into the archive under name.
|
||||
@@ -394,13 +398,13 @@ func writeTarBytes(tw *tar.Writer, name string, mode int64, data []byte) error {
|
||||
// the queue are not walked), and a rename leaves the file open here as it
|
||||
// was: the archive holds the old file or the new one, whole (assumptions
|
||||
// C-Q3).
|
||||
func writeTarFile(tw *tar.Writer, name string, mode fs.FileMode, srcPath string) error {
|
||||
func writeTarFile(a *archive, name string, mode fs.FileMode, srcPath string) error {
|
||||
f, err := os.Open(srcPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: open %s: %w", srcPath, err)
|
||||
}
|
||||
defer f.Close()
|
||||
if tw == nil {
|
||||
if a == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -408,20 +412,13 @@ func writeTarFile(tw *tar.Writer, name string, mode fs.FileMode, srcPath string)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: stat %s: %w", srcPath, err)
|
||||
}
|
||||
hdr := &tar.Header{
|
||||
return a.writeFile(&tar.Header{
|
||||
Typeflag: tar.TypeReg,
|
||||
Name: name,
|
||||
Mode: int64(mode.Perm()),
|
||||
Size: info.Size(),
|
||||
ModTime: info.ModTime(),
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return fmt.Errorf("backup: write header %s: %w", name, err)
|
||||
}
|
||||
if _, err := io.Copy(tw, f); err != nil {
|
||||
return fmt.Errorf("backup: copy %s: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}, f)
|
||||
}
|
||||
|
||||
// snapshotDB produces a consistent copy of the SQLite database at dbPath using
|
||||
@@ -463,10 +460,13 @@ func snapshotDB(dbPath string) (path string, cleanup func(), err error) {
|
||||
// CheckRestore enforces the backup version guard (architecture.md §
|
||||
// Persistence). If manifestPath exists (a backup was extracted into the data
|
||||
// directory), its version must match binaryVersion or the panel refuses to
|
||||
// start, telling the operator which image tag to use. On a match the manifest
|
||||
// is consumed (deleted) so it guards only the first boot after a restore and
|
||||
// never blocks a later in-place image upgrade, and restored is true so the
|
||||
// caller can heal drifted daemon maps once. Absence of the manifest is the
|
||||
// start, telling the operator which image tag to use; and the data directory
|
||||
// must hold every file the archive recorded at its end, as the archive holds
|
||||
// it, or the panel refuses to start on an archive unpacked in part (see
|
||||
// ContentsName). Then restored is true so the caller can heal drifted daemon
|
||||
// maps once. The manifest stays until the caller has done that and calls
|
||||
// ConsumeRestore: a start that fails on the way leaves it, and the next start
|
||||
// is a restore again (audit 2026-10-12, M1). Absence of the manifest is the
|
||||
// normal case and returns restored == false with a nil error.
|
||||
func CheckRestore(manifestPath, binaryVersion string) (restored bool, err error) {
|
||||
data, err := os.ReadFile(manifestPath)
|
||||
@@ -489,10 +489,24 @@ func CheckRestore(manifestPath, binaryVersion string) (restored bool, err error)
|
||||
"backup: this backup was created by SelfPost %s but this image is %s — restore into the matching image (selfpost:%s)",
|
||||
m.Version, binaryVersion, m.Version)
|
||||
}
|
||||
// Version matches: consume the manifest so subsequent normal starts (and
|
||||
// in-place upgrades) are not gated by it.
|
||||
if err := os.Remove(manifestPath); err != nil {
|
||||
return false, fmt.Errorf("backup: consume restore manifest: %w", err)
|
||||
if err := checkContents(filepath.Dir(manifestPath)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// ConsumeRestore ends a restore once its first start has done what a restore
|
||||
// start does: the manifest is removed, so later starts (and in-place
|
||||
// upgrades) are not gated by it, and then the record of the checked files. A
|
||||
// start stopped between the two leaves the record alone, which nothing reads
|
||||
// without a manifest and no backup carries.
|
||||
func ConsumeRestore(manifestPath string) error {
|
||||
if err := os.Remove(manifestPath); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("backup: consume restore manifest: %w", err)
|
||||
}
|
||||
checked := filepath.Join(filepath.Dir(manifestPath), contentsCheckedName)
|
||||
if err := os.Remove(checked); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("backup: consume restore manifest: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -200,9 +202,33 @@ func TestCheckRestoreNoManifestIsNormalStart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckRestoreMatchConsumesManifest(t *testing.T) {
|
||||
// unpackData writes the data directory of a Create archive into a new
|
||||
// directory and returns the manifest's path there.
|
||||
func unpackData(t *testing.T, archive []byte) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := writeManifest(t, dir, FormatFull, "1.0.0")
|
||||
for name, body := range readArchive(t, archive) {
|
||||
if rel, ok := strings.CutPrefix(name, DataArchivePrefix); ok {
|
||||
writeFile(t, filepath.Join(dir, filepath.FromSlash(rel)), body)
|
||||
}
|
||||
}
|
||||
return filepath.Join(dir, ManifestName)
|
||||
}
|
||||
|
||||
func createArchive(t *testing.T, version string) []byte {
|
||||
t.Helper()
|
||||
dataDir, dbPath, deployRoot := seedDataDir(t)
|
||||
var buf bytes.Buffer
|
||||
if err := Create(&buf, Params{DataDir: dataDir, DBPath: dbPath, Version: version, DeployRoot: deployRoot}); err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// A matching restore keeps its manifest until the start that runs it is done
|
||||
// with it (audit 2026-10-12, M1): CheckRestore reports, ConsumeRestore ends.
|
||||
func TestCheckRestoreMatchKeepsTheManifestUntilConsumed(t *testing.T) {
|
||||
path := unpackData(t, createArchive(t, "1.0.0"))
|
||||
restored, err := CheckRestore(path, "1.0.0")
|
||||
if err != nil {
|
||||
t.Fatalf("CheckRestore matching = %v, want nil", err)
|
||||
@@ -210,8 +236,145 @@ func TestCheckRestoreMatchConsumesManifest(t *testing.T) {
|
||||
if !restored {
|
||||
t.Fatal("CheckRestore matching did not report a restore")
|
||||
}
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Errorf("manifest should be consumed after a matching restore, stat err = %v", err)
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("CheckRestore consumed the manifest before the start was done with it: %v", err)
|
||||
}
|
||||
if err := ConsumeRestore(path); err != nil {
|
||||
t.Fatalf("ConsumeRestore: %v", err)
|
||||
}
|
||||
for _, name := range []string{ManifestName, ContentsName, contentsCheckedName} {
|
||||
if _, err := os.Stat(filepath.Join(filepath.Dir(path), name)); !os.IsNotExist(err) {
|
||||
t.Errorf("%s left after ConsumeRestore (stat err = %v)", name, err)
|
||||
}
|
||||
}
|
||||
if err := ConsumeRestore(path); err != nil {
|
||||
t.Errorf("ConsumeRestore on a consumed restore = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The archive's last entry records the data directory's files - every one,
|
||||
// with its size and digest - and nothing of the deploy files, which the
|
||||
// operator edits before the first start.
|
||||
func TestCreateEndsWithTheRecordOfItsData(t *testing.T) {
|
||||
archive := createArchive(t, "1.0.0")
|
||||
gz, err := gzip.NewReader(bytes.NewReader(archive))
|
||||
if err != nil {
|
||||
t.Fatalf("gzip: %v", err)
|
||||
}
|
||||
tr := tar.NewReader(gz)
|
||||
var last string
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("tar next: %v", err)
|
||||
}
|
||||
last = hdr.Name
|
||||
}
|
||||
if last != DataArchivePrefix+ContentsName {
|
||||
t.Fatalf("the last entry is %s, want %s", last, DataArchivePrefix+ContentsName)
|
||||
}
|
||||
|
||||
files := readArchive(t, archive)
|
||||
var c Contents
|
||||
if err := json.Unmarshal([]byte(files[DataArchivePrefix+ContentsName]), &c); err != nil {
|
||||
t.Fatalf("contents: %v", err)
|
||||
}
|
||||
recorded := map[string]FileSum{}
|
||||
for _, f := range c.Files {
|
||||
recorded[f.Name] = f
|
||||
}
|
||||
for name, body := range files {
|
||||
if name == DataArchivePrefix+ContentsName {
|
||||
continue
|
||||
}
|
||||
f, ok := recorded[name]
|
||||
if !strings.HasPrefix(name, DataArchivePrefix) {
|
||||
if ok {
|
||||
t.Errorf("the record names the deploy file %s", name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
sum := sha256.Sum256([]byte(body))
|
||||
if !ok || f.Size != int64(len(body)) || f.SHA256 != hex.EncodeToString(sum[:]) {
|
||||
t.Errorf("%s recorded as %+v, want size %d sha256 %x", name, f, len(body), sum)
|
||||
}
|
||||
delete(recorded, name)
|
||||
}
|
||||
for name := range recorded {
|
||||
t.Errorf("the record names %s, which the archive does not hold", name)
|
||||
}
|
||||
}
|
||||
|
||||
// Without the archive's last entry, or with a file that is not the one it
|
||||
// holds, a restore is refused, and the manifest stays for the next try.
|
||||
func TestCheckRestoreRefusesAnArchiveUnpackedInPart(t *testing.T) {
|
||||
sasldb := func(body string) func(string) {
|
||||
return func(dir string) { writeFile(t, filepath.Join(dir, "sasl", "sasldb2"), body) }
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
break_ func(dir string)
|
||||
want string
|
||||
}{
|
||||
{"no record", func(dir string) { os.Remove(filepath.Join(dir, ContentsName)) }, "contents.json, its last entry, is not there"},
|
||||
{"a record cut", func(dir string) { writeFile(t, filepath.Join(dir, ContentsName), `{"format":"selfpost-backup-con`) }, "is not a record"},
|
||||
{"a file missing", func(dir string) { os.Remove(filepath.Join(dir, "sasl", "sasldb2")) }, "data/sasl/sasldb2 is missing"},
|
||||
{"a file cut", sasldb("SASL"), "data/sasl/sasldb2 is not the file"},
|
||||
{"a file altered", sasldb("SASLDC"), "data/sasl/sasldb2 is not the file"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
path := unpackData(t, createArchive(t, "1.0.0"))
|
||||
c.break_(filepath.Dir(path))
|
||||
restored, err := CheckRestore(path, "1.0.0")
|
||||
if err == nil || !strings.Contains(err.Error(), c.want) || !strings.Contains(err.Error(), "not unpacked whole") {
|
||||
t.Fatalf("CheckRestore = %v, want a refusal saying %q", err, c.want)
|
||||
}
|
||||
if restored {
|
||||
t.Error("a refused restore reported a restore")
|
||||
}
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Errorf("the refusal removed the manifest: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A record names files of the data directory only; one that names another
|
||||
// path is refused, not followed.
|
||||
func TestCheckRestoreRefusesARecordOutsideTheDataDirectory(t *testing.T) {
|
||||
for _, name := range []string{"data/../.env", "data/", "docker-compose.yml", "data//sasl/sasldb2", "data/./x", "data/.."} {
|
||||
path := unpackData(t, createArchive(t, "1.0.0"))
|
||||
body, _ := json.Marshal(Contents{Format: FormatContents, Files: []FileSum{{Name: name}}})
|
||||
writeFile(t, filepath.Join(filepath.Dir(path), ContentsName), string(body))
|
||||
if _, err := CheckRestore(path, "1.0.0"); err == nil || !strings.Contains(err.Error(), "not a file of the data directory") {
|
||||
t.Errorf("a record naming %q = %v, want it refused", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The files are compared once per restore: a start that failed after the
|
||||
// check had the database open, and the next start must not take the changes
|
||||
// it made for damage. A different archive unpacked over the directory since
|
||||
// is compared anew.
|
||||
func TestCheckRestoreComparesTheFilesOncePerRestore(t *testing.T) {
|
||||
path := unpackData(t, createArchive(t, "1.0.0"))
|
||||
dir := filepath.Dir(path)
|
||||
if _, err := CheckRestore(path, "1.0.0"); err != nil {
|
||||
t.Fatalf("first CheckRestore: %v", err)
|
||||
}
|
||||
writeFile(t, filepath.Join(dir, "selfpost.db"), "changed by the start that failed")
|
||||
if _, err := CheckRestore(path, "1.0.0"); err != nil {
|
||||
t.Fatalf("the start after a failed one compared the files again: %v", err)
|
||||
}
|
||||
|
||||
// Another archive's manifest, its record lost on the way.
|
||||
b, _ := json.Marshal(Manifest{Format: FormatFull, Version: "1.0.0", CreatedAt: "another time"})
|
||||
writeFile(t, path, string(b))
|
||||
if _, err := CheckRestore(path, "1.0.0"); err == nil || !strings.Contains(err.Error(), "not unpacked whole") {
|
||||
t.Fatalf("another archive's manifest over a checked restore = %v, want it compared anew and refused", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// FormatContents identifies the record of what an archive's data directory
|
||||
// holds.
|
||||
const FormatContents = "selfpost-backup-contents"
|
||||
|
||||
// ContentsName is the record of the data directory's files, the last entry of
|
||||
// every archive (data/contents.json). The manifest is the first entry and
|
||||
// says a restore is under way; this one says the archive was unpacked to its
|
||||
// end. A download cut short, or a tar that stopped at a damaged block, leaves
|
||||
// a data directory with the manifest and without this file, and the panel
|
||||
// refuses to start on it instead of opening half a database (audit
|
||||
// 2026-10-12, stage 3).
|
||||
const ContentsName = "contents.json"
|
||||
|
||||
// contentsCheckedName is ContentsName once a start has checked the files
|
||||
// against it. The check is made once: the start that made it opens the
|
||||
// database, and from then on the files are the panel's to change, so a start
|
||||
// after a failed one (manifest still there) must not compare them again.
|
||||
const contentsCheckedName = "contents.checked.json"
|
||||
|
||||
// Contents is the record written as ContentsName.
|
||||
type Contents struct {
|
||||
Format string `json:"format"`
|
||||
Files []FileSum `json:"files"`
|
||||
}
|
||||
|
||||
// FileSum is one regular file of the data directory as the archive holds it,
|
||||
// named as in the archive (data/...).
|
||||
type FileSum struct {
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
// archive is the tar stream a backup is written to, and the record of the
|
||||
// data directory's files written to it so far. A nil *archive writes
|
||||
// nothing: every file is opened and closed, which is how Check finds what
|
||||
// cannot be read.
|
||||
//
|
||||
// Only the data directory is recorded. The deploy files are the operator's to
|
||||
// edit before the first start - the guide sends them to .env for a new
|
||||
// hostname and to docker-compose.yml for the tag - and the data directory is
|
||||
// what the panel opens.
|
||||
type archive struct {
|
||||
tw *tar.Writer
|
||||
files []FileSum
|
||||
}
|
||||
|
||||
// writeFile writes one regular file's header and content, read from r, and
|
||||
// records it when it belongs to the data directory.
|
||||
func (a *archive) writeFile(hdr *tar.Header, r io.Reader) error {
|
||||
if err := a.tw.WriteHeader(hdr); err != nil {
|
||||
return fmt.Errorf("backup: write header %s: %w", hdr.Name, err)
|
||||
}
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(io.MultiWriter(a.tw, h), r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: copy %s: %w", hdr.Name, err)
|
||||
}
|
||||
if strings.HasPrefix(hdr.Name, DataArchivePrefix) {
|
||||
a.files = append(a.files, FileSum{Name: hdr.Name, Size: n, SHA256: hex.EncodeToString(h.Sum(nil))})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeContents ends the archive's data with the record of what it holds.
|
||||
func (a *archive) writeContents() error {
|
||||
body, err := json.MarshalIndent(Contents{Format: FormatContents, Files: a.files}, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: encode contents: %w", err)
|
||||
}
|
||||
return writeTarBytes(&archive{tw: a.tw}, DataArchivePrefix+ContentsName, 0o600, body)
|
||||
}
|
||||
|
||||
// errUnpackedPart is what a restore that is not whole is refused with.
|
||||
func errUnpackedPart(detail string) error {
|
||||
return fmt.Errorf("backup: the archive was not unpacked whole (%s) - it was cut short or is damaged; nothing was opened. Unpack it again, from an intact copy, into an empty directory", detail)
|
||||
}
|
||||
|
||||
// checkContents compares the data directory with the archive's record, once
|
||||
// per restore (see contentsCheckedName).
|
||||
func checkContents(dataDir string) error {
|
||||
checked := filepath.Join(dataDir, contentsCheckedName)
|
||||
if c, err := readContents(checked); err == nil {
|
||||
// A start before this one checked the files. It is the same restore
|
||||
// when the manifest is still the one that record holds; a different
|
||||
// archive unpacked over the directory since is checked anew.
|
||||
if verifyFiles(dataDir, c, DataArchivePrefix+ManifestName) == nil {
|
||||
return nil
|
||||
}
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
|
||||
path := filepath.Join(dataDir, ContentsName)
|
||||
c, err := readContents(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return errUnpackedPart(DataArchivePrefix + ContentsName + ", its last entry, is not there")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verifyFiles(dataDir, c, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(path, checked); err != nil {
|
||||
return fmt.Errorf("backup: mark the restored files checked: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readContents reads a record of the archive's files.
|
||||
func readContents(path string) (Contents, error) {
|
||||
var c Contents
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
if err := json.Unmarshal(data, &c); err != nil || c.Format != FormatContents {
|
||||
return c, errUnpackedPart(filepath.Base(path) + " is not a record of the archive's files")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// verifyFiles compares the files c records with those under dataDir: each
|
||||
// present, of its size, with its digest. A non-empty only limits it to the
|
||||
// file of that name.
|
||||
func verifyFiles(dataDir string, c Contents, only string) error {
|
||||
found := false
|
||||
for _, f := range c.Files {
|
||||
if only != "" && f.Name != only {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
rel, ok := strings.CutPrefix(f.Name, DataArchivePrefix)
|
||||
if !ok || rel == "" || path.Clean(rel) != rel || rel == ".." || strings.HasPrefix(rel, "../") || path.IsAbs(rel) {
|
||||
return errUnpackedPart("the record names " + f.Name + ", which is not a file of the data directory")
|
||||
}
|
||||
file, err := os.Open(filepath.Join(dataDir, filepath.FromSlash(rel)))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return errUnpackedPart(f.Name + " is missing")
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: check %s: %w", f.Name, err)
|
||||
}
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(h, file)
|
||||
file.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: check %s: %w", f.Name, err)
|
||||
}
|
||||
if n != f.Size || hex.EncodeToString(h.Sum(nil)) != f.SHA256 {
|
||||
return errUnpackedPart(f.Name + " is not the file the archive holds")
|
||||
}
|
||||
}
|
||||
if only != "" && !found {
|
||||
return errUnpackedPart("the record does not name " + only)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package handlers
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"html"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -10,6 +11,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/mixeme/selfpost/internal/domain"
|
||||
"github.com/mixeme/selfpost/internal/secretfile"
|
||||
"github.com/mixeme/selfpost/internal/store"
|
||||
"github.com/mixeme/selfpost/internal/web/auth"
|
||||
)
|
||||
@@ -116,3 +118,57 @@ func TestDomainImportRefusedCreatesNothing(t *testing.T) {
|
||||
t.Fatalf("the refused import created %v", domains)
|
||||
}
|
||||
}
|
||||
|
||||
// A domain export cut short or altered on the way is refused with what is
|
||||
// wrong with it, before anything is created (audit 2026-10-12, stage 3).
|
||||
func TestDomainImportOfADamagedFileCreatesNothing(t *testing.T) {
|
||||
h, st := inboundHandlers(t)
|
||||
root := userWith(t, st, "root", store.RoleGlobal, store.Reach{})
|
||||
plain, _ := json.Marshal(domain.DomainExport{
|
||||
Format: domain.FormatDomainExport, Domain: "example.com", DKIMSelector: "mail",
|
||||
Applications: []domain.AppExport{{Login: "app1", AddressMode: store.AddressModeWildcard, Password: "imported-password"}},
|
||||
})
|
||||
const password = "a-long-enough-password"
|
||||
var sealed bytes.Buffer
|
||||
w, err := secretfile.NewWriter(&sealed, secretfile.TypeDomainExport, password)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write(plain)
|
||||
w.Close()
|
||||
flipped := bytes.Clone(sealed.Bytes())
|
||||
flipped[len(flipped)-20] ^= 1
|
||||
|
||||
for _, c := range []struct {
|
||||
name, file, password string
|
||||
body []byte
|
||||
want string
|
||||
}{
|
||||
{"plain, cut", "example.com.json", "", plain[:len(plain)/2], "That file is not a valid SelfPost domain export."},
|
||||
{"encrypted, cut", "example.com.spde", password, sealed.Bytes()[:sealed.Len()-10], "That file is damaged or incomplete."},
|
||||
{"encrypted, altered", "example.com.spde", password, flipped, "Wrong password, or the file has been altered since it was exported."},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
var body bytes.Buffer
|
||||
mw := multipart.NewWriter(&body)
|
||||
fw, _ := mw.CreateFormFile("file", c.file)
|
||||
fw.Write(c.body)
|
||||
mw.WriteField("import_password", c.password)
|
||||
mw.Close()
|
||||
req := httptest.NewRequest("POST", "/server/backup/import", &body)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
req = auth.RequestWithPrincipal(req, root)
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleImportDomain(rec, req)
|
||||
if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), html.EscapeString(c.want)) {
|
||||
t.Fatalf("import = %d, want 400 saying %q:\n%s", rec.Code, c.want, rec.Body.String())
|
||||
}
|
||||
if domains, _ := st.ListDomains(); len(domains) != 0 {
|
||||
t.Fatalf("the refused import created %v", domains)
|
||||
}
|
||||
if _, err := st.GetApplicationByLogin("app1"); err == nil {
|
||||
t.Fatal("the refused import created the application")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user