release: 1.3.0: 1.3.0-1.3.1: security pass, queue persist, full backup layout

[1.3.0] Security and quality after 1.2.5: domain-admin send-log authorization, fail-closed sign-in and application delete, level-2 rate-limit race fix, restore Resync, expanded tests, operator docs, release CI, and OFL for IBM Plex. Upgrading from 1.2.x is a tag bump; no migration. Added: - licence: the SIL Open Font License 1.1 text now travels with the IBM Plex - docs: agreed roadmap item **queue-retries** — show this Postfix's retry Security: - The independent security review of the send-log authorization and Fixed: - test (e2e): send-log status scrapers follow the badge markup in - security (panel): the Deliveries list is scoped to a domain administrator's - mail (level-2 rate limit): the ceiling is no longer overshot by messages that - panel (sign-in): a session that cannot be written to the database no longer - panel (applications): deleting an application removes its SASL credentials … (2 more in CHANGELOG) Changed: - docs: operator and as-built docs aligned with the code after a full - docs: [guide.md](docs/guide.md) reorganised into **Installation**, **Instance - docs: the 2026-08-13 full-tree review plan is complete — every phase (P0–P7) - licence: [NOTICE](NOTICE) tells modifiers to update `SourceURL` in - ci: the release image is published only for a **published** GitHub Release … (9 more in CHANGELOG) [1.3.1] Retry policy in the panel, persistent Postfix queue, and self-contained full backups. Mail queue and delivery history show this Postfix's first retry, backoff cap, and queue lifetime. Deferred mail survives container recreate. Full backups now archive `data/`, compose, `.env`, and `certs/`; the archive layout breaks 1.3.0 flat backups (unpack those with `tar xzf backup.tar.gz -C ./data` as before). Upgrading from 1.3.0 is a tag bump; no schema migration. Added: - Mail queue and a delivery's history show this Postfix's retry policy - docs: **Plan checklists** in [development.md](docs/development.md) — format, - docs: **Implementation checklists** with model routing in - docs: roadmap candidates **send-log-retention** (panel Settings for delivery - Postfix queue under `/data/postfix/queue` — deferred and active mail survive … (1 more in CHANGELOG) Changed: - **Breaking:** full-backup archive layout — paths are prefixed with `data/`; - docs: operator guide, architecture, security, and backup UI updated for - docs: HTML mockups for a full panel UI refresh Changed: - docs: panel UI mockups after review — Status restored to live readings - docs: panel UI design system and mockups rebuilt as separate pages - docs: panel UI mockups — add-domain sits in the list card (Domains and Fixed: - ci (GHCR): per-arch package tags (`X.Y.Z-amd64`, `X.Y.Z-arm64`) are dropped Squashed CHANGELOG sections: [1.3.0], [1.3.1]. Full detail: CHANGELOG.md (1.3.0, 1.3.1).
2026-09-22 15:24:52 +03:00
parent 60e08e802e
commit 0730b2090e
124 changed files with 8464 additions and 1121 deletions
@@ -0,0 +1,490 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/mixeme/selfpost/internal/backup"
+	"github.com/mixeme/selfpost/internal/buildinfo"
+	"github.com/mixeme/selfpost/internal/secretfile"
+	"github.com/mixeme/selfpost/internal/store"
+	"golang.org/x/crypto/bcrypt"
+)
+
+// Restoring a SelfPost backup is not a code path in the panel: the operator
+// extracts the archive into a project directory and starts the image, and the
+// panel is expected to come up on it (architecture.md § Persistence). Nothing
+// below stubs that story out — the archive is downloaded from a running panel
+// through /backup, unpacked the way `tar -xzf` unpacks it, and a second panel
+// is started on the result through the same startup sequence run() uses:
+// CheckRestore, store.Open, newPanel, Start.
+
+const (
+	restorePassword = "correct-horse-battery"
+	restoreDomain   = "bs.example.ru"
+	restoreSubject  = "Order confirmation"
+)
+
+// restored is the outcome of a full backup-and-restore round trip.
+type restored struct {
+	panel      http.Handler // panel booted on the restored data directory
+	deployRoot string       // the restored project directory
+	dataDir    string       // the restored /data
+	session    *http.Cookie // a session opened before the backup was taken
+}
+
+// restoreFromOwnBackup runs the operator's path end to end: seed a panel that
+// has been in use, sign in, download a backup from it, extract that archive
+// into an empty directory and boot a second panel there. A non-empty password
+// takes the encrypted download and decrypts it on the way in, which is what an
+// operator does with a .spbk file.
+func restoreFromOwnBackup(t *testing.T, password string) restored {
+	t.Helper()
+
+	live := seedPanelProject(t)
+	panel := bootPanel(t, live)
+	session := signIn(t, panel)
+	archive := downloadBackup(t, panel, session, password)
+
+	target := t.TempDir()
+	extract(t, archive, target)
+
+	return restored{
+		panel:      bootPanel(t, target),
+		deployRoot: target,
+		dataDir:    filepath.Join(target, "data"),
+		session:    session,
+	}
+}
+
+// The panel has to come up on the restored directory and show the state that
+// was in the archive, without the operator touching anything else: the domain
+// and its journal are in the database the archive carried, and the credentials
+// that worked before the restore still work after it.
+func TestPanelBootsOnADataDirectoryRestoredFromItsOwnBackup(t *testing.T) {
+	r := restoreFromOwnBackup(t, "")
+
+	body := getPage(t, r.panel, "/deliveries", signIn(t, r.panel))
+	for _, want := range []string{restoreDomain, restoreSubject} {
+		if !strings.Contains(body, want) {
+			t.Errorf("the restored panel's send log does not show %q:\n%s", want, body)
+		}
+	}
+
+	// The archive carries the daemon files, and the first boot after restore
+	// re-derives the maps from SQLite so they stay aligned with the database.
+	for path, want := range map[string]string{
+		filepath.Join("opendkim", "keys", restoreDomain, "selfpost.private"): "PRIVATE KEY",
+		filepath.Join("sasl", "sasldb2"):                                     "SASLDB",
+		filepath.Join("postfix", "sender_login_maps"):                        "@" + restoreDomain + " shop\n",
+	} {
+		got, err := os.ReadFile(filepath.Join(r.dataDir, path))
+		if err != nil {
+			t.Errorf("the restored data directory has no %s: %v", path, err)
+			continue
+		}
+		if string(got) != want {
+			t.Errorf("%s = %q, want %q", path, got, want)
+		}
+	}
+
+	for _, name := range []string{backup.ComposeFileName, backup.EnvFileName} {
+		if _, err := os.Stat(filepath.Join(r.deployRoot, name)); err != nil {
+			t.Errorf("the restored project directory has no %s: %v", name, err)
+		}
+	}
+}
+
+// The one-time setup link is closed by the presence of a panel user, and the
+// restored database has one. A restore that reopened it would publish a link
+// that creates a second global administrator on a server holding live mail
+// credentials (security.md).
+func TestARestoreDoesNotReopenTheSetupLink(t *testing.T) {
+	r := restoreFromOwnBackup(t, "")
+
+	if _, err := os.Stat(filepath.Join(r.dataDir, "setup-token")); !os.IsNotExist(err) {
+		t.Errorf("the restored panel wrote a setup token (stat err = %v)", err)
+	}
+	body := getPage(t, r.panel, "/login", nil)
+	if strings.Contains(body, "No administrator has been created yet") {
+		t.Errorf("the restored panel offers first-run setup:\n%s", body)
+	}
+}
+
+// Sessions live in the database, so they travel in the archive: a cookie that
+// was valid when the backup was taken is valid again on the restored panel.
+// That is the documented consequence of restoring an older backup (guide §
+// Backup and restore) — stated here so it cannot change by accident.
+func TestARestoredPanelHonoursSessionsFromTheArchive(t *testing.T) {
+	r := restoreFromOwnBackup(t, "")
+
+	rec := request(t, r.panel, http.MethodGet, "/deliveries", nil, r.session)
+	if rec.Code != http.StatusOK {
+		t.Errorf("a session from before the backup = %d on the restored panel, want 200", rec.Code)
+	}
+}
+
+// An encrypted download is the same archive inside an envelope, so it restores
+// the same way once the password is supplied. The archive is never written to
+// disk in the clear by the panel, so this is the only place the two paths can
+// be shown to agree.
+func TestAnEncryptedBackupRestoresTheSameWay(t *testing.T) {
+	r := restoreFromOwnBackup(t, "a-long-enough-password")
+
+	body := getPage(t, r.panel, "/deliveries", signIn(t, r.panel))
+	if !strings.Contains(body, restoreSubject) {
+		t.Errorf("the panel restored from an encrypted backup lost the send log:\n%s", body)
+	}
+}
+
+// A restore boot runs one Resync from SQLite. If the archive's Postfix map
+// drifted from the database, that step puts it back before mail flows.
+func TestResyncAfterRestoreHealsDriftedMaps(t *testing.T) {
+	deployRoot := seedPanelProject(t)
+	dataDir := filepath.Join(deployRoot, "data")
+	cfg := panelConfig(t, deployRoot)
+
+	mapPath := filepath.Join(dataDir, "postfix", "sender_login_maps")
+	if err := os.WriteFile(mapPath, []byte("stale map\n"), 0o640); err != nil {
+		t.Fatalf("write stale map: %v", err)
+	}
+
+	manifest, err := json.Marshal(backup.Manifest{
+		Format:    backup.FormatFull,
+		Version:   buildinfo.Version,
+		CreatedAt: "2026-08-14T00:00:00Z",
+	})
+	if err != nil {
+		t.Fatalf("marshal manifest: %v", err)
+	}
+	if err := os.WriteFile(cfg.manifestPath, manifest, 0o644); err != nil {
+		t.Fatalf("write manifest: %v", err)
+	}
+
+	restored, err := backup.CheckRestore(cfg.manifestPath, buildinfo.Version)
+	if err != nil {
+		t.Fatalf("CheckRestore: %v", err)
+	}
+	if !restored {
+		t.Fatal("CheckRestore did not report a restore")
+	}
+
+	st, err := store.Open(cfg.dbPath)
+	if err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+	defer st.Close()
+
+	if err := resyncAfterRestore(cfg, st, true); err != nil {
+		t.Fatalf("resync after restore: %v", err)
+	}
+
+	got, err := os.ReadFile(mapPath)
+	if err != nil {
+		t.Fatalf("read sender map: %v", err)
+	}
+	want := "@" + restoreDomain + " shop\n"
+	if string(got) != want {
+		t.Errorf("sender map = %q, want %q", got, want)
+	}
+}
+
+// The version guard is what stops a restore from being silently corrupted by
+// schema skew, and it runs before anything opens the database. The manifest
+// stays put on a mismatch: the operator's next move is to start the image the
+// backup names, and it has to be there when they do.
+func TestPanelRefusesADataDirectoryRestoredFromAnotherVersion(t *testing.T) {
+	deployRoot := seedPanelProject(t)
+	dataDir := filepath.Join(deployRoot, "data")
+	var archive bytes.Buffer
+	if err := backup.Create(&archive, backup.Params{
+		DataDir:    dataDir,
+		DBPath:     filepath.Join(dataDir, "selfpost.db"),
+		Version:    "9.9.9",
+		DeployRoot: deployRoot,
+	}); err != nil {
+		t.Fatalf("create backup: %v", err)
+	}
+	target := t.TempDir()
+	extract(t, archive.Bytes(), target)
+
+	cfg := panelConfig(t, target)
+	_, err := backup.CheckRestore(cfg.manifestPath, buildinfo.Version)
+	if err == nil {
+		t.Fatal("the panel booted on a data directory left by another version")
+	}
+	for _, want := range []string{"9.9.9", buildinfo.Version} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the refusal does not name %q, so the operator cannot tell which image to run: %v", want, err)
+		}
+	}
+	if _, statErr := os.Stat(cfg.manifestPath); statErr != nil {
+		t.Errorf("the manifest was consumed by a refused restore: %v", statErr)
+	}
+}
+
+// seedPanelProject builds an operator project tree: data/ with a panel that has
+// been in use, plus docker-compose.yml, .env, and certs/ for full backups.
+func seedPanelProject(t *testing.T) string {
+	t.Helper()
+	deployRoot := t.TempDir()
+	dataDir := filepath.Join(deployRoot, "data")
+	if err := os.MkdirAll(dataDir, 0o750); err != nil {
+		t.Fatalf("mkdir data: %v", err)
+	}
+
+	st, err := store.Open(filepath.Join(dataDir, "selfpost.db"))
+	if err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+	hash, err := bcrypt.GenerateFromPassword([]byte(restorePassword), bcrypt.MinCost)
+	if err != nil {
+		t.Fatalf("hash password: %v", err)
+	}
+	if err := st.CreateGlobalUser("admin", string(hash)); err != nil {
+		t.Fatalf("create user: %v", err)
+	}
+	dom, err := st.AddDomain(restoreDomain, "selfpost")
+	if err != nil {
+		t.Fatalf("add domain: %v", err)
+	}
+	if _, err := st.AddApplication(dom.ID, "shop", store.AddressModeWildcard, nil); err != nil {
+		t.Fatalf("add application: %v", err)
+	}
+	if err := st.InsertQueued(store.SendLogEntry{
+		QueueID: "4A1B2C3D", Domain: restoreDomain, AppLogin: "shop",
+		From: "noreply@" + restoreDomain, To: "customer@example.net", Subject: restoreSubject,
+	}); err != nil {
+		t.Fatalf("insert send-log row: %v", err)
+	}
+	if err := st.Close(); err != nil {
+		t.Fatalf("close store: %v", err)
+	}
+
+	for path, content := range map[string]string{
+		filepath.Join("opendkim", "keys", restoreDomain, "selfpost.private"): "PRIVATE KEY",
+		filepath.Join("sasl", "sasldb2"):                                     "SASLDB",
+		filepath.Join("postfix", "sender_login_maps"):                        "@" + restoreDomain + " shop\n",
+		filepath.Join("log", "mail.log"):                                     "postfix/smtp[1]: 4A1B2C3D: status=sent",
+	} {
+		full := filepath.Join(dataDir, path)
+		if err := os.MkdirAll(filepath.Dir(full), 0o750); err != nil {
+			t.Fatalf("mkdir %s: %v", full, err)
+		}
+		if err := os.WriteFile(full, []byte(content), 0o640); err != nil {
+			t.Fatalf("write %s: %v", full, err)
+		}
+	}
+
+	writeDeployFile(t, filepath.Join(deployRoot, backup.ComposeFileName), "services:\n  selfpost:\n    image: test\n")
+	writeDeployFile(t, filepath.Join(deployRoot, backup.EnvFileName), "SELFPOST_HOSTNAME=mail.example.ru\n")
+	writeDeployFile(t, filepath.Join(deployRoot, backup.CertsDirName, "fullchain.pem"), "CERT")
+	writeDeployFile(t, filepath.Join(deployRoot, backup.CertsDirName, "privkey.pem"), "KEY")
+	return deployRoot
+}
+
+func writeDeployFile(t *testing.T, path, content string) {
+	t.Helper()
+	if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
+		t.Fatalf("mkdir %s: %v", path, err)
+	}
+	if err := os.WriteFile(path, []byte(content), 0o640); err != nil {
+		t.Fatalf("write %s: %v", path, err)
+	}
+}
+
+// panelConfig resolves the panel's own configuration for a project directory, so
+// the test finds the files where the running binary would look for them rather
+// than where it put them. Cookies are marked insecure for the same reason the
+// e2e stand does it: the test client speaks plain HTTP.
+func panelConfig(t *testing.T, deployRoot string) config {
+	t.Helper()
+	dataDir := filepath.Join(deployRoot, "data")
+	t.Setenv("SELFPOST_DATA_DIR", dataDir)
+	t.Setenv("SELFPOST_DEPLOY_ROOT", deployRoot)
+	t.Setenv("PANEL_COOKIE_SECURE", "false")
+	t.Setenv("SELFPOST_HOSTNAME", "mail.example.ru")
+	// MAIL_LOG's default is an absolute path, not one derived from the data
+	// directory; without this the panel would read the host's /data.
+	t.Setenv("MAIL_LOG", filepath.Join(dataDir, "log", "mail.log"))
+	return loadConfig()
+}
+
+// bootPanel performs the startup sequence run() performs, in the same order,
+// and returns the panel's HTTP handler.
+func bootPanel(t *testing.T, deployRoot string) http.Handler {
+	t.Helper()
+	cfg := panelConfig(t, deployRoot)
+
+	restored, err := backup.CheckRestore(cfg.manifestPath, buildinfo.Version)
+	if err != nil {
+		t.Fatalf("the panel refused to start on %s: %v", deployRoot, err)
+	}
+	if restored {
+		if _, err := os.Stat(cfg.manifestPath); err == nil {
+			t.Errorf("the restore manifest was not consumed, so the next start is gated by it too")
+		}
+	}
+
+	st, err := store.Open(cfg.dbPath)
+	if err != nil {
+		t.Fatalf("open the restored database: %v", err)
+	}
+	t.Cleanup(func() { _ = st.Close() })
+
+	if restored {
+		if err := resyncAfterRestore(cfg, st, true); err != nil {
+			t.Fatalf("resync after restore: %v", err)
+		}
+	}
+
+	panel, err := newPanel(cfg, st)
+	if err != nil {
+		t.Fatalf("build the panel: %v", err)
+	}
+	if err := panel.Start(); err != nil {
+		t.Fatalf("start the panel: %v", err)
+	}
+	return panel.Handler()
+}
+
+// signIn signs in as the seeded administrator and returns the session cookie.
+func signIn(t *testing.T, h http.Handler) *http.Cookie {
+	t.Helper()
+	form := url.Values{"username": {"admin"}, "password": {restorePassword}}
+	rec := request(t, h, http.MethodPost, "/login", strings.NewReader(form.Encode()), nil)
+	if rec.Code != http.StatusSeeOther {
+		t.Fatalf("sign in = %d, want 303:\n%s", rec.Code, rec.Body.String())
+	}
+	cookies := rec.Result().Cookies()
+	if len(cookies) == 0 {
+		t.Fatal("sign in issued no session cookie")
+	}
+	return cookies[0]
+}
+
+// downloadBackup takes a backup through the panel's own /backup route, the way
+// the operator does. An empty password downloads the plain archive; otherwise
+// the response is a .spbk envelope, which is decrypted here.
+func downloadBackup(t *testing.T, h http.Handler, session *http.Cookie, password string) []byte {
+	t.Helper()
+	form := url.Values{}
+	if password != "" {
+		form.Set("encrypt", "1")
+		form.Set("password", password)
+		form.Set("password_confirm", password)
+	}
+	rec := request(t, h, http.MethodPost, "/backup", strings.NewReader(form.Encode()), session)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("download a backup = %d, want 200:\n%s", rec.Code, rec.Body.String())
+	}
+	if got := rec.Header().Get("Cache-Control"); got != "no-store" {
+		t.Errorf("Cache-Control = %q; an archive of every secret on the server must not be cached", got)
+	}
+
+	body := rec.Body.Bytes()
+	if password == "" {
+		if secretfile.HasMagic(body) {
+			t.Fatal("an unencrypted download came back as an envelope")
+		}
+		return body
+	}
+
+	if !secretfile.HasMagic(body) {
+		t.Fatal("the download is not an encrypted envelope, so the archive left the panel in the clear")
+	}
+	r, err := secretfile.NewReader(bytes.NewReader(body), password)
+	if err != nil {
+		t.Fatalf("open the encrypted backup: %v", err)
+	}
+	plain, err := io.ReadAll(r)
+	if err != nil {
+		t.Fatalf("decrypt the backup: %v", err)
+	}
+	return plain
+}
+
+// extract unpacks a backup archive into dir, as `tar -xzf` does onto the project
+// directory before the image is started.
+func extract(t *testing.T, archive []byte, dir string) {
+	t.Helper()
+	gz, err := gzip.NewReader(bytes.NewReader(archive))
+	if err != nil {
+		t.Fatalf("the download is not a gzip stream: %v", err)
+	}
+	tr := tar.NewReader(gz)
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatalf("read the archive: %v", err)
+		}
+		name := filepath.Clean(filepath.FromSlash(hdr.Name))
+		if strings.HasPrefix(name, "..") || filepath.IsAbs(name) {
+			t.Fatalf("the archive escapes the directory it is extracted into: %q", hdr.Name)
+		}
+		path := filepath.Join(dir, name)
+
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(path, hdr.FileInfo().Mode().Perm()); err != nil {
+				t.Fatalf("mkdir %s: %v", path, err)
+			}
+		case tar.TypeReg:
+			if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
+				t.Fatalf("mkdir %s: %v", filepath.Dir(path), err)
+			}
+			f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, hdr.FileInfo().Mode().Perm())
+			if err != nil {
+				t.Fatalf("create %s: %v", path, err)
+			}
+			if _, err := io.Copy(f, tr); err != nil {
+				f.Close()
+				t.Fatalf("write %s: %v", path, err)
+			}
+			if err := f.Close(); err != nil {
+				t.Fatalf("close %s: %v", path, err)
+			}
+		}
+	}
+}
+
+// getPage performs a GET and returns the body, failing on any non-200.
+func getPage(t *testing.T, h http.Handler, target string, session *http.Cookie) string {
+	t.Helper()
+	rec := request(t, h, http.MethodGet, target, nil, session)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("GET %s = %d, want 200:\n%s", target, rec.Code, rec.Body.String())
+	}
+	return rec.Body.String()
+}
+
+// request drives the panel's real handler chain, including the origin check,
+// with the headers a browser on the panel's own page would send.
+func request(t *testing.T, h http.Handler, method, target string, body io.Reader, session *http.Cookie) *httptest.ResponseRecorder {
+	t.Helper()
+	req := httptest.NewRequest(method, "http://mail.example.ru"+target, body)
+	req.Host = "mail.example.ru"
+	if method == http.MethodPost {
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+		req.Header.Set("Sec-Fetch-Site", "same-origin")
+	}
+	if session != nil {
+		req.AddCookie(session)
+	}
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+	return rec
+}