Persist Postfix queue and ship self-contained full backups.
Move the mail queue under /data so recreate no longer drops deferred mail, and archive data/, compose, .env, and certs/ together for restore on a fresh host. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -7,6 +7,20 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); version
|
|||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
|
- Postfix queue under `/data/postfix/queue` — deferred and active mail survive
|
||||||
|
container recreate and are included in full backups.
|
||||||
|
- Full backup archives the whole operator project: `data/`, `docker-compose.yml`,
|
||||||
|
`.env`, and `certs/` (requires `.:/selfpost-deploy:ro` in compose). Restore
|
||||||
|
by unpacking into an empty project directory.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- **Breaking:** full-backup archive layout — paths are prefixed with `data/`;
|
||||||
|
deploy files sit at the archive root. Old flat archives restore with
|
||||||
|
`tar xzf backup.tar.gz -C ./data` as before.
|
||||||
|
- docs: operator guide, architecture, security, and backup UI updated for
|
||||||
|
self-contained backups and persistent queue.
|
||||||
|
|
||||||
- docs: HTML mockups for a full panel UI refresh
|
- docs: HTML mockups for a full panel UI refresh
|
||||||
([docs/assets/panel-ui/](docs/assets/panel-ui/index.html)) — current screens,
|
([docs/assets/panel-ui/](docs/assets/panel-ui/index.html)) — current screens,
|
||||||
agreed and candidate roadmap surfaces (queue-retries, inbound-relay,
|
agreed and candidate roadmap surfaces (queue-retries, inbound-relay,
|
||||||
|
|||||||
+13
-6
@@ -69,7 +69,7 @@ chmod 755 /data
|
|||||||
# later phase deliberately hands to another service. /data/log is exempt: it is
|
# later phase deliberately hands to another service. /data/log is exempt: it is
|
||||||
# deliberately owned by postfix (postlogd writes the delivery log there) and is
|
# deliberately owned by postfix (postlogd writes the delivery log there) and is
|
||||||
# normalised on its own below.
|
# normalised on its own below.
|
||||||
find /data -mindepth 1 -maxdepth 1 ! -user panel ! -name log -exec chown -R panel:panel {} +
|
find /data -mindepth 1 -maxdepth 1 ! -user panel ! -name log ! -name postfix -exec chown -R panel:panel {} +
|
||||||
|
|
||||||
# DKIM key tree (spec 6, 9). The panel (user `panel`) generates keys and writes
|
# DKIM key tree (spec 6, 9). The panel (user `panel`) generates keys and writes
|
||||||
# the OpenDKIM tables; OpenDKIM (user `opendkim`) must read them. Normalise the
|
# the OpenDKIM tables; OpenDKIM (user `opendkim`) must read them. Normalise the
|
||||||
@@ -99,13 +99,13 @@ chown -R panel:selfpost /data/sasl
|
|||||||
chmod 2750 /data/sasl
|
chmod 2750 /data/sasl
|
||||||
[ -e /data/sasl/sasldb2 ] && chmod 0640 /data/sasl/sasldb2
|
[ -e /data/sasl/sasldb2 ] && chmod 0640 /data/sasl/sasldb2
|
||||||
|
|
||||||
# Postfix sender_login_maps (spec 5.1). The panel writes it; Postfix reads it.
|
# Postfix state under /data (spec 5.1, architecture.md § Persistence). The panel
|
||||||
# Ensure the file exists (empty is fine) before Postfix starts so a reload that
|
# writes sender_login_maps; Postfix owns the on-disk queue tree under queue/.
|
||||||
# references it never fails on a missing file, and keep it group-readable.
|
mkdir -p /data/postfix/queue
|
||||||
mkdir -p /data/postfix
|
|
||||||
[ -e /data/postfix/sender_login_maps ] || : > /data/postfix/sender_login_maps
|
[ -e /data/postfix/sender_login_maps ] || : > /data/postfix/sender_login_maps
|
||||||
chown -R panel:selfpost /data/postfix
|
chown panel:selfpost /data/postfix
|
||||||
chmod 2750 /data/postfix
|
chmod 2750 /data/postfix
|
||||||
|
chown panel:selfpost /data/postfix/sender_login_maps
|
||||||
chmod 0640 /data/postfix/sender_login_maps
|
chmod 0640 /data/postfix/sender_login_maps
|
||||||
|
|
||||||
# Delivery log (architecture.md § Log tailer). postlogd writes it as user
|
# Delivery log (architecture.md § Log tailer). postlogd writes it as user
|
||||||
@@ -144,4 +144,11 @@ chmod 2750 /run/opendkim /run/selfpost
|
|||||||
# start the same way the /data normalisation above is.
|
# start the same way the /data normalisation above is.
|
||||||
/usr/local/bin/postfix-config.sh
|
/usr/local/bin/postfix-config.sh
|
||||||
|
|
||||||
|
# Initialise the persistent queue tree on first start or after restore. postfix
|
||||||
|
# set-permissions reads queue_directory from main.cf (set by postfix-config.sh).
|
||||||
|
if [ ! -d /data/postfix/queue/active ]; then
|
||||||
|
postfix set-permissions
|
||||||
|
fi
|
||||||
|
chown -R postfix:postfix /data/postfix/queue
|
||||||
|
|
||||||
exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
|
exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
|
||||||
|
|||||||
@@ -57,6 +57,9 @@ JOURNAL_SOCK="${JOURNAL_MILTER_SOCKET:-/run/selfpost/journal.sock}"
|
|||||||
# Persistent panel-managed sender map (spec 5.1); texthash needs no postmap, so
|
# Persistent panel-managed sender map (spec 5.1); texthash needs no postmap, so
|
||||||
# the unprivileged panel can rewrite it and just ask for a reload.
|
# the unprivileged panel can rewrite it and just ask for a reload.
|
||||||
SENDER_LOGIN_MAPS="${POSTFIX_SENDER_LOGIN_MAPS:-/data/postfix/sender_login_maps}"
|
SENDER_LOGIN_MAPS="${POSTFIX_SENDER_LOGIN_MAPS:-/data/postfix/sender_login_maps}"
|
||||||
|
# Transit mail queue under /data so deferred/active messages survive container
|
||||||
|
# recreate (architecture.md § Persistence). Distinct from sender_login_maps.
|
||||||
|
QUEUE_DIR="${POSTFIX_QUEUE_DIR:-/data/postfix/queue}"
|
||||||
SASLDB_PATH="${SASL_DB_PATH:-/data/sasl/sasldb2}"
|
SASLDB_PATH="${SASL_DB_PATH:-/data/sasl/sasldb2}"
|
||||||
|
|
||||||
# Optional submission service on 587 (spec 5 p.1: off by default, enabled only
|
# Optional submission service on 587 (spec 5 p.1: off by default, enabled only
|
||||||
@@ -128,7 +131,8 @@ postconf -e \
|
|||||||
# resolves the full address first, then the "@domain" wildcard, so both address
|
# resolves the full address first, then the "@domain" wildcard, so both address
|
||||||
# modes work from the same map.
|
# modes work from the same map.
|
||||||
postconf -e \
|
postconf -e \
|
||||||
"smtpd_sender_login_maps=texthash:${SENDER_LOGIN_MAPS}"
|
"smtpd_sender_login_maps=texthash:${SENDER_LOGIN_MAPS}" \
|
||||||
|
"queue_directory=${QUEUE_DIR}"
|
||||||
|
|
||||||
# Restrictions: authenticated clients only, no relay to foreign destinations,
|
# Restrictions: authenticated clients only, no relay to foreign destinations,
|
||||||
# and every authenticated sender address must be owned by its login. NO
|
# and every authenticated sender address must be owned by its login. NO
|
||||||
|
|||||||
@@ -33,6 +33,8 @@ var documentedInternal = []string{
|
|||||||
"SASL_REALM",
|
"SASL_REALM",
|
||||||
"POSTFIX_DIR",
|
"POSTFIX_DIR",
|
||||||
"POSTFIX_SENDER_LOGIN_MAPS",
|
"POSTFIX_SENDER_LOGIN_MAPS",
|
||||||
|
"POSTFIX_QUEUE_DIR",
|
||||||
|
"SELFPOST_DEPLOY_ROOT",
|
||||||
"MILTER_CONNECT_TIMEOUT",
|
"MILTER_CONNECT_TIMEOUT",
|
||||||
"MILTER_COMMAND_TIMEOUT",
|
"MILTER_COMMAND_TIMEOUT",
|
||||||
"MILTER_CONTENT_TIMEOUT",
|
"MILTER_CONTENT_TIMEOUT",
|
||||||
@@ -70,6 +72,7 @@ var loadConfigKeys = []string{
|
|||||||
"SASL_DB_PATH",
|
"SASL_DB_PATH",
|
||||||
"SASL_REALM",
|
"SASL_REALM",
|
||||||
"POSTFIX_DIR",
|
"POSTFIX_DIR",
|
||||||
|
"SELFPOST_DEPLOY_ROOT",
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildScriptKeys is every ${VAR:-…} / os.Getenv used in build/*.sh and entrypoint.sh
|
// buildScriptKeys is every ${VAR:-…} / os.Getenv used in build/*.sh and entrypoint.sh
|
||||||
@@ -84,6 +87,7 @@ var buildScriptKeys = []string{
|
|||||||
"JOURNAL_MILTER_SOCKET",
|
"JOURNAL_MILTER_SOCKET",
|
||||||
"MAIL_LOG",
|
"MAIL_LOG",
|
||||||
"POSTFIX_SENDER_LOGIN_MAPS",
|
"POSTFIX_SENDER_LOGIN_MAPS",
|
||||||
|
"POSTFIX_QUEUE_DIR",
|
||||||
"SASL_DB_PATH",
|
"SASL_DB_PATH",
|
||||||
"SUBMISSION_ENABLE",
|
"SUBMISSION_ENABLE",
|
||||||
"MILTER_CONNECT_TIMEOUT",
|
"MILTER_CONNECT_TIMEOUT",
|
||||||
|
|||||||
@@ -73,6 +73,7 @@ func newPanel(cfg config, st *store.Store) (*web.Server, error) {
|
|||||||
MailLogPath: cfg.mailLog,
|
MailLogPath: cfg.mailLog,
|
||||||
DataDir: cfg.dataDir,
|
DataDir: cfg.dataDir,
|
||||||
DBPath: cfg.dbPath,
|
DBPath: cfg.dbPath,
|
||||||
|
DeployRoot: cfg.deployRoot,
|
||||||
Version: buildinfo.Version,
|
Version: buildinfo.Version,
|
||||||
TrustedProxyCIDRs: cfg.trustedProxies,
|
TrustedProxyCIDRs: cfg.trustedProxies,
|
||||||
TLSCertFile: cfg.tlsCertFile,
|
TLSCertFile: cfg.tlsCertFile,
|
||||||
|
|||||||
@@ -78,6 +78,7 @@ type config struct {
|
|||||||
saslDBPath string
|
saslDBPath string
|
||||||
saslRealm string
|
saslRealm string
|
||||||
postfixDir string
|
postfixDir string
|
||||||
|
deployRoot string
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadConfig() config {
|
func loadConfig() config {
|
||||||
@@ -138,6 +139,7 @@ func loadConfig() config {
|
|||||||
saslDBPath: envDefault("SASL_DB_PATH", filepath.Join(dataDir, "sasl", "sasldb2")),
|
saslDBPath: envDefault("SASL_DB_PATH", filepath.Join(dataDir, "sasl", "sasldb2")),
|
||||||
saslRealm: saslRealm(),
|
saslRealm: saslRealm(),
|
||||||
postfixDir: envDefault("POSTFIX_DIR", filepath.Join(dataDir, "postfix")),
|
postfixDir: envDefault("POSTFIX_DIR", filepath.Join(dataDir, "postfix")),
|
||||||
|
deployRoot: envDefault("SELFPOST_DEPLOY_ROOT", "/selfpost-deploy"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+56
-22
@@ -22,7 +22,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Restoring a SelfPost backup is not a code path in the panel: the operator
|
// Restoring a SelfPost backup is not a code path in the panel: the operator
|
||||||
// extracts the archive into the /data bind mount and starts the image, and the
|
// extracts the archive into a project directory and starts the image, and the
|
||||||
// panel is expected to come up on it (architecture.md § Persistence). Nothing
|
// panel is expected to come up on it (architecture.md § Persistence). Nothing
|
||||||
// below stubs that story out — the archive is downloaded from a running panel
|
// below stubs that story out — the archive is downloaded from a running panel
|
||||||
// through /backup, unpacked the way `tar -xzf` unpacks it, and a second panel
|
// through /backup, unpacked the way `tar -xzf` unpacks it, and a second panel
|
||||||
@@ -38,6 +38,7 @@ const (
|
|||||||
// restored is the outcome of a full backup-and-restore round trip.
|
// restored is the outcome of a full backup-and-restore round trip.
|
||||||
type restored struct {
|
type restored struct {
|
||||||
panel http.Handler // panel booted on the restored data directory
|
panel http.Handler // panel booted on the restored data directory
|
||||||
|
deployRoot string // the restored project directory
|
||||||
dataDir string // the restored /data
|
dataDir string // the restored /data
|
||||||
session *http.Cookie // a session opened before the backup was taken
|
session *http.Cookie // a session opened before the backup was taken
|
||||||
}
|
}
|
||||||
@@ -50,7 +51,7 @@ type restored struct {
|
|||||||
func restoreFromOwnBackup(t *testing.T, password string) restored {
|
func restoreFromOwnBackup(t *testing.T, password string) restored {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
live := seedPanelData(t)
|
live := seedPanelProject(t)
|
||||||
panel := bootPanel(t, live)
|
panel := bootPanel(t, live)
|
||||||
session := signIn(t, panel)
|
session := signIn(t, panel)
|
||||||
archive := downloadBackup(t, panel, session, password)
|
archive := downloadBackup(t, panel, session, password)
|
||||||
@@ -58,7 +59,12 @@ func restoreFromOwnBackup(t *testing.T, password string) restored {
|
|||||||
target := t.TempDir()
|
target := t.TempDir()
|
||||||
extract(t, archive, target)
|
extract(t, archive, target)
|
||||||
|
|
||||||
return restored{panel: bootPanel(t, target), dataDir: target, session: session}
|
return restored{
|
||||||
|
panel: bootPanel(t, target),
|
||||||
|
deployRoot: target,
|
||||||
|
dataDir: filepath.Join(target, "data"),
|
||||||
|
session: session,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The panel has to come up on the restored directory and show the state that
|
// The panel has to come up on the restored directory and show the state that
|
||||||
@@ -91,6 +97,12 @@ func TestPanelBootsOnADataDirectoryRestoredFromItsOwnBackup(t *testing.T) {
|
|||||||
t.Errorf("%s = %q, want %q", path, got, want)
|
t.Errorf("%s = %q, want %q", path, got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, name := range []string{backup.ComposeFileName, backup.EnvFileName} {
|
||||||
|
if _, err := os.Stat(filepath.Join(r.deployRoot, name)); err != nil {
|
||||||
|
t.Errorf("the restored project directory has no %s: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The one-time setup link is closed by the presence of a panel user, and the
|
// The one-time setup link is closed by the presence of a panel user, and the
|
||||||
@@ -138,8 +150,9 @@ func TestAnEncryptedBackupRestoresTheSameWay(t *testing.T) {
|
|||||||
// A restore boot runs one Resync from SQLite. If the archive's Postfix map
|
// A restore boot runs one Resync from SQLite. If the archive's Postfix map
|
||||||
// drifted from the database, that step puts it back before mail flows.
|
// drifted from the database, that step puts it back before mail flows.
|
||||||
func TestResyncAfterRestoreHealsDriftedMaps(t *testing.T) {
|
func TestResyncAfterRestoreHealsDriftedMaps(t *testing.T) {
|
||||||
dataDir := seedPanelData(t)
|
deployRoot := seedPanelProject(t)
|
||||||
cfg := panelConfig(t, dataDir)
|
dataDir := filepath.Join(deployRoot, "data")
|
||||||
|
cfg := panelConfig(t, deployRoot)
|
||||||
|
|
||||||
mapPath := filepath.Join(dataDir, "postfix", "sender_login_maps")
|
mapPath := filepath.Join(dataDir, "postfix", "sender_login_maps")
|
||||||
if err := os.WriteFile(mapPath, []byte("stale map\n"), 0o640); err != nil {
|
if err := os.WriteFile(mapPath, []byte("stale map\n"), 0o640); err != nil {
|
||||||
@@ -191,12 +204,14 @@ func TestResyncAfterRestoreHealsDriftedMaps(t *testing.T) {
|
|||||||
// stays put on a mismatch: the operator's next move is to start the image the
|
// stays put on a mismatch: the operator's next move is to start the image the
|
||||||
// backup names, and it has to be there when they do.
|
// backup names, and it has to be there when they do.
|
||||||
func TestPanelRefusesADataDirectoryRestoredFromAnotherVersion(t *testing.T) {
|
func TestPanelRefusesADataDirectoryRestoredFromAnotherVersion(t *testing.T) {
|
||||||
live := seedPanelData(t)
|
deployRoot := seedPanelProject(t)
|
||||||
|
dataDir := filepath.Join(deployRoot, "data")
|
||||||
var archive bytes.Buffer
|
var archive bytes.Buffer
|
||||||
if err := backup.Create(&archive, backup.Params{
|
if err := backup.Create(&archive, backup.Params{
|
||||||
DataDir: live,
|
DataDir: dataDir,
|
||||||
DBPath: filepath.Join(live, "selfpost.db"),
|
DBPath: filepath.Join(dataDir, "selfpost.db"),
|
||||||
Version: "9.9.9",
|
Version: "9.9.9",
|
||||||
|
DeployRoot: deployRoot,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatalf("create backup: %v", err)
|
t.Fatalf("create backup: %v", err)
|
||||||
}
|
}
|
||||||
@@ -218,13 +233,15 @@ func TestPanelRefusesADataDirectoryRestoredFromAnotherVersion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedPanelData builds the /data tree of a panel that has been in use: an
|
// seedPanelProject builds an operator project tree: data/ with a panel that has
|
||||||
// administrator, a sending domain with an application and one logged message,
|
// been in use, plus docker-compose.yml, .env, and certs/ for full backups.
|
||||||
// and the daemon state the mail path needs (a DKIM key, the SASL database and
|
func seedPanelProject(t *testing.T) string {
|
||||||
// Postfix's sender map).
|
|
||||||
func seedPanelData(t *testing.T) string {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dataDir := t.TempDir()
|
deployRoot := t.TempDir()
|
||||||
|
dataDir := filepath.Join(deployRoot, "data")
|
||||||
|
if err := os.MkdirAll(dataDir, 0o750); err != nil {
|
||||||
|
t.Fatalf("mkdir data: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
st, err := store.Open(filepath.Join(dataDir, "selfpost.db"))
|
st, err := store.Open(filepath.Join(dataDir, "selfpost.db"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -268,16 +285,33 @@ func seedPanelData(t *testing.T) string {
|
|||||||
t.Fatalf("write %s: %v", full, err)
|
t.Fatalf("write %s: %v", full, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return dataDir
|
|
||||||
|
writeDeployFile(t, filepath.Join(deployRoot, backup.ComposeFileName), "services:\n selfpost:\n image: test\n")
|
||||||
|
writeDeployFile(t, filepath.Join(deployRoot, backup.EnvFileName), "SELFPOST_HOSTNAME=mail.example.ru\n")
|
||||||
|
writeDeployFile(t, filepath.Join(deployRoot, backup.CertsDirName, "fullchain.pem"), "CERT")
|
||||||
|
writeDeployFile(t, filepath.Join(deployRoot, backup.CertsDirName, "privkey.pem"), "KEY")
|
||||||
|
return deployRoot
|
||||||
}
|
}
|
||||||
|
|
||||||
// panelConfig resolves the panel's own configuration for a data directory, so
|
func writeDeployFile(t *testing.T, path, content string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
||||||
|
t.Fatalf("mkdir %s: %v", path, err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o640); err != nil {
|
||||||
|
t.Fatalf("write %s: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// panelConfig resolves the panel's own configuration for a project directory, so
|
||||||
// the test finds the files where the running binary would look for them rather
|
// the test finds the files where the running binary would look for them rather
|
||||||
// than where it put them. Cookies are marked insecure for the same reason the
|
// than where it put them. Cookies are marked insecure for the same reason the
|
||||||
// e2e stand does it: the test client speaks plain HTTP.
|
// e2e stand does it: the test client speaks plain HTTP.
|
||||||
func panelConfig(t *testing.T, dataDir string) config {
|
func panelConfig(t *testing.T, deployRoot string) config {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
dataDir := filepath.Join(deployRoot, "data")
|
||||||
t.Setenv("SELFPOST_DATA_DIR", dataDir)
|
t.Setenv("SELFPOST_DATA_DIR", dataDir)
|
||||||
|
t.Setenv("SELFPOST_DEPLOY_ROOT", deployRoot)
|
||||||
t.Setenv("PANEL_COOKIE_SECURE", "false")
|
t.Setenv("PANEL_COOKIE_SECURE", "false")
|
||||||
t.Setenv("SELFPOST_HOSTNAME", "mail.example.ru")
|
t.Setenv("SELFPOST_HOSTNAME", "mail.example.ru")
|
||||||
// MAIL_LOG's default is an absolute path, not one derived from the data
|
// MAIL_LOG's default is an absolute path, not one derived from the data
|
||||||
@@ -288,13 +322,13 @@ func panelConfig(t *testing.T, dataDir string) config {
|
|||||||
|
|
||||||
// bootPanel performs the startup sequence run() performs, in the same order,
|
// bootPanel performs the startup sequence run() performs, in the same order,
|
||||||
// and returns the panel's HTTP handler.
|
// and returns the panel's HTTP handler.
|
||||||
func bootPanel(t *testing.T, dataDir string) http.Handler {
|
func bootPanel(t *testing.T, deployRoot string) http.Handler {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
cfg := panelConfig(t, dataDir)
|
cfg := panelConfig(t, deployRoot)
|
||||||
|
|
||||||
restored, err := backup.CheckRestore(cfg.manifestPath, buildinfo.Version)
|
restored, err := backup.CheckRestore(cfg.manifestPath, buildinfo.Version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("the panel refused to start on %s: %v", dataDir, err)
|
t.Fatalf("the panel refused to start on %s: %v", deployRoot, err)
|
||||||
}
|
}
|
||||||
if restored {
|
if restored {
|
||||||
if _, err := os.Stat(cfg.manifestPath); err == nil {
|
if _, err := os.Stat(cfg.manifestPath); err == nil {
|
||||||
@@ -380,8 +414,8 @@ func downloadBackup(t *testing.T, h http.Handler, session *http.Cookie, password
|
|||||||
return plain
|
return plain
|
||||||
}
|
}
|
||||||
|
|
||||||
// extract unpacks a backup archive into dir, as `tar -xzf` does onto the /data
|
// extract unpacks a backup archive into dir, as `tar -xzf` does onto the project
|
||||||
// bind mount before the image is started.
|
// directory before the image is started.
|
||||||
func extract(t *testing.T, archive []byte, dir string) {
|
func extract(t *testing.T, archive []byte, dir string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
gz, err := gzip.NewReader(bytes.NewReader(archive))
|
gz, err := gzip.NewReader(bytes.NewReader(archive))
|
||||||
|
|||||||
@@ -6,9 +6,9 @@
|
|||||||
//
|
//
|
||||||
// docker exec <container> selfpost-backup > selfpost-backup.tar.gz
|
// docker exec <container> selfpost-backup > selfpost-backup.tar.gz
|
||||||
//
|
//
|
||||||
// Use -o to write to a file instead. The resulting archive contains DKIM private
|
// Use -o to write to a file instead. The resulting archive is self-contained:
|
||||||
// keys, the admin password hash and SASL credentials — treat it as a secret
|
// data/, docker-compose.yml, .env, and certs/ when present — treat it as a
|
||||||
// (architecture.md § Persistence).
|
// secret (architecture.md § Persistence).
|
||||||
//
|
//
|
||||||
// Given a password (SELFPOST_BACKUP_PASSWORD or -password-file, never an
|
// Given a password (SELFPOST_BACKUP_PASSWORD or -password-file, never an
|
||||||
// argument, which would show up in the process list) the archive is written as
|
// argument, which would show up in the process list) the archive is written as
|
||||||
@@ -90,6 +90,10 @@ func run(outPath, password string) error {
|
|||||||
DataDir: dataDir,
|
DataDir: dataDir,
|
||||||
DBPath: dbPath,
|
DBPath: dbPath,
|
||||||
Version: buildinfo.Version,
|
Version: buildinfo.Version,
|
||||||
|
DeployRoot: envDefault("SELFPOST_DEPLOY_ROOT", "/selfpost-deploy"),
|
||||||
|
OnWarn: func(msg string) {
|
||||||
|
fmt.Fprintf(os.Stderr, "selfpost-backup: %s\n", msg)
|
||||||
|
},
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,13 +9,18 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/mixeme/selfpost/internal/backup"
|
||||||
"github.com/mixeme/selfpost/internal/store"
|
"github.com/mixeme/selfpost/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
// seedDataDir builds the minimum /data tree a backup can be taken from.
|
// seedProject builds the minimum operator project tree a backup can be taken from.
|
||||||
func seedDataDir(t *testing.T) string {
|
func seedProject(t *testing.T) (deployRoot, dataDir string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dataDir := t.TempDir()
|
deployRoot = t.TempDir()
|
||||||
|
dataDir = filepath.Join(deployRoot, "data")
|
||||||
|
if err := os.MkdirAll(dataDir, 0o750); err != nil {
|
||||||
|
t.Fatalf("mkdir data: %v", err)
|
||||||
|
}
|
||||||
st, err := store.Open(filepath.Join(dataDir, "selfpost.db"))
|
st, err := store.Open(filepath.Join(dataDir, "selfpost.db"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("open store: %v", err)
|
t.Fatalf("open store: %v", err)
|
||||||
@@ -26,16 +31,31 @@ func seedDataDir(t *testing.T) string {
|
|||||||
if err := st.Close(); err != nil {
|
if err := st.Close(); err != nil {
|
||||||
t.Fatalf("close store: %v", err)
|
t.Fatalf("close store: %v", err)
|
||||||
}
|
}
|
||||||
|
writeFile(t, filepath.Join(deployRoot, backup.ComposeFileName), "services:\n selfpost:\n image: test\n")
|
||||||
|
writeFile(t, filepath.Join(deployRoot, backup.EnvFileName), "SELFPOST_HOSTNAME=mail.example.com\n")
|
||||||
|
writeFile(t, filepath.Join(deployRoot, backup.CertsDirName, "fullchain.pem"), "CERT")
|
||||||
|
writeFile(t, filepath.Join(deployRoot, backup.CertsDirName, "privkey.pem"), "KEY")
|
||||||
t.Setenv("SELFPOST_DATA_DIR", dataDir)
|
t.Setenv("SELFPOST_DATA_DIR", dataDir)
|
||||||
t.Setenv("SELFPOST_DB_PATH", filepath.Join(dataDir, "selfpost.db"))
|
t.Setenv("SELFPOST_DB_PATH", filepath.Join(dataDir, "selfpost.db"))
|
||||||
return dataDir
|
t.Setenv("SELFPOST_DEPLOY_ROOT", deployRoot)
|
||||||
|
return deployRoot, dataDir
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeFile(t *testing.T, path, content string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
||||||
|
t.Fatalf("mkdir %s: %v", path, err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o640); err != nil {
|
||||||
|
t.Fatalf("write %s: %v", path, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// An encrypted backup is only worth having if the container it came from can
|
// An encrypted backup is only worth having if the container it came from can
|
||||||
// hand it back as an ordinary archive during a restore, so the two halves of
|
// hand it back as an ordinary archive during a restore, so the two halves of
|
||||||
// the CLI are tested as the one round trip an operator actually performs.
|
// the CLI are tested as the one round trip an operator actually performs.
|
||||||
func TestEncryptedBackupRoundTrip(t *testing.T) {
|
func TestEncryptedBackupRoundTrip(t *testing.T) {
|
||||||
seedDataDir(t)
|
seedProject(t)
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
encrypted := filepath.Join(dir, "backup.spbk")
|
encrypted := filepath.Join(dir, "backup.spbk")
|
||||||
plain := filepath.Join(dir, "backup.tar.gz")
|
plain := filepath.Join(dir, "backup.tar.gz")
|
||||||
@@ -81,7 +101,12 @@ func TestEncryptedBackupRoundTrip(t *testing.T) {
|
|||||||
}
|
}
|
||||||
names[hdr.Name] = true
|
names[hdr.Name] = true
|
||||||
}
|
}
|
||||||
for _, want := range []string{"manifest.json", "selfpost.db"} {
|
for _, want := range []string{
|
||||||
|
backup.DataArchivePrefix + backup.ManifestName,
|
||||||
|
backup.DataArchivePrefix + "selfpost.db",
|
||||||
|
backup.ComposeFileName,
|
||||||
|
backup.EnvFileName,
|
||||||
|
} {
|
||||||
if !names[want] {
|
if !names[want] {
|
||||||
t.Errorf("decrypted archive has no %s (entries: %v)", want, names)
|
t.Errorf("decrypted archive has no %s (entries: %v)", want, names)
|
||||||
}
|
}
|
||||||
@@ -91,7 +116,7 @@ func TestEncryptedBackupRoundTrip(t *testing.T) {
|
|||||||
// Without a password the CLI keeps producing the plain archive that existing
|
// Without a password the CLI keeps producing the plain archive that existing
|
||||||
// backup scripts consume.
|
// backup scripts consume.
|
||||||
func TestUnencryptedBackupStaysPlain(t *testing.T) {
|
func TestUnencryptedBackupStaysPlain(t *testing.T) {
|
||||||
seedDataDir(t)
|
seedProject(t)
|
||||||
out := filepath.Join(t.TempDir(), "backup.tar.gz")
|
out := filepath.Join(t.TempDir(), "backup.tar.gz")
|
||||||
if err := run(out, ""); err != nil {
|
if err := run(out, ""); err != nil {
|
||||||
t.Fatalf("create backup: %v", err)
|
t.Fatalf("create backup: %v", err)
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ services:
|
|||||||
TLS_KEY_FILE: /etc/postfix/tls/<hostname>.key
|
TLS_KEY_FILE: /etc/postfix/tls/<hostname>.key
|
||||||
volumes: !override
|
volumes: !override
|
||||||
- ./data:/data
|
- ./data:/data
|
||||||
|
- .:/selfpost-deploy:ro
|
||||||
# Replace <hostname> with your actual mail/panel hostname, matching
|
# Replace <hostname> with your actual mail/panel hostname, matching
|
||||||
# the Caddyfile below and SELFPOST_HOSTNAME.
|
# the Caddyfile below and SELFPOST_HOSTNAME.
|
||||||
- ./caddy/caddy-data/caddy/certificates/acme-v02.api.letsencrypt.org-directory/<hostname>:/etc/postfix/tls:ro
|
- ./caddy/caddy-data/caddy/certificates/acme-v02.api.letsencrypt.org-directory/<hostname>:/etc/postfix/tls:ro
|
||||||
|
|||||||
@@ -58,6 +58,9 @@ services:
|
|||||||
# kept) so the delivery lines that resolve a queued send-log row survive
|
# kept) so the delivery lines that resolve a queued send-log row survive
|
||||||
# a container recreate; it is diagnostics and is left out of backups.
|
# a container recreate; it is diagnostics and is left out of backups.
|
||||||
- ./data:/data
|
- ./data:/data
|
||||||
|
# Read-only project root for full backups: docker-compose.yml, .env, and
|
||||||
|
# certs/ are archived alongside /data (architecture.md § Persistence).
|
||||||
|
- .:/selfpost-deploy:ro
|
||||||
# Read-only: SelfPost only ever reads certificates, never manages them
|
# Read-only: SelfPost only ever reads certificates, never manages them
|
||||||
# (spec 10 p.2). Point this at wherever your reverse-proxy/certbot
|
# (spec 10 p.2). Point this at wherever your reverse-proxy/certbot
|
||||||
# writes PEM files, e.g. /etc/letsencrypt on the host.
|
# writes PEM files, e.g. /etc/letsencrypt on the host.
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ services:
|
|||||||
# network at selfpost:8080 instead (see nginx.conf.example).
|
# network at selfpost:8080 instead (see nginx.conf.example).
|
||||||
volumes: !override
|
volumes: !override
|
||||||
- ./data:/data
|
- ./data:/data
|
||||||
|
- .:/selfpost-deploy:ro
|
||||||
# Same host directory certbot below writes into — plain bind mount,
|
# Same host directory certbot below writes into — plain bind mount,
|
||||||
# no named volume, so the PEM files are as directly inspectable as in
|
# no named volume, so the PEM files are as directly inspectable as in
|
||||||
# the Apache scenario (spec 10.3).
|
# the Apache scenario (spec 10.3).
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ services:
|
|||||||
- "587:587"
|
- "587:587"
|
||||||
volumes: !override
|
volumes: !override
|
||||||
- ./data:/data
|
- ./data:/data
|
||||||
|
- .:/selfpost-deploy:ro
|
||||||
# Populated by extract-cert.sh from traefik's acme.json — see below.
|
# Populated by extract-cert.sh from traefik's acme.json — see below.
|
||||||
- ./traefik/extracted-certs:/etc/postfix/tls:ro
|
- ./traefik/extracted-certs:/etc/postfix/tls:ro
|
||||||
labels:
|
labels:
|
||||||
|
|||||||
+17
-10
@@ -291,23 +291,27 @@ single-connection trade-off that follows from it.
|
|||||||
| `opendkim/` | DKIM keys + tables |
|
| `opendkim/` | DKIM keys + tables |
|
||||||
| `sasl/sasldb2` | Application SASL credentials |
|
| `sasl/sasldb2` | Application SASL credentials |
|
||||||
| `postfix/sender_login_maps` | Login → From binding |
|
| `postfix/sender_login_maps` | Login → From binding |
|
||||||
|
| `postfix/queue/` | Postfix transit mail (deferred/active); survives container recreate |
|
||||||
| `log/mail.log` | Postfix delivery log + rotated copies (excluded from backups) |
|
| `log/mail.log` | Postfix delivery log + rotated copies (excluded from backups) |
|
||||||
| `manifest.json` | Backup version stamp (consumed on restore) |
|
| `manifest.json` | Backup version stamp (consumed on restore) |
|
||||||
|
|
||||||
Not in `/data`: TLS certificates (reverse-proxy mount), Postfix queue
|
Not in `/data`: TLS certificates for the panel (reverse-proxy mount) — though
|
||||||
(transit mail not migrated by design).
|
full backups also archive the operator's `./certs` PEM files when present.
|
||||||
|
|
||||||
**Rotation:** send-log retention `SEND_LOG_RETENTION_DAYS` (default 90);
|
**Rotation:** send-log retention `SEND_LOG_RETENTION_DAYS` (default 90);
|
||||||
`mail.log` via logrotate (14 rotated files, check every 6h, rename +
|
`mail.log` via logrotate (14 rotated files, check every 6h, rename +
|
||||||
`postfix reload` in `postrotate` — see § Log tailer above).
|
`postfix reload` in `postrotate` — see § Log tailer above).
|
||||||
|
|
||||||
**Restore:** panel button or `selfpost-backup` CLI — SQLite snapshot + tar of
|
**Restore:** panel button or `selfpost-backup` CLI — self-contained archive:
|
||||||
`/data` tree, minus `log/`, the setup token and any `tls/`; version check on
|
`data/` (SQLite snapshot + tree minus `log/`, the setup token and any `tls/`
|
||||||
restore. On the first successful boot after restore, the panel runs one
|
under `/data`), `docker-compose.yml`, `.env`, and `certs/` when present;
|
||||||
**Resync** — OpenDKIM's tables and Postfix's sender map are re-derived from
|
version check on restore. Requires the project directory mounted read-only at
|
||||||
SQLite and both daemons are reloaded, so drift between the extracted archive
|
`SELFPOST_DEPLOY_ROOT` (`/selfpost-deploy` in the default compose file). On the
|
||||||
and the database is healed before mail flows (same step as `POST /reload` on
|
first successful boot after restore, the panel runs one **Resync** — OpenDKIM's
|
||||||
demand). Stopped-container `tar` of `./data` is safe (see guide).
|
tables and Postfix's sender map are re-derived from SQLite and both daemons are
|
||||||
|
reloaded, so drift between the extracted archive and the database is healed
|
||||||
|
before mail flows (same step as `POST /reload` on demand). Stopped-container
|
||||||
|
`tar` of `./data` alone remains possible for state-only copies (see guide).
|
||||||
|
|
||||||
**Optional encryption** of the two secret-bearing downloads
|
**Optional encryption** of the two secret-bearing downloads
|
||||||
([internal/secretfile](../internal/secretfile/secretfile.go)): password →
|
([internal/secretfile](../internal/secretfile/secretfile.go)): password →
|
||||||
@@ -350,7 +354,10 @@ unsupported rather than as a missing doc:
|
|||||||
`POSTFIX_DIR` (`/data/postfix`), `POSTFIX_SENDER_LOGIN_MAPS`
|
`POSTFIX_DIR` (`/data/postfix`), `POSTFIX_SENDER_LOGIN_MAPS`
|
||||||
(`/data/postfix/sender_login_maps` — read by Postfix config only; the panel
|
(`/data/postfix/sender_login_maps` — read by Postfix config only; the panel
|
||||||
always writes `<POSTFIX_DIR>/sender_login_maps`, so overriding this env alone
|
always writes `<POSTFIX_DIR>/sender_login_maps`, so overriding this env alone
|
||||||
desyncs the map Postfix reads from the file the panel maintains).
|
desyncs the map Postfix reads from the file the panel maintains),
|
||||||
|
`POSTFIX_QUEUE_DIR` (`/data/postfix/queue` — set in `build/postfix-config.sh`),
|
||||||
|
`SELFPOST_DEPLOY_ROOT` (`/selfpost-deploy` — operator project directory for
|
||||||
|
full backups; mount `.:/selfpost-deploy:ro` in compose).
|
||||||
- **Milter and Postfix startup:** `MILTER_CONNECT_TIMEOUT` (`15s`),
|
- **Milter and Postfix startup:** `MILTER_CONNECT_TIMEOUT` (`15s`),
|
||||||
`MILTER_COMMAND_TIMEOUT` (`15s`), `MILTER_CONTENT_TIMEOUT` (`30s`),
|
`MILTER_COMMAND_TIMEOUT` (`15s`), `MILTER_CONTENT_TIMEOUT` (`30s`),
|
||||||
`MILTER_WAIT_TIMEOUT` (`30` seconds).
|
`MILTER_WAIT_TIMEOUT` (`30` seconds).
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
<div class="pair">
|
<div class="pair">
|
||||||
<div class="card">
|
<div class="card">
|
||||||
<h2>Full backup</h2>
|
<h2>Full backup</h2>
|
||||||
<p class="muted">Download a full backup of all persistent state — the database, every domain’s DKIM key and the application credentials. Restore into a container of the <strong>same SelfPost version</strong>, with the same data mount, before first start. TLS certificates and the mail queue are not included.</p>
|
<p class="muted">Download a self-contained backup — <code>data/</code> (database, DKIM keys, credentials, Postfix queue), <code>docker-compose.yml</code>, <code>.env</code>, and <code>certs/</code>. Extract into an empty project directory on a new machine, adjust hostname if needed, and start the <strong>same SelfPost version</strong> before first boot. The reverse-proxy vhost is not included.</p>
|
||||||
<p class="muted"><strong>The backup file is a secret.</strong> Encrypting it is the simplest way to store it: the download is then a <code>.spbk</code> that only the password opens.</p>
|
<p class="muted"><strong>The backup file is a secret.</strong> Encrypting it is the simplest way to store it: the download is then a <code>.spbk</code> that only the password opens.</p>
|
||||||
<div class="encrypt">
|
<div class="encrypt">
|
||||||
<label class="check"><input type="checkbox" checked> <span>Encrypt with a password</span></label>
|
<label class="check"><input type="checkbox" checked> <span>Encrypt with a password</span></label>
|
||||||
|
|||||||
+43
-34
@@ -417,25 +417,31 @@ journal-milter (level 2) is down. There is no per-IP bypass.
|
|||||||
|
|
||||||
### Full backup and restore
|
### Full backup and restore
|
||||||
|
|
||||||
**Full backup** (whole `/data` except `log/`: SQLite, all domains' DKIM keys,
|
**Full backup** is a self-contained project archive: `data/` (SQLite, all
|
||||||
all applications' SASL credentials, `manifest.json` with the version that
|
domains' DKIM keys, all applications' SASL credentials, the Postfix queue,
|
||||||
created it): panel button (*Backup* → *Full backup*), or from the host:
|
`manifest.json` with the version that created it), plus `docker-compose.yml`,
|
||||||
|
`.env`, and `certs/` from the operator directory next to `./data`. Delivery
|
||||||
|
logs under `data/log/` are excluded. The base compose file mounts the project
|
||||||
|
directory read-only at `/selfpost-deploy` so the panel and CLI can read those
|
||||||
|
deploy files — without that mount, *Full backup* refuses with an error.
|
||||||
|
|
||||||
|
Take a backup from the panel (*Backup* → *Full backup*) or from the host:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker exec <container> selfpost-backup > selfpost-backup.tar.gz
|
docker exec <container> selfpost-backup > selfpost-backup.tar.gz
|
||||||
```
|
```
|
||||||
|
|
||||||
**Restore** means unpacking that archive into a fresh `/data` bind mount and
|
**Restore** means unpacking that archive into an **empty project directory**
|
||||||
starting a container of the **exact same image version** that created it —
|
(not into `./data` alone) and starting a container of the **exact same image
|
||||||
SelfPost refuses to start otherwise and tells you which tag to use. On the
|
version** that created it — SelfPost refuses to start otherwise and tells you
|
||||||
first successful start after restore, `manifest.json` from the archive is
|
which tag to use. On the first successful start after restore, `data/manifest.json`
|
||||||
**deleted** — it guards only that one boot, so a later in-place upgrade is
|
from the archive is **deleted** — it guards only that one boot, so a later
|
||||||
not blocked. On that same first boot the panel also runs one **Resync** —
|
in-place upgrade is not blocked. On that same first boot the panel also runs
|
||||||
OpenDKIM's tables and Postfix's sender map are re-derived from SQLite and both
|
one **Resync** — OpenDKIM's tables and Postfix's sender map are re-derived from
|
||||||
daemons are reloaded, healing any drift between the extracted files and the
|
SQLite and both daemons are reloaded, healing any drift between the extracted
|
||||||
database (the Status page's *Reload configuration* button runs the same step
|
files and the database (the Status page's *Reload configuration* button runs
|
||||||
on demand). This is why the compose file pins a fixed tag rather than
|
the same step on demand). This is why the compose file pins a fixed tag rather
|
||||||
`:latest`: without a known version, there'd be no way to tell which image
|
than `:latest`: without a known version, there'd be no way to tell which image
|
||||||
restoring a given backup actually requires (see [Fixed image
|
restoring a given backup actually requires (see [Fixed image
|
||||||
tag](#fixed-image-tag)).
|
tag](#fixed-image-tag)).
|
||||||
|
|
||||||
@@ -446,16 +452,16 @@ back after a bad change):
|
|||||||
# 1. Stop the instance being replaced
|
# 1. Stop the instance being replaced
|
||||||
docker compose down
|
docker compose down
|
||||||
|
|
||||||
# 2. Move the current /data aside rather than deleting it, start from empty
|
# 2. Move the current project aside rather than deleting it
|
||||||
mv ./data ./data.before-restore
|
mv . ../selfpost.before-restore
|
||||||
mkdir ./data
|
mkdir selfpost && cd selfpost
|
||||||
|
|
||||||
# 3. Unpack the backup into the fresh directory
|
# 3. Unpack the backup into the fresh directory
|
||||||
tar xzf selfpost-backup.tar.gz -C ./data
|
tar xzf ../selfpost-backup.tar.gz
|
||||||
|
|
||||||
# 4. docker-compose.yml must already pin the exact tag the backup was made
|
# 4. docker-compose.yml in the archive must pin the exact tag the backup was
|
||||||
# with — check the archive's manifest if unsure:
|
# made with — check if unsure:
|
||||||
tar xzf selfpost-backup.tar.gz -O manifest.json
|
tar xzf ../selfpost-backup.tar.gz -O data/manifest.json
|
||||||
|
|
||||||
# 5. Start it and watch the boot
|
# 5. Start it and watch the boot
|
||||||
docker compose up -d
|
docker compose up -d
|
||||||
@@ -473,28 +479,31 @@ Fix the tag in `docker-compose.yml`, `docker compose pull && docker compose up
|
|||||||
-d` again — the manifest is still there because the failed boot never got to
|
-d` again — the manifest is still there because the failed boot never got to
|
||||||
delete it.
|
delete it.
|
||||||
|
|
||||||
**Moving to a different host** is the same five steps, just starting cold:
|
**Moving to a different host** is the same flow: create an empty project
|
||||||
bring the compose files and the correct pinned image tag to the new host
|
directory, unpack the backup there, edit `.env` (and `docker-compose.yml` if
|
||||||
(step 1 of [Full deployment](#full-deployment)), put the backup archive in
|
needed) for the new hostname or proxy, then `docker compose up -d`. The archive
|
||||||
place of step 3 above, then redo the reverse-proxy/TLS and DNS steps of a
|
carries `certs/` from the old host — re-issue certificates when the hostname or
|
||||||
[Full deployment](#full-deployment) — the PTR record and the certificate both
|
IP changes. Set up the reverse-proxy vhost separately (not in the backup).
|
||||||
belong to the old IP/host and have to be reissued for the new one; nothing in
|
|
||||||
the backup carries them.
|
|
||||||
|
|
||||||
**Restoring an encrypted (`.spbk`) backup** needs a running container to
|
**Restoring an encrypted (`.spbk`) backup** needs a running container to
|
||||||
decrypt it first — any container with the `selfpost-backup` CLI works; decryption
|
decrypt it first — any container with the `selfpost-backup` CLI works; decryption
|
||||||
does not read `/data` and performs no version check. Start one normally
|
does not read `/data` and performs no version check. Start one normally
|
||||||
(step 5, but on an empty `/data` you haven't unpacked yet), then:
|
(step 5, but on an empty project you have not unpacked yet), then:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker exec -i <container> selfpost-backup -decrypt < backup.spbk > selfpost-backup.tar.gz
|
docker exec -i <container> selfpost-backup -decrypt < backup.spbk > selfpost-backup.tar.gz
|
||||||
```
|
```
|
||||||
|
|
||||||
Stop it, wipe `/data` again, and continue from step 2 above with the
|
Stop it, wipe the project directory again, and continue from step 2 above with
|
||||||
resulting `.tar.gz` — see [Encrypting a backup or
|
the resulting `.tar.gz` — see [Encrypting a backup or
|
||||||
export](#encrypting-a-backup-or-export) for the decrypt command's password
|
export](#encrypting-a-backup-or-export) for the decrypt command's password
|
||||||
options.
|
options.
|
||||||
|
|
||||||
|
**Archives from older SelfPost versions** (flat layout: `manifest.json` and
|
||||||
|
`selfpost.db` at the archive root, no `data/` prefix, no deploy files) restore
|
||||||
|
with the previous procedure: `tar xzf backup.tar.gz -C ./data` into a project
|
||||||
|
that already has `docker-compose.yml` and `.env`.
|
||||||
|
|
||||||
Restoring an archive taken **before** a session row was removed can bring
|
Restoring an archive taken **before** a session row was removed can bring
|
||||||
that session back: session rows travel with the backup, and a browser that
|
that session back: session rows travel with the backup, and a browser that
|
||||||
still holds the matching cookie is signed in again on the next request if the
|
still holds the matching cookie is signed in again on the next request if the
|
||||||
@@ -520,9 +529,9 @@ domain](#exporting-and-importing-a-single-domain) — a different, domain-scoped
|
|||||||
operation that also lives on the *Backup* page (`/backup`).
|
operation that also lives on the *Backup* page (`/backup`).
|
||||||
|
|
||||||
Both a full backup and a domain export are **secrets** — they contain the
|
Both a full backup and a domain export are **secrets** — they contain the
|
||||||
admin password hash (full backup) or working application credentials (domain
|
admin password hash (full backup), TLS private keys and `.env` (full backup),
|
||||||
export) in the clear or in directly reversible form. Treat them like any
|
or working application credentials (domain export) in the clear or in
|
||||||
other credential material: restrict who can read them, don't email them
|
directly reversible form. Treat them like any other credential material: restrict who can read them, don't email them
|
||||||
around — and encrypt them, which SelfPost can do for you.
|
around — and encrypt them, which SelfPost can do for you.
|
||||||
|
|
||||||
#### Encrypting a backup or export
|
#### Encrypting a backup or export
|
||||||
|
|||||||
+5
-4
@@ -86,10 +86,11 @@ The panel is exposed to the internet — the items below are **not optional**.
|
|||||||
|
|
||||||
### Backup and domain export
|
### Backup and domain export
|
||||||
|
|
||||||
- Both files are secrets: a full backup carries DKIM keys, `sasldb2`, and the
|
- Both files are secrets: a full backup carries DKIM keys, `sasldb2`, the
|
||||||
administrator's password hash; a domain export carries the DKIM key and
|
administrator's password hash, `docker-compose.yml`, `.env`, and the TLS
|
||||||
**working** application passwords in the clear (otherwise a transfer without
|
private key from `certs/` when present; a domain export carries the DKIM key
|
||||||
recreating credentials would be impossible).
|
and **working** application passwords in the clear (otherwise a transfer
|
||||||
|
without recreating credentials would be impossible).
|
||||||
- Both downloads can be encrypted with a password (a checkbox on the form):
|
- Both downloads can be encrypted with a password (a checkbox on the form):
|
||||||
scrypt (N=2¹⁵, r=8, p=1) → AES-256-GCM, streamed in 64 KiB chunks, each
|
scrypt (N=2¹⁵, r=8, p=1) → AES-256-GCM, streamed in 64 KiB chunks, each
|
||||||
authenticated with the header, the chunk number, and an end-of-stream flag —
|
authenticated with the header, the chunk number, and an end-of-stream flag —
|
||||||
|
|||||||
+109
-39
@@ -1,15 +1,17 @@
|
|||||||
// Package backup implements SelfPost's full-server backup and the restore
|
// Package backup implements SelfPost's full-server backup and the restore
|
||||||
// version guard (architecture.md § Persistence). A full backup is a
|
// version guard (architecture.md § Persistence). A full backup is a
|
||||||
// gzip-compressed tar of the consolidated persistent state under /data — the
|
// gzip-compressed tar of the consolidated persistent state under /data — the
|
||||||
// SQLite database (as a consistent snapshot), the per-domain DKIM keys and the
|
// SQLite database (as a consistent snapshot), the per-domain DKIM keys, the
|
||||||
// SASL database — plus a manifest recording the SelfPost version that produced
|
// SASL database, and the Postfix queue — plus docker-compose.yml, .env, and
|
||||||
// it. TLS certificates (the reverse proxy's responsibility) and the Postfix
|
// certs/ from the operator's deploy directory, and a manifest recording the
|
||||||
// queue are deliberately excluded (architecture.md § Persistence).
|
// SelfPost version that produced it. Postfix delivery logs under log/ are
|
||||||
|
// excluded (diagnostics, not state).
|
||||||
//
|
//
|
||||||
// Restore is not a separate code path in the panel: a backup is extracted into
|
// Restore is not a separate code path in the panel: a backup is extracted into
|
||||||
// the /data bind mount before first start, and the archive already carries
|
// the operator's project directory (data/, docker-compose.yml, .env, certs/)
|
||||||
// everything the mail path needs — DKIM keys, sasldb2, and Postfix's sender
|
// before first start. The archive carries everything needed to bring the
|
||||||
// map — so the daemons can start on the extracted files alone. The
|
// instance back — DKIM keys, sasldb2, sender map, queue, and deploy files —
|
||||||
|
// so the operator only adjusts hostname or proxy settings on a new host. The
|
||||||
// restore-specific steps the panel runs are CheckRestore, which refuses to
|
// restore-specific steps the panel runs are CheckRestore, which refuses to
|
||||||
// boot if the manifest's version does not match the running binary so
|
// boot if the manifest's version does not match the running binary so
|
||||||
// schema/format skew between versions cannot silently corrupt state
|
// schema/format skew between versions cannot silently corrupt state
|
||||||
@@ -40,11 +42,21 @@ import (
|
|||||||
// FormatFull identifies a full-server backup manifest.
|
// FormatFull identifies a full-server backup manifest.
|
||||||
const FormatFull = "selfpost-full-backup"
|
const FormatFull = "selfpost-full-backup"
|
||||||
|
|
||||||
// ManifestName is the manifest's filename, both inside the archive and, after a
|
// ManifestName is the manifest's filename inside the data directory. After
|
||||||
// restore extraction, at the root of the data directory where CheckRestore
|
// restore extraction it lives at data/manifest.json under the project root;
|
||||||
// looks for it.
|
// CheckRestore reads it from the /data bind mount.
|
||||||
const ManifestName = "manifest.json"
|
const ManifestName = "manifest.json"
|
||||||
|
|
||||||
|
// DataArchivePrefix is the path prefix for every /data entry in the archive.
|
||||||
|
const DataArchivePrefix = "data/"
|
||||||
|
|
||||||
|
// ComposeFileName and EnvFileName are required deploy files at the archive root.
|
||||||
|
const (
|
||||||
|
ComposeFileName = "docker-compose.yml"
|
||||||
|
EnvFileName = ".env"
|
||||||
|
CertsDirName = "certs"
|
||||||
|
)
|
||||||
|
|
||||||
// Manifest is the small JSON document embedded in every backup archive. Its
|
// Manifest is the small JSON document embedded in every backup archive. Its
|
||||||
// Version is the single fact that makes restore safe: the panel refuses to
|
// Version is the single fact that makes restore safe: the panel refuses to
|
||||||
// boot a data directory whose manifest version does not match its own binary
|
// boot a data directory whose manifest version does not match its own binary
|
||||||
@@ -58,21 +70,23 @@ type Manifest struct {
|
|||||||
// Params configures a backup. DataDir is the consolidated state root (/data);
|
// Params configures a backup. DataDir is the consolidated state root (/data);
|
||||||
// DBPath is the live SQLite file within it, snapshotted consistently rather than
|
// DBPath is the live SQLite file within it, snapshotted consistently rather than
|
||||||
// copied byte-for-byte while it may be mid-write; Version is stamped into the
|
// copied byte-for-byte while it may be mid-write; Version is stamped into the
|
||||||
// manifest.
|
// manifest; DeployRoot is the host project directory mounted read-only (holds
|
||||||
|
// docker-compose.yml, .env, and optionally certs/). OnWarn is called for
|
||||||
|
// non-fatal issues such as a missing certs/ directory.
|
||||||
type Params struct {
|
type Params struct {
|
||||||
DataDir string
|
DataDir string
|
||||||
DBPath string
|
DBPath string
|
||||||
Version string
|
Version string
|
||||||
|
DeployRoot string
|
||||||
|
OnWarn func(string)
|
||||||
}
|
}
|
||||||
|
|
||||||
// excludedFromArchive lists the data-directory entries a backup never carries.
|
// excludedFromArchive lists the data-directory entries a backup never carries.
|
||||||
// The live database files are replaced by a consistent VACUUM INTO snapshot
|
// The live database files are replaced by a consistent VACUUM INTO snapshot
|
||||||
// written under the canonical name; the setup token is transient bootstrap
|
// written under the canonical name; the setup token is transient bootstrap
|
||||||
// state; a stale manifest from a previous restore must not be re-captured (a
|
// state; a stale manifest from a previous restore must not be re-captured (a
|
||||||
// fresh one is written instead); a "tls" directory holds the reverse proxy's
|
// fresh one is written instead); a "tls" directory under /data is skipped when
|
||||||
// certificates, which are explicitly out of scope for a SelfPost backup
|
// an operator pointed TLS_CERT_FILE inside /data; and "log" is Postfix's raw
|
||||||
// (architecture.md § Persistence) — excluding it keeps that guarantee even when
|
|
||||||
// an operator points TLS_CERT_FILE inside /data; and "log" is Postfix's raw
|
|
||||||
// delivery log plus its fourteen rotated files, which is diagnostic output, not
|
// delivery log plus its fourteen rotated files, which is diagnostic output, not
|
||||||
// state to restore, and by far the largest thing under /data.
|
// state to restore, and by far the largest thing under /data.
|
||||||
var excludedFromArchive = map[string]bool{
|
var excludedFromArchive = map[string]bool{
|
||||||
@@ -86,16 +100,22 @@ var excludedFromArchive = map[string]bool{
|
|||||||
ManifestName: true,
|
ManifestName: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create writes a gzip-compressed tar backup to w. Archive entries are named
|
// Create writes a gzip-compressed tar backup to w. Archive layout:
|
||||||
// relative to DataDir, so extracting the archive into the /data bind mount
|
//
|
||||||
// reconstructs the state in place (architecture.md § Persistence). The SQLite
|
// data/manifest.json, data/selfpost.db, data/<rest of /data>
|
||||||
// database is added as a consistent snapshot under "selfpost.db"; everything
|
// docker-compose.yml, .env, certs/...
|
||||||
// else under DataDir is copied as-is except the entries in
|
//
|
||||||
// excludedFromArchive.
|
// Extract the archive into an empty project directory, then docker compose up.
|
||||||
func Create(w io.Writer, p Params) error {
|
func Create(w io.Writer, p Params) error {
|
||||||
if p.DataDir == "" || p.DBPath == "" {
|
if p.DataDir == "" || p.DBPath == "" {
|
||||||
return fmt.Errorf("backup: DataDir and DBPath are required")
|
return fmt.Errorf("backup: DataDir and DBPath are required")
|
||||||
}
|
}
|
||||||
|
if p.DeployRoot == "" {
|
||||||
|
return fmt.Errorf("backup: DeployRoot is required (mount the project directory at SELFPOST_DEPLOY_ROOT)")
|
||||||
|
}
|
||||||
|
if err := validateDeployRoot(p.DeployRoot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
snapshot, cleanup, err := snapshotDB(p.DBPath)
|
snapshot, cleanup, err := snapshotDB(p.DBPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -115,17 +135,21 @@ func Create(w io.Writer, p Params) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("backup: encode manifest: %w", err)
|
return fmt.Errorf("backup: encode manifest: %w", err)
|
||||||
}
|
}
|
||||||
if err := writeTarBytes(tw, ManifestName, 0o600, manifestJSON); err != nil {
|
if err := writeTarBytes(tw, DataArchivePrefix+ManifestName, 0o600, manifestJSON); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// The consistent SQLite snapshot, under the canonical filename the panel
|
// The consistent SQLite snapshot, under the canonical filename the panel
|
||||||
// opens on start (the live file and its WAL/SHM are excluded from the walk).
|
// opens on start (the live file and its WAL/SHM are excluded from the walk).
|
||||||
if err := writeTarFile(tw, "selfpost.db", 0o640, snapshot); err != nil {
|
if err := writeTarFile(tw, DataArchivePrefix+"selfpost.db", 0o640, snapshot); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := addTree(tw, p.DataDir); err != nil {
|
if err := addTree(tw, p.DataDir, DataArchivePrefix, excludedFromArchive); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := addDeployFiles(tw, p.DeployRoot, p.OnWarn); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -138,33 +162,79 @@ func Create(w io.Writer, p Params) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// addTree walks dataDir and adds every regular file (and directory, to preserve
|
// ValidateDeployRoot checks that the operator project directory is mounted and
|
||||||
// empty ones and modes) to tw under its path relative to dataDir, skipping the
|
// contains the files a full backup requires. Call before streaming a response.
|
||||||
// excluded entries. Non-regular, non-directory entries (symlinks, sockets) are
|
func ValidateDeployRoot(deployRoot string) error {
|
||||||
// skipped: /data holds none in normal operation, and copying them into a backup
|
if deployRoot == "" {
|
||||||
// would be meaningless or unsafe.
|
return fmt.Errorf("backup: DeployRoot is required (mount the project directory at SELFPOST_DEPLOY_ROOT)")
|
||||||
func addTree(tw *tar.Writer, dataDir string) error {
|
}
|
||||||
return filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error {
|
return validateDeployRoot(deployRoot)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateDeployRoot(deployRoot string) error {
|
||||||
|
for _, name := range []string{ComposeFileName, EnvFileName} {
|
||||||
|
path := filepath.Join(deployRoot, name)
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
return fmt.Errorf("backup: deploy root %q is missing %s: %w", deployRoot, name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string)) error {
|
||||||
|
for _, name := range []string{ComposeFileName, EnvFileName} {
|
||||||
|
src := filepath.Join(deployRoot, name)
|
||||||
|
info, err := os.Stat(src)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("backup: stat deploy file %s: %w", name, err)
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("backup: deploy file %s is not a regular file", name)
|
||||||
|
}
|
||||||
|
if err := writeTarFile(tw, name, info.Mode().Perm(), src); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
certsDir := filepath.Join(deployRoot, CertsDirName)
|
||||||
|
if _, err := os.Stat(certsDir); err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
if onWarn != nil {
|
||||||
|
onWarn("certs/ not found in deploy root; backup will not include TLS material")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("backup: stat %s: %w", CertsDirName, err)
|
||||||
|
}
|
||||||
|
return addTree(tw, certsDir, CertsDirName+"/", nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// addTree walks root and adds every regular file (and directory, to preserve
|
||||||
|
// empty ones and modes) to tw under archivePrefix + path relative to root.
|
||||||
|
// When exclude is non-nil, top-level names relative to root are skipped.
|
||||||
|
// Non-regular, non-directory entries (symlinks, sockets) are skipped.
|
||||||
|
func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool) error {
|
||||||
|
return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
rel, err := filepath.Rel(dataDir, path)
|
rel, err := filepath.Rel(root, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if rel == "." {
|
if rel == "." {
|
||||||
return nil // the data root itself is implicit
|
return nil
|
||||||
}
|
}
|
||||||
// filepath.Rel yields OS separators; tar names use forward slashes.
|
name := archivePrefix + filepath.ToSlash(rel)
|
||||||
name := filepath.ToSlash(rel)
|
if exclude != nil {
|
||||||
// Exclude by top-level name (the live DB, setup token and stale manifest
|
top := strings.Split(filepath.ToSlash(rel), "/")[0]
|
||||||
// all live at the data root).
|
if exclude[top] {
|
||||||
if excludedFromArchive[name] {
|
|
||||||
if d.IsDir() {
|
if d.IsDir() {
|
||||||
return fs.SkipDir
|
return fs.SkipDir
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
info, err := d.Info()
|
info, err := d.Info()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -182,7 +252,7 @@ func addTree(tw *tar.Writer, dataDir string) error {
|
|||||||
case info.Mode().IsRegular():
|
case info.Mode().IsRegular():
|
||||||
return writeTarFile(tw, name, info.Mode().Perm(), path)
|
return writeTarFile(tw, name, info.Mode().Perm(), path)
|
||||||
default:
|
default:
|
||||||
return nil // skip symlinks/sockets/devices
|
return nil
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,10 +15,15 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// seedDataDir builds a realistic /data tree: a migrated SQLite database plus the
|
// seedDataDir builds a realistic /data tree: a migrated SQLite database plus the
|
||||||
// DKIM key, SASL and transient files a backup must include or exclude.
|
// DKIM key, SASL and transient files a backup must include or exclude. It also
|
||||||
func seedDataDir(t *testing.T) (dataDir, dbPath string) {
|
// seeds a deploy root beside data/ with compose, .env, and certs/.
|
||||||
|
func seedDataDir(t *testing.T) (dataDir, dbPath, deployRoot string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dataDir = t.TempDir()
|
deployRoot = t.TempDir()
|
||||||
|
dataDir = filepath.Join(deployRoot, "data")
|
||||||
|
if err := os.MkdirAll(dataDir, 0o750); err != nil {
|
||||||
|
t.Fatalf("mkdir data: %v", err)
|
||||||
|
}
|
||||||
dbPath = filepath.Join(dataDir, "selfpost.db")
|
dbPath = filepath.Join(dataDir, "selfpost.db")
|
||||||
|
|
||||||
st, err := store.Open(dbPath)
|
st, err := store.Open(dbPath)
|
||||||
@@ -35,6 +40,7 @@ func seedDataDir(t *testing.T) (dataDir, dbPath string) {
|
|||||||
writeFile(t, filepath.Join(dataDir, "opendkim", "keys", "example.com", "selfpost.private"), "PRIVATE KEY")
|
writeFile(t, filepath.Join(dataDir, "opendkim", "keys", "example.com", "selfpost.private"), "PRIVATE KEY")
|
||||||
writeFile(t, filepath.Join(dataDir, "sasl", "sasldb2"), "SASLDB")
|
writeFile(t, filepath.Join(dataDir, "sasl", "sasldb2"), "SASLDB")
|
||||||
writeFile(t, filepath.Join(dataDir, "postfix", "sender_login_maps"), "@example.com login")
|
writeFile(t, filepath.Join(dataDir, "postfix", "sender_login_maps"), "@example.com login")
|
||||||
|
writeFile(t, filepath.Join(dataDir, "postfix", "queue", "deferred", "sample"), "queue-file")
|
||||||
// Transient files that must NOT be archived.
|
// Transient files that must NOT be archived.
|
||||||
writeFile(t, filepath.Join(dataDir, "setup-token"), "secret-token")
|
writeFile(t, filepath.Join(dataDir, "setup-token"), "secret-token")
|
||||||
writeFile(t, filepath.Join(dataDir, "selfpost.db-wal"), "wal")
|
writeFile(t, filepath.Join(dataDir, "selfpost.db-wal"), "wal")
|
||||||
@@ -43,7 +49,11 @@ func seedDataDir(t *testing.T) (dataDir, dbPath string) {
|
|||||||
// state, and the bulkiest thing under /data.
|
// state, and the bulkiest thing under /data.
|
||||||
writeFile(t, filepath.Join(dataDir, "log", "mail.log"), "Aug 8 07:26:41 mail postfix/smtp[1]: ABC: to=<a@example.net>, status=sent (ok)")
|
writeFile(t, filepath.Join(dataDir, "log", "mail.log"), "Aug 8 07:26:41 mail postfix/smtp[1]: ABC: to=<a@example.net>, status=sent (ok)")
|
||||||
writeFile(t, filepath.Join(dataDir, "log", "mail.log.1"), "older")
|
writeFile(t, filepath.Join(dataDir, "log", "mail.log.1"), "older")
|
||||||
return dataDir, dbPath
|
writeFile(t, filepath.Join(deployRoot, ComposeFileName), "services:\n selfpost:\n image: test\n")
|
||||||
|
writeFile(t, filepath.Join(deployRoot, EnvFileName), "SELFPOST_HOSTNAME=mail.example.com\n")
|
||||||
|
writeFile(t, filepath.Join(deployRoot, CertsDirName, "fullchain.pem"), "CERT")
|
||||||
|
writeFile(t, filepath.Join(deployRoot, CertsDirName, "privkey.pem"), "KEY")
|
||||||
|
return dataDir, dbPath, deployRoot
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeFile(t *testing.T, path, content string) {
|
func writeFile(t *testing.T, path, content string) {
|
||||||
@@ -87,21 +97,26 @@ func readArchive(t *testing.T, data []byte) map[string]string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
||||||
dataDir, dbPath := seedDataDir(t)
|
dataDir, dbPath, deployRoot := seedDataDir(t)
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
if err := Create(&buf, Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3"}); err != nil {
|
if err := Create(&buf, Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3", DeployRoot: deployRoot}); err != nil {
|
||||||
t.Fatalf("Create: %v", err)
|
t.Fatalf("Create: %v", err)
|
||||||
}
|
}
|
||||||
files := readArchive(t, buf.Bytes())
|
files := readArchive(t, buf.Bytes())
|
||||||
|
|
||||||
// Present.
|
// Present.
|
||||||
for _, name := range []string{
|
for _, name := range []string{
|
||||||
ManifestName,
|
DataArchivePrefix + ManifestName,
|
||||||
"selfpost.db",
|
DataArchivePrefix + "selfpost.db",
|
||||||
"opendkim/keys/example.com/selfpost.private",
|
DataArchivePrefix + "opendkim/keys/example.com/selfpost.private",
|
||||||
"sasl/sasldb2",
|
DataArchivePrefix + "sasl/sasldb2",
|
||||||
"postfix/sender_login_maps",
|
DataArchivePrefix + "postfix/sender_login_maps",
|
||||||
|
DataArchivePrefix + "postfix/queue/deferred/sample",
|
||||||
|
ComposeFileName,
|
||||||
|
EnvFileName,
|
||||||
|
CertsDirName + "/fullchain.pem",
|
||||||
|
CertsDirName + "/privkey.pem",
|
||||||
} {
|
} {
|
||||||
if _, ok := files[name]; !ok {
|
if _, ok := files[name]; !ok {
|
||||||
t.Errorf("archive missing %s", name)
|
t.Errorf("archive missing %s", name)
|
||||||
@@ -109,8 +124,11 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
|||||||
}
|
}
|
||||||
// Excluded.
|
// Excluded.
|
||||||
for _, name := range []string{
|
for _, name := range []string{
|
||||||
"setup-token", "selfpost.db-wal", "selfpost.db-shm",
|
DataArchivePrefix + "setup-token",
|
||||||
"log/mail.log", "log/mail.log.1",
|
DataArchivePrefix + "selfpost.db-wal",
|
||||||
|
DataArchivePrefix + "selfpost.db-shm",
|
||||||
|
DataArchivePrefix + "log/mail.log",
|
||||||
|
DataArchivePrefix + "log/mail.log.1",
|
||||||
} {
|
} {
|
||||||
if _, ok := files[name]; ok {
|
if _, ok := files[name]; ok {
|
||||||
t.Errorf("archive should not contain %s", name)
|
t.Errorf("archive should not contain %s", name)
|
||||||
@@ -119,7 +137,7 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
|||||||
|
|
||||||
// Manifest is well-formed and carries the version.
|
// Manifest is well-formed and carries the version.
|
||||||
var m Manifest
|
var m Manifest
|
||||||
if err := json.Unmarshal([]byte(files[ManifestName]), &m); err != nil {
|
if err := json.Unmarshal([]byte(files[DataArchivePrefix+ManifestName]), &m); err != nil {
|
||||||
t.Fatalf("manifest json: %v", err)
|
t.Fatalf("manifest json: %v", err)
|
||||||
}
|
}
|
||||||
if m.Format != FormatFull || m.Version != "1.2.3" {
|
if m.Format != FormatFull || m.Version != "1.2.3" {
|
||||||
@@ -128,7 +146,7 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
|||||||
|
|
||||||
// The archived selfpost.db is a real, openable SQLite snapshot with our data.
|
// The archived selfpost.db is a real, openable SQLite snapshot with our data.
|
||||||
snapPath := filepath.Join(t.TempDir(), "restored.db")
|
snapPath := filepath.Join(t.TempDir(), "restored.db")
|
||||||
if err := os.WriteFile(snapPath, []byte(files["selfpost.db"]), 0o640); err != nil {
|
if err := os.WriteFile(snapPath, []byte(files[DataArchivePrefix+"selfpost.db"]), 0o640); err != nil {
|
||||||
t.Fatalf("write snapshot: %v", err)
|
t.Fatalf("write snapshot: %v", err)
|
||||||
}
|
}
|
||||||
st, err := store.Open(snapPath)
|
st, err := store.Open(snapPath)
|
||||||
@@ -145,6 +163,14 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCreateRequiresDeployRoot(t *testing.T) {
|
||||||
|
dataDir, dbPath, _ := seedDataDir(t)
|
||||||
|
var buf bytes.Buffer
|
||||||
|
if err := Create(&buf, Params{DataDir: dataDir, DBPath: dbPath, Version: "1.0.0"}); err == nil {
|
||||||
|
t.Fatal("Create without DeployRoot succeeded")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writeManifest(t *testing.T, dir, format, version string) string {
|
func writeManifest(t *testing.T, dir, format, version string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
path := filepath.Join(dir, ManifestName)
|
path := filepath.Join(dir, ManifestName)
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ type Config struct {
|
|||||||
MailLogPath string
|
MailLogPath string
|
||||||
DataDir string
|
DataDir string
|
||||||
DBPath string
|
DBPath string
|
||||||
|
DeployRoot string
|
||||||
Version string
|
Version string
|
||||||
TLSCertFile string
|
TLSCertFile string
|
||||||
OpenDKIMSocket string
|
OpenDKIMSocket string
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/mixeme/selfpost/internal/backup"
|
"github.com/mixeme/selfpost/internal/backup"
|
||||||
@@ -70,6 +71,11 @@ func (h *Handlers) HandleBackup(w http.ResponseWriter, r *http.Request) {
|
|||||||
h.renderBackupPageWith(w, r, http.StatusBadRequest, "", pwErr)
|
h.renderBackupPageWith(w, r, http.StatusBadRequest, "", pwErr)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if err := backup.ValidateDeployRoot(h.cfg.DeployRoot); err != nil {
|
||||||
|
logf("panel: full backup: %v", err)
|
||||||
|
h.renderBackupPageWith(w, r, http.StatusBadRequest, "", deployBackupErr(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
stamp := time.Now().UTC().Format("20060102-150405")
|
stamp := time.Now().UTC().Format("20060102-150405")
|
||||||
filename := fmt.Sprintf("selfpost-backup-%s.tar.gz", stamp)
|
filename := fmt.Sprintf("selfpost-backup-%s.tar.gz", stamp)
|
||||||
@@ -108,6 +114,10 @@ func (h *Handlers) HandleBackup(w http.ResponseWriter, r *http.Request) {
|
|||||||
DataDir: h.cfg.DataDir,
|
DataDir: h.cfg.DataDir,
|
||||||
DBPath: h.cfg.DBPath,
|
DBPath: h.cfg.DBPath,
|
||||||
Version: h.cfg.Version,
|
Version: h.cfg.Version,
|
||||||
|
DeployRoot: h.cfg.DeployRoot,
|
||||||
|
OnWarn: func(msg string) {
|
||||||
|
logf("panel: full backup: %s", msg)
|
||||||
|
},
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
logf("panel: full backup failed: %v", err)
|
logf("panel: full backup failed: %v", err)
|
||||||
return
|
return
|
||||||
@@ -310,6 +320,14 @@ func decryptErrorMessage(err error) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// deployBackupErr phrases a pre-flight backup failure for the operator.
|
||||||
|
func deployBackupErr(err error) string {
|
||||||
|
if strings.Contains(err.Error(), "DeployRoot") || strings.Contains(err.Error(), "deploy root") {
|
||||||
|
return "Full backup needs the project directory mounted read-only at /selfpost-deploy — add <code>.:/selfpost-deploy:ro</code> to docker-compose.yml and recreate the container."
|
||||||
|
}
|
||||||
|
return "Could not create the backup: " + err.Error()
|
||||||
|
}
|
||||||
|
|
||||||
// importErrorMessage maps a domain-import failure (already logged by the caller)
|
// importErrorMessage maps a domain-import failure (already logged by the caller)
|
||||||
// to an HTTP status and a user-facing message. Duplicate domain/login are called
|
// to an HTTP status and a user-facing message. Duplicate domain/login are called
|
||||||
// out specifically; other failures — validation errors describing what is wrong
|
// out specifically; other failures — validation errors describing what is wrong
|
||||||
|
|||||||
@@ -3,13 +3,15 @@
|
|||||||
|
|
||||||
<div class="card">
|
<div class="card">
|
||||||
<h2>Full backup</h2>
|
<h2>Full backup</h2>
|
||||||
<p class="muted">Download a full backup of all persistent state — the database,
|
<p class="muted">Download a self-contained backup of the whole instance —
|
||||||
every domain's DKIM key and the application credentials. Use it to move the
|
<code>data/</code> (database, DKIM keys, application credentials, and the
|
||||||
whole server to a new machine: restore it into a container of the
|
Postfix queue), <code>docker-compose.yml</code>, <code>.env</code>, and
|
||||||
<strong>same SelfPost version</strong>, with the same data mount, before first
|
<code>certs/</code>. Extract it into an empty project directory on a new
|
||||||
start. TLS certificates and the mail queue are not included.</p>
|
machine, adjust hostname or proxy settings if needed, and start a container of
|
||||||
|
the <strong>same SelfPost version</strong> before first boot. The reverse-proxy
|
||||||
|
vhost is not included — set that up separately on the new host.</p>
|
||||||
<p class="muted"><strong>The backup file is a secret</strong> (it contains
|
<p class="muted"><strong>The backup file is a secret</strong> (it contains
|
||||||
private keys and credentials). Store and transfer it securely and delete it
|
private keys, TLS material, and credentials). Store and transfer it securely and delete it
|
||||||
once the restore succeeds. Encrypting it below is the simplest way to do that:
|
once the restore succeeds. Encrypting it below is the simplest way to do that:
|
||||||
the download is then a <code>.spbk</code> file (SelfPost backup) that only
|
the download is then a <code>.spbk</code> file (SelfPost backup) that only
|
||||||
the password opens.</p>
|
the password opens.</p>
|
||||||
|
|||||||
+5
-2
@@ -40,10 +40,12 @@ type Config struct {
|
|||||||
// the log-tailer role follows in cmd/panel.
|
// the log-tailer role follows in cmd/panel.
|
||||||
MailLogPath string
|
MailLogPath string
|
||||||
// DataDir and DBPath locate the persistent state a full backup archives
|
// DataDir and DBPath locate the persistent state a full backup archives
|
||||||
// (architecture.md § Persistence); Version is stamped into the backup
|
// (architecture.md § Persistence); DeployRoot is the operator project
|
||||||
// manifest. They mirror the panel's own configuration.
|
// directory (docker-compose.yml, .env, certs/); Version is stamped into
|
||||||
|
// the backup manifest. They mirror the panel's own configuration.
|
||||||
DataDir string
|
DataDir string
|
||||||
DBPath string
|
DBPath string
|
||||||
|
DeployRoot string
|
||||||
Version string
|
Version string
|
||||||
// TrustedProxyCIDRs are the reverse-proxy addresses allowed to supply
|
// TrustedProxyCIDRs are the reverse-proxy addresses allowed to supply
|
||||||
// X-Forwarded-For (env TRUSTED_PROXY_CIDR). A request whose
|
// X-Forwarded-For (env TRUSTED_PROXY_CIDR). A request whose
|
||||||
@@ -106,6 +108,7 @@ func New(st *store.Store, domains *domain.Service, apps *app.Service, cfg Config
|
|||||||
MailLogPath: cfg.MailLogPath,
|
MailLogPath: cfg.MailLogPath,
|
||||||
DataDir: cfg.DataDir,
|
DataDir: cfg.DataDir,
|
||||||
DBPath: cfg.DBPath,
|
DBPath: cfg.DBPath,
|
||||||
|
DeployRoot: cfg.DeployRoot,
|
||||||
Version: cfg.Version,
|
Version: cfg.Version,
|
||||||
TLSCertFile: cfg.TLSCertFile,
|
TLSCertFile: cfg.TLSCertFile,
|
||||||
OpenDKIMSocket: cfg.OpenDKIMSocket,
|
OpenDKIMSocket: cfg.OpenDKIMSocket,
|
||||||
|
|||||||
Reference in New Issue
Block a user