Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b99f60c1d6 | ||
|
|
41c3e6e896 |
@@ -7,6 +7,25 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); version
|
|||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
||||||
|
- docs: roadmap candidates **send-log-retention** (panel Settings for delivery
|
||||||
|
journal retention) and **domain-stats-auto-ratelimit** (30-day send stats per
|
||||||
|
domain/application, auto level-2 rate limit from avg × multiplier) — plans in
|
||||||
|
[docs/plans/](docs/plans/).
|
||||||
|
|
||||||
|
- Postfix queue under `/data/postfix/queue` — deferred and active mail survive
|
||||||
|
container recreate and are included in full backups.
|
||||||
|
- Full backup archives the whole operator project: `data/`, `docker-compose.yml`,
|
||||||
|
`.env`, and `certs/` (requires `.:/selfpost-deploy:ro` in compose). Restore
|
||||||
|
by unpacking into an empty project directory.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- **Breaking:** full-backup archive layout — paths are prefixed with `data/`;
|
||||||
|
deploy files sit at the archive root. Old flat archives restore with
|
||||||
|
`tar xzf backup.tar.gz -C ./data` as before.
|
||||||
|
- docs: operator guide, architecture, security, and backup UI updated for
|
||||||
|
self-contained backups and persistent queue.
|
||||||
|
|
||||||
- docs: HTML mockups for a full panel UI refresh
|
- docs: HTML mockups for a full panel UI refresh
|
||||||
([docs/assets/panel-ui/](docs/assets/panel-ui/index.html)) — current screens,
|
([docs/assets/panel-ui/](docs/assets/panel-ui/index.html)) — current screens,
|
||||||
agreed and candidate roadmap surfaces (queue-retries, inbound-relay,
|
agreed and candidate roadmap surfaces (queue-retries, inbound-relay,
|
||||||
|
|||||||
+13
-6
@@ -69,7 +69,7 @@ chmod 755 /data
|
|||||||
# later phase deliberately hands to another service. /data/log is exempt: it is
|
# later phase deliberately hands to another service. /data/log is exempt: it is
|
||||||
# deliberately owned by postfix (postlogd writes the delivery log there) and is
|
# deliberately owned by postfix (postlogd writes the delivery log there) and is
|
||||||
# normalised on its own below.
|
# normalised on its own below.
|
||||||
find /data -mindepth 1 -maxdepth 1 ! -user panel ! -name log -exec chown -R panel:panel {} +
|
find /data -mindepth 1 -maxdepth 1 ! -user panel ! -name log ! -name postfix -exec chown -R panel:panel {} +
|
||||||
|
|
||||||
# DKIM key tree (spec 6, 9). The panel (user `panel`) generates keys and writes
|
# DKIM key tree (spec 6, 9). The panel (user `panel`) generates keys and writes
|
||||||
# the OpenDKIM tables; OpenDKIM (user `opendkim`) must read them. Normalise the
|
# the OpenDKIM tables; OpenDKIM (user `opendkim`) must read them. Normalise the
|
||||||
@@ -99,13 +99,13 @@ chown -R panel:selfpost /data/sasl
|
|||||||
chmod 2750 /data/sasl
|
chmod 2750 /data/sasl
|
||||||
[ -e /data/sasl/sasldb2 ] && chmod 0640 /data/sasl/sasldb2
|
[ -e /data/sasl/sasldb2 ] && chmod 0640 /data/sasl/sasldb2
|
||||||
|
|
||||||
# Postfix sender_login_maps (spec 5.1). The panel writes it; Postfix reads it.
|
# Postfix state under /data (spec 5.1, architecture.md § Persistence). The panel
|
||||||
# Ensure the file exists (empty is fine) before Postfix starts so a reload that
|
# writes sender_login_maps; Postfix owns the on-disk queue tree under queue/.
|
||||||
# references it never fails on a missing file, and keep it group-readable.
|
mkdir -p /data/postfix/queue
|
||||||
mkdir -p /data/postfix
|
|
||||||
[ -e /data/postfix/sender_login_maps ] || : > /data/postfix/sender_login_maps
|
[ -e /data/postfix/sender_login_maps ] || : > /data/postfix/sender_login_maps
|
||||||
chown -R panel:selfpost /data/postfix
|
chown panel:selfpost /data/postfix
|
||||||
chmod 2750 /data/postfix
|
chmod 2750 /data/postfix
|
||||||
|
chown panel:selfpost /data/postfix/sender_login_maps
|
||||||
chmod 0640 /data/postfix/sender_login_maps
|
chmod 0640 /data/postfix/sender_login_maps
|
||||||
|
|
||||||
# Delivery log (architecture.md § Log tailer). postlogd writes it as user
|
# Delivery log (architecture.md § Log tailer). postlogd writes it as user
|
||||||
@@ -144,4 +144,11 @@ chmod 2750 /run/opendkim /run/selfpost
|
|||||||
# start the same way the /data normalisation above is.
|
# start the same way the /data normalisation above is.
|
||||||
/usr/local/bin/postfix-config.sh
|
/usr/local/bin/postfix-config.sh
|
||||||
|
|
||||||
|
# Initialise the persistent queue tree on first start or after restore. postfix
|
||||||
|
# set-permissions reads queue_directory from main.cf (set by postfix-config.sh).
|
||||||
|
if [ ! -d /data/postfix/queue/active ]; then
|
||||||
|
postfix set-permissions
|
||||||
|
fi
|
||||||
|
chown -R postfix:postfix /data/postfix/queue
|
||||||
|
|
||||||
exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
|
exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
|
||||||
|
|||||||
@@ -57,6 +57,9 @@ JOURNAL_SOCK="${JOURNAL_MILTER_SOCKET:-/run/selfpost/journal.sock}"
|
|||||||
# Persistent panel-managed sender map (spec 5.1); texthash needs no postmap, so
|
# Persistent panel-managed sender map (spec 5.1); texthash needs no postmap, so
|
||||||
# the unprivileged panel can rewrite it and just ask for a reload.
|
# the unprivileged panel can rewrite it and just ask for a reload.
|
||||||
SENDER_LOGIN_MAPS="${POSTFIX_SENDER_LOGIN_MAPS:-/data/postfix/sender_login_maps}"
|
SENDER_LOGIN_MAPS="${POSTFIX_SENDER_LOGIN_MAPS:-/data/postfix/sender_login_maps}"
|
||||||
|
# Transit mail queue under /data so deferred/active messages survive container
|
||||||
|
# recreate (architecture.md § Persistence). Distinct from sender_login_maps.
|
||||||
|
QUEUE_DIR="${POSTFIX_QUEUE_DIR:-/data/postfix/queue}"
|
||||||
SASLDB_PATH="${SASL_DB_PATH:-/data/sasl/sasldb2}"
|
SASLDB_PATH="${SASL_DB_PATH:-/data/sasl/sasldb2}"
|
||||||
|
|
||||||
# Optional submission service on 587 (spec 5 p.1: off by default, enabled only
|
# Optional submission service on 587 (spec 5 p.1: off by default, enabled only
|
||||||
@@ -128,7 +131,8 @@ postconf -e \
|
|||||||
# resolves the full address first, then the "@domain" wildcard, so both address
|
# resolves the full address first, then the "@domain" wildcard, so both address
|
||||||
# modes work from the same map.
|
# modes work from the same map.
|
||||||
postconf -e \
|
postconf -e \
|
||||||
"smtpd_sender_login_maps=texthash:${SENDER_LOGIN_MAPS}"
|
"smtpd_sender_login_maps=texthash:${SENDER_LOGIN_MAPS}" \
|
||||||
|
"queue_directory=${QUEUE_DIR}"
|
||||||
|
|
||||||
# Restrictions: authenticated clients only, no relay to foreign destinations,
|
# Restrictions: authenticated clients only, no relay to foreign destinations,
|
||||||
# and every authenticated sender address must be owned by its login. NO
|
# and every authenticated sender address must be owned by its login. NO
|
||||||
|
|||||||
@@ -33,6 +33,8 @@ var documentedInternal = []string{
|
|||||||
"SASL_REALM",
|
"SASL_REALM",
|
||||||
"POSTFIX_DIR",
|
"POSTFIX_DIR",
|
||||||
"POSTFIX_SENDER_LOGIN_MAPS",
|
"POSTFIX_SENDER_LOGIN_MAPS",
|
||||||
|
"POSTFIX_QUEUE_DIR",
|
||||||
|
"SELFPOST_DEPLOY_ROOT",
|
||||||
"MILTER_CONNECT_TIMEOUT",
|
"MILTER_CONNECT_TIMEOUT",
|
||||||
"MILTER_COMMAND_TIMEOUT",
|
"MILTER_COMMAND_TIMEOUT",
|
||||||
"MILTER_CONTENT_TIMEOUT",
|
"MILTER_CONTENT_TIMEOUT",
|
||||||
@@ -70,6 +72,7 @@ var loadConfigKeys = []string{
|
|||||||
"SASL_DB_PATH",
|
"SASL_DB_PATH",
|
||||||
"SASL_REALM",
|
"SASL_REALM",
|
||||||
"POSTFIX_DIR",
|
"POSTFIX_DIR",
|
||||||
|
"SELFPOST_DEPLOY_ROOT",
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildScriptKeys is every ${VAR:-…} / os.Getenv used in build/*.sh and entrypoint.sh
|
// buildScriptKeys is every ${VAR:-…} / os.Getenv used in build/*.sh and entrypoint.sh
|
||||||
@@ -84,6 +87,7 @@ var buildScriptKeys = []string{
|
|||||||
"JOURNAL_MILTER_SOCKET",
|
"JOURNAL_MILTER_SOCKET",
|
||||||
"MAIL_LOG",
|
"MAIL_LOG",
|
||||||
"POSTFIX_SENDER_LOGIN_MAPS",
|
"POSTFIX_SENDER_LOGIN_MAPS",
|
||||||
|
"POSTFIX_QUEUE_DIR",
|
||||||
"SASL_DB_PATH",
|
"SASL_DB_PATH",
|
||||||
"SUBMISSION_ENABLE",
|
"SUBMISSION_ENABLE",
|
||||||
"MILTER_CONNECT_TIMEOUT",
|
"MILTER_CONNECT_TIMEOUT",
|
||||||
|
|||||||
@@ -73,6 +73,7 @@ func newPanel(cfg config, st *store.Store) (*web.Server, error) {
|
|||||||
MailLogPath: cfg.mailLog,
|
MailLogPath: cfg.mailLog,
|
||||||
DataDir: cfg.dataDir,
|
DataDir: cfg.dataDir,
|
||||||
DBPath: cfg.dbPath,
|
DBPath: cfg.dbPath,
|
||||||
|
DeployRoot: cfg.deployRoot,
|
||||||
Version: buildinfo.Version,
|
Version: buildinfo.Version,
|
||||||
TrustedProxyCIDRs: cfg.trustedProxies,
|
TrustedProxyCIDRs: cfg.trustedProxies,
|
||||||
TLSCertFile: cfg.tlsCertFile,
|
TLSCertFile: cfg.tlsCertFile,
|
||||||
|
|||||||
@@ -78,6 +78,7 @@ type config struct {
|
|||||||
saslDBPath string
|
saslDBPath string
|
||||||
saslRealm string
|
saslRealm string
|
||||||
postfixDir string
|
postfixDir string
|
||||||
|
deployRoot string
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadConfig() config {
|
func loadConfig() config {
|
||||||
@@ -138,6 +139,7 @@ func loadConfig() config {
|
|||||||
saslDBPath: envDefault("SASL_DB_PATH", filepath.Join(dataDir, "sasl", "sasldb2")),
|
saslDBPath: envDefault("SASL_DB_PATH", filepath.Join(dataDir, "sasl", "sasldb2")),
|
||||||
saslRealm: saslRealm(),
|
saslRealm: saslRealm(),
|
||||||
postfixDir: envDefault("POSTFIX_DIR", filepath.Join(dataDir, "postfix")),
|
postfixDir: envDefault("POSTFIX_DIR", filepath.Join(dataDir, "postfix")),
|
||||||
|
deployRoot: envDefault("SELFPOST_DEPLOY_ROOT", "/selfpost-deploy"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+60
-26
@@ -22,7 +22,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Restoring a SelfPost backup is not a code path in the panel: the operator
|
// Restoring a SelfPost backup is not a code path in the panel: the operator
|
||||||
// extracts the archive into the /data bind mount and starts the image, and the
|
// extracts the archive into a project directory and starts the image, and the
|
||||||
// panel is expected to come up on it (architecture.md § Persistence). Nothing
|
// panel is expected to come up on it (architecture.md § Persistence). Nothing
|
||||||
// below stubs that story out — the archive is downloaded from a running panel
|
// below stubs that story out — the archive is downloaded from a running panel
|
||||||
// through /backup, unpacked the way `tar -xzf` unpacks it, and a second panel
|
// through /backup, unpacked the way `tar -xzf` unpacks it, and a second panel
|
||||||
@@ -37,9 +37,10 @@ const (
|
|||||||
|
|
||||||
// restored is the outcome of a full backup-and-restore round trip.
|
// restored is the outcome of a full backup-and-restore round trip.
|
||||||
type restored struct {
|
type restored struct {
|
||||||
panel http.Handler // panel booted on the restored data directory
|
panel http.Handler // panel booted on the restored data directory
|
||||||
dataDir string // the restored /data
|
deployRoot string // the restored project directory
|
||||||
session *http.Cookie // a session opened before the backup was taken
|
dataDir string // the restored /data
|
||||||
|
session *http.Cookie // a session opened before the backup was taken
|
||||||
}
|
}
|
||||||
|
|
||||||
// restoreFromOwnBackup runs the operator's path end to end: seed a panel that
|
// restoreFromOwnBackup runs the operator's path end to end: seed a panel that
|
||||||
@@ -50,7 +51,7 @@ type restored struct {
|
|||||||
func restoreFromOwnBackup(t *testing.T, password string) restored {
|
func restoreFromOwnBackup(t *testing.T, password string) restored {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
live := seedPanelData(t)
|
live := seedPanelProject(t)
|
||||||
panel := bootPanel(t, live)
|
panel := bootPanel(t, live)
|
||||||
session := signIn(t, panel)
|
session := signIn(t, panel)
|
||||||
archive := downloadBackup(t, panel, session, password)
|
archive := downloadBackup(t, panel, session, password)
|
||||||
@@ -58,7 +59,12 @@ func restoreFromOwnBackup(t *testing.T, password string) restored {
|
|||||||
target := t.TempDir()
|
target := t.TempDir()
|
||||||
extract(t, archive, target)
|
extract(t, archive, target)
|
||||||
|
|
||||||
return restored{panel: bootPanel(t, target), dataDir: target, session: session}
|
return restored{
|
||||||
|
panel: bootPanel(t, target),
|
||||||
|
deployRoot: target,
|
||||||
|
dataDir: filepath.Join(target, "data"),
|
||||||
|
session: session,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The panel has to come up on the restored directory and show the state that
|
// The panel has to come up on the restored directory and show the state that
|
||||||
@@ -91,6 +97,12 @@ func TestPanelBootsOnADataDirectoryRestoredFromItsOwnBackup(t *testing.T) {
|
|||||||
t.Errorf("%s = %q, want %q", path, got, want)
|
t.Errorf("%s = %q, want %q", path, got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, name := range []string{backup.ComposeFileName, backup.EnvFileName} {
|
||||||
|
if _, err := os.Stat(filepath.Join(r.deployRoot, name)); err != nil {
|
||||||
|
t.Errorf("the restored project directory has no %s: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The one-time setup link is closed by the presence of a panel user, and the
|
// The one-time setup link is closed by the presence of a panel user, and the
|
||||||
@@ -138,8 +150,9 @@ func TestAnEncryptedBackupRestoresTheSameWay(t *testing.T) {
|
|||||||
// A restore boot runs one Resync from SQLite. If the archive's Postfix map
|
// A restore boot runs one Resync from SQLite. If the archive's Postfix map
|
||||||
// drifted from the database, that step puts it back before mail flows.
|
// drifted from the database, that step puts it back before mail flows.
|
||||||
func TestResyncAfterRestoreHealsDriftedMaps(t *testing.T) {
|
func TestResyncAfterRestoreHealsDriftedMaps(t *testing.T) {
|
||||||
dataDir := seedPanelData(t)
|
deployRoot := seedPanelProject(t)
|
||||||
cfg := panelConfig(t, dataDir)
|
dataDir := filepath.Join(deployRoot, "data")
|
||||||
|
cfg := panelConfig(t, deployRoot)
|
||||||
|
|
||||||
mapPath := filepath.Join(dataDir, "postfix", "sender_login_maps")
|
mapPath := filepath.Join(dataDir, "postfix", "sender_login_maps")
|
||||||
if err := os.WriteFile(mapPath, []byte("stale map\n"), 0o640); err != nil {
|
if err := os.WriteFile(mapPath, []byte("stale map\n"), 0o640); err != nil {
|
||||||
@@ -191,12 +204,14 @@ func TestResyncAfterRestoreHealsDriftedMaps(t *testing.T) {
|
|||||||
// stays put on a mismatch: the operator's next move is to start the image the
|
// stays put on a mismatch: the operator's next move is to start the image the
|
||||||
// backup names, and it has to be there when they do.
|
// backup names, and it has to be there when they do.
|
||||||
func TestPanelRefusesADataDirectoryRestoredFromAnotherVersion(t *testing.T) {
|
func TestPanelRefusesADataDirectoryRestoredFromAnotherVersion(t *testing.T) {
|
||||||
live := seedPanelData(t)
|
deployRoot := seedPanelProject(t)
|
||||||
|
dataDir := filepath.Join(deployRoot, "data")
|
||||||
var archive bytes.Buffer
|
var archive bytes.Buffer
|
||||||
if err := backup.Create(&archive, backup.Params{
|
if err := backup.Create(&archive, backup.Params{
|
||||||
DataDir: live,
|
DataDir: dataDir,
|
||||||
DBPath: filepath.Join(live, "selfpost.db"),
|
DBPath: filepath.Join(dataDir, "selfpost.db"),
|
||||||
Version: "9.9.9",
|
Version: "9.9.9",
|
||||||
|
DeployRoot: deployRoot,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatalf("create backup: %v", err)
|
t.Fatalf("create backup: %v", err)
|
||||||
}
|
}
|
||||||
@@ -218,13 +233,15 @@ func TestPanelRefusesADataDirectoryRestoredFromAnotherVersion(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedPanelData builds the /data tree of a panel that has been in use: an
|
// seedPanelProject builds an operator project tree: data/ with a panel that has
|
||||||
// administrator, a sending domain with an application and one logged message,
|
// been in use, plus docker-compose.yml, .env, and certs/ for full backups.
|
||||||
// and the daemon state the mail path needs (a DKIM key, the SASL database and
|
func seedPanelProject(t *testing.T) string {
|
||||||
// Postfix's sender map).
|
|
||||||
func seedPanelData(t *testing.T) string {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dataDir := t.TempDir()
|
deployRoot := t.TempDir()
|
||||||
|
dataDir := filepath.Join(deployRoot, "data")
|
||||||
|
if err := os.MkdirAll(dataDir, 0o750); err != nil {
|
||||||
|
t.Fatalf("mkdir data: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
st, err := store.Open(filepath.Join(dataDir, "selfpost.db"))
|
st, err := store.Open(filepath.Join(dataDir, "selfpost.db"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -268,16 +285,33 @@ func seedPanelData(t *testing.T) string {
|
|||||||
t.Fatalf("write %s: %v", full, err)
|
t.Fatalf("write %s: %v", full, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return dataDir
|
|
||||||
|
writeDeployFile(t, filepath.Join(deployRoot, backup.ComposeFileName), "services:\n selfpost:\n image: test\n")
|
||||||
|
writeDeployFile(t, filepath.Join(deployRoot, backup.EnvFileName), "SELFPOST_HOSTNAME=mail.example.ru\n")
|
||||||
|
writeDeployFile(t, filepath.Join(deployRoot, backup.CertsDirName, "fullchain.pem"), "CERT")
|
||||||
|
writeDeployFile(t, filepath.Join(deployRoot, backup.CertsDirName, "privkey.pem"), "KEY")
|
||||||
|
return deployRoot
|
||||||
}
|
}
|
||||||
|
|
||||||
// panelConfig resolves the panel's own configuration for a data directory, so
|
func writeDeployFile(t *testing.T, path, content string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
||||||
|
t.Fatalf("mkdir %s: %v", path, err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o640); err != nil {
|
||||||
|
t.Fatalf("write %s: %v", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// panelConfig resolves the panel's own configuration for a project directory, so
|
||||||
// the test finds the files where the running binary would look for them rather
|
// the test finds the files where the running binary would look for them rather
|
||||||
// than where it put them. Cookies are marked insecure for the same reason the
|
// than where it put them. Cookies are marked insecure for the same reason the
|
||||||
// e2e stand does it: the test client speaks plain HTTP.
|
// e2e stand does it: the test client speaks plain HTTP.
|
||||||
func panelConfig(t *testing.T, dataDir string) config {
|
func panelConfig(t *testing.T, deployRoot string) config {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
dataDir := filepath.Join(deployRoot, "data")
|
||||||
t.Setenv("SELFPOST_DATA_DIR", dataDir)
|
t.Setenv("SELFPOST_DATA_DIR", dataDir)
|
||||||
|
t.Setenv("SELFPOST_DEPLOY_ROOT", deployRoot)
|
||||||
t.Setenv("PANEL_COOKIE_SECURE", "false")
|
t.Setenv("PANEL_COOKIE_SECURE", "false")
|
||||||
t.Setenv("SELFPOST_HOSTNAME", "mail.example.ru")
|
t.Setenv("SELFPOST_HOSTNAME", "mail.example.ru")
|
||||||
// MAIL_LOG's default is an absolute path, not one derived from the data
|
// MAIL_LOG's default is an absolute path, not one derived from the data
|
||||||
@@ -288,13 +322,13 @@ func panelConfig(t *testing.T, dataDir string) config {
|
|||||||
|
|
||||||
// bootPanel performs the startup sequence run() performs, in the same order,
|
// bootPanel performs the startup sequence run() performs, in the same order,
|
||||||
// and returns the panel's HTTP handler.
|
// and returns the panel's HTTP handler.
|
||||||
func bootPanel(t *testing.T, dataDir string) http.Handler {
|
func bootPanel(t *testing.T, deployRoot string) http.Handler {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
cfg := panelConfig(t, dataDir)
|
cfg := panelConfig(t, deployRoot)
|
||||||
|
|
||||||
restored, err := backup.CheckRestore(cfg.manifestPath, buildinfo.Version)
|
restored, err := backup.CheckRestore(cfg.manifestPath, buildinfo.Version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("the panel refused to start on %s: %v", dataDir, err)
|
t.Fatalf("the panel refused to start on %s: %v", deployRoot, err)
|
||||||
}
|
}
|
||||||
if restored {
|
if restored {
|
||||||
if _, err := os.Stat(cfg.manifestPath); err == nil {
|
if _, err := os.Stat(cfg.manifestPath); err == nil {
|
||||||
@@ -380,8 +414,8 @@ func downloadBackup(t *testing.T, h http.Handler, session *http.Cookie, password
|
|||||||
return plain
|
return plain
|
||||||
}
|
}
|
||||||
|
|
||||||
// extract unpacks a backup archive into dir, as `tar -xzf` does onto the /data
|
// extract unpacks a backup archive into dir, as `tar -xzf` does onto the project
|
||||||
// bind mount before the image is started.
|
// directory before the image is started.
|
||||||
func extract(t *testing.T, archive []byte, dir string) {
|
func extract(t *testing.T, archive []byte, dir string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
gz, err := gzip.NewReader(bytes.NewReader(archive))
|
gz, err := gzip.NewReader(bytes.NewReader(archive))
|
||||||
|
|||||||
@@ -6,9 +6,9 @@
|
|||||||
//
|
//
|
||||||
// docker exec <container> selfpost-backup > selfpost-backup.tar.gz
|
// docker exec <container> selfpost-backup > selfpost-backup.tar.gz
|
||||||
//
|
//
|
||||||
// Use -o to write to a file instead. The resulting archive contains DKIM private
|
// Use -o to write to a file instead. The resulting archive is self-contained:
|
||||||
// keys, the admin password hash and SASL credentials — treat it as a secret
|
// data/, docker-compose.yml, .env, and certs/ when present — treat it as a
|
||||||
// (architecture.md § Persistence).
|
// secret (architecture.md § Persistence).
|
||||||
//
|
//
|
||||||
// Given a password (SELFPOST_BACKUP_PASSWORD or -password-file, never an
|
// Given a password (SELFPOST_BACKUP_PASSWORD or -password-file, never an
|
||||||
// argument, which would show up in the process list) the archive is written as
|
// argument, which would show up in the process list) the archive is written as
|
||||||
@@ -87,9 +87,13 @@ func run(outPath, password string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := backup.Create(sink, backup.Params{
|
if err := backup.Create(sink, backup.Params{
|
||||||
DataDir: dataDir,
|
DataDir: dataDir,
|
||||||
DBPath: dbPath,
|
DBPath: dbPath,
|
||||||
Version: buildinfo.Version,
|
Version: buildinfo.Version,
|
||||||
|
DeployRoot: envDefault("SELFPOST_DEPLOY_ROOT", "/selfpost-deploy"),
|
||||||
|
OnWarn: func(msg string) {
|
||||||
|
fmt.Fprintf(os.Stderr, "selfpost-backup: %s\n", msg)
|
||||||
|
},
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,13 +9,18 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/mixeme/selfpost/internal/backup"
|
||||||
"github.com/mixeme/selfpost/internal/store"
|
"github.com/mixeme/selfpost/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
// seedDataDir builds the minimum /data tree a backup can be taken from.
|
// seedProject builds the minimum operator project tree a backup can be taken from.
|
||||||
func seedDataDir(t *testing.T) string {
|
func seedProject(t *testing.T) (deployRoot, dataDir string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dataDir := t.TempDir()
|
deployRoot = t.TempDir()
|
||||||
|
dataDir = filepath.Join(deployRoot, "data")
|
||||||
|
if err := os.MkdirAll(dataDir, 0o750); err != nil {
|
||||||
|
t.Fatalf("mkdir data: %v", err)
|
||||||
|
}
|
||||||
st, err := store.Open(filepath.Join(dataDir, "selfpost.db"))
|
st, err := store.Open(filepath.Join(dataDir, "selfpost.db"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("open store: %v", err)
|
t.Fatalf("open store: %v", err)
|
||||||
@@ -26,16 +31,31 @@ func seedDataDir(t *testing.T) string {
|
|||||||
if err := st.Close(); err != nil {
|
if err := st.Close(); err != nil {
|
||||||
t.Fatalf("close store: %v", err)
|
t.Fatalf("close store: %v", err)
|
||||||
}
|
}
|
||||||
|
writeFile(t, filepath.Join(deployRoot, backup.ComposeFileName), "services:\n selfpost:\n image: test\n")
|
||||||
|
writeFile(t, filepath.Join(deployRoot, backup.EnvFileName), "SELFPOST_HOSTNAME=mail.example.com\n")
|
||||||
|
writeFile(t, filepath.Join(deployRoot, backup.CertsDirName, "fullchain.pem"), "CERT")
|
||||||
|
writeFile(t, filepath.Join(deployRoot, backup.CertsDirName, "privkey.pem"), "KEY")
|
||||||
t.Setenv("SELFPOST_DATA_DIR", dataDir)
|
t.Setenv("SELFPOST_DATA_DIR", dataDir)
|
||||||
t.Setenv("SELFPOST_DB_PATH", filepath.Join(dataDir, "selfpost.db"))
|
t.Setenv("SELFPOST_DB_PATH", filepath.Join(dataDir, "selfpost.db"))
|
||||||
return dataDir
|
t.Setenv("SELFPOST_DEPLOY_ROOT", deployRoot)
|
||||||
|
return deployRoot, dataDir
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeFile(t *testing.T, path, content string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
|
||||||
|
t.Fatalf("mkdir %s: %v", path, err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o640); err != nil {
|
||||||
|
t.Fatalf("write %s: %v", path, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// An encrypted backup is only worth having if the container it came from can
|
// An encrypted backup is only worth having if the container it came from can
|
||||||
// hand it back as an ordinary archive during a restore, so the two halves of
|
// hand it back as an ordinary archive during a restore, so the two halves of
|
||||||
// the CLI are tested as the one round trip an operator actually performs.
|
// the CLI are tested as the one round trip an operator actually performs.
|
||||||
func TestEncryptedBackupRoundTrip(t *testing.T) {
|
func TestEncryptedBackupRoundTrip(t *testing.T) {
|
||||||
seedDataDir(t)
|
seedProject(t)
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
encrypted := filepath.Join(dir, "backup.spbk")
|
encrypted := filepath.Join(dir, "backup.spbk")
|
||||||
plain := filepath.Join(dir, "backup.tar.gz")
|
plain := filepath.Join(dir, "backup.tar.gz")
|
||||||
@@ -81,7 +101,12 @@ func TestEncryptedBackupRoundTrip(t *testing.T) {
|
|||||||
}
|
}
|
||||||
names[hdr.Name] = true
|
names[hdr.Name] = true
|
||||||
}
|
}
|
||||||
for _, want := range []string{"manifest.json", "selfpost.db"} {
|
for _, want := range []string{
|
||||||
|
backup.DataArchivePrefix + backup.ManifestName,
|
||||||
|
backup.DataArchivePrefix + "selfpost.db",
|
||||||
|
backup.ComposeFileName,
|
||||||
|
backup.EnvFileName,
|
||||||
|
} {
|
||||||
if !names[want] {
|
if !names[want] {
|
||||||
t.Errorf("decrypted archive has no %s (entries: %v)", want, names)
|
t.Errorf("decrypted archive has no %s (entries: %v)", want, names)
|
||||||
}
|
}
|
||||||
@@ -91,7 +116,7 @@ func TestEncryptedBackupRoundTrip(t *testing.T) {
|
|||||||
// Without a password the CLI keeps producing the plain archive that existing
|
// Without a password the CLI keeps producing the plain archive that existing
|
||||||
// backup scripts consume.
|
// backup scripts consume.
|
||||||
func TestUnencryptedBackupStaysPlain(t *testing.T) {
|
func TestUnencryptedBackupStaysPlain(t *testing.T) {
|
||||||
seedDataDir(t)
|
seedProject(t)
|
||||||
out := filepath.Join(t.TempDir(), "backup.tar.gz")
|
out := filepath.Join(t.TempDir(), "backup.tar.gz")
|
||||||
if err := run(out, ""); err != nil {
|
if err := run(out, ""); err != nil {
|
||||||
t.Fatalf("create backup: %v", err)
|
t.Fatalf("create backup: %v", err)
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ services:
|
|||||||
TLS_KEY_FILE: /etc/postfix/tls/<hostname>.key
|
TLS_KEY_FILE: /etc/postfix/tls/<hostname>.key
|
||||||
volumes: !override
|
volumes: !override
|
||||||
- ./data:/data
|
- ./data:/data
|
||||||
|
- .:/selfpost-deploy:ro
|
||||||
# Replace <hostname> with your actual mail/panel hostname, matching
|
# Replace <hostname> with your actual mail/panel hostname, matching
|
||||||
# the Caddyfile below and SELFPOST_HOSTNAME.
|
# the Caddyfile below and SELFPOST_HOSTNAME.
|
||||||
- ./caddy/caddy-data/caddy/certificates/acme-v02.api.letsencrypt.org-directory/<hostname>:/etc/postfix/tls:ro
|
- ./caddy/caddy-data/caddy/certificates/acme-v02.api.letsencrypt.org-directory/<hostname>:/etc/postfix/tls:ro
|
||||||
|
|||||||
@@ -58,6 +58,9 @@ services:
|
|||||||
# kept) so the delivery lines that resolve a queued send-log row survive
|
# kept) so the delivery lines that resolve a queued send-log row survive
|
||||||
# a container recreate; it is diagnostics and is left out of backups.
|
# a container recreate; it is diagnostics and is left out of backups.
|
||||||
- ./data:/data
|
- ./data:/data
|
||||||
|
# Read-only project root for full backups: docker-compose.yml, .env, and
|
||||||
|
# certs/ are archived alongside /data (architecture.md § Persistence).
|
||||||
|
- .:/selfpost-deploy:ro
|
||||||
# Read-only: SelfPost only ever reads certificates, never manages them
|
# Read-only: SelfPost only ever reads certificates, never manages them
|
||||||
# (spec 10 p.2). Point this at wherever your reverse-proxy/certbot
|
# (spec 10 p.2). Point this at wherever your reverse-proxy/certbot
|
||||||
# writes PEM files, e.g. /etc/letsencrypt on the host.
|
# writes PEM files, e.g. /etc/letsencrypt on the host.
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ services:
|
|||||||
# network at selfpost:8080 instead (see nginx.conf.example).
|
# network at selfpost:8080 instead (see nginx.conf.example).
|
||||||
volumes: !override
|
volumes: !override
|
||||||
- ./data:/data
|
- ./data:/data
|
||||||
|
- .:/selfpost-deploy:ro
|
||||||
# Same host directory certbot below writes into — plain bind mount,
|
# Same host directory certbot below writes into — plain bind mount,
|
||||||
# no named volume, so the PEM files are as directly inspectable as in
|
# no named volume, so the PEM files are as directly inspectable as in
|
||||||
# the Apache scenario (spec 10.3).
|
# the Apache scenario (spec 10.3).
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ services:
|
|||||||
- "587:587"
|
- "587:587"
|
||||||
volumes: !override
|
volumes: !override
|
||||||
- ./data:/data
|
- ./data:/data
|
||||||
|
- .:/selfpost-deploy:ro
|
||||||
# Populated by extract-cert.sh from traefik's acme.json — see below.
|
# Populated by extract-cert.sh from traefik's acme.json — see below.
|
||||||
- ./traefik/extracted-certs:/etc/postfix/tls:ro
|
- ./traefik/extracted-certs:/etc/postfix/tls:ro
|
||||||
labels:
|
labels:
|
||||||
|
|||||||
+17
-10
@@ -291,23 +291,27 @@ single-connection trade-off that follows from it.
|
|||||||
| `opendkim/` | DKIM keys + tables |
|
| `opendkim/` | DKIM keys + tables |
|
||||||
| `sasl/sasldb2` | Application SASL credentials |
|
| `sasl/sasldb2` | Application SASL credentials |
|
||||||
| `postfix/sender_login_maps` | Login → From binding |
|
| `postfix/sender_login_maps` | Login → From binding |
|
||||||
|
| `postfix/queue/` | Postfix transit mail (deferred/active); survives container recreate |
|
||||||
| `log/mail.log` | Postfix delivery log + rotated copies (excluded from backups) |
|
| `log/mail.log` | Postfix delivery log + rotated copies (excluded from backups) |
|
||||||
| `manifest.json` | Backup version stamp (consumed on restore) |
|
| `manifest.json` | Backup version stamp (consumed on restore) |
|
||||||
|
|
||||||
Not in `/data`: TLS certificates (reverse-proxy mount), Postfix queue
|
Not in `/data`: TLS certificates for the panel (reverse-proxy mount) — though
|
||||||
(transit mail not migrated by design).
|
full backups also archive the operator's `./certs` PEM files when present.
|
||||||
|
|
||||||
**Rotation:** send-log retention `SEND_LOG_RETENTION_DAYS` (default 90);
|
**Rotation:** send-log retention `SEND_LOG_RETENTION_DAYS` (default 90);
|
||||||
`mail.log` via logrotate (14 rotated files, check every 6h, rename +
|
`mail.log` via logrotate (14 rotated files, check every 6h, rename +
|
||||||
`postfix reload` in `postrotate` — see § Log tailer above).
|
`postfix reload` in `postrotate` — see § Log tailer above).
|
||||||
|
|
||||||
**Restore:** panel button or `selfpost-backup` CLI — SQLite snapshot + tar of
|
**Restore:** panel button or `selfpost-backup` CLI — self-contained archive:
|
||||||
`/data` tree, minus `log/`, the setup token and any `tls/`; version check on
|
`data/` (SQLite snapshot + tree minus `log/`, the setup token and any `tls/`
|
||||||
restore. On the first successful boot after restore, the panel runs one
|
under `/data`), `docker-compose.yml`, `.env`, and `certs/` when present;
|
||||||
**Resync** — OpenDKIM's tables and Postfix's sender map are re-derived from
|
version check on restore. Requires the project directory mounted read-only at
|
||||||
SQLite and both daemons are reloaded, so drift between the extracted archive
|
`SELFPOST_DEPLOY_ROOT` (`/selfpost-deploy` in the default compose file). On the
|
||||||
and the database is healed before mail flows (same step as `POST /reload` on
|
first successful boot after restore, the panel runs one **Resync** — OpenDKIM's
|
||||||
demand). Stopped-container `tar` of `./data` is safe (see guide).
|
tables and Postfix's sender map are re-derived from SQLite and both daemons are
|
||||||
|
reloaded, so drift between the extracted archive and the database is healed
|
||||||
|
before mail flows (same step as `POST /reload` on demand). Stopped-container
|
||||||
|
`tar` of `./data` alone remains possible for state-only copies (see guide).
|
||||||
|
|
||||||
**Optional encryption** of the two secret-bearing downloads
|
**Optional encryption** of the two secret-bearing downloads
|
||||||
([internal/secretfile](../internal/secretfile/secretfile.go)): password →
|
([internal/secretfile](../internal/secretfile/secretfile.go)): password →
|
||||||
@@ -350,7 +354,10 @@ unsupported rather than as a missing doc:
|
|||||||
`POSTFIX_DIR` (`/data/postfix`), `POSTFIX_SENDER_LOGIN_MAPS`
|
`POSTFIX_DIR` (`/data/postfix`), `POSTFIX_SENDER_LOGIN_MAPS`
|
||||||
(`/data/postfix/sender_login_maps` — read by Postfix config only; the panel
|
(`/data/postfix/sender_login_maps` — read by Postfix config only; the panel
|
||||||
always writes `<POSTFIX_DIR>/sender_login_maps`, so overriding this env alone
|
always writes `<POSTFIX_DIR>/sender_login_maps`, so overriding this env alone
|
||||||
desyncs the map Postfix reads from the file the panel maintains).
|
desyncs the map Postfix reads from the file the panel maintains),
|
||||||
|
`POSTFIX_QUEUE_DIR` (`/data/postfix/queue` — set in `build/postfix-config.sh`),
|
||||||
|
`SELFPOST_DEPLOY_ROOT` (`/selfpost-deploy` — operator project directory for
|
||||||
|
full backups; mount `.:/selfpost-deploy:ro` in compose).
|
||||||
- **Milter and Postfix startup:** `MILTER_CONNECT_TIMEOUT` (`15s`),
|
- **Milter and Postfix startup:** `MILTER_CONNECT_TIMEOUT` (`15s`),
|
||||||
`MILTER_COMMAND_TIMEOUT` (`15s`), `MILTER_CONTENT_TIMEOUT` (`30s`),
|
`MILTER_COMMAND_TIMEOUT` (`15s`), `MILTER_CONTENT_TIMEOUT` (`30s`),
|
||||||
`MILTER_WAIT_TIMEOUT` (`30` seconds).
|
`MILTER_WAIT_TIMEOUT` (`30` seconds).
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
<div class="pair">
|
<div class="pair">
|
||||||
<div class="card">
|
<div class="card">
|
||||||
<h2>Full backup</h2>
|
<h2>Full backup</h2>
|
||||||
<p class="muted">Download a full backup of all persistent state — the database, every domain’s DKIM key and the application credentials. Restore into a container of the <strong>same SelfPost version</strong>, with the same data mount, before first start. TLS certificates and the mail queue are not included.</p>
|
<p class="muted">Download a self-contained backup — <code>data/</code> (database, DKIM keys, credentials, Postfix queue), <code>docker-compose.yml</code>, <code>.env</code>, and <code>certs/</code>. Extract into an empty project directory on a new machine, adjust hostname if needed, and start the <strong>same SelfPost version</strong> before first boot. The reverse-proxy vhost is not included.</p>
|
||||||
<p class="muted"><strong>The backup file is a secret.</strong> Encrypting it is the simplest way to store it: the download is then a <code>.spbk</code> that only the password opens.</p>
|
<p class="muted"><strong>The backup file is a secret.</strong> Encrypting it is the simplest way to store it: the download is then a <code>.spbk</code> that only the password opens.</p>
|
||||||
<div class="encrypt">
|
<div class="encrypt">
|
||||||
<label class="check"><input type="checkbox" checked> <span>Encrypt with a password</span></label>
|
<label class="check"><input type="checkbox" checked> <span>Encrypt with a password</span></label>
|
||||||
|
|||||||
+43
-34
@@ -417,25 +417,31 @@ journal-milter (level 2) is down. There is no per-IP bypass.
|
|||||||
|
|
||||||
### Full backup and restore
|
### Full backup and restore
|
||||||
|
|
||||||
**Full backup** (whole `/data` except `log/`: SQLite, all domains' DKIM keys,
|
**Full backup** is a self-contained project archive: `data/` (SQLite, all
|
||||||
all applications' SASL credentials, `manifest.json` with the version that
|
domains' DKIM keys, all applications' SASL credentials, the Postfix queue,
|
||||||
created it): panel button (*Backup* → *Full backup*), or from the host:
|
`manifest.json` with the version that created it), plus `docker-compose.yml`,
|
||||||
|
`.env`, and `certs/` from the operator directory next to `./data`. Delivery
|
||||||
|
logs under `data/log/` are excluded. The base compose file mounts the project
|
||||||
|
directory read-only at `/selfpost-deploy` so the panel and CLI can read those
|
||||||
|
deploy files — without that mount, *Full backup* refuses with an error.
|
||||||
|
|
||||||
|
Take a backup from the panel (*Backup* → *Full backup*) or from the host:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker exec <container> selfpost-backup > selfpost-backup.tar.gz
|
docker exec <container> selfpost-backup > selfpost-backup.tar.gz
|
||||||
```
|
```
|
||||||
|
|
||||||
**Restore** means unpacking that archive into a fresh `/data` bind mount and
|
**Restore** means unpacking that archive into an **empty project directory**
|
||||||
starting a container of the **exact same image version** that created it —
|
(not into `./data` alone) and starting a container of the **exact same image
|
||||||
SelfPost refuses to start otherwise and tells you which tag to use. On the
|
version** that created it — SelfPost refuses to start otherwise and tells you
|
||||||
first successful start after restore, `manifest.json` from the archive is
|
which tag to use. On the first successful start after restore, `data/manifest.json`
|
||||||
**deleted** — it guards only that one boot, so a later in-place upgrade is
|
from the archive is **deleted** — it guards only that one boot, so a later
|
||||||
not blocked. On that same first boot the panel also runs one **Resync** —
|
in-place upgrade is not blocked. On that same first boot the panel also runs
|
||||||
OpenDKIM's tables and Postfix's sender map are re-derived from SQLite and both
|
one **Resync** — OpenDKIM's tables and Postfix's sender map are re-derived from
|
||||||
daemons are reloaded, healing any drift between the extracted files and the
|
SQLite and both daemons are reloaded, healing any drift between the extracted
|
||||||
database (the Status page's *Reload configuration* button runs the same step
|
files and the database (the Status page's *Reload configuration* button runs
|
||||||
on demand). This is why the compose file pins a fixed tag rather than
|
the same step on demand). This is why the compose file pins a fixed tag rather
|
||||||
`:latest`: without a known version, there'd be no way to tell which image
|
than `:latest`: without a known version, there'd be no way to tell which image
|
||||||
restoring a given backup actually requires (see [Fixed image
|
restoring a given backup actually requires (see [Fixed image
|
||||||
tag](#fixed-image-tag)).
|
tag](#fixed-image-tag)).
|
||||||
|
|
||||||
@@ -446,16 +452,16 @@ back after a bad change):
|
|||||||
# 1. Stop the instance being replaced
|
# 1. Stop the instance being replaced
|
||||||
docker compose down
|
docker compose down
|
||||||
|
|
||||||
# 2. Move the current /data aside rather than deleting it, start from empty
|
# 2. Move the current project aside rather than deleting it
|
||||||
mv ./data ./data.before-restore
|
mv . ../selfpost.before-restore
|
||||||
mkdir ./data
|
mkdir selfpost && cd selfpost
|
||||||
|
|
||||||
# 3. Unpack the backup into the fresh directory
|
# 3. Unpack the backup into the fresh directory
|
||||||
tar xzf selfpost-backup.tar.gz -C ./data
|
tar xzf ../selfpost-backup.tar.gz
|
||||||
|
|
||||||
# 4. docker-compose.yml must already pin the exact tag the backup was made
|
# 4. docker-compose.yml in the archive must pin the exact tag the backup was
|
||||||
# with — check the archive's manifest if unsure:
|
# made with — check if unsure:
|
||||||
tar xzf selfpost-backup.tar.gz -O manifest.json
|
tar xzf ../selfpost-backup.tar.gz -O data/manifest.json
|
||||||
|
|
||||||
# 5. Start it and watch the boot
|
# 5. Start it and watch the boot
|
||||||
docker compose up -d
|
docker compose up -d
|
||||||
@@ -473,28 +479,31 @@ Fix the tag in `docker-compose.yml`, `docker compose pull && docker compose up
|
|||||||
-d` again — the manifest is still there because the failed boot never got to
|
-d` again — the manifest is still there because the failed boot never got to
|
||||||
delete it.
|
delete it.
|
||||||
|
|
||||||
**Moving to a different host** is the same five steps, just starting cold:
|
**Moving to a different host** is the same flow: create an empty project
|
||||||
bring the compose files and the correct pinned image tag to the new host
|
directory, unpack the backup there, edit `.env` (and `docker-compose.yml` if
|
||||||
(step 1 of [Full deployment](#full-deployment)), put the backup archive in
|
needed) for the new hostname or proxy, then `docker compose up -d`. The archive
|
||||||
place of step 3 above, then redo the reverse-proxy/TLS and DNS steps of a
|
carries `certs/` from the old host — re-issue certificates when the hostname or
|
||||||
[Full deployment](#full-deployment) — the PTR record and the certificate both
|
IP changes. Set up the reverse-proxy vhost separately (not in the backup).
|
||||||
belong to the old IP/host and have to be reissued for the new one; nothing in
|
|
||||||
the backup carries them.
|
|
||||||
|
|
||||||
**Restoring an encrypted (`.spbk`) backup** needs a running container to
|
**Restoring an encrypted (`.spbk`) backup** needs a running container to
|
||||||
decrypt it first — any container with the `selfpost-backup` CLI works; decryption
|
decrypt it first — any container with the `selfpost-backup` CLI works; decryption
|
||||||
does not read `/data` and performs no version check. Start one normally
|
does not read `/data` and performs no version check. Start one normally
|
||||||
(step 5, but on an empty `/data` you haven't unpacked yet), then:
|
(step 5, but on an empty project you have not unpacked yet), then:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
docker exec -i <container> selfpost-backup -decrypt < backup.spbk > selfpost-backup.tar.gz
|
docker exec -i <container> selfpost-backup -decrypt < backup.spbk > selfpost-backup.tar.gz
|
||||||
```
|
```
|
||||||
|
|
||||||
Stop it, wipe `/data` again, and continue from step 2 above with the
|
Stop it, wipe the project directory again, and continue from step 2 above with
|
||||||
resulting `.tar.gz` — see [Encrypting a backup or
|
the resulting `.tar.gz` — see [Encrypting a backup or
|
||||||
export](#encrypting-a-backup-or-export) for the decrypt command's password
|
export](#encrypting-a-backup-or-export) for the decrypt command's password
|
||||||
options.
|
options.
|
||||||
|
|
||||||
|
**Archives from older SelfPost versions** (flat layout: `manifest.json` and
|
||||||
|
`selfpost.db` at the archive root, no `data/` prefix, no deploy files) restore
|
||||||
|
with the previous procedure: `tar xzf backup.tar.gz -C ./data` into a project
|
||||||
|
that already has `docker-compose.yml` and `.env`.
|
||||||
|
|
||||||
Restoring an archive taken **before** a session row was removed can bring
|
Restoring an archive taken **before** a session row was removed can bring
|
||||||
that session back: session rows travel with the backup, and a browser that
|
that session back: session rows travel with the backup, and a browser that
|
||||||
still holds the matching cookie is signed in again on the next request if the
|
still holds the matching cookie is signed in again on the next request if the
|
||||||
@@ -520,9 +529,9 @@ domain](#exporting-and-importing-a-single-domain) — a different, domain-scoped
|
|||||||
operation that also lives on the *Backup* page (`/backup`).
|
operation that also lives on the *Backup* page (`/backup`).
|
||||||
|
|
||||||
Both a full backup and a domain export are **secrets** — they contain the
|
Both a full backup and a domain export are **secrets** — they contain the
|
||||||
admin password hash (full backup) or working application credentials (domain
|
admin password hash (full backup), TLS private keys and `.env` (full backup),
|
||||||
export) in the clear or in directly reversible form. Treat them like any
|
or working application credentials (domain export) in the clear or in
|
||||||
other credential material: restrict who can read them, don't email them
|
directly reversible form. Treat them like any other credential material: restrict who can read them, don't email them
|
||||||
around — and encrypt them, which SelfPost can do for you.
|
around — and encrypt them, which SelfPost can do for you.
|
||||||
|
|
||||||
#### Encrypting a backup or export
|
#### Encrypting a backup or export
|
||||||
|
|||||||
@@ -0,0 +1,187 @@
|
|||||||
|
# Plan: domain-stats-auto-ratelimit
|
||||||
|
|
||||||
|
**Status:** candidate
|
||||||
|
**Date:** 2026-08-17
|
||||||
|
**Version:** `1.x` MINOR; migrations must stay compatible with `1.0.0`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Give the operator **30-day sending statistics** per domain and per application
|
||||||
|
(total volume, peak and average rate), and an optional **auto** level-2 rate
|
||||||
|
limit that sets `max_messages` from the average rate (avg × multiplier) over the
|
||||||
|
level-1 window.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
**In:**
|
||||||
|
|
||||||
|
- Rolling **30-day** stats on each domain page (domain aggregate + per-app rows):
|
||||||
|
message count, peak msg/h, average msg/h.
|
||||||
|
- Level-2 rate limit mode **manual** (today) or **auto** for domain and
|
||||||
|
application scopes.
|
||||||
|
- Auto formula: `max_messages = ceil(avg_hourly × multiplier)`, window =
|
||||||
|
level-1 window (`RATE_LIMIT_WINDOW_SECONDS`); capped at level 1.
|
||||||
|
- Background recalculation (e.g. every 6 h, alongside send-log prune); milter
|
||||||
|
reads stored `max_messages` / `window_seconds` only (no aggregates on the
|
||||||
|
hot path).
|
||||||
|
- Panel UI: stats card, manual/auto toggle, multiplier field, read-only computed
|
||||||
|
limit in auto mode, optional «Recalculate now».
|
||||||
|
- RBAC: domain-admin sees stats and may configure auto/manual for assigned
|
||||||
|
domains only; same authz as existing rate-limit handlers.
|
||||||
|
- Tests, [guide.md](../guide.md), [CHANGELOG.md](../../CHANGELOG.md); security
|
||||||
|
review (Fable) for rate-limit path changes.
|
||||||
|
|
||||||
|
**Out:**
|
||||||
|
|
||||||
|
- Changing level 1 (Postfix env) — auto only fills level 2 under the L1 cap.
|
||||||
|
- Automated IP warmup schedules ([guide.md](../guide.md) § IP warmup stays
|
||||||
|
operator-driven).
|
||||||
|
- Prometheus/Grafana, alerting, APIs.
|
||||||
|
- Counting level-1 refusals or `rejected` rows as sent volume.
|
||||||
|
- Per-client-IP analytics.
|
||||||
|
|
||||||
|
## Data source
|
||||||
|
|
||||||
|
All metrics come from SQLite `send_log`, same rules as
|
||||||
|
[`CountMessages`](../../internal/store/ratelimits.go):
|
||||||
|
|
||||||
|
- One message = one distinct `queue_id` (many recipients = one count).
|
||||||
|
- `status != rejected` (level-2 refusals never queued).
|
||||||
|
- Level-1 refusals are **not** in `send_log` — stats under-count refusals;
|
||||||
|
document in UI copy.
|
||||||
|
|
||||||
|
Retention today is env `SEND_LOG_RETENTION_DAYS` (default 90). Stats use the
|
||||||
|
last **30 days** of rows still present. If retention < 30 days (after
|
||||||
|
[send-log-retention](send-log-retention.md)), the stats window is
|
||||||
|
`min(30, retention)` with a warning.
|
||||||
|
|
||||||
|
### Metrics
|
||||||
|
|
||||||
|
| Metric | Definition |
|
||||||
|
|---|---|
|
||||||
|
| **total** | `COUNT(DISTINCT queue_id)` in the stats window |
|
||||||
|
| **peak rate** | maximum messages in any **hourly** bucket in that window (msg/h) |
|
||||||
|
| **avg rate** | `total / hours_in_window`, where `hours_in_window = min(720, age of domain/app in hours, retention hours)` |
|
||||||
|
|
||||||
|
Keys: `send_log.domain` (domain scope), `send_log.app_login` (application scope).
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart LR
|
||||||
|
milter["journal-milter"] --> sendLog["send_log"]
|
||||||
|
sendLog --> statsQuery["StatsQuery 30d"]
|
||||||
|
statsQuery --> domainPage["domain page stats card"]
|
||||||
|
statsQuery --> autoJob["auto recalc job"]
|
||||||
|
autoJob --> rateLimits["rate_limits"]
|
||||||
|
rateLimits --> milter
|
||||||
|
```
|
||||||
|
|
||||||
|
1. **[`internal/store/stats.go`](../../internal/store/stats.go)** — `SendStats`
|
||||||
|
with `Total`, `PeakPerHour`, `AvgPerHour`; `DomainSendStats(name, since)` /
|
||||||
|
`AppSendStats(login, since)`.
|
||||||
|
2. Hourly buckets: `strftime('%Y-%m-%d %H', created_at)` + `GROUP BY`; subquery
|
||||||
|
for peak; total via distinct `queue_id`.
|
||||||
|
3. Indexes `idx_send_log_domain` and `idx_send_log_created_at` exist; add
|
||||||
|
composite `(domain, created_at)` only if profiling shows need.
|
||||||
|
4. **Auto recalc** — panel goroutine (same interval as send-log prune): for each
|
||||||
|
`rate_limits` row with `mode = auto`, recompute `max_messages`, set
|
||||||
|
`auto_updated_at`. Milter unchanged except reading new columns via existing
|
||||||
|
`RateLimit` lookup.
|
||||||
|
|
||||||
|
### Auto rate limit
|
||||||
|
|
||||||
|
Extend [`RateLimit`](../../internal/store/ratelimits.go):
|
||||||
|
|
||||||
|
```go
|
||||||
|
type RateLimit struct {
|
||||||
|
// existing: Scope, RefID, AllowedIPs, MaxMessages, WindowSeconds
|
||||||
|
Mode string // "manual" | "auto"
|
||||||
|
AutoMultiplier float64 // default 2.5 when Mode == "auto"
|
||||||
|
AutoUpdatedAt time.Time
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Formula:**
|
||||||
|
|
||||||
|
```
|
||||||
|
avg_hourly = total_messages_in_window / hours_in_window
|
||||||
|
max_messages = ceil(avg_hourly * auto_multiplier)
|
||||||
|
window_seconds = L1 window (not editable in auto mode)
|
||||||
|
max_messages = min(max_messages, L1 max)
|
||||||
|
```
|
||||||
|
|
||||||
|
When `total == 0`: auto limit stays **inactive** (same as empty manual limit);
|
||||||
|
UI explains that traffic is required before auto can apply.
|
||||||
|
|
||||||
|
**Application overrides** ([`handlers_ratelimit.go`](../../internal/web/handlers/handlers_ratelimit.go)):
|
||||||
|
|
||||||
|
- Trusted IPs required.
|
||||||
|
- Auto app ceiling **strictly above** domain limit when domain limit is active.
|
||||||
|
- Ceiling ≤ L1.
|
||||||
|
|
||||||
|
**Fail-open:** store errors during recalc must not weaken enforcement of the
|
||||||
|
last successfully written limit; recalc failures are logged only.
|
||||||
|
|
||||||
|
### Migration (`0006_rate_limit_auto.sql`)
|
||||||
|
|
||||||
|
```sql
|
||||||
|
ALTER TABLE rate_limits ADD COLUMN mode TEXT NOT NULL DEFAULT 'manual'
|
||||||
|
CHECK (mode IN ('manual', 'auto'));
|
||||||
|
ALTER TABLE rate_limits ADD COLUMN auto_multiplier REAL;
|
||||||
|
ALTER TABLE rate_limits ADD COLUMN auto_updated_at TEXT;
|
||||||
|
```
|
||||||
|
|
||||||
|
Existing rows → `manual`.
|
||||||
|
|
||||||
|
### Domain export
|
||||||
|
|
||||||
|
Today rate limits are **not** exported. This plan adds them (including
|
||||||
|
`mode`, `auto_multiplier`) to domain transfer JSON — document as a boundary
|
||||||
|
change in [guide.md](../guide.md) § Export.
|
||||||
|
|
||||||
|
## Panel UI
|
||||||
|
|
||||||
|
- Domain page ([`domain_detail.html`](../../internal/web/view/templates/domain_detail.html)):
|
||||||
|
- **Sending statistics (30 days)** — total, peak msg/h, avg msg/h.
|
||||||
|
- Per-application stats in the app list.
|
||||||
|
- Rate limit: Manual / Auto, multiplier (e.g. 1.5–5.0, default 2.5), read-only
|
||||||
|
computed max/window in auto mode, «Recalculate now».
|
||||||
|
- Optional later: «30d» column on domain list (global admin only).
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
- `internal/store/stats_test.go` — fixtures → total / peak / avg.
|
||||||
|
- `internal/store/ratelimits_test.go` — auto recalc, L1 cap, app > domain.
|
||||||
|
- Handler tests — auto form validation, multiplier bounds.
|
||||||
|
- Milter tests — enforced limit matches last recalculated values.
|
||||||
|
|
||||||
|
`go test` / `go vet` on touched packages.
|
||||||
|
|
||||||
|
## Done when
|
||||||
|
|
||||||
|
- Domain and app 30-day stats visible on the domain page; domain-admin scoping
|
||||||
|
enforced.
|
||||||
|
- Manual/auto toggle works for domain and app; auto recalc updates `rate_limits`
|
||||||
|
and milter enforces stored ceilings.
|
||||||
|
- Zero-traffic auto stays inactive with clear UI copy.
|
||||||
|
- [guide.md](../guide.md) and [CHANGELOG.md](../../CHANGELOG.md) updated;
|
||||||
|
security review passed.
|
||||||
|
|
||||||
|
## Risks
|
||||||
|
|
||||||
|
- Heavy aggregation on large `send_log` tables — mitigate with indexes or
|
||||||
|
nightly rollups (phase 2).
|
||||||
|
- Stats without level-1 visibility — mitigate with operator-facing caveat.
|
||||||
|
- Auto limit too tight after a spike — multiplier is operator-tuned; show peak
|
||||||
|
alongside avg in auto UI.
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
- [`send-log-retention`](send-log-retention.md) is a separate roadmap item but
|
||||||
|
should land before or in parallel so operators can set retention ≥ 30 days
|
||||||
|
from the panel.
|
||||||
|
|
||||||
|
**Version:** `1.x` MINOR.
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
# Plan: send-log-retention
|
||||||
|
|
||||||
|
**Status:** candidate
|
||||||
|
**Date:** 2026-08-17
|
||||||
|
**Version:** `1.x` MINOR; no schema migration required (uses existing `settings` table).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Let the **global administrator** change how long delivery journal rows
|
||||||
|
(`send_log`, `/deliveries`) are kept, from the panel — without editing `.env`.
|
||||||
|
|
||||||
|
## Context (as-built)
|
||||||
|
|
||||||
|
Retention **already exists**, but only via environment:
|
||||||
|
|
||||||
|
- `SEND_LOG_RETENTION_DAYS` (default **90**) in `.env` / Compose.
|
||||||
|
- [`cmd/panel/main.go`](../../cmd/panel/main.go) passes it to
|
||||||
|
[`logtail.Run`](../../internal/logtail/logtail.go).
|
||||||
|
- [`retentionLoop`](../../internal/logtail/logtail.go) prunes via
|
||||||
|
[`DeleteSendLogBefore`](../../internal/store/sendlog.go) every **6 hours**.
|
||||||
|
- No panel control; [`handlers_monitor.go`](../../internal/web/handlers/handlers_monitor.go)
|
||||||
|
hardcodes «ninety days» in copy.
|
||||||
|
- Migration `0001_init.sql` describes `settings` as the place for «retention
|
||||||
|
overrides», but no UI writes that key yet.
|
||||||
|
|
||||||
|
This plan moves the **effective** retention into SQLite `settings`, with env as
|
||||||
|
bootstrap only.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
**In:**
|
||||||
|
|
||||||
|
- Settings card on `/settings` (global administrator only): **Send log
|
||||||
|
retention (days)**.
|
||||||
|
- Key `send_log_retention_days` in [`settings`](../../internal/store/settings.go).
|
||||||
|
- Validation: integer range **7–365** (exact bounds fixed at implementation).
|
||||||
|
- On first use: if setting missing, seed from env
|
||||||
|
(`SEND_LOG_RETENTION_DAYS`, default 90) at panel start or first save.
|
||||||
|
- Log-tailer reads the setting **each prune cycle** (no container restart).
|
||||||
|
- Delivery pages and guide copy show the **current** retention, not a hardcoded
|
||||||
|
90.
|
||||||
|
- Tests; [guide.md](../guide.md); [CHANGELOG.md](../../CHANGELOG.md).
|
||||||
|
|
||||||
|
**Out:**
|
||||||
|
|
||||||
|
- Per-domain retention (instance-wide only).
|
||||||
|
- `mail.log` rotation (logrotate, 14 daily files — unchanged).
|
||||||
|
- Immediate prune on save when lowering retention (next 6 h cycle is enough;
|
||||||
|
optional «Prune now» not in v1).
|
||||||
|
- Domain-admin access to this setting.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart LR
|
||||||
|
settingsPage["/settings form"] --> sqlite["settings.send_log_retention_days"]
|
||||||
|
env["SEND_LOG_RETENTION_DAYS bootstrap"] --> sqlite
|
||||||
|
sqlite --> retentionLoop["logtail retentionLoop"]
|
||||||
|
retentionLoop --> prune["DeleteSendLogBefore"]
|
||||||
|
```
|
||||||
|
|
||||||
|
1. **Read path** — `GetSendLogRetentionDays()`: settings value if valid, else env
|
||||||
|
default.
|
||||||
|
2. **Write path** — POST `/settings` (global admin): validate, `SetSetting`,
|
||||||
|
flash confirmation.
|
||||||
|
3. **Prune path** — change [`logtail.retentionLoop`](../../internal/logtail/logtail.go)
|
||||||
|
to accept `func() int` or `RetentionReader` that queries settings each cycle
|
||||||
|
(same 6 h ticker).
|
||||||
|
4. **Copy** — inject retention days into delivery list/detail templates and
|
||||||
|
remove hardcoded «ninety days».
|
||||||
|
|
||||||
|
`SEND_LOG_RETENTION_DAYS` remains documented in [guide.md](../guide.md) as the
|
||||||
|
**initial default** until changed in Settings.
|
||||||
|
|
||||||
|
## Relation to domain-stats-auto-ratelimit
|
||||||
|
|
||||||
|
[domain-stats-auto-ratelimit](domain-stats-auto-ratelimit.md) uses a **30-day**
|
||||||
|
stats window. Requires effective retention ≥ 30 for full accuracy. When
|
||||||
|
retention < 30:
|
||||||
|
|
||||||
|
- Stats UI shows a warning and uses `min(30, retention)` as the window, or
|
||||||
|
- Settings validation warns when saving a value below 30 while stats/auto are
|
||||||
|
enabled (pick one at implementation; document in guide).
|
||||||
|
|
||||||
|
Recommended roadmap order: **send-log-retention** before or parallel with
|
||||||
|
domain-stats-auto-ratelimit.
|
||||||
|
|
||||||
|
## Panel UI
|
||||||
|
|
||||||
|
New card on [`settings.html`](../../internal/web/view/templates/settings.html)
|
||||||
|
(global admin block, near rate limits or under a «Deliveries» heading):
|
||||||
|
|
||||||
|
- Number input: retention days (7–365).
|
||||||
|
- Muted copy: rows older than this are deleted from `/deliveries`; main driver
|
||||||
|
of `/data` growth; does not affect `mail.log` rotation.
|
||||||
|
|
||||||
|
Domain administrators keep the narrow credentials-only settings page.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
- Save/load setting; reject out-of-range values.
|
||||||
|
- `retentionLoop` uses updated value without process restart (mock reader).
|
||||||
|
- Bootstrap: empty settings → env default used for prune.
|
||||||
|
- Template/delivery copy reflects configured days.
|
||||||
|
|
||||||
|
`go test` / `go vet` on touched packages.
|
||||||
|
|
||||||
|
## Done when
|
||||||
|
|
||||||
|
- Global admin can set retention on `/settings`; value persists in SQLite.
|
||||||
|
- Prune uses the panel value on the next cycle; env remains bootstrap default.
|
||||||
|
- Guide documents panel vs env; CHANGELOG entry added.
|
||||||
|
- Hardcoded «ninety days» removed from delivery UI.
|
||||||
|
|
||||||
|
## Risks
|
||||||
|
|
||||||
|
- Operator lowers retention while bookmarking old delivery URLs — existing
|
||||||
|
behaviour; copy already notes pruned rows are gone.
|
||||||
|
- Settings change without restart — must be tested so log-tailer never keeps a
|
||||||
|
stale int from panel start only.
|
||||||
|
|
||||||
|
**Version:** `1.x` MINOR.
|
||||||
+3
-1
@@ -59,7 +59,9 @@ domain administrators with assigned domains). The optional **inbound relay** is
|
|||||||
the main agreed **1.x+** extension still on the
|
the main agreed **1.x+** extension still on the
|
||||||
[roadmap](roadmap.md) — it targets a 1.x MINOR bump by default; a 2.x major
|
[roadmap](roadmap.md) — it targets a 1.x MINOR bump by default; a 2.x major
|
||||||
remains possible pending implementation. Items marked *candidate* in the
|
remains possible pending implementation. Items marked *candidate* in the
|
||||||
roadmap require explicit approval before coding.
|
roadmap require explicit approval before coding (including **send-log retention
|
||||||
|
in Settings**, **30-day send statistics**, and **auto level-2 rate limits** —
|
||||||
|
see linked plans there).
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
+43
-1
@@ -30,6 +30,8 @@ in `git log` and [CHANGELOG.md](../CHANGELOG.md).
|
|||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| queue-retries | Postfix retry policy in the panel (queue lifetime, backoff) | **agreed** | [plans/queue-retries.md](plans/queue-retries.md) |
|
| queue-retries | Postfix retry policy in the panel (queue lifetime, backoff) | **agreed** | [plans/queue-retries.md](plans/queue-retries.md) |
|
||||||
| inbound-relay | Inbound relay (backup-MX / forwarding) | **agreed** | [plans/inbound-relay.md](plans/inbound-relay.md) |
|
| inbound-relay | Inbound relay (backup-MX / forwarding) | **agreed** | [plans/inbound-relay.md](plans/inbound-relay.md) |
|
||||||
|
| send-log-retention | Send-log retention days in panel Settings | candidate | [plans/send-log-retention.md](plans/send-log-retention.md) |
|
||||||
|
| domain-stats-auto-ratelimit | 30-day send stats + auto level-2 rate limit | candidate | [plans/domain-stats-auto-ratelimit.md](plans/domain-stats-auto-ratelimit.md) |
|
||||||
| contributing | `CONTRIBUTING.md` | candidate | — |
|
| contributing | `CONTRIBUTING.md` | candidate | — |
|
||||||
| dmarc-reports | DMARC aggregate report ingestion and panel UI | candidate | [plans/dmarc-reports.md](plans/dmarc-reports.md) |
|
| dmarc-reports | DMARC aggregate report ingestion and panel UI | candidate | [plans/dmarc-reports.md](plans/dmarc-reports.md) |
|
||||||
| panel-docs | In-panel operator documentation | candidate | — |
|
| panel-docs | In-panel operator documentation | candidate | — |
|
||||||
@@ -37,7 +39,9 @@ in `git log` and [CHANGELOG.md](../CHANGELOG.md).
|
|||||||
|
|
||||||
**Recommended order** (not binding): **queue-retries** is a small panel item
|
**Recommended order** (not binding): **queue-retries** is a small panel item
|
||||||
that can land first or in parallel; the next feature is **inbound-relay**.
|
that can land first or in parallel; the next feature is **inbound-relay**.
|
||||||
The 2026-08-13 full-tree review follow-ups (send-log authorization,
|
**send-log-retention** can land before or beside **domain-stats-auto-ratelimit**
|
||||||
|
(panel retention ≥ 30 days helps the stats window). The 2026-08-13 full-tree
|
||||||
|
review follow-ups (send-log authorization,
|
||||||
fail-closed paths, docs, GUI, tests, licence) are closed — history in
|
fail-closed paths, docs, GUI, tests, licence) are closed — history in
|
||||||
[CHANGELOG.md](../CHANGELOG.md) `[Unreleased]` and git. Candidates need
|
[CHANGELOG.md](../CHANGELOG.md) `[Unreleased]` and git. Candidates need
|
||||||
explicit agreement before they join the queue.
|
explicit agreement before they join the queue.
|
||||||
@@ -89,6 +93,44 @@ implementation lands.
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## send-log-retention
|
||||||
|
|
||||||
|
**Goal:** global administrator sets how many days of delivery journal rows
|
||||||
|
(`send_log`, `/deliveries`) are kept, from `/settings` — stored in SQLite
|
||||||
|
`settings`, with `SEND_LOG_RETENTION_DAYS` as bootstrap default only.
|
||||||
|
|
||||||
|
**Boundary:** instance-wide retention; not per-domain. Does not change
|
||||||
|
`mail.log` rotation. Today retention is env-only (default 90 days); the panel
|
||||||
|
has no control.
|
||||||
|
|
||||||
|
**Done when:** see [plans/send-log-retention.md](plans/send-log-retention.md).
|
||||||
|
|
||||||
|
**Dependencies / risks:** log-tailer must re-read settings each prune cycle;
|
||||||
|
delivery UI must stop hardcoding «ninety days».
|
||||||
|
**Version:** `1.x` MINOR; `candidate` until explicitly agreed.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## domain-stats-auto-ratelimit
|
||||||
|
|
||||||
|
**Goal:** 30-day sending statistics per domain and application (total, peak and
|
||||||
|
average msg/h) on the domain page, plus optional **auto** level-2 rate limits
|
||||||
|
computed as `ceil(avg_hourly × multiplier)` over the level-1 window.
|
||||||
|
|
||||||
|
**Boundary:** extension of v1.0 manual L2 limits; does not change Postfix
|
||||||
|
level 1. Stats exclude level-1 refusals (not in send log). Auto with zero
|
||||||
|
traffic stays inactive.
|
||||||
|
|
||||||
|
**Done when:** see
|
||||||
|
[plans/domain-stats-auto-ratelimit.md](plans/domain-stats-auto-ratelimit.md).
|
||||||
|
|
||||||
|
**Dependencies / risks:** send log data and preferably
|
||||||
|
[send-log-retention](#send-log-retention) ≥ 30 days; aggregate query cost on
|
||||||
|
large logs; rate-limit path needs security review.
|
||||||
|
**Version:** `1.x` MINOR; `candidate` until explicitly agreed.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## contributing
|
## contributing
|
||||||
|
|
||||||
**Goal:** `CONTRIBUTING.md` in the root — the dev loop, the checks to run
|
**Goal:** `CONTRIBUTING.md` in the root — the dev loop, the checks to run
|
||||||
|
|||||||
+5
-4
@@ -86,10 +86,11 @@ The panel is exposed to the internet — the items below are **not optional**.
|
|||||||
|
|
||||||
### Backup and domain export
|
### Backup and domain export
|
||||||
|
|
||||||
- Both files are secrets: a full backup carries DKIM keys, `sasldb2`, and the
|
- Both files are secrets: a full backup carries DKIM keys, `sasldb2`, the
|
||||||
administrator's password hash; a domain export carries the DKIM key and
|
administrator's password hash, `docker-compose.yml`, `.env`, and the TLS
|
||||||
**working** application passwords in the clear (otherwise a transfer without
|
private key from `certs/` when present; a domain export carries the DKIM key
|
||||||
recreating credentials would be impossible).
|
and **working** application passwords in the clear (otherwise a transfer
|
||||||
|
without recreating credentials would be impossible).
|
||||||
- Both downloads can be encrypted with a password (a checkbox on the form):
|
- Both downloads can be encrypted with a password (a checkbox on the form):
|
||||||
scrypt (N=2¹⁵, r=8, p=1) → AES-256-GCM, streamed in 64 KiB chunks, each
|
scrypt (N=2¹⁵, r=8, p=1) → AES-256-GCM, streamed in 64 KiB chunks, each
|
||||||
authenticated with the header, the chunk number, and an end-of-stream flag —
|
authenticated with the header, the chunk number, and an end-of-stream flag —
|
||||||
|
|||||||
+116
-46
@@ -1,15 +1,17 @@
|
|||||||
// Package backup implements SelfPost's full-server backup and the restore
|
// Package backup implements SelfPost's full-server backup and the restore
|
||||||
// version guard (architecture.md § Persistence). A full backup is a
|
// version guard (architecture.md § Persistence). A full backup is a
|
||||||
// gzip-compressed tar of the consolidated persistent state under /data — the
|
// gzip-compressed tar of the consolidated persistent state under /data — the
|
||||||
// SQLite database (as a consistent snapshot), the per-domain DKIM keys and the
|
// SQLite database (as a consistent snapshot), the per-domain DKIM keys, the
|
||||||
// SASL database — plus a manifest recording the SelfPost version that produced
|
// SASL database, and the Postfix queue — plus docker-compose.yml, .env, and
|
||||||
// it. TLS certificates (the reverse proxy's responsibility) and the Postfix
|
// certs/ from the operator's deploy directory, and a manifest recording the
|
||||||
// queue are deliberately excluded (architecture.md § Persistence).
|
// SelfPost version that produced it. Postfix delivery logs under log/ are
|
||||||
|
// excluded (diagnostics, not state).
|
||||||
//
|
//
|
||||||
// Restore is not a separate code path in the panel: a backup is extracted into
|
// Restore is not a separate code path in the panel: a backup is extracted into
|
||||||
// the /data bind mount before first start, and the archive already carries
|
// the operator's project directory (data/, docker-compose.yml, .env, certs/)
|
||||||
// everything the mail path needs — DKIM keys, sasldb2, and Postfix's sender
|
// before first start. The archive carries everything needed to bring the
|
||||||
// map — so the daemons can start on the extracted files alone. The
|
// instance back — DKIM keys, sasldb2, sender map, queue, and deploy files —
|
||||||
|
// so the operator only adjusts hostname or proxy settings on a new host. The
|
||||||
// restore-specific steps the panel runs are CheckRestore, which refuses to
|
// restore-specific steps the panel runs are CheckRestore, which refuses to
|
||||||
// boot if the manifest's version does not match the running binary so
|
// boot if the manifest's version does not match the running binary so
|
||||||
// schema/format skew between versions cannot silently corrupt state
|
// schema/format skew between versions cannot silently corrupt state
|
||||||
@@ -40,11 +42,21 @@ import (
|
|||||||
// FormatFull identifies a full-server backup manifest.
|
// FormatFull identifies a full-server backup manifest.
|
||||||
const FormatFull = "selfpost-full-backup"
|
const FormatFull = "selfpost-full-backup"
|
||||||
|
|
||||||
// ManifestName is the manifest's filename, both inside the archive and, after a
|
// ManifestName is the manifest's filename inside the data directory. After
|
||||||
// restore extraction, at the root of the data directory where CheckRestore
|
// restore extraction it lives at data/manifest.json under the project root;
|
||||||
// looks for it.
|
// CheckRestore reads it from the /data bind mount.
|
||||||
const ManifestName = "manifest.json"
|
const ManifestName = "manifest.json"
|
||||||
|
|
||||||
|
// DataArchivePrefix is the path prefix for every /data entry in the archive.
|
||||||
|
const DataArchivePrefix = "data/"
|
||||||
|
|
||||||
|
// ComposeFileName and EnvFileName are required deploy files at the archive root.
|
||||||
|
const (
|
||||||
|
ComposeFileName = "docker-compose.yml"
|
||||||
|
EnvFileName = ".env"
|
||||||
|
CertsDirName = "certs"
|
||||||
|
)
|
||||||
|
|
||||||
// Manifest is the small JSON document embedded in every backup archive. Its
|
// Manifest is the small JSON document embedded in every backup archive. Its
|
||||||
// Version is the single fact that makes restore safe: the panel refuses to
|
// Version is the single fact that makes restore safe: the panel refuses to
|
||||||
// boot a data directory whose manifest version does not match its own binary
|
// boot a data directory whose manifest version does not match its own binary
|
||||||
@@ -58,21 +70,23 @@ type Manifest struct {
|
|||||||
// Params configures a backup. DataDir is the consolidated state root (/data);
|
// Params configures a backup. DataDir is the consolidated state root (/data);
|
||||||
// DBPath is the live SQLite file within it, snapshotted consistently rather than
|
// DBPath is the live SQLite file within it, snapshotted consistently rather than
|
||||||
// copied byte-for-byte while it may be mid-write; Version is stamped into the
|
// copied byte-for-byte while it may be mid-write; Version is stamped into the
|
||||||
// manifest.
|
// manifest; DeployRoot is the host project directory mounted read-only (holds
|
||||||
|
// docker-compose.yml, .env, and optionally certs/). OnWarn is called for
|
||||||
|
// non-fatal issues such as a missing certs/ directory.
|
||||||
type Params struct {
|
type Params struct {
|
||||||
DataDir string
|
DataDir string
|
||||||
DBPath string
|
DBPath string
|
||||||
Version string
|
Version string
|
||||||
|
DeployRoot string
|
||||||
|
OnWarn func(string)
|
||||||
}
|
}
|
||||||
|
|
||||||
// excludedFromArchive lists the data-directory entries a backup never carries.
|
// excludedFromArchive lists the data-directory entries a backup never carries.
|
||||||
// The live database files are replaced by a consistent VACUUM INTO snapshot
|
// The live database files are replaced by a consistent VACUUM INTO snapshot
|
||||||
// written under the canonical name; the setup token is transient bootstrap
|
// written under the canonical name; the setup token is transient bootstrap
|
||||||
// state; a stale manifest from a previous restore must not be re-captured (a
|
// state; a stale manifest from a previous restore must not be re-captured (a
|
||||||
// fresh one is written instead); a "tls" directory holds the reverse proxy's
|
// fresh one is written instead); a "tls" directory under /data is skipped when
|
||||||
// certificates, which are explicitly out of scope for a SelfPost backup
|
// an operator pointed TLS_CERT_FILE inside /data; and "log" is Postfix's raw
|
||||||
// (architecture.md § Persistence) — excluding it keeps that guarantee even when
|
|
||||||
// an operator points TLS_CERT_FILE inside /data; and "log" is Postfix's raw
|
|
||||||
// delivery log plus its fourteen rotated files, which is diagnostic output, not
|
// delivery log plus its fourteen rotated files, which is diagnostic output, not
|
||||||
// state to restore, and by far the largest thing under /data.
|
// state to restore, and by far the largest thing under /data.
|
||||||
var excludedFromArchive = map[string]bool{
|
var excludedFromArchive = map[string]bool{
|
||||||
@@ -86,16 +100,22 @@ var excludedFromArchive = map[string]bool{
|
|||||||
ManifestName: true,
|
ManifestName: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create writes a gzip-compressed tar backup to w. Archive entries are named
|
// Create writes a gzip-compressed tar backup to w. Archive layout:
|
||||||
// relative to DataDir, so extracting the archive into the /data bind mount
|
//
|
||||||
// reconstructs the state in place (architecture.md § Persistence). The SQLite
|
// data/manifest.json, data/selfpost.db, data/<rest of /data>
|
||||||
// database is added as a consistent snapshot under "selfpost.db"; everything
|
// docker-compose.yml, .env, certs/...
|
||||||
// else under DataDir is copied as-is except the entries in
|
//
|
||||||
// excludedFromArchive.
|
// Extract the archive into an empty project directory, then docker compose up.
|
||||||
func Create(w io.Writer, p Params) error {
|
func Create(w io.Writer, p Params) error {
|
||||||
if p.DataDir == "" || p.DBPath == "" {
|
if p.DataDir == "" || p.DBPath == "" {
|
||||||
return fmt.Errorf("backup: DataDir and DBPath are required")
|
return fmt.Errorf("backup: DataDir and DBPath are required")
|
||||||
}
|
}
|
||||||
|
if p.DeployRoot == "" {
|
||||||
|
return fmt.Errorf("backup: DeployRoot is required (mount the project directory at SELFPOST_DEPLOY_ROOT)")
|
||||||
|
}
|
||||||
|
if err := validateDeployRoot(p.DeployRoot); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
snapshot, cleanup, err := snapshotDB(p.DBPath)
|
snapshot, cleanup, err := snapshotDB(p.DBPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -115,17 +135,21 @@ func Create(w io.Writer, p Params) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("backup: encode manifest: %w", err)
|
return fmt.Errorf("backup: encode manifest: %w", err)
|
||||||
}
|
}
|
||||||
if err := writeTarBytes(tw, ManifestName, 0o600, manifestJSON); err != nil {
|
if err := writeTarBytes(tw, DataArchivePrefix+ManifestName, 0o600, manifestJSON); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// The consistent SQLite snapshot, under the canonical filename the panel
|
// The consistent SQLite snapshot, under the canonical filename the panel
|
||||||
// opens on start (the live file and its WAL/SHM are excluded from the walk).
|
// opens on start (the live file and its WAL/SHM are excluded from the walk).
|
||||||
if err := writeTarFile(tw, "selfpost.db", 0o640, snapshot); err != nil {
|
if err := writeTarFile(tw, DataArchivePrefix+"selfpost.db", 0o640, snapshot); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := addTree(tw, p.DataDir); err != nil {
|
if err := addTree(tw, p.DataDir, DataArchivePrefix, excludedFromArchive); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := addDeployFiles(tw, p.DeployRoot, p.OnWarn); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -138,33 +162,79 @@ func Create(w io.Writer, p Params) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// addTree walks dataDir and adds every regular file (and directory, to preserve
|
// ValidateDeployRoot checks that the operator project directory is mounted and
|
||||||
// empty ones and modes) to tw under its path relative to dataDir, skipping the
|
// contains the files a full backup requires. Call before streaming a response.
|
||||||
// excluded entries. Non-regular, non-directory entries (symlinks, sockets) are
|
func ValidateDeployRoot(deployRoot string) error {
|
||||||
// skipped: /data holds none in normal operation, and copying them into a backup
|
if deployRoot == "" {
|
||||||
// would be meaningless or unsafe.
|
return fmt.Errorf("backup: DeployRoot is required (mount the project directory at SELFPOST_DEPLOY_ROOT)")
|
||||||
func addTree(tw *tar.Writer, dataDir string) error {
|
}
|
||||||
return filepath.WalkDir(dataDir, func(path string, d fs.DirEntry, err error) error {
|
return validateDeployRoot(deployRoot)
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateDeployRoot(deployRoot string) error {
|
||||||
|
for _, name := range []string{ComposeFileName, EnvFileName} {
|
||||||
|
path := filepath.Join(deployRoot, name)
|
||||||
|
if _, err := os.Stat(path); err != nil {
|
||||||
|
return fmt.Errorf("backup: deploy root %q is missing %s: %w", deployRoot, name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func addDeployFiles(tw *tar.Writer, deployRoot string, onWarn func(string)) error {
|
||||||
|
for _, name := range []string{ComposeFileName, EnvFileName} {
|
||||||
|
src := filepath.Join(deployRoot, name)
|
||||||
|
info, err := os.Stat(src)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("backup: stat deploy file %s: %w", name, err)
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("backup: deploy file %s is not a regular file", name)
|
||||||
|
}
|
||||||
|
if err := writeTarFile(tw, name, info.Mode().Perm(), src); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
certsDir := filepath.Join(deployRoot, CertsDirName)
|
||||||
|
if _, err := os.Stat(certsDir); err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
if onWarn != nil {
|
||||||
|
onWarn("certs/ not found in deploy root; backup will not include TLS material")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("backup: stat %s: %w", CertsDirName, err)
|
||||||
|
}
|
||||||
|
return addTree(tw, certsDir, CertsDirName+"/", nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// addTree walks root and adds every regular file (and directory, to preserve
|
||||||
|
// empty ones and modes) to tw under archivePrefix + path relative to root.
|
||||||
|
// When exclude is non-nil, top-level names relative to root are skipped.
|
||||||
|
// Non-regular, non-directory entries (symlinks, sockets) are skipped.
|
||||||
|
func addTree(tw *tar.Writer, root, archivePrefix string, exclude map[string]bool) error {
|
||||||
|
return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
rel, err := filepath.Rel(dataDir, path)
|
rel, err := filepath.Rel(root, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if rel == "." {
|
if rel == "." {
|
||||||
return nil // the data root itself is implicit
|
|
||||||
}
|
|
||||||
// filepath.Rel yields OS separators; tar names use forward slashes.
|
|
||||||
name := filepath.ToSlash(rel)
|
|
||||||
// Exclude by top-level name (the live DB, setup token and stale manifest
|
|
||||||
// all live at the data root).
|
|
||||||
if excludedFromArchive[name] {
|
|
||||||
if d.IsDir() {
|
|
||||||
return fs.SkipDir
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
name := archivePrefix + filepath.ToSlash(rel)
|
||||||
|
if exclude != nil {
|
||||||
|
top := strings.Split(filepath.ToSlash(rel), "/")[0]
|
||||||
|
if exclude[top] {
|
||||||
|
if d.IsDir() {
|
||||||
|
return fs.SkipDir
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
info, err := d.Info()
|
info, err := d.Info()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -182,7 +252,7 @@ func addTree(tw *tar.Writer, dataDir string) error {
|
|||||||
case info.Mode().IsRegular():
|
case info.Mode().IsRegular():
|
||||||
return writeTarFile(tw, name, info.Mode().Perm(), path)
|
return writeTarFile(tw, name, info.Mode().Perm(), path)
|
||||||
default:
|
default:
|
||||||
return nil // skip symlinks/sockets/devices
|
return nil
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,10 +15,15 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// seedDataDir builds a realistic /data tree: a migrated SQLite database plus the
|
// seedDataDir builds a realistic /data tree: a migrated SQLite database plus the
|
||||||
// DKIM key, SASL and transient files a backup must include or exclude.
|
// DKIM key, SASL and transient files a backup must include or exclude. It also
|
||||||
func seedDataDir(t *testing.T) (dataDir, dbPath string) {
|
// seeds a deploy root beside data/ with compose, .env, and certs/.
|
||||||
|
func seedDataDir(t *testing.T) (dataDir, dbPath, deployRoot string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
dataDir = t.TempDir()
|
deployRoot = t.TempDir()
|
||||||
|
dataDir = filepath.Join(deployRoot, "data")
|
||||||
|
if err := os.MkdirAll(dataDir, 0o750); err != nil {
|
||||||
|
t.Fatalf("mkdir data: %v", err)
|
||||||
|
}
|
||||||
dbPath = filepath.Join(dataDir, "selfpost.db")
|
dbPath = filepath.Join(dataDir, "selfpost.db")
|
||||||
|
|
||||||
st, err := store.Open(dbPath)
|
st, err := store.Open(dbPath)
|
||||||
@@ -35,6 +40,7 @@ func seedDataDir(t *testing.T) (dataDir, dbPath string) {
|
|||||||
writeFile(t, filepath.Join(dataDir, "opendkim", "keys", "example.com", "selfpost.private"), "PRIVATE KEY")
|
writeFile(t, filepath.Join(dataDir, "opendkim", "keys", "example.com", "selfpost.private"), "PRIVATE KEY")
|
||||||
writeFile(t, filepath.Join(dataDir, "sasl", "sasldb2"), "SASLDB")
|
writeFile(t, filepath.Join(dataDir, "sasl", "sasldb2"), "SASLDB")
|
||||||
writeFile(t, filepath.Join(dataDir, "postfix", "sender_login_maps"), "@example.com login")
|
writeFile(t, filepath.Join(dataDir, "postfix", "sender_login_maps"), "@example.com login")
|
||||||
|
writeFile(t, filepath.Join(dataDir, "postfix", "queue", "deferred", "sample"), "queue-file")
|
||||||
// Transient files that must NOT be archived.
|
// Transient files that must NOT be archived.
|
||||||
writeFile(t, filepath.Join(dataDir, "setup-token"), "secret-token")
|
writeFile(t, filepath.Join(dataDir, "setup-token"), "secret-token")
|
||||||
writeFile(t, filepath.Join(dataDir, "selfpost.db-wal"), "wal")
|
writeFile(t, filepath.Join(dataDir, "selfpost.db-wal"), "wal")
|
||||||
@@ -43,7 +49,11 @@ func seedDataDir(t *testing.T) (dataDir, dbPath string) {
|
|||||||
// state, and the bulkiest thing under /data.
|
// state, and the bulkiest thing under /data.
|
||||||
writeFile(t, filepath.Join(dataDir, "log", "mail.log"), "Aug 8 07:26:41 mail postfix/smtp[1]: ABC: to=<a@example.net>, status=sent (ok)")
|
writeFile(t, filepath.Join(dataDir, "log", "mail.log"), "Aug 8 07:26:41 mail postfix/smtp[1]: ABC: to=<a@example.net>, status=sent (ok)")
|
||||||
writeFile(t, filepath.Join(dataDir, "log", "mail.log.1"), "older")
|
writeFile(t, filepath.Join(dataDir, "log", "mail.log.1"), "older")
|
||||||
return dataDir, dbPath
|
writeFile(t, filepath.Join(deployRoot, ComposeFileName), "services:\n selfpost:\n image: test\n")
|
||||||
|
writeFile(t, filepath.Join(deployRoot, EnvFileName), "SELFPOST_HOSTNAME=mail.example.com\n")
|
||||||
|
writeFile(t, filepath.Join(deployRoot, CertsDirName, "fullchain.pem"), "CERT")
|
||||||
|
writeFile(t, filepath.Join(deployRoot, CertsDirName, "privkey.pem"), "KEY")
|
||||||
|
return dataDir, dbPath, deployRoot
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeFile(t *testing.T, path, content string) {
|
func writeFile(t *testing.T, path, content string) {
|
||||||
@@ -87,21 +97,26 @@ func readArchive(t *testing.T, data []byte) map[string]string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
||||||
dataDir, dbPath := seedDataDir(t)
|
dataDir, dbPath, deployRoot := seedDataDir(t)
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
if err := Create(&buf, Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3"}); err != nil {
|
if err := Create(&buf, Params{DataDir: dataDir, DBPath: dbPath, Version: "1.2.3", DeployRoot: deployRoot}); err != nil {
|
||||||
t.Fatalf("Create: %v", err)
|
t.Fatalf("Create: %v", err)
|
||||||
}
|
}
|
||||||
files := readArchive(t, buf.Bytes())
|
files := readArchive(t, buf.Bytes())
|
||||||
|
|
||||||
// Present.
|
// Present.
|
||||||
for _, name := range []string{
|
for _, name := range []string{
|
||||||
ManifestName,
|
DataArchivePrefix + ManifestName,
|
||||||
"selfpost.db",
|
DataArchivePrefix + "selfpost.db",
|
||||||
"opendkim/keys/example.com/selfpost.private",
|
DataArchivePrefix + "opendkim/keys/example.com/selfpost.private",
|
||||||
"sasl/sasldb2",
|
DataArchivePrefix + "sasl/sasldb2",
|
||||||
"postfix/sender_login_maps",
|
DataArchivePrefix + "postfix/sender_login_maps",
|
||||||
|
DataArchivePrefix + "postfix/queue/deferred/sample",
|
||||||
|
ComposeFileName,
|
||||||
|
EnvFileName,
|
||||||
|
CertsDirName + "/fullchain.pem",
|
||||||
|
CertsDirName + "/privkey.pem",
|
||||||
} {
|
} {
|
||||||
if _, ok := files[name]; !ok {
|
if _, ok := files[name]; !ok {
|
||||||
t.Errorf("archive missing %s", name)
|
t.Errorf("archive missing %s", name)
|
||||||
@@ -109,8 +124,11 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
|||||||
}
|
}
|
||||||
// Excluded.
|
// Excluded.
|
||||||
for _, name := range []string{
|
for _, name := range []string{
|
||||||
"setup-token", "selfpost.db-wal", "selfpost.db-shm",
|
DataArchivePrefix + "setup-token",
|
||||||
"log/mail.log", "log/mail.log.1",
|
DataArchivePrefix + "selfpost.db-wal",
|
||||||
|
DataArchivePrefix + "selfpost.db-shm",
|
||||||
|
DataArchivePrefix + "log/mail.log",
|
||||||
|
DataArchivePrefix + "log/mail.log.1",
|
||||||
} {
|
} {
|
||||||
if _, ok := files[name]; ok {
|
if _, ok := files[name]; ok {
|
||||||
t.Errorf("archive should not contain %s", name)
|
t.Errorf("archive should not contain %s", name)
|
||||||
@@ -119,7 +137,7 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
|||||||
|
|
||||||
// Manifest is well-formed and carries the version.
|
// Manifest is well-formed and carries the version.
|
||||||
var m Manifest
|
var m Manifest
|
||||||
if err := json.Unmarshal([]byte(files[ManifestName]), &m); err != nil {
|
if err := json.Unmarshal([]byte(files[DataArchivePrefix+ManifestName]), &m); err != nil {
|
||||||
t.Fatalf("manifest json: %v", err)
|
t.Fatalf("manifest json: %v", err)
|
||||||
}
|
}
|
||||||
if m.Format != FormatFull || m.Version != "1.2.3" {
|
if m.Format != FormatFull || m.Version != "1.2.3" {
|
||||||
@@ -128,7 +146,7 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
|||||||
|
|
||||||
// The archived selfpost.db is a real, openable SQLite snapshot with our data.
|
// The archived selfpost.db is a real, openable SQLite snapshot with our data.
|
||||||
snapPath := filepath.Join(t.TempDir(), "restored.db")
|
snapPath := filepath.Join(t.TempDir(), "restored.db")
|
||||||
if err := os.WriteFile(snapPath, []byte(files["selfpost.db"]), 0o640); err != nil {
|
if err := os.WriteFile(snapPath, []byte(files[DataArchivePrefix+"selfpost.db"]), 0o640); err != nil {
|
||||||
t.Fatalf("write snapshot: %v", err)
|
t.Fatalf("write snapshot: %v", err)
|
||||||
}
|
}
|
||||||
st, err := store.Open(snapPath)
|
st, err := store.Open(snapPath)
|
||||||
@@ -145,6 +163,14 @@ func TestCreateIncludesStateExcludesTransient(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCreateRequiresDeployRoot(t *testing.T) {
|
||||||
|
dataDir, dbPath, _ := seedDataDir(t)
|
||||||
|
var buf bytes.Buffer
|
||||||
|
if err := Create(&buf, Params{DataDir: dataDir, DBPath: dbPath, Version: "1.0.0"}); err == nil {
|
||||||
|
t.Fatal("Create without DeployRoot succeeded")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func writeManifest(t *testing.T, dir, format, version string) string {
|
func writeManifest(t *testing.T, dir, format, version string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
path := filepath.Join(dir, ManifestName)
|
path := filepath.Join(dir, ManifestName)
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ type Config struct {
|
|||||||
MailLogPath string
|
MailLogPath string
|
||||||
DataDir string
|
DataDir string
|
||||||
DBPath string
|
DBPath string
|
||||||
|
DeployRoot string
|
||||||
Version string
|
Version string
|
||||||
TLSCertFile string
|
TLSCertFile string
|
||||||
OpenDKIMSocket string
|
OpenDKIMSocket string
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/mixeme/selfpost/internal/backup"
|
"github.com/mixeme/selfpost/internal/backup"
|
||||||
@@ -70,6 +71,11 @@ func (h *Handlers) HandleBackup(w http.ResponseWriter, r *http.Request) {
|
|||||||
h.renderBackupPageWith(w, r, http.StatusBadRequest, "", pwErr)
|
h.renderBackupPageWith(w, r, http.StatusBadRequest, "", pwErr)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if err := backup.ValidateDeployRoot(h.cfg.DeployRoot); err != nil {
|
||||||
|
logf("panel: full backup: %v", err)
|
||||||
|
h.renderBackupPageWith(w, r, http.StatusBadRequest, "", deployBackupErr(err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
stamp := time.Now().UTC().Format("20060102-150405")
|
stamp := time.Now().UTC().Format("20060102-150405")
|
||||||
filename := fmt.Sprintf("selfpost-backup-%s.tar.gz", stamp)
|
filename := fmt.Sprintf("selfpost-backup-%s.tar.gz", stamp)
|
||||||
@@ -105,9 +111,13 @@ func (h *Handlers) HandleBackup(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := backup.Create(sink, backup.Params{
|
if err := backup.Create(sink, backup.Params{
|
||||||
DataDir: h.cfg.DataDir,
|
DataDir: h.cfg.DataDir,
|
||||||
DBPath: h.cfg.DBPath,
|
DBPath: h.cfg.DBPath,
|
||||||
Version: h.cfg.Version,
|
Version: h.cfg.Version,
|
||||||
|
DeployRoot: h.cfg.DeployRoot,
|
||||||
|
OnWarn: func(msg string) {
|
||||||
|
logf("panel: full backup: %s", msg)
|
||||||
|
},
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
logf("panel: full backup failed: %v", err)
|
logf("panel: full backup failed: %v", err)
|
||||||
return
|
return
|
||||||
@@ -310,6 +320,14 @@ func decryptErrorMessage(err error) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// deployBackupErr phrases a pre-flight backup failure for the operator.
|
||||||
|
func deployBackupErr(err error) string {
|
||||||
|
if strings.Contains(err.Error(), "DeployRoot") || strings.Contains(err.Error(), "deploy root") {
|
||||||
|
return "Full backup needs the project directory mounted read-only at /selfpost-deploy — add <code>.:/selfpost-deploy:ro</code> to docker-compose.yml and recreate the container."
|
||||||
|
}
|
||||||
|
return "Could not create the backup: " + err.Error()
|
||||||
|
}
|
||||||
|
|
||||||
// importErrorMessage maps a domain-import failure (already logged by the caller)
|
// importErrorMessage maps a domain-import failure (already logged by the caller)
|
||||||
// to an HTTP status and a user-facing message. Duplicate domain/login are called
|
// to an HTTP status and a user-facing message. Duplicate domain/login are called
|
||||||
// out specifically; other failures — validation errors describing what is wrong
|
// out specifically; other failures — validation errors describing what is wrong
|
||||||
|
|||||||
@@ -3,13 +3,15 @@
|
|||||||
|
|
||||||
<div class="card">
|
<div class="card">
|
||||||
<h2>Full backup</h2>
|
<h2>Full backup</h2>
|
||||||
<p class="muted">Download a full backup of all persistent state — the database,
|
<p class="muted">Download a self-contained backup of the whole instance —
|
||||||
every domain's DKIM key and the application credentials. Use it to move the
|
<code>data/</code> (database, DKIM keys, application credentials, and the
|
||||||
whole server to a new machine: restore it into a container of the
|
Postfix queue), <code>docker-compose.yml</code>, <code>.env</code>, and
|
||||||
<strong>same SelfPost version</strong>, with the same data mount, before first
|
<code>certs/</code>. Extract it into an empty project directory on a new
|
||||||
start. TLS certificates and the mail queue are not included.</p>
|
machine, adjust hostname or proxy settings if needed, and start a container of
|
||||||
|
the <strong>same SelfPost version</strong> before first boot. The reverse-proxy
|
||||||
|
vhost is not included — set that up separately on the new host.</p>
|
||||||
<p class="muted"><strong>The backup file is a secret</strong> (it contains
|
<p class="muted"><strong>The backup file is a secret</strong> (it contains
|
||||||
private keys and credentials). Store and transfer it securely and delete it
|
private keys, TLS material, and credentials). Store and transfer it securely and delete it
|
||||||
once the restore succeeds. Encrypting it below is the simplest way to do that:
|
once the restore succeeds. Encrypting it below is the simplest way to do that:
|
||||||
the download is then a <code>.spbk</code> file (SelfPost backup) that only
|
the download is then a <code>.spbk</code> file (SelfPost backup) that only
|
||||||
the password opens.</p>
|
the password opens.</p>
|
||||||
|
|||||||
+8
-5
@@ -40,11 +40,13 @@ type Config struct {
|
|||||||
// the log-tailer role follows in cmd/panel.
|
// the log-tailer role follows in cmd/panel.
|
||||||
MailLogPath string
|
MailLogPath string
|
||||||
// DataDir and DBPath locate the persistent state a full backup archives
|
// DataDir and DBPath locate the persistent state a full backup archives
|
||||||
// (architecture.md § Persistence); Version is stamped into the backup
|
// (architecture.md § Persistence); DeployRoot is the operator project
|
||||||
// manifest. They mirror the panel's own configuration.
|
// directory (docker-compose.yml, .env, certs/); Version is stamped into
|
||||||
DataDir string
|
// the backup manifest. They mirror the panel's own configuration.
|
||||||
DBPath string
|
DataDir string
|
||||||
Version string
|
DBPath string
|
||||||
|
DeployRoot string
|
||||||
|
Version string
|
||||||
// TrustedProxyCIDRs are the reverse-proxy addresses allowed to supply
|
// TrustedProxyCIDRs are the reverse-proxy addresses allowed to supply
|
||||||
// X-Forwarded-For (env TRUSTED_PROXY_CIDR). A request whose
|
// X-Forwarded-For (env TRUSTED_PROXY_CIDR). A request whose
|
||||||
// direct peer (RemoteAddr) is not in this list never has its XFF header
|
// direct peer (RemoteAddr) is not in this list never has its XFF header
|
||||||
@@ -106,6 +108,7 @@ func New(st *store.Store, domains *domain.Service, apps *app.Service, cfg Config
|
|||||||
MailLogPath: cfg.MailLogPath,
|
MailLogPath: cfg.MailLogPath,
|
||||||
DataDir: cfg.DataDir,
|
DataDir: cfg.DataDir,
|
||||||
DBPath: cfg.DBPath,
|
DBPath: cfg.DBPath,
|
||||||
|
DeployRoot: cfg.DeployRoot,
|
||||||
Version: cfg.Version,
|
Version: cfg.Version,
|
||||||
TLSCertFile: cfg.TLSCertFile,
|
TLSCertFile: cfg.TLSCertFile,
|
||||||
OpenDKIMSocket: cfg.OpenDKIMSocket,
|
OpenDKIMSocket: cfg.OpenDKIMSocket,
|
||||||
|
|||||||
Reference in New Issue
Block a user