diff --git a/CHANGELOG.md b/CHANGELOG.md index 14b7b3f..340f980 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,41 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); version time. It is now accepted and discarded - no retry, and no bounce, since the envelope sender of such a message is whatever its author wrote. It counts as one parse failure and the reason is in the mail log, on the message's - own delivery line (Server › System log, `dropped: …`). Only a failure of the database is still - retried. + own delivery line (Server › System log, `dropped: …`). Only a failure of + the database is still retried. +- DMARC reports are kept 500 **for each domain** (and 90 days), not 500 for + the whole server - see Security. + +### Security + +Review of the DMARC report path (Fable, 2026-10-11), which started to run for +the first time in 2.0.0. Nothing Critical or High. + +- **A report is bounded before it is stored.** The XML comes from anyone who + knows a report address, and its 8 MiB limit left room for some 160 000 + records or a single name of megabytes - rows in the database and cells of + the report page. A report holds at most 5 000 records (a larger one is + dropped), every text field is cut to 255 bytes, and a record's count is + capped so the sums cannot wrap. +- **One domain's reports cannot push out another's.** The cap of 500 was for + the whole server, so a stranger filing reports under one domain's address + could evict every other domain's history. It is counted per domain now. +- The delivery command runs one at a time, not up to a hundred in parallel. +- The reason written to the mail log for a dropped message is cut to 200 + bytes and anything outside printable ASCII is replaced: parts of it are the + sender's own bytes (a header line, a file name inside an archive). +- `SELFPOST_DATA_DIR` / `SELFPOST_DB_PATH` with whitespace or `$` are refused + at start when DMARC ingest is on, instead of handing the delivery command + another path and so an empty database of its own. + +### Added + +- e2e: the run now ends on port 25 with both optional features on. A DMARC + report is mailed to a domain's hosted address and has to appear under + Outbound › DMARC reports; a report on another domain and a message with no + report are accepted and dropped; mail for an inbound domain reaches its + upstream and other recipients are refused. Until now the suite only read + back the generated configuration of these features. ## [2.0.0] - 2026-10-11 diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 97b5693..88df52f 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -70,6 +70,12 @@ const ( recipient = "rcpt@sink.e2e.test" ) +// The one administrator of the run. +const ( + adminUser = "e2eadmin" + adminPassword = "correct-horse-battery-staple" +) + // scenario carries state forward between the ordered subtests of TestE2E: // each later step needs something an earlier one produced (the domain id, the // DKIM record, the application credentials, an observed client IP). @@ -131,10 +137,10 @@ func TestE2E(t *testing.T) { if err != nil { t.Fatal(err) } - if err := p.setup(token, "e2eadmin", "correct-horse-battery-staple"); err != nil { + if err := p.setup(token, adminUser, adminPassword); err != nil { t.Fatalf("setup: %v", err) } - if err := p.login("e2eadmin", "correct-horse-battery-staple"); err != nil { + if err := p.login(adminUser, adminPassword); err != nil { t.Fatalf("login: %v", err) } sc.panel = p @@ -268,6 +274,20 @@ func TestE2E(t *testing.T) { run("negative_level1_ratelimit", func(t *testing.T) { testLevel1RateLimit(t, sc) }) + + // From here on the stand accepts mail on port 25. + run("enable_port25_features", func(t *testing.T) { + testEnablePort25Features(t, sc) + }) + run("dmarc_report_reaches_the_panel", func(t *testing.T) { + testDMARCReportReachesThePanel(t, sc) + }) + run("dmarc_junk_is_dropped", func(t *testing.T) { + testDMARCJunkIsDropped(t, sc) + }) + run("inbound_relay_delivers_and_refuses", func(t *testing.T) { + testInboundRelay(t, sc) + }) } // trimZoneSuffix strips the "e2e.test" suffix a fully-qualified DKIM record diff --git a/test/e2e/port25_client.go b/test/e2e/port25_client.go new file mode 100644 index 0000000..6727ce9 --- /dev/null +++ b/test/e2e/port25_client.go @@ -0,0 +1,81 @@ +package e2e + +import ( + "fmt" + "net" + "net/smtp" + "time" +) + +// port25Addr is where compose.override.yml publishes port 25: no TLS required, +// no AUTH offered - a stranger's mail server talking to this one. +const port25Addr = "127.0.0.1:20025" + +// deliverPort25 is one transaction on port 25 the way another mail server makes +// it: HELO, MAIL, RCPT, DATA with message as it stands. The result says at +// which stage it was refused, like attemptSend's. +func deliverPort25(from, to, message string) sendResult { + var res sendResult + conn, err := net.DialTimeout("tcp", port25Addr, 10*time.Second) + if err != nil { + res.dialErr = fmt.Errorf("dial %s: %w", port25Addr, err) + return res + } + c, err := smtp.NewClient(conn, selfpostHostname) + if err != nil { + res.dialErr = fmt.Errorf("smtp handshake: %w", err) + _ = conn.Close() + return res + } + defer c.Close() + if err := c.Hello("mx.stranger.e2e.test"); err != nil { + res.dialErr = err + return res + } + if err := c.Mail(from); err != nil { + res.mailErr = err + _ = c.Quit() + return res + } + if err := c.Rcpt(to); err != nil { + res.rcptErr = err + _ = c.Quit() + return res + } + w, err := c.Data() + if err != nil { + res.dataErr = err + _ = c.Quit() + return res + } + if _, err := w.Write([]byte(message)); err != nil { + res.dataErr = err + } else if err := w.Close(); err != nil { + res.dataErr = err + } + _ = c.Quit() + return res +} + +// waitForPort25Ready polls port 25 until Postfix greets on it. A bare connect +// is not enough: Docker's proxy accepts on the published port whether or not +// anything listens behind it. +func waitForPort25Ready(timeout time.Duration) error { + return waitFor("port 25 to greet", timeout, 500*time.Millisecond, func() (bool, error) { + conn, err := net.DialTimeout("tcp", port25Addr, 2*time.Second) + if err != nil { + return false, err + } + defer conn.Close() + _ = conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + buf := make([]byte, 4) + n, err := conn.Read(buf) + if err != nil { + return false, err + } + if string(buf[:n]) != "220 " { + return false, fmt.Errorf("greeting starts with %q", buf[:n]) + } + return true, nil + }) +} diff --git a/test/e2e/port25_features_test.go b/test/e2e/port25_features_test.go new file mode 100644 index 0000000..66ef830 --- /dev/null +++ b/test/e2e/port25_features_test.go @@ -0,0 +1,274 @@ +package e2e + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "fmt" + "net/url" + "regexp" + "strings" + "testing" + "time" +) + +// The stand starts with both port-25 features off, as a new installation does. +// The last part of the run switches them on and uses them from outside, the +// way a stranger's mail server would: until 2.0.1 the suite only read back the +// generated configuration, and DMARC ingest shipped in 1.7.0 with a delivery +// service Postfix could not start - nothing ever sent a report through it. + +// testEnablePort25Features recreates the container with the inbound relay and +// DMARC ingest on, over the same /data. +func testEnablePort25Features(t *testing.T, sc *scenario) { + h.env = append(h.env, "INBOUND_RELAY_ENABLE=true", "DMARC_REPORTS_ENABLE=true") + if err := h.recreateSelfpost(); err != nil { + t.Fatalf("recreate selfpost with the port-25 features on: %v", err) + } + if err := waitForPanelReady(); err != nil { + t.Fatalf("panel after enabling the port-25 features: %v\n%s", err, h.logs("selfpost")) + } + if err := waitForSMTPSReady(60 * time.Second); err != nil { + t.Fatal(err) + } + if err := waitForPort25Ready(60 * time.Second); err != nil { + t.Fatalf("%v\n%s", err, h.logs("selfpost")) + } + if err := checkSupervisorProcesses(h); err != nil { + t.Fatal(err) + } + // Nothing in the container's own output may complain about the generated + // configuration: the broken delivery service of 1.7.0-1.9.5 was announced + // there at every start ("unmatched service_name/type") and went unread. + logs := h.logs("selfpost") + for _, bad := range []string{"unmatched service_name", "postconf: warning", "postconf: fatal"} { + if strings.Contains(logs, bad) { + t.Fatalf("the container log has %q with the port-25 features on:\n%s", bad, logs) + } + } + resp, err := sc.panel.status() + if err != nil { + t.Fatal(err) + } + if resp.Request.URL.Path != "/overview" { + t.Fatalf("the session did not survive the recreate: landed on %s", resp.Request.URL.Path) + } +} + +const reportReporter = "reporter.e2e.test" + +// aggregateXML is a DMARC aggregate report on domain, as a receiver sends it. +func aggregateXML(domain, reportID string) string { + return ` + + + ` + reportReporter + ` + noreply@` + reportReporter + ` + ` + reportID + ` + 17235936001723680000 + + + ` + domain + ` +

none

none100 + rr + + + + 203.0.113.10 + 7 + + none + passpass + + + ` + domain + ` + + +` +} + +// reportMessage is the mail a receiver sends a report in: one gzip attachment, +// base64 in 76-column lines. +func reportMessage(t *testing.T, to, xml string) string { + t.Helper() + var gz bytes.Buffer + zw := gzip.NewWriter(&gz) + if _, err := zw.Write([]byte(xml)); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + encoded := base64.StdEncoding.EncodeToString(gz.Bytes()) + var b strings.Builder + fmt.Fprintf(&b, "From: noreply@%s\r\nTo: %s\r\nSubject: Report domain: %s\r\nDate: %s\r\nMIME-Version: 1.0\r\n", + reportReporter, to, senderDomain, time.Now().UTC().Format(time.RFC1123Z)) + b.WriteString("Content-Type: application/gzip; name=\"report.xml.gz\"\r\nContent-Transfer-Encoding: base64\r\n\r\n") + for len(encoded) > 76 { + b.WriteString(encoded[:76] + "\r\n") + encoded = encoded[76:] + } + b.WriteString(encoded + "\r\n") + return b.String() +} + +// testDMARCReportReachesThePanel gives the sending domain a SelfPost-hosted +// report address and mails a report to it on port 25. It has to show up under +// Outbound > DMARC reports. +func testDMARCReportReachesThePanel(t *testing.T, sc *scenario) { + address := "dmarc-reports+" + senderDomain + "@" + selfpostHostname + resp, body, err := sc.panel.postForm("/outbound/domains/"+sc.domainID+"/settings/reports", url.Values{"dmarc_rua_mode": {"hosted"}}) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != 200 || resp.Request.URL.Path != "/outbound/domains/"+sc.domainID { + t.Fatalf("choosing the hosted report address: status %d, landed on %s:\n%.800s", resp.StatusCode, resp.Request.URL.Path, body) + } + + // An address nobody chose is refused before DATA, hosted form or not. + res := deliverPort25("noreply@"+reportReporter, "dmarc-reports+nobody.e2e.test@"+selfpostHostname, "Subject: x\r\n\r\nx\r\n") + if res.rcptErr == nil { + t.Fatalf("port 25 accepted a report address no domain uses (first error: %v)", res.firstErr()) + } + + res = deliverPort25("noreply@"+reportReporter, address, reportMessage(t, address, aggregateXML(senderDomain, "e2e-report-1"))) + if !res.ok() { + t.Fatalf("port 25 did not accept a report at %s: %v", address, res.firstErr()) + } + if err := waitFor("the report to appear under Outbound > DMARC reports", 30*time.Second, 500*time.Millisecond, func() (bool, error) { + resp, body, err := sc.panel.get("/outbound/dmarc") + if err != nil { + return false, err + } + if resp.StatusCode != 200 { + return false, fmt.Errorf("GET /outbound/dmarc = %d", resp.StatusCode) + } + if strings.Contains(body, reportReporter) { + return true, nil + } + return false, fmt.Errorf("no report from %s on the page yet; mail log:\n%s", reportReporter, mailLogTail(h, "dmarc-ingest")) + }); err != nil { + t.Fatal(err) + } +} + +// testDMARCJunkIsDropped mails two things that are not reports this relay +// keeps to the same address: a report on somebody else's domain, and a message +// with no report in it. Each is accepted, dropped at once with the reason in +// the mail log, and counted; neither stays in the queue to be tried again and +// neither becomes a report. +func testDMARCJunkIsDropped(t *testing.T, sc *scenario) { + address := "dmarc-reports+" + senderDomain + "@" + selfpostHostname + junk := map[string]string{ + "a report on another domain": reportMessage(t, address, aggregateXML("stranger.e2e.test", "e2e-report-2")), + "a message with no report": "From: someone@stranger.e2e.test\r\nTo: " + address + "\r\nSubject: hello\r\n\r\nnot a report\r\n", + } + for what, msg := range junk { + if res := deliverPort25("someone@stranger.e2e.test", address, msg); !res.ok() { + t.Fatalf("%s was refused on port 25 instead of being accepted and dropped: %v", what, res.firstErr()) + } + } + + dropped := regexp.MustCompile(`status=sent \(delivered via dmarc-ingest service \(dropped: [^)]`) + if err := waitFor("both messages to be dropped", 30*time.Second, 500*time.Millisecond, func() (bool, error) { + log := mailLogTail(h, "dmarc-ingest") + if n := len(dropped.FindAllString(log, -1)); n != len(junk) { + return false, fmt.Errorf("%d of %d dropped so far; mail log:\n%s", n, len(junk), log) + } + return true, nil + }); err != nil { + t.Fatal(err) + } + + queue, err := h.execIn("selfpost", "postqueue", "-p") + if err != nil { + t.Fatalf("postqueue -p: %v\n%s", err, queue) + } + if !strings.Contains(queue, "Mail queue is empty") { + t.Fatalf("a dropped message is still in the queue:\n%s", queue) + } + + _, body, err := sc.panel.get("/outbound/dmarc") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(body, "2 parse failures") { + t.Fatalf("the DMARC page does not count the two dropped messages as parse failures:\n%s", body) + } + if strings.Contains(body, "stranger.e2e.test") { + t.Fatal("the report on another domain was stored") + } +} + +// mailLogTail returns the lines of the end of the mail log that contain filter. +func mailLogTail(s *stack, filter string) string { + out, err := s.execIn("selfpost", "tail", "-n", "400", "/data/log/mail.log") + if err != nil { + return "(mail log unreadable: " + err.Error() + ")" + } + var b strings.Builder + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, filter) { + b.WriteString(line + "\n") + } + } + return b.String() +} + +const ( + inboundDomain = "in.e2e.test" + inboundRecipient = "user@" + inboundDomain +) + +// testInboundRelay sets up an inbound domain with the sink as its upstream and +// one listed recipient, then talks to port 25 as a stranger: mail for the +// listed recipient reaches the upstream, mail for anyone else at the domain +// and for a domain that is not configured is refused before DATA. +func testInboundRelay(t *testing.T, sc *scenario) { + resp, body, err := sc.panel.postForm("/inbound/domains", url.Values{"name": {inboundDomain}}) + if err != nil { + t.Fatal(err) + } + m := regexp.MustCompile(`^/inbound/domains/(\d+)$`).FindStringSubmatch(resp.Request.URL.Path) + if m == nil { + t.Fatalf("add inbound domain: landed on %s:\n%.800s", resp.Request.URL.Path, body) + } + base := "/inbound/domains/" + m[1] + + // The sink by address: an upstream named by an IP is delivered to as it + // stands, with no MX lookup in between. + resp, body, err = sc.panel.postForm(base+"/upstream", url.Values{"host": {"10.77.0.11"}, "port": {"25"}, "tls_mode": {"none"}}) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != 200 || resp.Request.URL.Path != base { + t.Fatalf("set the upstream: status %d, landed on %s:\n%.800s", resp.StatusCode, resp.Request.URL.Path, body) + } + resp, body, err = sc.panel.postForm(base+"/recipients", url.Values{"recipient_mode": {"list"}, "addresses": {inboundRecipient}}) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != 200 || resp.Request.URL.Path != base { + t.Fatalf("set the recipients: status %d, landed on %s:\n%.800s", resp.StatusCode, resp.Request.URL.Path, body) + } + + message := func(to, token string) string { + return fmt.Sprintf("From: someone@stranger.e2e.test\r\nTo: %s\r\nSubject: e2e %s\r\nDate: %s\r\n\r\nmail for the inbound domain\r\n", + to, token, time.Now().UTC().Format(time.RFC1123Z)) + } + + token := uniqueToken("inbound") + if res := deliverPort25("someone@stranger.e2e.test", inboundRecipient, message(inboundRecipient, token)); !res.ok() { + t.Fatalf("port 25 did not accept mail for %s: %v", inboundRecipient, res.firstErr()) + } + if _, err := findSinkMessage(h.stageDir, token, 30*time.Second); err != nil { + t.Fatalf("mail for the inbound domain never reached its upstream: %v\n%s", err, mailLogTail(h, "postfix/")) + } + + for _, to := range []string{"nobody@" + inboundDomain, "someone@elsewhere.e2e.test"} { + res := deliverPort25("someone@stranger.e2e.test", to, message(to, uniqueToken("refused"))) + if res.rcptErr == nil { + t.Fatalf("port 25 accepted the recipient %s (first error: %v)", to, res.firstErr()) + } + } +} diff --git a/test/e2e/stack.go b/test/e2e/stack.go index 879af14..1bc98e8 100644 --- a/test/e2e/stack.go +++ b/test/e2e/stack.go @@ -24,6 +24,10 @@ type stack struct { baseComposeFile string overrideFile string stageDir string + // env is added to compose's own environment. The shipped compose file takes + // the optional features from there (${INBOUND_RELAY_ENABLE:-false} and the + // like), so this is how a later part of the run switches them on. + env []string } func newStack() (*stack, error) { @@ -64,6 +68,7 @@ func (s *stack) compose(args ...string) ([]byte, error) { // looked at (that mapping only reaches the container, not compose's own // variable interpolation), so it has to be set here too. cmd.Env = append(os.Environ(), "SELFPOST_HOSTNAME="+selfpostHostname) + cmd.Env = append(cmd.Env, s.env...) var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &out @@ -102,6 +107,14 @@ func (s *stack) restartSelfpost() error { return err } +// recreateSelfpost replaces the selfpost container with one started from the +// same /data and the environment as it is now (see env), leaving the DNS and +// sink sidecars and their state alone. +func (s *stack) recreateSelfpost() error { + _, err := s.compose("up", "-d", "--force-recreate", "--no-deps", "selfpost") + return err +} + func (s *stack) down() { _, _ = s.compose("down", "-v", "--remove-orphans") } diff --git a/test/e2e/stage.go b/test/e2e/stage.go index 3f51391..b03c265 100644 --- a/test/e2e/stage.go +++ b/test/e2e/stage.go @@ -39,6 +39,9 @@ func prepareStage(s *stack) error { ); err != nil { return err } + if err := writeDeployFiles(s); err != nil { + return err + } corefile, err := os.ReadFile(filepath.Join("dns", "Corefile")) if err != nil { return fmt.Errorf("read Corefile: %w", err) @@ -49,6 +52,22 @@ func prepareStage(s *stack) error { return writeZone(s.stageDir, nil) } +// writeDeployFiles puts docker-compose.yml and .env into the stage directory. +// The stage directory is what the container sees as /selfpost-deploy, the +// operator's project directory, and a full backup refuses to run unless both +// files are there to be archived. compose reads neither: the stand names its +// files with -f, and the .env repeats what compose is given anyway. +func writeDeployFiles(s *stack) error { + compose, err := os.ReadFile(s.baseComposeFile) + if err != nil { + return fmt.Errorf("read the shipped compose file: %w", err) + } + if err := os.WriteFile(filepath.Join(s.stageDir, "docker-compose.yml"), compose, 0o644); err != nil { + return err + } + return os.WriteFile(filepath.Join(s.stageDir, ".env"), []byte("SELFPOST_HOSTNAME="+selfpostHostname+"\n"), 0o644) +} + // removeAllBestEffort deletes path; if a previous run left container-UID files // on the bind mount, a root alpine one-shot removes them first. func removeAllBestEffort(path string) error {