dmarc: a report is bounded, kept per domain, and ingested one at a time
Security review of the DMARC report path (Fable, 2026-10-11), which ran for the first time in 2.0.0. No Critical or High finding. Fixed: - Medium: a report was bounded only as 8 MiB of XML - some 160 000 records or one name of megabytes, all of it rows and table cells. At most 5 000 records (more is ErrRejected, dropped), text fields cut to 255 bytes on a character boundary, a record's count capped. - Medium: the cap of 500 reports was for the whole server, so a stranger filing under one domain's address evicted every other domain's reports. PruneDMARCReports counts per domain. - Low: the pipe service ran up to default_process_limit (100) commands at once against the panel's database; maxproc is 1. - Low: the reason logged for a dropped message carried the sender's bytes (header line, zip entry name) as they came; cut to 200 bytes and reduced to printable ASCII, zip entry names quoted. - Low: SELFPOST_DATA_DIR / SELFPOST_DB_PATH with whitespace or "$" would reach the command through export_environment as another path; refused at start when DMARC ingest is on. Left as they are, told to the owner: the parse-failure count is a lifetime figure shown under "This week"; a report larger than the command reads is dropped as unreadable; nested multipart is not searched. The hub's Retention line follows the design commit before this one. security.md: the review, and the accepted-risk entry says what a stranger can and cannot do now. The roadmap no longer calls the review pending. go vet ./cmd/... ./internal/dmarc/ ./internal/store/: ok go test ./cmd/... ./internal/dmarc/ ./internal/store/ ./internal/web/...: ok make e2e on the test server (with the port-25 scenarios of the next commit): ok Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+13
-1
@@ -291,11 +291,23 @@ if [ "${INBOUND_RELAY_ENABLE}" = "true" ] || [ "${DMARC_REPORTS_ENABLE}" = "true
|
|||||||
# attribute" on every report. They go into the service line itself.
|
# attribute" on every report. They go into the service line itself.
|
||||||
# ${recipient} is expanded by pipe(8), not by the shell: the command
|
# ${recipient} is expanded by pipe(8), not by the shell: the command
|
||||||
# learns which report address the message came to from its argument.
|
# learns which report address the message came to from its argument.
|
||||||
postconf -M 'dmarc-ingest/unix=dmarc-ingest unix - n n - - pipe flags=Rq user=panel argv=/usr/local/bin/dmarc-ingest ${recipient}'
|
# One at a time (the 1 before "pipe"): reports are in no hurry, every run
|
||||||
|
# writes to the database the panel writes to, and a hundred of them in
|
||||||
|
# parallel - Postfix's default - is what a flood of reports would start.
|
||||||
|
postconf -M 'dmarc-ingest/unix=dmarc-ingest unix - n n - 1 pipe flags=Rq user=panel argv=/usr/local/bin/dmarc-ingest ${recipient}'
|
||||||
# One recipient per run, so ${recipient} is one argument.
|
# One recipient per run, so ${recipient} is one argument.
|
||||||
postconf -e "dmarc-ingest_destination_recipient_limit=1"
|
postconf -e "dmarc-ingest_destination_recipient_limit=1"
|
||||||
# pipe(8) starts the command with export_environment only, so what the
|
# pipe(8) starts the command with export_environment only, so what the
|
||||||
# command reads from the container's environment is named there.
|
# command reads from the container's environment is named there.
|
||||||
|
# export_environment is a list split at whitespace in which Postfix
|
||||||
|
# expands "$": a path with either would reach the command as another
|
||||||
|
# path, and it would quietly open a database of its own.
|
||||||
|
case "${SELFPOST_DATA_DIR:-}${SELFPOST_DB_PATH:-}" in
|
||||||
|
*[[:space:]]* | *'$'*)
|
||||||
|
echo "postfix-config: SELFPOST_DATA_DIR and SELFPOST_DB_PATH must not contain whitespace or \$ when DMARC_REPORTS_ENABLE=true" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
DMARC_INGEST_ENV="DMARC_REPORTS_ENABLE=true"
|
DMARC_INGEST_ENV="DMARC_REPORTS_ENABLE=true"
|
||||||
[ -z "${SELFPOST_DATA_DIR:-}" ] || DMARC_INGEST_ENV="${DMARC_INGEST_ENV} SELFPOST_DATA_DIR=${SELFPOST_DATA_DIR}"
|
[ -z "${SELFPOST_DATA_DIR:-}" ] || DMARC_INGEST_ENV="${DMARC_INGEST_ENV} SELFPOST_DATA_DIR=${SELFPOST_DATA_DIR}"
|
||||||
[ -z "${SELFPOST_DB_PATH:-}" ] || DMARC_INGEST_ENV="${DMARC_INGEST_ENV} SELFPOST_DB_PATH=${SELFPOST_DB_PATH}"
|
[ -z "${SELFPOST_DB_PATH:-}" ] || DMARC_INGEST_ENV="${DMARC_INGEST_ENV} SELFPOST_DB_PATH=${SELFPOST_DB_PATH}"
|
||||||
|
|||||||
@@ -50,8 +50,24 @@ func runDMARCIngestMode() error {
|
|||||||
// all of which that line already has. postlog is no use here: pipe(8) starts
|
// all of which that line already has. postlog is no use here: pipe(8) starts
|
||||||
// the command without the group that may write to Postfix's log socket.
|
// the command without the group that may write to Postfix's log socket.
|
||||||
func logRejected(reason error) {
|
func logRejected(reason error) {
|
||||||
text := strings.TrimPrefix(reason.Error(), dmarc.ErrRejected.Error()+": ")
|
fmt.Fprintln(os.Stderr, "dropped: "+logSafe(strings.TrimPrefix(reason.Error(), dmarc.ErrRejected.Error()+": ")))
|
||||||
fmt.Fprintln(os.Stderr, "dropped: "+text)
|
}
|
||||||
|
|
||||||
|
// logSafe makes a reason fit for a log line. Parts of it are the sender's own
|
||||||
|
// bytes - a header line, a file name inside an archive, a piece of XML - so it
|
||||||
|
// is cut short and anything that is not printable ASCII becomes "?", whatever
|
||||||
|
// Postfix would have done with it.
|
||||||
|
func logSafe(s string) string {
|
||||||
|
const max = 200
|
||||||
|
if len(s) > max {
|
||||||
|
s = s[:max]
|
||||||
|
}
|
||||||
|
return strings.Map(func(r rune) rune {
|
||||||
|
if r < 0x20 || r > 0x7e {
|
||||||
|
return '?'
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}, s)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ingestRecipient is the report address the message was delivered to. Postfix's
|
// ingestRecipient is the report address the message was delivered to. Postfix's
|
||||||
|
|||||||
@@ -18,3 +18,17 @@ func TestIngestRecipient(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLogSafe(t *testing.T) {
|
||||||
|
got := logSafe("zip entry \"a\nstatus=bounced\tx\" \x1b[31m é")
|
||||||
|
if want := "zip entry \"a?status=bounced?x\" ?[31m ?"; got != want {
|
||||||
|
t.Errorf("logSafe = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
long := make([]byte, 1000)
|
||||||
|
for i := range long {
|
||||||
|
long[i] = 'a'
|
||||||
|
}
|
||||||
|
if got := logSafe(string(long)); len(got) != 200 {
|
||||||
|
t.Errorf("logSafe kept %d bytes of 1000, want 200", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+1
-1
@@ -891,7 +891,7 @@ sees the *Ingest* box — whether ingest works, the last report received, report
|
|||||||
kept this week and parse failures. Open a domain's roll-up from the list or
|
kept this week and parse failures. Open a domain's roll-up from the list or
|
||||||
from *DMARC reports* in the domain page's menu. A user with the *domain* role
|
from *DMARC reports* in the domain page's menu. A user with the *domain* role
|
||||||
sees the reports of the domains assigned to them, without the *Ingest* box.
|
sees the reports of the domains assigned to them, without the *Ingest* box.
|
||||||
Summaries are pruned (500 kept, 90 days max).
|
Summaries are pruned (500 kept for each domain, 90 days max).
|
||||||
|
|
||||||
A message that is not a report the relay keeps — unreadable as an aggregate
|
A message that is not a report the relay keeps — unreadable as an aggregate
|
||||||
report, about a domain not sent from here, or delivered to another domain's
|
report, about a domain not sent from here, or delivered to another domain's
|
||||||
|
|||||||
+2
-1
@@ -47,7 +47,8 @@ development convenience): **panel-redesign** shipped as `2.0.0` (2026-10-11); th
|
|||||||
**inbound-antispam-panel** (`2.1.0`), built in the new component kit —
|
**inbound-antispam-panel** (`2.1.0`), built in the new component kit —
|
||||||
styling its two screens once instead of twice. panel-docs shipped in
|
styling its two screens once instead of twice. panel-docs shipped in
|
||||||
[CHANGELOG.md](../CHANGELOG.md) `[1.8.0]`; dmarc-reports in `[1.7.0]`
|
[CHANGELOG.md](../CHANGELOG.md) `[1.8.0]`; dmarc-reports in `[1.7.0]`
|
||||||
(security review of the ingest path pending); domain-stats-auto-ratelimit in
|
(its ingest path first ran in `2.0.0` and was reviewed on 2026-10-11, see
|
||||||
|
[security.md](security.md)); domain-stats-auto-ratelimit in
|
||||||
`[1.6.0]`; send-log-retention in `[1.5.0]`; inbound-relay in `[1.4.0]`;
|
`[1.6.0]`; send-log-retention in `[1.5.0]`; inbound-relay in `[1.4.0]`;
|
||||||
queue-retries in `[1.3.1]`; the 2026-08-13 full-tree review follow-ups are in
|
queue-retries in `[1.3.1]`; the 2026-08-13 full-tree review follow-ups are in
|
||||||
`[1.3.0]`. Candidates need explicit agreement before they join the queue.
|
`[1.3.0]`. Candidates need explicit agreement before they join the queue.
|
||||||
|
|||||||
+17
-3
@@ -26,7 +26,14 @@ the user form did not end that user's sessions, internal errors reached forms a
|
|||||||
(see [Authentication and sessions](#authentication-and-sessions) and [Errors
|
(see [Authentication and sessions](#authentication-and-sessions) and [Errors
|
||||||
shown to users](#errors-shown-to-users)); two deliberate behaviours are
|
shown to users](#errors-shown-to-users)); two deliberate behaviours are
|
||||||
recorded under Accepted risks (upstream of a delegated inbound domain, profile
|
recorded under Accepted risks (upstream of a delegated inbound domain, profile
|
||||||
saved without the password).
|
saved without the password). The 2026-10-11 review (Fable) covered the DMARC
|
||||||
|
report path from port 25 to the report pages, which ran for the first time in
|
||||||
|
2.0.0 - the 2026-08-19 pass read code that Postfix could not yet start: no
|
||||||
|
Critical or High finding; a report's size was bounded only as bytes (now 5 000
|
||||||
|
records, 255 bytes a field), the retention cap was one for the whole server
|
||||||
|
(now per domain), the delivery command could run a hundred at once (now one),
|
||||||
|
and the reason logged for a dropped message carried the sender's bytes as they
|
||||||
|
came (now cut and reduced to printable ASCII).
|
||||||
(2) **Accepted risks** —
|
(2) **Accepted risks** —
|
||||||
deliberate departures beyond the mandatory, recorded so the decision is not
|
deliberate departures beyond the mandatory, recorded so the decision is not
|
||||||
lost.
|
lost.
|
||||||
@@ -274,8 +281,15 @@ deferred item from the roadmap.
|
|||||||
reporters are arbitrary third parties and the mailbox address is published in
|
reporters are arbitrary third parties and the mailbox address is published in
|
||||||
DNS. The ingest already constrains the meaningful part — a report cannot be
|
DNS. The ingest already constrains the meaningful part — a report cannot be
|
||||||
filed under a domain this instance does not host, and the claimed domain is
|
filed under a domain this instance does not host, and the claimed domain is
|
||||||
cross-checked against the `+tag` of the delivery address — and the data is
|
cross-checked against the `+tag` of the delivery address (effective since
|
||||||
advisory statistics only; nothing enforces off it. The trigger to revisit is
|
2.0.0: before it the recipient never reached the command) — and the data is
|
||||||
|
advisory statistics only; nothing enforces off it. What a stranger can do is
|
||||||
|
limited to a domain that chose a SelfPost-hosted address, through that
|
||||||
|
address: file reports under it, replace one whose reporter and id they know,
|
||||||
|
and fill that domain's 500 — not another domain's. A message that is not a
|
||||||
|
report the relay keeps is dropped, not bounced (the envelope sender is the
|
||||||
|
author's to write); a genuine report the parser cannot read is lost the same
|
||||||
|
way, with a count and a line in the mail log. The trigger to revisit is
|
||||||
any feature that lets report contents drive an automatic action (policy
|
any feature that lets report contents drive an automatic action (policy
|
||||||
changes, rate limiting, alerting an operator into a config change).
|
changes, rate limiting, alerting an operator into a config change).
|
||||||
- **The application auth-IP allow-list is enforced fail-open.** The check runs
|
- **The application auth-IP allow-list is enforced fail-open.** The check runs
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
package dmarc
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/mixeme/selfpost/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func reportXML(orgName, domain, records string) string {
|
||||||
|
return `<?xml version="1.0"?><feedback><report_metadata><org_name>` + orgName +
|
||||||
|
`</org_name><email>a@example.net</email><report_id>1</report_id>` +
|
||||||
|
`<date_range><begin>1723593600</begin><end>1723680000</end></date_range></report_metadata>` +
|
||||||
|
`<policy_published><domain>` + domain + `</domain><p>none</p></policy_published>` + records + `</feedback>`
|
||||||
|
}
|
||||||
|
|
||||||
|
const oneRecord = `<record><row><source_ip>203.0.113.10</source_ip><count>3</count>` +
|
||||||
|
`<policy_evaluated><disposition>none</disposition><dkim>pass</dkim><spf>pass</spf></policy_evaluated></row>` +
|
||||||
|
`<identifiers><header_from>example.com</header_from></identifiers></record>`
|
||||||
|
|
||||||
|
// The XML is a stranger's: what it may make the relay store is bounded.
|
||||||
|
func TestParseAggregateBoundsWhatAReportHolds(t *testing.T) {
|
||||||
|
long := strings.Repeat("é", 4000)
|
||||||
|
got, err := ParseAggregate([]byte(reportXML(long, "example.com", oneRecord)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got.Reporter) > maxFieldLen || !strings.HasPrefix(long, got.Reporter) || got.Reporter == "" {
|
||||||
|
t.Fatalf("reporter name is %d bytes, want the start of the name in at most %d", len(got.Reporter), maxFieldLen)
|
||||||
|
}
|
||||||
|
if strings.ContainsRune(got.Reporter, '�') {
|
||||||
|
t.Fatal("the name was cut in the middle of a character")
|
||||||
|
}
|
||||||
|
|
||||||
|
huge := `<record><row><source_ip>203.0.113.10</source_ip><count>9223372036854775807</count>` +
|
||||||
|
`<policy_evaluated><dkim>pass</dkim></policy_evaluated></row></record>`
|
||||||
|
got, err = ParseAggregate([]byte(reportXML("r", "example.com", huge+huge)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.PassCount != 2*maxRecordCount {
|
||||||
|
t.Fatalf("pass count = %d, want two records clamped to %d each", got.PassCount, maxRecordCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := ParseAggregate([]byte(reportXML("r", "example.com", strings.Repeat(oneRecord, maxReportRecords)))); err != nil {
|
||||||
|
t.Fatalf("a report of %d records was refused: %v", maxReportRecords, err)
|
||||||
|
}
|
||||||
|
if _, err := ParseAggregate([]byte(reportXML("r", "example.com", strings.Repeat(oneRecord, maxReportRecords+1)))); err == nil {
|
||||||
|
t.Fatalf("a report of %d records was accepted", maxReportRecords+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A report over the bound is one the relay does not keep: dropped, not retried.
|
||||||
|
func TestIngestMessageRejectsAReportWithTooManyRecords(t *testing.T) {
|
||||||
|
st, err := store.Open(t.TempDir() + "/test.db")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer st.Close()
|
||||||
|
if _, err := st.AddDomain("example.com", "selfpost"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
xml := reportXML("r", "example.com", strings.Repeat(oneRecord, maxReportRecords+1))
|
||||||
|
err = IngestMessage(st, strings.NewReader(gzipMessage(t, xml)), "dmarc-reports+example.com@mail.example.com", time.Now().UTC())
|
||||||
|
if !errors.Is(err, ErrRejected) {
|
||||||
|
t.Fatalf("err = %v, want ErrRejected", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reports filed under one domain do not push another domain's reports out.
|
||||||
|
func TestIngestKeepsEachDomainsReportsApart(t *testing.T) {
|
||||||
|
st, err := store.Open(t.TempDir() + "/test.db")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer st.Close()
|
||||||
|
now := time.Now().UTC()
|
||||||
|
if _, err := st.InsertDMARCReport(store.DMARCReport{
|
||||||
|
Domain: "quiet.example", Reporter: "genuine", ReportID: "1", PeriodBegin: now, PeriodEnd: now, ReceivedAt: now.Add(-time.Hour),
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for i := 0; i < store.DMARCReportsMaxKeep+20; i++ {
|
||||||
|
if _, err := st.InsertDMARCReport(store.DMARCReport{
|
||||||
|
Domain: "flooded.example", Reporter: "stranger", ReportID: strings.Repeat("x", 1+i), PeriodBegin: now, PeriodEnd: now, ReceivedAt: now,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := st.PruneDMARCReports("flooded.example"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
quiet, err := st.ListDMARCReports(store.DMARCReportScope{Domains: []string{"quiet.example"}}, 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(quiet) != 1 {
|
||||||
|
t.Fatalf("the other domain has %d reports after the flood, want its 1", len(quiet))
|
||||||
|
}
|
||||||
|
flooded, err := st.ListDMARCReports(store.DMARCReportScope{Domains: []string{"flooded.example"}}, 10000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(flooded) > store.DMARCReportsMaxKeep {
|
||||||
|
t.Fatalf("the flooded domain keeps %d reports, want at most %d", len(flooded), store.DMARCReportsMaxKeep)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -84,7 +84,7 @@ func IngestMessage(st *store.Store, r io.Reader, recipient string, receivedAt ti
|
|||||||
if _, err := st.InsertDMARCReport(rep); err != nil {
|
if _, err := st.InsertDMARCReport(rep); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return st.PruneDMARCReports()
|
return st.PruneDMARCReports(rep.Domain)
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkReportDomain refuses a report whose policy domain is not one of this
|
// checkReportDomain refuses a report whose policy domain is not one of this
|
||||||
|
|||||||
+50
-15
@@ -9,6 +9,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ParsedReport is the structured form of a DMARC aggregate XML payload.
|
// ParsedReport is the structured form of a DMARC aggregate XML payload.
|
||||||
@@ -74,6 +75,31 @@ type feedbackXML struct {
|
|||||||
} `xml:"record"`
|
} `xml:"record"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// What a report may hold. The XML comes from anyone who knows a report address,
|
||||||
|
// and eight megabytes of it (maxAggregateXML) fit some 160 000 empty records or
|
||||||
|
// one name of eight megabytes - every one of them a row or a table cell later.
|
||||||
|
// A real report has tens of records, a few thousand from the largest
|
||||||
|
// receivers, and no field longer than a line.
|
||||||
|
const (
|
||||||
|
maxReportRecords = 5000
|
||||||
|
maxFieldLen = 255
|
||||||
|
maxRecordCount = 1_000_000_000
|
||||||
|
)
|
||||||
|
|
||||||
|
// field trims a text value of the report and cuts it to maxFieldLen bytes, on
|
||||||
|
// a character boundary.
|
||||||
|
func field(s string) string {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
if len(s) <= maxFieldLen {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
s = s[:maxFieldLen]
|
||||||
|
for len(s) > 0 && !utf8.ValidString(s) {
|
||||||
|
s = s[:len(s)-1]
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
// ParseAggregate decodes gzip- or zip-compressed, or raw, DMARC aggregate XML.
|
// ParseAggregate decodes gzip- or zip-compressed, or raw, DMARC aggregate XML.
|
||||||
func ParseAggregate(raw []byte) (ParsedReport, error) {
|
func ParseAggregate(raw []byte) (ParsedReport, error) {
|
||||||
data, err := decompress(raw)
|
data, err := decompress(raw)
|
||||||
@@ -87,34 +113,43 @@ func ParseAggregate(raw []byte) (ParsedReport, error) {
|
|||||||
if doc.Metadata.ReportID == "" || doc.Policy.Domain == "" {
|
if doc.Metadata.ReportID == "" || doc.Policy.Domain == "" {
|
||||||
return ParsedReport{}, fmt.Errorf("dmarc xml: missing report_id or domain")
|
return ParsedReport{}, fmt.Errorf("dmarc xml: missing report_id or domain")
|
||||||
}
|
}
|
||||||
|
if len(doc.Records) > maxReportRecords {
|
||||||
|
return ParsedReport{}, fmt.Errorf("dmarc xml: %d records, more than the %d a report may hold", len(doc.Records), maxReportRecords)
|
||||||
|
}
|
||||||
|
// Cut, not refused: a domain name is at most 253 bytes, so a longer one
|
||||||
|
// names nothing this relay sends for and is turned away as such.
|
||||||
out := ParsedReport{
|
out := ParsedReport{
|
||||||
Reporter: strings.TrimSpace(doc.Metadata.OrgName),
|
Reporter: field(doc.Metadata.OrgName),
|
||||||
ReportID: strings.TrimSpace(doc.Metadata.ReportID),
|
ReportID: field(doc.Metadata.ReportID),
|
||||||
ContactEmail: strings.TrimSpace(doc.Metadata.Email),
|
ContactEmail: field(doc.Metadata.Email),
|
||||||
PeriodBegin: time.Unix(doc.Metadata.DateRange.Begin, 0).UTC(),
|
PeriodBegin: time.Unix(doc.Metadata.DateRange.Begin, 0).UTC(),
|
||||||
PeriodEnd: time.Unix(doc.Metadata.DateRange.End, 0).UTC(),
|
PeriodEnd: time.Unix(doc.Metadata.DateRange.End, 0).UTC(),
|
||||||
Domain: strings.ToLower(strings.TrimSpace(doc.Policy.Domain)),
|
Domain: strings.ToLower(field(doc.Policy.Domain)),
|
||||||
PolicyP: strings.TrimSpace(doc.Policy.P),
|
PolicyP: field(doc.Policy.P),
|
||||||
PolicySP: strings.TrimSpace(doc.Policy.SP),
|
PolicySP: field(doc.Policy.SP),
|
||||||
PolicyPct: doc.Policy.Pct,
|
PolicyPct: doc.Policy.Pct,
|
||||||
PolicyADKIM: strings.TrimSpace(doc.Policy.ADKIM),
|
PolicyADKIM: field(doc.Policy.ADKIM),
|
||||||
PolicyASPF: strings.TrimSpace(doc.Policy.ASPF),
|
PolicyASPF: field(doc.Policy.ASPF),
|
||||||
}
|
}
|
||||||
if out.PolicyPct == 0 {
|
if out.PolicyPct == 0 {
|
||||||
out.PolicyPct = 100
|
out.PolicyPct = 100
|
||||||
}
|
}
|
||||||
for _, rec := range doc.Records {
|
for _, rec := range doc.Records {
|
||||||
row := ParsedRecord{
|
row := ParsedRecord{
|
||||||
SourceIP: strings.TrimSpace(rec.Row.SourceIP),
|
SourceIP: field(rec.Row.SourceIP),
|
||||||
Count: rec.Row.Count,
|
Count: rec.Row.Count,
|
||||||
Disposition: strings.TrimSpace(rec.Row.Policy.Disposition),
|
Disposition: field(rec.Row.Policy.Disposition),
|
||||||
SPFResult: strings.TrimSpace(rec.Row.Policy.SPF),
|
SPFResult: field(rec.Row.Policy.SPF),
|
||||||
DKIMResult: strings.TrimSpace(rec.Row.Policy.DKIM),
|
DKIMResult: field(rec.Row.Policy.DKIM),
|
||||||
HeaderFrom: strings.TrimSpace(rec.Identifiers.HeaderFrom),
|
HeaderFrom: field(rec.Identifiers.HeaderFrom),
|
||||||
}
|
}
|
||||||
if row.Count <= 0 {
|
if row.Count <= 0 {
|
||||||
row.Count = 1
|
row.Count = 1
|
||||||
}
|
}
|
||||||
|
// Bounded so that the sums of a report cannot wrap around.
|
||||||
|
if row.Count > maxRecordCount {
|
||||||
|
row.Count = maxRecordCount
|
||||||
|
}
|
||||||
if dmarcRecordPasses(row) {
|
if dmarcRecordPasses(row) {
|
||||||
out.PassCount += row.Count
|
out.PassCount += row.Count
|
||||||
} else {
|
} else {
|
||||||
@@ -172,12 +207,12 @@ func unzip(raw []byte) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
rc, err := f.Open()
|
rc, err := f.Open()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("dmarc zip open %s: %w", f.Name, err)
|
return nil, fmt.Errorf("dmarc zip open %q: %w", field(f.Name), err)
|
||||||
}
|
}
|
||||||
data, err := io.ReadAll(io.LimitReader(rc, maxAggregateXML))
|
data, err := io.ReadAll(io.LimitReader(rc, maxAggregateXML))
|
||||||
rc.Close()
|
rc.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("dmarc zip read %s: %w", f.Name, err)
|
return nil, fmt.Errorf("dmarc zip read %q: %w", field(f.Name), err)
|
||||||
}
|
}
|
||||||
return data, nil
|
return data, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -131,14 +131,18 @@ func (s *Store) InsertDMARCReport(rep DMARCReport) (int64, error) {
|
|||||||
return id, nil
|
return id, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PruneDMARCReports enforces count and age caps.
|
// PruneDMARCReports enforces the age cap for every domain and the count cap
|
||||||
func (s *Store) PruneDMARCReports() error {
|
// for domain. The count is kept per domain on purpose: reports are accepted
|
||||||
|
// from anyone who knows a domain's report address, and with one cap for the
|
||||||
|
// whole instance a stranger filing reports under one domain would push every
|
||||||
|
// other domain's reports out.
|
||||||
|
func (s *Store) PruneDMARCReports(domain string) error {
|
||||||
cutoff := time.Now().UTC().AddDate(0, 0, -DMARCReportsMaxAgeDays).Format(time.RFC3339)
|
cutoff := time.Now().UTC().AddDate(0, 0, -DMARCReportsMaxAgeDays).Format(time.RFC3339)
|
||||||
if _, err := s.db.Exec(`DELETE FROM dmarc_reports WHERE received_at < ?`, cutoff); err != nil {
|
if _, err := s.db.Exec(`DELETE FROM dmarc_reports WHERE received_at < ?`, cutoff); err != nil {
|
||||||
return fmt.Errorf("prune dmarc by age: %w", err)
|
return fmt.Errorf("prune dmarc by age: %w", err)
|
||||||
}
|
}
|
||||||
var count int
|
var count int
|
||||||
if err := s.db.QueryRow(`SELECT COUNT(*) FROM dmarc_reports`).Scan(&count); err != nil {
|
if err := s.db.QueryRow(`SELECT COUNT(*) FROM dmarc_reports WHERE domain = ?`, domain).Scan(&count); err != nil {
|
||||||
return fmt.Errorf("count dmarc reports: %w", err)
|
return fmt.Errorf("count dmarc reports: %w", err)
|
||||||
}
|
}
|
||||||
if count <= DMARCReportsMaxKeep {
|
if count <= DMARCReportsMaxKeep {
|
||||||
@@ -147,8 +151,8 @@ func (s *Store) PruneDMARCReports() error {
|
|||||||
excess := count - DMARCReportsMaxKeep
|
excess := count - DMARCReportsMaxKeep
|
||||||
_, err := s.db.Exec(`
|
_, err := s.db.Exec(`
|
||||||
DELETE FROM dmarc_reports WHERE id IN (
|
DELETE FROM dmarc_reports WHERE id IN (
|
||||||
SELECT id FROM dmarc_reports ORDER BY received_at ASC LIMIT ?
|
SELECT id FROM dmarc_reports WHERE domain = ? ORDER BY received_at ASC LIMIT ?
|
||||||
)`, excess)
|
)`, domain, excess)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("prune dmarc by count: %w", err)
|
return fmt.Errorf("prune dmarc by count: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ func TestPruneDMARCReports(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := st.PruneDMARCReports(); err != nil {
|
if err := st.PruneDMARCReports("example.com"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
list, _ := st.ListDMARCReports(DMARCReportScope{AllDomains: true}, 100)
|
list, _ := st.ListDMARCReports(DMARCReportScope{AllDomains: true}, 100)
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ func TestDMARCHubShowsIngestAndRecentReports(t *testing.T) {
|
|||||||
body := getBody(t, s.h.HandleDMARCList, "/outbound/dmarc")
|
body := getBody(t, s.h.HandleDMARCList, "/outbound/dmarc")
|
||||||
has(t, "hub", body,
|
has(t, "hub", body,
|
||||||
`<h1 class="title is-3">DMARC reports</h1>`, `<h2>Ingest</h2>`, `<span class="tag is-success is-light">ok</span>`,
|
`<h1 class="title is-3">DMARC reports</h1>`, `<h2>Ingest</h2>`, `<span class="tag is-success is-light">ok</span>`,
|
||||||
`1 kept · 0 parse failures`, `Hosted addresses`, `dmarc-reports+<domain>@mail.example.org`, `500 reports or 90 days`,
|
`1 kept · 0 parse failures`, `Hosted addresses`, `dmarc-reports+<domain>@mail.example.org`, `500 reports a domain or 90 days`,
|
||||||
`<h2>Recent reports</h2>`, `<a href="/outbound/dmarc/domains/1">example.org</a>`, `Outlook.com`,
|
`<h2>Recent reports</h2>`, `<a href="/outbound/dmarc/domains/1">example.org</a>`, `Outlook.com`,
|
||||||
"<td>1\u00a0903</td>", `<span class="tag is-warning is-light">7</span>`, `href="/outbound/dmarc/reports/`+idStr(s.reportID)+`">View</a>`)
|
"<td>1\u00a0903</td>", `<span class="tag is-warning is-light">7</span>`, `href="/outbound/dmarc/reports/`+idStr(s.reportID)+`">View</a>`)
|
||||||
lacks(t, "hub", body, `None yet`)
|
lacks(t, "hub", body, `None yet`)
|
||||||
|
|||||||
@@ -113,7 +113,7 @@ func NewDMARCHub(m Meta, in IngestInput) *DMARCHub {
|
|||||||
last,
|
last,
|
||||||
{Label: "This week", Value: Plain(FormatCount(int64(in.KeptThisWeek)) + " kept · " + plural(int64(in.ParseFailures), "parse failure", "parse failures"))},
|
{Label: "This week", Value: Plain(FormatCount(int64(in.KeptThisWeek)) + " kept · " + plural(int64(in.ParseFailures), "parse failure", "parse failures"))},
|
||||||
{Label: "Hosted addresses", Value: Plain(in.Hosted), Mono: true, Small: true},
|
{Label: "Hosted addresses", Value: Plain(in.Hosted), Mono: true, Small: true},
|
||||||
{Label: "Retention", Value: Plain(FormatCount(int64(in.RetentionMax)) + " reports or " + plural(int64(in.RetentionDays), "day", "days"))},
|
{Label: "Retention", Value: Plain(FormatCount(int64(in.RetentionMax)) + " reports a domain or " + plural(int64(in.RetentionDays), "day", "days"))},
|
||||||
},
|
},
|
||||||
Recent: Box{No: "02", Title: "Recent reports"},
|
Recent: Box{No: "02", Title: "Recent reports"},
|
||||||
Empty: EmptyState{Icon: "ti-report-analytics", Text: Plain("No reports yet.")},
|
Empty: EmptyState{Icon: "ti-report-analytics", Text: Plain("No reports yet.")},
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ func TestDMARCHubShowsIngestAndTheLatestReports(t *testing.T) {
|
|||||||
pageHas(t, "DMARC reports", out,
|
pageHas(t, "DMARC reports", out,
|
||||||
`<h1 class="title is-3">DMARC reports</h1>`, `<code>rua=</code>`,
|
`<h1 class="title is-3">DMARC reports</h1>`, `<code>rua=</code>`,
|
||||||
`<h2>Ingest</h2>`, `<span class="tag is-success is-light">ok</span>`,
|
`<h2>Ingest</h2>`, `<span class="tag is-success is-light">ok</span>`,
|
||||||
`<dd>2026-09-21 06:12 UTC</dd>`, `<dd>14 kept · 0 parse failures</dd>`, `<dt>Hosted addresses</dt>`, `<dd class="sp-mono sp-small">dmarc-reports+<domain>@mail.example.org</dd>`, `<dd>500 reports or 90 days</dd>`,
|
`<dd>2026-09-21 06:12 UTC</dd>`, `<dd>14 kept · 0 parse failures</dd>`, `<dt>Hosted addresses</dt>`, `<dd class="sp-mono sp-small">dmarc-reports+<domain>@mail.example.org</dd>`, `<dd>500 reports a domain or 90 days</dd>`,
|
||||||
`<th>Received, UTC</th>`, `09-21 06:12`, `<a href="/outbound/dmarc/domains/1">example.org</a>`,
|
`<th>Received, UTC</th>`, `09-21 06:12`, `<a href="/outbound/dmarc/domains/1">example.org</a>`,
|
||||||
"<td>1 903</td>", `<span class="tag is-warning is-light">7</span>`, `<td>0</td><td class="sp-actions"><a href="/outbound/dmarc/reports/41">View</a>`,
|
"<td>1 903</td>", `<span class="tag is-warning is-light">7</span>`, `<td>0</td><td class="sp-actions"><a href="/outbound/dmarc/reports/41">View</a>`,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ type port25Case struct {
|
|||||||
// flags=, user= and argv= are arguments of pipe(8); set as -o parameters they
|
// flags=, user= and argv= are arguments of pipe(8); set as -o parameters they
|
||||||
// are dropped, and a bare "pipe" dies on every report with "fatal: missing
|
// are dropped, and a bare "pipe" dies on every report with "fatal: missing
|
||||||
// user= command-line attribute" - the state of 1.7.0 through 1.9.5.
|
// user= command-line attribute" - the state of 1.7.0 through 1.9.5.
|
||||||
const ingestService = "dmarc-ingest unix - n n - - pipe flags=Rq user=panel argv=/usr/local/bin/dmarc-ingest ${recipient}"
|
const ingestService = "dmarc-ingest unix - n n - 1 pipe flags=Rq user=panel argv=/usr/local/bin/dmarc-ingest ${recipient}"
|
||||||
|
|
||||||
var port25Cases = []port25Case{
|
var port25Cases = []port25Case{
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user