CI showed exit=1 in ~3s with empty docker logs after /data/opendkim was created. Log each entrypoint step and trap ERR with LINENO; also read the container LogPath and docker logs by container id when compose logs are empty. Co-Authored-By: Composer <noreply@cursor.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -128,6 +128,21 @@ func runEntrypointBackground(t *testing.T, hostnameEnv string) (output string, s
|
||||
inspect, _ := exec.Command("docker", "inspect", name, "--format",
|
||||
"status={{.State.Status}} exit={{.State.ExitCode}} err={{.State.Error}} oom={{.State.OOMKilled}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}").CombinedOutput()
|
||||
logs, logErr := exec.Command("docker", "logs", name).CombinedOutput()
|
||||
// Exited containers sometimes yield empty `docker logs` in CI; also read
|
||||
// the JSON-file log path docker recorded on the container.
|
||||
logPathOut, _ := exec.Command("docker", "inspect", name, "--format", "{{.LogPath}}").CombinedOutput()
|
||||
logPath := strings.TrimSpace(string(logPathOut))
|
||||
logFileHead := ""
|
||||
if logPath != "" {
|
||||
if b, e := os.ReadFile(logPath); e == nil {
|
||||
logFileHead = truncateForDebug(string(b), 4000)
|
||||
if strings.TrimSpace(string(logs)) == "" && len(b) > 0 {
|
||||
logs = b
|
||||
}
|
||||
} else {
|
||||
logFileHead = "read LogPath: " + e.Error()
|
||||
}
|
||||
}
|
||||
dataMode, _ := exec.Command("docker", "exec", name, "stat", "-c", "%a %U:%G", "/data").CombinedOutput()
|
||||
opendkimStat, _ := exec.Command("docker", "exec", name, "stat", "-c", "%a %U:%G", "/data/opendkim").CombinedOutput()
|
||||
keyTableStat, _ := exec.Command("docker", "exec", name, "stat", "-c", "%a %U:%G", "/data/opendkim/KeyTable").CombinedOutput()
|
||||
@@ -147,16 +162,18 @@ func runEntrypointBackground(t *testing.T, hostnameEnv string) (output string, s
|
||||
hostDataListing = strings.Join(parts, ",")
|
||||
}
|
||||
agentDebugLog("H1", "hostname_gate_test.go:post-wait", "data permissions", map[string]any{
|
||||
"dataMode": strings.TrimSpace(string(dataMode)),
|
||||
"opendkimStat": strings.TrimSpace(string(opendkimStat)),
|
||||
"keyTableStat": strings.TrimSpace(string(keyTableStat)),
|
||||
"dataMode": strings.TrimSpace(string(dataMode)),
|
||||
"opendkimStat": strings.TrimSpace(string(opendkimStat)),
|
||||
"keyTableStat": strings.TrimSpace(string(keyTableStat)),
|
||||
"hostDataListing": hostDataListing,
|
||||
})
|
||||
agentDebugLog("H2", "hostname_gate_test.go:post-wait", "container inspect", map[string]any{
|
||||
"inspect": strings.TrimSpace(string(inspect)),
|
||||
"waitErr": fmt.Sprintf("%v", err),
|
||||
"logErr": fmt.Sprintf("%v", logErr),
|
||||
"logLen": len(logs),
|
||||
"inspect": strings.TrimSpace(string(inspect)),
|
||||
"waitErr": fmt.Sprintf("%v", err),
|
||||
"logErr": fmt.Sprintf("%v", logErr),
|
||||
"logLen": len(logs),
|
||||
"logPath": logPath,
|
||||
"logFileHead": logFileHead,
|
||||
})
|
||||
agentDebugLog("H3", "hostname_gate_test.go:post-wait", "supervisor and logs", map[string]any{
|
||||
"supervisorctl": strings.TrimSpace(string(supStatus)),
|
||||
@@ -167,8 +184,11 @@ func runEntrypointBackground(t *testing.T, hostnameEnv string) (output string, s
|
||||
diag := string(logs)
|
||||
if strings.TrimSpace(diag) == "" {
|
||||
diag = "(empty docker logs)\ninspect: " + strings.TrimSpace(string(inspect)) +
|
||||
"\nlogPath: " + logPath +
|
||||
"\nlogFileHead: " + logFileHead +
|
||||
"\nsupervisorctl: " + strings.TrimSpace(string(supStatus)) +
|
||||
"\n/data mode: " + strings.TrimSpace(string(dataMode))
|
||||
"\n/data mode: " + strings.TrimSpace(string(dataMode)) +
|
||||
"\nhost /data: " + hostDataListing
|
||||
}
|
||||
return diag, err == nil
|
||||
}
|
||||
|
||||
@@ -54,17 +54,36 @@ func checkSupervisorProcesses(s *stack) error {
|
||||
ps, _ := s.compose("ps", "-a", "--format", "json")
|
||||
inspectOut, _ := exec.Command("docker", "ps", "-a", "--filter", "name=selfpost-e2e",
|
||||
"--format", "{{.Names}} {{.Status}} {{.ID}}").CombinedOutput()
|
||||
// compose logs can be empty for a fast-exit container; pull docker logs by id.
|
||||
rawDockerLogs := ""
|
||||
for _, line := range strings.Split(string(inspectOut), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 3 || !strings.Contains(fields[0], "selfpost-1") {
|
||||
continue
|
||||
}
|
||||
id := fields[len(fields)-1]
|
||||
if b, e := exec.Command("docker", "logs", id).CombinedOutput(); e == nil {
|
||||
rawDockerLogs = string(b)
|
||||
} else {
|
||||
rawDockerLogs = e.Error() + "\n" + string(b)
|
||||
}
|
||||
if strings.TrimSpace(logs) == "" && strings.TrimSpace(rawDockerLogs) != "" {
|
||||
logs = rawDockerLogs
|
||||
}
|
||||
break
|
||||
}
|
||||
dataPath := filepath.Join(s.stageDir, "data")
|
||||
dataMode := ""
|
||||
if fi, e := os.Stat(dataPath); e == nil {
|
||||
dataMode = fmt.Sprintf("%04o", fi.Mode().Perm())
|
||||
}
|
||||
entrypointSnippet, _ := exec.Command("docker", "run", "--rm", "--entrypoint", "sh", "selfpost:e2e",
|
||||
"-c", "grep -n 'chmod 755 /data\\|SELFPOST_HOSTNAME is not set' /usr/local/bin/entrypoint.sh | head -20").CombinedOutput()
|
||||
"-c", "grep -n 'chmod 755 /data\\|entrypoint: command failed\\|step()' /usr/local/bin/entrypoint.sh | head -30").CombinedOutput()
|
||||
agentDebugLog("H4", "process_check.go:fail", "compose selfpost not ready", map[string]any{
|
||||
"waitErr": err.Error(),
|
||||
"logsLen": len(logs),
|
||||
"logsHead": truncateForDebug(logs, 4000),
|
||||
"rawDockerLogsHead": truncateForDebug(rawDockerLogs, 4000),
|
||||
"composePs": truncateForDebug(string(ps), 2000),
|
||||
"dockerPs": strings.TrimSpace(string(inspectOut)),
|
||||
"hostDataMode": dataMode,
|
||||
@@ -74,8 +93,8 @@ func checkSupervisorProcesses(s *stack) error {
|
||||
"snippet": strings.TrimSpace(string(entrypointSnippet)),
|
||||
})
|
||||
// #endregion
|
||||
return fmt.Errorf("%w\n==== selfpost logs ====\n%s\n==== docker ps ====\n%s\n==== entrypoint markers ====\n%s",
|
||||
err, logs, inspectOut, entrypointSnippet)
|
||||
return fmt.Errorf("%w\n==== selfpost logs ====\n%s\n==== docker logs (by id) ====\n%s\n==== docker ps ====\n%s\n==== entrypoint markers ====\n%s",
|
||||
err, logs, rawDockerLogs, inspectOut, entrypointSnippet)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user