perf: keep file I/O off Service.mu and untie StartOnly from the app context
Phase 7 of the whole-project review (findings 3.2 and 3.3). Service.mu is the lock the Fyne main thread takes on every Jobs() and Runtime() call, so anything blocking inside it makes a UI refresh wait on the disk. Three things did: - Every SaveJobs/SaveConfig was a marshal, fsync, and rename under mu. Writes are now prepared under the lock (Store.PrepareSaveJobs / PrepareSaveConfig snapshot the payload and target path) and run after it is released. deferSaveLocked takes saveMu while mu is still held, so writes still reach the file in the order their snapshots were taken and an older snapshot can never land on top of a newer one. - executeRun ran runner.CleanupLogs under mu after every run. It needs only the values already snapshotted into runEnv, so it now runs after the unlock — including when the job is gone, since the run still wrote a log file that retention covers. - adoptJobsLocked ran runner.SeedStats under mu, reached from UpdateSettings on the UI thread. Seeding moved out into applySeededStatsLocked; UpdateSettings now reads the new jobs file and seeds its statistics before taking the lock, and re-checks the "no jobs-file switch while running" guard once it has it. SeedStats also opened every log file twice — once to find the job, again to read the result. readLogSummary reads job_id, state, and duration in one pass, so each log is opened once. StartOnly runs were built with exec.CommandContext on the app's lifecycle context. os/exec keeps a watcher goroutine alive until Wait returns or the context is done, and StartOnly never calls Wait, so one goroutine leaked per run and would then try to kill a process whose handle startJobOnly had already released. The invocation now uses context.Background(), whose nil Done channel means no watcher is started at all. Regression tests: TestRunJobStartOnlyLeavesNoContextWatcher (fails with 5 leaked goroutines on the old code), TestConcurrentJobOperationsLeaveTheFileMatchingMemory, and TestUpdateSettingsSeedsAdoptedJobsFromLogs. STANDARDS gains the no-I/O-under-mu rule and the "a StartOnly process outlives GoSentry" entry. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
+61
-32
@@ -43,15 +43,23 @@ func (s *Service) CreateJob(job domain.Job) (domain.Job, error) {
|
||||
s.parseScheduleLocked(&job)
|
||||
record := uiRecord(job.ID, job.Name, "Created", "Job was added")
|
||||
prependLog(runtime, record)
|
||||
err := s.store.SaveJobs(s.jobs)
|
||||
if err != nil {
|
||||
s.jobs = s.jobs[:len(s.jobs)-1]
|
||||
save := s.deferSaveLocked(s.store.PrepareSaveJobs(s.jobs))
|
||||
s.mu.Unlock()
|
||||
|
||||
if err := save(); err != nil {
|
||||
// The write is atomic, so a failure left the file holding the previous
|
||||
// list: take the job back out so memory matches what is on disk. Another
|
||||
// operation may have run in between, so it is removed by ID rather than by
|
||||
// truncating the slice.
|
||||
s.mu.Lock()
|
||||
if index := s.indexByIDLocked(job.ID); index >= 0 {
|
||||
s.jobs = append(s.jobs[:index], s.jobs[index+1:]...)
|
||||
}
|
||||
delete(s.runtimes, job.ID)
|
||||
delete(s.schedules, job.ID)
|
||||
s.mu.Unlock()
|
||||
return domain.Job{}, err
|
||||
}
|
||||
s.mu.Unlock()
|
||||
s.emit(RunRecorded{Record: record})
|
||||
s.emit(JobChanged{JobID: job.ID})
|
||||
return job, nil
|
||||
@@ -87,10 +95,10 @@ func (s *Service) UpdateJob(job domain.Job) error {
|
||||
s.refreshNextRunLocked(existing, runtime)
|
||||
record := uiRecord(job.ID, job.Name, "Updated", "Job settings changed")
|
||||
prependLog(runtime, record)
|
||||
err := s.store.SaveJobs(s.jobs)
|
||||
save := s.deferSaveLocked(s.store.PrepareSaveJobs(s.jobs))
|
||||
s.mu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
if err := save(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.emit(RunRecorded{Record: record})
|
||||
@@ -113,10 +121,10 @@ func (s *Service) DeleteJob(id int) error {
|
||||
delete(s.runtimes, id)
|
||||
delete(s.schedules, id)
|
||||
record := uiRecord(id, deleted.Name, "Deleted", "Job was removed")
|
||||
err := s.store.SaveJobs(s.jobs)
|
||||
save := s.deferSaveLocked(s.store.PrepareSaveJobs(s.jobs))
|
||||
s.mu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
if err := save(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.emit(RunRecorded{Record: record})
|
||||
@@ -155,10 +163,10 @@ func (s *Service) SetEnabled(id int, enabled bool) error {
|
||||
record = uiRecord(id, job.Name, "Paused", "Job was disabled")
|
||||
}
|
||||
prependLog(runtime, record)
|
||||
err := s.store.SaveJobs(s.jobs)
|
||||
save := s.deferSaveLocked(s.store.PrepareSaveJobs(s.jobs))
|
||||
s.mu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
if err := save(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.emit(RunRecorded{Record: record})
|
||||
@@ -188,10 +196,10 @@ func (s *Service) SetGlobalPause(paused bool) error {
|
||||
}
|
||||
s.refreshNextRunFromLocked(job, runtime, now)
|
||||
}
|
||||
err := s.store.SaveConfig()
|
||||
save := s.deferSaveLocked(s.store.PrepareSaveConfig())
|
||||
s.mu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
if err := save(); err != nil {
|
||||
return err
|
||||
}
|
||||
state, detail := "Resumed", "All job execution resumed"
|
||||
@@ -219,9 +227,9 @@ func (s *Service) SetJobListView(view domain.JobListView) error {
|
||||
return nil
|
||||
}
|
||||
s.store.Config.JobListView = view
|
||||
err := s.store.SaveConfig()
|
||||
save := s.deferSaveLocked(s.store.PrepareSaveConfig())
|
||||
s.mu.Unlock()
|
||||
return err
|
||||
return save()
|
||||
}
|
||||
|
||||
// ShouldNotifyOnFailure reports whether the user has enabled desktop
|
||||
@@ -251,54 +259,75 @@ func (s *Service) UpdateSettings(config domain.Config) error {
|
||||
config.JobsFile = strings.TrimSpace(config.JobsFile)
|
||||
|
||||
s.mu.Lock()
|
||||
jobsPath := storage.ResolveConfiguredPath(s.store.Paths.AppDir, config.JobsFile)
|
||||
// AppDir is fixed for the process and only UpdateSettings itself — a UI
|
||||
// action — can move JobsPath, so this snapshot stays valid across the reads
|
||||
// below.
|
||||
appDir := s.store.Paths.AppDir
|
||||
jobsPath := storage.ResolveConfiguredPath(appDir, config.JobsFile)
|
||||
switching := jobsPath != s.store.Paths.JobsPath
|
||||
if switching && s.anyRunningLocked() {
|
||||
s.mu.Unlock()
|
||||
running := s.anyRunningLocked()
|
||||
s.mu.Unlock()
|
||||
|
||||
if switching && running {
|
||||
return errors.New("cannot change the jobs file while a job is running")
|
||||
}
|
||||
// Read the new file before anything is written, so a file that cannot be
|
||||
// parsed leaves both the config and the current jobs untouched.
|
||||
// Read the new file, and reconstruct its jobs' statistics from the logs the
|
||||
// new config points at, before anything is written and while no lock is held:
|
||||
// both are file I/O, and SeedStats opens every log in the directory. A file
|
||||
// that cannot be parsed leaves both the config and the current jobs untouched.
|
||||
var adopted []domain.Job
|
||||
var seeds map[int]runner.SeededStats
|
||||
if switching {
|
||||
jobs, found, err := storage.LoadJobsFile(jobsPath)
|
||||
if err != nil {
|
||||
s.mu.Unlock()
|
||||
return fmt.Errorf("read jobs file %s: %w", jobsPath, err)
|
||||
}
|
||||
if found {
|
||||
adopted = jobs
|
||||
seeds = runner.SeedStats(storage.ResolveConfiguredPath(appDir, config.LogsDir), jobs, config.MaxLogFiles)
|
||||
}
|
||||
}
|
||||
|
||||
s.store.Config = config
|
||||
if err := s.store.SaveConfig(); err != nil {
|
||||
s.mu.Lock()
|
||||
// The guard above was evaluated before the reads, off the lock, so re-check
|
||||
// it: a scheduled run may have started in the meantime, and adoption drops
|
||||
// every runtime.
|
||||
if switching && s.anyRunningLocked() {
|
||||
s.mu.Unlock()
|
||||
return err
|
||||
return errors.New("cannot change the jobs file while a job is running")
|
||||
}
|
||||
s.store.Config = config
|
||||
saveConfig := s.store.PrepareSaveConfig()
|
||||
if adopted != nil {
|
||||
s.adoptJobsLocked(adopted)
|
||||
s.applySeededStatsLocked(seeds)
|
||||
}
|
||||
// SaveConfig re-resolved the paths from the new config, so SaveJobs writes to
|
||||
// the (possibly new) jobs file and cleanup targets the new logs dir. Adopted
|
||||
// jobs are written back too, which persists the IDs and defaults that
|
||||
// normalization filled in, exactly as loading them at startup would.
|
||||
if err := s.store.SaveJobs(s.jobs); err != nil {
|
||||
s.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
// PrepareSaveConfig re-resolved the paths from the new config, so the jobs
|
||||
// write targets the (possibly new) jobs file and cleanup targets the new logs
|
||||
// dir. Adopted jobs are written back too, which persists the IDs and defaults
|
||||
// that normalization filled in, exactly as loading them at startup would. The
|
||||
// jobs write is skipped when the config write fails, because both writes run
|
||||
// in the order prepared and stop at the first error.
|
||||
save := s.deferSaveLocked(saveConfig, s.store.PrepareSaveJobs(s.jobs))
|
||||
loaded := len(s.jobs)
|
||||
logsDir := s.store.Paths.LogsDir
|
||||
maxFiles := s.store.Config.MaxLogFiles
|
||||
maxAge := s.store.Config.MaxLogAgeDays
|
||||
s.mu.Unlock()
|
||||
|
||||
saveErr := save()
|
||||
if adopted != nil {
|
||||
// A broad JobChanged redraws the job list; JobsLoaded tells the user in
|
||||
// History which file those jobs came from, since nothing was asked.
|
||||
// History which file those jobs came from, since nothing was asked. Both
|
||||
// are emitted even when the write failed: the adopted jobs are already the
|
||||
// in-memory list, and a job list the user cannot see would be worse than
|
||||
// the error they are about to be shown.
|
||||
s.emit(JobsLoaded{Path: jobsPath, Count: loaded})
|
||||
s.emit(JobChanged{})
|
||||
}
|
||||
if saveErr != nil {
|
||||
return saveErr
|
||||
}
|
||||
return runner.CleanupLogs(logsDir, maxFiles, maxAge)
|
||||
}
|
||||
|
||||
|
||||
@@ -3,8 +3,10 @@ package app
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -725,6 +727,102 @@ func TestUpdateSettingsRefusesJobsFileSwitchWhileRunning(t *testing.T) {
|
||||
waitRecord(t, done)
|
||||
}
|
||||
|
||||
// Adoption reconstructs the adopted jobs' aggregate statistics from the log
|
||||
// files the new configuration points at. That scan opens every log in the
|
||||
// directory, so UpdateSettings runs it before taking the state lock; this pins
|
||||
// that its result still reaches the runtime map.
|
||||
func TestUpdateSettingsSeedsAdoptedJobsFromLogs(t *testing.T) {
|
||||
svc := newTempService(t, []domain.Job{{ID: 1, Name: "Local", Schedule: "@every 1m", Command: "echo local", Enabled: true}})
|
||||
|
||||
logsDir := svc.store.Paths.LogsDir
|
||||
if err := os.MkdirAll(logsDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
log := "time: 2026-08-05 10:00:00\njob_id: 7\njob_name: Adopted\ntrigger: Schedule\nstate: Failed\ndetail: boom\nduration: 1500\n\nstdout:\n<empty>\n"
|
||||
if err := os.WriteFile(filepath.Join(logsDir, "20260805-100000_Adopted.log"), []byte(log), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
shared := filepath.Join(svc.store.Paths.AppDir, "shared.json")
|
||||
data, err := json.Marshal(domain.JobsFile{Jobs: []domain.Job{
|
||||
{ID: 7, Name: "Adopted", Schedule: "@every 5m", Command: "echo adopted", Enabled: true},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(shared, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
config := svc.store.Config
|
||||
config.JobsFile = shared
|
||||
if err := svc.UpdateSettings(config); err != nil {
|
||||
t.Fatalf("UpdateSettings: %v", err)
|
||||
}
|
||||
|
||||
runtime := svc.Runtime(7)
|
||||
if runtime == nil {
|
||||
t.Fatal("the adopted job has no runtime")
|
||||
}
|
||||
if runtime.RunCount != 1 || runtime.FailCount != 1 || runtime.LastDurationMS != 1500 {
|
||||
t.Errorf("seeded stats: RunCount=%d FailCount=%d LastDurationMS=%d, want 1/1/1500",
|
||||
runtime.RunCount, runtime.FailCount, runtime.LastDurationMS)
|
||||
}
|
||||
}
|
||||
|
||||
// Job saves run after mu is released, so one operation can be writing while
|
||||
// another mutates state. deferSaveLocked takes its own lock while mu is still
|
||||
// held, which is what keeps writes in mutation order: whatever changed the list
|
||||
// last also wrote it last, so the file ends up matching memory instead of
|
||||
// holding an older snapshot.
|
||||
func TestConcurrentJobOperationsLeaveTheFileMatchingMemory(t *testing.T) {
|
||||
svc := newTempService(t, nil)
|
||||
|
||||
const workers = 8
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
job, err := svc.CreateJob(domain.Job{Name: fmt.Sprintf("Job %d", i), Schedule: "@every 1m", Command: "echo hi", Enabled: true})
|
||||
if err != nil {
|
||||
t.Errorf("CreateJob %d: %v", i, err)
|
||||
return
|
||||
}
|
||||
if err := svc.SetEnabled(job.ID, false); err != nil {
|
||||
t.Errorf("SetEnabled %d: %v", job.ID, err)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
memory := svc.Jobs()
|
||||
if len(memory) != workers {
|
||||
t.Fatalf("jobs in memory = %d, want %d", len(memory), workers)
|
||||
}
|
||||
saved, found, err := storage.LoadJobsFile(svc.store.Paths.JobsPath)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("read jobs file: found=%v err=%v", found, err)
|
||||
}
|
||||
if len(saved) != len(memory) {
|
||||
t.Fatalf("jobs on disk = %d, want %d: the last write must be the last mutation", len(saved), len(memory))
|
||||
}
|
||||
onDisk := make(map[int]domain.Job, len(saved))
|
||||
for _, job := range saved {
|
||||
onDisk[job.ID] = job
|
||||
}
|
||||
for _, job := range memory {
|
||||
got, ok := onDisk[job.ID]
|
||||
if !ok {
|
||||
t.Errorf("job %d (%q) is in memory but missing from the file", job.ID, job.Name)
|
||||
continue
|
||||
}
|
||||
if got.Name != job.Name || got.Enabled != job.Enabled {
|
||||
t.Errorf("job %d on disk = %q/%v, want %q/%v", job.ID, got.Name, got.Enabled, job.Name, job.Enabled)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrependLogCapsActivityList(t *testing.T) {
|
||||
runtime := &domain.JobRuntime{}
|
||||
for i := 0; i < maxJobLogs+10; i++ {
|
||||
|
||||
+6
-2
@@ -147,7 +147,6 @@ func (s *Service) executeRun(ctx context.Context, jobCopy domain.Job, trigger st
|
||||
record, logErr := s.runJob(ctx, &jobCopy, trigger, env.logsDir, env.timeout)
|
||||
|
||||
s.mu.Lock()
|
||||
var cleanupErr error
|
||||
var rerunStarted bool
|
||||
if current := s.findByIDLocked(jobCopy.ID); current != nil {
|
||||
runtime := s.runtimeForLocked(current)
|
||||
@@ -166,10 +165,15 @@ func (s *Service) executeRun(ctx context.Context, jobCopy domain.Job, trigger st
|
||||
} else {
|
||||
s.refreshNextRunLocked(current, runtime)
|
||||
}
|
||||
cleanupErr = runner.CleanupLogs(env.logsDir, env.maxFiles, env.maxAge)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
// Cleanup is a directory scan plus up to MaxLogFiles unlinks. It needs only
|
||||
// the values already snapshotted into runEnv, so it runs after mu is released
|
||||
// rather than making every UI refresh wait behind it. It runs even when the
|
||||
// job is gone, because the run still wrote a log file that retention covers.
|
||||
cleanupErr := runner.CleanupLogs(env.logsDir, env.maxFiles, env.maxAge)
|
||||
|
||||
if logErr != nil {
|
||||
s.emit(ErrorOccurred{Err: fmt.Errorf("write run log for %q: %w", jobCopy.Name, logErr)})
|
||||
}
|
||||
|
||||
+48
-8
@@ -26,7 +26,10 @@ import (
|
||||
// it; unexported helpers ending in "Locked" assume the caller already holds it.
|
||||
// The Service must never call back into the UI (or any code that might re-enter
|
||||
// the Service) while holding mu — in particular emit() is always called after
|
||||
// mu is released.
|
||||
// mu is released. Blocking file I/O follows the same rule: mu is the lock the
|
||||
// Fyne main thread takes on every Jobs() and Runtime() call, so a JSON write, a
|
||||
// log-directory scan, or a pass over every log header must not happen inside it
|
||||
// (see deferSaveLocked, executeRun, and applySeededStatsLocked).
|
||||
type Service struct {
|
||||
mu sync.Mutex
|
||||
store *storage.Store
|
||||
@@ -56,6 +59,13 @@ type Service struct {
|
||||
// do not exercise autostart; Open() wires it via autostart.New().
|
||||
manager autostart.Manager
|
||||
|
||||
// saveMu serializes the store writes that operations prepare under mu and run
|
||||
// after releasing it. It is taken while mu is still held and released once the
|
||||
// write is done, so writes reach the file in the same order their snapshots
|
||||
// were taken and an older snapshot can never land on top of a newer one.
|
||||
// Nothing may take mu while holding saveMu.
|
||||
saveMu sync.Mutex
|
||||
|
||||
// observers and their guard live in events.go. dispatchMu is separate from mu
|
||||
// so that emitting an event never requires (or is held under) the state lock:
|
||||
// the Service must release mu before dispatching, per the locking contract.
|
||||
@@ -63,6 +73,26 @@ type Service struct {
|
||||
observers []Observer
|
||||
}
|
||||
|
||||
// deferSaveLocked prepares the store writes for the caller to run after mu is
|
||||
// released, and takes saveMu now so a later operation's write cannot overtake
|
||||
// this one. The caller must hold mu, must unlock it before calling the returned
|
||||
// function, and must call that function exactly once. Keeping the marshal, the
|
||||
// fsync, and the rename out of the critical section is what stops a settings
|
||||
// change or a job edit from blocking a scheduler tick or a finishing run. The
|
||||
// writes run in the order given and stop at the first error.
|
||||
func (s *Service) deferSaveLocked(writes ...func() error) func() error {
|
||||
s.saveMu.Lock()
|
||||
return func() error {
|
||||
defer s.saveMu.Unlock()
|
||||
for _, write := range writes {
|
||||
if err := write(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewService wires the Service to a loaded store and its jobs. It builds the
|
||||
// initial runtime map from the durable jobs so every job has transient state
|
||||
// from the moment the Service exists, and parses each job's schedule once. The
|
||||
@@ -77,18 +107,19 @@ func NewService(store *storage.Store, jobs []domain.Job) *Service {
|
||||
// No lock is needed here: construction is single-threaded, before Start
|
||||
// launches the timing loop.
|
||||
s.adoptJobsLocked(jobs)
|
||||
s.applySeededStatsLocked(runner.SeedStats(store.Paths.LogsDir, s.jobs, store.Config.MaxLogFiles))
|
||||
return s
|
||||
}
|
||||
|
||||
// adoptJobsLocked makes jobs the Service's durable state and rebuilds everything
|
||||
// derived from it: the runtime map, the parsed-schedule cache, each job's first
|
||||
// next-run — so the Service is ready to schedule the moment it exists, mirroring
|
||||
// the old scheduler's reset-on-construction — and the statistics seeded from
|
||||
// existing log files, so the details panel shows accumulated run history
|
||||
// immediately rather than only runs since this process started.
|
||||
// derived from it: the runtime map, the parsed-schedule cache, and each job's
|
||||
// first next-run — so the Service is ready to schedule the moment it exists,
|
||||
// mirroring the old scheduler's reset-on-construction.
|
||||
//
|
||||
// It backs both construction and a Settings change that points at a different
|
||||
// jobs file. The caller must hold mu.
|
||||
// jobs file. Statistics seeded from existing log files are applied separately by
|
||||
// applySeededStatsLocked, because reconstructing them is file I/O. The caller
|
||||
// must hold mu.
|
||||
func (s *Service) adoptJobsLocked(jobs []domain.Job) {
|
||||
s.jobs = jobs
|
||||
s.runtimes = domain.NewRuntimes(jobs)
|
||||
@@ -100,7 +131,16 @@ func (s *Service) adoptJobsLocked(jobs []domain.Job) {
|
||||
s.parseScheduleLocked(job)
|
||||
s.refreshNextRunFromLocked(job, s.runtimes[job.ID], now)
|
||||
}
|
||||
for id, seed := range runner.SeedStats(s.store.Paths.LogsDir, s.jobs, s.store.Config.MaxLogFiles) {
|
||||
}
|
||||
|
||||
// applySeededStatsLocked folds statistics reconstructed from existing log files
|
||||
// into the runtime map, so the details panel shows accumulated run history
|
||||
// immediately rather than only runs since this process started. It is separate
|
||||
// from adoptJobsLocked because producing the seeds opens every log file in the
|
||||
// directory, which must not happen under mu: callers compute the map first and
|
||||
// apply it here. The caller must hold mu.
|
||||
func (s *Service) applySeededStatsLocked(seeds map[int]runner.SeededStats) {
|
||||
for id, seed := range seeds {
|
||||
runtime := s.runtimes[id]
|
||||
if runtime == nil {
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user