fix: remove no-op SaveJobs calls, fix PendingRuns lifecycle and cap

Implements items 4-5 of the whole-project review's suggested order
(docs/PROJECT_REVIEW_PLAN.md):

- Drop the three SaveJobs calls in the run lifecycle (startRunLocked,
  executeRun, SetGlobalPause): none of them change a durable Job field,
  everything they touch lives on JobRuntime, which is never persisted.
  Retire TestStartRunLockedRollbackOnSaveFailure with the rollback it
  guarded, since a run can no longer fail to start this way.
- Clear PendingRuns (the "queue" overlap policy's backlog) when a job is
  disabled or the scheduler is globally paused, so resuming or
  re-enabling a job no longer replays a deferred run left over from
  before the pause/disable. Cap it at maxPendingRuns (10) so a job whose
  runs take longer than its own interval stops accumulating an unbounded
  backlog. Surface the queued count in the details pane via DisplayStats.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-06 17:06:55 +03:00
parent 1242b22e4f
commit 263717874c
9 changed files with 147 additions and 82 deletions
+24 -41
View File
@@ -11,6 +11,13 @@ import (
"gitea.mixdep.ru/mix/gosentry/src/runner"
)
// maxPendingRuns bounds how many missed occurrences the "queue" overlap policy
// will defer for one job. Without a ceiling a job whose runs take longer than
// its interval would queue one more occurrence on every tick forever, so once
// the cap is reached further overlaps are dropped exactly as the "skip" policy
// would drop them, until the backlog drains below the cap again.
const maxPendingRuns = 10
// RunNow starts a manual run of a job. Global pause stops only the scheduler's
// automatic runs (see RunDue), so a manual "Run now" is allowed even while
// paused — it is the user's explicit, one-off action. It will not start a job
@@ -36,14 +43,12 @@ func (s *Service) RunNow(id int) error {
s.mu.Unlock()
return errors.New("another job is already running (sequential mode)")
}
err := s.startRunLocked(job, runtime, "Manual", time.Now())
s.startRunLocked(job, runtime, "Manual", time.Now())
s.mu.Unlock()
if err == nil {
// Reflect the "Running" transition; the run's completion emits again later.
s.emit(JobChanged{JobID: id})
}
return err
// Reflect the "Running" transition; the run's completion emits again later.
s.emit(JobChanged{JobID: id})
return nil
}
// RunDue is the scheduler's per-tick entry point: it starts whatever is due at
@@ -63,7 +68,6 @@ func (s *Service) RunNow(id int) error {
func (s *Service) RunDue(now time.Time) {
s.mu.Lock()
var started []int
var startErr error
if !s.paused {
sequential := s.store.Config.ExecutionMode == domain.ExecutionModeSequential
running := s.anyRunningLocked()
@@ -77,7 +81,7 @@ func (s *Service) RunDue(now time.Time) {
// The job came due again while its own run is still in flight.
// Apply the effective overlap policy and step past this
// occurrence.
if s.effectiveOverlapPolicy(job) == domain.OverlapPolicyQueue {
if s.effectiveOverlapPolicy(job) == domain.OverlapPolicyQueue && runtime.PendingRuns < maxPendingRuns {
runtime.PendingRuns++
}
s.advanceNextDueLocked(job, runtime, now)
@@ -88,19 +92,13 @@ func (s *Service) RunDue(now time.Time) {
// tick once the in-flight run has finished.
continue
}
if err := s.startRunLocked(job, runtime, "Schedule", now); err != nil {
startErr = err
continue
}
s.startRunLocked(job, runtime, "Schedule", now)
started = append(started, job.ID)
running = true
}
}
s.mu.Unlock()
if startErr != nil {
s.emit(ErrorOccurred{Err: fmt.Errorf("save jobs before scheduled run: %w", startErr)})
}
for _, id := range started {
s.emit(JobChanged{JobID: id})
}
@@ -116,29 +114,19 @@ type runEnv struct {
}
// startRunLocked transitions a job to "Running", advances its NextDue to the next
// scheduled occurrence, persists that, and launches the run on a background
// goroutine. Advancing (rather than zeroing) NextDue keeps the schedule marching
// while the run is in flight, which is what lets RunDue notice a fresh occurrence
// firing during a long run and apply the overlap policy. The caller must hold mu.
// now is the reference time for next-due advancement and the running placeholder.
func (s *Service) startRunLocked(job *domain.Job, runtime *domain.JobRuntime, trigger string, now time.Time) error {
// scheduled occurrence, and launches the run on a background goroutine. Neither
// step touches a durable field — both live on JobRuntime, which is never
// persisted — so there is nothing to save here. Advancing (rather than zeroing)
// NextDue keeps the schedule marching while the run is in flight, which is what
// lets RunDue notice a fresh occurrence firing during a long run and apply the
// overlap policy. The caller must hold mu. now is the reference time for
// next-due advancement and the running placeholder.
func (s *Service) startRunLocked(job *domain.Job, runtime *domain.JobRuntime, trigger string, now time.Time) {
jobCopy := *job
prevState := runtime.LastState
prevNextRun := runtime.NextRun
prevOutput := runtime.Output
prevNextDue := runtime.NextDue
runtime.LastState = "Running"
runtime.NextRun = "Running"
runtime.Output = runningOutput(jobCopy, trigger, now)
s.advanceNextDueLocked(job, runtime, now)
if err := s.store.SaveJobs(s.jobs); err != nil {
runtime.LastState = prevState
runtime.NextRun = prevNextRun
runtime.Output = prevOutput
runtime.NextDue = prevNextDue
return err
}
env := runEnv{
logsDir: s.store.Paths.LogsDir,
maxFiles: s.store.Config.MaxLogFiles,
@@ -148,7 +136,6 @@ func (s *Service) startRunLocked(job *domain.Job, runtime *domain.JobRuntime, tr
// Capture ctx under the lock so a concurrent Start/Stop cannot swap it out
// from under the goroutine after we release mu.
go s.executeRun(s.ctx, jobCopy, trigger, env)
return nil
}
// executeRun runs the job off the lock, then records the result back through the
@@ -160,7 +147,7 @@ func (s *Service) executeRun(ctx context.Context, jobCopy domain.Job, trigger st
record, logErr := s.runJob(ctx, &jobCopy, trigger, env.logsDir, env.timeout)
s.mu.Lock()
var cleanupErr, saveErr error
var cleanupErr error
var rerunStarted bool
if current := s.findByIDLocked(jobCopy.ID); current != nil {
runtime := s.runtimeForLocked(current)
@@ -174,11 +161,10 @@ func (s *Service) executeRun(ctx context.Context, jobCopy domain.Job, trigger st
runtime.PendingRuns--
// A scheduled occurrence fired while this run was active under the
// "queue" policy; start one deferred run now.
saveErr = s.startRunLocked(current, runtime, "Schedule", time.Now())
rerunStarted = saveErr == nil
s.startRunLocked(current, runtime, "Schedule", time.Now())
rerunStarted = true
} else {
s.refreshNextRunLocked(current, runtime)
saveErr = s.store.SaveJobs(s.jobs)
}
cleanupErr = runner.CleanupLogs(env.logsDir, env.maxFiles, env.maxAge)
}
@@ -190,9 +176,6 @@ func (s *Service) executeRun(ctx context.Context, jobCopy domain.Job, trigger st
if cleanupErr != nil {
s.emit(ErrorOccurred{Err: fmt.Errorf("log cleanup after run %q: %w", jobCopy.Name, cleanupErr)})
}
if saveErr != nil {
s.emit(ErrorOccurred{Err: fmt.Errorf("save jobs after run %q: %w", jobCopy.Name, saveErr)})
}
s.emit(RunRecorded{Record: record})
if !rerunStarted {
s.emit(JobChanged{JobID: jobCopy.ID})