From 7d298e05bea16efeed369db3c35e6258cd5a0d2a Mon Sep 17 00:00:00 2001 From: nxshock Date: Tue, 29 Mar 2022 21:38:04 +0500 Subject: [PATCH] Add on error restart job options --- README.md | 2 +- index.htm | 4 +- job.go | 124 ++++++++++++++++++++++++++------------------ job_test.go | 24 +++++++++ main.go | 2 +- restartrule.go | 36 +++++++++++++ restartrule_test.go | 40 ++++++++++++++ tests/job.conf | 3 ++ 8 files changed, 180 insertions(+), 55 deletions(-) create mode 100644 job_test.go create mode 100644 restartrule.go create mode 100644 restartrule_test.go diff --git a/README.md b/README.md index d9a46e4..f6d87ce 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # gron -*cron-like job scheduler* +*SystemD and cron inspired job scheduler* ## Usage diff --git a/index.htm b/index.htm index 40136b1..6518e46 100644 --- a/index.htm +++ b/index.htm @@ -207,9 +207,9 @@
{{.Name}} - {{.Description}} + {{.JobConfig.Description}} - 
{{.Cron}}
+ 
{{.JobConfig.Cron}}
{{if gt .CurrentRunningCount 1}}running {{.CurrentRunningCount}} jobs{{else}}{{if .CurrentRunningCount}}▶ running{{else}}◼ inactive{{end}}{{end}} {{.LastStartTime}} diff --git a/job.go b/job.go index b312a15..52bd66a 100644 --- a/job.go +++ b/job.go @@ -1,8 +1,7 @@ package main import ( - "bytes" - "io/ioutil" + "io" "os" "os/exec" "path/filepath" @@ -11,20 +10,24 @@ import ( "time" "github.com/BurntSushi/toml" + formatter "github.com/antonfisher/nested-logrus-formatter" log "github.com/sirupsen/logrus" ) // JobConfig is a TOML representation of job type JobConfig struct { - Cron string // cron decription - Command string // command for execution - Description string // job description + Cron string // cron decription + Command string // command for execution + Description string // job description + NumberOfRestartAttemts int + RestartSec int // the time to sleep before restarting a job (seconds) + RestartRule RestartRule // Configures whether the job shall be restarted when the job process exits } type Job struct { Name string // from filename - JobConfig + JobConfig JobConfig // Fields for stats CurrentRunningCount int @@ -52,13 +55,7 @@ func readJob(filePath string) (*Job, error) { return job, nil } -func (js *JobConfig) Write() { - buf := new(bytes.Buffer) - toml.NewEncoder(buf).Encode(*js) - ioutil.WriteFile("job.conf", buf.Bytes(), 0644) -} - -func (j *Job) CommandAndParams() (command string, params []string) { +func (j *Job) commandAndParams() (command string, params []string) { quoted := false items := strings.FieldsFunc(j.JobConfig.Command, func(r rune) bool { if r == '"' { @@ -74,51 +71,76 @@ func (j *Job) CommandAndParams() (command string, params []string) { } func (j *Job) Run() { - startTime := time.Now() + jobLogFile, _ := os.OpenFile(filepath.Join(config.LogFilesPath, j.Name+".log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + jobLogFile.WriteString("\n") - globalMutex.Lock() - j.CurrentRunningCount++ - j.LastStartTime = startTime.Format(config.TimeFormat) - globalMutex.Unlock() + logWriter := io.MultiWriter(logFile, jobLogFile) - defer jobLogFile.Close() - defer jobLogFile.WriteString("\n") + log := log.New() + log.SetFormatter(&formatter.Formatter{ + TimestampFormat: config.TimeFormat, + HideKeys: true, + NoColors: true, + TrimMessages: true}) + log.SetOutput(logWriter) + logEntry := log.WithField("job", j.Name) - l := log.New() - l.SetOutput(jobLogFile) - l.SetFormatter(log.StandardLogger().Formatter) - - log.WithField("job", j.Name).Info("started") - l.Info("started") - - command, params := j.CommandAndParams() - - cmd := exec.Command(command, params...) - cmd.Stdout = jobLogFile - cmd.Stderr = jobLogFile - jobLogFile, _ := os.OpenFile(filepath.Join(config.LogFilesPath, j.Name+".log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) - - err := cmd.Run() - if err != nil { - log.WithField("job", j.Name).Error(err.Error()) - l.WithField("job", j.Name).Error(err.Error()) + for i := 0; i < j.JobConfig.NumberOfRestartAttemts+1; i++ { + logEntry.Info("Started.") + startTime := time.Now() globalMutex.Lock() - j.LastError = err.Error() + j.CurrentRunningCount++ + j.LastStartTime = startTime.Format(config.TimeFormat) globalMutex.Unlock() - } else { + + /**/ + command, params := j.commandAndParams() + + cmd := exec.Command(command, params...) + cmd.Stdout = jobLogFile + cmd.Stderr = jobLogFile + + err := cmd.Run() + if err != nil { + logEntry.Error(err.Error()) + + globalMutex.Lock() + j.LastError = err.Error() + globalMutex.Unlock() + } else { + globalMutex.Lock() + j.LastError = "" + globalMutex.Unlock() + } + + endTime := time.Now() + logEntry.Infof("Finished (%s).", endTime.Sub(startTime).Truncate(time.Second).String()) + globalMutex.Lock() - j.LastError = "" + j.CurrentRunningCount-- + j.LastEndTime = endTime.Format(config.TimeFormat) + j.LastExecutionDuration = endTime.Sub(startTime).Truncate(time.Second).String() globalMutex.Unlock() + + if err == nil { + break + } + + if j.JobConfig.RestartRule == No || j.JobConfig.NumberOfRestartAttemts == 0 { + break + } + + if i == 0 { + logEntry.Printf("Job failed, restarting in %d seconds.", j.JobConfig.RestartSec) + } else if i+1 < j.JobConfig.NumberOfRestartAttemts { + logEntry.Printf("Retry attempt №%d of %d failed, restarting in %d seconds.", i, j.JobConfig.NumberOfRestartAttemts, j.JobConfig.RestartSec) + } else { + logEntry.Printf("Retry attempt №%d of %d failed.", i, j.JobConfig.NumberOfRestartAttemts) + } + + time.Sleep(time.Duration(j.JobConfig.RestartSec) * time.Second) + } - - endTime := time.Now() - log.WithField("job", j.Name).Infof("finished (%s)", endTime.Sub(startTime).Truncate(time.Second).String()) - l.Infof("finished (%s)", endTime.Sub(startTime).Truncate(time.Second).String()) - - globalMutex.Lock() - j.CurrentRunningCount-- - j.LastEndTime = endTime.Format(config.TimeFormat) - j.LastExecutionDuration = endTime.Sub(startTime).Truncate(time.Second).String() - globalMutex.Unlock() + jobLogFile.Close() } diff --git a/job_test.go b/job_test.go new file mode 100644 index 0000000..e305ee3 --- /dev/null +++ b/job_test.go @@ -0,0 +1,24 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestReadJob(t *testing.T) { + expectedJob := &Job{ + Name: "job", + JobConfig: JobConfig{ + Cron: "* * * * *", + Command: `command "param1 param1" param2`, + Description: "comment", + NumberOfRestartAttemts: 3, + RestartSec: 5, + RestartRule: OnError}} + + job, err := readJob("tests/job.conf") + assert.NoError(t, err) + + assert.Equal(t, expectedJob, job) +} diff --git a/main.go b/main.go index c5b9896..82335bb 100644 --- a/main.go +++ b/main.go @@ -64,7 +64,7 @@ func initJobs() error { return err } - _, err = c.AddJob(job.Cron, job) + _, err = c.AddJob(job.JobConfig.Cron, job) if err != nil { return err } diff --git a/restartrule.go b/restartrule.go new file mode 100644 index 0000000..0357d41 --- /dev/null +++ b/restartrule.go @@ -0,0 +1,36 @@ +package main + +import ( + "fmt" +) + +type RestartRule int + +const ( + No RestartRule = iota + OnError +) + +func (r *RestartRule) MarshalText() (text []byte, err error) { + switch *r { + case No: + return []byte("no"), nil + case OnError: + return []byte("on-error"), nil + } + + return nil, fmt.Errorf("unknown restart rule: %v", r) +} + +func (r *RestartRule) UnmarshalText(text []byte) error { + switch string(text) { + case "no": + *r = No + return nil + case "on-error": + *r = OnError + return nil + } + + return fmt.Errorf("unknown restart rule: %s", string(text)) +} diff --git a/restartrule_test.go b/restartrule_test.go new file mode 100644 index 0000000..1771b5e --- /dev/null +++ b/restartrule_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRestartRuleMarshalText(t *testing.T) { + var tests = []struct { + expected []byte + value RestartRule + }{ + {[]byte("no"), No}, + {[]byte("on-error"), OnError}, + } + + for _, test := range tests { + got, err := test.value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, test.expected, got) + } +} + +func TestRestartRuleUnmarshalText(t *testing.T) { + var tests = []struct { + expected RestartRule + value []byte + }{ + {No, []byte("no")}, + {OnError, []byte("on-error")}, + } + + for _, test := range tests { + var r RestartRule + err := r.UnmarshalText(test.value) + assert.NoError(t, err) + assert.Equal(t, test.expected, r) + } +} diff --git a/tests/job.conf b/tests/job.conf index 95605dd..2bfae77 100644 --- a/tests/job.conf +++ b/tests/job.conf @@ -1,3 +1,6 @@ Cron = "* * * * *" Command = 'command "param1 param1" param2' Description = "comment" +NumberOfRestartAttemts = 3 +RestartSec = 5 +RestartRule = "on-error"