Add on error restart job options

README.md

@@ -1,6 +1,6 @@
 # gron
 
-*cron-like job scheduler*
+*SystemD and cron inspired job scheduler*
 
 ## Usage
 

index.htm

@@ -207,9 +207,9 @@
 					<form action="/start" method="get" id="form-{{.Name}}"></form>
 					<button{{if gt .CurrentRunningCount 0}} class="runningbg" {{else}}{{if .LastError}} class="errorbg" {{end}}{{end}} type="submit" form="form-{{.Name}}" name="jobName" value="{{.Name}}" {{if gt .CurrentRunningCount 0}} disabled{{end}}>{{.Name}}</button>
 				</td>
-				<td class="smaller">{{.Description}}</td>
+				<td class="smaller">{{.JobConfig.Description}}</td>
 				<td class="nowrap" align="right">
-					<pre>{{.Cron}}</pre>
+					<pre>{{.JobConfig.Cron}}</pre>
 				</td>
 				<td class="nowrap">{{if gt .CurrentRunningCount 1}}<span class="red">running {{.CurrentRunningCount}} jobs</span>{{else}}{{if .CurrentRunningCount}}<span class="green">&#x25b6; running</span>{{else}}&#x25fc; inactive{{end}}{{end}}</td>
 				<td>{{.LastStartTime}}</td>

job.go

@@ -1,8 +1,7 @@
 package main
 
 import (
-	"bytes"
+	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -11,20 +10,24 @@ import (
 	"time"
 
 	"github.com/BurntSushi/toml"
+	formatter "github.com/antonfisher/nested-logrus-formatter"
 	log "github.com/sirupsen/logrus"
 )
 
 // JobConfig is a TOML representation of job
 type JobConfig struct {
-	Cron        string // cron decription
+	Cron                   string // cron decription
-	Command     string // command for execution
+	Command                string // command for execution
-	Description string // job description
+	Description            string // job description
+	NumberOfRestartAttemts int
+	RestartSec             int         // the time to sleep before restarting a job (seconds)
+	RestartRule            RestartRule // Configures whether the job shall be restarted when the job process exits
 }
 
 type Job struct {
 	Name string // from filename
 
-	JobConfig
+	JobConfig JobConfig
 
 	// Fields for stats
 	CurrentRunningCount   int
@@ -52,13 +55,7 @@ func readJob(filePath string) (*Job, error) {
 	return job, nil
 }
 
-func (js *JobConfig) Write() {
+func (j *Job) commandAndParams() (command string, params []string) {
-	buf := new(bytes.Buffer)
-	toml.NewEncoder(buf).Encode(*js)
-	ioutil.WriteFile("job.conf", buf.Bytes(), 0644)
-}
-
-func (j *Job) CommandAndParams() (command string, params []string) {
 	quoted := false
 	items := strings.FieldsFunc(j.JobConfig.Command, func(r rune) bool {
 		if r == '"' {
@@ -74,51 +71,76 @@ func (j *Job) CommandAndParams() (command string, params []string) {
 }
 
 func (j *Job) Run() {
-	startTime := time.Now()
+	jobLogFile, _ := os.OpenFile(filepath.Join(config.LogFilesPath, j.Name+".log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
+	jobLogFile.WriteString("\n")
 
-	globalMutex.Lock()
+	logWriter := io.MultiWriter(logFile, jobLogFile)
-	j.CurrentRunningCount++
-	j.LastStartTime = startTime.Format(config.TimeFormat)
-	globalMutex.Unlock()
 
-	defer jobLogFile.Close()
+	log := log.New()
-	defer jobLogFile.WriteString("\n")
+	log.SetFormatter(&formatter.Formatter{
+		TimestampFormat: config.TimeFormat,
+		HideKeys:        true,
+		NoColors:        true,
+		TrimMessages:    true})
+	log.SetOutput(logWriter)
+	logEntry := log.WithField("job", j.Name)
 
-	l := log.New()
+	for i := 0; i < j.JobConfig.NumberOfRestartAttemts+1; i++ {
-	l.SetOutput(jobLogFile)
+		logEntry.Info("Started.")
-	l.SetFormatter(log.StandardLogger().Formatter)
+		startTime := time.Now()
-
-	log.WithField("job", j.Name).Info("started")
-	l.Info("started")
-
-	command, params := j.CommandAndParams()
-
-	cmd := exec.Command(command, params...)
-	cmd.Stdout = jobLogFile
-	cmd.Stderr = jobLogFile
-		jobLogFile, _ := os.OpenFile(filepath.Join(config.LogFilesPath, j.Name+".log"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
-
-	err := cmd.Run()
-	if err != nil {
-		log.WithField("job", j.Name).Error(err.Error())
-		l.WithField("job", j.Name).Error(err.Error())
 
 		globalMutex.Lock()
-		j.LastError = err.Error()
+		j.CurrentRunningCount++
+		j.LastStartTime = startTime.Format(config.TimeFormat)
 		globalMutex.Unlock()
-	} else {
+
+		/**/
+		command, params := j.commandAndParams()
+
+		cmd := exec.Command(command, params...)
+		cmd.Stdout = jobLogFile
+		cmd.Stderr = jobLogFile
+
+		err := cmd.Run()
+		if err != nil {
+			logEntry.Error(err.Error())
+
+			globalMutex.Lock()
+			j.LastError = err.Error()
+			globalMutex.Unlock()
+		} else {
+			globalMutex.Lock()
+			j.LastError = ""
+			globalMutex.Unlock()
+		}
+
+		endTime := time.Now()
+		logEntry.Infof("Finished (%s).", endTime.Sub(startTime).Truncate(time.Second).String())
+
 		globalMutex.Lock()
-		j.LastError = ""
+		j.CurrentRunningCount--
+		j.LastEndTime = endTime.Format(config.TimeFormat)
+		j.LastExecutionDuration = endTime.Sub(startTime).Truncate(time.Second).String()
 		globalMutex.Unlock()
+
+		if err == nil {
+			break
+		}
+
+		if j.JobConfig.RestartRule == No || j.JobConfig.NumberOfRestartAttemts == 0 {
+			break
+		}
+
+		if i == 0 {
+			logEntry.Printf("Job failed, restarting in %d seconds.", j.JobConfig.RestartSec)
+		} else if i+1 < j.JobConfig.NumberOfRestartAttemts {
+			logEntry.Printf("Retry attempt №%d of %d failed, restarting in %d seconds.", i, j.JobConfig.NumberOfRestartAttemts, j.JobConfig.RestartSec)
+		} else {
+			logEntry.Printf("Retry attempt №%d of %d failed.", i, j.JobConfig.NumberOfRestartAttemts)
+		}
+
+		time.Sleep(time.Duration(j.JobConfig.RestartSec) * time.Second)
+
 	}
-
+	jobLogFile.Close()
-	endTime := time.Now()
-	log.WithField("job", j.Name).Infof("finished (%s)", endTime.Sub(startTime).Truncate(time.Second).String())
-	l.Infof("finished (%s)", endTime.Sub(startTime).Truncate(time.Second).String())
-
-	globalMutex.Lock()
-	j.CurrentRunningCount--
-	j.LastEndTime = endTime.Format(config.TimeFormat)
-	j.LastExecutionDuration = endTime.Sub(startTime).Truncate(time.Second).String()
-	globalMutex.Unlock()
 }

job_test.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReadJob(t *testing.T) {
+	expectedJob := &Job{
+		Name: "job",
+		JobConfig: JobConfig{
+			Cron:                   "* * * * *",
+			Command:                `command "param1 param1" param2`,
+			Description:            "comment",
+			NumberOfRestartAttemts: 3,
+			RestartSec:             5,
+			RestartRule:            OnError}}
+
+	job, err := readJob("tests/job.conf")
+	assert.NoError(t, err)
+
+	assert.Equal(t, expectedJob, job)
+}

main.go

@@ -64,7 +64,7 @@ func initJobs() error {
 				return err
 			}
 
-			_, err = c.AddJob(job.Cron, job)
+			_, err = c.AddJob(job.JobConfig.Cron, job)
 			if err != nil {
 				return err
 			}

restartrule.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"fmt"
+)
+
+type RestartRule int
+
+const (
+	No RestartRule = iota
+	OnError
+)
+
+func (r *RestartRule) MarshalText() (text []byte, err error) {
+	switch *r {
+	case No:
+		return []byte("no"), nil
+	case OnError:
+		return []byte("on-error"), nil
+	}
+
+	return nil, fmt.Errorf("unknown restart rule: %v", r)
+}
+
+func (r *RestartRule) UnmarshalText(text []byte) error {
+	switch string(text) {
+	case "no":
+		*r = No
+		return nil
+	case "on-error":
+		*r = OnError
+		return nil
+	}
+
+	return fmt.Errorf("unknown restart rule: %s", string(text))
+}

restartrule_test.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRestartRuleMarshalText(t *testing.T) {
+	var tests = []struct {
+		expected []byte
+		value    RestartRule
+	}{
+		{[]byte("no"), No},
+		{[]byte("on-error"), OnError},
+	}
+
+	for _, test := range tests {
+		got, err := test.value.MarshalText()
+		assert.NoError(t, err)
+		assert.Equal(t, test.expected, got)
+	}
+}
+
+func TestRestartRuleUnmarshalText(t *testing.T) {
+	var tests = []struct {
+		expected RestartRule
+		value    []byte
+	}{
+		{No, []byte("no")},
+		{OnError, []byte("on-error")},
+	}
+
+	for _, test := range tests {
+		var r RestartRule
+		err := r.UnmarshalText(test.value)
+		assert.NoError(t, err)
+		assert.Equal(t, test.expected, r)
+	}
+}

tests/job.conf

@@ -1,3 +1,6 @@
 Cron = "* * * * *"
 Command = 'command "param1 param1" param2'
 Description = "comment"
+NumberOfRestartAttemts = 3
+RestartSec = 5
+RestartRule = "on-error"