README.md

@@ -205,10 +205,10 @@ command="systemctl restart foo.service",no-port-forwarding,no-x11-forwarding,no-
 <details>
   <summary>{{ .Name }} and other template variables</summary>
 
-- `{{ .Name }}` is the name of your site.
+`{{ .Name }}` is the name of your site.
-- `{{ .Message }}` is either `went down` or `came back up`.
+`{{ .Message }}` is either `went down` or `came back up`.
-- `{{ .Status }}` is either `up` or `down`.
+`{{ .Status }}` is either `up` or `down`.
-- `{{ .Watchdog }}` is the name of your watchdog (useful if you have multiple).
+`{{ .Watchdog }}` is the name of your watchdog (useful if you have multiple).
 
 It refers to the name of the watch, which is "Example Site" in the sample config below.
 

build-all.sh

@@ -5,44 +5,41 @@
 
 export CGO_ENABLED=0
 exe=watchdog
-distpre=../..
 gocmd=.
 
 echo ""
 go generate -mod=vendor ./...
 
-pushd cmd/${exe}
 echo ""
 echo "Windows amd64"
-#GOOS=windows GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/windows/amd64/${exe}.exe -ldflags "-H=windowsgui" $gocmd
+#GOOS=windows GOARCH=amd64 go build -mod=vendor -o dist/windows/amd64/${exe}.exe -ldflags "-H=windowsgui" $gocmd
-#GOOS=windows GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/windows/amd64/${exe}.debug.exe
+#GOOS=windows GOARCH=amd64 go build -mod=vendor -o dist/windows/amd64/${exe}.debug.exe
-GOOS=windows GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/windows/amd64/${exe}.exe
+GOOS=windows GOARCH=amd64 go build -mod=vendor -o dist/windows/amd64/${exe}.exe
 echo "Windows 386"
-#GOOS=windows GOARCH=386 go build -mod=vendor -o ${distpre}/dist/windows/386/${exe}.exe -ldflags "-H=windowsgui" $gocmd
+#GOOS=windows GOARCH=386 go build -mod=vendor -o dist/windows/386/${exe}.exe -ldflags "-H=windowsgui" $gocmd
-#GOOS=windows GOARCH=386 go build -mod=vendor -o ${distpre}/dist/windows/386/${exe}.debug.exe
+#GOOS=windows GOARCH=386 go build -mod=vendor -o dist/windows/386/${exe}.debug.exe
-GOOS=windows GOARCH=386 go build -mod=vendor -o ${distpre}/dist/windows/386/${exe}.exe
+GOOS=windows GOARCH=386 go build -mod=vendor -o dist/windows/386/${exe}.exe
 
 echo ""
 echo "Darwin (macOS) amd64"
-GOOS=darwin GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/darwin/amd64/${exe} $gocmd
+GOOS=darwin GOARCH=amd64 go build -mod=vendor -o dist/darwin/amd64/${exe} $gocmd
 
 echo ""
 echo "Linux amd64"
-GOOS=linux GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/linux/amd64/${exe} $gocmd
+GOOS=linux GOARCH=amd64 go build -mod=vendor -o dist/linux/amd64/${exe} $gocmd
 echo "Linux 386"
-GOOS=linux GOARCH=386 go build -mod=vendor -o ${distpre}/dist/linux/386/${exe} $gocmd
+GOOS=linux GOARCH=386 go build -mod=vendor -o dist/linux/386/${exe} $gocmd
 
 echo ""
 echo "RPi 4 (64-bit) ARMv8"
-GOOS=linux GOARCH=arm64 go build -mod=vendor -o ${distpre}/dist/linux/armv8/${exe} $gocmd
+GOOS=linux GOARCH=arm64 go build -mod=vendor -o dist/linux/armv8/${exe} $gocmd
 echo "RPi 3 B+ ARMv7"
-GOOS=linux GOARCH=arm GOARM=7 go build -mod=vendor -o ${distpre}/dist/linux/armv7/${exe} $gocmd
+GOOS=linux GOARCH=arm GOARM=7 go build -mod=vendor -o dist/linux/armv7/${exe} $gocmd
 echo "ARMv6"
-GOOS=linux GOARCH=arm GOARM=6 go build -mod=vendor -o ${distpre}/dist/linux/armv6/${exe} $gocmd
+GOOS=linux GOARCH=arm GOARM=6 go build -mod=vendor -o dist/linux/armv6/${exe} $gocmd
 echo "RPi Zero ARMv5"
-GOOS=linux GOARCH=arm GOARM=5 go build -mod=vendor -o ${distpre}/dist/linux/armv5/${exe} $gocmd
+GOOS=linux GOARCH=arm GOARM=5 go build -mod=vendor -o dist/linux/armv5/${exe} $gocmd
 
 echo ""
-popd
 rsync -av ./dist/ ubuntu@rootprojects.org:/srv/www/rootprojects.org/$exe/dist/
 # https://rootprojects.org/serviceman/dist/windows/amd64/serviceman.exe

watchdog.go

@@ -33,12 +33,6 @@ func (s Status) String() string {
 	}
 }
 
-const (
-	MessageDown   = "went down"
-	MessageUp     = "came back up"
-	MessageHiccup = "hiccupped"
-)
-
 type Dog struct {
 	Watchdog      string
 	Name          string
@@ -53,15 +47,15 @@ type Dog struct {
 	status        Status
 	changed       bool
 	error         error
-	//failures      int
+	failures      int
-	//passes        int
+	passes        int
-	//lastFailed    time.Time
+	lastFailed    time.Time
-	//lastPassed    time.Time
+	lastPassed    time.Time
-	//lastNotified time.Time
+	lastNotified  time.Time
 }
 
 func New(d *Dog) *Dog {
-	//d.lastPassed = time.Now().Add(-5 * time.Minute)
+	d.lastPassed = time.Now().Add(-5 * time.Minute)
 	d.status = StatusUp
 	d.changed = false
 	return d
@@ -76,87 +70,88 @@ func (d *Dog) Watch() {
 	}
 }
 
-// Now that I've added the ability to notify when a server is back up
-// this definitely needs some refactoring. It's bad now.
 func (d *Dog) watch() {
 	d.Logger <- fmt.Sprintf("Check: '%s'", d.Name)
 
-	// This may be up or down
+	err := d.check()
-	err := d.hardcheck()
 	if nil == err {
-		d.Logger <- fmt.Sprintf("Up: '%s'", d.Name)
-		// if it's down, coming up, notify
 		if d.changed {
-			d.notify(MessageUp)
+			d.notify("came back up")
 		}
 		return
 	}
 
-	// If being down is a change, check to see if it's just a hiccup
-	if d.changed {
 	time.Sleep(time.Duration(5) * time.Second)
-		err2 := d.softcheck()
+
+	err2 := d.check()
 	if nil != err2 {
-			// it's really down
 		d.Logger <- fmt.Sprintf("Down: '%s': %s", d.Name, err2)
 	} else {
-			// it's not really down, so reset the change info
-			d.changed = false
-			d.status = StatusUp
-			// and notify of the hiccup
 		d.Logger <- fmt.Sprintf("Hiccup: '%s': %s", d.Name, err)
-			d.notify(MessageHiccup)
 		return
 	}
+
+	t := 10
+	for {
+		d.recover()
+		time.Sleep(time.Duration(t) * time.Second)
+		// backoff
+		t *= 2
+		err := d.check()
+		if nil != err {
+			d.Logger <- fmt.Sprintf("Unrecoverable: '%s': %s", d.Name, err)
 		}
 
+		// We should notify if
+		// * The status has changed
+		//
 		// TODO what if the server is flip-flopping rapidly?
 		// how to rate limit?
 		// "{{ .Server }} is on cooldown for 30 minutes"
+		if d.changed {
+			d.notify("went down")
+			if StatusUp == d.status {
+				break
+			}
 
 			// * We've had success since the last notification
 			// * It's been at least 5 minutes since the last notification
 			//fiveMinutesAgo := time.Now().Add(-5 * time.Minute)
 			//if d.lastPassed.After(d.lastNotified) && d.lastNotified.Before(fiveMinutesAgo) {
 			//}
-
+			//if !failure || d.failures >= 5 {
-	t := 10
+			// go back to the main 5-minute loop
-	for {
+			//	break
-		// try to recover, then backoff exponentially
+			//}
-		d.recover()
+		}
-		time.Sleep(time.Duration(t) * time.Second)
+	}
-		t *= 2
-		if t > 120 {
-			t = 120
 }
 
-		err := d.softcheck()
+func (d *Dog) check() error {
+	previousStatus := d.status
+
+	var err error
+	defer func() {
+		// Are we up, or down?
 		if nil != err {
-			// this is down, and we know it's down
 			d.status = StatusDown
-			d.Logger <- fmt.Sprintf("Unrecoverable: '%s': %s", d.Name, err)
+			d.failures += 1
-			if d.changed {
+			d.lastFailed = time.Now()
-				d.changed = false
-				d.notify(MessageDown)
-			}
 		} else {
-			// it came back up
 			d.status = StatusUp
+			d.lastPassed = time.Now()
+			d.passes += 1
 			d.Logger <- fmt.Sprintf("Up: '%s'", d.Name)
-			if d.changed {
-				// and the downtime was short - just a recovery
-				d.notify(MessageHiccup)
-			} else {
-				// and the downtime was some time
-				d.notify(MessageUp)
-			}
-			d.changed = false
-			break
-		}
-	}
 		}
 
-func (d *Dog) softcheck() error {
+		// Has that changed?
+		if previousStatus != d.status {
+			d.changed = true
+		} else {
+			d.changed = false
+		}
+	}()
+
 	client := NewHTTPClient()
 	response, err := client.Get(d.CheckURL)
 	if nil != err {
@@ -179,7 +174,7 @@ func (d *Dog) softcheck() error {
 	}
 
 	if "" != d.Badwords {
-		if bytes.Contains(b, []byte(d.Badwords)) {
+		if !bytes.Contains(b, []byte(d.Badwords)) {
 			err = fmt.Errorf("Down: '%s' Found for '%s'", d.Badwords, d.Name)
 			d.Logger <- fmt.Sprintf("%s", err)
 			d.error = err
@@ -190,32 +185,6 @@ func (d *Dog) softcheck() error {
 	return nil
 }
 
-func (d *Dog) hardcheck() error {
-	previousStatus := d.status
-
-	err := d.softcheck()
-
-	// Are we up, or down?
-	if nil != err {
-		d.status = StatusDown
-		//d.failures += 1
-		//d.lastFailed = time.Now()
-	} else {
-		d.status = StatusUp
-		//d.lastPassed = time.Now()
-		//d.passes += 1
-	}
-
-	// Has that changed?
-	if previousStatus != d.status {
-		d.changed = true
-	} else {
-		d.changed = false
-	}
-
-	return err
-}
-
 func (d *Dog) recover() {
 	if "" == d.Recover {
 		return
@@ -245,7 +214,7 @@ func (d *Dog) recover() {
 
 func (d *Dog) notify(msg string) {
 	d.Logger <- fmt.Sprintf("Notifying the authorities of %s's status change", d.Name)
-	//d.lastNotified = time.Now()
+	d.lastNotified = time.Now()
 
 	for i := range d.Webhooks {
 		name := d.Webhooks[i]