v1.2.2: improved change detection

.gitignore

@@ -1,3 +1,4 @@
 /watchdog
 /cmd/watchdog/watchdog
 xversion.go
+*.json

README.md

@@ -11,7 +11,85 @@ Can work with email, text (sms), push notifications, etc.
 
 # Install
 
-Git:
+## Downloads
+
+### MacOS
+
+MacOS (darwin): [64-bit Download ](https://rootprojects.org/watchdog/dist/darwin/amd64/watchdog)
+
+```
+curl https://rootprojects.org/watchdog/dist/darwin/amd64/watchdog -o watchdog
+```
+
+### Windows
+
+<details>
+<summary>See download options</summary>
+Windows 10: [64-bit Download](https://rootprojects.org/watchdog/dist/windows/amd64/watchdog.exe)
+
+```
+powershell.exe $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest https://rootprojects.org/watchdog/dist/windows/amd64/watchdog.exe -OutFile watchdog.exe
+```
+
+Windows 7: [32-bit Download](https://rootprojects.org/watchdog/dist/windows/386/watchdog.exe)
+
+```
+powershell.exe "(New-Object Net.WebClient).DownloadFile('https://rootprojects.org/watchdog/dist/windows/386/watchdog.exe', 'watchdog.exe')"
+```
+
+</details>
+
+### Linux
+
+<details>
+<summary>See download options</summary>
+
+Linux (64-bit): [Download](https://rootprojects.org/watchdog/dist/linux/amd64/watchdog)
+
+```
+curl https://rootprojects.org/watchdog/dist/linux/amd64/watchdog -o watchdog
+```
+
+Linux (32-bit): [Download](https://rootprojects.org/watchdog/dist/linux/386/watchdog)
+
+```
+curl https://rootprojects.org/watchdog/dist/linux/386/watchdog -o watchdog
+```
+
+</details>
+
+### Raspberry Pi (Linux ARM)
+
+<details>
+<summary>See download options</summary>
+
+RPi 4 (64-bit armv8): [Download](https://rootprojects.org/watchdog/dist/linux/armv8/watchdog)
+
+```
+curl https://rootprojects.org/watchdog/dist/linux/armv8/watchdog -o watchdog`
+```
+
+RPi 3 (armv7): [Download](https://rootprojects.org/watchdog/dist/linux/armv7/watchdog)
+
+```
+curl https://rootprojects.org/watchdog/dist/linux/armv7/watchdog -o watchdog
+```
+
+ARMv6: [Download](https://rootprojects.org/watchdog/dist/linux/armv6/watchdog)
+
+```
+curl https://rootprojects.org/watchdog/dist/linux/armv6/watchdog -o watchdog
+```
+
+RPi Zero (armv5): [Download](https://rootprojects.org/watchdog/dist/linux/armv5/watchdog)
+
+```
+curl https://rootprojects.org/watchdog/dist/linux/armv5/watchdog -o watchdog
+```
+
+</details>
+
+## Git:
 
 ```bash
 git clone https://git.coolaj86.com/coolaj86/watchdog.go.git
@@ -21,19 +99,6 @@ pushd cmd/watchdog
 go build -mod=vendor
 ```
 
-Zip:
-
-- Linux
-  - [watchdog-v1.1.0-linux-amd64.zip](https://git.rootprojects.org/root/watchdog.go/releases/download/v1.1.0-linux/watchdog-v1.1.0-linux-amd64.zip)
-  - [watchdog-v1.1.0-linux-386.zip](https://git.rootprojects.org/root/watchdog.go/releases/download/v1.1.0-linux/watchdog-v1.1.0-linux-386.zip)
-  - [watchdog-v1.1.0-linux-armv7.zip](https://git.rootprojects.org/root/watchdog.go/releases/download/v1.1.0-linux/watchdog-v1.1.0-linux-armv7.zip)
-  - [watchdog-v1.1.0-linux-armv5.zip](https://git.rootprojects.org/root/watchdog.go/releases/download/v1.1.0-linux/watchdog-v1.1.0-linux-armv5.zip)
-- MacOS
-  - [watchdog-v1.1.0-darwin-amd64.zip](https://git.rootprojects.org/root/watchdog.go/releases/download/v1.1.0-macos/watchdog-v1.1.0-darwin-amd64.zip)
-- Windows
-  - [watchdog-v1.1.0-windows-amd64.zip](https://git.rootprojects.org/root/watchdog.go/releases/download/v1.1.0-windows/watchdog-v1.1.0-windows-amd64.zip)
-  - [watchdog-v1.1.0-windows-386.zip](https://git.rootprojects.org/root/watchdog.go/releases/download/v1.1.0-windows/watchdog-v1.1.0-windows-386.zip)
-
 # Usage
 
 Mac, Linux:
@@ -48,6 +113,15 @@ Windows:
 watchdog.exe -c config.json
 ```
 
+# Changelog
+
+- v1.2.0
+  - report when sites come back up
+  - and more template vars
+  - and localization for status
+- v1.1.0 support `json` request bodies (for Pushbullet)
+- v1.0.0 support Twilio and Mailgun
+
 # Getting Started
 
 <details>
@@ -78,6 +152,23 @@ Be careful of "smart quotes" and HTML entities:
 - `We’re Open!` is not `We're Open!`
 - Neither is `We&apos;re Open!` nor `We&#39;re Open!`
 
+Leave empty for No Content pages, such as redirects.
+
+### `badwords`
+
+The opposite of `keywords`.
+
+If a literal, exact match of badwords exists as part of the response, the site is considered to be down.
+
+Ignored if empty.
+
+### `localizations`
+
+Normally `{{ .Status }}` will be `"up"` or `"down"` and `{{ .Message }}` will be `"is down"` or `"came back up"`.
+Localizations allow you to swap that out for something else.
+
+I added this so that I could use "🔥🔥🔥" and "👍" for myself without imposing upon others.
+
 ### `webhooks`
 
 This references the arbitrary `name` of a webhook in the `webhooks` array.
@@ -114,7 +205,10 @@ command="systemctl restart foo.service",no-port-forwarding,no-x11-forwarding,no-
 <details>
   <summary>{{ .Name }} and other template variables</summary>
 
-`{{ .Name }}` is the only template variable right now.
+- `{{ .Name }}` is the name of your site.
+- `{{ .Message }}` is either `went down` or `came back up`.
+- `{{ .Status }}` is either `up` or `down`.
+- `{{ .Watchdog }}` is the name of your watchdog (useful if you have multiple).
 
 It refers to the name of the watch, which is "Example Site" in the sample config below.
 
@@ -234,11 +328,13 @@ The examples below are shown with Mailgun, Pushbullet, and Twilio, as taken from
 
 ```json
 {
+  "watchdog": "Monitor A",
   "watches": [
     {
       "name": "Example Site",
       "url": "https://example.com/",
       "keywords": "My Site",
+      "badwords": "Could not connect to database.",
       "webhooks": ["my_mailgun", "my_pushbullet", "my_twilio"],
       "recover_script": "systemctl restart example-site"
     }
@@ -258,8 +354,8 @@ The examples below are shown with Mailgun, Pushbullet, and Twilio, as taken from
       "form": {
         "from": "Watchdog <watchdog@my.example.com>",
         "to": "jon.doe@gmail.com",
-        "subject": "{{ .Name }} is down.",
+        "subject": "[{{ .Watchdog }}] {{ .Name }} {{ .Message }}.",
-        "text": "The system is down. Check up on {{ .Name }} ASAP."
+        "text": "{{ .Name }} {{ .Message }}. Reported by {{ .Watchdog }}."
       }
     },
     {
@@ -271,8 +367,8 @@ The examples below are shown with Mailgun, Pushbullet, and Twilio, as taken from
         "User-Agent": "Watchdog/1.0"
       },
       "json": {
-        "body": "The system is down. Check up on {{ .Name }} ASAP.",
+        "body": "The system {{ .Message }}. Check up on {{ .Name }} ASAP.",
-        "title": "{{ .Name }} is down.",
+        "title": "{{ .Name }} {{ .Message }}.",
         "type": "note"
       }
     },
@@ -293,7 +389,11 @@ The examples below are shown with Mailgun, Pushbullet, and Twilio, as taken from
         "Body": "[{{ .Name }}] The system is down. The system is down."
       }
     }
-  ]
+  ],
+  "localizations": {
+    "up": "👍",
+    "down": "🔥🔥🔥"
+  }
 }
 ```
 

build-all.sh

@@ -0,0 +1,48 @@
+#GOOS=windows GOARCH=amd64 go install
+#go tool dist list
+
+# TODO move this into tools/build.go
+
+export CGO_ENABLED=0
+exe=watchdog
+distpre=../..
+gocmd=.
+
+echo ""
+go generate -mod=vendor ./...
+
+pushd cmd/${exe}
+echo ""
+echo "Windows amd64"
+#GOOS=windows GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/windows/amd64/${exe}.exe -ldflags "-H=windowsgui" $gocmd
+#GOOS=windows GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/windows/amd64/${exe}.debug.exe
+GOOS=windows GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/windows/amd64/${exe}.exe
+echo "Windows 386"
+#GOOS=windows GOARCH=386 go build -mod=vendor -o ${distpre}/dist/windows/386/${exe}.exe -ldflags "-H=windowsgui" $gocmd
+#GOOS=windows GOARCH=386 go build -mod=vendor -o ${distpre}/dist/windows/386/${exe}.debug.exe
+GOOS=windows GOARCH=386 go build -mod=vendor -o ${distpre}/dist/windows/386/${exe}.exe
+
+echo ""
+echo "Darwin (macOS) amd64"
+GOOS=darwin GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/darwin/amd64/${exe} $gocmd
+
+echo ""
+echo "Linux amd64"
+GOOS=linux GOARCH=amd64 go build -mod=vendor -o ${distpre}/dist/linux/amd64/${exe} $gocmd
+echo "Linux 386"
+GOOS=linux GOARCH=386 go build -mod=vendor -o ${distpre}/dist/linux/386/${exe} $gocmd
+
+echo ""
+echo "RPi 4 (64-bit) ARMv8"
+GOOS=linux GOARCH=arm64 go build -mod=vendor -o ${distpre}/dist/linux/armv8/${exe} $gocmd
+echo "RPi 3 B+ ARMv7"
+GOOS=linux GOARCH=arm GOARM=7 go build -mod=vendor -o ${distpre}/dist/linux/armv7/${exe} $gocmd
+echo "ARMv6"
+GOOS=linux GOARCH=arm GOARM=6 go build -mod=vendor -o ${distpre}/dist/linux/armv6/${exe} $gocmd
+echo "RPi Zero ARMv5"
+GOOS=linux GOARCH=arm GOARM=5 go build -mod=vendor -o ${distpre}/dist/linux/armv5/${exe} $gocmd
+
+echo ""
+popd
+rsync -av ./dist/ ubuntu@rootprojects.org:/srv/www/rootprojects.org/$exe/dist/
+# https://rootprojects.org/serviceman/dist/windows/amd64/serviceman.exe

build.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-export CGO_ENABLED=0
-#GOOS=windows GOARCH=amd64 go install
-go tool dist list
-
-gocmd=watchdog.go
-golib=""
-echo ""
-
-echo ""
-echo "Windows amd64"
-GOOS=windows GOARCH=amd64 go build -o dist/windows-amd64/watchdog.exe $gocmd $golib
-echo "Windows 386"
-GOOS=windows GOARCH=386 go build -o dist/windows-386/watchdog.exe $gocmd $golib
-
-echo ""
-echo "Darwin (macOS) amd64"
-GOOS=darwin GOARCH=amd64 go build -o dist/darwin-amd64/watchdog $gocmd $golib
-
-echo ""
-echo "Linux amd64"
-GOOS=linux GOARCH=amd64 go build -o dist/linux-amd64/watchdog $gocmd $golib
-echo "Linux 386"
-
-echo ""
-GOOS=linux GOARCH=386 go build -o dist/linux-386/watchdog $gocmd $golib
-echo "RPi 3 B+ ARMv7"
-GOOS=linux GOARCH=arm GOARM=7 go build -o dist/linux-armv7/watchdog $gocmd $golib
-echo "RPi Zero ARMv5"
-GOOS=linux GOARCH=arm GOARM=5 go build -o dist/linux-armv5/watchdog $gocmd $golib
-
-my_ver=$(git describe --tags)
-pushd dist
-  ls -d *-* | while read my_dist
-  do
-    if [ -d "$my_dist" ]; then
-      #tar -czvf watchdog-$my_ver-$my_dist.tar.gz $my_dist
-      zip -r watchdog-$my_ver-$my_dist.zip $my_dist
-    fi
-  done
-popd
-
-echo ""
-echo ""

cmd/watchdog/version.go

@@ -0,0 +1,8 @@
+package main
+
+// Fallback to recent version if not in a git repository
+func init() {
+	GitRev = "d5c026948cf134997c7260e78d4bd5864ac5b9b3"
+	GitVersion = "v1.1.3"
+	GitTimestamp = "2019-06-21T01:03:19-06:00"
+}

cmd/watchdog/watchdog.go

@@ -11,14 +11,10 @@ import (
 	"os"
 	"strings"
 
-	watchdog "git.rootprojects.org/root/watchdog.go"
+	watchdog "git.rootprojects.org/root/go-watchdog"
 )
 
-var (
+var GitRev, GitVersion, GitTimestamp string
-	GitRev       = "00000000"
-	GitVersion   = "v0.0.0"
-	GitTimestamp = "0000-00-00T00:00:00Z"
-)
 
 func usage() {
 	fmt.Println("Usage: watchdog -c config.json")
@@ -73,7 +69,7 @@ func main() {
 
 	done := make(chan struct{}, 1)
 
-	allWebhooks := make(map[string]watchdog.ConfigWebhook)
+	allWebhooks := make(map[string]watchdog.Webhook)
 
 	for i := range config.Webhooks {
 		h := config.Webhooks[i]
@@ -87,13 +83,16 @@ func main() {
 		logQueue <- fmt.Sprintf("Watching '%s'", c.Name)
 		go func(c watchdog.ConfigWatch) {
 			d := watchdog.New(&watchdog.Dog{
-				Name:        c.Name,
+				Watchdog:      config.Watchdog,
-				CheckURL:    c.URL,
+				Name:          c.Name,
-				Keywords:    c.Keywords,
+				CheckURL:      c.URL,
-				Recover:     c.RecoverScript,
+				Keywords:      c.Keywords,
-				Webhooks:    c.Webhooks,
+				Badwords:      c.Badwords,
-				AllWebhooks: allWebhooks,
+				Localizations: config.Localizations,
-				Logger:      logQueue,
+				Recover:       c.RecoverScript,
+				Webhooks:      c.Webhooks,
+				AllWebhooks:   allWebhooks,
+				Logger:        logQueue,
 			})
 			d.Watch()
 		}(config.Watches[i])

doc.go

@@ -5,5 +5,5 @@
 // The git tag version describes the state of the binary,
 // not the state of the library. The API is not yet stable.
 //
-// See https://git.rootproject.org/root/watchdog.go for pre-built binaries.
+// See https://git.rootproject.org/root/go-watchdog for pre-built binaries.
 package watchdog

go.mod

@@ -1,5 +1,5 @@
-module git.rootprojects.org/root/watchdog.go
+module git.rootprojects.org/root/go-watchdog
 
 go 1.12
 
-require git.rootprojects.org/root/go-gitver v1.1.0
+require git.rootprojects.org/root/go-gitver v1.1.1

go.sum

@@ -1,2 +1,2 @@
-git.rootprojects.org/root/go-gitver v1.1.0 h1:ANQUnUXYgbDR+WaMcI+PQQjLnxlCbAZCD/zivkrf8fY=
+git.rootprojects.org/root/go-gitver v1.1.1 h1:5b0lxnTYnft5hqpln0XCrJaGPH0SKzhPaazVAvAlZ8I=
-git.rootprojects.org/root/go-gitver v1.1.0/go.mod h1:Rj1v3TBhvdaSphFEqMynUYwAz/4f+wY/+syBTvRrmlI=
+git.rootprojects.org/root/go-gitver v1.1.1/go.mod h1:Rj1v3TBhvdaSphFEqMynUYwAz/4f+wY/+syBTvRrmlI=

vendor/git.rootprojects.org/root/go-gitver/README.md

@@ -60,7 +60,7 @@ You don't have to use `mod vendor`, but I highly recommend it.
 
 # Options
 
-```
+```txt
 version           print version and exit
 --fail            exit with non-zero status code on failure
 --package <name>  will set the package name
@@ -69,7 +69,7 @@ version           print version and exit
 
 ENVs
 
-```
+```bash
 # Alias for --fail
 GITVER_FAIL=true
 ```
@@ -142,7 +142,7 @@ the repository in itself and that would be... weird.
 
 # Why a tools package?
 
->     import "git.rootprojects.org/root/go-gitver" is a program, not an importable package
+> import "git.rootprojects.org/root/go-gitver" is a program, not an importable package
 
 Having a tools package with a build tag that you don't use is a nice way to add exact
 versions of a command package used for tooling to your `go.mod` with `go mod tidy`,

vendor/git.rootprojects.org/root/go-gitver/gitver.go

@@ -209,9 +209,9 @@ package {{ .Package }}
 
 func init() {
 	GitRev = "{{ .GitRev }}"
-	if "" != "{{ .Version }}" {
+  {{- if .Version }}
-		GitVersion = "{{ .Version }}"
+	GitVersion = "{{ .Version }}"
-	}
+	{{ end -}}
 	GitTimestamp = "{{ .Timestamp }}"
 }
 `))

vendor/modules.txt

@@ -1,2 +1,2 @@
-# git.rootprojects.org/root/go-gitver v1.1.0
+# git.rootprojects.org/root/go-gitver v1.1.1
 git.rootprojects.org/root/go-gitver

watchdog.go

@@ -14,24 +14,56 @@ import (
 	"time"
 )
 
+type Status int
+
+const (
+	StatusDown Status = iota
+	StatusUp
+)
+
+func (s Status) String() string {
+	// ... just wishing Go had enums like Rust...
+	switch s {
+	case StatusUp:
+		return "up"
+	case StatusDown:
+		return "down"
+	default:
+		return "[[internal error]]"
+	}
+}
+
+const (
+	MessageDown   = "went down"
+	MessageUp     = "came back up"
+	MessageHiccup = "hiccupped"
+)
+
 type Dog struct {
-	Name         string
+	Watchdog      string
-	CheckURL     string
+	Name          string
-	Keywords     string
+	CheckURL      string
-	Recover      string
+	Keywords      string
-	Webhooks     []string
+	Badwords      string
-	AllWebhooks  map[string]ConfigWebhook
+	Localizations map[string]string
-	Logger       chan string
+	Recover       string
-	error        error
+	Webhooks      []string
-	failures     int
+	AllWebhooks   map[string]Webhook
-	passes       int
+	Logger        chan string
-	lastFailed   time.Time
+	status        Status
-	lastPassed   time.Time
+	changed       bool
-	lastNotified time.Time
+	error         error
+	//failures      int
+	//passes        int
+	//lastFailed    time.Time
+	//lastPassed    time.Time
+	//lastNotified time.Time
 }
 
 func New(d *Dog) *Dog {
-	d.lastPassed = time.Now().Add(-5 * time.Minute)
+	//d.lastPassed = time.Now().Add(-5 * time.Minute)
+	d.status = StatusUp
+	d.changed = false
 	return d
 }
 
@@ -44,64 +76,87 @@ func (d *Dog) Watch() {
 	}
 }
 
+// Now that I've added the ability to notify when a server is back up
+// this definitely needs some refactoring. It's bad now.
 func (d *Dog) watch() {
 	d.Logger <- fmt.Sprintf("Check: '%s'", d.Name)
 
-	err := d.check()
+	// This may be up or down
+	err := d.hardcheck()
 	if nil == err {
+		d.Logger <- fmt.Sprintf("Up: '%s'", d.Name)
+		// if it's down, coming up, notify
+		if d.changed {
+			d.notify(MessageUp)
+		}
 		return
 	}
 
-	time.Sleep(time.Duration(2) * time.Second)
+	// If being down is a change, check to see if it's just a hiccup
-	err2 := d.check()
+	if d.changed {
-	if nil != err2 {
+		time.Sleep(time.Duration(5) * time.Second)
-		d.Logger <- fmt.Sprintf("Down: '%s': %s", d.Name, err2)
+		err2 := d.softcheck()
-	} else {
+		if nil != err2 {
-		d.Logger <- fmt.Sprintf("Hiccup: '%s': %s", d.Name, err)
+			// it's really down
-		return
+			d.Logger <- fmt.Sprintf("Down: '%s': %s", d.Name, err2)
+		} else {
+			// it's not really down, so reset the change info
+			d.changed = false
+			d.status = StatusUp
+			// and notify of the hiccup
+			d.Logger <- fmt.Sprintf("Hiccup: '%s': %s", d.Name, err)
+			d.notify(MessageHiccup)
+			return
+		}
 	}
 
-	failure := false
+	// TODO what if the server is flip-flopping rapidly?
+	// how to rate limit?
+	// "{{ .Server }} is on cooldown for 30 minutes"
+
+	// * We've had success since the last notification
+	// * It's been at least 5 minutes since the last notification
+	//fiveMinutesAgo := time.Now().Add(-5 * time.Minute)
+	//if d.lastPassed.After(d.lastNotified) && d.lastNotified.Before(fiveMinutesAgo) {
+	//}
+
 	t := 10
 	for {
+		// try to recover, then backoff exponentially
 		d.recover()
 		time.Sleep(time.Duration(t) * time.Second)
-		// backoff
 		t *= 2
-		err := d.check()
+		if t > 120 {
-		if nil != err {
+			t = 120
-			d.Logger <- fmt.Sprintf("Unrecoverable: '%s': %s", d.Name, err)
-			failure = true
-		} else {
-			failure = false
 		}
 
-		// We should notify if
+		err := d.softcheck()
-		// * We've had success since the last notification
+		if nil != err {
-		// * It's been at least 5 minutes since the last notification
+			// this is down, and we know it's down
-		fiveMinutesAgo := time.Now().Add(-5 * time.Minute)
+			d.status = StatusDown
-		if d.lastPassed.After(d.lastNotified) && d.lastNotified.Before(fiveMinutesAgo) {
+			d.Logger <- fmt.Sprintf("Unrecoverable: '%s': %s", d.Name, err)
-			d.notify(failure)
+			if d.changed {
-		}
+				d.changed = false
-		if !failure || d.failures >= 5 {
+				d.notify(MessageDown)
-			// go back to the main 5-minute loop
+			}
+		} else {
+			// it came back up
+			d.status = StatusUp
+			d.Logger <- fmt.Sprintf("Up: '%s'", d.Name)
+			if d.changed {
+				// and the downtime was short - just a recovery
+				d.notify(MessageHiccup)
+			} else {
+				// and the downtime was some time
+				d.notify(MessageUp)
+			}
+			d.changed = false
 			break
 		}
 	}
 }
 
-func (d *Dog) check() error {
+func (d *Dog) softcheck() error {
-	var err error
-	defer func() {
-		if nil != err {
-			d.failures += 1
-			d.lastFailed = time.Now()
-		} else {
-			d.lastPassed = time.Now()
-			d.passes += 1
-		}
-	}()
-
 	client := NewHTTPClient()
 	response, err := client.Get(d.CheckURL)
 	if nil != err {
@@ -115,18 +170,52 @@ func (d *Dog) check() error {
 		return err
 	}
 
+	// Note: empty matches empty as true, so this works for checking redirects
 	if !bytes.Contains(b, []byte(d.Keywords)) {
 		err = fmt.Errorf("Down: '%s' Not Found for '%s'", d.Keywords, d.Name)
 		d.Logger <- fmt.Sprintf("%s", err)
 		d.error = err
 		return err
-	} else {
+	}
-		d.Logger <- fmt.Sprintf("Up: '%s'", d.Name)
+
+	if "" != d.Badwords {
+		if bytes.Contains(b, []byte(d.Badwords)) {
+			err = fmt.Errorf("Down: '%s' Found for '%s'", d.Badwords, d.Name)
+			d.Logger <- fmt.Sprintf("%s", err)
+			d.error = err
+			return err
+		}
 	}
 
 	return nil
 }
 
+func (d *Dog) hardcheck() error {
+	previousStatus := d.status
+
+	err := d.softcheck()
+
+	// Are we up, or down?
+	if nil != err {
+		d.status = StatusDown
+		//d.failures += 1
+		//d.lastFailed = time.Now()
+	} else {
+		d.status = StatusUp
+		//d.lastPassed = time.Now()
+		//d.passes += 1
+	}
+
+	// Has that changed?
+	if previousStatus != d.status {
+		d.changed = true
+	} else {
+		d.changed = false
+	}
+
+	return err
+}
+
 func (d *Dog) recover() {
 	if "" == d.Recover {
 		return
@@ -154,9 +243,9 @@ func (d *Dog) recover() {
 	}
 }
 
-func (d *Dog) notify(hardFail bool) {
+func (d *Dog) notify(msg string) {
-	d.Logger <- fmt.Sprintf("Notifying the authorities of %s's failure", d.Name)
+	d.Logger <- fmt.Sprintf("Notifying the authorities of %s's status change", d.Name)
-	d.lastNotified = time.Now()
+	//d.lastNotified = time.Now()
 
 	for i := range d.Webhooks {
 		name := d.Webhooks[i]
@@ -172,105 +261,128 @@ func (d *Dog) notify(hardFail bool) {
 			continue
 		}
 
-		// TODO do this in main on config init
+		d.notifyOne(h, msg)
-		if "" == h.Method {
-			h.Method = "POST"
-		}
-
-		var body *strings.Reader
-		var err error
-		// TODO real templates
-		if 0 != len(h.Form) {
-			form := url.Values{}
-			for k := range h.Form {
-				v := h.Form[k]
-				// because `{{` gets urlencoded
-				//k = strings.Replace(k, "{{ .Name }}", d.Name, -1)
-				v = strings.Replace(v, "{{ .Name }}", d.Name, -1)
-				d.Logger <- fmt.Sprintf("[HEADER] %s: %s", k, v)
-				form.Set(k, v)
-			}
-			body = strings.NewReader(form.Encode())
-		} else if 0 != len(h.JSON) {
-			bodyBuf, err := json.Marshal(h.JSON)
-			if nil != err {
-				d.Logger <- fmt.Sprintf("[Notify] JSON Marshal Error for '%s': %s", h.Name, err)
-				continue
-			}
-			// `{{` should be left alone
-			body = strings.NewReader(strings.Replace(string(bodyBuf), "{{ .Name }}", d.Name, -1))
-		}
-
-		client := NewHTTPClient()
-		req, err := http.NewRequest(h.Method, h.URL, body)
-		if nil != err {
-			d.Logger <- fmt.Sprintf("[Notify] HTTP Client Network Error for '%s': %s", h.Name, err)
-			continue
-		}
-
-		if 0 != len(h.Form) {
-			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-		} else if 0 != len(h.JSON) {
-			req.Header.Set("Content-Type", "application/json")
-		}
-
-		if 0 != len(h.Auth) {
-			user := h.Auth["user"]
-			if "" == user {
-				user = h.Auth["username"]
-			}
-			pass := h.Auth["pass"]
-			if "" == user {
-				pass = h.Auth["password"]
-			}
-			req.SetBasicAuth(user, pass)
-		}
-
-		req.Header.Set("User-Agent", "Watchdog/1.0")
-		for k := range h.Headers {
-			req.Header.Set(k, h.Headers[k])
-		}
-
-		resp, err := client.Do(req)
-		if nil != err {
-			d.Logger <- fmt.Sprintf("[Notify] HTTP Client Error for '%s': %s", h.Name, err)
-			continue
-		}
-
-		if !(resp.StatusCode >= 200 && resp.StatusCode < 300) {
-			d.Logger <- fmt.Sprintf("[Notify] Response Error for '%s': %s", h.Name, resp.Status)
-			continue
-		}
-
-		// TODO json vs xml vs txt
-		var data map[string]interface{}
-		req.Header.Add("Accept", "application/json")
-		decoder := json.NewDecoder(resp.Body)
-		err = decoder.Decode(&data)
-		if err != nil {
-			d.Logger <- fmt.Sprintf("[Notify] Response Body Error for '%s': %s", h.Name, resp.Status)
-			continue
-		}
-
-		// TODO some sort of way to determine if data is successful (keywords)
-		d.Logger <- fmt.Sprintf("[Notify] Success? %#v", data)
 	}
 }
 
+func (d *Dog) notifyOne(h Webhook, msg string) {
+	// TODO do this in main on config init
+	if "" == h.Method {
+		h.Method = "POST"
+	}
+
+	var body *strings.Reader
+	var err error
+	// TODO real templates
+	if 0 != len(h.Form) {
+		form := url.Values{}
+		for k := range h.Form {
+			v := h.Form[k]
+			// because `{{` gets urlencoded
+			//k = strings.Replace(k, "{{ .Name }}", d.Name, -1)
+			v = strings.Replace(v, "{{ .Watchdog }}", d.Watchdog, -1)
+			v = strings.Replace(v, "{{ .Name }}", d.Name, -1)
+			v = strings.Replace(v, "{{ .Status }}", d.localize(d.status.String()), -1)
+			v = strings.Replace(v, "{{ .Message }}", d.localize(msg), -1)
+			d.Logger <- fmt.Sprintf("[HEADER] %s: %s", k, v)
+			form.Set(k, v)
+		}
+		body = strings.NewReader(form.Encode())
+	} else if 0 != len(h.JSON) {
+		bodyBuf, err := json.Marshal(h.JSON)
+		if nil != err {
+			d.Logger <- fmt.Sprintf("[Notify] JSON Marshal Error for '%s': %s", h.Name, err)
+			return
+		}
+		// `{{` should be left alone
+		v := string(bodyBuf)
+		v = strings.Replace(v, "{{ .Watchdog }}", d.Watchdog, -1)
+		v = strings.Replace(v, "{{ .Name }}", d.Name, -1)
+		v = strings.Replace(v, "{{ .Status }}", d.localize(d.status.String()), -1)
+		v = strings.Replace(v, "{{ .Message }}", d.localize(msg), -1)
+		body = strings.NewReader(v)
+	}
+
+	client := NewHTTPClient()
+	req, err := http.NewRequest(h.Method, h.URL, body)
+	if nil != err {
+		d.Logger <- fmt.Sprintf("[Notify] HTTP Client Network Error for '%s': %s", h.Name, err)
+		return
+	}
+
+	if 0 != len(h.Form) {
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	} else if 0 != len(h.JSON) {
+		req.Header.Set("Content-Type", "application/json")
+	}
+
+	if 0 != len(h.Auth) {
+		user := h.Auth["user"]
+		if "" == user {
+			user = h.Auth["username"]
+		}
+		pass := h.Auth["pass"]
+		if "" == user {
+			pass = h.Auth["password"]
+		}
+		req.SetBasicAuth(user, pass)
+	}
+
+	req.Header.Set("User-Agent", "Watchdog/1.0")
+	for k := range h.Headers {
+		req.Header.Set(k, h.Headers[k])
+	}
+
+	resp, err := client.Do(req)
+	if nil != err {
+		d.Logger <- fmt.Sprintf("[Notify] HTTP Client Error for '%s': %s", h.Name, err)
+		return
+	}
+
+	if !(resp.StatusCode >= 200 && resp.StatusCode < 300) {
+		d.Logger <- fmt.Sprintf("[Notify] Response Error for '%s': %s", h.Name, resp.Status)
+		return
+	}
+
+	// TODO json vs xml vs txt
+	var data map[string]interface{}
+	req.Header.Add("Accept", "application/json")
+	decoder := json.NewDecoder(resp.Body)
+	err = decoder.Decode(&data)
+	if err != nil {
+		d.Logger <- fmt.Sprintf("[Notify] Response Body Error for '%s': %s", h.Name, resp.Status)
+		return
+	}
+
+	// TODO some sort of way to determine if data is successful (keywords)
+	d.Logger <- fmt.Sprintf("[Notify] Success? %#v", data)
+}
+func (d *Dog) localize(msg string) string {
+	for k := range d.Localizations {
+		if k == msg {
+			return d.Localizations[k]
+		}
+	}
+	return msg
+}
+
 type Config struct {
-	Watches  []ConfigWatch   `json:"watches"`
+	Watchdog      string            `json:"watchdog"`
-	Webhooks []ConfigWebhook `json:"webhooks"`
+	Watches       []ConfigWatch     `json:"watches"`
+	Webhooks      []Webhook         `json:"webhooks"`
+	Localizations map[string]string `json:"localizations"`
 }
 
 type ConfigWatch struct {
 	Name          string   `json:"name"`
 	URL           string   `json:"url"`
 	Keywords      string   `json:"keywords"`
+	Badwords      string   `json:"badwords"`
 	Webhooks      []string `json:"webhooks"`
 	RecoverScript string   `json:"recover_script"`
 }
 
-type ConfigWebhook struct {
+type Webhook struct {
 	Name    string              `json:"name"`
 	Method  string              `json:"method"`
 	URL     string              `json:"url"`