Initial work on deadmanswatch (#1)

* Initial work on deadmanswatch

* Change the entrypoint

.dockerignore

@@ -0,0 +1 @@
+bin/

.travis.yml

@@ -0,0 +1,29 @@
+# This is a weird way of telling Travis to use the fast container-based test
+# runner instead of the slow VM-based runner.
+sudo: false
+
+language: go
+
+go:
+  - 1.11.x
+
+# Skip the install step. Don't `go get` dependencies. Only build with the code
+# in vendor/
+install: true
+
+# Only clone the most recent commit.
+git:
+  depth: 1
+
+# Don't email me the results of the test runs.
+notifications:
+  email: false
+
+before_script:
+  - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $GOPATH/bin v1.12.3
+  - go get -u github.com/golang/dep/cmd/dep
+
+script:
+  - dep check
+  - golangci-lint run       # run a bunch of code checkers/linters in parallel
+  - go test -v -race ./...  # Run all the tests with the race detector enabled

Dockerfile

@@ -0,0 +1,21 @@
+FROM golang:1.10 AS builder
+LABEL maintainer="[email protected]" \
+      description="Forwards prometheus DeadMansSwitch alerts to CloudWatch" \
+      version="0.0.1"
+
+RUN useradd -u 10001 deadmanswatch
+
+# Copy the code from the host and compile it
+WORKDIR $GOPATH/src/github.com/kierranm/deadmanswatch
+COPY ./vendor $GOPATH/src/github.com/kierranm/deadmanswatch/vendor
+COPY ./main.go $GOPATH/src/github.com/kierranm/deadmanswatch/main.go
+COPY ./cmd $GOPATH/src/github.com/kierranm/deadmanswatch/cmd
+RUN go test ./...
+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix nocgo -o /deadmanswatch .
+
+FROM scratch
+COPY --from=builder /deadmanswatch ./
+COPY --from=builder /etc/passwd /etc/passwd
+USER deadmanswatch
+WORKDIR /
+ENTRYPOINT ["./deadmanswatch"]

Gopkg.toml

@@ -0,0 +1,46 @@
+# Gopkg.toml example
+#
+# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[[constraint]]
+  name = "github.com/aws/aws-sdk-go"
+  version = "1.15.82"
+
+[[constraint]]
+  name = "github.com/gorilla/mux"
+  version = "1.6.2"
+
+[[constraint]]
+  name = "github.com/sirupsen/logrus"
+  version = "1.2.0"
+
+[[constraint]]
+  name = "github.com/spf13/cobra"
+  version = "0.0.3"
+
+[prune]
+  go-tests = true
+  unused-packages = true

README.md

@@ -1,2 +1,76 @@
-# deadmanswatch
-CloudWatch Alarms from Prometheus DeadMansSwitch alerts 
+# DeadMansWatch
+
+DeadMansWatch is a tiny tool for forwarding Prometheus DeadMansWatch alerts from AlertManager
+to CloudWatch as metrics, these metrics can be used to create CloudWatch alarms to notify you
+when Prometheus is down.
+
+It also sends it's own dead mans switch to CloudWatch so that you can alarm when DeadMansWatch is down.
+
+## Usage
+To run deadmanswatch, use the `watch` command
+```
+All software has versions. This is DeadMansWatch's
+
+Usage:
+  deadmanswatch watch [flags]
+
+Flags:
+      --alert-source-label string           The alert label to use for the 'source' dimension. If unset the 'source' will always be 'prometheus'
+      --graceful-timeout duration           Time to wait for the server to gracefully shutdown (default 15s)
+      --heartbeat-interval duration         Time between sending metrics for DeadMansWatchs own DeadMansSwitch (default 1m0s)
+  -h, --help                                help for watch
+  -a, --listen-address ip                   Address to bind to (default 0.0.0.0)
+      --log-level string                    The level at which to log. Valid values are debug, info, warn, error (default "info")
+      --metric-dimensions stringToString    Dimensions for the metrics in CloudWatch (default [])
+      --metric-name string                  metric name for DeadManWatch's own DeadManSwitch metric (default "DeadMansSwitch")
+      --metric-namespace string             Metric namespace in CloudWatch (default "DeadMansWatch")
+  -p, --port int                            Port to listen on (default 8080)
+  -r, --region string                       AWS Region for CloudWatch
+```
+
+This will start the deadmanswatch server and listen for connections that match the alertmanager [webhook payload](https://prometheus.io/docs/alerting/configuration/#%3Cwebhook_config%3E)
+
+### AWS Credentials
+DeadMansWatch uses the aws sdk for go, which supports the following authentication methods:
+- IAM Instance Profile
+- Environment variables
+- Shared credentials file (`~/.aws/credentials`)
+
+## Deploying
+### Service
+#### Kubes
+The `deploy/kubes` folder contains kubernetes manifests to get DeadMansWatch up and running in kubernetes.
+
+#### Helm
+The `deploy/helm` directory contains a helm chart so that you can deploy without having to modify the manifests manually.
+
+### CloudWatch Alarm
+The main idea behind this tool is to have CloudWatch alarm when the dead mans switch metric is no longer being received,
+you could create such an alarm with terraform like this:
+```hcl
+resource "aws_cloudwatch_metric_alarm" "deadmanswatch" {
+  alarm_name = "deadmansswitch-missing"
+  comparison_operator = "LessThanThreshold"
+  metric_name = "DeadMansSwitch"
+  namespace = "DeadMansWatch"
+  evaluation_periods = 3
+  treat_missing_data = "breaching"
+  threshold = 1
+  dimensions {
+      source = "prometheus"
+  }
+  alarm_description = "This alarm fires when prometheus is down in a kubernetes"
+  alarm_actions = [] # SNS Arn or something
+  ok_actions = [] # SNS ARN or something
+}
+```
+
+## Contributing
+
+1. Fork it
+2. Download your fork to your PC (`git clone https://github.com/your_username/deadmanswatch && cd deadmanswatch`)
+3. Create your feature branch (`git checkout -b my-new-feature`)
+4. Make changes and add them (`git add .`)
+5. Commit your changes (`git commit -m 'Add some feature'`)
+6. Push to the branch (`git push origin my-new-feature`)
+7. Create new pull request

cmd/root.go

@@ -0,0 +1,31 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/spf13/cobra"
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "deadmanswatch",
+	Short: "Forwards Prometheus DeadManSwitch alerts to CloudWatch metrics",
+	Long: `Listens for DeadMansSwitch alerts for AlertManager and forwards
+them as metrcs to CloudWatch so that you can create alarms from them.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		err := cmd.Help()
+		logrus.Warnf("Failed to display help: %v", err)
+	},
+}
+
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func init() {
+}

cmd/version.go

@@ -0,0 +1,22 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+const Version = "0.0.1"
+
+func init() {
+	rootCmd.AddCommand(versionCmd)
+}
+
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Print the version of deadmanswatch",
+	Long:  `All software has versions. This is DeadMansWatch's`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Printf("DeadMansWatch v%s\n", Version)
+	},
+}