switch webhook configuration from sns to argus (#202)

* switch webhook configuration from sns to argus * bump argus client to v0.2.1 * leverage new argus client * bump argus to v0.3.0 * Apply suggestions from code review Co-authored-by: kristinaspring <[email protected]> Co-authored-by: Joel Unzain <[email protected]> * add requested changes * removed hooks endpoint * gofmt * adding pr url to changelog Co-authored-by: Joel Unzain <[email protected]> * added unit tests * bump argus and webpa-common Co-authored-by: kristinaspring <[email protected]> Co-authored-by: Joel Unzain <[email protected]>
xmidt-org · Jul 7, 2020 · 453f141 · 453f141
1 parent d92f6ad
commit 453f141
Show file tree

Hide file tree

Showing 13 changed files with 442 additions and 268 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+- switch webhook configuration from sns to argus [#202](https://github.com/xmidt-org/caduceus/pull/202)
+- removed `/hooks` endpoint [#202](https://github.com/xmidt-org/caduceus/pull/202)
 
 ## [v0.4.0]
 - Moved and renamed configuration variable for outgoing hostname validation [#223](https://github.com/xmidt-org/caduceus/pull/223)

diff --git a/README.md b/README.md
@@ -88,11 +88,6 @@ The following is an example request. Note: this is not a valid json because of t
 }
 ```
 
-#### Get Webhooks - `/hooks` endpoint
-To speed up caduceus start up time and test the registration of webhooks, the
-`/hooks` endpoint was created. This is a simple `GET` request which will return
-all the webhooks and their configuration.
-
 ## Usage
 Once everything is up and running you can start sending requests. Bellow are
 a few examples.

diff --git a/caduceus.yaml b/caduceus.yaml
@@ -186,11 +186,11 @@
     # file is the name of the most recent log file.  If set to "stdout" this
     # will log to os.Stdout.
     # (Optional) defaults to os.TempDir()
-    file: "/var/log/caduceus/caduceus.log"
+    file: "stdout"
 
     # level is the logging level to use - INFO, DEBUG, WARN, ERROR
     # (Optional) defaults to ERROR
-    level: "ERROR"
+    level: "DEBUG"
 
     # maxsize is the maximum file size in MB
     # (Optional) defaults to max 100MB
@@ -230,59 +230,47 @@
   # authHeader provides the list of basic auth headers that caduceus will accept
   # as authorization
   # (Optional)
-  authHeader: ["xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=","xxxxxxxxxxxxxxxxxxxxxx=="]
-
-########################################
-#   Startup Related Configuration
-########################################
-
-  # start is a way to bypass the need to wait for the duration before accepting
-  # events.  The wait is needed to ensure caduceus knows all the hooks it needs
-  # to deliver events to.  Presently the only 2 ways to do that for an SNS
-  # backed service is to:
-  #
-  #     1. Wait until all the hooks have timed out and re-registered (duration)
-  #     2. Ask some server that has the entire list to share it (apiPath)
-  #
-  # TBD add details
-  start:
-    duration: 60000000000
-    apiPath: "https://tr1d1um.example.com:443/api/v2/hooks"
-    sat:
-      path: "https://jwt.example.com/get"
-      id: "magic-id"
-      secret: "xxx"
-      capabilities: "webpa:api:.*:all"
-
-  # waitForDns TBD
-  waitForDns: 0
-
-  # soa is a way for the caduceus machine to check to see if it's DNS name is
-  # present before it tries to use it's DNS name to register with services like
-  # AWS Route53.  This check goes directly to the SOA record authority and
-  # bypasses the various caching servers that could cache the non-existence
-  # record which generally have VERY long TTLs.  Ultimately this way caduceus
-  # can restart much faster.
-  # (Optional)
-  soa:
-    provider: "example.awsdns-19.com:53"
+  authHeader: ["xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=","dXNlcjpwYXNz"]
 
 ########################################
 #   Webhook Related Configuration
 ########################################
+  # webhookStore provides configuration for storing and obtaining webhook 
+  # information using argus.
+  webhookStore:
+
+    # bucket to store and retrieve webhooks.
+    bucket: "webhooks-caduceus"
+
+    # address it the the location to talk to argus.
+    address: "http://argus:6600"
+
+    # pullInterval is how often to call argus to update the webhook structure.
+    pullInterval: "2s"
+
+    # auth the authentication method for argus.
+    auth:
+      # basic configures basic authentication for argus.
+      # Must be of form: 'Basic xyz=='
+      basic: "Basic dXNlcjpwYXNz"
+#
+#      # jwt configures jwt style authentication for argus.
+#      JWT:
+#        # requestHeaders are added to the request for the token.
+#        # (Optional)
+#        # requestHeaders:
+#        #   "": ""
+#
+#        # authURL is the URL to access for the token.
+#        authURL: ""
+#
+#        # timeout is how long the request to get the token will take before
+#        # timing out.
+#        timeout: "1m"
+#
+#        # buffer is the length of time before a token expires to get a new token.
+#        buffer: "2m"
 
-  # aws provides the details needed to subscribe to the SNS topic that sends
-  # webhook registration events out to all listeners
-  #
-  # TBD add details
-  aws:
-    accessKey: "xxxxxxxxxxxxxxxxxxxx"
-    secretKey: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-    env: "xmidt-prod"
-    sns:
-      region: "us-east-1"
-      topicArn: "arn:aws:sns:us-east-1:000000000000:xmidt-prod"
-      urlPath: "/api/v2/aws/sns"
 
 ########################################
 #   Delivery Pipeline Related Configuration
@@ -357,3 +345,4 @@
   #   - 100
   #   - 1000
   #   - 10000
+
diff --git a/caduceus_type.go b/caduceus_type.go
@@ -17,6 +17,7 @@
 package main
 
 import (
+	"github.com/xmidt-org/argus/chrysom"
 	"time"
 
 	"github.com/go-kit/kit/log"
@@ -34,6 +35,8 @@ type CaduceusConfig struct {
 	JobQueueSize     int
 	Sender           SenderConfig
 	JWTValidators    []JWTValidator
+	WebhookStore     chrysom.ClientConfig
+	AllowInsecureTLS bool
 }
 
 type SenderConfig struct {

diff --git a/deploy/packaging/caduceus_spruce.yaml b/deploy/packaging/caduceus_spruce.yaml
@@ -235,64 +235,18 @@ jwtValidators:
 # as authorization
 # (Optional)
 authHeader:
-  - (( grab $AUTH_HEADER || "YXV0aEhlYWRlcg==" ))
-
-########################################
-#   Startup Related Configuration
-########################################
-
-# start contains configuration for the logic by which Tr1d1um can
-# fetch the current WebPA webhooks without having to wait for SNS
-# It does so by pinging the rest of the cluter at the specified apiPath
-# More details at https://godoc.org/github.com/xmidt-org/webpa-common/webhook#StartConfig
-start:
-  # duration is the max amount of time allowed to wait for webhooks data to be retrieved
-  duration: "20s"
-
-  # path used to query the existing webhooks
-  apiPath: http://localhost:6100/hooks
-
-# waitForDns TBD
-waitForDns: 0
-
-# soa is a way for the caduceus machine to check to see if it's DNS name is
-# present before it tries to use it's DNS name to register with services like
-# AWS Route53.  This check goes directly to the SOA record authority and
-# bypasses the various caching servers that could cache the non-existence
-# record which generally have VERY long TTLs.  Ultimately this way caduceus
-# can restart much faster.
-# (Optional)
-soa:
-  provider: "example.awsdns-19.com:53"
+  - (( grab $AUTH_HEADER || "dXNlcjpwYXNz" ))
 
 ########################################
 #   Webhook Related Configuration
 ########################################
-
-# aws provides the AWS SNS configurations the webhooks library needs
-aws:
-  #AWS access key
-  accessKey: (( grab $AWS_ACCESS_KEY || "" ))
-
-  #AWS secret key
-  secretKey: (( grab $AWS_SECRET_KEY || "" ))
-
-  env: (( grab $AWS_WEBHOOK_ENV || "local-dev" ))
-
-  sns:
-    # awsEndpoint is the AWS endpoint
-    # this must be left out in produ
-    awsEndpoint: (( grab $AWS_ENDPOINT || "http://goaws:4100" ))
-
-    #region is the AWS SNS region
-    region: (( grab $AWS_REGION || "us-east-1" ))
-
-    # topicArn describes the SNS topic this server needs to subscribe to
-    topicArn: (( grab $AWS_SNS_ARN || "arn:aws:sns:us-east-1:000000000000:xmidt-local-caduceus"))
-
-    #urlPath is the URL path SNS will use to confirm a subscription with this server
-    urlPath: "/api/v2/aws/sns"
-
+# Argus Config for storing the webhook information
+webhookStore:
+  bucket: "webhooks-caduceus"
+  address:  (( grab $ARGUS_HOST || "argus:6600" ))
+  pullInterval: "2s"
+  auth:
+    basic: (( grab $AUTH_HEADER || "Basic dXNlcjpwYXNz" ))
 ########################################
 #   Delivery Pipeline Related Configuration
 ########################################

diff --git a/go.mod b/go.mod
@@ -1,26 +1,23 @@
 module github.com/xmidt-org/caduceus
 
-go 1.12
+go 1.14
 
 require (
 	github.com/SermoDigital/jose v0.9.2-0.20161205224733-f6df55f235c2
-	github.com/aws/aws-sdk-go v1.23.11 // indirect
 	github.com/c9s/goprocinfo v0.0.0-20190309065803-0b2ad9ac246b // indirect
 	github.com/davecgh/go-spew v1.1.1
-	github.com/go-kit/kit v0.8.0
+	github.com/go-kit/kit v0.9.0
 	github.com/gorilla/mux v1.7.3
-	github.com/influxdata/influxdb v1.7.7 // indirect
 	github.com/justinas/alice v1.2.0
-	github.com/prometheus/client_golang v1.0.0 // indirect
 	github.com/samuel/go-zookeeper v0.0.0-20190810000440-0ceca61e4d75 // indirect
 	github.com/satori/go.uuid v1.2.0
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/spf13/pflag v1.0.3
-	github.com/spf13/viper v1.6.1
-	github.com/stretchr/testify v1.3.0
-	github.com/xmidt-org/webpa-common v1.8.1
+	github.com/spf13/pflag v1.0.5
+	github.com/spf13/viper v1.6.2
+	github.com/stretchr/testify v1.5.1
+	github.com/xmidt-org/argus v0.3.3
+	github.com/xmidt-org/webpa-common v1.10.2
 	github.com/xmidt-org/wrp-go/v2 v2.0.0
 	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect
-	golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 // indirect
 	golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 // indirect
 )