From 10d40f389339f967f516e97ce97084adb1a9ecf8 Mon Sep 17 00:00:00 2001 From: George Angel Date: Thu, 2 Mar 2017 21:32:10 +1000 Subject: [PATCH 01/11] Enable setting multiple targets and specifying them via a label --- ingress_test.go | 83 +++++++++++++++++++++++--------------- main.go | 55 +++++++++++++++++++------ registrator.go | 75 ++++++++++++++++++++-------------- registrator_test.go | 98 +++++++++++++++++++++++++-------------------- 4 files changed, 193 insertions(+), 118 deletions(-) diff --git a/ingress_test.go b/ingress_test.go index a85dccd..b2bbea1 100644 --- a/ingress_test.go +++ b/ingress_test.go @@ -15,87 +15,106 @@ import ( ) var ( - testIngressA = &v1beta1.Ingress{ + privateIngressHostsAB = &v1beta1.Ingress{ ObjectMeta: v1.ObjectMeta{ - Name: "exampleA", + Name: "privateIngressHostsAB", Namespace: api.NamespaceDefault, - Labels: map[string]string{}, + Labels: map[string]string{ + LabelName: PrivateTarget, + }, }, Spec: v1beta1.IngressSpec{ Rules: []v1beta1.IngressRule{ - {Host: "foo1.example.com"}, - {Host: "foo2.example.com"}, + {Host: "a.example.com"}, + {Host: "b.example.com"}, }, }, } - testIngressB = &v1beta1.Ingress{ + publicIngressHostC = &v1beta1.Ingress{ ObjectMeta: v1.ObjectMeta{ - Name: "exampleB", + Name: "publicIngressHostCD", Namespace: api.NamespaceDefault, Labels: map[string]string{ - "public": "true", + LabelName: PublicTarget, }, }, Spec: v1beta1.IngressSpec{ Rules: []v1beta1.IngressRule{ - {Host: "bar.example.com"}, + {Host: "c.example.com"}, }, }, } - testIngressB2 = &v1beta1.Ingress{ + publicIngressHostD = &v1beta1.Ingress{ ObjectMeta: v1.ObjectMeta{ - Name: "exampleB", + Name: "publicIngressHostCD", Namespace: api.NamespaceDefault, Labels: map[string]string{ - "public": "true", + LabelName: PublicTarget, }, }, Spec: v1beta1.IngressSpec{ Rules: []v1beta1.IngressRule{ - {Host: "bar2.example.com"}, + {Host: "d.example.com"}, }, }, } - testIngressC1 = &v1beta1.Ingress{ + privateIngressHostE = &v1beta1.Ingress{ ObjectMeta: v1.ObjectMeta{ - Name: "exampleC1", + Name: "ingressHostE", Namespace: api.NamespaceDefault, - Labels: map[string]string{}, + Labels: map[string]string{ + LabelName: PrivateTarget, + }, }, Spec: v1beta1.IngressSpec{ Rules: []v1beta1.IngressRule{ - {Host: "baz.example.com"}, + {Host: "e.example.com"}, }, }, } - testIngressC2 = &v1beta1.Ingress{ + privateIngressHostEDup = &v1beta1.Ingress{ ObjectMeta: v1.ObjectMeta{ - Name: "exampleC2", + Name: "ingressHostE", Namespace: api.NamespaceDefault, - Labels: map[string]string{}, + Labels: map[string]string{ + LabelName: PrivateTarget, + }, }, Spec: v1beta1.IngressSpec{ Rules: []v1beta1.IngressRule{ - {Host: "baz.example.com"}, + {Host: "e.example.com"}, }, }, } - testIngressC3 = &v1beta1.Ingress{ + publicIngressHostEDup = &v1beta1.Ingress{ ObjectMeta: v1.ObjectMeta{ - Name: "exampleC3", + Name: "ingressHostE", Namespace: api.NamespaceDefault, Labels: map[string]string{ - "public": "true", + LabelName: PublicTarget, }, }, Spec: v1beta1.IngressSpec{ Rules: []v1beta1.IngressRule{ - {Host: "baz.example.com"}, + {Host: "e.example.com"}, + }, + }, + } + + ingressNoLabels = &v1beta1.Ingress{ + ObjectMeta: v1.ObjectMeta{ + Name: "ingressNoLabels", + Namespace: api.NamespaceDefault, + Labels: map[string]string{}, + }, + Spec: v1beta1.IngressSpec{ + Rules: []v1beta1.IngressRule{ + {Host: "no-labels.example.com"}, }, }, } @@ -180,13 +199,13 @@ func waitForTrue(test func() bool, timeout time.Duration) error { func TestIngressWatcher(t *testing.T) { expected := []testIngressEvent{ - {watch.Added, nil, testIngressA}, - {watch.Added, nil, testIngressB}, - {watch.Deleted, testIngressA, nil}, - {watch.Modified, testIngressB, testIngressB2}, + {watch.Added, nil, privateIngressHostsAB}, + {watch.Added, nil, publicIngressHostC}, + {watch.Deleted, privateIngressHostsAB, nil}, + {watch.Modified, publicIngressHostC, publicIngressHostD}, } - client, watcher := newTestIngressWatcherClient(*testIngressA, *testIngressB) + client, watcher := newTestIngressWatcherClient(*privateIngressHostsAB, *publicIngressHostC) pM := &sync.Mutex{} processed := []testIngressEvent{} @@ -219,11 +238,11 @@ func TestIngressWatcher(t *testing.T) { if err := waitForTrue(pLenIs(2), 10*time.Second); err != nil { t.Fatalf("timed out waiting for ingressWatcher to process events") } - watcher.Delete(testIngressA) + watcher.Delete(privateIngressHostsAB) if err := waitForTrue(pLenIs(3), 10*time.Second); err != nil { t.Fatalf("timed out waiting for ingressWatcher to process events") } - watcher.Modify(testIngressB2) + watcher.Modify(publicIngressHostD) if err := waitForTrue(pLenIs(4), 10*time.Second); err != nil { t.Fatalf("timed out waiting for ingressWatcher to process events") } diff --git a/main.go b/main.go index 93484f0..90247f0 100644 --- a/main.go +++ b/main.go @@ -2,10 +2,12 @@ package main import ( "flag" + "fmt" "log" "net/http" "os" "os/signal" + "strings" "k8s.io/client-go/1.5/tools/clientcmd" @@ -15,20 +17,46 @@ import ( "github.com/utilitywarehouse/go-operational/op" ) +// Define a type named "strslice" as a slice of strings +type strslice []string + +// Now, for our new type, implement the two methods of +// the flag.Value interface... +// The first method is String() string +func (s *strslice) String() string { + return fmt.Sprint(*s) +} + +// Set is the method to set the flag value, part of the flag.Value interface. +// Set's argument is a string to be parsed to set the flag. +// It's a comma-separated list, so we split it. +func (s *strslice) Set(value string) error { + for _, target := range strings.Split(value, ",") { + *s = append(*s, target) + } + return nil +} + +// Define a flag to accumulate durations. Because it has a special type, +// we need to use the Var function and therefore create the flag during +// init. + +var targets strslice + var ( appGitHash = "master" - kubeConfig = flag.String("kubernetes-config", "", "path to the kubeconfig file, if unspecified then in-cluster config will be used") - publicSelectorString = flag.String("public-ingress-selector", "", "selector for ingresses that should point to the public target") - targetPublic = flag.String("target-public", "", "target hostname for public ingresses") - targetPrivate = flag.String("target-private", "", "target hostnam for private ingresses") - r53ZoneID = flag.String("route53-zone-id", "", "route53 hosted DNS zone id") - debugLogs = flag.Bool("debug", false, "enables debug logs") - dryRun = flag.Bool("dry-run", false, "if set, ingress53 will not make any Route53 changes") + kubeConfig = flag.String("kubernetes-config", "", "path to the kubeconfig file, if unspecified then in-cluster config will be used") + labelName = flag.String("label-name", "", "Kubernetes key of the label that specifies the target type") + defaultTarget = flag.String("default-target", "", "Default target to use in the absense of matching labels") + r53ZoneID = flag.String("route53-zone-id", "", "route53 hosted DNS zone id") + debugLogs = flag.Bool("debug", false, "enables debug logs") + dryRun = flag.Bool("dry-run", false, "if set, ingress53 will not make any Route53 changes") metricUpdatesApplied = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "ingress53", + Subsystem: "route53", Name: "updates_applied", Help: "number of route53 updates", @@ -57,7 +85,7 @@ var ( ) func init() { - flag.Parse() + flag.Var(&targets, "targets", "List of endpoints (ELB) targets to map ingress records to") luf := &logutils.LevelFilter{ Levels: []logutils.LogLevel{"DEBUG", "INFO", "ERROR"}, @@ -77,13 +105,14 @@ func init() { } func main() { + flag.Parse() + ro := registratorOptions{ - PrivateHostname: *targetPrivate, - PublicHostname: *targetPublic, - PublicResourceSelector: *publicSelectorString, - Route53ZoneID: *r53ZoneID, + Targets: targets, + LabelName: *labelName, + DefaultTarget: *defaultTarget, + Route53ZoneID: *r53ZoneID, } - if *kubeConfig != "" { config, err := clientcmd.BuildConfigFromFlags("", *kubeConfig) if err != nil { diff --git a/registrator.go b/registrator.go index bbcc785..b409a0a 100644 --- a/registrator.go +++ b/registrator.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "errors" "fmt" "log" @@ -22,6 +23,7 @@ import ( var ( errRegistratorMissingOption = errors.New("missing required registrator option") errDNSEmptyAnswer = errors.New("DNS nameserver returned an empty answer") + errInvalidTarget = errors.New("Target specified with invalid format, should be: \"private:private-aws-elb.com\"") defaultResyncPeriod = 15 * time.Minute defaultBatchProcessCycle = 5 * time.Second dnsClient = &dns.Client{} @@ -47,44 +49,55 @@ type cnameRecord struct { type registrator struct { dnsZone *ingressWatcher - options registratorOptions - publicSelector labels.Selector - updateQueue chan cnameChange + options registratorOptions + sats []selectorAndTarget + updateQueue chan cnameChange } type registratorOptions struct { - AWSSessionOptions *session.Options - KubernetesConfig *rest.Config - PrivateHostname string // required - PublicHostname string // required - PublicResourceSelector string - Route53ZoneID string // required - ResyncPeriod time.Duration + AWSSessionOptions *session.Options + KubernetesConfig *rest.Config + Targets []string // required + LabelName string // required + DefaultTarget string // required + Route53ZoneID string // required + ResyncPeriod time.Duration } -func newRegistrator(zoneID, publicHostname, privateHostname, publicSelector string) (*registrator, error) { - return newRegistratorWithOptions(registratorOptions{ - PrivateHostname: privateHostname, - PublicHostname: publicHostname, - PublicResourceSelector: publicSelector, - Route53ZoneID: zoneID, - }) +type selectorAndTarget struct { + Selector labels.Selector + Target string +} + +func newRegistrator(zoneID string, targets []string, labelName, defaultTarget string) (*registrator, error) { + return newRegistratorWithOptions( + registratorOptions{ + Route53ZoneID: zoneID, + Targets: targets, + LabelName: labelName, + DefaultTarget: defaultTarget, + }) } func newRegistratorWithOptions(options registratorOptions) (*registrator, error) { - if options.PrivateHostname == "" || options.PublicHostname == "" || options.Route53ZoneID == "" { + // check required options are set + if len(options.Targets) == 0 || options.Route53ZoneID == "" || options.LabelName == "" { return nil, errRegistratorMissingOption } - var publicSelector labels.Selector - if options.PublicResourceSelector == "" { - publicSelector = labels.Nothing() - } else { - s, err := labels.Parse(options.PublicResourceSelector) + var sats []selectorAndTarget + + for _, target := range options.Targets { + var sb bytes.Buffer + sb.WriteString(options.LabelName) + sb.WriteString("=") + sb.WriteString(target) + + s, err := labels.Parse(sb.String()) if err != nil { return nil, err } - publicSelector = s + sats = append(sats, selectorAndTarget{Selector: s, Target: target}) } if options.AWSSessionOptions == nil { @@ -104,9 +117,9 @@ func newRegistratorWithOptions(options registratorOptions) (*registrator, error) } return ®istrator{ - options: options, - publicSelector: publicSelector, - updateQueue: make(chan cnameChange, 64), + options: options, + sats: sats, + updateQueue: make(chan cnameChange, 64), }, nil } @@ -256,10 +269,12 @@ func (r *registrator) applyBatch(changes []cnameChange) { } func (r *registrator) getTargetForIngress(ingress *v1beta1.Ingress) string { - if r.publicSelector.Matches(labels.Set(ingress.Labels)) { - return r.options.PublicHostname + for _, sat := range r.sats { + if sat.Selector.Matches(labels.Set(ingress.Labels)) { + return sat.Target + } } - return r.options.PrivateHostname + return r.options.DefaultTarget } func (r *registrator) pruneBatch(action string, records []cnameRecord) []cnameRecord { diff --git a/registrator_test.go b/registrator_test.go index 46957a7..04db8c2 100644 --- a/registrator_test.go +++ b/registrator_test.go @@ -17,8 +17,14 @@ import ( "k8s.io/client-go/1.5/rest" ) +const ( + PrivateTarget string = "private.cluster-entrypoint.com" + PublicTarget string = "public.cluster-entrypoint.com" + LabelName string = "ingress53.target" +) + func TestNewRegistrator_defaults(t *testing.T) { - _, err := newRegistrator("z", "a", "b", "") + _, err := newRegistrator("z", []string{PrivateTarget, PublicTarget}, LabelName, PrivateTarget) if err == nil || err.Error() != "unable to load in-cluster configuration, KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be defined" { t.Errorf("newRegistrator did not return expected error") } @@ -29,35 +35,38 @@ func TestNewRegistrator_defaults(t *testing.T) { t.Errorf("newRegistrator did not return expected error") } - // invalid selector - _, err = newRegistrator("z", "a", "b", "a^b") + // invalid label name + _, err = newRegistrator("z", []string{PrivateTarget, PublicTarget}, "!^7", "") if err == nil { t.Errorf("newRegistrator did not return expected error") } // working - _, err = newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}, PublicHostname: "a", PrivateHostname: "b", Route53ZoneID: "c"}) + _, err = newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}, Targets: []string{PrivateTarget, PublicTarget}, LabelName: LabelName, Route53ZoneID: "c"}) if err != nil { t.Errorf("newRegistrator returned an unexpected error: %+v", err) } } func TestRegistrator_GetTargetForIngress(t *testing.T) { - // empty selector - r, err := newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}, PublicHostname: "a", PrivateHostname: "b", Route53ZoneID: "c"}) + // ingress b + r, err := newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}, Targets: []string{PrivateTarget, PublicTarget}, LabelName: LabelName, Route53ZoneID: "c"}) if err != nil { t.Errorf("newRegistrator returned an unexpected error: %+v", err) } - if r.getTargetForIngress(testIngressB) != "b" { + + target := r.getTargetForIngress(privateIngressHostsAB) + if target != PrivateTarget { t.Errorf("getTargetForIngress returned unexpected value") } - // proper selector - r, err = newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}, PublicHostname: "a", PrivateHostname: "b", Route53ZoneID: "c", PublicResourceSelector: "public=true"}) + // ingress a + r, err = newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}, Targets: []string{PrivateTarget, PublicTarget}, LabelName: LabelName, Route53ZoneID: "c"}) if err != nil { t.Errorf("newRegistrator returned an unexpected error: %+v", err) } - if r.getTargetForIngress(testIngressB) != "a" { + target = r.getTargetForIngress(publicIngressHostC) + if target != PublicTarget { t.Errorf("getTargetForIngress returned unexpected value") } } @@ -141,7 +150,10 @@ type mockEvent struct { } func TestRegistratorHandler(t *testing.T) { - s, _ := labels.Parse("public=true") + privateSelector, _ := labels.Parse(fmt.Sprintf("%s=%s", LabelName, PrivateTarget)) + publicSelector, _ := labels.Parse(fmt.Sprintf("%s=%s", LabelName, PublicTarget)) + sats := []selectorAndTarget{selectorAndTarget{Selector: privateSelector, Target: PrivateTarget}, selectorAndTarget{Selector: publicSelector, Target: PublicTarget}} + mdz := &mockDNSZone{} server, err := mdz.startMockDNSServer() defer server.Shutdown() @@ -150,16 +162,16 @@ func TestRegistratorHandler(t *testing.T) { } r := ®istrator{ - dnsZone: mdz, - publicSelector: s, - updateQueue: make(chan cnameChange, 16), + dnsZone: mdz, + sats: sats, + updateQueue: make(chan cnameChange, 16), ingressWatcher: &ingressWatcher{ stopChannel: make(chan struct{}), }, options: registratorOptions{ - PrivateHostname: "priv.example.com", - PublicHostname: "pub.example.com", - Route53ZoneID: "c", + Targets: []string{PrivateTarget, PublicTarget}, + LabelName: LabelName, + Route53ZoneID: "c", }, } @@ -176,85 +188,85 @@ func TestRegistratorHandler(t *testing.T) { { "example.com.", []mockEvent{ - {watch.Added, nil, testIngressA}, + {watch.Added, nil, privateIngressHostsAB}, }, map[string]string{ - "foo1.example.com": "priv.example.com", - "foo2.example.com": "priv.example.com", + "a.example.com": PrivateTarget, + "b.example.com": PrivateTarget, }, }, { "example.com.", []mockEvent{ - {watch.Added, nil, testIngressA}, - {watch.Deleted, testIngressA, nil}, + {watch.Added, nil, privateIngressHostsAB}, + {watch.Deleted, privateIngressHostsAB, nil}, }, map[string]string{}, }, { "example.com.", []mockEvent{ - {watch.Added, nil, testIngressA}, - {watch.Modified, testIngressA, testIngressB}, + {watch.Added, nil, privateIngressHostsAB}, + {watch.Modified, privateIngressHostsAB, publicIngressHostC}, }, map[string]string{ - "bar.example.com": "pub.example.com", + "c.example.com": PublicTarget, }, }, { "example.com.", []mockEvent{ - {watch.Added, nil, testIngressA}, - {watch.Deleted, testIngressA, nil}, - {watch.Added, nil, testIngressB}, + {watch.Added, nil, privateIngressHostsAB}, + {watch.Deleted, privateIngressHostsAB, nil}, + {watch.Added, nil, publicIngressHostC}, }, map[string]string{ - "bar.example.com": "pub.example.com", + "c.example.com": PublicTarget, }, }, { "an.example.com.", []mockEvent{ - {watch.Added, nil, testIngressA}, + {watch.Added, nil, privateIngressHostsAB}, }, map[string]string{}, }, { "example.com.", []mockEvent{ - {watch.Added, nil, testIngressC1}, + {watch.Added, nil, privateIngressHostE}, }, map[string]string{ - "baz.example.com": "priv.example.com", + "e.example.com": PrivateTarget, }, }, { "example.com.", []mockEvent{ - {watch.Added, nil, testIngressC1}, - {watch.Added, nil, testIngressC2}, + {watch.Added, nil, privateIngressHostE}, + {watch.Added, nil, privateIngressHostEDup}, }, map[string]string{ - "baz.example.com": "priv.example.com", + "e.example.com": PrivateTarget, }, }, { "example.com.", []mockEvent{ - {watch.Added, nil, testIngressC1}, - {watch.Added, nil, testIngressC3}, + {watch.Added, nil, privateIngressHostE}, + {watch.Added, nil, publicIngressHostEDup}, }, map[string]string{}, }, { "example.com.", []mockEvent{ - {watch.Added, nil, testIngressC1}, - {watch.Deleted, testIngressA, nil}, - {watch.Modified, testIngressC1, testIngressC3}, + {watch.Added, nil, privateIngressHostE}, + {watch.Deleted, privateIngressHostsAB, nil}, + {watch.Modified, privateIngressHostE, publicIngressHostEDup}, }, map[string]string{ - "baz.example.com": "pub.example.com", + "e.example.com": PublicTarget, }, }, } @@ -273,11 +285,11 @@ func TestRegistratorHandler(t *testing.T) { defer wg.Done() r.processUpdateQueue() }() - time.Sleep(1000 * time.Millisecond) // XXX + time.Sleep(10 * time.Millisecond) // XXX close(r.stopChannel) wg.Wait() if !reflect.DeepEqual(mdz.zoneData, test.data) { - t.Errorf("handler produced unexcepted zone data for test case #%02d: %+v", i, mdz.zoneData) + t.Errorf("handler produced unexcepted zone data for test case #%02d: %+v, expected: %+v", i, mdz.zoneData, test.data) } } } From 129d0bff63d1da9a7b8a7e4d1b0262ba22aae946 Mon Sep 17 00:00:00 2001 From: George Angel Date: Fri, 3 Mar 2017 10:47:11 +1000 Subject: [PATCH 02/11] Revert sleep times --- registrator_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registrator_test.go b/registrator_test.go index 04db8c2..9f075e1 100644 --- a/registrator_test.go +++ b/registrator_test.go @@ -285,7 +285,7 @@ func TestRegistratorHandler(t *testing.T) { defer wg.Done() r.processUpdateQueue() }() - time.Sleep(10 * time.Millisecond) // XXX + time.Sleep(1000 * time.Millisecond) // XXX close(r.stopChannel) wg.Wait() if !reflect.DeepEqual(mdz.zoneData, test.data) { From ba7da5bea7e1c6f328b7bf82345e8aa5ae906cb1 Mon Sep 17 00:00:00 2001 From: George Angel Date: Fri, 3 Mar 2017 11:02:13 +1000 Subject: [PATCH 03/11] Update README --- README.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d7a321e..4797330 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ingress53 is a service designed to run in kubernetes and maintain DNS records for the cluster's ingress resources in AWS Route53. -It will watch the kubernetes API (using the service token) for any Ingress resource changes and try to apply those records to route53 in Amazon, mapping the record to the "target name", which is the dns name of the ingress endpoint for your cluster. +It will watch the kubernetes API (using the service token) for any Ingress resource changes and try to apply those records to route53 in Amazon, mapping the record to the "target", which is the dns name of the ingress endpoint for your cluster. # Requirements @@ -42,7 +42,7 @@ The minimum AWS policy you can use: # Usage -ingress53 is slightly opinionated in that it assumes there are two kinds of ingress endpoints: public and private. A kubernetes selector is used to select public ingresses, while all others default to being private. +A kubernetes selector is used to specify the target (entry point of the cluster). You will need to create a dns record that points to your ingress endpoint[s]. We will use this to CNAME all ingress resource entries to that "target". @@ -50,7 +50,7 @@ Your set up might look like this: - A ingress controller (nginx/traefik) kubernetes service running on a nodePort (:8080) - ELB that serves all worker nodes on :8080 - - A CNAME for the elb `private.example.com` > `my-loadbalancer-1234567890.us-west-2.elb.amazonaws.com` + - A CNAME for the elb `private.cluster-entrypoint.com` > `my-loadbalancer-1234567890.us-west-2.elb.amazonaws.com` - ingress53 service running inside the cluster Now, if you were to create an ingress kubernetes resource: @@ -60,6 +60,8 @@ apiVersion: extensions/v1beta1 kind: Ingress metadata: name: my-app + labels: + ingress53.target: private.cluster-entrypoint.com spec: rules: - host: my-app.example.com @@ -71,15 +73,17 @@ spec: servicePort: 80 ``` -ingress53 will create a CNAME record in route53: `my-app.example.com` > `private.example.com` +ingress53 will create a CNAME record in route53: `my-app.example.com` > `private.cluster-entrypoint.com` You can test it locally (please refer to the command line help for more options): ```sh ./ingress53 \ -route53-zone-id=XXXXXXXXXXXXXX \ - -target-private=private.example.com \ - -target-public=public.example.com \ + -label-name=ingress53.target \ + -target=private.cluster-entrypoint.com \ + -target=public.cluster-entrypoint.com \ + -default-target=private.cluster-entrypoint.com \ -kubernetes-config=$HOME/.kube/config \ -dry-run ``` @@ -129,10 +133,11 @@ spec: - name: ingress53 image: utilitywarehouse/ingress53:v1.0.0 args: - - -route53-zone-id=XXXXXX - - -target-private=private.example.com - - -target-public=public.example.com - - -public-ingress-selector=ingress-tag-name:ingress-tag-value + - -route53-zone-id=XXXXXXXXXXXXXX \ + - -label-name=ingress53.target \ + - -target=private.cluster-entrypoint.com \ + - -target=public.cluster-entrypoint.com \ + - -default-target=private.cluster-entrypoint.com \ resources: requests: cpu: 10m From 0f968118bcc454651039615b3d3f1e9f86a46e19 Mon Sep 17 00:00:00 2001 From: George Angel Date: Fri, 3 Mar 2017 11:55:58 +1000 Subject: [PATCH 04/11] Better flow for default target --- registrator.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/registrator.go b/registrator.go index b409a0a..f01daaa 100644 --- a/registrator.go +++ b/registrator.go @@ -23,7 +23,6 @@ import ( var ( errRegistratorMissingOption = errors.New("missing required registrator option") errDNSEmptyAnswer = errors.New("DNS nameserver returned an empty answer") - errInvalidTarget = errors.New("Target specified with invalid format, should be: \"private:private-aws-elb.com\"") defaultResyncPeriod = 15 * time.Minute defaultBatchProcessCycle = 5 * time.Second dnsClient = &dns.Client{} @@ -162,31 +161,31 @@ func (r *registrator) handler(eventType watch.EventType, oldIngress *v1beta1.Ing case watch.Added: hostnames := getHostnamesFromIngress(newIngress) target := r.getTargetForIngress(newIngress) - metricUpdatesReceived.WithLabelValues(newIngress.Name, "add").Inc() - if len(hostnames) > 0 { + if len(hostnames) > 0 && target != "" { + metricUpdatesReceived.WithLabelValues(newIngress.Name, "add").Inc() log.Printf("[DEBUG] queued update of %d records for ingress %s, pointing to %s", len(hostnames), newIngress.Name, target) r.queueUpdates(route53.ChangeActionUpsert, hostnames, target) } case watch.Modified: newHostnames := getHostnamesFromIngress(newIngress) newTarget := r.getTargetForIngress(newIngress) - metricUpdatesReceived.WithLabelValues(newIngress.Name, "modify").Inc() - if len(newHostnames) > 0 { + if len(newHostnames) > 0 && newTarget != "" { + metricUpdatesReceived.WithLabelValues(newIngress.Name, "modify").Inc() log.Printf("[DEBUG] queued update of %d records for ingress %s, pointing to %s", len(newHostnames), newIngress.Name, newTarget) r.queueUpdates(route53.ChangeActionUpsert, newHostnames, newTarget) } oldHostnames := getHostnamesFromIngress(oldIngress) oldTarget := r.getTargetForIngress(oldIngress) diffHostnames := diffStringSlices(oldHostnames, newHostnames) - if len(diffHostnames) > 0 { + if len(diffHostnames) > 0 && oldTarget != "" { log.Printf("[DEBUG] queued deletion of %d records for ingress %s", len(diffHostnames), oldIngress.Name) r.queueUpdates(route53.ChangeActionDelete, diffHostnames, oldTarget) } case watch.Deleted: hostnames := getHostnamesFromIngress(oldIngress) target := r.getTargetForIngress(oldIngress) - metricUpdatesReceived.WithLabelValues(oldIngress.Name, "delete").Inc() - if len(hostnames) > 0 { + if len(hostnames) > 0 && target != "" { + metricUpdatesReceived.WithLabelValues(oldIngress.Name, "delete").Inc() log.Printf("[DEBUG] queued deletion of %d records for ingress %s", len(hostnames), oldIngress.Name) r.queueUpdates(route53.ChangeActionDelete, hostnames, target) } @@ -274,7 +273,14 @@ func (r *registrator) getTargetForIngress(ingress *v1beta1.Ingress) string { return sat.Target } } - return r.options.DefaultTarget + + if r.options.DefaultTarget != "" { + log.Printf("[WARN] didn't find a valid selector for ingress: %v, using default: %s", ingress, r.options.DefaultTarget) + return r.options.DefaultTarget + } else { + // no valid selector and no default target specified. Do nothing + return "" + } } func (r *registrator) pruneBatch(action string, records []cnameRecord) []cnameRecord { From a36e630ca23c5d4586afd4b983dc0ba153472732 Mon Sep 17 00:00:00 2001 From: George Angel Date: Fri, 3 Mar 2017 17:32:45 +1000 Subject: [PATCH 05/11] Add tests for new scenarios --- ingress_test.go | 15 +++++++++++++++ registrator_test.go | 24 ++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/ingress_test.go b/ingress_test.go index b2bbea1..2c30f18 100644 --- a/ingress_test.go +++ b/ingress_test.go @@ -118,6 +118,21 @@ var ( }, }, } + + nonRegisteredIngress = &v1beta1.Ingress{ + ObjectMeta: v1.ObjectMeta{ + Name: "nonRegisteredIngress", + Namespace: api.NamespaceDefault, + Labels: map[string]string{ + LabelName: "non-registered-target.aws.com", + }, + }, + Spec: v1beta1.IngressSpec{ + Rules: []v1beta1.IngressRule{ + {Host: "non-registered-target.example.com"}, + }, + }, + } ) func Test_getHostnamesFromIngress(t *testing.T) { diff --git a/registrator_test.go b/registrator_test.go index 9f075e1..b5b4e62 100644 --- a/registrator_test.go +++ b/registrator_test.go @@ -49,7 +49,7 @@ func TestNewRegistrator_defaults(t *testing.T) { } func TestRegistrator_GetTargetForIngress(t *testing.T) { - // ingress b + // ingress ab r, err := newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}, Targets: []string{PrivateTarget, PublicTarget}, LabelName: LabelName, Route53ZoneID: "c"}) if err != nil { t.Errorf("newRegistrator returned an unexpected error: %+v", err) @@ -60,7 +60,7 @@ func TestRegistrator_GetTargetForIngress(t *testing.T) { t.Errorf("getTargetForIngress returned unexpected value") } - // ingress a + // ingress c r, err = newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}, Targets: []string{PrivateTarget, PublicTarget}, LabelName: LabelName, Route53ZoneID: "c"}) if err != nil { t.Errorf("newRegistrator returned an unexpected error: %+v", err) @@ -69,6 +69,26 @@ func TestRegistrator_GetTargetForIngress(t *testing.T) { if target != PublicTarget { t.Errorf("getTargetForIngress returned unexpected value") } + + // ingress default + r, err = newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}, Targets: []string{PrivateTarget, PublicTarget}, DefaultTarget: PrivateTarget, LabelName: LabelName, Route53ZoneID: "c"}) + if err != nil { + t.Errorf("newRegistrator returned an unexpected error: %+v", err) + } + target = r.getTargetForIngress(ingressNoLabels) + if target != PrivateTarget { + t.Errorf("getTargetForIngress returned unexpected value") + } + + // ingress target not registered with ingress53 + r, err = newRegistratorWithOptions(registratorOptions{KubernetesConfig: &rest.Config{}, Targets: []string{PrivateTarget, PublicTarget}, LabelName: LabelName, Route53ZoneID: "c"}) + if err != nil { + t.Errorf("newRegistrator returned an unexpected error: %+v", err) + } + target = r.getTargetForIngress(nonRegisteredIngress) + if target != "" { + t.Errorf("getTargetForIngress returned unexpected value") + } } type mockDNSZone struct { From 003a3a242fa1f07baf9318b4b81152bdf139c59c Mon Sep 17 00:00:00 2001 From: George Angel Date: Fri, 3 Mar 2017 19:40:50 +1000 Subject: [PATCH 06/11] Slashes.. --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4797330..a1b3afe 100644 --- a/README.md +++ b/README.md @@ -133,11 +133,11 @@ spec: - name: ingress53 image: utilitywarehouse/ingress53:v1.0.0 args: - - -route53-zone-id=XXXXXXXXXXXXXX \ - - -label-name=ingress53.target \ - - -target=private.cluster-entrypoint.com \ - - -target=public.cluster-entrypoint.com \ - - -default-target=private.cluster-entrypoint.com \ + - -route53-zone-id=XXXXXXXXXXXXXX + - -label-name=ingress53.target + - -target=private.cluster-entrypoint.com + - -target=public.cluster-entrypoint.com + - -default-target=private.cluster-entrypoint.com resources: requests: cpu: 10m From 88f6ec511e167571d4b39b0f01d5bc7298044ff8 Mon Sep 17 00:00:00 2001 From: George Angel Date: Fri, 3 Mar 2017 21:58:50 +1000 Subject: [PATCH 07/11] Move update metric Its counting updates recieved, therefore should be outside the conditional that determines if it will be acted upon --- registrator.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/registrator.go b/registrator.go index f01daaa..39ae9fd 100644 --- a/registrator.go +++ b/registrator.go @@ -161,16 +161,16 @@ func (r *registrator) handler(eventType watch.EventType, oldIngress *v1beta1.Ing case watch.Added: hostnames := getHostnamesFromIngress(newIngress) target := r.getTargetForIngress(newIngress) + metricUpdatesReceived.WithLabelValues(newIngress.Name, "add").Inc() if len(hostnames) > 0 && target != "" { - metricUpdatesReceived.WithLabelValues(newIngress.Name, "add").Inc() log.Printf("[DEBUG] queued update of %d records for ingress %s, pointing to %s", len(hostnames), newIngress.Name, target) r.queueUpdates(route53.ChangeActionUpsert, hostnames, target) } case watch.Modified: newHostnames := getHostnamesFromIngress(newIngress) newTarget := r.getTargetForIngress(newIngress) + metricUpdatesReceived.WithLabelValues(newIngress.Name, "modify").Inc() if len(newHostnames) > 0 && newTarget != "" { - metricUpdatesReceived.WithLabelValues(newIngress.Name, "modify").Inc() log.Printf("[DEBUG] queued update of %d records for ingress %s, pointing to %s", len(newHostnames), newIngress.Name, newTarget) r.queueUpdates(route53.ChangeActionUpsert, newHostnames, newTarget) } @@ -184,8 +184,8 @@ func (r *registrator) handler(eventType watch.EventType, oldIngress *v1beta1.Ing case watch.Deleted: hostnames := getHostnamesFromIngress(oldIngress) target := r.getTargetForIngress(oldIngress) + metricUpdatesReceived.WithLabelValues(oldIngress.Name, "delete").Inc() if len(hostnames) > 0 && target != "" { - metricUpdatesReceived.WithLabelValues(oldIngress.Name, "delete").Inc() log.Printf("[DEBUG] queued deletion of %d records for ingress %s", len(hostnames), oldIngress.Name) r.queueUpdates(route53.ChangeActionDelete, hostnames, target) } From 1fb579be13c7d61402bc2a0439ae54ed76318c98 Mon Sep 17 00:00:00 2001 From: George Angel Date: Fri, 3 Mar 2017 22:30:52 +1000 Subject: [PATCH 08/11] rename var --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 90247f0..462896f 100644 --- a/main.go +++ b/main.go @@ -85,7 +85,7 @@ var ( ) func init() { - flag.Var(&targets, "targets", "List of endpoints (ELB) targets to map ingress records to") + flag.Var(&targets, "target", "List of endpoints (ELB) targets to map ingress records to") luf := &logutils.LevelFilter{ Levels: []logutils.LogLevel{"DEBUG", "INFO", "ERROR"}, From d735e4344a5c0f48741d707c02e6e0647307a2f7 Mon Sep 17 00:00:00 2001 From: George Angel Date: Fri, 3 Mar 2017 23:20:40 +1000 Subject: [PATCH 09/11] Reduce logging verbosity --- registrator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registrator.go b/registrator.go index 39ae9fd..474524a 100644 --- a/registrator.go +++ b/registrator.go @@ -275,7 +275,7 @@ func (r *registrator) getTargetForIngress(ingress *v1beta1.Ingress) string { } if r.options.DefaultTarget != "" { - log.Printf("[WARN] didn't find a valid selector for ingress: %v, using default: %s", ingress, r.options.DefaultTarget) + log.Printf("[WARN] didn't find a valid selector for ingress: %s, using default: %s", ingress.Name, r.options.DefaultTarget) return r.options.DefaultTarget } else { // no valid selector and no default target specified. Do nothing From 65a1f3349a07c588c2669b1dc37baf155d06e276 Mon Sep 17 00:00:00 2001 From: Dimitrios Karagiannis Date: Mon, 6 Mar 2017 13:19:05 +0000 Subject: [PATCH 10/11] merged init() into main() --- main.go | 28 +++++++++++----------------- registrator.go | 2 +- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/main.go b/main.go index 462896f..632f5e8 100644 --- a/main.go +++ b/main.go @@ -37,15 +37,14 @@ func (s *strslice) Set(value string) error { return nil } -// Define a flag to accumulate durations. Because it has a special type, -// we need to use the Var function and therefore create the flag during -// init. - -var targets strslice - var ( appGitHash = "master" + // Define a flag to accumulate durations. Because it has a special type, + // we need to use the Var function and therefore create the flag during + // init. + targets strslice + kubeConfig = flag.String("kubernetes-config", "", "path to the kubeconfig file, if unspecified then in-cluster config will be used") labelName = flag.String("label-name", "", "Kubernetes key of the label that specifies the target type") defaultTarget = flag.String("default-target", "", "Default target to use in the absense of matching labels") @@ -84,29 +83,24 @@ var ( ) ) -func init() { +func main() { + prometheus.MustRegister(metricUpdatesApplied) + prometheus.MustRegister(metricUpdatesReceived) + prometheus.MustRegister(metricUpdatesRejected) + flag.Var(&targets, "target", "List of endpoints (ELB) targets to map ingress records to") + flag.Parse() luf := &logutils.LevelFilter{ Levels: []logutils.LogLevel{"DEBUG", "INFO", "ERROR"}, MinLevel: logutils.LogLevel("INFO"), Writer: os.Stdout, } - if *debugLogs { luf.MinLevel = logutils.LogLevel("DEBUG") } - log.SetOutput(luf) - prometheus.MustRegister(metricUpdatesApplied) - prometheus.MustRegister(metricUpdatesReceived) - prometheus.MustRegister(metricUpdatesRejected) -} - -func main() { - flag.Parse() - ro := registratorOptions{ Targets: targets, LabelName: *labelName, diff --git a/registrator.go b/registrator.go index 474524a..4875ba5 100644 --- a/registrator.go +++ b/registrator.go @@ -275,7 +275,7 @@ func (r *registrator) getTargetForIngress(ingress *v1beta1.Ingress) string { } if r.options.DefaultTarget != "" { - log.Printf("[WARN] didn't find a valid selector for ingress: %s, using default: %s", ingress.Name, r.options.DefaultTarget) + log.Printf("[INFO] didn't find a valid selector for ingress: %s, using default: %s", ingress.Name, r.options.DefaultTarget) return r.options.DefaultTarget } else { // no valid selector and no default target specified. Do nothing From 532cf64e0b6e242aae68ac7492c3b568c14ae46a Mon Sep 17 00:00:00 2001 From: Dimitrios Karagiannis Date: Mon, 6 Mar 2017 13:30:06 +0000 Subject: [PATCH 11/11] more debug logs --- registrator.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/registrator.go b/registrator.go index 4875ba5..bf5e07a 100644 --- a/registrator.go +++ b/registrator.go @@ -275,10 +275,11 @@ func (r *registrator) getTargetForIngress(ingress *v1beta1.Ingress) string { } if r.options.DefaultTarget != "" { - log.Printf("[INFO] didn't find a valid selector for ingress: %s, using default: %s", ingress.Name, r.options.DefaultTarget) + log.Printf("[DEBUG] didn't find a valid selector for ingress: %s, using default: %s", ingress.Name, r.options.DefaultTarget) return r.options.DefaultTarget } else { // no valid selector and no default target specified. Do nothing + log.Printf("[DEBUG] cannot compute target for ingress: %s, invalid selector and no default target set", ingress.Name, r.options.DefaultTarget) return "" } }