From db5c64e58c82cea832c8f2dd5b0f4e81fefd4545 Mon Sep 17 00:00:00 2001 From: Easwar Swaminathan Date: Thu, 12 Aug 2021 14:00:48 -0700 Subject: [PATCH] Add deployment info as part of Node metadata (#20) --- deployment_info.go | 62 +++++++++++++++++++++++++++++ go.mod | 5 ++- go.sum | 11 +++--- main.go | 99 ++++++++++++++++++++++++++++++++++++++++------ main_test.go | 90 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 249 insertions(+), 18 deletions(-) create mode 100644 deployment_info.go diff --git a/deployment_info.go b/deployment_info.go new file mode 100644 index 0000000..5a78a55 --- /dev/null +++ b/deployment_info.go @@ -0,0 +1,62 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net/http" + "net/url" + "time" +) + +type deploymentType int + +const ( + deploymentTypeUnknown deploymentType = iota + deploymentTypeGKE + deploymentTypeGCE +) + +// getDeploymentType tries to talk the metadata server at +// http://metadata.google.internal and uses a response header with key "Server" +// to determine the deployment type. +func getDeploymentType() deploymentType { + parsedUrl, err := url.Parse("http://metadata.google.internal") + if err != nil { + return deploymentTypeUnknown + } + client := &http.Client{Timeout: 5 * time.Second} + req := &http.Request{ + Method: "GET", + URL: parsedUrl, + Header: http.Header{"Metadata-Flavor": {"Google"}}, + } + resp, err := client.Do(req) + if err != nil { + return deploymentTypeUnknown + } + resp.Body.Close() + + // Read the "Server" header to determine the deployment type. + vals := resp.Header.Values("Server") + for _, val := range vals { + switch val { + case "GKE Metadata Server": + return deploymentTypeGKE + case "Metadata Server for VM": + return deploymentTypeGCE + } + } + return deploymentTypeUnknown +} diff --git a/go.mod b/go.mod index e3f11ff..d6164d2 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module td-grpc-bootstrap go 1.13 require ( - github.com/google/go-cmp v0.5.2 - github.com/google/uuid v1.1.1 + github.com/google/go-cmp v0.5.4 + github.com/google/uuid v1.1.2 + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect ) diff --git a/go.sum b/go.sum index c440cfa..dfe67e0 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index 2cec0b2..788ef21 100644 --- a/main.go +++ b/main.go @@ -36,11 +36,16 @@ var ( outputName = flag.String("output", "-", "output file name") gcpProjectNumber = flag.Int64("gcp-project-number", 0, "the gcp project number. If unknown, can be found via 'gcloud projects list'") - vpcNetworkName = flag.String("vpc-network-name", "default", "VPC network name") - localityZone = flag.String("locality-zone", "", "the locality zone to use, instead of retrieving it from the metadata server. Useful when not running on GCP and/or for testing") - includeV3Features = flag.Bool("include-v3-features-experimental", true, "whether or not to generate configs which works with the xDS v3 implementation in TD. This flag is EXPERIMENTAL and may be changed or removed in a later release.") - includePSMSecurity = flag.Bool("include-psm-security-experimental", false, "whether or not to generate config required for PSM security. This flag is EXPERIMENTAL and may be changed or removed in a later release.") - secretsDir = flag.String("secrets-dir-experimental", "/var/run/secrets/workload-spiffe-credentials", "path to a directory containing TLS certificates and keys required for PSM security. Used only if --include-psm-security-experimental is set. This flag is EXPERIMENTAL and may be changed or removed in a later release.") + vpcNetworkName = flag.String("vpc-network-name", "default", "VPC network name") + localityZone = flag.String("locality-zone", "", "the locality zone to use, instead of retrieving it from the metadata server. Useful when not running on GCP and/or for testing") + includeV3Features = flag.Bool("include-v3-features-experimental", true, "whether or not to generate configs which works with the xDS v3 implementation in TD. This flag is EXPERIMENTAL and may be changed or removed in a later release.") + includePSMSecurity = flag.Bool("include-psm-security-experimental", false, "whether or not to generate config required for PSM security. This flag is EXPERIMENTAL and may be changed or removed in a later release.") + secretsDir = flag.String("secrets-dir-experimental", "/var/run/secrets/workload-spiffe-credentials", "path to a directory containing TLS certificates and keys required for PSM security. Used only if --include-psm-security-experimental is set. This flag is EXPERIMENTAL and may be changed or removed in a later release.") + includeDeploymentInfo = flag.Bool("include-deployment-info-experimental", false, "whether or not to generate config which contains deployment related information. This flag is EXPERIMENTAL and may be changed or removed in a later release.") + gkeClusterName = flag.String("gke-cluster-name-experimental", "", "GKE cluster name to use, instead of retrieving it from the metadata server. This flag is EXPERIMENTAL and may be changed or removed in a later release.") + gkePodName = flag.String("gke-pod-name-experimental", "", "GKE pod name to use, instead of reading it from $HOSTNAME or /etc/hostname file. This flag is EXPERIMENTAL and may be changed or removed in a later release.") + gkeNamespace = flag.String("gke-namespace-experimental", "", "GKE namespace to use. This flag is EXPERIMENTAL and may be changed or removed in a later release.") + gceVM = flag.String("gce-vm-experimental", "", "GCE VM name to use, instead of reading it from the metadata server. This flag is EXPERIMENTAL and may be changed or removed in a later release.") ) func main() { @@ -70,6 +75,42 @@ func main() { zone = "" } } + + // Generate deployment info from metadata server or from command-line + // arguments, with the latter taking preference. + var deploymentInfo map[string]string + if *includeDeploymentInfo { + dType := getDeploymentType() + switch dType { + case deploymentTypeGKE: + cluster := *gkeClusterName + if cluster == "" { + cluster = getClusterName() + } + pod := *gkePodName + if pod == "" { + pod = getPodName() + } + deploymentInfo = map[string]string{ + "GKE-CLUSTER": cluster, + "GCP-ZONE": zone, + "INSTANCE-IP": ip, + "GKE-POD": pod, + "GKE-NAMESPACE": *gkeNamespace, + } + case deploymentTypeGCE: + vmName := *gceVM + if vmName == "" { + vmName = getVMName() + } + deploymentInfo = map[string]string{ + "GCE-VM": vmName, + "GCP-ZONE": zone, + "INSTANCE-IP": ip, + } + } + } + config, err := generate(configInput{ xdsServerUri: *xdsServerUri, gcpProjectNumber: *gcpProjectNumber, @@ -80,6 +121,7 @@ func main() { includePSMSecurity: *includePSMSecurity, secretsDir: *secretsDir, metadataLabels: nodeMetadata, + deploymentInfo: deploymentInfo, }) if err != nil { fmt.Fprintf(os.Stderr, "Failed to generate config: %s\n", err) @@ -122,6 +164,7 @@ type configInput struct { includePSMSecurity bool secretsDir string metadataLabels map[string]string + deploymentInfo map[string]string } func generate(in configInput) ([]byte, error) { @@ -140,7 +183,7 @@ func generate(in configInput) ([]byte, error) { Locality: &locality{ Zone: in.zone, }, - Metadata: map[string]string{ + Metadata: map[string]interface{}{ "TRAFFICDIRECTOR_NETWORK_NAME": in.vpcNetworkName, "TRAFFICDIRECTOR_GCP_PROJECT_NUMBER": strconv.FormatInt(in.gcpProjectNumber, 10), }, @@ -176,6 +219,9 @@ func generate(in configInput) ([]byte, error) { } c.ServerListenerResourceNameTemplate = "grpc/server?xds.resource.listening_address=%s" } + if in.deploymentInfo != nil { + c.Node.Metadata["TRAFFIC_DIRECTOR_CLIENT_ENVIRONMENT"] = in.deploymentInfo + } return json.MarshalIndent(c, "", " ") } @@ -219,6 +265,37 @@ func getProjectId() (int64, error) { return projectId, nil } +func getClusterName() string { + cluster, err := getFromMetadata("http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-name") + if err != nil { + fmt.Fprintf(os.Stderr, "could not discover GKE cluster name: %v", err) + return "" + } + return string(cluster) +} + +// For overriding in unit tests. +var readHostNameFile = func() ([]byte, error) { + return ioutil.ReadFile("/etc/hostname") +} + +func getPodName() string { + pod, err := os.Hostname() + if err != nil { + fmt.Fprintf(os.Stderr, "could not discover GKE pod name: %v", err) + } + return pod +} + +func getVMName() string { + vm, err := getFromMetadata("http://metadata.google.internal/computeMetadata/v1/instance/name") + if err != nil { + fmt.Fprintf(os.Stderr, "could not discover GCE VM name: %v", err) + return "" + } + return string(vm) +} + func getFromMetadata(urlStr string) ([]byte, error) { parsedUrl, err := url.Parse(urlStr) if err != nil { @@ -265,11 +342,11 @@ type creds struct { } type node struct { - Id string `json:"id,omitempty"` - Cluster string `json:"cluster,omitempty"` - Metadata map[string]string `json:"metadata,omitempty"` - Locality *locality `json:"locality,omitempty"` - BuildVersion string `json:"build_version,omitempty"` + Id string `json:"id,omitempty"` + Cluster string `json:"cluster,omitempty"` + Metadata map[string]interface{} `json:"metadata,omitempty"` + Locality *locality `json:"locality,omitempty"` + BuildVersion string `json:"build_version,omitempty"` } type locality struct { diff --git a/main_test.go b/main_test.go index 2cd1dd8..25fcb5b 100644 --- a/main_test.go +++ b/main_test.go @@ -157,6 +157,60 @@ func TestGenerate(t *testing.T) { } }, "server_listener_resource_name_template": "grpc/server?xds.resource.listening_address=%s" +}`, + }, + { + desc: "happy case with deployment info", + input: configInput{ + xdsServerUri: "example.com:443", + gcpProjectNumber: 123456789012345, + vpcNetworkName: "thedefault", + ip: "10.9.8.7", + zone: "uscentral-5", + includeV3Features: true, + deploymentInfo: map[string]string{ + "GCP-ZONE": "uscentral-5", + "GKE-CLUSTER": "test-gke-cluster", + "GKE-NAMESPACE": "test-gke-namespace", + "GKE-POD": "test-gke-pod", + "INSTANCE-IP": "10.9.8.7", + "GCE-VM": "test-gce-vm", + }, + }, + wantOutput: `{ + "xds_servers": [ + { + "server_uri": "example.com:443", + "channel_creds": [ + { + "type": "google_default" + } + ], + "server_features": [ + "xds_v3" + ] + } + ], + "node": { + "id": "projects/123456789012345/networks/thedefault/nodes/9566c74d-1003-4c4d-bbbb-0407d1e2c649", + "cluster": "cluster", + "metadata": { + "INSTANCE_IP": "10.9.8.7", + "TRAFFICDIRECTOR_GCP_PROJECT_NUMBER": "123456789012345", + "TRAFFICDIRECTOR_NETWORK_NAME": "thedefault", + "TRAFFIC_DIRECTOR_CLIENT_ENVIRONMENT": { + "GCE-VM": "test-gce-vm", + "GCP-ZONE": "uscentral-5", + "GKE-CLUSTER": "test-gke-cluster", + "GKE-NAMESPACE": "test-gke-namespace", + "GKE-POD": "test-gke-pod", + "INSTANCE-IP": "10.9.8.7" + } + }, + "locality": { + "zone": "uscentral-5" + } + } }`, }, } @@ -220,6 +274,42 @@ func TestGetProjectId(t *testing.T) { } } +func TestGetClusterName(t *testing.T) { + server := httptest.NewServer(nil) + defer server.Close() + overrideHTTP(server) + want := "test-cluster" + http.HandleFunc("metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-name", + func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Metadata-Flavor") != "Google" { + http.Error(w, "Missing Metadata-Flavor", 403) + return + } + w.Write([]byte("test-cluster")) + }) + if got := getClusterName(); got != want { + t.Fatalf("getClusterName() = %s, want: %s", got, want) + } +} + +func TestGetVMName(t *testing.T) { + server := httptest.NewServer(nil) + defer server.Close() + overrideHTTP(server) + want := "test-vm" + http.HandleFunc("metadata.google.internal/computeMetadata/v1/instance/name", + func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Metadata-Flavor") != "Google" { + http.Error(w, "Missing Metadata-Flavor", 403) + return + } + w.Write([]byte("test-vm")) + }) + if got := getVMName(); got != want { + t.Fatalf("getVMName() = %s, want: %s", got, want) + } +} + func overrideHTTP(s *httptest.Server) { http.DefaultTransport = &http.Transport{ DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {