From eabb1388729ac2e663446d9506be27203f27b432 Mon Sep 17 00:00:00 2001 From: Paul Crooks Date: Thu, 8 Jun 2023 16:23:47 +0000 Subject: [PATCH] feat(serverless-agent) Add support for ignored containers to the TF data source This commit contains: - TF premodifications should update the name parameter to CF style - Add the optional `ignore_containers` field to the Sysdig TF data source - Update docs to reflect the changes --- sysdig/cfn_preprocess_template.go | 13 +++ sysdig/data_source_sysdig_fargate_ECS_test.go | 32 ++++- ...ta_source_sysdig_fargate_workload_agent.go | 39 ++++++- sysdig/testfiles/ECSInstrumented.json | 2 +- .../testfiles/fargate_cmd_test_expected.json | 2 +- .../fargate_combined_test_expected.json | 2 +- .../fargate_entrypoint_test_expected.json | 2 +- .../testfiles/fargate_env_test_expected.json | 2 +- sysdig/testfiles/fargate_field_case_test.json | 16 +++ .../fargate_field_case_test_expected.json | 110 ++++++++++++++++++ .../fargate_ignore_container_test.json | 23 ++++ ...argate_ignore_container_test_expected.json | 75 ++++++++++++ ...fargate_linuxparameters_test_expected.json | 2 +- .../testfiles/fargate_log_group_expected.json | 2 +- .../fargate_volumesfrom_test_expected.json | 2 +- website/docs/d/fargate_workload_agent.md | 1 + 16 files changed, 312 insertions(+), 13 deletions(-) create mode 100644 sysdig/testfiles/fargate_field_case_test.json create mode 100644 sysdig/testfiles/fargate_field_case_test_expected.json create mode 100644 sysdig/testfiles/fargate_ignore_container_test.json create mode 100644 sysdig/testfiles/fargate_ignore_container_test_expected.json diff --git a/sysdig/cfn_preprocess_template.go b/sysdig/cfn_preprocess_template.go index 7abbbaa3..0f35ffef 100644 --- a/sysdig/cfn_preprocess_template.go +++ b/sysdig/cfn_preprocess_template.go @@ -96,6 +96,19 @@ func terraformPreModifications(ctx context.Context, patchedStack []byte) ([]byte } } + if container.Exists("name") { + passthrough, _ := GetValueFromTemplate(container.S("name")) + _, err = container.Set(passthrough, "Name") + if err != nil { + return nil, fmt.Errorf("Could not update Name field: %v", err) + } + + err = container.Delete("name") + if err != nil { + return nil, fmt.Errorf("could not delete name in the Container definition: %w", err) + } + } + if container.Exists("environment") { for _, env := range container.S("environment").Children() { if env.Exists("name") { diff --git a/sysdig/data_source_sysdig_fargate_ECS_test.go b/sysdig/data_source_sysdig_fargate_ECS_test.go index 97cfbdd3..01a8bccb 100644 --- a/sysdig/data_source_sysdig_fargate_ECS_test.go +++ b/sysdig/data_source_sysdig_fargate_ECS_test.go @@ -25,6 +25,8 @@ var ( SysdigLogging: "sysdig_logging", } + testIgnoreContainers = []string{} + testContainerDefinitionFiles = []string{ "fargate_entrypoint_test", "fargate_env_test", @@ -32,6 +34,7 @@ var ( "fargate_linuxparameters_test", "fargate_combined_test", "fargate_volumesfrom_test", + "fargate_field_case_test", } ) @@ -86,7 +89,7 @@ func TestECStransformation(t *testing.T) { RecipeConfig: string(jsonConf), } - patchedOutput, err := patchFargateTaskDefinition(context.Background(), string(inputfile), kiltConfig, nil) + patchedOutput, err := patchFargateTaskDefinition(context.Background(), string(inputfile), kiltConfig, nil, &testIgnoreContainers) if err != nil { t.Fatalf("Cannot execute PatchFargateTaskDefinition : %v", err.Error()) } @@ -105,6 +108,7 @@ func TestECStransformation(t *testing.T) { VolumesFrom []interface{} `json:"VolumesFrom"` LogConfiguration interface{} `json:"LogConfiguration"` Name string `json:"Name"` + Name2 string `json:"name"` Image2 string `json:"image"` EntryPoint2 string `json:"entryPoint"` } @@ -120,7 +124,9 @@ func TestECStransformation(t *testing.T) { t.Fatalf("Error Unmarshaling expected Container definitions: %v", err.Error()) } + // Check if Name key is correct assert.Equal(t, expectedContainerDefinitions[0].Name, patchedContainerDefinitions[0].Name) + assert.Equal(t, expectedContainerDefinitions[0].Name2, "") // The order received from patchedOutput changes continuously hence it is important to check if the arrays of expected and actual are equal without order being correct. This check also // helps with checking if key/value is named "Name" and "Value" accordingly. @@ -148,7 +154,7 @@ func TestTransform(t *testing.T) { } inputContainerDefinition, _ := os.ReadFile("testfiles/" + testName + ".json") - patched, _ := patchFargateTaskDefinition(context.Background(), string(inputContainerDefinition), kiltConfig, nil) + patched, _ := patchFargateTaskDefinition(context.Background(), string(inputContainerDefinition), kiltConfig, nil, &testIgnoreContainers) expectedContainerDefinition, _ := os.ReadFile("testfiles/" + testName + "_expected.json") sortAndCompare(t, expectedContainerDefinition, []byte(*patched)) @@ -173,8 +179,28 @@ func TestLogGroup(t *testing.T) { } inputContainerDefinition, _ := os.ReadFile("testfiles/fargate_log_group.json") - patched, _ := patchFargateTaskDefinition(context.Background(), string(inputContainerDefinition), kiltConfig, logConfig) + patched, _ := patchFargateTaskDefinition(context.Background(), string(inputContainerDefinition), kiltConfig, logConfig, &testIgnoreContainers) expectedContainerDefinition, _ := os.ReadFile("testfiles/fargate_log_group_expected.json") sortAndCompare(t, expectedContainerDefinition, []byte(*patched)) } + +func TestIgnoreContainers(t *testing.T) { + jsonConfig, _ := json.Marshal(testKiltDefinition) + kiltConfig := &cfnpatcher.Configuration{ + Kilt: agentinoKiltDefinition, + ImageAuthSecret: "image_auth_secret", + OptIn: false, + UseRepositoryHints: true, + RecipeConfig: string(jsonConfig), + } + + fileTemplate := "fargate_ignore_container_test" + ignoreContainers := []string{"other", "another"} + + inputContainerDefinition, _ := os.ReadFile("testfiles/" + fileTemplate + ".json") + patched, _ := patchFargateTaskDefinition(context.Background(), string(inputContainerDefinition), kiltConfig, nil, &ignoreContainers) + expectedContainerDefinition, _ := os.ReadFile("testfiles/" + fileTemplate + "_expected.json") + + sortAndCompare(t, expectedContainerDefinition, []byte(*patched)) +} diff --git a/sysdig/data_source_sysdig_fargate_workload_agent.go b/sysdig/data_source_sysdig_fargate_workload_agent.go index e6a9dcdd..c5a71669 100644 --- a/sysdig/data_source_sysdig_fargate_workload_agent.go +++ b/sysdig/data_source_sysdig_fargate_workload_agent.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "github.com/Jeffail/gabs/v2" "github.com/aws/aws-sdk-go/aws" @@ -81,6 +82,12 @@ func dataSourceSysdigFargateWorkloadAgent() *schema.Resource { Description: "the collector port to connect to", Optional: true, }, + "ignore_containers": { + Type: schema.TypeList, + Description: "list of containers to not add instrumentation to", + Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, "log_configuration": { Type: schema.TypeSet, MaxItems: 1, @@ -119,9 +126,15 @@ func dataSourceSysdigFargateWorkloadAgent() *schema.Resource { } } +type cfnTag struct { + Key string `json:"Key"` + Value string `json:"Value"` +} + type cfnProperties struct { RequiresCompatibilities []string `json:"RequiresCompatibilities"` ContainerDefinitions []map[string]interface{} `json:"ContainerDefinitions"` + Tags []cfnTag `json:"Tags"` } type cfnResource struct { @@ -171,13 +184,23 @@ func fargatePostKiltModifications(patchedBytes []byte, logConfig map[string]inte } // PatchFargateTaskDefinition modifies the container definitions -func patchFargateTaskDefinition(ctx context.Context, containerDefinitions string, kiltConfig *cfnpatcher.Configuration, logConfig map[string]interface{}) (patched *string, err error) { +func patchFargateTaskDefinition(ctx context.Context, containerDefinitions string, kiltConfig *cfnpatcher.Configuration, logConfig map[string]interface{}, ignoreContainers *[]string) (patched *string, err error) { var cdefs []map[string]interface{} err = json.Unmarshal([]byte(containerDefinitions), &cdefs) if err != nil { return nil, err } + // Convert the ignore containers list into Kilt tags for the patcher + tags := []cfnTag{} + if len(*ignoreContainers) > 0 { + containerTagValue := strings.Join(*ignoreContainers, ":") + tags = append(tags, cfnTag{ + Key: "kilt-ignore-containers", + Value: containerTagValue, + }) + } + stack := cfnStack{ Resources: map[string]cfnResource{ "kilt": { @@ -185,6 +208,7 @@ func patchFargateTaskDefinition(ctx context.Context, containerDefinitions string Properties: cfnProperties{ RequiresCompatibilities: []string{"FARGATE"}, ContainerDefinitions: cdefs, + Tags: tags, }, }, }, @@ -270,12 +294,23 @@ func dataSourceSysdigFargateWorkloadAgentRead(ctx context.Context, d *schema.Res containerDefinitions := d.Get("container_definitions").(string) + ignoreContainersField := d.Get("ignore_containers") + ignoreContainers := []string{} + if ignoreContainersField != nil { + for _, value := range ignoreContainersField.([]interface{}) { + if value_str, ok := value.(string); ok { + value_str = strings.TrimSpace(value_str) + ignoreContainers = append(ignoreContainers, value_str) + } + } + } + logConfig := map[string]interface{}{} if logConfiguration := d.Get("log_configuration").(*schema.Set).List(); len(logConfiguration) > 0 { logConfig = logConfiguration[0].(map[string]interface{}) } - outputContainerDefinitions, err := patchFargateTaskDefinition(ctx, containerDefinitions, kiltConfig, logConfig) + outputContainerDefinitions, err := patchFargateTaskDefinition(ctx, containerDefinitions, kiltConfig, logConfig, &ignoreContainers) if err != nil { return diag.Errorf("Error applying configuration patch: %v", err.Error()) } diff --git a/sysdig/testfiles/ECSInstrumented.json b/sysdig/testfiles/ECSInstrumented.json index 73ecf856..c26b3833 100644 --- a/sysdig/testfiles/ECSInstrumented.json +++ b/sysdig/testfiles/ECSInstrumented.json @@ -66,7 +66,7 @@ "awslogs-stream-prefix": "ecs" } }, - "name": "busybox" + "Name": "busybox" }, { "EntryPoint": [ diff --git a/sysdig/testfiles/fargate_cmd_test_expected.json b/sysdig/testfiles/fargate_cmd_test_expected.json index 2eab9dff..7e548810 100644 --- a/sysdig/testfiles/fargate_cmd_test_expected.json +++ b/sysdig/testfiles/fargate_cmd_test_expected.json @@ -1,6 +1,6 @@ [ { - "name": "test", + "Name": "test", "Image": "test_image:latest", "EntryPoint": [ "/opt/draios/bin/instrument" diff --git a/sysdig/testfiles/fargate_combined_test_expected.json b/sysdig/testfiles/fargate_combined_test_expected.json index 64e906d2..2e08b411 100644 --- a/sysdig/testfiles/fargate_combined_test_expected.json +++ b/sysdig/testfiles/fargate_combined_test_expected.json @@ -1,6 +1,6 @@ [ { - "name": "test", + "Name": "test", "Image": "test_image:latest", "EntryPoint": [ "/opt/draios/bin/instrument" diff --git a/sysdig/testfiles/fargate_entrypoint_test_expected.json b/sysdig/testfiles/fargate_entrypoint_test_expected.json index a4282e75..d6f0c8cb 100644 --- a/sysdig/testfiles/fargate_entrypoint_test_expected.json +++ b/sysdig/testfiles/fargate_entrypoint_test_expected.json @@ -1,6 +1,6 @@ [ { - "name": "test", + "Name": "test", "Image": "test_image:latest", "EntryPoint": [ "/opt/draios/bin/instrument" diff --git a/sysdig/testfiles/fargate_env_test_expected.json b/sysdig/testfiles/fargate_env_test_expected.json index 45110166..46d0ceba 100644 --- a/sysdig/testfiles/fargate_env_test_expected.json +++ b/sysdig/testfiles/fargate_env_test_expected.json @@ -1,6 +1,6 @@ [ { - "name": "test", + "Name": "test", "Image": "test_image:latest", "EntryPoint": [ "/opt/draios/bin/instrument" diff --git a/sysdig/testfiles/fargate_field_case_test.json b/sysdig/testfiles/fargate_field_case_test.json new file mode 100644 index 00000000..19b84800 --- /dev/null +++ b/sysdig/testfiles/fargate_field_case_test.json @@ -0,0 +1,16 @@ +[ + { + "name": "test", + "image": "test_image:latest", + "entryPoint": [ + "/bin/test" + ] + }, + { + "Name": "other", + "Image": "other_image:latest", + "entryPoint": [ + "/bin/other" + ] + } +] \ No newline at end of file diff --git a/sysdig/testfiles/fargate_field_case_test_expected.json b/sysdig/testfiles/fargate_field_case_test_expected.json new file mode 100644 index 00000000..26504b7d --- /dev/null +++ b/sysdig/testfiles/fargate_field_case_test_expected.json @@ -0,0 +1,110 @@ +[ + { + "Name": "test", + "Image": "test_image:latest", + "EntryPoint": [ + "/opt/draios/bin/instrument" + ], + "Command": [ + "/bin/test" + ], + "Environment": [ + { + "Name": "SYSDIG_ORCHESTRATOR_PORT", + "Value": "orchestrator_port" + }, + { + "Name": "SYSDIG_COLLECTOR", + "Value": "collector_host" + }, + { + "Name": "SYSDIG_COLLECTOR_PORT", + "Value": "collector_port" + }, + { + "Name": "SYSDIG_ACCESS_KEY", + "Value": "sysdig_access_key" + }, + { + "Name": "SYSDIG_LOGGING", + "Value": "sysdig_logging" + }, + { + "Name": "SYSDIG_ORCHESTRATOR", + "Value": "orchestrator_host" + } + ], + "LinuxParameters": { + "Capabilities": { + "Add": [ + "SYS_PTRACE" + ] + } + }, + "VolumesFrom": [ + { + "ReadOnly": true, + "SourceContainer": "SysdigInstrumentation" + } + ] + }, + { + "Name": "other", + "Image": "other_image:latest", + "EntryPoint": [ + "/opt/draios/bin/instrument" + ], + "Command": [ + "/bin/other" + ], + "Environment": [ + { + "Name": "SYSDIG_ORCHESTRATOR_PORT", + "Value": "orchestrator_port" + }, + { + "Name": "SYSDIG_COLLECTOR", + "Value": "collector_host" + }, + { + "Name": "SYSDIG_COLLECTOR_PORT", + "Value": "collector_port" + }, + { + "Name": "SYSDIG_ACCESS_KEY", + "Value": "sysdig_access_key" + }, + { + "Name": "SYSDIG_LOGGING", + "Value": "sysdig_logging" + }, + { + "Name": "SYSDIG_ORCHESTRATOR", + "Value": "orchestrator_host" + } + ], + "LinuxParameters": { + "Capabilities": { + "Add": [ + "SYS_PTRACE" + ] + } + }, + "VolumesFrom": [ + { + "ReadOnly": true, + "SourceContainer": "SysdigInstrumentation" + } + ] + }, + { + "EntryPoint": [ + "/opt/draios/bin/logwriter" + ], + "Image": "workload_agent_image", + "Name": "SysdigInstrumentation", + "RepositoryCredentials": { + "CredentialsParameter": "image_auth_secret" + } + } +] \ No newline at end of file diff --git a/sysdig/testfiles/fargate_ignore_container_test.json b/sysdig/testfiles/fargate_ignore_container_test.json new file mode 100644 index 00000000..535cc286 --- /dev/null +++ b/sysdig/testfiles/fargate_ignore_container_test.json @@ -0,0 +1,23 @@ +[ + { + "name": "test", + "image": "test_image:latest", + "entryPoint": [ + "/bin/test" + ] + }, + { + "name": "other", + "image": "other_image:latest", + "entryPoint": [ + "/bin/other" + ] + }, + { + "name": "another", + "image": "another_image:latest", + "entryPoint": [ + "/bin/another" + ] + } +] \ No newline at end of file diff --git a/sysdig/testfiles/fargate_ignore_container_test_expected.json b/sysdig/testfiles/fargate_ignore_container_test_expected.json new file mode 100644 index 00000000..9dc46c31 --- /dev/null +++ b/sysdig/testfiles/fargate_ignore_container_test_expected.json @@ -0,0 +1,75 @@ +[ + { + "Name": "test", + "Image": "test_image:latest", + "EntryPoint": [ + "/opt/draios/bin/instrument" + ], + "Command": [ + "/bin/test" + ], + "Environment": [ + { + "Name": "SYSDIG_ORCHESTRATOR_PORT", + "Value": "orchestrator_port" + }, + { + "Name": "SYSDIG_COLLECTOR", + "Value": "collector_host" + }, + { + "Name": "SYSDIG_COLLECTOR_PORT", + "Value": "collector_port" + }, + { + "Name": "SYSDIG_ACCESS_KEY", + "Value": "sysdig_access_key" + }, + { + "Name": "SYSDIG_LOGGING", + "Value": "sysdig_logging" + }, + { + "Name": "SYSDIG_ORCHESTRATOR", + "Value": "orchestrator_host" + } + ], + "LinuxParameters": { + "Capabilities": { + "Add": [ + "SYS_PTRACE" + ] + } + }, + "VolumesFrom": [ + { + "ReadOnly": true, + "SourceContainer": "SysdigInstrumentation" + } + ] + }, + { + "Name": "other", + "Image": "other_image:latest", + "EntryPoint": [ + "/bin/other" + ] + }, + { + "Name": "another", + "Image": "another_image:latest", + "EntryPoint": [ + "/bin/another" + ] + }, + { + "EntryPoint": [ + "/opt/draios/bin/logwriter" + ], + "Image": "workload_agent_image", + "Name": "SysdigInstrumentation", + "RepositoryCredentials": { + "CredentialsParameter": "image_auth_secret" + } + } +] \ No newline at end of file diff --git a/sysdig/testfiles/fargate_linuxparameters_test_expected.json b/sysdig/testfiles/fargate_linuxparameters_test_expected.json index 4db87a25..7d8c970b 100644 --- a/sysdig/testfiles/fargate_linuxparameters_test_expected.json +++ b/sysdig/testfiles/fargate_linuxparameters_test_expected.json @@ -1,6 +1,6 @@ [ { - "name": "test", + "Name": "test", "Image": "test_image:latest", "EntryPoint": [ "/opt/draios/bin/instrument" diff --git a/sysdig/testfiles/fargate_log_group_expected.json b/sysdig/testfiles/fargate_log_group_expected.json index e9657926..d641745c 100644 --- a/sysdig/testfiles/fargate_log_group_expected.json +++ b/sysdig/testfiles/fargate_log_group_expected.json @@ -1,6 +1,6 @@ [ { - "name": "test", + "Name": "test", "Image": "test_image:latest", "EntryPoint": [ "/opt/draios/bin/instrument" diff --git a/sysdig/testfiles/fargate_volumesfrom_test_expected.json b/sysdig/testfiles/fargate_volumesfrom_test_expected.json index 3962f440..f410e048 100644 --- a/sysdig/testfiles/fargate_volumesfrom_test_expected.json +++ b/sysdig/testfiles/fargate_volumesfrom_test_expected.json @@ -1,6 +1,6 @@ [ { - "name": "test", + "Name": "test", "Image": "test_image:latest", "EntryPoint": [ "/opt/draios/bin/instrument" diff --git a/website/docs/d/fargate_workload_agent.md b/website/docs/d/fargate_workload_agent.md index 84abf1b1..28d12cd2 100644 --- a/website/docs/d/fargate_workload_agent.md +++ b/website/docs/d/fargate_workload_agent.md @@ -42,6 +42,7 @@ data "sysdig_fargate_workload_agent" "instrumented_containers" { * `stream_prefix` - Prefix for the instrumentation log stream * `region` - The AWS region where the target log group resides * `sysdig_logging` - (Optional) The instrumentation logging level: `trace`, `debug`, `info`, `warning`, `error`, `silent`. +* `ignore_containers` - (Optional) A list of containers in this data source that should not be instrumented. ## Attributes Reference