Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add tag support to LogGroup resource #53

Merged
merged 7 commits into from
May 20, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 44 additions & 4 deletions aws-logs-loggroup/aws-logs-loggroup.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,32 @@
"typeName": "AWS::Logs::LogGroup",
"description": "Resource schema for AWS::Logs::LogGroup",
"sourceUrl": "https://github.com/aws-cloudformation/aws-cloudformation-resource-providers-logs.git",
"definitions": {
"Tag": {
"description": "A key-value pair to associate with a resource.",
"type": "object",
"properties": {
"Key": {
"type": "string",
"description": "The key name of the tag. You can specify a value that is 1 to 128 Unicode characters in length and cannot be prefixed with aws:. You can use any of the following characters: the set of Unicode letters, digits, whitespace, _, ., :, /, =, +, - and @.",
"minLength": 1,
"maxLength": 128,
"pattern": "^([\\w\\s\\d_.:/=+\\-@]+)$"
},
"Value": {
"type": "string",
"description": "The value for the tag. You can specify a value that is 0 to 256 Unicode characters in length. You can use any of the following characters: the set of Unicode letters, digits, whitespace, _, ., :, /, =, +, - and @.",
"minLength": 0,
"maxLength": 256,
"pattern": "^([\\w\\s\\d_.:/=+\\-@]*)$"
}
},
"required": [
"Key",
"Value"
]
}
},
"properties": {
"LogGroupName": {
"description": "The name of the log group. If you don't specify a name, AWS CloudFormation generates a unique ID for the log group.",
Expand Down Expand Up @@ -39,6 +65,15 @@
3653
]
},
"Tags": {
"description": "An array of key-value pairs to apply to this resource.",
"type": "array",
"uniqueItems": true,
"insertionOrder": false,
"items": {
"$ref": "#/definitions/Tag"
}
},
"Arn": {
"description": "The CloudWatch log group ARN.",
"type": "string"
Expand All @@ -49,12 +84,14 @@
"permissions": [
"logs:DescribeLogGroups",
"logs:CreateLogGroup",
"logs:PutRetentionPolicy"
"logs:PutRetentionPolicy",
"logs:TagLogGroup"
]
},
"read": {
"permissions": [
"logs:DescribeLogGroups"
"logs:DescribeLogGroups",
"logs:ListTagsLogGroup"
]
},
"update": {
Expand All @@ -63,7 +100,9 @@
"logs:AssociateKmsKey",
"logs:DisassociateKmsKey",
"logs:PutRetentionPolicy",
"logs:DeleteRetentionPolicy"
"logs:DeleteRetentionPolicy",
"logs:TagLogGroup",
"logs:UntagLogGroup"
]
},
"delete": {
Expand All @@ -74,7 +113,8 @@
},
"list": {
"permissions": [
"logs:DescribeLogGroups"
"logs:DescribeLogGroups",
"logs:ListTagsLogGroup"
]
}
},
Expand Down
3 changes: 3 additions & 0 deletions aws-logs-loggroup/resource-role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,10 @@ Resources:
- "logs:DeleteRetentionPolicy"
- "logs:DescribeLogGroups"
- "logs:DisassociateKmsKey"
- "logs:ListTagsLogGroup"
- "logs:PutRetentionPolicy"
- "logs:TagLogGroup"
- "logs:UntagLogGroup"
Resource: "*"
Outputs:
ExecutionRoleArn:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
package software.amazon.logs.loggroup;

import software.amazon.awssdk.services.cloudwatchlogs.model.ListTagsLogGroupResponse;
import software.amazon.awssdk.services.cloudwatchlogs.model.LogGroup;
import software.amazon.cloudformation.proxy.AmazonWebServicesClientProxy;
import software.amazon.cloudformation.proxy.Logger;
import software.amazon.cloudformation.proxy.OperationStatus;
import software.amazon.cloudformation.proxy.ProgressEvent;
import software.amazon.cloudformation.proxy.ResourceHandlerRequest;
import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsResponse;

import java.util.Map;
import java.util.stream.Collectors;

public class ListHandler extends BaseHandler<CallbackContext> {

@Override
Expand All @@ -19,9 +24,17 @@ public ProgressEvent<ResourceModel, CallbackContext> handleRequest(
final DescribeLogGroupsResponse response =
proxy.injectCredentialsAndInvokeV2(Translator.translateToListRequest(request.getNextToken()),
ClientBuilder.getClient()::describeLogGroups);

final Map<String, ListTagsLogGroupResponse> tagResponses = Translator.streamOfOrEmpty(response.logGroups())
.collect(Collectors.toMap(
LogGroup::logGroupName,
logGroup -> proxy.injectCredentialsAndInvokeV2(Translator.translateToListTagsLogGroupRequest(logGroup.logGroupName()),
ClientBuilder.getClient()::listTagsLogGroup))
);

return ProgressEvent.<ResourceModel, CallbackContext>builder()
.status(OperationStatus.SUCCESS)
.resourceModels(Translator.translateForList(response))
.resourceModels(Translator.translateForList(response, tagResponses))
.nextToken(response.nextToken())
.build();
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package software.amazon.logs.loggroup;

import software.amazon.awssdk.services.cloudwatchlogs.model.ListTagsLogGroupResponse;
import software.amazon.cloudformation.proxy.AmazonWebServicesClientProxy;
import software.amazon.cloudformation.proxy.Logger;
import software.amazon.cloudformation.proxy.ProgressEvent;
Expand All @@ -25,14 +26,17 @@ public ProgressEvent<ResourceModel, CallbackContext> handleRequest(
}

DescribeLogGroupsResponse response = null;
ListTagsLogGroupResponse tagsResponse = null;
try {
response = proxy.injectCredentialsAndInvokeV2(Translator.translateToReadRequest(model),
ClientBuilder.getClient()::describeLogGroups);
ClientBuilder.getClient()::describeLogGroups);
tagsResponse = proxy.injectCredentialsAndInvokeV2(Translator.translateToListTagsLogGroupRequest(model.getLogGroupName()),
Copy link
Contributor

@wbingli wbingli Nov 24, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To have backward compatibility, the ListTagsLogGroups call should be soft-fail if no permission for tagging. Basically catch the permission error of this API call and ignore it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure, I can add it, but I don't get why I should support here backward compatibility? Shouldn't I then do it everywhere, where I interact with tags in some way? I'm just really curious about it, hope to get some more insights 😄

Copy link

@PatMyron PatMyron Nov 24, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

particularly issues with ReadHandlers since permissions expand without customer changes. We have an internal issue to eventually find a platform-wide solution to this problem

Copy link
Contributor

@wbingli wbingli Dec 3, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@gruebel Some users may not have tagging permissions while creating the log group resource before, especially they use stack role and restrict the permissions to operate a stack .

In this way, it will break people if this change go out. It makes senses to fail if users specify tags during create or update time. However, if they creates the log group successfully before, it will be bad experience to break them. BTW, the READ will be used for GetAtt function to get the attribute data as well as for drift detection for the resource.

Copy link
Contributor Author

@gruebel gruebel Dec 3, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the insight. Makes sense with the stach role permission for stack updates. I also used it in my previous work.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What I'm still curious about is, what is actually the List handler used for. The Read one I understand and it is used also during stack resource import, but the List one?!

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This List handler is not used in CloudFormation yet, it intends for future features, so having it hard fail for permission issue is right to do.

ClientBuilder.getClient()::listTagsLogGroup);
} catch (final ResourceNotFoundException e) {
throwNotFoundException(model);
}

final ResourceModel modelFromReadResult = Translator.translateForRead(response);
final ResourceModel modelFromReadResult = Translator.translateForRead(response, tagsResponse);
if (modelFromReadResult.getLogGroupName() == null) {
throwNotFoundException(model);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,21 @@
import software.amazon.awssdk.services.cloudwatchlogs.model.DeleteRetentionPolicyRequest;
import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsRequest;
import software.amazon.awssdk.services.cloudwatchlogs.model.DescribeLogGroupsResponse;
import software.amazon.awssdk.services.cloudwatchlogs.model.ListTagsLogGroupRequest;
import software.amazon.awssdk.services.cloudwatchlogs.model.ListTagsLogGroupResponse;
import software.amazon.awssdk.services.cloudwatchlogs.model.PutRetentionPolicyRequest;
import software.amazon.awssdk.services.cloudwatchlogs.model.DisassociateKmsKeyRequest;
import software.amazon.awssdk.services.cloudwatchlogs.model.AssociateKmsKeyRequest;
import software.amazon.awssdk.services.cloudwatchlogs.model.TagLogGroupRequest;
import software.amazon.awssdk.services.cloudwatchlogs.model.UntagLogGroupRequest;

import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand Down Expand Up @@ -42,6 +49,7 @@ static CreateLogGroupRequest translateToCreateRequest(final ResourceModel model)
return CreateLogGroupRequest.builder()
.logGroupName(model.getLogGroupName())
.kmsKeyId(model.getKmsKeyId())
.tags(translateTagsToSdk(model.getTags()))
.build();
}

Expand Down Expand Up @@ -71,7 +79,27 @@ static AssociateKmsKeyRequest translateToAssociateKmsKeyRequest(final ResourceMo
.build();
}

static ResourceModel translateForRead(final DescribeLogGroupsResponse response) {
static ListTagsLogGroupRequest translateToListTagsLogGroupRequest(final String logGroupName) {
return ListTagsLogGroupRequest.builder()
.logGroupName(logGroupName)
.build();
}

static TagLogGroupRequest translateToTagLogGroupRequest(final String logGroupName, final Set<Tag> tags) {
return TagLogGroupRequest.builder()
.logGroupName(logGroupName)
.tags(translateTagsToSdk(tags))
.build();
}

static UntagLogGroupRequest translateToUntagLogGroupRequest(final String logGroupName, final List<String> tagKeys) {
return UntagLogGroupRequest.builder()
.logGroupName(logGroupName)
.tags(tagKeys)
.build();
}

static ResourceModel translateForRead(final DescribeLogGroupsResponse response, final ListTagsLogGroupResponse tagsResponse) {
final String logGroupName = streamOfOrEmpty(response.logGroups())
.map(software.amazon.awssdk.services.cloudwatchlogs.model.LogGroup::logGroupName)
.filter(Objects::nonNull)
Expand All @@ -92,26 +120,29 @@ static ResourceModel translateForRead(final DescribeLogGroupsResponse response)
.filter(Objects::nonNull)
.findAny()
.orElse(null);
final Set<Tag> tags = translateSdkToTags(tagsResponse.tags());
return ResourceModel.builder()
.arn(logGroupArn)
.logGroupName(logGroupName)
.retentionInDays(retentionInDays)
.kmsKeyId(kmsKeyId)
.tags(tags)
.build();
}

static List<ResourceModel> translateForList(final DescribeLogGroupsResponse response) {
static List<ResourceModel> translateForList(final DescribeLogGroupsResponse response, final Map<String, ListTagsLogGroupResponse> tagResponses) {
return streamOfOrEmpty(response.logGroups())
.map(logGroup -> ResourceModel.builder()
.arn(logGroup.arn())
.logGroupName(logGroup.logGroupName())
.retentionInDays(logGroup.retentionInDays())
.kmsKeyId(logGroup.kmsKeyId())
.tags(translateSdkToTags(tagResponses.get(logGroup.logGroupName()).tags()))
.build())
.collect(Collectors.toList());
}

private static <T> Stream<T> streamOfOrEmpty(final Collection<T> collection) {
static <T> Stream<T> streamOfOrEmpty(final Collection<T> collection) {
return Optional.ofNullable(collection)
.map(Collection::stream)
.orElseGet(Stream::empty);
Expand All @@ -128,4 +159,19 @@ static String buildResourceDoesNotExistErrorMessage(final String resourceIdentif
ResourceModel.TYPE_NAME,
resourceIdentifier);
}

static Map<String, String> translateTagsToSdk(final Set<Tag> tags) {
if (tags == null || tags.isEmpty()) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The tags.isEmpty() check is redundant. It should be handled by the stream() call

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will result in same error as below

return null;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of returning null can we return an empty map?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if I don't return null here some of the contract tests will fail with

{'status': 'FAILED', 'errorCode': 'GeneralServiceException', 'message': "1 validation error detected: Value '{}' at 'tags' failed to satisfy constraint: Member must have length greater than or equal to 1 (Service: CloudWatchLogs, Status Code: 400, Request ID: 710f70c1-0dad-4f89-91ad-3eadd83b6e9c, Extended Request ID: null)", 'callbackDelaySeconds': 0}

}
return tags.stream().collect(Collectors.toMap(Tag::getKey, Tag::getValue));
}

static Set<Tag> translateSdkToTags(final Map<String, String> tags) {
if (tags == null || tags.isEmpty()) {
return null;
}
return tags.entrySet().stream().map(tag -> new Tag(tag.getKey(), tag.getValue()))
.collect(Collectors.toSet());
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package software.amazon.logs.loggroup;

import com.google.common.collect.Sets;
import software.amazon.cloudformation.exceptions.CfnInternalFailureException;
import software.amazon.cloudformation.exceptions.CfnResourceConflictException;
import software.amazon.cloudformation.exceptions.CfnServiceInternalErrorException;
Expand All @@ -16,7 +17,12 @@
import software.amazon.awssdk.services.cloudwatchlogs.model.ResourceNotFoundException;
import software.amazon.awssdk.services.cloudwatchlogs.model.ServiceUnavailableException;

import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;

public class UpdateHandler extends BaseHandler<CallbackContext> {

Expand All @@ -27,11 +33,12 @@ public ProgressEvent<ResourceModel, CallbackContext> handleRequest(
final CallbackContext callbackContext,
final Logger logger) {

// Everything except RetentionPolicyInDays and KmsKeyId is createOnly
// Everything except RetentionPolicyInDays, KmsKeyId and Tags is createOnly
gruebel marked this conversation as resolved.
Show resolved Hide resolved
final ResourceModel model = request.getDesiredResourceState();
final ResourceModel previousModel = request.getPreviousResourceState();
final boolean retentionChanged = ! retentionUnchanged(previousModel, model);
final boolean kmsKeyChanged = ! kmsKeyUnchanged(previousModel, model);
final boolean tagsChanged = ! tagsUnchanged(previousModel, model);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should use getDesiredResourceTags which will combine both stack tags and resource tag. And getPreviousResourceTags for both previous stack tags and resource tags.

You will need to override the resourceDefinedTags (example) so that the getDesiredResourceTags will merge the stacks and resource defined tags together.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, makes sense and will try to add it.

if (retentionChanged && model.getRetentionInDays() == null) {
deleteRetentionPolicy(proxy, request, logger);
} else if (retentionChanged){
Expand All @@ -47,6 +54,10 @@ public ProgressEvent<ResourceModel, CallbackContext> handleRequest(
associateKmsKey(proxy, request, logger);
}

if (tagsChanged) {
updateTags(proxy, previousModel, model, logger);
}

return ProgressEvent.defaultSuccessHandler(model);
}

Expand Down Expand Up @@ -148,6 +159,39 @@ private void associateKmsKey(final AmazonWebServicesClientProxy proxy,
logger.log(kmsKeyMessage);
}

private void updateTags(final AmazonWebServicesClientProxy proxy,
final ResourceModel previousModel,
final ResourceModel model,
final Logger logger) {
final Set<Tag> previousTags = Optional.ofNullable(previousModel).map(ResourceModel::getTags).orElse(new HashSet<>());
final Set<Tag> newTags = Optional.ofNullable(model.getTags()).orElse(new HashSet<>());
final Set<Tag> tagsToRemove = Sets.difference(previousTags, newTags);
final Set<Tag> tagsToAdd = Sets.difference(newTags, previousTags);
try {
if (!tagsToRemove.isEmpty()) {
final List<String> tagKeys = tagsToRemove.stream().map(Tag::getKey).collect(Collectors.toList());
proxy.injectCredentialsAndInvokeV2(Translator.translateToUntagLogGroupRequest(model.getLogGroupName(), tagKeys),
ClientBuilder.getClient()::untagLogGroup);

final String message =
String.format("%s [%s] successfully removed tags: [%s]",
ResourceModel.TYPE_NAME, model.getLogGroupName(), tagKeys);
logger.log(message);
}
if(!tagsToAdd.isEmpty()) {
proxy.injectCredentialsAndInvokeV2(Translator.translateToTagLogGroupRequest(model.getLogGroupName(), tagsToAdd),
ClientBuilder.getClient()::tagLogGroup);

final String message =
String.format("%s [%s] successfully added tags: [%s]",
ResourceModel.TYPE_NAME, model.getLogGroupName(), tagsToAdd);
logger.log(message);
}
} catch (final ResourceNotFoundException e) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added one more exception handling, because the link provided is actually the wrong one, it is for CloudWatch not CloudWatch Logs. That's the correct one 😄
https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_TagLogGroup.html#API_TagLogGroup_Errors

throwNotFoundException(model);
}
}

private void throwNotFoundException(final ResourceModel model) {
throw new software.amazon.cloudformation.exceptions.ResourceNotFoundException(ResourceModel.TYPE_NAME,
Objects.toString(model.getPrimaryIdentifier()));
Expand All @@ -161,4 +205,11 @@ private static boolean retentionUnchanged(final ResourceModel previousModel, fin
private static boolean kmsKeyUnchanged(final ResourceModel previousModel, final ResourceModel model) {
return (previousModel != null && Objects.equals(model.getKmsKeyId(), previousModel.getKmsKeyId()));
}

private static boolean tagsUnchanged(final ResourceModel previousModel, final ResourceModel model) {
if (previousModel == null && model.getTags() == null) {
return true;
}
return (previousModel != null && Objects.equals(model.getTags(), previousModel.getTags()));
}
}
Loading