-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Pulumi module for expiring stuff instead of deleting it.
- Loading branch information
Showing
9 changed files
with
270 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| load("@aspect_bazel_lib//lib:testing.bzl", "assert_archive_contains") | ||
| load("@rules_oci//oci:defs.bzl", "oci_image") | ||
| load("@rules_pkg//:pkg.bzl", "pkg_tar") | ||
| load("//bzl:rules.bzl", "bazel_lint") | ||
| load("//go:rules.bzl", "go_binary", "go_library") | ||
| load("//ts/pulumi/lib/oci:rules.bzl", "pulumi_image") | ||
|
|
||
| bazel_lint( | ||
| name = "bazel_lint", | ||
| srcs = ["BUILD.bazel"], | ||
| ) | ||
|
|
||
| pulumi_image( | ||
| name = "expire_on_delete_image", | ||
| src = ":oci_image", | ||
| ) | ||
|
|
||
| go_library( | ||
| name = "lambda_lib", | ||
| srcs = ["main.go"], | ||
| importpath = "github.com/zemn-me/monorepo/ts/pulumi/lib/expire_on_delete/lambda", | ||
| visibility = ["//visibility:private"], | ||
| deps = [ | ||
| "@com_github_aws_aws_lambda_go//events", | ||
| "@com_github_aws_aws_lambda_go//lambda", | ||
| "@com_github_aws_aws_sdk_go//aws", | ||
| "@com_github_aws_aws_sdk_go//aws/session", | ||
| "@com_github_aws_aws_sdk_go//service/s3", | ||
| ], | ||
| ) | ||
|
|
||
| go_binary( | ||
| name = "app", | ||
| embed = [":lambda_lib"], | ||
| visibility = ["//visibility:public"], | ||
| ) | ||
|
|
||
| # Put app go_binary into a tar layer. | ||
| pkg_tar( | ||
| name = "app_layer", | ||
| srcs = [":app"], | ||
| # If the binary depends on RUNFILES, uncomment the attribute below. | ||
| # include_runfiles = True | ||
| ) | ||
|
|
||
| # Prove that the application is at the path we expect in that tar. | ||
| assert_archive_contains( | ||
| name = "test_app_layer", | ||
| archive = "app_layer.tar", | ||
| expected = ["app"], | ||
| ) | ||
|
|
||
| oci_image( | ||
| name = "oci_image", | ||
| base = "@distroless_base", | ||
| entrypoint = ["/app"], | ||
| tars = [":app_layer"], | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| // Package main implements an AWS Lambda function that handles S3 delete marker events. | ||
| // This function removes delete markers and sets expiration tags on versioned objects. | ||
| // It is intended for use in scenarios such as managing a CDN backing store where objects | ||
| // need to be retained for a short period after delete markers are created, ensuring proper | ||
| // propagation of updates before objects are fully expired. | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "log" | ||
| "time" | ||
|
|
||
| "github.com/aws/aws-lambda-go/events" | ||
| "github.com/aws/aws-lambda-go/lambda" | ||
| "github.com/aws/aws-sdk-go/aws" | ||
| "github.com/aws/aws-sdk-go/aws/session" | ||
| "github.com/aws/aws-sdk-go/service/s3" | ||
| ) | ||
|
|
||
| const expirationDays = 7 | ||
|
|
||
| type S3EventRecord struct { | ||
| BucketName string | ||
| ObjectKey string | ||
| VersionID string | ||
| } | ||
|
|
||
| func handleRequest(ctx context.Context, event events.S3Event) (map[string]string, error) { | ||
| sess := session.Must(session.NewSession()) | ||
| s3Client := s3.New(sess) | ||
|
|
||
| // Extract bucket name, object key, and version ID from the event | ||
| record := event.Records[0] | ||
| bucketName := record.S3.Bucket.Name | ||
| objectKey := record.S3.Object.Key | ||
| versionID := record.S3.Object.VersionID | ||
|
|
||
| // Check if this event corresponds to a delete marker | ||
| headObjectInput := &s3.HeadObjectInput{ | ||
| Bucket: aws.String(bucketName), | ||
| Key: aws.String(objectKey), | ||
| VersionId: aws.String(versionID), | ||
| } | ||
|
|
||
| headObjectOutput, err := s3Client.HeadObject(headObjectInput) | ||
| if err != nil { | ||
| log.Printf("Error getting object metadata for %s: %v", objectKey, err) | ||
| return map[string]string{ | ||
| "statusCode": "500", | ||
| "body": fmt.Sprintf("Error: %v", err), | ||
| }, err | ||
| } | ||
|
|
||
| if aws.BoolValue(headObjectOutput.DeleteMarker) { | ||
| // Remove the delete marker | ||
| deleteObjectInput := &s3.DeleteObjectInput{ | ||
| Bucket: aws.String(bucketName), | ||
| Key: aws.String(objectKey), | ||
| VersionId: aws.String(versionID), | ||
| } | ||
|
|
||
| _, err := s3Client.DeleteObject(deleteObjectInput) | ||
| if err != nil { | ||
| log.Printf("Error removing delete marker for %s: %v", objectKey, err) | ||
| return map[string]string{ | ||
| "statusCode": "500", | ||
| "body": fmt.Sprintf("Error: %v", err), | ||
| }, err | ||
| } | ||
| log.Printf("Removed delete marker for %s in %s", objectKey, bucketName) | ||
|
|
||
| // Calculate the expiration date | ||
| expirationDate := time.Now().Add(expirationDays * 24 * time.Hour).Format(time.RFC3339) | ||
|
|
||
| // Add an expiration tag to the object | ||
| taggingInput := &s3.PutObjectTaggingInput{ | ||
| Bucket: aws.String(bucketName), | ||
| Key: aws.String(objectKey), | ||
| Tagging: &s3.Tagging{ | ||
| TagSet: []*s3.Tag{ | ||
| { | ||
| Key: aws.String("ExpireOn"), | ||
| Value: aws.String(expirationDate), | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| _, err = s3Client.PutObjectTagging(taggingInput) | ||
| if err != nil { | ||
| log.Printf("Error setting expiration tag for %s: %v", objectKey, err) | ||
| return map[string]string{ | ||
| "statusCode": "500", | ||
| "body": fmt.Sprintf("Error: %v", err), | ||
| }, err | ||
| } | ||
| log.Printf("Set expiration tag for %s to %s", objectKey, expirationDate) | ||
|
|
||
| return map[string]string{ | ||
| "statusCode": "200", | ||
| "body": fmt.Sprintf("Removed delete marker and set expiration for %s.", objectKey), | ||
| }, nil | ||
| } | ||
|
|
||
| return map[string]string{ | ||
| "statusCode": "200", | ||
| "body": "No action needed.", | ||
| }, nil | ||
| } | ||
|
|
||
| func main() { | ||
| lambda.Start(handleRequest) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| /** | ||
| * Pulumi component for a bucket that expires files after some time | ||
| * instead of deleting them. | ||
| */ | ||
|
|
||
| import * as fs from 'node:fs/promises'; | ||
|
|
||
| import { lambda, s3 } from "@pulumi/aws"; | ||
| import { ComponentResource, ComponentResourceOptions, interpolate, output } from "@pulumi/pulumi"; | ||
|
|
||
| import { ExpireOnDeleteImage } from "#root/ts/pulumi/lib/expire_on_delete/lambda/ExpireOnDeleteImage.js"; | ||
|
|
||
|
|
||
| export interface Args { | ||
| bucketId: string | ||
| ECRBaseURI: string | ||
| } | ||
|
|
||
| /** | ||
| * Expires files after some time instead of deleting them. | ||
| */ | ||
| export class FileExpirer extends ComponentResource { | ||
| constructor( | ||
| name: string, | ||
| args: Args, | ||
| opts?: ComponentResourceOptions | ||
| ) { | ||
| super('ts:pulumi:lib:ExpiringBucket', name, args, opts); | ||
|
|
||
| const img = new ExpireOnDeleteImage(`${name}_img`, { | ||
| repository: args.ECRBaseURI | ||
| }, { parent: this }); | ||
|
|
||
| const fn = new lambda.Function(`${name}_lambda`, { | ||
| "imageUri": interpolate`${args.ECRBaseURI}@${ | ||
| img.digest.path.then(p => output( | ||
| fs.readFile(p))) | ||
| }` | ||
| }, { parent: this }); | ||
|
|
||
|
|
||
| new s3.BucketNotification( | ||
| `${name}_bucketnotification`, | ||
| { | ||
| bucket: args.bucketId, | ||
| lambdaFunctions: [ | ||
| { | ||
| lambdaFunctionArn: fn.arn, | ||
| events: ["s3:ObjectRemoved:*"], | ||
| } | ||
| ] | ||
| }, | ||
| { parent: this} | ||
| ) | ||
| } | ||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.