From 3ff3afbc5ba2c9e11e815c21f13d156debe1bd0d Mon Sep 17 00:00:00 2001 From: Thomas Neil James Shadwell Date: Wed, 27 Nov 2024 21:50:28 -0800 Subject: [PATCH] Pulumi module for expiring stuff instead of deleting it. --- MODULE.bazel | 3 + gazelle_python.yaml | 6 +- go.mod | 6 + go.sum | 9 ++ .../lib/expire_on_delete/lambda/BUILD.bazel | 58 +++++++++ ts/pulumi/lib/expire_on_delete/lambda/main.go | 114 ++++++++++++++++++ ts/pulumi/lib/expiring_bucket.ts | 57 +++++++++ ts/pulumi/lib/oci/oci_image.tmpl.ts | 16 ++- ts/pulumi/lib/oci/rules.bzl | 14 ++- 9 files changed, 270 insertions(+), 13 deletions(-) create mode 100644 ts/pulumi/lib/expire_on_delete/lambda/BUILD.bazel create mode 100644 ts/pulumi/lib/expire_on_delete/lambda/main.go create mode 100644 ts/pulumi/lib/expiring_bucket.ts diff --git a/MODULE.bazel b/MODULE.bazel index e738054f97..6e4036aeaa 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -120,6 +120,8 @@ go_deps.from_file(go_mod = "//:go.mod") use_repo( go_deps, "co_honnef_go_tools", + "com_github_aws_aws_lambda_go", + "com_github_aws_aws_sdk_go", "com_github_bazelbuild_bazel_watcher", "com_github_bazelbuild_buildtools", "com_github_go_delve_delve", @@ -131,6 +133,7 @@ use_repo( "com_github_oapi_codegen_oapi_codegen_v2", "com_github_oapi_codegen_runtime", "com_github_sergi_go_diff", + "com_github_stretchr_testify", "com_github_tdewolff_parse_v2", "com_github_twilio_twilio_go", "org_golang_x_sync", diff --git a/gazelle_python.yaml b/gazelle_python.yaml index ce5b429866..bf5d71e70e 100644 --- a/gazelle_python.yaml +++ b/gazelle_python.yaml @@ -14,6 +14,7 @@ manifest: async_lru: async_lru attr: attrs attrs: attrs + awslambdaric: awslambdaric babel: babel bleach: bleach bs4: beautifulsoup4 @@ -95,9 +96,12 @@ manifest: rfc3986_validator: rfc3986_validator rpds: rpds_py ruff: ruff + runtime_client: awslambdaric send2trash: Send2Trash setuptools: setuptools + simplejson: simplejson six: six + snapshot_restore_py: snapshot_restore_py sniffio: sniffio soupsieve: soupsieve stack_data: stack_data @@ -120,4 +124,4 @@ manifest: zmq: pyzmq pip_repository: name: pip -integrity: 68ff42ea0ba2e790a9fd502896e6d171515924aca1a8bffcc4cbd31c71b83755 +integrity: 43b6176e90c98d9be1b57350a392d23ceb8edc6089cee2d3ea293cd27fa0a1dc diff --git a/go.mod b/go.mod index aa59f5e24a..2c6e830752 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,8 @@ go 1.22.7 toolchain go1.23.3 require ( + github.com/aws/aws-lambda-go v1.47.0 + github.com/aws/aws-sdk-go v1.55.5 github.com/bazelbuild/bazel-gazelle v0.39.1 github.com/bazelbuild/bazel-watcher v0.25.3 github.com/bazelbuild/buildtools v0.0.0-20240827154017-dd10159baa91 @@ -20,6 +22,7 @@ require ( github.com/oapi-codegen/oapi-codegen/v2 v2.4.1 github.com/oapi-codegen/runtime v1.1.1 github.com/sergi/go-diff v1.3.1 + github.com/stretchr/testify v1.9.0 github.com/tdewolff/parse/v2 v2.7.19 github.com/twilio/twilio-go v1.23.6 golang.org/x/sync v0.9.0 @@ -35,6 +38,7 @@ require ( github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/cosiner/argv v0.1.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/derekparker/trie v0.0.0-20230829180723-39f4de51ef7d // indirect github.com/docker/cli v27.1.1+incompatible // indirect github.com/docker/distribution v2.8.2+incompatible // indirect @@ -62,6 +66,7 @@ require ( github.com/invopop/yaml v0.3.1 // indirect github.com/itchyny/timefmt-go v0.1.6 // indirect github.com/jaschaephraim/lrserver v0.0.0-20171129202958-50d19f603f71 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -77,6 +82,7 @@ require ( github.com/opencontainers/image-spec v1.1.0-rc3 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect diff --git a/go.sum b/go.sum index 360208b53f..9703a1a74a 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,10 @@ github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI= +github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/bazelbuild/bazel-gazelle v0.39.1 h1:N1StaCeJidwDOTQSWzzJNNHX5bl1DwTDWT5kYX96lo0= github.com/bazelbuild/bazel-gazelle v0.39.1/go.mod h1:zvqhNATbJ+IXplnP8BnR9Eojj7BCcqD/JK+iU8x9bA0= github.com/bazelbuild/bazel-watcher v0.25.3 h1:qX33Z4DDPXpe9Ry0KGTvPkuuTekrB1b59E5fQk5BjiY= @@ -134,6 +138,10 @@ github.com/jaschaephraim/lrserver v0.0.0-20171129202958-50d19f603f71 h1:24NdJ5N6 github.com/jaschaephraim/lrserver v0.0.0-20171129202958-50d19f603f71/go.mod h1:ozZLfjiLmXytkIUh200wMeuoQJ4ww06wN+KZtFP6j3g= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -365,6 +373,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/ts/pulumi/lib/expire_on_delete/lambda/BUILD.bazel b/ts/pulumi/lib/expire_on_delete/lambda/BUILD.bazel new file mode 100644 index 0000000000..92dd3ac6be --- /dev/null +++ b/ts/pulumi/lib/expire_on_delete/lambda/BUILD.bazel @@ -0,0 +1,58 @@ +load("@aspect_bazel_lib//lib:testing.bzl", "assert_archive_contains") +load("@rules_oci//oci:defs.bzl", "oci_image") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("//bzl:rules.bzl", "bazel_lint") +load("//go:rules.bzl", "go_binary", "go_library") +load("//ts/pulumi/lib/oci:rules.bzl", "pulumi_image") + +bazel_lint( + name = "bazel_lint", + srcs = ["BUILD.bazel"], +) + +pulumi_image( + name = "expire_on_delete_image", + src = ":oci_image", +) + +go_library( + name = "lambda_lib", + srcs = ["main.go"], + importpath = "github.com/zemn-me/monorepo/ts/pulumi/lib/expire_on_delete/lambda", + visibility = ["//visibility:private"], + deps = [ + "@com_github_aws_aws_lambda_go//events", + "@com_github_aws_aws_lambda_go//lambda", + "@com_github_aws_aws_sdk_go//aws", + "@com_github_aws_aws_sdk_go//aws/session", + "@com_github_aws_aws_sdk_go//service/s3", + ], +) + +go_binary( + name = "app", + embed = [":lambda_lib"], + visibility = ["//visibility:public"], +) + +# Put app go_binary into a tar layer. +pkg_tar( + name = "app_layer", + srcs = [":app"], + # If the binary depends on RUNFILES, uncomment the attribute below. + # include_runfiles = True +) + +# Prove that the application is at the path we expect in that tar. +assert_archive_contains( + name = "test_app_layer", + archive = "app_layer.tar", + expected = ["app"], +) + +oci_image( + name = "oci_image", + base = "@distroless_base", + entrypoint = ["/app"], + tars = [":app_layer"], +) diff --git a/ts/pulumi/lib/expire_on_delete/lambda/main.go b/ts/pulumi/lib/expire_on_delete/lambda/main.go new file mode 100644 index 0000000000..9a63c7c486 --- /dev/null +++ b/ts/pulumi/lib/expire_on_delete/lambda/main.go @@ -0,0 +1,114 @@ +// Package main implements an AWS Lambda function that handles S3 delete marker events. +// This function removes delete markers and sets expiration tags on versioned objects. +// It is intended for use in scenarios such as managing a CDN backing store where objects +// need to be retained for a short period after delete markers are created, ensuring proper +// propagation of updates before objects are fully expired. +package main + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/aws/aws-lambda-go/events" + "github.com/aws/aws-lambda-go/lambda" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" +) + +const expirationDays = 7 + +type S3EventRecord struct { + BucketName string + ObjectKey string + VersionID string +} + +func handleRequest(ctx context.Context, event events.S3Event) (map[string]string, error) { + sess := session.Must(session.NewSession()) + s3Client := s3.New(sess) + + // Extract bucket name, object key, and version ID from the event + record := event.Records[0] + bucketName := record.S3.Bucket.Name + objectKey := record.S3.Object.Key + versionID := record.S3.Object.VersionID + + // Check if this event corresponds to a delete marker + headObjectInput := &s3.HeadObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionID), + } + + headObjectOutput, err := s3Client.HeadObject(headObjectInput) + if err != nil { + log.Printf("Error getting object metadata for %s: %v", objectKey, err) + return map[string]string{ + "statusCode": "500", + "body": fmt.Sprintf("Error: %v", err), + }, err + } + + if aws.BoolValue(headObjectOutput.DeleteMarker) { + // Remove the delete marker + deleteObjectInput := &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionID), + } + + _, err := s3Client.DeleteObject(deleteObjectInput) + if err != nil { + log.Printf("Error removing delete marker for %s: %v", objectKey, err) + return map[string]string{ + "statusCode": "500", + "body": fmt.Sprintf("Error: %v", err), + }, err + } + log.Printf("Removed delete marker for %s in %s", objectKey, bucketName) + + // Calculate the expiration date + expirationDate := time.Now().Add(expirationDays * 24 * time.Hour).Format(time.RFC3339) + + // Add an expiration tag to the object + taggingInput := &s3.PutObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + Tagging: &s3.Tagging{ + TagSet: []*s3.Tag{ + { + Key: aws.String("ExpireOn"), + Value: aws.String(expirationDate), + }, + }, + }, + } + + _, err = s3Client.PutObjectTagging(taggingInput) + if err != nil { + log.Printf("Error setting expiration tag for %s: %v", objectKey, err) + return map[string]string{ + "statusCode": "500", + "body": fmt.Sprintf("Error: %v", err), + }, err + } + log.Printf("Set expiration tag for %s to %s", objectKey, expirationDate) + + return map[string]string{ + "statusCode": "200", + "body": fmt.Sprintf("Removed delete marker and set expiration for %s.", objectKey), + }, nil + } + + return map[string]string{ + "statusCode": "200", + "body": "No action needed.", + }, nil +} + +func main() { + lambda.Start(handleRequest) +} diff --git a/ts/pulumi/lib/expiring_bucket.ts b/ts/pulumi/lib/expiring_bucket.ts new file mode 100644 index 0000000000..8b8fff74d3 --- /dev/null +++ b/ts/pulumi/lib/expiring_bucket.ts @@ -0,0 +1,57 @@ +/** + * Pulumi component for a bucket that expires files after some time + * instead of deleting them. + */ + +import * as fs from 'node:fs/promises'; + +import { lambda, s3 } from "@pulumi/aws"; +import { ComponentResource, ComponentResourceOptions, interpolate, output } from "@pulumi/pulumi"; + +import { ExpireOnDeleteImage } from "#root/ts/pulumi/lib/expire_on_delete/lambda/ExpireOnDeleteImage.js"; + + +export interface Args { + bucketId: string + ECRBaseURI: string +} + +/** + * Expires files after some time instead of deleting them. + */ +export class FileExpirer extends ComponentResource { + constructor( + name: string, + args: Args, + opts?: ComponentResourceOptions + ) { + super('ts:pulumi:lib:ExpiringBucket', name, args, opts); + + const img = new ExpireOnDeleteImage(`${name}_img`, { + repository: args.ECRBaseURI + }, { parent: this }); + + const fn = new lambda.Function(`${name}_lambda`, { + "imageUri": interpolate`${args.ECRBaseURI}@${ + img.digest.path.then(p => output( + fs.readFile(p))) + }` + }, { parent: this }); + + + new s3.BucketNotification( + `${name}_bucketnotification`, + { + bucket: args.bucketId, + lambdaFunctions: [ + { + lambdaFunctionArn: fn.arn, + events: ["s3:ObjectRemoved:*"], + } + ] + }, + { parent: this} + ) + } +} + diff --git a/ts/pulumi/lib/oci/oci_image.tmpl.ts b/ts/pulumi/lib/oci/oci_image.tmpl.ts index afd32a41bc..be584713f0 100644 --- a/ts/pulumi/lib/oci/oci_image.tmpl.ts +++ b/ts/pulumi/lib/oci/oci_image.tmpl.ts @@ -1,12 +1,14 @@ import { local } from '@pulumi/command'; -import { ComponentResource, ComponentResourceOptions } from "@pulumi/pulumi"; +import { ComponentResource, ComponentResourceOptions, Input, output } from "@pulumi/pulumi"; +import { FileAsset } from '@pulumi/pulumi/asset'; export interface Args { - repository: string + repository: Input } export class __ClassName extends ComponentResource { + digest: FileAsset constructor( name: string, args: Args, @@ -14,16 +16,18 @@ export class __ClassName extends ComponentResource { ) { super('__TYPE', name, args, opts); + this.digest = new FileAsset("_DIGEST_PATH"); + const upload = new local.Command(`${name}_push`, { - create: [ + create: output(args.repository).apply(repo => [ "__PUSH_BIN", "--repository", - args.repository, - ].join(" ") + repo, + ].join(" ")) }, { parent: this }) - super.registerOutputs({ upload }) + super.registerOutputs({ upload, digest: this.digest }) } diff --git a/ts/pulumi/lib/oci/rules.bzl b/ts/pulumi/lib/oci/rules.bzl index 5f39a8508f..415895221e 100644 --- a/ts/pulumi/lib/oci/rules.bzl +++ b/ts/pulumi/lib/oci/rules.bzl @@ -36,26 +36,28 @@ def pulumi_image( repository = "why is this mandatory", ) + tsprojdeps = [ + ":" + name + "_push_bin", + src + ".digest", + ] + expand_template_rule( name = name + "_tsfiles", template = "//ts/pulumi/lib/oci:oci_image.tmpl.ts", out = out, - data = [ - ":" + name + "_push_bin", - ], + data = tsprojdeps, substitutions = { "__ClassName": component_name, "__TYPE": native.package_name().replace("/", ":"), "__PUSH_BIN": "$(rootpath :" + name + "_push_bin" + ")", + "_DIGEST_PATH": "$(rootpath " + src + ".digest)", }, ) ts_project( name = name, srcs = [name + "_tsfiles"], - data = [ - ":" + name + "_push_bin", - ], + data = tsprojdeps, deps = [ "//:node_modules/@pulumi/pulumi", "//:node_modules/@pulumi/command",