From 490b1c78b82a54516c8fe0645289685967b864cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=AD=90=E5=81=A5?= <3025104405@qq.com> Date: Sun, 4 Aug 2024 19:02:33 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=BC=93=E5=AD=98=E6=8F=92?= =?UTF-8?q?=E4=BB=B6=E5=B8=83=E9=9A=86=E8=BF=87=E6=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Former-commit-id: 18dc59682f42179bf25d67a986dc9ed32a09d349 --- go.mod | 13 +++------ go.sum | 25 ++++++----------- pkg/cachebloom/cachebloom.go | 53 ++++++++++++++++++++++++++++++++++++ pkg/limiterp/limit.go | 2 +- 4 files changed, 67 insertions(+), 26 deletions(-) create mode 100644 pkg/cachebloom/cachebloom.go diff --git a/go.mod b/go.mod index 9ff7cdc..f6a0363 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,13 @@ go 1.22.3 require ( github.com/IBM/sarama v1.43.2 + github.com/bits-and-blooms/bloom/v3 v3.7.0 github.com/bsm/redislock v0.9.4 github.com/bwmarrin/snowflake v0.3.0 github.com/casbin/casbin/v2 v2.93.0 github.com/casbin/gorm-adapter/v3 v3.25.0 github.com/dlclark/regexp2 v1.11.0 + github.com/elastic/go-elasticsearch/v8 v8.14.0 github.com/gin-contrib/cors v1.7.2 github.com/gin-gonic/gin v1.10.0 github.com/go-sql-driver/mysql v1.8.1 @@ -19,7 +21,6 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 - github.com/swaggo/swag v1.16.3 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.933 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/sms v1.0.933 go.mongodb.org/mongo-driver v1.15.0 @@ -34,6 +35,7 @@ require ( require ( filippo.io/edwards25519 v1.1.0 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/casbin/govaluate v1.1.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -43,7 +45,6 @@ require ( github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect - github.com/elastic/go-elasticsearch/v8 v8.14.0 // indirect github.com/glebarez/go-sqlite v1.22.0 // indirect github.com/glebarez/sqlite v1.11.0 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -77,6 +78,7 @@ require ( go.opentelemetry.io/otel v1.28.0 // indirect go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect + golang.org/x/mod v0.18.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gorm.io/driver/postgres v1.5.7 // indirect gorm.io/driver/sqlserver v1.5.3 // indirect @@ -88,7 +90,6 @@ require ( ) require ( - github.com/KyleBanks/depth v1.2.1 // indirect github.com/bytedance/sonic v1.11.8 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cloudwego/base64x v0.1.4 // indirect @@ -96,10 +97,6 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.4 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.21.0 // indirect - github.com/go-openapi/spec v0.21.0 // indirect - github.com/go-openapi/swag v0.23.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.22.0 // indirect @@ -108,12 +105,10 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/go.sum b/go.sum index a180c64..3151e8c 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,11 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaC github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/IBM/sarama v1.43.2 h1:HABeEqRUh32z8yzY2hGB/j8mHSzC/HA9zlEjqFNCzSw= github.com/IBM/sarama v1.43.2/go.mod h1:Kyo4WkF24Z+1nz7xeVUFWIuKVV8RS3wM8mkvPKMdXFQ= -github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= -github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bloom/v3 v3.7.0 h1:VfknkqV4xI+PsaDIsoHueyxVDZrfvMn56jeWUzvzdls= +github.com/bits-and-blooms/bloom/v3 v3.7.0/go.mod h1:VKlUSvp0lFIYqxJjzdnSsZEw4iHb1kOL2tfHTgyJBHg= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -98,14 +101,6 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= -github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= -github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= -github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -182,8 +177,6 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= @@ -204,8 +197,6 @@ github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/microsoft/go-mssqldb v1.6.0/go.mod h1:00mDtPbeQCRGC1HwOOR5K/gr30P1NcEG0vx6Kbv2aJU= @@ -273,14 +264,14 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= -github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.933 h1:U9cMP5WSO7GNZoQ0Br2MlzG31KU+WSeyCaXeUNV4Mow= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.933/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/sms v1.0.933 h1:JVkDpW7y/Rl+FYmO6kuYv9Q12KJ3AAMrG0QqSMzwIHQ= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/sms v1.0.933/go.mod h1:nJzJ+3+BDMeYd34uziUJK5KMv2QEHgYJkGLxUpVMNc0= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= +github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -299,6 +290,8 @@ go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= diff --git a/pkg/cachebloom/cachebloom.go b/pkg/cachebloom/cachebloom.go new file mode 100644 index 0000000..9aa2224 --- /dev/null +++ b/pkg/cachebloom/cachebloom.go @@ -0,0 +1,53 @@ +package cachebloom + +import ( + "context" + "errors" + "github.com/bits-and-blooms/bloom/v3" + "github.com/redis/go-redis/v9" + "time" +) + +// CacheBloom 是一个包含布隆过滤器和 Redis 客户端的结构体 +type CacheBloom struct { + bf *bloom.BloomFilter + client *redis.Client +} + +// NewCacheBloom 创建并初始化一个 CacheBloom 实例 +func NewCacheBloom(client *redis.Client, filterSize uint, falsePositiveRate float64) *CacheBloom { + return &CacheBloom{ + bf: bloom.NewWithEstimates(filterSize, falsePositiveRate), + client: client, + } +} + +// QueryData 查询数据，如果缓存和布隆过滤器都没有命中，则查询数据库 +func (cb *CacheBloom) QueryData(ctx context.Context, key string, queryDB func(string) string, ttl time.Duration) (string, error) { + // 检查布隆过滤器 + if !cb.bf.TestString(key) { + return "", nil // 数据不存在，直接返回空结果 + } + // 检查缓存 + data, err := cb.client.Get(ctx, key).Result() + if err != nil { + if errors.Is(err, redis.Nil) { + // 缓存中没有数据，查询数据库 + data = queryDB(key) + if data != "" { + if er := cb.client.Set(ctx, key, data, ttl).Err(); er != nil { + return "", er // 返回设置缓存时的错误 + } + cb.bf.AddString(key) // 更新布隆过滤器 + } else { + if er := cb.client.Set(ctx, key, "", ttl).Err(); er != nil { + return "", er // 返回设置缓存空对象时的错误 + } + cb.bf.AddString(key) // 更新布隆过滤器 + } + } else { + return "", err // 返回Redis操作时发生的其他错误 + } + } + return data, nil +} diff --git a/pkg/limiterp/limit.go b/pkg/limiterp/limit.go index a07617b..bec8727 100644 --- a/pkg/limiterp/limit.go +++ b/pkg/limiterp/limit.go @@ -31,7 +31,7 @@ func NewRedisSlidingWindowLimiter(cmd redis.Cmdable, interval time.Duration, rat } } -func (l limiter) Limit(ctx context.Context, key string) (bool, error) { +func (l *limiter) Limit(ctx context.Context, key string) (bool, error) { return l.cmd.Eval(ctx, luaScript, []string{key}, l.interval.Milliseconds(), l.rate, time.Now().UnixMilli()).Bool() }