From 7873a189792b6570e7ff9c80b9fdc92906a3f43e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=88=90=E9=94=B4?= Date: Sun, 19 Feb 2023 21:42:51 +0800 Subject: [PATCH 01/13] feat: use cache to solve cache penetration --- cache/afterQuery.go | 12 ++++++++++++ cache/beforeQuery.go | 4 ++++ config/config.go | 3 +++ 3 files changed, 19 insertions(+) diff --git a/cache/afterQuery.go b/cache/afterQuery.go index 6e18ffd..69c16aa 100644 --- a/cache/afterQuery.go +++ b/cache/afterQuery.go @@ -92,6 +92,18 @@ func AfterQuery(cache *Gorm2Cache) func(db *gorm.DB) { return } + if !cache.Config.DisableCachePenetrationProtect { + if errors.Is(db.Error, gorm.ErrRecordNotFound) { // 应对缓存穿透 未来可能考虑使用其他过滤器实现：如布隆过滤器 + cache.Logger.CtxInfo(ctx, "[AfterQuery] set cache: %v", "recordNotFound") + err := cache.SetSearchCache(ctx, "recordNotFound", tableName, sql, vars...) + if err != nil { + cache.Logger.CtxError(ctx, "[AfterQuery] set search cache for sql: %s error: %v", sql, err) + return + } + cache.Logger.CtxInfo(ctx, "[AfterQuery] sql %s cached", sql) + } + } + if errors.Is(db.Error, util.SearchCacheHit) { // search cache hit db.Error = nil diff --git a/cache/beforeQuery.go b/cache/beforeQuery.go index 9d4d0a0..3d2de48 100644 --- a/cache/beforeQuery.go +++ b/cache/beforeQuery.go @@ -41,6 +41,10 @@ func BeforeQuery(cache *Gorm2Cache) func(db *gorm.DB) { return } cache.Logger.CtxInfo(ctx, "[BeforeQuery] get value: %s", cacheValue) + if cacheValue == "recordNotFound" { // 应对缓存穿透 + db.Error = gorm.ErrRecordNotFound + return + } rowsAffectedPos := strings.Index(cacheValue, "|") db.RowsAffected, err = strconv.ParseInt(cacheValue[:rowsAffectedPos], 10, 64) if err != nil { diff --git a/config/config.go b/config/config.go index f5c8061..9703b84 100644 --- a/config/config.go +++ b/config/config.go @@ -25,6 +25,9 @@ type CacheConfig struct { // then we choose not to cache for this query. 0 represents caching all queries. CacheMaxItemCnt int64 + // DisableCachePenetration if true, then we will not cache nil result + DisableCachePenetrationProtect bool + // CacheSize maximal items in primary cache (only works in MEMORY storage) CacheSize int From 9b36523c13e217f4009fa7e52fd1391a5e62fdb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=88=90=E9=94=B4?= Date: Sun, 19 Feb 2023 22:08:46 +0800 Subject: [PATCH 02/13] fix: fix cache penetration protect recache bug --- cache/afterQuery.go | 4 ++++ cache/beforeQuery.go | 2 +- util/definition.go | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cache/afterQuery.go b/cache/afterQuery.go index 69c16aa..bab20ab 100644 --- a/cache/afterQuery.go +++ b/cache/afterQuery.go @@ -103,6 +103,10 @@ func AfterQuery(cache *Gorm2Cache) func(db *gorm.DB) { cache.Logger.CtxInfo(ctx, "[AfterQuery] sql %s cached", sql) } } + if errors.Is(db.Error, util.RecordNotFoundCacheHit) { + db.Error = gorm.ErrRecordNotFound + return + } if errors.Is(db.Error, util.SearchCacheHit) { // search cache hit diff --git a/cache/beforeQuery.go b/cache/beforeQuery.go index 3d2de48..b200f38 100644 --- a/cache/beforeQuery.go +++ b/cache/beforeQuery.go @@ -42,7 +42,7 @@ func BeforeQuery(cache *Gorm2Cache) func(db *gorm.DB) { } cache.Logger.CtxInfo(ctx, "[BeforeQuery] get value: %s", cacheValue) if cacheValue == "recordNotFound" { // 应对缓存穿透 - db.Error = gorm.ErrRecordNotFound + db.Error = util.RecordNotFoundCacheHit return } rowsAffectedPos := strings.Index(cacheValue, "|") diff --git a/util/definition.go b/util/definition.go index 06ad5d8..0a275a6 100644 --- a/util/definition.go +++ b/util/definition.go @@ -2,6 +2,7 @@ package util import "errors" +var RecordNotFoundCacheHit = errors.New("record not found cache hit") var PrimaryCacheHit = errors.New("primary cache hit") var SearchCacheHit = errors.New("search cache hit") From 0d6a1619810b3e17c65a778291cddde0dbf65fbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=88=90=E9=94=B4?= Date: Sun, 19 Feb 2023 22:30:47 +0800 Subject: [PATCH 03/13] update: add auto release workflow --- .github/workflows/release.yaml | 42 ++++++++++++++++++++++++++++++++++ .releaserc.js | 8 +++++++ 2 files changed, 50 insertions(+) create mode 100644 .github/workflows/release.yaml create mode 100644 .releaserc.js diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..866b2b5 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,42 @@ +name: Release +on: + push: +# pull_request: +# branches: +# - '**' + workflow_dispatch: + +jobs: +# test: +# runs-on: ubuntu-22.04 +# strategy: +# matrix: +# go: ['1.17', '1.18', '1.19'] +# name: Go ${{ matrix.go }} test +# steps: +# - uses: actions/checkout@v3 +# - name: Setup go +# uses: actions/setup-go@v3 +# with: +# go-version: ${{ matrix.go }} +# - run: go test -race -v -coverprofile=profile.cov ./pkg/... +# - uses: codecov/codecov-action@v3.1.1 +# with: +# file: ./profile.cov +# name: codecov-go + release: + runs-on: ubuntu-22.04 +# needs: [test] +# if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Source checkout + uses: actions/checkout@v3 + - name: Semantic Release + uses: cycjimmy/semantic-release-action@v3 + with: + dry_run: false + semantic_version: 18.0.1 + extra_plugins: | + @semantic-release/exec@6.0.3 \ No newline at end of file diff --git a/.releaserc.js b/.releaserc.js new file mode 100644 index 0000000..f09fe95 --- /dev/null +++ b/.releaserc.js @@ -0,0 +1,8 @@ +module.exports = { + branches: ["main", {name: 'dev', prerelease: true}], + plugins: [ + "@semantic-release/commit-analyzer", + "@semantic-release/release-notes-generator", + "@semantic-release/github" + ] +}; \ No newline at end of file From 9b60c3ab2cdd63f69ccaf10194e7a49e28fdfeb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=88=90=E9=94=B4?= Date: Mon, 20 Feb 2023 00:44:24 +0800 Subject: [PATCH 04/13] feat: new way to set storage layer --- cache/cache.go | 16 +++++----------- cache/entrance.go | 15 --------------- config/config.go | 17 +++-------------- config/redisConfig.go | 31 ------------------------------- storage/memory.go | 16 +++++++++++++++- storage/redis.go | 16 ++++++++++------ 6 files changed, 33 insertions(+), 78 deletions(-) delete mode 100644 config/redisConfig.go diff --git a/cache/cache.go b/cache/cache.go index 1e4e037..753807d 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -65,24 +65,18 @@ func (c *Gorm2Cache) Initialize(db *gorm.DB) (err error) { } func (c *Gorm2Cache) AttachToDB(db *gorm.DB) { - c.Initialize(db) + _ = c.Initialize(db) } func (c *Gorm2Cache) Init() error { - if c.Config.CacheStorage == config.CacheStorageRedis { - if c.Config.RedisConfig == nil { - panic("please init redis config!") - } - c.Config.RedisConfig.InitClient() - } c.InstanceId = util.GenInstanceId() prefix := util.GormCachePrefix + ":" + c.InstanceId - if c.Config.CacheStorage == config.CacheStorageRedis { - c.cache = &storage.RedisLayer{} - } else if c.Config.CacheStorage == config.CacheStorageMemory { - c.cache = &storage.MemoryLayer{} + if c.cache != nil { + c.cache = c.Config.CacheStorage + } else { + c.cache = storage.NewMem(storage.DefaultMemStoreConfig) } if c.Config.DebugLogger == nil { diff --git a/cache/entrance.go b/cache/entrance.go index f17ff95..bc68ded 100644 --- a/cache/entrance.go +++ b/cache/entrance.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/asjdf/gorm-cache/config" - "github.com/redis/go-redis/v9" ) func NewGorm2Cache(cacheConfig *config.CacheConfig) (*Gorm2Cache, error) { @@ -20,17 +19,3 @@ func NewGorm2Cache(cacheConfig *config.CacheConfig) (*Gorm2Cache, error) { } return cache, nil } - -func NewRedisConfigWithOptions(options *redis.Options) *config.RedisConfig { - return &config.RedisConfig{ - Mode: config.RedisConfigModeOptions, - Options: options, - } -} - -func NewRedisConfigWithClient(client *redis.Client) *config.RedisConfig { - return &config.RedisConfig{ - Mode: config.RedisConfigModeRaw, - Client: client, - } -} diff --git a/config/config.go b/config/config.go index 9703b84..b71f3a7 100644 --- a/config/config.go +++ b/config/config.go @@ -1,14 +1,13 @@ package config +import "github.com/asjdf/gorm-cache/storage" + type CacheConfig struct { // CacheLevel there are 2 types of cache and 4 kinds of cache option CacheLevel CacheLevel // CacheStorage choose proper storage medium - CacheStorage CacheStorage - - // RedisConfig if storage is redis, then this config needs to be setup - RedisConfig *RedisConfig + CacheStorage storage.DataStorage // Tables only cache data within given data tables (cache all if empty) Tables []string @@ -28,9 +27,6 @@ type CacheConfig struct { // DisableCachePenetration if true, then we will not cache nil result DisableCachePenetrationProtect bool - // CacheSize maximal items in primary cache (only works in MEMORY storage) - CacheSize int - // DebugMode indicate if we're in debug mode (will print access log) DebugMode bool @@ -46,10 +42,3 @@ const ( CacheLevelOnlySearch CacheLevel = 2 CacheLevelAll CacheLevel = 3 ) - -type CacheStorage int - -const ( - CacheStorageMemory CacheStorage = 0 - CacheStorageRedis CacheStorage = 1 -) diff --git a/config/redisConfig.go b/config/redisConfig.go deleted file mode 100644 index b56c055..0000000 --- a/config/redisConfig.go +++ /dev/null @@ -1,31 +0,0 @@ -package config - -import ( - "github.com/redis/go-redis/v9" - "sync" -) - -type RedisConfigMode int - -const ( - RedisConfigModeOptions RedisConfigMode = 0 - RedisConfigModeRaw RedisConfigMode = 1 -) - -type RedisConfig struct { - Mode RedisConfigMode - - Options *redis.Options - Client *redis.Client - - once sync.Once -} - -func (c *RedisConfig) InitClient() *redis.Client { - c.once.Do(func() { - if c.Mode == RedisConfigModeOptions { - c.Client = redis.NewClient(c.Options) - } - }) - return c.Client -} diff --git a/storage/memory.go b/storage/memory.go index f79d2b6..615e2a4 100644 --- a/storage/memory.go +++ b/storage/memory.go @@ -10,13 +10,27 @@ import ( "github.com/asjdf/gorm-cache/util" ) +type MemStoreConfig struct { + MaxSize int64 // maximal items in primary cache +} + +var DefaultMemStoreConfig = &MemStoreConfig{ + MaxSize: 1000, +} + +func NewMem(config *MemStoreConfig) *MemoryLayer { + return &MemoryLayer{config: config} +} + type MemoryLayer struct { + config *MemStoreConfig + cache *ccache.Cache[string] ttl int64 } func (m *MemoryLayer) Init(conf *config.CacheConfig, prefix string) error { - c := ccache.New(ccache.Configure[string]().MaxSize(int64(conf.CacheSize))) + c := ccache.New(ccache.Configure[string]().MaxSize(m.config.MaxSize)) m.cache = c m.ttl = conf.CacheTTL return nil diff --git a/storage/redis.go b/storage/redis.go index f69f6d9..233c183 100644 --- a/storage/redis.go +++ b/storage/redis.go @@ -9,6 +9,16 @@ import ( "github.com/redis/go-redis/v9" ) +func NewRedisWithClient(client *redis.Client) *RedisLayer { + return &RedisLayer{ + client: client, + } +} + +func NewRedisWithOptions(options *redis.Options) *RedisLayer { + return NewRedisWithClient(redis.NewClient(options)) +} + type RedisLayer struct { client *redis.Client ttl int64 @@ -20,12 +30,6 @@ type RedisLayer struct { } func (r *RedisLayer) Init(conf *config.CacheConfig, prefix string) error { - if conf.RedisConfig.Mode == config.RedisConfigModeOptions { - r.client = redis.NewClient(conf.RedisConfig.Options) - } else { - r.client = conf.RedisConfig.Client - } - r.ttl = conf.CacheTTL r.logger = conf.DebugLogger r.logger.SetIsDebug(conf.DebugMode) From d36549aac044350ceaab4dbc7e29c8e751e8ebd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=88=90=E9=94=B4?= Date: Mon, 20 Feb 2023 01:46:33 +0800 Subject: [PATCH 05/13] feat: support gcache as storager --- README.md | 21 ++++++--- go.mod | 1 + go.sum | 2 + storage/gcache.go | 105 +++++++++++++++++++++++++++++++++++++++++++ storage/interface.go | 2 +- storage/memory.go | 28 ++++++------ storage/redis.go | 32 ++++++------- 7 files changed, 153 insertions(+), 38 deletions(-) create mode 100644 storage/gcache.go diff --git a/README.md b/README.md index e38fef8..97e35ef 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,11 @@ `gorm-cache` 旨在为gorm v2用户提供一个即插即用的旁路缓存解决方案。本缓存只适用于数据库表单主键时的场景。 -本库支持使用2种 cache 存储介质： - -1. 内存 (所有数据存储在单服务器的内存中) -2. Redis (所有数据存储在redis中，如果你有多个实例使用本缓存，那么他们不共享redis存储空间) +## 特性 +- 即插即用 +- 旁路缓存 +- 穿透防护 +- 多存储介质（内存/redis） ## 使用说明 @@ -26,11 +27,10 @@ func main() { cache, _ := cache.NewGorm2Cache(&config.CacheConfig{ CacheLevel: config.CacheLevelAll, - CacheStorage: config.CacheStorageRedis, - RedisConfig: cache.NewRedisConfigWithClient(redisClient), + CacheStorage: storage.NewRedisWithClient(redisClient), InvalidateWhenUpdate: true, // when you create/update/delete objects, invalidate cache CacheTTL: 5000, // 5000 ms - CacheMaxItemCnt: 5, // if length of objects retrieved one single time + CacheMaxItemCnt: 50, // if length of objects retrieved one single time // exceeds this number, then don't cache }) // More options in `config.config.go` @@ -56,3 +56,10 @@ func main() { 5. Row (Row/Rows/Scan) 本库不支持Row操作的缓存。 + +## 存储介质细节 + +本库支持使用2种 cache 存储介质： + +1. 内存 (所有数据存储在单服务器的内存中) +2. Redis (所有数据存储在redis中，如果你有多个实例使用本缓存，那么他们不共享redis存储空间) diff --git a/go.mod b/go.mod index f1985c7..4216023 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( ) require ( + github.com/bluele/gcache v0.0.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect diff --git a/go.sum b/go.sum index 48dc67b..b96806d 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= +github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/bsm/ginkgo/v2 v2.5.0 h1:aOAnND1T40wEdAtkGSkvSICWeQ8L3UASX7YVCqQx+eQ= github.com/bsm/gomega v1.20.0 h1:JhAwLmtRzXFTx2AkALSLa8ijZafntmhSoU63Ok18Uq8= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= diff --git a/storage/gcache.go b/storage/gcache.go new file mode 100644 index 0000000..cbb4d4f --- /dev/null +++ b/storage/gcache.go @@ -0,0 +1,105 @@ +package storage + +import ( + "context" + "github.com/asjdf/gorm-cache/config" + "github.com/asjdf/gorm-cache/util" + "github.com/bluele/gcache" + "strings" + "time" +) + +var _ DataStorage = &Gcache{} + +func NewGcache(builder *gcache.CacheBuilder) *Gcache { + if builder == nil { + builder = gcache.New(1000).ARC() + } + return &Gcache{builder: builder} +} + +type Gcache struct { + builder *gcache.CacheBuilder + cache gcache.Cache +} + +func (g *Gcache) Init(config *config.CacheConfig, prefix string) error { + if config.CacheTTL != 0 { + g.builder.Expiration(time.Duration(config.CacheTTL) * time.Microsecond) + } + g.cache = g.builder.Build() + return nil +} + +func (g *Gcache) BatchKeyExist(ctx context.Context, keys []string) (bool, error) { + for _, key := range keys { + if !g.cache.Has(key) { + return false, nil + } + } + return true, nil +} + +func (g *Gcache) KeyExists(ctx context.Context, key string) (bool, error) { + return g.cache.Has(key), nil +} + +func (g *Gcache) GetValue(ctx context.Context, key string) (string, error) { + v, err := g.cache.Get(key) + if err != nil { + return "", err + } + return v.(string), nil +} + +func (g *Gcache) BatchGetValues(ctx context.Context, keys []string) ([]string, error) { + values := make([]string, 0, len(keys)) + for _, key := range keys { + v, err := g.cache.Get(key) + if err != nil { + return nil, err + } + values = append(values, v.(string)) + } + return values, nil +} + +func (g *Gcache) CleanCache(ctx context.Context) error { + g.cache.Purge() + return nil +} + +func (g *Gcache) DeleteKeysWithPrefix(ctx context.Context, keyPrefix string) error { + all := g.cache.GetALL(false) + for k, _ := range all { + if key, ok := k.(string); ok && strings.HasPrefix(key, keyPrefix) { + g.cache.Remove(key) + } + } + return nil +} + +func (g *Gcache) DeleteKey(ctx context.Context, key string) error { + g.cache.Remove(key) + return nil +} + +func (g *Gcache) BatchDeleteKeys(ctx context.Context, keys []string) error { + for _, key := range keys { + g.cache.Remove(key) + } + return nil +} + +func (g *Gcache) BatchSetKeys(ctx context.Context, kvs []util.Kv) error { + for _, kv := range kvs { + if err := g.SetKey(ctx, kv); err != nil { + return err + } + } + return nil +} + +func (g *Gcache) SetKey(ctx context.Context, kv util.Kv) error { + return g.cache.Set(kv.Key, kv.Value) +} diff --git a/storage/interface.go b/storage/interface.go index e19f07a..4082857 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -13,6 +13,7 @@ var ( type DataStorage interface { Init(config *config.CacheConfig, prefix string) error + CleanCache(ctx context.Context) error // read BatchKeyExist(ctx context.Context, keys []string) (bool, error) @@ -21,7 +22,6 @@ type DataStorage interface { BatchGetValues(ctx context.Context, keys []string) ([]string, error) // write - CleanCache(ctx context.Context) error DeleteKeysWithPrefix(ctx context.Context, keyPrefix string) error DeleteKey(ctx context.Context, key string) error BatchDeleteKeys(ctx context.Context, keys []string) error diff --git a/storage/memory.go b/storage/memory.go index 615e2a4..549f3dc 100644 --- a/storage/memory.go +++ b/storage/memory.go @@ -18,30 +18,30 @@ var DefaultMemStoreConfig = &MemStoreConfig{ MaxSize: 1000, } -func NewMem(config *MemStoreConfig) *MemoryLayer { - return &MemoryLayer{config: config} +func NewMem(config *MemStoreConfig) *Memory { + return &Memory{config: config} } -type MemoryLayer struct { +type Memory struct { config *MemStoreConfig cache *ccache.Cache[string] ttl int64 } -func (m *MemoryLayer) Init(conf *config.CacheConfig, prefix string) error { +func (m *Memory) Init(conf *config.CacheConfig, prefix string) error { c := ccache.New(ccache.Configure[string]().MaxSize(m.config.MaxSize)) m.cache = c m.ttl = conf.CacheTTL return nil } -func (m *MemoryLayer) CleanCache(ctx context.Context) error { +func (m *Memory) CleanCache(ctx context.Context) error { m.cache.Clear() return nil } -func (m *MemoryLayer) BatchKeyExist(ctx context.Context, keys []string) (bool, error) { +func (m *Memory) BatchKeyExist(ctx context.Context, keys []string) (bool, error) { for _, key := range keys { item := m.cache.Get(key) if item == nil || item.Expired() { @@ -51,12 +51,12 @@ func (m *MemoryLayer) BatchKeyExist(ctx context.Context, keys []string) (bool, e return true, nil } -func (m *MemoryLayer) KeyExists(ctx context.Context, key string) (bool, error) { +func (m *Memory) KeyExists(ctx context.Context, key string) (bool, error) { item := m.cache.Get(key) return item != nil && !item.Expired(), nil } -func (m *MemoryLayer) GetValue(ctx context.Context, key string) (string, error) { +func (m *Memory) GetValue(ctx context.Context, key string) (string, error) { item := m.cache.Get(key) if item == nil || item.Expired() { return "", ErrCacheNotFound @@ -64,7 +64,7 @@ func (m *MemoryLayer) GetValue(ctx context.Context, key string) (string, error) return item.Value(), nil } -func (m *MemoryLayer) BatchGetValues(ctx context.Context, keys []string) ([]string, error) { +func (m *Memory) BatchGetValues(ctx context.Context, keys []string) ([]string, error) { values := make([]string, 0, len(keys)) for _, key := range keys { item := m.cache.Get(key) @@ -78,24 +78,24 @@ func (m *MemoryLayer) BatchGetValues(ctx context.Context, keys []string) ([]stri return values, nil } -func (m *MemoryLayer) DeleteKeysWithPrefix(ctx context.Context, keyPrefix string) error { +func (m *Memory) DeleteKeysWithPrefix(ctx context.Context, keyPrefix string) error { m.cache.DeletePrefix(keyPrefix) return nil } -func (m *MemoryLayer) DeleteKey(ctx context.Context, key string) error { +func (m *Memory) DeleteKey(ctx context.Context, key string) error { m.cache.Delete(key) return nil } -func (m *MemoryLayer) BatchDeleteKeys(ctx context.Context, keys []string) error { +func (m *Memory) BatchDeleteKeys(ctx context.Context, keys []string) error { for _, key := range keys { m.cache.Delete(key) } return nil } -func (m *MemoryLayer) BatchSetKeys(ctx context.Context, kvs []util.Kv) error { +func (m *Memory) BatchSetKeys(ctx context.Context, kvs []util.Kv) error { for _, kv := range kvs { if m.ttl > 0 { m.cache.Set(kv.Key, kv.Value, time.Duration(util.RandFloatingInt64(m.ttl))*time.Millisecond) @@ -106,7 +106,7 @@ func (m *MemoryLayer) BatchSetKeys(ctx context.Context, kvs []util.Kv) error { return nil } -func (m *MemoryLayer) SetKey(ctx context.Context, kv util.Kv) error { +func (m *Memory) SetKey(ctx context.Context, kv util.Kv) error { if m.ttl > 0 { m.cache.Set(kv.Key, kv.Value, time.Duration(util.RandFloatingInt64(m.ttl))*time.Millisecond) } else { diff --git a/storage/redis.go b/storage/redis.go index 233c183..cb4b231 100644 --- a/storage/redis.go +++ b/storage/redis.go @@ -9,17 +9,17 @@ import ( "github.com/redis/go-redis/v9" ) -func NewRedisWithClient(client *redis.Client) *RedisLayer { - return &RedisLayer{ +func NewRedisWithClient(client *redis.Client) *Redis { + return &Redis{ client: client, } } -func NewRedisWithOptions(options *redis.Options) *RedisLayer { +func NewRedisWithOptions(options *redis.Options) *Redis { return NewRedisWithClient(redis.NewClient(options)) } -type RedisLayer struct { +type Redis struct { client *redis.Client ttl int64 logger config.LoggerInterface @@ -29,7 +29,7 @@ type RedisLayer struct { cleanCacheSha string } -func (r *RedisLayer) Init(conf *config.CacheConfig, prefix string) error { +func (r *Redis) Init(conf *config.CacheConfig, prefix string) error { r.ttl = conf.CacheTTL r.logger = conf.DebugLogger r.logger.SetIsDebug(conf.DebugMode) @@ -37,7 +37,7 @@ func (r *RedisLayer) Init(conf *config.CacheConfig, prefix string) error { return r.initScripts() } -func (r *RedisLayer) initScripts() error { +func (r *Redis) initScripts() error { batchKeyExistScript := ` for idx, val in pairs(KEYS) do local exists = redis.call('EXISTS', val) @@ -72,7 +72,7 @@ func (r *RedisLayer) initScripts() error { return nil } -func (r *RedisLayer) CleanCache(ctx context.Context) error { +func (r *Redis) CleanCache(ctx context.Context) error { result := r.client.EvalSha(ctx, r.cleanCacheSha, []string{"0"}, r.keyPrefix+":*") if result.Err() != nil { r.logger.CtxError(ctx, "[CleanCache] clean cache error: %v", result.Err()) @@ -81,7 +81,7 @@ func (r *RedisLayer) CleanCache(ctx context.Context) error { return nil } -func (r *RedisLayer) BatchKeyExist(ctx context.Context, keys []string) (bool, error) { +func (r *Redis) BatchKeyExist(ctx context.Context, keys []string) (bool, error) { result := r.client.EvalSha(ctx, r.batchExistSha, keys) if result.Err() != nil { r.logger.CtxError(ctx, "[BatchKeyExist] eval script error: %v", result.Err()) @@ -90,7 +90,7 @@ func (r *RedisLayer) BatchKeyExist(ctx context.Context, keys []string) (bool, er return result.Bool() } -func (r *RedisLayer) KeyExists(ctx context.Context, key string) (bool, error) { +func (r *Redis) KeyExists(ctx context.Context, key string) (bool, error) { result := r.client.Exists(ctx, key) if result.Err() != nil { r.logger.CtxError(ctx, "[KeyExists] exists error: %v", result.Err()) @@ -102,7 +102,7 @@ func (r *RedisLayer) KeyExists(ctx context.Context, key string) (bool, error) { return false, nil } -func (r *RedisLayer) GetValue(ctx context.Context, key string) (data string, err error) { +func (r *Redis) GetValue(ctx context.Context, key string) (data string, err error) { data, err = r.client.Get(ctx, key).Result() if err == redis.Nil { err = ErrCacheNotFound @@ -110,7 +110,7 @@ func (r *RedisLayer) GetValue(ctx context.Context, key string) (data string, err return } -func (r *RedisLayer) BatchGetValues(ctx context.Context, keys []string) ([]string, error) { +func (r *Redis) BatchGetValues(ctx context.Context, keys []string) ([]string, error) { result := r.client.MGet(ctx, keys...) if result.Err() != nil { r.logger.CtxError(ctx, "[BatchGetValues] mget error: %v", result.Err()) @@ -126,20 +126,20 @@ func (r *RedisLayer) BatchGetValues(ctx context.Context, keys []string) ([]strin return strs, nil } -func (r *RedisLayer) DeleteKeysWithPrefix(ctx context.Context, keyPrefix string) error { +func (r *Redis) DeleteKeysWithPrefix(ctx context.Context, keyPrefix string) error { result := r.client.EvalSha(ctx, r.cleanCacheSha, []string{"0"}, keyPrefix+":*") return result.Err() } -func (r *RedisLayer) DeleteKey(ctx context.Context, key string) error { +func (r *Redis) DeleteKey(ctx context.Context, key string) error { return r.client.Del(ctx, key).Err() } -func (r *RedisLayer) BatchDeleteKeys(ctx context.Context, keys []string) error { +func (r *Redis) BatchDeleteKeys(ctx context.Context, keys []string) error { return r.client.Del(ctx, keys...).Err() } -func (r *RedisLayer) BatchSetKeys(ctx context.Context, kvs []util.Kv) error { +func (r *Redis) BatchSetKeys(ctx context.Context, kvs []util.Kv) error { if r.ttl == 0 { spreads := make([]interface{}, 0, len(kvs)) for _, kv := range kvs { @@ -161,6 +161,6 @@ func (r *RedisLayer) BatchSetKeys(ctx context.Context, kvs []util.Kv) error { return err } -func (r *RedisLayer) SetKey(ctx context.Context, kv util.Kv) error { +func (r *Redis) SetKey(ctx context.Context, kv util.Kv) error { return r.client.Set(ctx, kv.Key, kv.Value, time.Duration(util.RandFloatingInt64(r.ttl))*time.Millisecond).Err() } From 865f6f7f86a33909fb88a012358ef204a892db97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=88=90=E9=94=B4?= Date: Mon, 20 Feb 2023 20:48:27 +0800 Subject: [PATCH 06/13] update: check rows affected before invalidate cache --- cache/afterCreate.go | 4 ++++ cache/afterDelete.go | 4 ++++ cache/afterUpdate.go | 4 ++++ cache/cache.go | 8 ++++---- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/cache/afterCreate.go b/cache/afterCreate.go index d4e7c10..97906bb 100644 --- a/cache/afterCreate.go +++ b/cache/afterCreate.go @@ -8,6 +8,10 @@ import ( func AfterCreate(cache *Gorm2Cache) func(db *gorm.DB) { return func(db *gorm.DB) { + if db.RowsAffected == 0 { + return // no rows affected, no need to invalidate cache + } + tableName := "" if db.Statement.Schema != nil { tableName = db.Statement.Schema.Table diff --git a/cache/afterDelete.go b/cache/afterDelete.go index 7e5348c..ddbf024 100644 --- a/cache/afterDelete.go +++ b/cache/afterDelete.go @@ -10,6 +10,10 @@ import ( func AfterDelete(cache *Gorm2Cache) func(db *gorm.DB) { return func(db *gorm.DB) { + if db.RowsAffected == 0 { + return // no rows affected, no need to invalidate cache + } + tableName := "" if db.Statement.Schema != nil { tableName = db.Statement.Schema.Table diff --git a/cache/afterUpdate.go b/cache/afterUpdate.go index e17aed1..fa89bcc 100644 --- a/cache/afterUpdate.go +++ b/cache/afterUpdate.go @@ -10,6 +10,10 @@ import ( func AfterUpdate(cache *Gorm2Cache) func(db *gorm.DB) { return func(db *gorm.DB) { + if db.RowsAffected == 0 { + return // no rows affected, no need to invalidate cache + } + tableName := "" if db.Statement.Schema != nil { tableName = db.Statement.Schema.Table diff --git a/cache/cache.go b/cache/cache.go index 753807d..99299cd 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -36,17 +36,17 @@ func (c *Gorm2Cache) Name() string { } func (c *Gorm2Cache) Initialize(db *gorm.DB) (err error) { - err = db.Callback().Create().After("*").Register("gorm:cache:after_create", AfterCreate(c)) + err = db.Callback().Create().After("gorm:create").Register("gorm:cache:after_create", AfterCreate(c)) if err != nil { return err } - err = db.Callback().Delete().After("*").Register("gorm:cache:after_delete", AfterDelete(c)) + err = db.Callback().Delete().After("gorm:delete").Register("gorm:cache:after_delete", AfterDelete(c)) if err != nil { return err } - err = db.Callback().Update().After("*").Register("gorm:cache:after_update", AfterUpdate(c)) + err = db.Callback().Update().After("gorm:update").Register("gorm:cache:after_update", AfterUpdate(c)) if err != nil { return err } @@ -56,7 +56,7 @@ func (c *Gorm2Cache) Initialize(db *gorm.DB) (err error) { return err } - err = db.Callback().Query().After("*").Register("gorm:cache:after_query", AfterQuery(c)) + err = db.Callback().Query().After("gorm:query").Register("gorm:cache:after_query", AfterQuery(c)) if err != nil { return err } From e9b2c0ff26366c90a7dba380421b419ff0859bb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=88=90=E9=94=B4?= Date: Tue, 21 Feb 2023 01:31:26 +0800 Subject: [PATCH 07/13] feat: reach stats support and support reuse storage solve #5 and #4 --- cache/beforeQuery.go | 2 +- cache/cache.go | 28 ++++++++++------------ cache/entrance.go | 3 ++- cache/stats.go | 56 ++++++++++++++++++++++++++++++++++++++++++++ go.mod | 15 ++++++++++-- go.sum | 27 +++++++++++++++++++-- storage/redis.go | 17 ++++++++++---- 7 files changed, 122 insertions(+), 26 deletions(-) create mode 100644 cache/stats.go diff --git a/cache/beforeQuery.go b/cache/beforeQuery.go index b200f38..dbc3a19 100644 --- a/cache/beforeQuery.go +++ b/cache/beforeQuery.go @@ -31,12 +31,12 @@ func BeforeQuery(cache *Gorm2Cache) func(db *gorm.DB) { if util.ShouldCache(tableName, cache.Config.Tables) { if cache.Config.CacheLevel == config.CacheLevelAll || cache.Config.CacheLevel == config.CacheLevelOnlySearch { // search cache hit - cacheValue, err := cache.GetSearchCache(ctx, tableName, sql, db.Statement.Vars...) if err != nil { if !errors.Is(err, storage.ErrCacheNotFound) { cache.Logger.CtxError(ctx, "[BeforeQuery] get cache value for sql %s error: %v", sql, err) } + cache.stats.IncrMissCount() db.Error = nil return } diff --git a/cache/cache.go b/cache/cache.go index 99299cd..d20a212 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -2,8 +2,6 @@ package cache import ( "context" - "sync/atomic" - "github.com/asjdf/gorm-cache/config" "github.com/asjdf/gorm-cache/storage" "github.com/asjdf/gorm-cache/util" @@ -13,6 +11,7 @@ import ( var ( _ gorm.Plugin = &Gorm2Cache{} + _ Cache = &Gorm2Cache{} json = jsoniter.Config{ EscapeHTML: true, @@ -21,6 +20,15 @@ var ( }.Froze() ) +type Cache interface { + Name() string + Initialize(db *gorm.DB) error + AttachToDB(db *gorm.DB) + + ResetCache() error + StatsAccessor +} + type Gorm2Cache struct { Config *config.CacheConfig Logger config.LoggerInterface @@ -29,6 +37,8 @@ type Gorm2Cache struct { db *gorm.DB cache storage.DataStorage hitCount int64 + + *stats } func (c *Gorm2Cache) Name() string { @@ -93,20 +103,8 @@ func (c *Gorm2Cache) Init() error { return nil } -func (c *Gorm2Cache) GetHitCount() int64 { - return atomic.LoadInt64(&c.hitCount) -} - -func (c *Gorm2Cache) ResetHitCount() { - atomic.StoreInt64(&c.hitCount, 0) -} - -func (c *Gorm2Cache) IncrHitCount() { - atomic.AddInt64(&c.hitCount, 1) -} - func (c *Gorm2Cache) ResetCache() error { - c.ResetHitCount() + c.stats.ResetHitCount() ctx := context.Background() err := c.cache.CleanCache(ctx) if err != nil { diff --git a/cache/entrance.go b/cache/entrance.go index bc68ded..640cae1 100644 --- a/cache/entrance.go +++ b/cache/entrance.go @@ -6,12 +6,13 @@ import ( "github.com/asjdf/gorm-cache/config" ) -func NewGorm2Cache(cacheConfig *config.CacheConfig) (*Gorm2Cache, error) { +func NewGorm2Cache(cacheConfig *config.CacheConfig) (Cache, error) { if cacheConfig == nil { return nil, fmt.Errorf("you pass a nil config") } cache := &Gorm2Cache{ Config: cacheConfig, + stats: &stats{}, } err := cache.Init() if err != nil { diff --git a/cache/stats.go b/cache/stats.go new file mode 100644 index 0000000..370c838 --- /dev/null +++ b/cache/stats.go @@ -0,0 +1,56 @@ +package cache + +import "sync/atomic" + +type StatsAccessor interface { + HitCount() uint64 + MissCount() uint64 + LookupCount() uint64 + HitRate() float64 +} + +// statistics +type stats struct { + hitCount uint64 + missCount uint64 +} + +func (st *stats) ResetHitCount() { + atomic.StoreUint64(&st.hitCount, 0) + atomic.StoreUint64(&st.missCount, 0) +} + +// IncrHitCount increase hit count +func (st *stats) IncrHitCount() uint64 { + return atomic.AddUint64(&st.hitCount, 1) +} + +// IncrMissCount increase miss count +func (st *stats) IncrMissCount() uint64 { + return atomic.AddUint64(&st.missCount, 1) +} + +// HitCount returns hit count +func (st *stats) HitCount() uint64 { + return atomic.LoadUint64(&st.hitCount) +} + +// MissCount returns miss count +func (st *stats) MissCount() uint64 { + return atomic.LoadUint64(&st.missCount) +} + +// LookupCount returns lookup count +func (st *stats) LookupCount() uint64 { + return st.HitCount() + st.MissCount() +} + +// HitRate returns rate for cache hitting +func (st *stats) HitRate() float64 { + hc, mc := st.HitCount(), st.MissCount() + total := hc + mc + if total == 0 { + return 0.0 + } + return float64(hc) / float64(total) +} diff --git a/go.mod b/go.mod index 4216023..1e8d73d 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,9 @@ module github.com/asjdf/gorm-cache go 1.18 require ( + github.com/bluele/gcache v0.0.2 + github.com/glebarez/sqlite v1.7.0 + github.com/json-iterator/go v1.1.12 github.com/karlseguin/ccache/v3 v3.0.3 github.com/redis/go-redis/v9 v9.0.2 github.com/smartystreets/goconvey v1.7.2 @@ -11,16 +14,24 @@ require ( ) require ( - github.com/bluele/gcache v0.0.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/glebarez/go-sqlite v1.20.3 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 // indirect github.com/smartystreets/assertions v1.13.0 // indirect + golang.org/x/sys v0.4.0 // indirect + modernc.org/libc v1.22.2 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.5.0 // indirect + modernc.org/sqlite v1.20.3 // indirect ) diff --git a/go.sum b/go.sum index b96806d..c61eb8b 100644 --- a/go.sum +++ b/go.sum @@ -9,10 +9,18 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/glebarez/go-sqlite v1.20.3 h1:89BkqGOXR9oRmG58ZrzgoY/Fhy5x0M+/WV48U5zVrZ4= +github.com/glebarez/go-sqlite v1.20.3/go.mod h1:u3N6D/wftiAzIOJtZl6BmedqxmmkDfH3q+ihjqxC9u0= +github.com/glebarez/sqlite v1.7.0 h1:A7Xj/KN2Lvie4Z4rrgQHY8MsbebX3NyWsL3n2i82MVI= +github.com/glebarez/sqlite v1.7.0/go.mod h1:PkeevrRlF/1BhQBCnzcMWzgrIk7IOop+qS2jUYLfHhk= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= @@ -27,6 +35,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/karlseguin/ccache/v3 v3.0.3 h1:cz+3tSdTrovp00xHPP3Y6ca/YuSl5kchhYG83wUPYN0= github.com/karlseguin/ccache/v3 v3.0.3/go.mod h1:qxC372+Qn+IBj8Pe3KvGjHPj0sWwEF7AeZVhsNPZ6uY= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= @@ -35,7 +45,9 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE= github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps= -github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 h1:VstopitMQi3hZP0fzvnsLmzXZdQGc4bEcgu24cp+d4M= +github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/assertions v1.13.0 h1:Dx1kYM01xsSqKPno3aqLnrwac2LetPvN23diwyr69Qs= github.com/smartystreets/assertions v1.13.0/go.mod h1:wDmR7qL282YbGsPy6H/yAsesrxfxaaSlJazyFLYVFx8= @@ -47,6 +59,9 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= @@ -55,3 +70,11 @@ gorm.io/driver/mysql v1.4.7/go.mod h1:SxzItlnT1cb6e1e4ZRpgJN2VYtcqJgqnHxWr4wsP8o gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.24.5 h1:g6OPREKqqlWq4kh/3MCQbZKImeB9e6Xgc4zD+JgNZGE= gorm.io/gorm v1.24.5/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0= +modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/sqlite v1.20.3 h1:SqGJMMxjj1PHusLxdYxeQSodg7Jxn9WWkaAQjKrntZs= +modernc.org/sqlite v1.20.3/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A= diff --git a/storage/redis.go b/storage/redis.go index cb4b231..3e84eca 100644 --- a/storage/redis.go +++ b/storage/redis.go @@ -2,6 +2,7 @@ package storage import ( "context" + "sync" "time" "github.com/asjdf/gorm-cache/config" @@ -27,14 +28,20 @@ type Redis struct { batchExistSha string cleanCacheSha string + + once sync.Once } func (r *Redis) Init(conf *config.CacheConfig, prefix string) error { - r.ttl = conf.CacheTTL - r.logger = conf.DebugLogger - r.logger.SetIsDebug(conf.DebugMode) - r.keyPrefix = prefix - return r.initScripts() + var err error + r.once.Do(func() { + r.ttl = conf.CacheTTL + r.logger = conf.DebugLogger + r.logger.SetIsDebug(conf.DebugMode) + r.keyPrefix = prefix + err = r.initScripts() + }) + return err } func (r *Redis) initScripts() error { From 4d082c2dce4e7a1cb6e8a24a28e8c24a7e29871f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=88=90=E9=94=B4?= Date: Tue, 21 Feb 2023 14:27:47 +0800 Subject: [PATCH 08/13] feat: support async update cache --- README.md | 2 +- cache/afterCreate.go | 25 ++++++++++++++++--------- cache/afterDelete.go | 4 +++- cache/afterQuery.go | 4 +++- cache/afterUpdate.go | 4 +++- config/config.go | 3 +++ 6 files changed, 29 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 97e35ef..8ccc180 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ func main() { 4. Update (Update/Updates/UpdateColumn/UpdateColumns/Save) 5. Row (Row/Rows/Scan) -本库不支持Row操作的缓存。 +本库不支持Row操作的缓存。（WIP） ## 存储介质细节 diff --git a/cache/afterCreate.go b/cache/afterCreate.go index 97906bb..831cc38 100644 --- a/cache/afterCreate.go +++ b/cache/afterCreate.go @@ -22,16 +22,23 @@ func AfterCreate(cache *Gorm2Cache) func(db *gorm.DB) { if db.Error == nil && cache.Config.InvalidateWhenUpdate && util.ShouldCache(tableName, cache.Config.Tables) { if cache.Config.CacheLevel == config.CacheLevelAll || cache.Config.CacheLevel == config.CacheLevelOnlySearch { - // We invalidate search cache here, - // because any newly created objects may cause search cache results to be outdated and invalid. - cache.Logger.CtxInfo(ctx, "[AfterCreate] now start to invalidate search cache for table: %s", tableName) - err := cache.InvalidateSearchCache(ctx, tableName) - if err != nil { - cache.Logger.CtxError(ctx, "[AfterCreate] invalidating search cache for table %s error: %v", - tableName, err) - return + invalidSearchCache := func() { + // We invalidate search cache here, + // because any newly created objects may cause search cache results to be outdated and invalid. + cache.Logger.CtxInfo(ctx, "[AfterCreate] now start to invalidate search cache for table: %s", tableName) + err := cache.InvalidateSearchCache(ctx, tableName) + if err != nil { + cache.Logger.CtxError(ctx, "[AfterCreate] invalidating search cache for table %s error: %v", + tableName, err) + return + } + cache.Logger.CtxInfo(ctx, "[AfterCreate] invalidating search cache for table: %s finished.", tableName) + } + if cache.Config.AsyncWrite { + go invalidSearchCache() + } else { + invalidSearchCache() } - cache.Logger.CtxInfo(ctx, "[AfterCreate] invalidating search cache for table: %s finished.", tableName) } } } diff --git a/cache/afterDelete.go b/cache/afterDelete.go index ddbf024..cd3f3f7 100644 --- a/cache/afterDelete.go +++ b/cache/afterDelete.go @@ -70,7 +70,9 @@ func AfterDelete(cache *Gorm2Cache) func(db *gorm.DB) { } }() - wg.Wait() + if !cache.Config.AsyncWrite { + wg.Wait() + } } } } diff --git a/cache/afterQuery.go b/cache/afterQuery.go index bab20ab..3c27269 100644 --- a/cache/afterQuery.go +++ b/cache/afterQuery.go @@ -88,7 +88,9 @@ func AfterQuery(cache *Gorm2Cache) func(db *gorm.DB) { } } }() - wg.Wait() + if !cache.Config.AsyncWrite { + wg.Wait() + } return } diff --git a/cache/afterUpdate.go b/cache/afterUpdate.go index fa89bcc..0deb1c2 100644 --- a/cache/afterUpdate.go +++ b/cache/afterUpdate.go @@ -71,7 +71,9 @@ func AfterUpdate(cache *Gorm2Cache) func(db *gorm.DB) { } }() - wg.Wait() + if !cache.Config.AsyncWrite { + wg.Wait() + } } } } diff --git a/config/config.go b/config/config.go index b71f3a7..250a4c3 100644 --- a/config/config.go +++ b/config/config.go @@ -17,6 +17,9 @@ type CacheConfig struct { // else we do nothing to outdated cache. InvalidateWhenUpdate bool + // AsyncWrite if true, then we will write cache in async mode + AsyncWrite bool + // CacheTTL cache ttl in ms, where 0 represents forever CacheTTL int64 From c8a6d2e693d2a95231f7d15fead114e29d57966d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=88=90=E9=94=B4?= Date: Tue, 21 Feb 2023 14:31:50 +0800 Subject: [PATCH 09/13] feat: support reuse storage --- storage/gcache.go | 13 +++++++++---- storage/memory.go | 11 ++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/storage/gcache.go b/storage/gcache.go index cbb4d4f..abd9f24 100644 --- a/storage/gcache.go +++ b/storage/gcache.go @@ -6,6 +6,7 @@ import ( "github.com/asjdf/gorm-cache/util" "github.com/bluele/gcache" "strings" + "sync" "time" ) @@ -21,13 +22,17 @@ func NewGcache(builder *gcache.CacheBuilder) *Gcache { type Gcache struct { builder *gcache.CacheBuilder cache gcache.Cache + + once sync.Once } func (g *Gcache) Init(config *config.CacheConfig, prefix string) error { - if config.CacheTTL != 0 { - g.builder.Expiration(time.Duration(config.CacheTTL) * time.Microsecond) - } - g.cache = g.builder.Build() + g.once.Do(func() { + if config.CacheTTL != 0 { + g.builder.Expiration(time.Duration(config.CacheTTL) * time.Microsecond) + } + g.cache = g.builder.Build() + }) return nil } diff --git a/storage/memory.go b/storage/memory.go index 549f3dc..c6b648a 100644 --- a/storage/memory.go +++ b/storage/memory.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/karlseguin/ccache/v3" + "sync" "time" "github.com/asjdf/gorm-cache/config" @@ -27,12 +28,16 @@ type Memory struct { cache *ccache.Cache[string] ttl int64 + + once sync.Once } func (m *Memory) Init(conf *config.CacheConfig, prefix string) error { - c := ccache.New(ccache.Configure[string]().MaxSize(m.config.MaxSize)) - m.cache = c - m.ttl = conf.CacheTTL + m.once.Do(func() { + c := ccache.New(ccache.Configure[string]().MaxSize(m.config.MaxSize)) + m.cache = c + m.ttl = conf.CacheTTL + }) return nil } From 659646158c6d86dac2abb2dd53e875957589f8ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=88=90=E9=94=B4?= Date: Tue, 21 Feb 2023 14:34:49 +0800 Subject: [PATCH 10/13] update: update readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8ccc180..af56197 100644 --- a/README.md +++ b/README.md @@ -61,5 +61,7 @@ func main() { 本库支持使用2种 cache 存储介质： -1. 内存 (所有数据存储在单服务器的内存中) +1. 内存 (ccache/gcache) 2. Redis (所有数据存储在redis中，如果你有多个实例使用本缓存，那么他们不共享redis存储空间) + +并且允许多个gorm-cache公用一个存储池，以确保同一数据库的多个gorm实例共享缓存。 From 77944ab07bad4b34a384bb7a87b1581b78180418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=88=90=E9=94=B4?= Date: Tue, 21 Feb 2023 22:26:34 +0800 Subject: [PATCH 11/13] fix: fix import circle --- cache/cache.go | 10 +++++++--- config/config.go | 7 +++++-- go.mod | 2 -- go.sum | 5 ----- storage/gcache.go | 7 +++---- storage/interface.go | 9 +++++++-- storage/memory.go | 5 ++--- storage/redis.go | 13 +++++++------ {config => util}/logger.go | 10 +++++----- 9 files changed, 36 insertions(+), 32 deletions(-) rename {config => util}/logger.go (65%) diff --git a/cache/cache.go b/cache/cache.go index d20a212..9fb530c 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -31,7 +31,7 @@ type Cache interface { type Gorm2Cache struct { Config *config.CacheConfig - Logger config.LoggerInterface + Logger util.LoggerInterface InstanceId string db *gorm.DB @@ -90,12 +90,16 @@ func (c *Gorm2Cache) Init() error { } if c.Config.DebugLogger == nil { - c.Config.DebugLogger = &config.DefaultLoggerImpl{} + c.Config.DebugLogger = &util.DefaultLogger{} } c.Logger = c.Config.DebugLogger c.Logger.SetIsDebug(c.Config.DebugMode) - err := c.cache.Init(c.Config, prefix) + err := c.cache.Init(&storage.Config{ + TTL: c.Config.CacheTTL, + Debug: c.Config.DebugMode, + Logger: c.Logger, + }, prefix) if err != nil { c.Logger.CtxError(context.Background(), "[Init] cache init error: %v", err) return err diff --git a/config/config.go b/config/config.go index 250a4c3..399d208 100644 --- a/config/config.go +++ b/config/config.go @@ -1,6 +1,9 @@ package config -import "github.com/asjdf/gorm-cache/storage" +import ( + "github.com/asjdf/gorm-cache/storage" + "github.com/asjdf/gorm-cache/util" +) type CacheConfig struct { // CacheLevel there are 2 types of cache and 4 kinds of cache option @@ -34,7 +37,7 @@ type CacheConfig struct { DebugMode bool // DebugLogger - DebugLogger LoggerInterface + DebugLogger util.LoggerInterface } type CacheLevel int diff --git a/go.mod b/go.mod index 1e8d73d..a58898d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/karlseguin/ccache/v3 v3.0.3 github.com/redis/go-redis/v9 v9.0.2 github.com/smartystreets/goconvey v1.7.2 - gorm.io/driver/mysql v1.4.7 gorm.io/gorm v1.24.5 ) @@ -18,7 +17,6 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/glebarez/go-sqlite v1.20.3 // indirect - github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect diff --git a/go.sum b/go.sum index c61eb8b..fcae857 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,6 @@ github.com/glebarez/go-sqlite v1.20.3 h1:89BkqGOXR9oRmG58ZrzgoY/Fhy5x0M+/WV48U5z github.com/glebarez/go-sqlite v1.20.3/go.mod h1:u3N6D/wftiAzIOJtZl6BmedqxmmkDfH3q+ihjqxC9u0= github.com/glebarez/sqlite v1.7.0 h1:A7Xj/KN2Lvie4Z4rrgQHY8MsbebX3NyWsL3n2i82MVI= github.com/glebarez/sqlite v1.7.0/go.mod h1:PkeevrRlF/1BhQBCnzcMWzgrIk7IOop+qS2jUYLfHhk= -github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= -github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -65,9 +63,6 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gorm.io/driver/mysql v1.4.7 h1:rY46lkCspzGHn7+IYsNpSfEv9tA+SU4SkkB+GFX125Y= -gorm.io/driver/mysql v1.4.7/go.mod h1:SxzItlnT1cb6e1e4ZRpgJN2VYtcqJgqnHxWr4wsP8oc= -gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.24.5 h1:g6OPREKqqlWq4kh/3MCQbZKImeB9e6Xgc4zD+JgNZGE= gorm.io/gorm v1.24.5/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0= diff --git a/storage/gcache.go b/storage/gcache.go index abd9f24..b7c9541 100644 --- a/storage/gcache.go +++ b/storage/gcache.go @@ -2,7 +2,6 @@ package storage import ( "context" - "github.com/asjdf/gorm-cache/config" "github.com/asjdf/gorm-cache/util" "github.com/bluele/gcache" "strings" @@ -26,10 +25,10 @@ type Gcache struct { once sync.Once } -func (g *Gcache) Init(config *config.CacheConfig, prefix string) error { +func (g *Gcache) Init(config *Config, prefix string) error { g.once.Do(func() { - if config.CacheTTL != 0 { - g.builder.Expiration(time.Duration(config.CacheTTL) * time.Microsecond) + if config.TTL != 0 { + g.builder.Expiration(time.Duration(config.TTL) * time.Microsecond) } g.cache = g.builder.Build() }) diff --git a/storage/interface.go b/storage/interface.go index 4082857..04bd2ce 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -3,7 +3,6 @@ package storage import ( "context" "errors" - "github.com/asjdf/gorm-cache/config" "github.com/asjdf/gorm-cache/util" ) @@ -11,8 +10,14 @@ var ( ErrCacheNotFound = errors.New("cache not found") ) +type Config struct { + TTL int64 + Debug bool + Logger util.LoggerInterface +} + type DataStorage interface { - Init(config *config.CacheConfig, prefix string) error + Init(config *Config, prefix string) error CleanCache(ctx context.Context) error // read diff --git a/storage/memory.go b/storage/memory.go index c6b648a..b0db836 100644 --- a/storage/memory.go +++ b/storage/memory.go @@ -7,7 +7,6 @@ import ( "sync" "time" - "github.com/asjdf/gorm-cache/config" "github.com/asjdf/gorm-cache/util" ) @@ -32,11 +31,11 @@ type Memory struct { once sync.Once } -func (m *Memory) Init(conf *config.CacheConfig, prefix string) error { +func (m *Memory) Init(conf *Config, prefix string) error { m.once.Do(func() { c := ccache.New(ccache.Configure[string]().MaxSize(m.config.MaxSize)) m.cache = c - m.ttl = conf.CacheTTL + m.ttl = conf.TTL }) return nil } diff --git a/storage/redis.go b/storage/redis.go index 3e84eca..f3b35cd 100644 --- a/storage/redis.go +++ b/storage/redis.go @@ -5,11 +5,12 @@ import ( "sync" "time" - "github.com/asjdf/gorm-cache/config" "github.com/asjdf/gorm-cache/util" "github.com/redis/go-redis/v9" ) +var _ DataStorage = &Redis{} + func NewRedisWithClient(client *redis.Client) *Redis { return &Redis{ client: client, @@ -23,7 +24,7 @@ func NewRedisWithOptions(options *redis.Options) *Redis { type Redis struct { client *redis.Client ttl int64 - logger config.LoggerInterface + logger util.LoggerInterface keyPrefix string batchExistSha string @@ -32,12 +33,12 @@ type Redis struct { once sync.Once } -func (r *Redis) Init(conf *config.CacheConfig, prefix string) error { +func (r *Redis) Init(conf *Config, prefix string) error { var err error r.once.Do(func() { - r.ttl = conf.CacheTTL - r.logger = conf.DebugLogger - r.logger.SetIsDebug(conf.DebugMode) + r.ttl = conf.TTL + r.logger = conf.Logger + r.logger.SetIsDebug(conf.Debug) r.keyPrefix = prefix err = r.initScripts() }) diff --git a/config/logger.go b/util/logger.go similarity index 65% rename from config/logger.go rename to util/logger.go index dd0eabd..686366f 100644 --- a/config/logger.go +++ b/util/logger.go @@ -1,4 +1,4 @@ -package config +package util import ( "context" @@ -12,22 +12,22 @@ type LoggerInterface interface { CtxError(ctx context.Context, format string, v ...interface{}) } -type DefaultLoggerImpl struct { +type DefaultLogger struct { isDebug bool } -func (l *DefaultLoggerImpl) SetIsDebug(d bool) { +func (l *DefaultLogger) SetIsDebug(d bool) { l.isDebug = d } -func (l *DefaultLoggerImpl) CtxInfo(ctx context.Context, format string, v ...interface{}) { +func (l *DefaultLogger) CtxInfo(ctx context.Context, format string, v ...interface{}) { if l.isDebug { timePrefix := time.Now().Format("2006-01-02 15:04:05.999") fmt.Printf(timePrefix+" [INFO] "+format+"\n", v...) } } -func (l *DefaultLoggerImpl) CtxError(ctx context.Context, format string, v ...interface{}) { +func (l *DefaultLogger) CtxError(ctx context.Context, format string, v ...interface{}) { if l.isDebug { timePrefix := time.Now().Format("2006-01-02 15:04:05.999") fmt.Printf(timePrefix+" [ERROR] "+format+"\n", v...) From 11de307e3015b068f6e34f01ae3bc927d867358f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=88=90=E9=94=B4?= Date: Tue, 21 Feb 2023 22:27:21 +0800 Subject: [PATCH 12/13] feat: add test for lib --- test/create_test.go | 28 +++++++ test/delete_test.go | 74 +++++++++++++++++ test/functionality_test.go | 37 +++++++++ test/model.go | 22 +++++ test/prepare_test.go | 34 ++++++++ test/query_test.go | 118 ++++++++++++++++++++++++++ test/setup_test.go | 165 +++++++++++++++++++++++++++++++++++++ test/update_test.go | 66 +++++++++++++++ test/util.go | 20 +++++ 9 files changed, 564 insertions(+) create mode 100644 test/create_test.go create mode 100644 test/delete_test.go create mode 100644 test/functionality_test.go create mode 100644 test/model.go create mode 100644 test/prepare_test.go create mode 100644 test/query_test.go create mode 100644 test/setup_test.go create mode 100644 test/update_test.go create mode 100644 test/util.go diff --git a/test/create_test.go b/test/create_test.go new file mode 100644 index 0000000..00632fa --- /dev/null +++ b/test/create_test.go @@ -0,0 +1,28 @@ +package test + +import ( + . "github.com/smartystreets/goconvey/convey" + + "github.com/asjdf/gorm-cache/cache" + "gorm.io/gorm" +) + +func testSearchCreate(cache cache.Cache, db *gorm.DB) { + err := cache.ResetCache() + So(err, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + var model = new(TestModel) + + result := db.Where("id = ?", 1).First(model) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + So(model.ID, ShouldEqual, 1) + + result = db.Create(&TestModel{}) + So(result.Error, ShouldBeNil) + + result = db.Where("id = ?", 1).First(model) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) +} diff --git a/test/delete_test.go b/test/delete_test.go new file mode 100644 index 0000000..41d4965 --- /dev/null +++ b/test/delete_test.go @@ -0,0 +1,74 @@ +package test + +import ( + . "github.com/smartystreets/goconvey/convey" + + "github.com/asjdf/gorm-cache/cache" + "gorm.io/gorm" +) + +func testPrimaryDelete(cache cache.Cache, db *gorm.DB) { + err := cache.ResetCache() + So(err, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + models := make([]*TestModel, 0) + result := db.Where("id IN (?)", []int{101, 102, 103, 104, 105}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + models = make([]*TestModel, 0) + result = db.Where("id IN (?)", []int{101, 102, 103}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 1) + So(len(models), ShouldEqual, 3) + + result = db.Delete(&TestModel{ID: 105}) + So(result.Error, ShouldBeNil) + + models = make([]*TestModel, 0) + result = db.Where("id IN (?)", []int{101, 102, 103, 104}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 2) + + models = make([]*TestModel, 0) + result = db.Where("id IN (?)", []int{101, 102, 103, 104, 105}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 2) + + result = db.Delete([]*TestModel{{ID: 103}, {ID: 104}}) + So(result.Error, ShouldBeNil) + + models = make([]*TestModel, 0) + result = db.Where("id IN (?)", []int{101, 102}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 3) + + result = db.Where("id = 102").Delete(&TestModel{}) + So(result.Error, ShouldBeNil) + + result = db.Where("id IN (?)", []int{101}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 4) +} + +func testSearchDelete(cache cache.Cache, db *gorm.DB) { + err := cache.ResetCache() + So(err, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + models := make([]TestModel, 0) + result := db.Where("id IN (?)", []int{51, 52}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + So(len(models), ShouldEqual, 2) + + result = db.Delete(&TestModel{ID: 53}) + So(result.Error, ShouldBeNil) + + models = make([]TestModel, 0) + result = db.Where("id IN (?)", []int{51, 52}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + So(len(models), ShouldEqual, 2) +} diff --git a/test/functionality_test.go b/test/functionality_test.go new file mode 100644 index 0000000..9e1dc48 --- /dev/null +++ b/test/functionality_test.go @@ -0,0 +1,37 @@ +package test + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestPrimaryCacheFunctionality(t *testing.T) { + Convey("test primary cache functionality", t, func() { + testFirst(primaryCache, primaryDB) + + testFind(primaryCache, primaryDB) + + testPrimaryFind(primaryCache, primaryDB) + + testPrimaryUpdate(primaryCache, primaryDB) + + testPrimaryDelete(primaryCache, primaryDB) + }) +} + +func TestSearchCacheFunctionality(t *testing.T) { + Convey("test search cache functionality", t, func() { + testFirst(searchCache, searchDB) + + testFind(searchCache, searchDB) + + testSearchFind(searchCache, searchDB) + + testSearchCreate(searchCache, searchDB) + + testSearchDelete(searchCache, searchDB) + + testSearchUpdate(searchCache, searchDB) + }) +} diff --git a/test/model.go b/test/model.go new file mode 100644 index 0000000..7db7e54 --- /dev/null +++ b/test/model.go @@ -0,0 +1,22 @@ +package test + +type TestModel struct { + ID int64 `gorm:"column:id;primary_key"` + Value1 int64 `gorm:"column:value1"` + Value2 int64 `gorm:"column:value2"` + Value3 int64 `gorm:"column:value3"` + Value4 int64 `gorm:"column:value4"` + Value5 int64 `gorm:"column:value5"` + Value6 int64 `gorm:"column:value6"` + Value7 int64 `gorm:"column:value7"` + Value8 int64 `gorm:"column:value8"` + PtrValue1 *int64 `gorm:"column:ptr_value1"` +} + +const ( + TestModelTableName = "gorm_cache_model" +) + +func (m *TestModel) TableName() string { + return TestModelTableName +} diff --git a/test/prepare_test.go b/test/prepare_test.go new file mode 100644 index 0000000..c7c095a --- /dev/null +++ b/test/prepare_test.go @@ -0,0 +1,34 @@ +package test + +import "gorm.io/gorm" + +func PrepareTableAndData(db *gorm.DB) error { + err := db.AutoMigrate(&TestModel{}) + if err != nil { + return err + } + + models := make([]TestModel, 0, testSize) + for i := 1; i <= testSize; i++ { + _pValue := int64(i) + model := TestModel{ + ID: int64(i), + Value1: int64(i), + Value2: int64(i), + Value3: int64(i), + Value4: int64(i), + Value5: int64(i), + Value6: int64(i), + Value7: int64(i), + Value8: int64(i), + PtrValue1: &_pValue, + } + models = append(models, model) + } + + return db.CreateInBatches(models, 2000).Error +} + +func CleanTable(db *gorm.DB) error { + return db.Migrator().DropTable(&TestModel{}) +} diff --git a/test/query_test.go b/test/query_test.go new file mode 100644 index 0000000..c3b1b57 --- /dev/null +++ b/test/query_test.go @@ -0,0 +1,118 @@ +package test + +import ( + . "github.com/smartystreets/goconvey/convey" + + "github.com/asjdf/gorm-cache/cache" + "gorm.io/gorm" +) + +func testFirst(cache cache.Cache, db *gorm.DB) { + err := cache.ResetCache() + So(err, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + var model = new(TestModel) + + result := db.Where("id = ?", 1).First(model) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + So(model.ID, ShouldEqual, 1) + + model = new(TestModel) + result = db.Where("id = ?", 1).First(model) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 1) + + targetModel := &TestModel{ + ID: 2, + } + + result = db.First(targetModel) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 1) + + result = db.First(targetModel) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 2) +} + +func testFind(cache cache.Cache, db *gorm.DB) { + err := cache.ResetCache() + So(err, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + models := make([]*TestModel, 0) + result := db.Where("id IN (?)", []int{1, 2}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + models = make([]*TestModel, 0) + result = db.Where("id IN (?)", []int{1, 2}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 1) + So(len(models), ShouldEqual, 2) + So(models[0].Value1, ShouldEqual, 1) + So(models[1].Value1, ShouldEqual, 2) +} + +func testPtrFind(cache cache.Cache, db *gorm.DB) { + err := cache.ResetCache() + So(err, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + _prtValue := int64(1) + model := &TestModel{ + PtrValue1: &_prtValue, + } + result := db.Model(model).Find(model) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 1) + So(model.Value1, ShouldEqual, 1) +} + +func testPrimaryFind(cache cache.Cache, db *gorm.DB) { + err := cache.ResetCache() + So(err, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + models := make([]*TestModel, 0) + result := db.Where("id IN (?)", []int{1, 2, 3}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + models = make([]*TestModel, 0) + result = db.Where("id IN (?)", []int{1, 2}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 1) + So(len(models), ShouldEqual, 2) + So(models[0].Value1, ShouldEqual, 1) + So(models[1].Value1, ShouldEqual, 2) + + models = make([]*TestModel, 0) + result = db.Where("id < ?", 3).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 1) + + models = make([]*TestModel, 0) + result = db.Where("id IN (?)", []int64{1, 4}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 1) +} + +func testSearchFind(cache cache.Cache, db *gorm.DB) { + err := cache.ResetCache() + So(err, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + models := make([]*TestModel, 0) + result := db.Where("id >= ?", 1).Where("id <= ?", 10).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + models = make([]*TestModel, 0) + result = db.Where("id >= ?", 1).Where("id <= ?", 10).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 1) + So(len(models), ShouldEqual, 10) +} diff --git a/test/setup_test.go b/test/setup_test.go new file mode 100644 index 0000000..c1fa195 --- /dev/null +++ b/test/setup_test.go @@ -0,0 +1,165 @@ +package test + +import ( + "github.com/asjdf/gorm-cache/storage" + "github.com/bluele/gcache" + "os" + "testing" + + "gorm.io/gorm/logger" + + "github.com/asjdf/gorm-cache/cache" + + "github.com/asjdf/gorm-cache/config" + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +var ( + searchCache cache.Cache + primaryCache cache.Cache + allCache cache.Cache + + searchDB *gorm.DB + primaryDB *gorm.DB + allDB *gorm.DB + originalDB *gorm.DB +) + +var ( + testSize = 200 // minimum 200 +) + +func TestMain(m *testing.M) { + log("test setup ...") + + var err error + //logger.Default.LogMode(logger.Info) + + f, err := os.CreateTemp("", "gormCacheTest.*.db") + if err != nil { + log("create temp db error: %v", err) + os.Exit(-1) + } + defer os.Remove(f.Name()) + originalDB, err = gorm.Open(sqlite.Open(f.Name()), &gorm.Config{ + CreateBatchSize: 1000, + Logger: logger.Default, + }) + if err != nil { + log("open db error: %v", err) + os.Exit(-1) + } + + searchDB, err = forkDB(originalDB) + if err != nil { + log("open db error: %v", err) + os.Exit(-1) + } + + primaryDB, err = forkDB(originalDB) + if err != nil { + log("open db error: %v", err) + os.Exit(-1) + } + + allDB, err = forkDB(originalDB) + if err != nil { + log("open db error: %v", err) + os.Exit(-1) + } + + searchCache, err = cache.NewGorm2Cache(&config.CacheConfig{ + CacheLevel: config.CacheLevelOnlySearch, + CacheStorage: storage.NewGcache(gcache.New(1000)), + InvalidateWhenUpdate: true, + CacheTTL: 5000, + CacheMaxItemCnt: 5000, + DebugMode: false, + }) + if err != nil { + log("setup search cache error: %v", err) + os.Exit(-1) + } + + primaryCache, err = cache.NewGorm2Cache(&config.CacheConfig{ + CacheLevel: config.CacheLevelOnlyPrimary, + CacheStorage: storage.NewGcache(gcache.New(1000)), + InvalidateWhenUpdate: true, + CacheTTL: 5000, + CacheMaxItemCnt: 5000, + DebugMode: false, + }) + if err != nil { + log("setup primary cache error: %v", err) + os.Exit(-1) + } + + allCache, err = cache.NewGorm2Cache(&config.CacheConfig{ + CacheLevel: config.CacheLevelAll, + CacheStorage: storage.NewGcache(gcache.New(1000)), + InvalidateWhenUpdate: true, + CacheTTL: 5000, + CacheMaxItemCnt: 5000, + DebugMode: false, + }) + if err != nil { + log("setup all cache error: %v", err) + os.Exit(-1) + } + + primaryDB.Use(primaryCache) + searchDB.Use(searchCache) + allDB.Use(allCache) + // primaryCache.AttachToDB(primaryDB)+ + // searchCache.AttachToDB(searchDB) + // allCache.AttachToDB(allDB) + + err = timer("prepare table and data", func() error { + return PrepareTableAndData(originalDB) + }) + if err != nil { + log("setup table and data error: %v", err) + os.Exit(-1) + } + + result := m.Run() + + err = timer("clean table and data", func() error { + return CleanTable(originalDB) + }) + if err != nil { + log("clean table and data error: %v", err) + os.Exit(-1) + } + + log("integration test end.") + os.Exit(result) +} + +func forkDB(db *gorm.DB) (newDB *gorm.DB, err error) { + plugins := map[string]gorm.Plugin{} + for k, v := range db.Config.Plugins { + plugins[k] = v + } + newDB, err = gorm.Open(db.Dialector, &gorm.Config{ + SkipDefaultTransaction: db.Config.SkipDefaultTransaction, + NamingStrategy: db.Config.NamingStrategy, + FullSaveAssociations: db.Config.FullSaveAssociations, + Logger: db.Config.Logger, + NowFunc: db.Config.NowFunc, + DryRun: db.Config.DryRun, + PrepareStmt: db.Config.PrepareStmt, + DisableAutomaticPing: db.Config.DisableAutomaticPing, + DisableForeignKeyConstraintWhenMigrating: db.Config.DisableForeignKeyConstraintWhenMigrating, + IgnoreRelationshipsWhenMigrating: db.Config.IgnoreRelationshipsWhenMigrating, + DisableNestedTransaction: db.Config.DisableNestedTransaction, + AllowGlobalUpdate: db.Config.AllowGlobalUpdate, + QueryFields: db.Config.QueryFields, + CreateBatchSize: db.Config.CreateBatchSize, + ClauseBuilders: db.Config.ClauseBuilders, + ConnPool: db.Config.ConnPool, + Plugins: plugins, + }) + return +} diff --git a/test/update_test.go b/test/update_test.go new file mode 100644 index 0000000..80c75b3 --- /dev/null +++ b/test/update_test.go @@ -0,0 +1,66 @@ +package test + +import ( + . "github.com/smartystreets/goconvey/convey" + + "github.com/asjdf/gorm-cache/cache" + "gorm.io/gorm" +) + +func testPrimaryUpdate(cache cache.Cache, db *gorm.DB) { + err := cache.ResetCache() + So(err, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + models := make([]*TestModel, 0) + result := db.Where("id IN (?)", []int{1, 2, 3}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + models = make([]*TestModel, 0) + result = db.Where("id IN (?)", []int{1, 2}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 1) + So(len(models), ShouldEqual, 2) + + result = db.Model(models[0]).Where("id IN (1)").Updates(map[string]interface{}{"value8": -1}) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 1) + + models = make([]*TestModel, 0) + result = db.Where("id IN (1,2)").Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 1) + So(len(models), ShouldEqual, 2) + So(models[0].Value8, ShouldEqual, -1) + + result = db.Table(TestModelTableName).Where("value8 = -1").UpdateColumn("value8", 1) + So(result.Error, ShouldBeNil) + + models = make([]*TestModel, 0) + result = db.Where("id IN (1,2)").Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 1) +} + +func testSearchUpdate(cache cache.Cache, db *gorm.DB) { + err := cache.ResetCache() + So(err, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + + models := make([]TestModel, 0) + result := db.Where("id IN (?)", []int{51, 52}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 0) + So(len(models), ShouldEqual, 2) + + result = db.Table(TestModelTableName). + Where("id IN (?)", []int{53}).UpdateColumn("value5", 5) + So(result.Error, ShouldBeNil) + + models = make([]TestModel, 0) + result = db.Where("id IN (?)", []int{51, 52}).Find(&models) + So(result.Error, ShouldBeNil) + So(cache.HitCount(), ShouldEqual, 1) + So(len(models), ShouldEqual, 2) +} diff --git a/test/util.go b/test/util.go new file mode 100644 index 0000000..5a1eb4f --- /dev/null +++ b/test/util.go @@ -0,0 +1,20 @@ +package test + +import ( + "fmt" + "time" +) + +func log(format string, a ...interface{}) { + timeStr := time.Now().Format("2006-01-02 15:04:05.999") + fmt.Printf(timeStr+" "+format+"\n", a...) +} + +func timer(name string, f func() error) error { + start := time.Now() + fmt.Printf("[%s] start ...\n", name) + err := f() + duration := time.Now().Sub(start) + fmt.Printf("[%s] finished. cost: %.3fs\n", name, duration.Seconds()) + return err +} From 7eecba4e57526a9bcd0a1b07909e51e35f6b034c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=88=90=E9=94=B4?= Date: Tue, 21 Feb 2023 22:32:28 +0800 Subject: [PATCH 13/13] update: update docs --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index af56197..65f16a1 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ import ( "context" "github.com/asjdf/gorm-cache/cache" + "github.com/asjdf/gorm-cache/storage" "github.com/redis/go-redis/v9" ) @@ -33,7 +34,7 @@ func main() { CacheMaxItemCnt: 50, // if length of objects retrieved one single time // exceeds this number, then don't cache }) - // More options in `config.config.go` + // More options in `config/config.go` db.Use(cache) // use gorm plugin // cache.AttachToDB(db)