Merge pull request #6 from asjdf/dev

first std release

.github/workflows/release.yaml

@@ -0,0 +1,42 @@
+name: Release
+on:
+  push:
+#  pull_request:
+#    branches:
+#      - '**'
+  workflow_dispatch:
+
+jobs:
+#  test:
+#    runs-on: ubuntu-22.04
+#    strategy:
+#      matrix:
+#        go: ['1.17', '1.18', '1.19']
+#    name: Go ${{ matrix.go }} test
+#    steps:
+#      - uses: actions/checkout@v3
+#      - name: Setup go
+#        uses: actions/setup-go@v3
+#        with:
+#          go-version: ${{ matrix.go }}
+#      - run: go test -race -v -coverprofile=profile.cov ./pkg/...
+#      - uses: codecov/[email protected]
+#        with:
+#          file: ./profile.cov
+#          name: codecov-go
+  release:
+    runs-on: ubuntu-22.04
+#    needs: [test]
+#    if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/main' }}
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Source checkout
+        uses: actions/checkout@v3
+      - name: Semantic Release
+        uses: cycjimmy/semantic-release-action@v3
+        with:
+          dry_run: false
+          semantic_version: 18.0.1
+          extra_plugins: |
+            @semantic-release/[email protected]

.releaserc.js

@@ -0,0 +1,8 @@
+module.exports = {
+    branches: ["main", {name: 'dev', prerelease: true}],
+    plugins: [
+        "@semantic-release/commit-analyzer",
+        "@semantic-release/release-notes-generator",
+        "@semantic-release/github"
+    ]
+};

README.md

@@ -2,17 +2,19 @@
 
 `gorm-cache` 旨在为gorm v2用户提供一个即插即用的旁路缓存解决方案。本缓存只适用于数据库表单主键时的场景。
 
-本库支持使用2种 cache 存储介质：
-
-1. 内存 (所有数据存储在单服务器的内存中)
-2. Redis (所有数据存储在redis中，如果你有多个实例使用本缓存，那么他们不共享redis存储空间)
+## 特性
+- 即插即用
+- 旁路缓存
+- 穿透防护
+- 多存储介质（内存/redis）
 
 ## 使用说明
 
 ```go
 import (
     "context"
     "github.com/asjdf/gorm-cache/cache"
+    "github.com/asjdf/gorm-cache/storage"
     "github.com/redis/go-redis/v9"
 )
 
@@ -26,14 +28,13 @@ func main() {
 
     cache, _ := cache.NewGorm2Cache(&config.CacheConfig{
         CacheLevel:           config.CacheLevelAll,
-        CacheStorage:         config.CacheStorageRedis,
-        RedisConfig:          cache.NewRedisConfigWithClient(redisClient),
+        CacheStorage:         storage.NewRedisWithClient(redisClient),
         InvalidateWhenUpdate: true, // when you create/update/delete objects, invalidate cache
         CacheTTL:             5000, // 5000 ms
-        CacheMaxItemCnt:      5,    // if length of objects retrieved one single time 
+        CacheMaxItemCnt:      50,   // if length of objects retrieved one single time 
                                     // exceeds this number, then don't cache
     })
-    // More options in `config.config.go`
+    // More options in `config/config.go`
     db.Use(cache)    // use gorm plugin
     // cache.AttachToDB(db)
 
@@ -55,4 +56,13 @@ func main() {
 4. Update (Update/Updates/UpdateColumn/UpdateColumns/Save)
 5. Row (Row/Rows/Scan)
 
-本库不支持Row操作的缓存。
+本库不支持Row操作的缓存。（WIP）
+
+## 存储介质细节
+
+本库支持使用2种 cache 存储介质：
+
+1. 内存 (ccache/gcache)
+2. Redis (所有数据存储在redis中，如果你有多个实例使用本缓存，那么他们不共享redis存储空间)
+
+并且允许多个gorm-cache公用一个存储池，以确保同一数据库的多个gorm实例共享缓存。

cache/afterCreate.go

@@ -8,6 +8,10 @@ import (
 
 func AfterCreate(cache *Gorm2Cache) func(db *gorm.DB) {
 	return func(db *gorm.DB) {
+		if db.RowsAffected == 0 {
+			return // no rows affected, no need to invalidate cache
+		}
+
 		tableName := ""
 		if db.Statement.Schema != nil {
 			tableName = db.Statement.Schema.Table
@@ -18,16 +22,23 @@ func AfterCreate(cache *Gorm2Cache) func(db *gorm.DB) {
 
 		if db.Error == nil && cache.Config.InvalidateWhenUpdate && util.ShouldCache(tableName, cache.Config.Tables) {
 			if cache.Config.CacheLevel == config.CacheLevelAll || cache.Config.CacheLevel == config.CacheLevelOnlySearch {
-				// We invalidate search cache here,
-				// because any newly created objects may cause search cache results to be outdated and invalid.
-				cache.Logger.CtxInfo(ctx, "[AfterCreate] now start to invalidate search cache for table: %s", tableName)
-				err := cache.InvalidateSearchCache(ctx, tableName)
-				if err != nil {
-					cache.Logger.CtxError(ctx, "[AfterCreate] invalidating search cache for table %s error: %v",
-						tableName, err)
-					return
+				invalidSearchCache := func() {
+					// We invalidate search cache here,
+					// because any newly created objects may cause search cache results to be outdated and invalid.
+					cache.Logger.CtxInfo(ctx, "[AfterCreate] now start to invalidate search cache for table: %s", tableName)
+					err := cache.InvalidateSearchCache(ctx, tableName)
+					if err != nil {
+						cache.Logger.CtxError(ctx, "[AfterCreate] invalidating search cache for table %s error: %v",
+							tableName, err)
+						return
+					}
+					cache.Logger.CtxInfo(ctx, "[AfterCreate] invalidating search cache for table: %s finished.", tableName)
+				}
+				if cache.Config.AsyncWrite {
+					go invalidSearchCache()
+				} else {
+					invalidSearchCache()
 				}
-				cache.Logger.CtxInfo(ctx, "[AfterCreate] invalidating search cache for table: %s finished.", tableName)
 			}
 		}
 	}

cache/afterDelete.go

@@ -10,6 +10,10 @@ import (
 
 func AfterDelete(cache *Gorm2Cache) func(db *gorm.DB) {
 	return func(db *gorm.DB) {
+		if db.RowsAffected == 0 {
+			return // no rows affected, no need to invalidate cache
+		}
+
 		tableName := ""
 		if db.Statement.Schema != nil {
 			tableName = db.Statement.Schema.Table
@@ -66,7 +70,9 @@ func AfterDelete(cache *Gorm2Cache) func(db *gorm.DB) {
 				}
 			}()
 
-			wg.Wait()
+			if !cache.Config.AsyncWrite {
+				wg.Wait()
+			}
 		}
 	}
 }

cache/afterQuery.go

@@ -88,7 +88,25 @@ func AfterQuery(cache *Gorm2Cache) func(db *gorm.DB) {
 					}
 				}
 			}()
-			wg.Wait()
+			if !cache.Config.AsyncWrite {
+				wg.Wait()
+			}
+			return
+		}
+
+		if !cache.Config.DisableCachePenetrationProtect {
+			if errors.Is(db.Error, gorm.ErrRecordNotFound) { // 应对缓存穿透 未来可能考虑使用其他过滤器实现：如布隆过滤器
+				cache.Logger.CtxInfo(ctx, "[AfterQuery] set cache: %v", "recordNotFound")
+				err := cache.SetSearchCache(ctx, "recordNotFound", tableName, sql, vars...)
+				if err != nil {
+					cache.Logger.CtxError(ctx, "[AfterQuery] set search cache for sql: %s error: %v", sql, err)
+					return
+				}
+				cache.Logger.CtxInfo(ctx, "[AfterQuery] sql %s cached", sql)
+			}
+		}
+		if errors.Is(db.Error, util.RecordNotFoundCacheHit) {
+			db.Error = gorm.ErrRecordNotFound
 			return
 		}
 

cache/afterUpdate.go

@@ -10,6 +10,10 @@ import (
 
 func AfterUpdate(cache *Gorm2Cache) func(db *gorm.DB) {
 	return func(db *gorm.DB) {
+		if db.RowsAffected == 0 {
+			return // no rows affected, no need to invalidate cache
+		}
+
 		tableName := ""
 		if db.Statement.Schema != nil {
 			tableName = db.Statement.Schema.Table
@@ -67,7 +71,9 @@ func AfterUpdate(cache *Gorm2Cache) func(db *gorm.DB) {
 				}
 			}()
 
-			wg.Wait()
+			if !cache.Config.AsyncWrite {
+				wg.Wait()
+			}
 		}
 	}
 }

cache/beforeQuery.go

@@ -31,16 +31,20 @@ func BeforeQuery(cache *Gorm2Cache) func(db *gorm.DB) {
 		if util.ShouldCache(tableName, cache.Config.Tables) {
 			if cache.Config.CacheLevel == config.CacheLevelAll || cache.Config.CacheLevel == config.CacheLevelOnlySearch {
 				// search cache hit
-
 				cacheValue, err := cache.GetSearchCache(ctx, tableName, sql, db.Statement.Vars...)
 				if err != nil {
 					if !errors.Is(err, storage.ErrCacheNotFound) {
 						cache.Logger.CtxError(ctx, "[BeforeQuery] get cache value for sql %s error: %v", sql, err)
 					}
+					cache.stats.IncrMissCount()
 					db.Error = nil
 					return
 				}
 				cache.Logger.CtxInfo(ctx, "[BeforeQuery] get value: %s", cacheValue)
+				if cacheValue == "recordNotFound" { // 应对缓存穿透
+					db.Error = util.RecordNotFoundCacheHit
+					return
+				}
 				rowsAffectedPos := strings.Index(cacheValue, "|")
 				db.RowsAffected, err = strconv.ParseInt(cacheValue[:rowsAffectedPos], 10, 64)
 				if err != nil {

cache/cache.go

@@ -2,8 +2,6 @@ package cache
 
 import (
 	"context"
-	"sync/atomic"
-
 	"github.com/asjdf/gorm-cache/config"
 	"github.com/asjdf/gorm-cache/storage"
 	"github.com/asjdf/gorm-cache/util"
@@ -13,6 +11,7 @@ import (
 
 var (
 	_ gorm.Plugin = &Gorm2Cache{}
+	_ Cache       = &Gorm2Cache{}
 
 	json = jsoniter.Config{
 		EscapeHTML:             true,
@@ -21,32 +20,43 @@ var (
 	}.Froze()
 )
 
+type Cache interface {
+	Name() string
+	Initialize(db *gorm.DB) error
+	AttachToDB(db *gorm.DB)
+
+	ResetCache() error
+	StatsAccessor
+}
+
 type Gorm2Cache struct {
 	Config     *config.CacheConfig
-	Logger     config.LoggerInterface
+	Logger     util.LoggerInterface
 	InstanceId string
 
 	db       *gorm.DB
 	cache    storage.DataStorage
 	hitCount int64
+
+	*stats
 }
 
 func (c *Gorm2Cache) Name() string {
 	return util.GormCachePrefix
 }
 
 func (c *Gorm2Cache) Initialize(db *gorm.DB) (err error) {
-	err = db.Callback().Create().After("*").Register("gorm:cache:after_create", AfterCreate(c))
+	err = db.Callback().Create().After("gorm:create").Register("gorm:cache:after_create", AfterCreate(c))
 	if err != nil {
 		return err
 	}
 
-	err = db.Callback().Delete().After("*").Register("gorm:cache:after_delete", AfterDelete(c))
+	err = db.Callback().Delete().After("gorm:delete").Register("gorm:cache:after_delete", AfterDelete(c))
 	if err != nil {
 		return err
 	}
 
-	err = db.Callback().Update().After("*").Register("gorm:cache:after_update", AfterUpdate(c))
+	err = db.Callback().Update().After("gorm:update").Register("gorm:cache:after_update", AfterUpdate(c))
 	if err != nil {
 		return err
 	}
@@ -56,7 +66,7 @@ func (c *Gorm2Cache) Initialize(db *gorm.DB) (err error) {
 		return err
 	}
 
-	err = db.Callback().Query().After("*").Register("gorm:cache:after_query", AfterQuery(c))
+	err = db.Callback().Query().After("gorm:query").Register("gorm:cache:after_query", AfterQuery(c))
 	if err != nil {
 		return err
 	}
@@ -65,54 +75,40 @@ func (c *Gorm2Cache) Initialize(db *gorm.DB) (err error) {
 }
 
 func (c *Gorm2Cache) AttachToDB(db *gorm.DB) {
-	c.Initialize(db)
+	_ = c.Initialize(db)
 }
 
 func (c *Gorm2Cache) Init() error {
-	if c.Config.CacheStorage == config.CacheStorageRedis {
-		if c.Config.RedisConfig == nil {
-			panic("please init redis config!")
-		}
-		c.Config.RedisConfig.InitClient()
-	}
 	c.InstanceId = util.GenInstanceId()
 
 	prefix := util.GormCachePrefix + ":" + c.InstanceId
 
-	if c.Config.CacheStorage == config.CacheStorageRedis {
-		c.cache = &storage.RedisLayer{}
-	} else if c.Config.CacheStorage == config.CacheStorageMemory {
-		c.cache = &storage.MemoryLayer{}
+	if c.cache != nil {
+		c.cache = c.Config.CacheStorage
+	} else {
+		c.cache = storage.NewMem(storage.DefaultMemStoreConfig)
 	}
 
 	if c.Config.DebugLogger == nil {
-		c.Config.DebugLogger = &config.DefaultLoggerImpl{}
+		c.Config.DebugLogger = &util.DefaultLogger{}
 	}
 	c.Logger = c.Config.DebugLogger
 	c.Logger.SetIsDebug(c.Config.DebugMode)
 
-	err := c.cache.Init(c.Config, prefix)
+	err := c.cache.Init(&storage.Config{
+		TTL:    c.Config.CacheTTL,
+		Debug:  c.Config.DebugMode,
+		Logger: c.Logger,
+	}, prefix)
 	if err != nil {
 		c.Logger.CtxError(context.Background(), "[Init] cache init error: %v", err)
 		return err
 	}
 	return nil
 }
 
-func (c *Gorm2Cache) GetHitCount() int64 {
-	return atomic.LoadInt64(&c.hitCount)
-}
-
-func (c *Gorm2Cache) ResetHitCount() {
-	atomic.StoreInt64(&c.hitCount, 0)
-}
-
-func (c *Gorm2Cache) IncrHitCount() {
-	atomic.AddInt64(&c.hitCount, 1)
-}
-
 func (c *Gorm2Cache) ResetCache() error {
-	c.ResetHitCount()
+	c.stats.ResetHitCount()
 	ctx := context.Background()
 	err := c.cache.CleanCache(ctx)
 	if err != nil {