From dc04b8dc2b1c0f50d9b42e82205d5933f00c5f64 Mon Sep 17 00:00:00 2001 From: xing Date: Thu, 2 Feb 2023 13:19:43 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E7=AE=80=E5=8C=96=E7=89=88=20pagination?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- model/querycondition.go | 9 +++++- model/querycondition_test.go | 57 ++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/model/querycondition.go b/model/querycondition.go index 1976754..33f0484 100644 --- a/model/querycondition.go +++ b/model/querycondition.go @@ -7,7 +7,7 @@ import ( "strings" ) -// Finds can use offset +// Finds 比 Find 多一个offset // // Conditions 中可用 Where Fields Group Having Join Order Offset Limit In 函数 func Finds[T Model](ctx context.Context, q *QueryCondition) (r []T, err error) { @@ -132,3 +132,10 @@ func Chunk[T Model, R any](ctx context.Context, perLimit int, fn func(rows T) (R } return } + +// Pagination 同 SimplePagination +// +// Condition 中可使用 Where Fields Group Having Join Order Page Limit In 函数 +func Pagination[T Model](ctx context.Context, q *QueryCondition) ([]T, int, error) { + return SimplePagination[T](ctx, q.where, q.fields, q.group, q.page, q.limit, q.order, q.join, q.having, q.in...) +} diff --git a/model/querycondition_test.go b/model/querycondition_test.go index 2cabfb0..9829844 100644 --- a/model/querycondition_test.go +++ b/model/querycondition_test.go @@ -2,6 +2,7 @@ package model import ( "context" + "database/sql" "github.com/fthvgb1/wp-go/helper/number" "github.com/fthvgb1/wp-go/helper/slice" "reflect" @@ -173,3 +174,59 @@ func TestChunk(t *testing.T) { }) } } + +func TestPagination(t *testing.T) { + type args struct { + ctx context.Context + q *QueryCondition + } + type testCase[T Model] struct { + name string + args args + want []T + want1 int + wantErr bool + } + tests := []testCase[post]{ + { + name: "t1", + args: args{ + ctx: ctx, + q: Conditions( + Where(SqlBuilder{ + {"ID", "in", ""}, + }), + Page(1), + Limit(5), + In([][]any{slice.ToAnySlice(number.Range(431, 440, 1))}...), + ), + }, + want: func() (r []post) { + r, err := Select[post](ctx, "select * from "+post{}.Table()+" where ID in (?,?,?,?,?)", slice.ToAnySlice(number.Range(431, 435, 1))...) + if err != nil && err != sql.ErrNoRows { + panic(err) + } else if err == sql.ErrNoRows { + err = nil + } + return + }(), + want1: 10, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1, err := Pagination[post](tt.args.ctx, tt.args.q) + if (err != nil) != tt.wantErr { + t.Errorf("Pagination() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Pagination() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("Pagination() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} From 490abf0bf675bf0ce61c868f9bbd3d83540bd031 Mon Sep 17 00:00:00 2001 From: xing Date: Thu, 2 Feb 2023 19:16:18 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E6=8A=BD=E8=B1=A1=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cache/interface.go | 16 +++ cache/map.go | 189 +++++++++++------------------------- cache/memorymapcache.go | 82 ++++++++++++++++ internal/actions/comment.go | 2 +- internal/actions/detail.go | 2 +- internal/actions/feed.go | 4 +- internal/pkg/cache/cache.go | 79 ++++++++------- internal/plugins/digest.go | 9 +- 8 files changed, 209 insertions(+), 174 deletions(-) create mode 100644 cache/interface.go create mode 100644 cache/memorymapcache.go diff --git a/cache/interface.go b/cache/interface.go new file mode 100644 index 0000000..e7fe619 --- /dev/null +++ b/cache/interface.go @@ -0,0 +1,16 @@ +package cache + +import ( + "context" + "time" +) + +type Cache[K comparable, V any] interface { + Get(ctx context.Context, key K) (V, bool) + Set(ctx context.Context, key K, val V, expire time.Duration) + Ttl(ctx context.Context, key K, expire time.Duration) time.Duration + Ver(ctx context.Context, key K) int + Flush(ctx context.Context) + Delete(ctx context.Context, key K) + ClearExpired(ctx context.Context, expire time.Duration) +} diff --git a/cache/map.go b/cache/map.go index 50e661a..b374f7c 100644 --- a/cache/map.go +++ b/cache/map.go @@ -4,42 +4,32 @@ import ( "context" "errors" "fmt" - "github.com/fthvgb1/wp-go/safety" + "github.com/fthvgb1/wp-go/helper/slice" "sync" "time" ) type MapCache[K comparable, V any] struct { - data safety.Map[K, mapCacheStruct[V]] - mutex *sync.Mutex + data Cache[K, V] + mux sync.Mutex cacheFunc func(...any) (V, error) batchCacheFn func(...any) (map[K]V, error) expireTime time.Duration } -func NewMapCache[K comparable, V any](expireTime time.Duration) *MapCache[K, V] { - return &MapCache[K, V]{expireTime: expireTime} -} - -type mapCacheStruct[T any] struct { - setTime time.Time - incr int - data T -} - func (m *MapCache[K, V]) SetCacheFunc(fn func(...any) (V, error)) { m.cacheFunc = fn } -func (m *MapCache[K, V]) GetLastSetTime(k K) (t time.Time) { - r, ok := m.data.Load(k) - if ok { - t = r.setTime +func (m *MapCache[K, V]) GetLastSetTime(ctx context.Context, k K) (t time.Time) { + tt := m.data.Ttl(ctx, k, m.expireTime) + if tt <= 0 { + return } - return + return time.Now().Add(m.data.Ttl(ctx, k, m.expireTime)) } -func (m *MapCache[K, V]) SetCacheBatchFunc(fn func(...any) (map[K]V, error)) { +func (m *MapCache[K, V]) SetCacheBatchFn(fn func(...any) (map[K]V, error)) { m.batchCacheFn = fn if m.cacheFunc == nil { m.setCacheFn(fn) @@ -68,102 +58,56 @@ func (m *MapCache[K, V]) setCacheFn(fn func(...any) (map[K]V, error)) { } } -func NewMapCacheByFn[K comparable, V any](fn func(...any) (V, error), expireTime time.Duration) *MapCache[K, V] { +func NewMemoryMapCacheByFn[K comparable, V any](fn func(...any) (V, error), expireTime time.Duration) *MapCache[K, V] { return &MapCache[K, V]{ - mutex: &sync.Mutex{}, + data: NewMemoryMapCache[K, V](), cacheFunc: fn, expireTime: expireTime, - data: safety.NewMap[K, mapCacheStruct[V]](), + mux: sync.Mutex{}, } } -func NewMapCacheByBatchFn[K comparable, V any](fn func(...any) (map[K]V, error), expireTime time.Duration) *MapCache[K, V] { +func NewMemoryMapCacheByBatchFn[K comparable, V any](fn func(...any) (map[K]V, error), expireTime time.Duration) *MapCache[K, V] { r := &MapCache[K, V]{ - mutex: &sync.Mutex{}, + data: NewMemoryMapCache[K, V](), batchCacheFn: fn, expireTime: expireTime, - data: safety.NewMap[K, mapCacheStruct[V]](), + mux: sync.Mutex{}, } r.setCacheFn(fn) return r } -func (m *MapCache[K, V]) Flush() { - m.mutex.Lock() - defer m.mutex.Unlock() - m.data = safety.NewMap[K, mapCacheStruct[V]]() +func (m *MapCache[K, V]) Flush(ctx context.Context) { + m.mux.Lock() + defer m.mux.Unlock() + m.data.Flush(ctx) } -func (m *MapCache[K, V]) Get(k K) (V, bool) { - r, ok := m.data.Load(k) - if ok { - return r.data, ok - } - var rr V - return rr, false +func (m *MapCache[K, V]) Get(ctx context.Context, k K) (V, bool) { + return m.data.Get(ctx, k) } -func (m *MapCache[K, V]) Set(k K, v V) { - m.set(k, v) -} - -func (m *MapCache[K, V]) SetByBatchFn(params ...any) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - r, err := m.batchCacheFn(params...) - if err != nil { - return err - } - for k, v := range r { - m.set(k, v) - } - return nil -} - -func (m *MapCache[K, V]) set(k K, v V) { - data, ok := m.data.Load(k) - t := time.Now() - if !ok { - data.data = v - data.setTime = t - data.incr++ - m.data.Store(k, data) - } else { - m.data.Store(k, mapCacheStruct[V]{ - data: v, - setTime: t, - }) - - } +func (m *MapCache[K, V]) Set(ctx context.Context, k K, v V) { + m.data.Set(ctx, k, v, m.expireTime) } func (m *MapCache[K, V]) GetCache(c context.Context, key K, timeout time.Duration, params ...any) (V, error) { - data, ok := m.data.Load(key) - if !ok { - data = mapCacheStruct[V]{} - } - now := time.Duration(time.Now().UnixNano()) + data, ok := m.data.Get(c, key) var err error - expired := time.Duration(data.setTime.UnixNano())+m.expireTime < now - //todo 这里应该判断下取出的值是否为零值，不过怎么操作呢？ - if !ok || (ok && m.expireTime >= 0 && expired) { - t := data.incr + if !ok || m.data.Ttl(c, key, m.expireTime) <= 0 { + ver := m.data.Ver(c, key) call := func() { - m.mutex.Lock() - defer m.mutex.Unlock() - da, ok := m.data.Load(key) - if ok && da.incr > t { + m.mux.Lock() + defer m.mux.Unlock() + if m.data.Ver(c, key) > ver { + data, _ = m.data.Get(c, key) return - } else { - da = data } - r, er := m.cacheFunc(params...) + data, err = m.cacheFunc(params...) if err != nil { - err = er return } - m.set(key, r) - data.data = r + m.Set(c, key, data) } if timeout > 0 { ctx, cancel := context.WithTimeout(c, timeout) @@ -183,48 +127,42 @@ func (m *MapCache[K, V]) GetCache(c context.Context, key K, timeout time.Duratio } } - return data.data, err + return data, err } func (m *MapCache[K, V]) GetCacheBatch(c context.Context, key []K, timeout time.Duration, params ...any) ([]V, error) { - var needFlush []K var res []V - t := 0 - now := time.Duration(time.Now().UnixNano()) - for _, k := range key { - d, ok := m.data.Load(k) - if !ok { - needFlush = append(needFlush, k) - continue + ver := 0 + needFlush := slice.FilterAndMap(key, func(k K) (r K, ok bool) { + if _, ok := m.data.Get(c, k); !ok { + return k, true } - expired := time.Duration(d.setTime.UnixNano())+m.expireTime < now - if expired { - needFlush = append(needFlush, k) - } - t = t + d.incr - } + ver += m.data.Ver(c, k) + return + }) + var err error - //todo 这里应该判断下取出的值是否为零值，不过怎么操作呢？ if len(needFlush) > 0 { call := func() { - m.mutex.Lock() - defer m.mutex.Unlock() - tt := 0 - for _, dd := range needFlush { - if ddd, ok := m.data.Load(dd); ok { - tt = tt + ddd.incr - } - } - if tt > t { + m.mux.Lock() + defer m.mux.Unlock() + + vers := slice.Reduce(needFlush, func(t K, r int) int { + r += m.data.Ver(c, t) + return r + }, 0) + + if vers > ver { return } + r, er := m.batchCacheFn(params...) if err != nil { err = er return } for k, v := range r { - m.set(k, v) + m.Set(c, k, v) } } if timeout > 0 { @@ -244,23 +182,14 @@ func (m *MapCache[K, V]) GetCacheBatch(c context.Context, key []K, timeout time. call() } } - for _, k := range key { - d, ok := m.data.Load(k) - if ok { - res = append(res, d.data) - } - } + res = slice.FilterAndMap(key, func(k K) (V, bool) { + return m.data.Get(c, k) + }) return res, err } -func (m *MapCache[K, V]) ClearExpired() { - now := time.Duration(time.Now().UnixNano()) - m.mutex.Lock() - defer m.mutex.Unlock() - m.data.Range(func(k K, v mapCacheStruct[V]) bool { - if now > time.Duration(v.setTime.UnixNano())+m.expireTime { - m.data.Delete(k) - } - return true - }) +func (m *MapCache[K, V]) ClearExpired(ctx context.Context) { + m.mux.Lock() + defer m.mux.Unlock() + m.data.ClearExpired(ctx, m.expireTime) } diff --git a/cache/memorymapcache.go b/cache/memorymapcache.go new file mode 100644 index 0000000..624813f --- /dev/null +++ b/cache/memorymapcache.go @@ -0,0 +1,82 @@ +package cache + +import ( + "context" + "github.com/fthvgb1/wp-go/helper/number" + "github.com/fthvgb1/wp-go/safety" + "time" +) + +type MemoryMapCache[K comparable, V any] struct { + safety.Map[K, mapVal[V]] +} + +func NewMemoryMapCache[K comparable, V any]() *MemoryMapCache[K, V] { + return &MemoryMapCache[K, V]{Map: safety.NewMap[K, mapVal[V]]()} +} + +type mapVal[T any] struct { + setTime time.Time + ver int + data T +} + +func (m *MemoryMapCache[K, V]) Get(_ context.Context, key K) (r V, ok bool) { + v, ok := m.Load(key) + if ok { + return v.data, true + } + return +} + +func (m *MemoryMapCache[K, V]) Set(_ context.Context, key K, val V, _ time.Duration) { + v, ok := m.Load(key) + t := time.Now() + if ok { + v.data = val + v.setTime = t + v.ver++ + } else { + v = mapVal[V]{ + setTime: t, + ver: 1, + data: val, + } + } + m.Store(key, v) +} + +func (m *MemoryMapCache[K, V]) Ttl(_ context.Context, key K, expire time.Duration) time.Duration { + v, ok := m.Load(key) + if !ok { + return -1 + } + return number.Max(time.Duration(0), expire-time.Duration(time.Now().UnixNano()-v.setTime.UnixNano())) +} + +func (m *MemoryMapCache[K, V]) Ver(_ context.Context, key K) int { + v, ok := m.Load(key) + if !ok { + return -1 + } + return v.ver +} + +func (m *MemoryMapCache[K, V]) Flush(context.Context) { + m.Map = safety.NewMap[K, mapVal[V]]() +} + +func (m *MemoryMapCache[K, V]) Delete(_ context.Context, key K) { + m.Map.Delete(key) +} + +func (m *MemoryMapCache[K, V]) ClearExpired(_ context.Context, expire time.Duration) { + now := time.Duration(time.Now().UnixNano()) + + m.Range(func(k K, v mapVal[V]) bool { + if now > time.Duration(v.setTime.UnixNano())+expire { + m.Map.Delete(k) + } + return true + }) +} diff --git a/internal/actions/comment.go b/internal/actions/comment.go index 4b01a16..40a7079 100644 --- a/internal/actions/comment.go +++ b/internal/actions/comment.go @@ -111,7 +111,7 @@ func PostComment(c *gin.Context) { err = er return } - cache.NewCommentCache().Set(up.RawQuery, string(s)) + cache.NewCommentCache().Set(c, up.RawQuery, string(s)) c.Redirect(http.StatusFound, res.Header.Get("Location")) return } diff --git a/internal/actions/detail.go b/internal/actions/detail.go index 7b898ec..d63ad4f 100644 --- a/internal/actions/detail.go +++ b/internal/actions/detail.go @@ -76,7 +76,7 @@ func Detail(c *gin.Context) { if post.PostPassword != "" && pw != post.PostPassword { plugins.PasswdProjectContent(&post) showComment = false - } else if s, ok := cache.NewCommentCache().Get(c.Request.URL.RawQuery); ok && s != "" && (post.PostPassword == "" || post.PostPassword != "" && pw == post.PostPassword) { + } else if s, ok := cache.NewCommentCache().Get(c, c.Request.URL.RawQuery); ok && s != "" && (post.PostPassword == "" || post.PostPassword != "" && pw == post.PostPassword) { c.Writer.WriteHeader(http.StatusOK) c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8") _, err = c.Writer.WriteString(s) diff --git a/internal/actions/feed.go b/internal/actions/feed.go index e2e74fd..1095bdb 100644 --- a/internal/actions/feed.go +++ b/internal/actions/feed.go @@ -50,7 +50,7 @@ func setFeed(s string, c *gin.Context, t time.Time) { func PostFeed(c *gin.Context) { id := c.Param("id") - if !isCacheExpired(c, cache.PostFeedCache().GetLastSetTime(id)) { + if !isCacheExpired(c, cache.PostFeedCache().GetLastSetTime(c, id)) { c.Status(http.StatusNotModified) } else { s, err := cache.PostFeedCache().GetCache(c, id, time.Second, c, id) @@ -60,7 +60,7 @@ func PostFeed(c *gin.Context) { c.Error(err) return } - setFeed(s, c, cache.PostFeedCache().GetLastSetTime(id)) + setFeed(s, c, cache.PostFeedCache().GetLastSetTime(c, id)) } } diff --git a/internal/pkg/cache/cache.go b/internal/pkg/cache/cache.go index 11f6f7f..8c2e992 100644 --- a/internal/pkg/cache/cache.go +++ b/internal/pkg/cache/cache.go @@ -43,6 +43,8 @@ var allUsernameCache *cache.VarCache[map[string]struct{}] var headerImagesCache *cache.MapCache[string, []models.PostThumbnail] +var ctx context.Context + func InitActionsCommonCache() { c := config.Conf.Load() archivesCaches = &Arch{ @@ -50,17 +52,17 @@ func InitActionsCommonCache() { setCacheFunc: dao.Archives, } - searchPostIdsCache = cache.NewMapCacheByFn[string](dao.SearchPostIds, c.SearchPostCacheTime) + searchPostIdsCache = cache.NewMemoryMapCacheByFn[string](dao.SearchPostIds, c.SearchPostCacheTime) - postListIdsCache = cache.NewMapCacheByFn[string](dao.SearchPostIds, c.PostListCacheTime) + postListIdsCache = cache.NewMemoryMapCacheByFn[string](dao.SearchPostIds, c.PostListCacheTime) - monthPostsCache = cache.NewMapCacheByFn[string](dao.MonthPost, c.MonthPostCacheTime) + monthPostsCache = cache.NewMemoryMapCacheByFn[string](dao.MonthPost, c.MonthPostCacheTime) - postContextCache = cache.NewMapCacheByFn[uint64](dao.GetPostContext, c.ContextPostCacheTime) + postContextCache = cache.NewMemoryMapCacheByFn[uint64](dao.GetPostContext, c.ContextPostCacheTime) - postsCache = cache.NewMapCacheByBatchFn(dao.GetPostsByIds, c.PostDataCacheTime) + postsCache = cache.NewMemoryMapCacheByBatchFn(dao.GetPostsByIds, c.PostDataCacheTime) - postMetaCache = cache.NewMapCacheByBatchFn(dao.GetPostMetaByPostIds, c.PostDataCacheTime) + postMetaCache = cache.NewMemoryMapCacheByBatchFn(dao.GetPostMetaByPostIds, c.PostDataCacheTime) categoryAndTagsCaches = cache.NewVarCache(dao.CategoriesAndTags, c.CategoryCacheTime) @@ -68,58 +70,61 @@ func InitActionsCommonCache() { recentCommentsCaches = cache.NewVarCache(dao.RecentComments, c.RecentCommentsCacheTime) - postCommentCaches = cache.NewMapCacheByFn[uint64](dao.PostComments, c.PostCommentsCacheTime) + postCommentCaches = cache.NewMemoryMapCacheByFn[uint64](dao.PostComments, c.PostCommentsCacheTime) maxPostIdCache = cache.NewVarCache(dao.GetMaxPostId, c.MaxPostIdCacheTime) - usersCache = cache.NewMapCacheByFn[uint64](dao.GetUserById, c.UserInfoCacheTime) + usersCache = cache.NewMemoryMapCacheByFn[uint64](dao.GetUserById, c.UserInfoCacheTime) - usersNameCache = cache.NewMapCacheByFn[string](dao.GetUserByName, c.UserInfoCacheTime) + usersNameCache = cache.NewMemoryMapCacheByFn[string](dao.GetUserByName, c.UserInfoCacheTime) - commentsCache = cache.NewMapCacheByBatchFn(dao.GetCommentByIds, c.CommentsCacheTime) + commentsCache = cache.NewMemoryMapCacheByBatchFn(dao.GetCommentByIds, c.CommentsCacheTime) feedCache = cache.NewVarCache(feed, time.Hour) - postFeedCache = cache.NewMapCacheByFn[string](postFeed, time.Hour) + postFeedCache = cache.NewMemoryMapCacheByFn[string](postFeed, time.Hour) commentsFeedCache = cache.NewVarCache(commentsFeed, time.Hour) - newCommentCache = cache.NewMapCacheByFn[string, string](nil, 15*time.Minute) + newCommentCache = cache.NewMemoryMapCacheByFn[string, string](nil, 15*time.Minute) allUsernameCache = cache.NewVarCache(dao.AllUsername, c.UserInfoCacheTime) - headerImagesCache = cache.NewMapCacheByFn[string](getHeaderImages, c.ThemeHeaderImagCacheTime) + headerImagesCache = cache.NewMemoryMapCacheByFn[string](getHeaderImages, c.ThemeHeaderImagCacheTime) + + ctx = context.Background() InitFeed() } func ClearCache() { - searchPostIdsCache.ClearExpired() - postsCache.ClearExpired() - postMetaCache.ClearExpired() - postListIdsCache.ClearExpired() - monthPostsCache.ClearExpired() - postContextCache.ClearExpired() - usersCache.ClearExpired() - commentsCache.ClearExpired() - usersNameCache.ClearExpired() - postFeedCache.ClearExpired() - newCommentCache.ClearExpired() - headerImagesCache.ClearExpired() + searchPostIdsCache.ClearExpired(ctx) + searchPostIdsCache.ClearExpired(ctx) + postsCache.ClearExpired(ctx) + postMetaCache.ClearExpired(ctx) + postListIdsCache.ClearExpired(ctx) + monthPostsCache.ClearExpired(ctx) + postContextCache.ClearExpired(ctx) + usersCache.ClearExpired(ctx) + commentsCache.ClearExpired(ctx) + usersNameCache.ClearExpired(ctx) + postFeedCache.ClearExpired(ctx) + newCommentCache.ClearExpired(ctx) + headerImagesCache.ClearExpired(ctx) } func FlushCache() { - searchPostIdsCache.Flush() - postsCache.Flush() - postMetaCache.Flush() - postListIdsCache.Flush() - monthPostsCache.Flush() - postContextCache.Flush() - usersCache.Flush() - commentsCache.Flush() - usersCache.Flush() - postFeedCache.Flush() - newCommentCache.Flush() - headerImagesCache.Flush() + searchPostIdsCache.Flush(ctx) + postsCache.Flush(ctx) + postMetaCache.Flush(ctx) + postListIdsCache.Flush(ctx) + monthPostsCache.Flush(ctx) + postContextCache.Flush(ctx) + usersCache.Flush(ctx) + commentsCache.Flush(ctx) + usersCache.Flush(ctx) + postFeedCache.Flush(ctx) + newCommentCache.Flush(ctx) + headerImagesCache.Flush(ctx) } func Archives(ctx context.Context) (r []models.PostArchive) { diff --git a/internal/plugins/digest.go b/internal/plugins/digest.go index ced314e..90311ce 100644 --- a/internal/plugins/digest.go +++ b/internal/plugins/digest.go @@ -1,6 +1,7 @@ package plugins import ( + "context" "fmt" "github.com/fthvgb1/wp-go/cache" "github.com/fthvgb1/wp-go/internal/pkg/config" @@ -12,16 +13,18 @@ import ( ) var digestCache *cache.MapCache[uint64, string] +var ctx context.Context func InitDigestCache() { - digestCache = cache.NewMapCacheByFn[uint64](digestRaw, config.Conf.Load().DigestCacheTime) + ctx = context.Background() + digestCache = cache.NewMemoryMapCacheByFn[uint64](digestRaw, config.Conf.Load().DigestCacheTime) } func ClearDigestCache() { - digestCache.ClearExpired() + digestCache.ClearExpired(ctx) } func FlushCache() { - digestCache.Flush() + digestCache.Flush(ctx) } func digestRaw(arg ...any) (string, error) { From 8ecf8bf94e632d8cdbd53ff47f801d143ba8f960 Mon Sep 17 00:00:00 2001 From: xing Date: Thu, 2 Feb 2023 22:56:09 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cache/map.go | 18 +- cache/map_test.go | 370 +++++++++++++++++++++++++++++++++++ cache/memorymapcache.go | 25 ++- cache/memorymapcache_test.go | 246 +++++++++++++++++++++++ 4 files changed, 649 insertions(+), 10 deletions(-) create mode 100644 cache/map_test.go create mode 100644 cache/memorymapcache_test.go diff --git a/cache/map.go b/cache/map.go index b374f7c..e4e2dbb 100644 --- a/cache/map.go +++ b/cache/map.go @@ -21,12 +21,16 @@ func (m *MapCache[K, V]) SetCacheFunc(fn func(...any) (V, error)) { m.cacheFunc = fn } +func (m *MapCache[K, V]) Ttl(ctx context.Context, k K) time.Duration { + return m.data.Ttl(ctx, k, m.expireTime) +} + func (m *MapCache[K, V]) GetLastSetTime(ctx context.Context, k K) (t time.Time) { tt := m.data.Ttl(ctx, k, m.expireTime) if tt <= 0 { return } - return time.Now().Add(m.data.Ttl(ctx, k, m.expireTime)) + return time.Now().Add(m.data.Ttl(ctx, k, m.expireTime)).Add(-m.expireTime) } func (m *MapCache[K, V]) SetCacheBatchFn(fn func(...any) (map[K]V, error)) { @@ -58,20 +62,20 @@ func (m *MapCache[K, V]) setCacheFn(fn func(...any) (map[K]V, error)) { } } -func NewMemoryMapCacheByFn[K comparable, V any](fn func(...any) (V, error), expireTime time.Duration) *MapCache[K, V] { +func NewMapCacheByFn[K comparable, V any](cacheType Cache[K, V], fn func(...any) (V, error), expireTime time.Duration) *MapCache[K, V] { return &MapCache[K, V]{ - data: NewMemoryMapCache[K, V](), + mux: sync.Mutex{}, cacheFunc: fn, expireTime: expireTime, - mux: sync.Mutex{}, + data: cacheType, } } -func NewMemoryMapCacheByBatchFn[K comparable, V any](fn func(...any) (map[K]V, error), expireTime time.Duration) *MapCache[K, V] { +func NewMapCacheByBatchFn[K comparable, V any](cacheType Cache[K, V], fn func(...any) (map[K]V, error), expireTime time.Duration) *MapCache[K, V] { r := &MapCache[K, V]{ - data: NewMemoryMapCache[K, V](), + mux: sync.Mutex{}, batchCacheFn: fn, expireTime: expireTime, - mux: sync.Mutex{}, + data: cacheType, } r.setCacheFn(fn) return r diff --git a/cache/map_test.go b/cache/map_test.go new file mode 100644 index 0000000..feed65e --- /dev/null +++ b/cache/map_test.go @@ -0,0 +1,370 @@ +package cache + +import ( + "context" + "fmt" + "github.com/fthvgb1/wp-go/helper/slice" + "reflect" + "strings" + "testing" + "time" +) + +var ca MapCache[string, string] +var fn func(a ...any) (string, error) +var batchFn func(a ...any) (map[string]string, error) +var ct context.Context + +func init() { + fn = func(a ...any) (string, error) { + aa := a[1].(string) + return strings.Repeat(aa, 2), nil + } + ct = context.Background() + batchFn = func(a ...any) (map[string]string, error) { + arr := a[1].([]string) + return slice.SimpleToMap(arr, func(t string) string { + return strings.Repeat(t, 2) + }), nil + } + ca = *NewMemoryMapCacheByFn[string, string](fn, time.Second*2) + ca.SetCacheBatchFn(batchFn) + _, _ = ca.GetCache(ct, "aa", time.Second, ct, "aa") + _, _ = ca.GetCache(ct, "bb", time.Second, ct, "bb") +} +func TestMapCache_ClearExpired(t *testing.T) { + type args struct { + ct context.Context + } + type testCase[K comparable, V any] struct { + name string + m MapCache[K, V] + args args + } + tests := []testCase[string, string]{ + { + name: "t1", + m: ca, + args: args{ + ct: ct, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fmt.Println(ca.Get(ct, "aa")) + fmt.Println(ca.Get(ct, "bb")) + time.Sleep(time.Second * 3) + tt.m.ClearExpired(tt.args.ct) + fmt.Println(ca.Get(ct, "bb")) + }) + } +} + +func TestMapCache_Flush(t *testing.T) { + type args struct { + ct context.Context + } + type testCase[K comparable, V any] struct { + name string + m MapCache[K, V] + args args + } + ca := *NewMemoryMapCacheByFn[string, string](fn, time.Second) + _, _ = ca.GetCache(ct, "aa", time.Second, ct, "aa") + tests := []testCase[string, string]{ + { + name: "t1", + m: ca, + args: args{ + ct, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fmt.Println(ca.Get(ct, "aa")) + tt.m.Flush(tt.args.ct) + fmt.Println(ca.Get(ct, "aa")) + }) + } +} + +func TestMapCache_Get(t *testing.T) { + type args[K comparable] struct { + ct context.Context + k K + } + type testCase[K comparable, V any] struct { + name string + m MapCache[K, V] + args args[K] + want V + want1 bool + } + tests := []testCase[string, string]{ + { + name: "t1", + m: ca, + args: args[string]{ct, "aa"}, + want: "aaaa", + want1: true, + }, + { + name: "t2", + m: ca, + args: args[string]{ct, "cc"}, + want: "", + want1: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := tt.m.Get(tt.args.ct, tt.args.k) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Get() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("Get() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func TestMapCache_GetCache(t *testing.T) { + type args[K comparable] struct { + c context.Context + key K + timeout time.Duration + params []any + } + type testCase[K comparable, V any] struct { + name string + m MapCache[K, V] + args args[K] + want V + wantErr bool + } + tests := []testCase[string, string]{ + { + name: "t1", + m: ca, + args: args[string]{c: ct, key: "xx", timeout: time.Second, params: []any{ct, "xx"}}, + want: "xxxx", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.m.GetCache(tt.args.c, tt.args.key, tt.args.timeout, tt.args.params...) + if (err != nil) != tt.wantErr { + t.Errorf("GetCache() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetCache() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMapCache_GetCacheBatch(t *testing.T) { + type args[K comparable] struct { + c context.Context + key []K + timeout time.Duration + params []any + } + type testCase[K comparable, V any] struct { + name string + m MapCache[K, V] + args args[K] + want []V + wantErr bool + } + tests := []testCase[string, string]{ + { + name: "t1", + m: ca, + args: args[string]{ + c: ct, + key: []string{"xx", "oo"}, + timeout: time.Second, + params: []any{ct, []string{"xx", "oo"}}, + }, + want: []string{"xxxx", "oooo"}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.m.GetCacheBatch(tt.args.c, tt.args.key, tt.args.timeout, tt.args.params...) + if (err != nil) != tt.wantErr { + t.Errorf("GetCacheBatch() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetCacheBatch() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMapCache_GetLastSetTime(t *testing.T) { + type args[K comparable] struct { + ct context.Context + k K + } + type testCase[K comparable, V any] struct { + name string + m MapCache[K, V] + args args[K] + wantT time.Time + } + tests := []testCase[string, string]{ + { + name: "t1", + m: ca, + args: args[string]{ct, "aa"}, + wantT: ttt, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotT := tt.m.GetLastSetTime(tt.args.ct, tt.args.k); !reflect.DeepEqual(gotT, tt.wantT) { + t.Errorf("GetLastSetTime() = %v, want %v", gotT, tt.wantT) + } + }) + } +} + +func TestMapCache_Set(t *testing.T) { + type args[K comparable, V any] struct { + ct context.Context + k K + v V + } + type testCase[K comparable, V any] struct { + name string + m MapCache[K, V] + args args[K, V] + } + tests := []testCase[string, string]{ + { + name: "t1", + m: ca, + args: args[string, string]{ + ct, "xx", "yy", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fmt.Println(ca.Get(ct, "xx")) + tt.m.Set(tt.args.ct, tt.args.k, tt.args.v) + fmt.Println(ca.Get(ct, "xx")) + }) + } +} + +func TestMapCache_SetCacheBatchFn(t *testing.T) { + type args[K comparable, V any] struct { + fn func(...any) (map[K]V, error) + } + type testCase[K comparable, V any] struct { + name string + m MapCache[K, V] + args args[K, V] + } + tests := []testCase[string, string]{ + { + name: "t1", + m: ca, + args: args[string, string]{batchFn}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.m.SetCacheBatchFn(tt.args.fn) + }) + } +} + +func TestMapCache_SetCacheFunc(t *testing.T) { + type args[V any] struct { + fn func(...any) (V, error) + } + type testCase[K comparable, V any] struct { + name string + m MapCache[K, V] + args args[V] + } + tests := []testCase[string, string]{ + { + name: "t1", + m: ca, + args: args[string]{fn: fn}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.m.SetCacheFunc(tt.args.fn) + }) + } +} + +func TestMapCache_Ttl(t *testing.T) { + type args[K comparable] struct { + ct context.Context + k K + } + type testCase[K comparable, V any] struct { + name string + m MapCache[K, V] + args args[K] + want time.Duration + } + tx := time.Now() + txx := ca.GetLastSetTime(ct, "aa") + tests := []testCase[string, string]{ + { + name: "t1", + m: ca, + args: args[string]{ct, "aa"}, + want: ca.expireTime - tx.Sub(txx), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fmt.Printf("过期时间=%v \nttl=%v \n当前时间 =%v\n最后设置时间=%v\n当时时间-最后设置时间=%v ", ca.expireTime, ca.Ttl(ct, "aa"), tx, txx, tx.Sub(txx)) + if got := tt.m.Ttl(tt.args.ct, tt.args.k); got != tt.want { + t.Errorf("Ttl() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMapCache_setCacheFn(t *testing.T) { + type args[K comparable, V any] struct { + fn func(...any) (map[K]V, error) + } + type testCase[K comparable, V any] struct { + name string + m MapCache[K, V] + args args[K, V] + } + tests := []testCase[string, string]{ + { + name: "t1", + m: ca, + args: args[string, string]{batchFn}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ca.cacheFunc = nil + tt.m.setCacheFn(tt.args.fn) + fmt.Println(ca.GetCache(ct, "xx", time.Second, ct, "xx")) + }) + } +} diff --git a/cache/memorymapcache.go b/cache/memorymapcache.go index 624813f..a2de458 100644 --- a/cache/memorymapcache.go +++ b/cache/memorymapcache.go @@ -2,8 +2,8 @@ package cache import ( "context" - "github.com/fthvgb1/wp-go/helper/number" "github.com/fthvgb1/wp-go/safety" + "sync" "time" ) @@ -11,6 +11,25 @@ type MemoryMapCache[K comparable, V any] struct { safety.Map[K, mapVal[V]] } +func NewMemoryMapCacheByFn[K comparable, V any](fn func(...any) (V, error), expireTime time.Duration) *MapCache[K, V] { + return &MapCache[K, V]{ + data: NewMemoryMapCache[K, V](), + cacheFunc: fn, + expireTime: expireTime, + mux: sync.Mutex{}, + } +} +func NewMemoryMapCacheByBatchFn[K comparable, V any](fn func(...any) (map[K]V, error), expireTime time.Duration) *MapCache[K, V] { + r := &MapCache[K, V]{ + data: NewMemoryMapCache[K, V](), + batchCacheFn: fn, + expireTime: expireTime, + mux: sync.Mutex{}, + } + r.setCacheFn(fn) + return r +} + func NewMemoryMapCache[K comparable, V any]() *MemoryMapCache[K, V] { return &MemoryMapCache[K, V]{Map: safety.NewMap[K, mapVal[V]]()} } @@ -49,9 +68,9 @@ func (m *MemoryMapCache[K, V]) Set(_ context.Context, key K, val V, _ time.Durat func (m *MemoryMapCache[K, V]) Ttl(_ context.Context, key K, expire time.Duration) time.Duration { v, ok := m.Load(key) if !ok { - return -1 + return time.Duration(-1) } - return number.Max(time.Duration(0), expire-time.Duration(time.Now().UnixNano()-v.setTime.UnixNano())) + return expire - time.Now().Sub(v.setTime) } func (m *MemoryMapCache[K, V]) Ver(_ context.Context, key K) int { diff --git a/cache/memorymapcache_test.go b/cache/memorymapcache_test.go new file mode 100644 index 0000000..9eb5b12 --- /dev/null +++ b/cache/memorymapcache_test.go @@ -0,0 +1,246 @@ +package cache + +import ( + "context" + "fmt" + "reflect" + "testing" + "time" +) + +var mm MemoryMapCache[string, string] +var ctx context.Context + +var ttt time.Time + +func init() { + ctx = context.Background() + mm = *NewMemoryMapCache[string, string]() + ttt = time.Now() + mm.Store("aa", mapVal[string]{ + setTime: ttt, + ver: 1, + data: "bb", + }) + time.Sleep(60 * time.Millisecond) + mm.Store("cc", mapVal[string]{ + setTime: time.Now(), + ver: 1, + data: "dd", + }) +} + +func TestMemoryMapCache_ClearExpired(t *testing.T) { + type args struct { + in0 context.Context + expire time.Duration + } + type testCase[K string, V string] struct { + name string + m MemoryMapCache[K, V] + args args + } + tests := []testCase[string, string]{ + { + name: "t1", + m: mm, + args: args{ + in0: ctx, + expire: time.Second, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fmt.Println(tt.m) + tt.m.ClearExpired(tt.args.in0, tt.args.expire) + time.Sleep(time.Second) + fmt.Println(tt.m) + }) + } +} + +func TestMemoryMapCache_Delete(t *testing.T) { + type args[K comparable] struct { + in0 context.Context + key K + } + type testCase[K comparable, V any] struct { + name string + m MemoryMapCache[K, V] + args args[K] + } + tests := []testCase[string, string]{ + { + name: "t1", + m: mm, + args: args[string]{ + in0: ctx, + key: "aa", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fmt.Println(mm.Get(ctx, "aa")) + tt.m.Delete(tt.args.in0, tt.args.key) + fmt.Println(mm.Get(ctx, "aa")) + + }) + } +} + +func TestMemoryMapCache_Flush(t *testing.T) { + type args struct { + in0 context.Context + } + type testCase[K comparable, V any] struct { + name string + m MemoryMapCache[K, V] + args args + } + tests := []testCase[string, string]{ + { + name: "t1", + m: mm, + args: args{ + in0: ctx, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.m.Flush(tt.args.in0) + mm.Set(ctx, "aa", "xx", time.Second) + fmt.Println(mm.Get(ctx, "aa")) + }) + } +} + +func TestMemoryMapCache_Get(t *testing.T) { + type args[K comparable] struct { + in0 context.Context + key K + } + type testCase[K comparable, V any] struct { + name string + m MemoryMapCache[K, V] + args args[K] + wantR V + wantOk bool + } + tests := []testCase[string, string]{ + { + name: "t1", + m: mm, + args: args[string]{ + in0: ctx, + key: "aa", + }, + wantOk: true, + wantR: "bb", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotR, gotOk := tt.m.Get(tt.args.in0, tt.args.key) + if !reflect.DeepEqual(gotR, tt.wantR) { + t.Errorf("Get() gotR = %v, want %v", gotR, tt.wantR) + } + if gotOk != tt.wantOk { + t.Errorf("Get() gotOk = %v, want %v", gotOk, tt.wantOk) + } + }) + } +} + +func TestMemoryMapCache_Set(t *testing.T) { + type args[K comparable, V any] struct { + in0 context.Context + key K + val V + in3 time.Duration + } + type testCase[K comparable, V any] struct { + name string + m MemoryMapCache[K, V] + args args[K, V] + } + tests := []testCase[string, string]{ + { + name: "t1", + m: mm, + args: args[string, string]{ + in0: ctx, + key: "ee", + val: "ff", + in3: time.Second, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.m.Set(tt.args.in0, tt.args.key, tt.args.val, tt.args.in3) + fmt.Println(tt.m.Get(ctx, tt.args.key)) + }) + } +} + +func TestMemoryMapCache_Ttl(t *testing.T) { + type args[K comparable] struct { + in0 context.Context + key K + expire time.Duration + } + type testCase[K comparable, V any] struct { + name string + m MemoryMapCache[K, V] + args args[K] + want time.Duration + } + tt := time.Now() + tests := []testCase[string, string]{ + { + name: "t1", + m: mm, + args: args[string]{key: "aa", in0: ctx, expire: time.Second * 4}, + want: 4*time.Second - tt.Sub(ttt), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.m.Ttl(tt.args.in0, tt.args.key, tt.args.expire); got != tt.want { + t.Errorf("Ttl() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMemoryMapCache_Ver(t *testing.T) { + type args[K comparable] struct { + in0 context.Context + key K + } + type testCase[K comparable, V any] struct { + name string + m MemoryMapCache[K, V] + args args[K] + want int + } + mm.Set(ctx, "aa", "ff", time.Second) + tests := []testCase[string, string]{ + { + name: "t1", + m: mm, + args: args[string]{ctx, "aa"}, + want: 2, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.m.Ver(tt.args.in0, tt.args.key); got != tt.want { + t.Errorf("Ver() = %v, want %v", got, tt.want) + } + }) + } +} From f831499341ecb8d9a7468afd69d350a3da1a4688 Mon Sep 17 00:00:00 2001 From: xing Date: Fri, 3 Feb 2023 14:27:32 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/actions/detail.go | 1 - internal/actions/index.go | 1 - 2 files changed, 2 deletions(-) diff --git a/internal/actions/detail.go b/internal/actions/detail.go index d63ad4f..d06b68f 100644 --- a/internal/actions/detail.go +++ b/internal/actions/detail.go @@ -25,7 +25,6 @@ func Detail(c *gin.Context) { recentComments := cache.RecentComments(c, 5) var ginH = gin.H{ "title": wpconfig.Options.Value("blogname"), - "options": wpconfig.Options, "recentPosts": recent, "archives": archive, "categories": categoryItems, diff --git a/internal/actions/index.go b/internal/actions/index.go index a05ea4d..ec19906 100644 --- a/internal/actions/index.go +++ b/internal/actions/index.go @@ -241,7 +241,6 @@ func Index(c *gin.Context) { recentComments := cache.RecentComments(c, 5) ginH := gin.H{ "err": err, - "options": wpconfig.Options, "recentPosts": recent, "archives": archive, "categories": categoryItems,