From 490abf0bf675bf0ce61c868f9bbd3d83540bd031 Mon Sep 17 00:00:00 2001 From: xing Date: Thu, 2 Feb 2023 19:16:18 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8A=BD=E8=B1=A1=E7=BC=93=E5=AD=98=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cache/interface.go | 16 +++ cache/map.go | 189 +++++++++++------------------------- cache/memorymapcache.go | 82 ++++++++++++++++ internal/actions/comment.go | 2 +- internal/actions/detail.go | 2 +- internal/actions/feed.go | 4 +- internal/pkg/cache/cache.go | 79 ++++++++------- internal/plugins/digest.go | 9 +- 8 files changed, 209 insertions(+), 174 deletions(-) create mode 100644 cache/interface.go create mode 100644 cache/memorymapcache.go diff --git a/cache/interface.go b/cache/interface.go new file mode 100644 index 0000000..e7fe619 --- /dev/null +++ b/cache/interface.go @@ -0,0 +1,16 @@ +package cache + +import ( + "context" + "time" +) + +type Cache[K comparable, V any] interface { + Get(ctx context.Context, key K) (V, bool) + Set(ctx context.Context, key K, val V, expire time.Duration) + Ttl(ctx context.Context, key K, expire time.Duration) time.Duration + Ver(ctx context.Context, key K) int + Flush(ctx context.Context) + Delete(ctx context.Context, key K) + ClearExpired(ctx context.Context, expire time.Duration) +} diff --git a/cache/map.go b/cache/map.go index 50e661a..b374f7c 100644 --- a/cache/map.go +++ b/cache/map.go @@ -4,42 +4,32 @@ import ( "context" "errors" "fmt" - "github.com/fthvgb1/wp-go/safety" + "github.com/fthvgb1/wp-go/helper/slice" "sync" "time" ) type MapCache[K comparable, V any] struct { - data safety.Map[K, mapCacheStruct[V]] - mutex *sync.Mutex + data Cache[K, V] + mux sync.Mutex cacheFunc func(...any) (V, error) batchCacheFn func(...any) (map[K]V, error) expireTime time.Duration } -func NewMapCache[K comparable, V any](expireTime time.Duration) *MapCache[K, V] { - return &MapCache[K, V]{expireTime: expireTime} -} - -type mapCacheStruct[T any] struct { - setTime time.Time - incr int - data T -} - func (m *MapCache[K, V]) SetCacheFunc(fn func(...any) (V, error)) { m.cacheFunc = fn } -func (m *MapCache[K, V]) GetLastSetTime(k K) (t time.Time) { - r, ok := m.data.Load(k) - if ok { - t = r.setTime +func (m *MapCache[K, V]) GetLastSetTime(ctx context.Context, k K) (t time.Time) { + tt := m.data.Ttl(ctx, k, m.expireTime) + if tt <= 0 { + return } - return + return time.Now().Add(m.data.Ttl(ctx, k, m.expireTime)) } -func (m *MapCache[K, V]) SetCacheBatchFunc(fn func(...any) (map[K]V, error)) { +func (m *MapCache[K, V]) SetCacheBatchFn(fn func(...any) (map[K]V, error)) { m.batchCacheFn = fn if m.cacheFunc == nil { m.setCacheFn(fn) @@ -68,102 +58,56 @@ func (m *MapCache[K, V]) setCacheFn(fn func(...any) (map[K]V, error)) { } } -func NewMapCacheByFn[K comparable, V any](fn func(...any) (V, error), expireTime time.Duration) *MapCache[K, V] { +func NewMemoryMapCacheByFn[K comparable, V any](fn func(...any) (V, error), expireTime time.Duration) *MapCache[K, V] { return &MapCache[K, V]{ - mutex: &sync.Mutex{}, + data: NewMemoryMapCache[K, V](), cacheFunc: fn, expireTime: expireTime, - data: safety.NewMap[K, mapCacheStruct[V]](), + mux: sync.Mutex{}, } } -func NewMapCacheByBatchFn[K comparable, V any](fn func(...any) (map[K]V, error), expireTime time.Duration) *MapCache[K, V] { +func NewMemoryMapCacheByBatchFn[K comparable, V any](fn func(...any) (map[K]V, error), expireTime time.Duration) *MapCache[K, V] { r := &MapCache[K, V]{ - mutex: &sync.Mutex{}, + data: NewMemoryMapCache[K, V](), batchCacheFn: fn, expireTime: expireTime, - data: safety.NewMap[K, mapCacheStruct[V]](), + mux: sync.Mutex{}, } r.setCacheFn(fn) return r } -func (m *MapCache[K, V]) Flush() { - m.mutex.Lock() - defer m.mutex.Unlock() - m.data = safety.NewMap[K, mapCacheStruct[V]]() +func (m *MapCache[K, V]) Flush(ctx context.Context) { + m.mux.Lock() + defer m.mux.Unlock() + m.data.Flush(ctx) } -func (m *MapCache[K, V]) Get(k K) (V, bool) { - r, ok := m.data.Load(k) - if ok { - return r.data, ok - } - var rr V - return rr, false +func (m *MapCache[K, V]) Get(ctx context.Context, k K) (V, bool) { + return m.data.Get(ctx, k) } -func (m *MapCache[K, V]) Set(k K, v V) { - m.set(k, v) -} - -func (m *MapCache[K, V]) SetByBatchFn(params ...any) error { - m.mutex.Lock() - defer m.mutex.Unlock() - - r, err := m.batchCacheFn(params...) - if err != nil { - return err - } - for k, v := range r { - m.set(k, v) - } - return nil -} - -func (m *MapCache[K, V]) set(k K, v V) { - data, ok := m.data.Load(k) - t := time.Now() - if !ok { - data.data = v - data.setTime = t - data.incr++ - m.data.Store(k, data) - } else { - m.data.Store(k, mapCacheStruct[V]{ - data: v, - setTime: t, - }) - - } +func (m *MapCache[K, V]) Set(ctx context.Context, k K, v V) { + m.data.Set(ctx, k, v, m.expireTime) } func (m *MapCache[K, V]) GetCache(c context.Context, key K, timeout time.Duration, params ...any) (V, error) { - data, ok := m.data.Load(key) - if !ok { - data = mapCacheStruct[V]{} - } - now := time.Duration(time.Now().UnixNano()) + data, ok := m.data.Get(c, key) var err error - expired := time.Duration(data.setTime.UnixNano())+m.expireTime < now - //todo 这里应该判断下取出的值是否为零值，不过怎么操作呢？ - if !ok || (ok && m.expireTime >= 0 && expired) { - t := data.incr + if !ok || m.data.Ttl(c, key, m.expireTime) <= 0 { + ver := m.data.Ver(c, key) call := func() { - m.mutex.Lock() - defer m.mutex.Unlock() - da, ok := m.data.Load(key) - if ok && da.incr > t { + m.mux.Lock() + defer m.mux.Unlock() + if m.data.Ver(c, key) > ver { + data, _ = m.data.Get(c, key) return - } else { - da = data } - r, er := m.cacheFunc(params...) + data, err = m.cacheFunc(params...) if err != nil { - err = er return } - m.set(key, r) - data.data = r + m.Set(c, key, data) } if timeout > 0 { ctx, cancel := context.WithTimeout(c, timeout) @@ -183,48 +127,42 @@ func (m *MapCache[K, V]) GetCache(c context.Context, key K, timeout time.Duratio } } - return data.data, err + return data, err } func (m *MapCache[K, V]) GetCacheBatch(c context.Context, key []K, timeout time.Duration, params ...any) ([]V, error) { - var needFlush []K var res []V - t := 0 - now := time.Duration(time.Now().UnixNano()) - for _, k := range key { - d, ok := m.data.Load(k) - if !ok { - needFlush = append(needFlush, k) - continue + ver := 0 + needFlush := slice.FilterAndMap(key, func(k K) (r K, ok bool) { + if _, ok := m.data.Get(c, k); !ok { + return k, true } - expired := time.Duration(d.setTime.UnixNano())+m.expireTime < now - if expired { - needFlush = append(needFlush, k) - } - t = t + d.incr - } + ver += m.data.Ver(c, k) + return + }) + var err error - //todo 这里应该判断下取出的值是否为零值，不过怎么操作呢？ if len(needFlush) > 0 { call := func() { - m.mutex.Lock() - defer m.mutex.Unlock() - tt := 0 - for _, dd := range needFlush { - if ddd, ok := m.data.Load(dd); ok { - tt = tt + ddd.incr - } - } - if tt > t { + m.mux.Lock() + defer m.mux.Unlock() + + vers := slice.Reduce(needFlush, func(t K, r int) int { + r += m.data.Ver(c, t) + return r + }, 0) + + if vers > ver { return } + r, er := m.batchCacheFn(params...) if err != nil { err = er return } for k, v := range r { - m.set(k, v) + m.Set(c, k, v) } } if timeout > 0 { @@ -244,23 +182,14 @@ func (m *MapCache[K, V]) GetCacheBatch(c context.Context, key []K, timeout time. call() } } - for _, k := range key { - d, ok := m.data.Load(k) - if ok { - res = append(res, d.data) - } - } + res = slice.FilterAndMap(key, func(k K) (V, bool) { + return m.data.Get(c, k) + }) return res, err } -func (m *MapCache[K, V]) ClearExpired() { - now := time.Duration(time.Now().UnixNano()) - m.mutex.Lock() - defer m.mutex.Unlock() - m.data.Range(func(k K, v mapCacheStruct[V]) bool { - if now > time.Duration(v.setTime.UnixNano())+m.expireTime { - m.data.Delete(k) - } - return true - }) +func (m *MapCache[K, V]) ClearExpired(ctx context.Context) { + m.mux.Lock() + defer m.mux.Unlock() + m.data.ClearExpired(ctx, m.expireTime) } diff --git a/cache/memorymapcache.go b/cache/memorymapcache.go new file mode 100644 index 0000000..624813f --- /dev/null +++ b/cache/memorymapcache.go @@ -0,0 +1,82 @@ +package cache + +import ( + "context" + "github.com/fthvgb1/wp-go/helper/number" + "github.com/fthvgb1/wp-go/safety" + "time" +) + +type MemoryMapCache[K comparable, V any] struct { + safety.Map[K, mapVal[V]] +} + +func NewMemoryMapCache[K comparable, V any]() *MemoryMapCache[K, V] { + return &MemoryMapCache[K, V]{Map: safety.NewMap[K, mapVal[V]]()} +} + +type mapVal[T any] struct { + setTime time.Time + ver int + data T +} + +func (m *MemoryMapCache[K, V]) Get(_ context.Context, key K) (r V, ok bool) { + v, ok := m.Load(key) + if ok { + return v.data, true + } + return +} + +func (m *MemoryMapCache[K, V]) Set(_ context.Context, key K, val V, _ time.Duration) { + v, ok := m.Load(key) + t := time.Now() + if ok { + v.data = val + v.setTime = t + v.ver++ + } else { + v = mapVal[V]{ + setTime: t, + ver: 1, + data: val, + } + } + m.Store(key, v) +} + +func (m *MemoryMapCache[K, V]) Ttl(_ context.Context, key K, expire time.Duration) time.Duration { + v, ok := m.Load(key) + if !ok { + return -1 + } + return number.Max(time.Duration(0), expire-time.Duration(time.Now().UnixNano()-v.setTime.UnixNano())) +} + +func (m *MemoryMapCache[K, V]) Ver(_ context.Context, key K) int { + v, ok := m.Load(key) + if !ok { + return -1 + } + return v.ver +} + +func (m *MemoryMapCache[K, V]) Flush(context.Context) { + m.Map = safety.NewMap[K, mapVal[V]]() +} + +func (m *MemoryMapCache[K, V]) Delete(_ context.Context, key K) { + m.Map.Delete(key) +} + +func (m *MemoryMapCache[K, V]) ClearExpired(_ context.Context, expire time.Duration) { + now := time.Duration(time.Now().UnixNano()) + + m.Range(func(k K, v mapVal[V]) bool { + if now > time.Duration(v.setTime.UnixNano())+expire { + m.Map.Delete(k) + } + return true + }) +} diff --git a/internal/actions/comment.go b/internal/actions/comment.go index 4b01a16..40a7079 100644 --- a/internal/actions/comment.go +++ b/internal/actions/comment.go @@ -111,7 +111,7 @@ func PostComment(c *gin.Context) { err = er return } - cache.NewCommentCache().Set(up.RawQuery, string(s)) + cache.NewCommentCache().Set(c, up.RawQuery, string(s)) c.Redirect(http.StatusFound, res.Header.Get("Location")) return } diff --git a/internal/actions/detail.go b/internal/actions/detail.go index 7b898ec..d63ad4f 100644 --- a/internal/actions/detail.go +++ b/internal/actions/detail.go @@ -76,7 +76,7 @@ func Detail(c *gin.Context) { if post.PostPassword != "" && pw != post.PostPassword { plugins.PasswdProjectContent(&post) showComment = false - } else if s, ok := cache.NewCommentCache().Get(c.Request.URL.RawQuery); ok && s != "" && (post.PostPassword == "" || post.PostPassword != "" && pw == post.PostPassword) { + } else if s, ok := cache.NewCommentCache().Get(c, c.Request.URL.RawQuery); ok && s != "" && (post.PostPassword == "" || post.PostPassword != "" && pw == post.PostPassword) { c.Writer.WriteHeader(http.StatusOK) c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8") _, err = c.Writer.WriteString(s) diff --git a/internal/actions/feed.go b/internal/actions/feed.go index e2e74fd..1095bdb 100644 --- a/internal/actions/feed.go +++ b/internal/actions/feed.go @@ -50,7 +50,7 @@ func setFeed(s string, c *gin.Context, t time.Time) { func PostFeed(c *gin.Context) { id := c.Param("id") - if !isCacheExpired(c, cache.PostFeedCache().GetLastSetTime(id)) { + if !isCacheExpired(c, cache.PostFeedCache().GetLastSetTime(c, id)) { c.Status(http.StatusNotModified) } else { s, err := cache.PostFeedCache().GetCache(c, id, time.Second, c, id) @@ -60,7 +60,7 @@ func PostFeed(c *gin.Context) { c.Error(err) return } - setFeed(s, c, cache.PostFeedCache().GetLastSetTime(id)) + setFeed(s, c, cache.PostFeedCache().GetLastSetTime(c, id)) } } diff --git a/internal/pkg/cache/cache.go b/internal/pkg/cache/cache.go index 11f6f7f..8c2e992 100644 --- a/internal/pkg/cache/cache.go +++ b/internal/pkg/cache/cache.go @@ -43,6 +43,8 @@ var allUsernameCache *cache.VarCache[map[string]struct{}] var headerImagesCache *cache.MapCache[string, []models.PostThumbnail] +var ctx context.Context + func InitActionsCommonCache() { c := config.Conf.Load() archivesCaches = &Arch{ @@ -50,17 +52,17 @@ func InitActionsCommonCache() { setCacheFunc: dao.Archives, } - searchPostIdsCache = cache.NewMapCacheByFn[string](dao.SearchPostIds, c.SearchPostCacheTime) + searchPostIdsCache = cache.NewMemoryMapCacheByFn[string](dao.SearchPostIds, c.SearchPostCacheTime) - postListIdsCache = cache.NewMapCacheByFn[string](dao.SearchPostIds, c.PostListCacheTime) + postListIdsCache = cache.NewMemoryMapCacheByFn[string](dao.SearchPostIds, c.PostListCacheTime) - monthPostsCache = cache.NewMapCacheByFn[string](dao.MonthPost, c.MonthPostCacheTime) + monthPostsCache = cache.NewMemoryMapCacheByFn[string](dao.MonthPost, c.MonthPostCacheTime) - postContextCache = cache.NewMapCacheByFn[uint64](dao.GetPostContext, c.ContextPostCacheTime) + postContextCache = cache.NewMemoryMapCacheByFn[uint64](dao.GetPostContext, c.ContextPostCacheTime) - postsCache = cache.NewMapCacheByBatchFn(dao.GetPostsByIds, c.PostDataCacheTime) + postsCache = cache.NewMemoryMapCacheByBatchFn(dao.GetPostsByIds, c.PostDataCacheTime) - postMetaCache = cache.NewMapCacheByBatchFn(dao.GetPostMetaByPostIds, c.PostDataCacheTime) + postMetaCache = cache.NewMemoryMapCacheByBatchFn(dao.GetPostMetaByPostIds, c.PostDataCacheTime) categoryAndTagsCaches = cache.NewVarCache(dao.CategoriesAndTags, c.CategoryCacheTime) @@ -68,58 +70,61 @@ func InitActionsCommonCache() { recentCommentsCaches = cache.NewVarCache(dao.RecentComments, c.RecentCommentsCacheTime) - postCommentCaches = cache.NewMapCacheByFn[uint64](dao.PostComments, c.PostCommentsCacheTime) + postCommentCaches = cache.NewMemoryMapCacheByFn[uint64](dao.PostComments, c.PostCommentsCacheTime) maxPostIdCache = cache.NewVarCache(dao.GetMaxPostId, c.MaxPostIdCacheTime) - usersCache = cache.NewMapCacheByFn[uint64](dao.GetUserById, c.UserInfoCacheTime) + usersCache = cache.NewMemoryMapCacheByFn[uint64](dao.GetUserById, c.UserInfoCacheTime) - usersNameCache = cache.NewMapCacheByFn[string](dao.GetUserByName, c.UserInfoCacheTime) + usersNameCache = cache.NewMemoryMapCacheByFn[string](dao.GetUserByName, c.UserInfoCacheTime) - commentsCache = cache.NewMapCacheByBatchFn(dao.GetCommentByIds, c.CommentsCacheTime) + commentsCache = cache.NewMemoryMapCacheByBatchFn(dao.GetCommentByIds, c.CommentsCacheTime) feedCache = cache.NewVarCache(feed, time.Hour) - postFeedCache = cache.NewMapCacheByFn[string](postFeed, time.Hour) + postFeedCache = cache.NewMemoryMapCacheByFn[string](postFeed, time.Hour) commentsFeedCache = cache.NewVarCache(commentsFeed, time.Hour) - newCommentCache = cache.NewMapCacheByFn[string, string](nil, 15*time.Minute) + newCommentCache = cache.NewMemoryMapCacheByFn[string, string](nil, 15*time.Minute) allUsernameCache = cache.NewVarCache(dao.AllUsername, c.UserInfoCacheTime) - headerImagesCache = cache.NewMapCacheByFn[string](getHeaderImages, c.ThemeHeaderImagCacheTime) + headerImagesCache = cache.NewMemoryMapCacheByFn[string](getHeaderImages, c.ThemeHeaderImagCacheTime) + + ctx = context.Background() InitFeed() } func ClearCache() { - searchPostIdsCache.ClearExpired() - postsCache.ClearExpired() - postMetaCache.ClearExpired() - postListIdsCache.ClearExpired() - monthPostsCache.ClearExpired() - postContextCache.ClearExpired() - usersCache.ClearExpired() - commentsCache.ClearExpired() - usersNameCache.ClearExpired() - postFeedCache.ClearExpired() - newCommentCache.ClearExpired() - headerImagesCache.ClearExpired() + searchPostIdsCache.ClearExpired(ctx) + searchPostIdsCache.ClearExpired(ctx) + postsCache.ClearExpired(ctx) + postMetaCache.ClearExpired(ctx) + postListIdsCache.ClearExpired(ctx) + monthPostsCache.ClearExpired(ctx) + postContextCache.ClearExpired(ctx) + usersCache.ClearExpired(ctx) + commentsCache.ClearExpired(ctx) + usersNameCache.ClearExpired(ctx) + postFeedCache.ClearExpired(ctx) + newCommentCache.ClearExpired(ctx) + headerImagesCache.ClearExpired(ctx) } func FlushCache() { - searchPostIdsCache.Flush() - postsCache.Flush() - postMetaCache.Flush() - postListIdsCache.Flush() - monthPostsCache.Flush() - postContextCache.Flush() - usersCache.Flush() - commentsCache.Flush() - usersCache.Flush() - postFeedCache.Flush() - newCommentCache.Flush() - headerImagesCache.Flush() + searchPostIdsCache.Flush(ctx) + postsCache.Flush(ctx) + postMetaCache.Flush(ctx) + postListIdsCache.Flush(ctx) + monthPostsCache.Flush(ctx) + postContextCache.Flush(ctx) + usersCache.Flush(ctx) + commentsCache.Flush(ctx) + usersCache.Flush(ctx) + postFeedCache.Flush(ctx) + newCommentCache.Flush(ctx) + headerImagesCache.Flush(ctx) } func Archives(ctx context.Context) (r []models.PostArchive) { diff --git a/internal/plugins/digest.go b/internal/plugins/digest.go index ced314e..90311ce 100644 --- a/internal/plugins/digest.go +++ b/internal/plugins/digest.go @@ -1,6 +1,7 @@ package plugins import ( + "context" "fmt" "github.com/fthvgb1/wp-go/cache" "github.com/fthvgb1/wp-go/internal/pkg/config" @@ -12,16 +13,18 @@ import ( ) var digestCache *cache.MapCache[uint64, string] +var ctx context.Context func InitDigestCache() { - digestCache = cache.NewMapCacheByFn[uint64](digestRaw, config.Conf.Load().DigestCacheTime) + ctx = context.Background() + digestCache = cache.NewMemoryMapCacheByFn[uint64](digestRaw, config.Conf.Load().DigestCacheTime) } func ClearDigestCache() { - digestCache.ClearExpired() + digestCache.ClearExpired(ctx) } func FlushCache() { - digestCache.Flush() + digestCache.Flush(ctx) } func digestRaw(arg ...any) (string, error) {