xing/wp-go
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

optimize and expand map cache remove ver add dynamic set expired time

Browse Source
This commit is contained in:
xing 2023-10-30 21:52:15 +08:00
parent 86d1616732
commit 64a2c2e33b
11 changed files with 188 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
app/cmd/cachemanager/flush.go
View File

@ -3,6 +3,7 @@ package cachemanager
import (
"context"
"errors"
"github.com/fthvgb1/wp-go/app/cmd/reload"
"github.com/fthvgb1/wp-go/cache"
str "github.com/fthvgb1/wp-go/helper/strings"
"github.com/fthvgb1/wp-go/safety"
@ -16,6 +17,14 @@ var getSingleFn = safety.NewMap[string, func(context.Context, any, time.Duration
var getBatchFn = safety.NewMap[string, func(context.Context, any, time.Duration, ...any) (any, error)]()
var anyFlush = safety.NewMap[string, func()]()
var expiredTime = safety.NewMap[string, expire]()
type expire struct {
fn func() time.Duration
p *safety.Var[time.Duration]
isUseManger *safety.Var[bool]
}
type flush interface {
Flush(ctx context.Context)
}
@ -52,7 +61,7 @@ func FlushAnyVal(name ...string) {
}
func pushFlushMap[K comparable, V any](m *cache.MapCache[K, V], args ...any) {
name := parseArgs(args...)
name, _ := parseArgs(args...)
if name != "" {
anyFlush.Store(name, func() {
m.Flush(ctx)
@ -108,15 +117,22 @@ func GetMultiple[T, K any](name string, ct context.Context, key []K, timeout tim
return
}
func parseArgs(args ...any) string {
func parseArgs(args ...any) (string, func() time.Duration) {
var name string
var fn func() time.Duration
for _, arg := range args {
v, ok := arg.(string)
if ok {
name = v
continue
}
vv, ok := arg.(func() time.Duration)
if ok {
fn = vv
}
}
return name
return name, fn
}
func NewMapCache[K comparable, V any](data cache.Cache[K, V], batchFn cache.MapBatchFn[K, V], fn cache.MapSingleFn[K, V], args ...any) *cache.MapCache[K, V] {
@ -128,7 +144,48 @@ func NewMapCache[K comparable, V any](data cache.Cache[K, V], batchFn cache.MapB
}
func NewMemoryMapCache[K comparable, V any](batchFn cache.MapBatchFn[K, V],
fn cache.MapSingleFn[K, V], expireTime time.Duration, args ...any) *cache.MapCache[K, V] {
return NewMapCache[K, V](cache.NewMemoryMapCache[K, V](expireTime), batchFn, fn, args...)
name, f := parseArgs(args...)
var t, tt func() time.Duration
t = f
if t == nil {
t = func() time.Duration {
return expireTime
}
}
tt = t
if name != "" {
expireTime = t()
p := safety.NewVar(expireTime)
e := expire{
fn: t,
p: p,
isUseManger: safety.NewVar(false),
}
expiredTime.Store(name, e)
reload.Push(func() {
if !e.isUseManger.Load() {
e.p.Store(tt())
}
}, str.Join("cacheManger-", name, "-expiredTime"))
t = func() time.Duration {
return e.p.Load()
}
}
return NewMapCache[K, V](cache.NewMemoryMapCache[K, V](t), batchFn, fn, args...)
}
func SetExpireTime(t time.Duration, name ...string) {
for _, s := range name {
v, ok := expiredTime.Load(s)
if !ok {
continue
}
v.p.Store(t)
if !v.isUseManger.Load() {
v.isUseManger.Store(true)
}
}
}
func FlushPush(f ...flush) {

9
app/cmd/reload/reload.go
View File

@ -13,7 +13,7 @@ type queue struct {
name string
}
var calls []queue
var calls = safety.NewSlice(make([]queue, 0))
var callsM = safety.NewMap[string, func()]()
var anyMap = safety.NewMap[string, any]()
@ -252,7 +252,8 @@ func SafeMap[K comparable, T any](args ...any) *safety.Map[K, T] {
func Push(fn func(), a ...any) {
ord, name := parseArgs(a...)
calls = append(calls, queue{fn, ord, name})
calls.Append(queue{fn, ord, name})
//calls = append(calls, queue{fn, ord, name})
if name != "" {
callsM.Store(name, fn)
}
@ -263,10 +264,10 @@ func Reload() {
safetyMaps.Flush()
callsM.Flush()
flushMapFn.Flush()
slice.Sort(calls, func(i, j queue) bool {
slice.Sort(calls.Load(), func(i, j queue) bool {
return i.order > j.order
})
for _, call := range calls {
for _, call := range calls.Load() {
call.fn()
}
return

44
app/pkg/cache/cache.go vendored
View File

@ -41,33 +41,55 @@ var allUsernameCache *cache.VarCache[map[string]struct{}]
func InitActionsCommonCache() {
c := config.GetConfig()
searchPostIdsCache = cachemanager.NewMemoryMapCache(nil, dao.SearchPostIds, c.CacheTime.SearchPostCacheTime, "searchPostIds")
searchPostIdsCache = cachemanager.NewMemoryMapCache(nil, dao.SearchPostIds, c.CacheTime.SearchPostCacheTime, "searchPostIds", func() time.Duration {
return config.GetConfig().CacheTime.SearchPostCacheTime
})
postListIdsCache = cachemanager.NewMemoryMapCache(nil, dao.SearchPostIds, c.CacheTime.PostListCacheTime, "listPostIds")
postListIdsCache = cachemanager.NewMemoryMapCache(nil, dao.SearchPostIds, c.CacheTime.PostListCacheTime, "listPostIds", func() time.Duration {
return config.GetConfig().CacheTime.PostListCacheTime
})
monthPostsCache = cachemanager.NewMemoryMapCache(nil, dao.MonthPost, c.CacheTime.MonthPostCacheTime, "monthPostIds")
monthPostsCache = cachemanager.NewMemoryMapCache(nil, dao.MonthPost, c.CacheTime.MonthPostCacheTime, "monthPostIds", func() time.Duration {
return config.GetConfig().CacheTime.MonthPostCacheTime
})
postContextCache = cachemanager.NewMemoryMapCache(nil, dao.GetPostContext, c.CacheTime.ContextPostCacheTime, "postContext")
postContextCache = cachemanager.NewMemoryMapCache(nil, dao.GetPostContext, c.CacheTime.ContextPostCacheTime, "postContext", func() time.Duration {
return config.GetConfig().CacheTime.ContextPostCacheTime
})
postsCache = cachemanager.NewMemoryMapCache(dao.GetPostsByIds, nil, c.CacheTime.PostDataCacheTime, "postData")
postsCache = cachemanager.NewMemoryMapCache(dao.GetPostsByIds, nil, c.CacheTime.PostDataCacheTime, "postData", func() time.Duration {
return config.GetConfig().CacheTime.PostDataCacheTime
})
postMetaCache = cachemanager.NewMemoryMapCache(dao.GetPostMetaByPostIds, nil, c.CacheTime.PostDataCacheTime, "postMetaData")
postMetaCache = cachemanager.NewMemoryMapCache(dao.GetPostMetaByPostIds, nil, c.CacheTime.PostDataCacheTime, "postMetaData", func() time.Duration {
return config.GetConfig().CacheTime.PostDataCacheTime
})
categoryAndTagsCaches = cachemanager.NewMemoryMapCache(nil, dao.CategoriesAndTags, c.CacheTime.CategoryCacheTime, "categoryAndTagsData")
categoryAndTagsCaches = cachemanager.NewMemoryMapCache(nil, dao.CategoriesAndTags, c.CacheTime.CategoryCacheTime, "categoryAndTagsData", func() time.Duration {
return config.GetConfig().CacheTime.CategoryCacheTime
})
recentPostsCaches = cache.NewVarCache(dao.RecentPosts, c.CacheTime.RecentPostCacheTime)
recentCommentsCaches = cache.NewVarCache(dao.RecentComments, c.CacheTime.RecentCommentsCacheTime)
postCommentCaches = cachemanager.NewMemoryMapCache(nil, dao.PostComments, c.CacheTime.PostCommentsCacheTime, "postCommentIds")
postCommentCaches = cachemanager.NewMemoryMapCache(nil, dao.PostComments, c.CacheTime.PostCommentsCacheTime, "postCommentIds", func() time.Duration {
return config.GetConfig().CacheTime.PostCommentsCacheTime
})
maxPostIdCache = cache.NewVarCache(dao.GetMaxPostId, c.CacheTime.MaxPostIdCacheTime)
cachemanager.NewMemoryMapCache(nil, dao.GetUserById, c.CacheTime.UserInfoCacheTime, "userData")
cachemanager.NewMemoryMapCache(nil, dao.GetUserById, c.CacheTime.UserInfoCacheTime, "userData", func() time.Duration {
return config.GetConfig().CacheTime.UserInfoCacheTime
})
usersNameCache = cachemanager.NewMemoryMapCache(nil, dao.GetUserByName, c.CacheTime.UserInfoCacheTime, "usernameMapToUserData")
usersNameCache = cachemanager.NewMemoryMapCache(nil, dao.GetUserByName, c.CacheTime.UserInfoCacheTime, "usernameMapToUserData", func() time.Duration {
return config.GetConfig().CacheTime.UserInfoCacheTime
})
cachemanager.NewMemoryMapCache(dao.GetCommentByIds, nil, c.CacheTime.CommentsCacheTime, "commentData")
cachemanager.NewMemoryMapCache(dao.GetCommentByIds, nil, c.CacheTime.CommentsCacheTime, "commentData", func() time.Duration {
return config.GetConfig().CacheTime.CommentsCacheTime
})
allUsernameCache = cache.NewVarCache(dao.AllUsername, c.CacheTime.UserInfoCacheTime)

4
app/plugins/digest.go
View File

@ -22,7 +22,9 @@ var more = regexp.MustCompile("<!--more(.*?)?-->")
var removeWpBlock = regexp.MustCompile("<!-- /?wp:.*-->")
func InitDigestCache() {
digestCache = cachemanager.NewMemoryMapCache(nil, digestRaw, config.GetConfig().CacheTime.DigestCacheTime, "digestPlugin")
digestCache = cachemanager.NewMemoryMapCache(nil, digestRaw, config.GetConfig().CacheTime.DigestCacheTime, "digestPlugin", func() time.Duration {
return config.GetConfig().CacheTime.DigestCacheTime
})
}
func RemoveWpBlock(s string) string {

17
app/theme/wp/fn.go
View File

@ -2,13 +2,14 @@ package wp
import (
"errors"
"github.com/fthvgb1/wp-go/safety"
)
var fnMap map[string]map[string]any
var fnHook map[string]map[string]any
var fnMap = safety.NewMap[string, map[string]any]() //map[string]map[string]any
var fnHook = safety.NewMap[string, map[string]any]() // map[string]map[string]any
func GetFn[T any](fnType string, name string) []T {
v, ok := fnMap[fnType]
v, ok := fnMap.Load(fnType)
if !ok {
return nil
}
@ -19,7 +20,7 @@ func GetFn[T any](fnType string, name string) []T {
return vv.([]T)
}
func GetFnHook[T any](fnType string, name string) []T {
v, ok := fnHook[fnType]
v, ok := fnHook.Load(fnType)
if !ok {
return nil
}
@ -31,10 +32,10 @@ func GetFnHook[T any](fnType string, name string) []T {
}
func PushFn[T any](fnType string, name string, fns ...T) error {
v, ok := fnMap[fnType]
v, ok := fnMap.Load(fnType)
if !ok {
v = make(map[string]any)
fnMap[fnType] = v
fnMap.Store(fnType, v)
v[name] = fns
return nil
}
@ -52,10 +53,10 @@ func PushFn[T any](fnType string, name string, fns ...T) error {
}
func PushFnHook[T any](fnType string, name string, fns ...T) error {
v, ok := fnHook[fnType]
v, ok := fnHook.Load(fnType)
if !ok {
v = make(map[string]any)
fnHook[fnType] = v
fnHook.Store(fnType, v)
v[name] = fns
return nil
}

4
app/theme/wp/wp.go
View File

@ -107,8 +107,8 @@ func InitHandle(fn func(*Handle), h *Handle) {
h.handlers = make(map[string]map[string][]HandleCall)
h.handleHook = make(map[string][]func(HandleCall) (HandleCall, bool))
h.ginH = gin.H{}
fnMap = map[string]map[string]any{}
fnHook = map[string]map[string]any{}
fnMap.Flush()
fnHook.Flush()
fn(h)
v := apply.UsePlugins()
pluginFn, ok := v.(func(*Handle))

2
cache/cache.go vendored
View File

@ -10,7 +10,6 @@ type Cache[K comparable, V any] interface {
Set(ctx context.Context, key K, val V)
GetExpireTime(ctx context.Context) time.Duration
Ttl(ctx context.Context, key K) time.Duration
Ver(ctx context.Context, key K) int
Flush(ctx context.Context)
Del(ctx context.Context, key ...K)
ClearExpired(ctx context.Context)
@ -18,6 +17,5 @@ type Cache[K comparable, V any] interface {
type Expend[K comparable, V any] interface {
Gets(ctx context.Context, k []K) (map[K]V, error)
Vers(ctx context.Context, k []K) map[K]int
Sets(ctx context.Context, m map[K]V)
}

116
cache/map.go vendored
View File

@ -98,39 +98,39 @@ func (m *MapCache[K, V]) Flush(ctx context.Context) {
func (m *MapCache[K, V]) GetCache(c context.Context, key K, timeout time.Duration, params ...any) (V, error) {
data, ok := m.Get(c, key)
if ok {
return data, nil
}
var err error
if !ok {
ver := m.Ver(c, key)
call := func() {
m.mux.Lock()
defer m.mux.Unlock()
if m.Ver(c, key) > ver {
data, _ = m.Get(c, key)
return
}
data, err = m.cacheFunc(c, key, params...)
if err != nil {
return
}
m.Set(c, key, data)
call := func() {
m.mux.Lock()
defer m.mux.Unlock()
if data, ok = m.Get(c, key); ok {
return
}
if timeout > 0 {
ctx, cancel := context.WithTimeout(c, timeout)
defer cancel()
done := make(chan struct{}, 1)
go func() {
call()
done <- struct{}{}
}()
select {
case <-ctx.Done():
err = errors.New(fmt.Sprintf("get cache %v %s", key, ctx.Err().Error()))
case <-done:
}
} else {
data, err = m.cacheFunc(c, key, params...)
if err != nil {
return
}
m.Set(c, key, data)
}
if timeout > 0 {
ctx, cancel := context.WithTimeout(c, timeout)
defer cancel()
done := make(chan struct{}, 1)
go func() {
call()
done <- struct{}{}
}()
select {
case <-ctx.Done():
err = errors.New(fmt.Sprintf("get cache %v %s", key, ctx.Err().Error()))
var vv V
return vv, err
case <-done:
}
} else {
call()
}
return data, err
}
@ -142,11 +142,9 @@ func (m *MapCache[K, V]) GetCacheBatch(c context.Context, key []K, timeout time.
func (m *MapCache[K, V]) getCacheBatchs(c context.Context, key []K, timeout time.Duration, params ...any) ([]V, error) {
var res = make([]V, 0, len(key))
var needIndex = make(map[K]int)
var ver = make(map[K]int)
for i, k := range key {
v, ok := m.Get(c, k)
if !ok {
ver[k] = m.Ver(c, k)
needIndex[k] = i
}
res = append(res, v)
@ -160,13 +158,16 @@ func (m *MapCache[K, V]) getCacheBatchs(c context.Context, key []K, timeout time
m.mux.Lock()
defer m.mux.Unlock()
needFlushs := maps.FilterToSlice(needIndex, func(k K, v int) (K, bool) {
return k, ver[k] >= m.Ver(c, k)
vv, ok := m.Get(c, k)
if ok {
res[needIndex[k]] = vv
delete(needIndex, k)
return k, false
}
return k, true
})
if len(needFlushs) < 1 {
for k, i := range needIndex {
res[i], _ = m.Get(c, k)
}
return
}
@ -180,11 +181,6 @@ func (m *MapCache[K, V]) getCacheBatchs(c context.Context, key []K, timeout time
if ok {
res[i] = v
m.Set(c, k, v)
} else {
v, ok = m.Get(c, k)
if ok {
res[i] = v
}
}
}
}
@ -199,6 +195,7 @@ func (m *MapCache[K, V]) getCacheBatchs(c context.Context, key []K, timeout time
select {
case <-ctx.Done():
err = errors.New(fmt.Sprintf("get cache %v %s", key, ctx.Err().Error()))
return nil, err
case <-done:
}
} else {
@ -232,37 +229,29 @@ func (m *MapCache[K, V]) getBatches(e Expend[K, V]) func(ctx context.Context, ke
if len(needIndex) < 1 {
return res, nil
}
vers := cc.Vers(ctx, flushKeys)
call := func() {
m.mux.Lock()
defer m.mux.Unlock()
verss := cc.Vers(ctx, flushKeys)
needFlushs := maps.FilterToSlice(needIndex, func(k K, v int) (K, bool) {
vv, ok := vers[k]
vvv, ook := verss[k]
if !ok || !ook || vv >= vvv {
return k, true
}
return k, false
})
mmm, er := cc.Gets(ctx, maps.FilterToSlice(needIndex, func(k K, v int) (K, bool) {
return k, true
}))
if er != nil {
err = er
return
}
for k, v := range mmm {
res[needIndex[k]] = v
delete(needIndex, k)
}
if len(needFlushs) < 1 {
vv, er := cc.Gets(ctx, needFlushs)
if er != nil {
err = er
return
}
for k, i := range needIndex {
v, ok := vv[k]
if ok {
res[i] = v
}
}
if len(needIndex) < 1 {
return
}
r, er := m.batchCacheFn(ctx, needFlushs, params...)
r, er := m.batchCacheFn(ctx, maps.FilterToSlice(needIndex, func(k K, v int) (K, bool) {
return k, true
}), params...)
if err != nil {
err = er
return
@ -287,6 +276,7 @@ func (m *MapCache[K, V]) getBatches(e Expend[K, V]) func(ctx context.Context, ke
select {
case <-ctx.Done():
err = errors.New(fmt.Sprintf("get cache %v %s", key, ctx.Err().Error()))
return nil, err
case <-done:
}
} else {

9
cache/map_test.go vendored
View File

@ -27,10 +27,7 @@ func init() {
return t, strings.Repeat(t, 2), true
}), nil
}
ca = *NewMemoryMapCacheByFn[string, string](fn, time.Second*2)
ca.SetCacheBatchFn(batchFn)
_, _ = ca.GetCache(ct, "aa", time.Second, ct, "aa")
_, _ = ca.GetCache(ct, "bb", time.Second, ct, "bb")
}
func TestMapCache_ClearExpired(t *testing.T) {
type args struct {
@ -70,7 +67,9 @@ func TestMapCache_Flush(t *testing.T) {
m MapCache[K, V]
args args
}
ca := *NewMemoryMapCacheByFn[string, string](fn, time.Second)
ca := *NewMapCache[string, string](NewMemoryMapCache[string, string](func() time.Duration {
return time.Second
}), fn, nil)
_, _ = ca.GetCache(ct, "aa", time.Second, ct, "aa")
tests := []testCase[string, string]{
{

21
cache/memorymapcache.go vendored
View File

@ -3,24 +3,15 @@ package cache
import (
"context"
"github.com/fthvgb1/wp-go/safety"
"sync"
"time"
)
type MemoryMapCache[K comparable, V any] struct {
*safety.Map[K, mapVal[V]]
expireTime time.Duration
expireTime func() time.Duration
}
func NewMemoryMapCacheByFn[K comparable, V any](fn MapSingleFn[K, V], expireTime time.Duration) *MapCache[K, V] {
return &MapCache[K, V]{
Cache: NewMemoryMapCache[K, V](expireTime),
cacheFunc: fn,
mux: sync.Mutex{},
}
}
func NewMemoryMapCache[K comparable, V any](expireTime time.Duration) *MemoryMapCache[K, V] {
func NewMemoryMapCache[K comparable, V any](expireTime func() time.Duration) *MemoryMapCache[K, V] {
return &MemoryMapCache[K, V]{
Map: safety.NewMap[K, mapVal[V]](),
expireTime: expireTime,
@ -34,7 +25,7 @@ type mapVal[T any] struct {
}
func (m *MemoryMapCache[K, V]) GetExpireTime(_ context.Context) time.Duration {
return m.expireTime
return m.expireTime()
}
func (m *MemoryMapCache[K, V]) Get(_ context.Context, key K) (r V, ok bool) {
@ -43,7 +34,7 @@ func (m *MemoryMapCache[K, V]) Get(_ context.Context, key K) (r V, ok bool) {
return
}
r = v.data
t := m.expireTime - time.Now().Sub(v.setTime)
t := m.expireTime() - time.Now().Sub(v.setTime)
if t <= 0 {
ok = false
}
@ -72,7 +63,7 @@ func (m *MemoryMapCache[K, V]) Ttl(_ context.Context, key K) time.Duration {
if !ok {
return time.Duration(-1)
}
return m.expireTime - time.Now().Sub(v.setTime)
return m.expireTime() - time.Now().Sub(v.setTime)
}
func (m *MemoryMapCache[K, V]) Ver(_ context.Context, key K) int {
@ -96,7 +87,7 @@ func (m *MemoryMapCache[K, V]) Del(_ context.Context, keys ...K) {
func (m *MemoryMapCache[K, V]) ClearExpired(_ context.Context) {
now := time.Duration(time.Now().UnixNano())
m.Range(func(k K, v mapVal[V]) bool {
if now > time.Duration(v.setTime.UnixNano())+m.expireTime {
if now > time.Duration(v.setTime.UnixNano())+m.expireTime() {
m.Map.Delete(k)
}
return true

12
helper/maps/map.go
View File

@ -24,6 +24,18 @@ func StructToAnyMap[K comparable, T any](s T) (r map[K]any, err error) {
return
}
func Filter[M ~map[K]V, K comparable, V any](fn func(K, V) bool, m ...M) M {
var r = map[K]V{}
for _, mm := range m {
for k, v := range mm {
if ok := fn(k, v); ok {
r[k] = v
}
}
}
return r
}
func FilterToSlice[T any, K comparable, V any](m map[K]V, fn func(K, V) (T, bool)) (r []T) {
for k, v := range m {
vv, ok := fn(k, v)