xing/wp-go
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

optimize cachemanger improve var cache

Browse Source
This commit is contained in:
xing 2023-11-02 22:40:13 +08:00
parent d1fb560578
commit 041d06104b
13 changed files with 271 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
app/actions/feed.go
View File

@ -25,19 +25,20 @@ func isCacheExpired(c *gin.Context, lastTime time.Time) bool {
}
func Feed(c *gin.Context) {
if !isCacheExpired(c, cache.FeedCache().GetLastSetTime()) {
feed := cache.FeedCache()
if !isCacheExpired(c, feed.GetLastSetTime(c)) {
c.Status(http.StatusNotModified)
return
}
r, err := cache.FeedCache().GetCache(c, time.Second, c)
r, err := feed.GetCache(c, time.Second, c)
if err != nil {
c.Status(http.StatusInternalServerError)
c.Abort()
c.Error(err)
return
}
setFeed(r[0], c, cache.FeedCache().GetLastSetTime())
setFeed(r[0], c, feed.GetLastSetTime(c))
}
func setFeed(s string, c *gin.Context, t time.Time) {
@ -51,31 +52,33 @@ func setFeed(s string, c *gin.Context, t time.Time) {
func PostFeed(c *gin.Context) {
id := c.Param("id")
if !isCacheExpired(c, cache.PostFeedCache().GetLastSetTime(c, id)) {
postFeed := cache.PostFeedCache()
if !isCacheExpired(c, postFeed.GetLastSetTime(c, id)) {
c.Status(http.StatusNotModified)
return
}
s, err := cache.PostFeedCache().GetCache(c, id, time.Second, c, id)
s, err := postFeed.GetCache(c, id, time.Second, c, id)
if err != nil {
c.Status(http.StatusInternalServerError)
c.Abort()
c.Error(err)
return
}
setFeed(s, c, cache.PostFeedCache().GetLastSetTime(c, id))
setFeed(s, c, postFeed.GetLastSetTime(c, id))
}
func CommentsFeed(c *gin.Context) {
if !isCacheExpired(c, cache.CommentsFeedCache().GetLastSetTime()) {
feed := cache.CommentsFeedCache()
if !isCacheExpired(c, feed.GetLastSetTime(c)) {
c.Status(http.StatusNotModified)
return
}
r, err := cache.CommentsFeedCache().GetCache(c, time.Second, c)
r, err := feed.GetCache(c, time.Second, c)
if err != nil {
c.Status(http.StatusInternalServerError)
c.Abort()
c.Error(err)
return
}
setFeed(r[0], c, cache.CommentsFeedCache().GetLastSetTime())
setFeed(r[0], c, feed.GetLastSetTime(c))
}

177
app/cmd/cachemanager/flush.go → app/cmd/cachemanager/manger.go
View File

@ -17,8 +17,13 @@ var getSingleFn = safety.NewMap[string, func(context.Context, any, time.Duration
var getBatchFn = safety.NewMap[string, func(context.Context, any, time.Duration, ...any) (any, error)]()
var anyFlush = safety.NewMap[string, func()]()
var getVar = safety.NewMap[string, func(context.Context, time.Duration, ...any) (any, error)]()
var expiredTime = safety.NewMap[string, expire]()
var varCache = safety.NewMap[string, any]()
var mapCache = safety.NewMap[string, any]()
type expire struct {
fn func() time.Duration
p *safety.Var[time.Duration]
@ -62,35 +67,38 @@ func FlushAnyVal(name ...string) {
func pushFlushMap[K comparable, V any](m *cache.MapCache[K, V], args ...any) {
name, _ := parseArgs(args...)
if name != "" {
anyFlush.Store(name, func() {
m.Flush(ctx)
})
mapFlush.Store(name, func(a any) {
k, ok := a.([]K)
if ok && len(k) > 0 {
m.Del(ctx, k...)
}
})
getSingleFn.Store(name, func(ct context.Context, k any, t time.Duration, a ...any) (any, error) {
kk, ok := k.(K)
if !ok {
return nil, errors.New(str.Join("cache ", name, " key type err"))
}
return m.GetCache(ct, kk, t, a...)
})
getBatchFn.Store(name, func(ct context.Context, k any, t time.Duration, a ...any) (any, error) {
kk, ok := k.([]K)
if !ok {
return nil, errors.New(str.Join("cache ", name, " key type err"))
}
return m.GetCacheBatch(ct, kk, t, a...)
})
FlushPush()
if name == "" {
return
}
mapCache.Store(name, m)
anyFlush.Store(name, func() {
m.Flush(ctx)
})
mapFlush.Store(name, func(a any) {
k, ok := a.([]K)
if ok && len(k) > 0 {
m.Del(ctx, k...)
}
})
getSingleFn.Store(name, func(ct context.Context, k any, t time.Duration, a ...any) (any, error) {
kk, ok := k.(K)
if !ok {
return nil, errors.New(str.Join("cache ", name, " key type err"))
}
return m.GetCache(ct, kk, t, a...)
})
getBatchFn.Store(name, func(ct context.Context, k any, t time.Duration, a ...any) (any, error) {
kk, ok := k.([]K)
if !ok {
return nil, errors.New(str.Join("cache ", name, " key type err"))
}
return m.GetCacheBatch(ct, kk, t, a...)
})
FlushPush()
}
func Get[T, K any](name string, ct context.Context, key K, timeout time.Duration, params ...any) (r T, err error) {
ct = context.WithValue(ct, "getCache", name)
v, ok := getSingleFn.Load(name)
if !ok {
err = errors.New(str.Join("cache ", name, " doesn't exist"))
@ -104,6 +112,7 @@ func Get[T, K any](name string, ct context.Context, key K, timeout time.Duration
return
}
func GetMultiple[T, K any](name string, ct context.Context, key []K, timeout time.Duration, params ...any) (r []T, err error) {
ct = context.WithValue(ct, "getCache", name)
v, ok := getBatchFn.Load(name)
if !ok {
err = errors.New(str.Join("cache ", name, " doesn't exist"))
@ -140,42 +149,53 @@ func NewMapCache[K comparable, V any](data cache.Cache[K, V], batchFn cache.MapB
pushFlushMap(m, args...)
FlushPush(m)
ClearPush(m)
name, f := parseArgs(args...)
if f != nil && name != "" {
SetExpireTime(any(data).(cache.SetTime), name, 0, f)
}
return m
}
func NewMemoryMapCache[K comparable, V any](batchFn cache.MapBatchFn[K, V],
fn cache.MapSingleFn[K, V], expireTime time.Duration, args ...any) *cache.MapCache[K, V] {
name, f := parseArgs(args...)
c := NewMapCache[K, V](cache.NewMemoryMapCache[K, V](func() time.Duration {
return expireTime
}), batchFn, fn, args...)
return c
}
func SetExpireTime(c cache.SetTime, name string, expireTime time.Duration, expireTimeFn func() time.Duration) {
if name == "" {
return
}
var t, tt func() time.Duration
t = f
t = expireTimeFn
if t == nil {
t = func() time.Duration {
return expireTime
}
}
tt = t
if name != "" {
expireTime = t()
p := safety.NewVar(expireTime)
e := expire{
fn: t,
p: p,
isUseManger: safety.NewVar(false),
}
expiredTime.Store(name, e)
reload.Push(func() {
if !e.isUseManger.Load() {
e.p.Store(tt())
}
}, str.Join("cacheManger-", name, "-expiredTime"))
t = func() time.Duration {
return e.p.Load()
}
expireTime = t()
p := safety.NewVar(expireTime)
e := expire{
fn: t,
p: p,
isUseManger: safety.NewVar(false),
}
return NewMapCache[K, V](cache.NewMemoryMapCache[K, V](t), batchFn, fn, args...)
expiredTime.Store(name, e)
reload.Push(func() {
if !e.isUseManger.Load() {
e.p.Store(tt())
}
}, str.Join("cacheManger-", name, "-expiredTime"))
t = func() time.Duration {
return e.p.Load()
}
c.SetExpiredTime(t)
}
func SetExpireTime(t time.Duration, name ...string) {
func ChangeExpireTime(t time.Duration, name ...string) {
for _, s := range name {
v, ok := expiredTime.Load(s)
if !ok {
@ -200,3 +220,66 @@ func ClearExpired() {
c.ClearExpired(ctx)
}
}
func NewVarCache[T any](c cache.AnyCache[T], fn func(context.Context, ...any) (T, error), a ...any) *cache.VarCache[T] {
v := cache.NewVarCache(c, fn)
FlushPush(v)
name, _ := parseArgs(a...)
if name != "" {
varCache.Store(name, v)
getVar.Store(name, func(c context.Context, duration time.Duration, a ...any) (any, error) {
return v.GetCache(c, duration, a...)
})
}
cc, ok := any(c).(clearExpired)
if ok {
ClearPush(cc)
}
return v
}
func GetVarVal[T any](name string, ctx context.Context, duration time.Duration, a ...any) (r T, err error) {
ctx = context.WithValue(ctx, "getCache", name)
fn, ok := getVar.Load(name)
if !ok {
err = errors.New(str.Join("cache ", name, " is not exist"))
return
}
v, err := fn(ctx, duration, a...)
if err != nil {
return
}
vv, ok := v.(T)
if !ok {
err = errors.New(str.Join("cache ", name, " value wanted can't match got"))
return
}
r = vv
return
}
func NewVarMemoryCache[T any](fn func(context.Context, ...any) (T, error), expired time.Duration, a ...any) *cache.VarCache[T] {
c := cache.NewVarMemoryCache[T](nil)
name, e := parseArgs(a...)
SetExpireTime(c, name, expired, e)
v := NewVarCache[T](c, fn, a...)
return v
}
func GetVarCache[T any](name string) (*cache.VarCache[T], bool) {
v, ok := varCache.Load(name)
if !ok {
return nil, false
}
vv, ok := v.(*cache.VarCache[T])
return vv, ok
}
func GetMapCache[K comparable, V any](name string) (*cache.MapCache[K, V], bool) {
v, ok := mapCache.Load(name)
if !ok {
return nil, false
}
vv, ok := v.(*cache.MapCache[K, V])
return vv, ok
}

2
app/cmd/cachemanager/flush_test.go → app/cmd/cachemanager/manger_test.go
View File

@ -61,7 +61,7 @@ func TestSetExpireTime(t *testing.T) {
fmt.Println(c.Get(ctx, "xx"))
time.Sleep(time.Second)
fmt.Println(c.Get(ctx, "xx"))
SetExpireTime(3*time.Second, "xx")
ChangeExpireTime(3*time.Second, "xx")
c.Set(ctx, "xx", "yyy")
time.Sleep(time.Second)
fmt.Println(c.Get(ctx, "xx"))

20
app/pkg/cache/cache.go vendored
View File

@ -69,15 +69,21 @@ func InitActionsCommonCache() {
return config.GetConfig().CacheTime.CategoryCacheTime
})
recentPostsCaches = cache.NewVarCache(dao.RecentPosts, c.CacheTime.RecentPostCacheTime)
recentPostsCaches = cachemanager.NewVarMemoryCache(dao.RecentPosts, c.CacheTime.RecentPostCacheTime, "recentPosts", func() time.Duration {
return config.GetConfig().CacheTime.RecentPostCacheTime
})
recentCommentsCaches = cache.NewVarCache(dao.RecentComments, c.CacheTime.RecentCommentsCacheTime)
recentCommentsCaches = cachemanager.NewVarMemoryCache(dao.RecentComments, c.CacheTime.RecentCommentsCacheTime, "recentComments", func() time.Duration {
return config.GetConfig().CacheTime.RecentCommentsCacheTime
})
postCommentCaches = cachemanager.NewMemoryMapCache(nil, dao.PostComments, c.CacheTime.PostCommentsCacheTime, "postCommentIds", func() time.Duration {
return config.GetConfig().CacheTime.PostCommentsCacheTime
})
maxPostIdCache = cache.NewVarCache(dao.GetMaxPostId, c.CacheTime.MaxPostIdCacheTime)
maxPostIdCache = cachemanager.NewVarMemoryCache(dao.GetMaxPostId, c.CacheTime.MaxPostIdCacheTime, "maxPostId", func() time.Duration {
return config.GetConfig().CacheTime.MaxPostIdCacheTime
})
cachemanager.NewMemoryMapCache(nil, dao.GetUserById, c.CacheTime.UserInfoCacheTime, "userData", func() time.Duration {
return config.GetConfig().CacheTime.UserInfoCacheTime
@ -91,13 +97,15 @@ func InitActionsCommonCache() {
return config.GetConfig().CacheTime.CommentsCacheTime
})
allUsernameCache = cache.NewVarCache(dao.AllUsername, c.CacheTime.UserInfoCacheTime)
allUsernameCache = cachemanager.NewVarMemoryCache(dao.AllUsername, c.CacheTime.UserInfoCacheTime, "allUsername", func() time.Duration {
return config.GetConfig().CacheTime.UserInfoCacheTime
})
feedCache = cache.NewVarCache(feed, time.Hour)
feedCache = cachemanager.NewVarMemoryCache(feed, time.Hour, "feed")
postFeedCache = cachemanager.NewMemoryMapCache(nil, postFeed, time.Hour, "postFeed")
commentsFeedCache = cache.NewVarCache(commentsFeed, time.Hour)
commentsFeedCache = cachemanager.NewVarMemoryCache(commentsFeed, time.Hour, "commentsFeed")
newCommentCache = cachemanager.NewMemoryMapCache[string, string](nil, nil, 15*time.Minute, "NewComment")

7
app/pkg/cache/feed.go vendored
View File

@ -13,7 +13,6 @@ import (
str "github.com/fthvgb1/wp-go/helper/strings"
"github.com/fthvgb1/wp-go/plugin/digest"
"github.com/fthvgb1/wp-go/rss2"
"github.com/gin-gonic/gin"
"strings"
"time"
)
@ -46,8 +45,7 @@ func PostFeedCache() *cache.MapCache[string, string] {
return postFeedCache
}
func feed(arg ...any) (xml []string, err error) {
c := arg[0].(*gin.Context)
func feed(c context.Context, _ ...any) (xml []string, err error) {
r := RecentPosts(c, 10)
ids := slice.Map(r, func(t models.Posts) uint64 {
return t.Id
@ -150,8 +148,7 @@ func postFeed(c context.Context, id string, arg ...any) (x string, err error) {
return
}
func commentsFeed(args ...any) (r []string, err error) {
c := args[0].(*gin.Context)
func commentsFeed(c context.Context, _ ...any) (r []string, err error) {
commens := RecentComments(c, 10)
rs := templateRss
rs.Title = fmt.Sprintf("\"%s\"的评论", wpconfig.GetOption("blogname"))

3
app/pkg/dao/comments.go
View File

@ -10,8 +10,7 @@ import (
// RecentComments
// param context.Context
func RecentComments(a ...any) (r []models.Comments, err error) {
ctx := a[0].(context.Context)
func RecentComments(ctx context.Context, a ...any) (r []models.Comments, err error) {
n := a[1].(int)
return model.Finds[models.Comments](ctx, model.Conditions(
model.Where(model.SqlBuilder{

6
app/pkg/dao/posts.go
View File

@ -116,8 +116,7 @@ func SearchPostIds(ctx context.Context, _ string, args ...any) (ids PostIds, err
return
}
func GetMaxPostId(a ...any) (uint64, error) {
ctx := a[0].(context.Context)
func GetMaxPostId(ctx context.Context, a ...any) (uint64, error) {
r, err := model.SimpleFind[models.Posts](ctx,
model.SqlBuilder{{"post_type", "post"}, {"post_status", "publish"}},
"max(ID) ID",
@ -129,8 +128,7 @@ func GetMaxPostId(a ...any) (uint64, error) {
return id, err
}
func RecentPosts(a ...any) (r []models.Posts, err error) {
ctx := a[0].(context.Context)
func RecentPosts(ctx context.Context, a ...any) (r []models.Posts, err error) {
num := a[1].(int)
r, err = model.Finds[models.Posts](ctx, model.Conditions(
model.Where(model.SqlBuilder{

3
app/pkg/dao/users.go
View File

@ -12,8 +12,7 @@ func GetUserById(ctx context.Context, uid uint64, _ ...any) (r models.Users, err
return
}
func AllUsername(a ...any) (map[string]struct{}, error) {
ctx := a[0].(context.Context)
func AllUsername(ctx context.Context, _ ...any) (map[string]struct{}, error) {
r, err := model.SimpleFind[models.Users](ctx, model.SqlBuilder{
{"user_status", "=", "0", "int"},
}, "user_login")

2
app/theme/wp/wp.go
View File

@ -118,7 +118,7 @@ func InitHandle(fn func(*Handle), h *Handle) {
h.C.Set("inited", true)
inited = true
return *h
}, float64(-100))
})
h.ginH = maps.Copy(hh.ginH)
h.ginH["calPostClass"] = postClass(h)
h.ginH["calBodyClass"] = bodyClass(h)

11
cache/cache.go vendored
View File

@ -19,3 +19,14 @@ type Expend[K comparable, V any] interface {
Gets(ctx context.Context, k []K) (map[K]V, error)
Sets(ctx context.Context, m map[K]V)
}
type SetTime interface {
SetExpiredTime(func() time.Duration)
}
type AnyCache[T any] interface {
Get(ctx context.Context) (T, bool)
Set(ctx context.Context, v T)
Flush(ctx context.Context)
GetLastSetTime(ctx context.Context) time.Time
}

4
cache/memorymapcache.go vendored
View File

@ -24,6 +24,10 @@ type mapVal[T any] struct {
data T
}
func (m *MemoryMapCache[K, V]) SetExpiredTime(f func() time.Duration) {
m.expireTime = f
}
func (m *MemoryMapCache[K, V]) GetExpireTime(_ context.Context) time.Duration {
return m.expireTime()
}

164
cache/vars.go vendored
View File

@ -10,90 +10,100 @@ import (
)
type VarCache[T any] struct {
v *safety.Var[vars[T]]
AnyCache[T]
setCacheFunc func(context.Context, ...any) (T, error)
mutex sync.Mutex
}
type vars[T any] struct {
data T
mutex *sync.Mutex
setCacheFunc func(...any) (T, error)
expireTime time.Duration
setTime time.Time
incr int
}
func (c *VarCache[T]) GetLastSetTime() time.Time {
return c.v.Load().setTime
}
func NewVarCache[T any](fun func(...any) (T, error), duration time.Duration) *VarCache[T] {
return &VarCache[T]{
v: safety.NewVar(vars[T]{
mutex: &sync.Mutex{},
setCacheFunc: fun,
expireTime: duration,
}),
func (t *VarCache[T]) GetCache(ctx context.Context, timeout time.Duration, params ...any) (T, error) {
data, ok := t.Get(ctx)
if ok {
return data, nil
}
}
func (c *VarCache[T]) IsExpired() bool {
v := c.v.Load()
return time.Duration(v.setTime.UnixNano())+v.expireTime < time.Duration(time.Now().UnixNano())
}
func (c *VarCache[T]) Flush() {
v := c.v.Load()
mu := v.mutex
mu.Lock()
defer mu.Unlock()
var vv T
v.data = vv
c.v.Store(v)
}
func (c *VarCache[T]) GetCache(ctx context.Context, timeout time.Duration, params ...any) (T, error) {
v := c.v.Load()
data := v.data
var err error
if v.expireTime <= 0 || ((time.Duration(v.setTime.UnixNano()) + v.expireTime) < time.Duration(time.Now().UnixNano())) {
t := v.incr
call := func() {
v.mutex.Lock()
defer v.mutex.Unlock()
vv := c.v.Load()
if vv.incr > t {
return
}
r, er := vv.setCacheFunc(params...)
if err != nil {
err = er
return
}
vv.setTime = time.Now()
vv.data = r
data = r
vv.incr++
c.v.Store(vv)
call := func() {
t.mutex.Lock()
defer t.mutex.Unlock()
dat, ok := t.Get(ctx)
if ok {
data = dat
return
}
if timeout > 0 {
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
done := make(chan struct{}, 1)
go func() {
call()
done <- struct{}{}
close(done)
}()
select {
case <-ctx.Done():
err = errors.New(fmt.Sprintf("get cache %s", ctx.Err().Error()))
case <-done:
}
} else {
r, er := t.setCacheFunc(ctx, params...)
if er != nil {
err = er
return
}
t.Set(ctx, r)
data = r
}
if timeout > 0 {
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
done := make(chan struct{}, 1)
go func() {
call()
}
done <- struct{}{}
close(done)
}()
select {
case <-ctx.Done():
err = errors.New(fmt.Sprintf("get cache %s", ctx.Err().Error()))
case <-done:
}
} else {
call()
}
return data, err
}
type VarMemoryCache[T any] struct {
v *safety.Var[vars[T]]
expireTime func() time.Duration
}
func (c *VarMemoryCache[T]) ClearExpired(ctx context.Context) {
c.Flush(ctx)
}
func NewVarMemoryCache[T any](expireTime func() time.Duration) *VarMemoryCache[T] {
return &VarMemoryCache[T]{v: safety.NewVar(vars[T]{}), expireTime: expireTime}
}
func (c *VarMemoryCache[T]) Get(_ context.Context) (T, bool) {
v := c.v.Load()
return v.data, c.expireTime() >= time.Now().Sub(v.setTime)
}
func (c *VarMemoryCache[T]) Set(_ context.Context, v T) {
vv := c.v.Load()
vv.data = v
vv.setTime = time.Now()
vv.incr++
c.v.Store(vv)
}
func (c *VarMemoryCache[T]) SetExpiredTime(f func() time.Duration) {
c.expireTime = f
}
type vars[T any] struct {
data T
setTime time.Time
incr int
}
func (c *VarMemoryCache[T]) GetLastSetTime(_ context.Context) time.Time {
return c.v.Load().setTime
}
func NewVarCache[T any](cache AnyCache[T], fn func(context.Context, ...any) (T, error)) *VarCache[T] {
return &VarCache[T]{
AnyCache: cache, setCacheFunc: fn, mutex: sync.Mutex{},
}
}
func (c *VarMemoryCache[T]) Flush(_ context.Context) {
c.v.Flush()
}

41
cache/vars_test.go vendored
View File

@ -7,9 +7,11 @@ import (
"time"
)
var cc = *NewVarCache(func(a ...any) (int, error) {
var cc = *NewVarCache[int](NewVarMemoryCache[int](func() time.Duration {
return time.Minute
}), func(ctx context.Context, a ...any) (int, error) {
return 1, nil
}, time.Minute)
})
func TestVarCache_Flush(t *testing.T) {
type testCase[T any] struct {
@ -26,41 +28,8 @@ func TestVarCache_Flush(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
fmt.Println(tt.c.GetCache(c, time.Second))
tt.c.Flush()
tt.c.Flush(ctx)
fmt.Println(tt.c.GetCache(c, time.Second))
})
}
}
func TestVarCache_IsExpired(t *testing.T) {
type testCase[T any] struct {
name string
c VarCache[T]
want bool
}
tests := []testCase[int]{
{
name: "expired",
c: cc,
want: true,
},
{
name: "not expired",
c: func() VarCache[int] {
v := *NewVarCache(func(a ...any) (int, error) {
return 1, nil
}, time.Minute)
_, _ = v.GetCache(context.Background(), time.Second)
return v
}(),
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := tt.c.IsExpired(); got != tt.want {
t.Errorf("IsExpired() = %v, want %v", got, tt.want)
}
})
}
}