From 42d2795a05bcae0868ba27e86e711327b2971a75 Mon Sep 17 00:00:00 2001 From: xing Date: Tue, 7 Nov 2023 15:18:34 +0800 Subject: [PATCH] map cache add func get map --- app/cmd/main.go | 2 +- app/pkg/cache/cache.go | 2 +- app/pkg/cache/comments.go | 2 +- app/pkg/cache/users.go | 2 +- app/plugins/digest.go | 2 +- .../cmd => cache}/cachemanager/manger_test.go | 23 ++- cache/map.go | 158 +++++++++++++++++- 7 files changed, 181 insertions(+), 10 deletions(-) rename {app/cmd => cache}/cachemanager/manger_test.go (61%) diff --git a/app/cmd/main.go b/app/cmd/main.go index c0474ec..c99e884 100644 --- a/app/cmd/main.go +++ b/app/cmd/main.go @@ -3,7 +3,6 @@ package main import ( "flag" "fmt" - "github.com/fthvgb1/wp-go/app/cmd/cachemanager" "github.com/fthvgb1/wp-go/app/cmd/reload" "github.com/fthvgb1/wp-go/app/cmd/route" "github.com/fthvgb1/wp-go/app/mail" @@ -15,6 +14,7 @@ import ( "github.com/fthvgb1/wp-go/app/plugins/wphandle" "github.com/fthvgb1/wp-go/app/theme" "github.com/fthvgb1/wp-go/app/wpconfig" + "github.com/fthvgb1/wp-go/cache/cachemanager" "github.com/fthvgb1/wp-go/model" "log" "os" diff --git a/app/pkg/cache/cache.go b/app/pkg/cache/cache.go index cb9171d..7a7241a 100644 --- a/app/pkg/cache/cache.go +++ b/app/pkg/cache/cache.go @@ -2,12 +2,12 @@ package cache import ( "context" - "github.com/fthvgb1/wp-go/app/cmd/cachemanager" "github.com/fthvgb1/wp-go/app/pkg/config" "github.com/fthvgb1/wp-go/app/pkg/dao" "github.com/fthvgb1/wp-go/app/pkg/logs" "github.com/fthvgb1/wp-go/app/pkg/models" "github.com/fthvgb1/wp-go/cache" + "github.com/fthvgb1/wp-go/cache/cachemanager" "github.com/fthvgb1/wp-go/helper/slice" "github.com/fthvgb1/wp-go/safety" "time" diff --git a/app/pkg/cache/comments.go b/app/pkg/cache/comments.go index 6d002ba..6704b10 100644 --- a/app/pkg/cache/comments.go +++ b/app/pkg/cache/comments.go @@ -2,10 +2,10 @@ package cache import ( "context" - "github.com/fthvgb1/wp-go/app/cmd/cachemanager" "github.com/fthvgb1/wp-go/app/pkg/logs" "github.com/fthvgb1/wp-go/app/pkg/models" "github.com/fthvgb1/wp-go/cache" + "github.com/fthvgb1/wp-go/cache/cachemanager" "github.com/fthvgb1/wp-go/helper/number" "time" ) diff --git a/app/pkg/cache/users.go b/app/pkg/cache/users.go index c546e66..3ce84bf 100644 --- a/app/pkg/cache/users.go +++ b/app/pkg/cache/users.go @@ -2,9 +2,9 @@ package cache import ( "context" - "github.com/fthvgb1/wp-go/app/cmd/cachemanager" "github.com/fthvgb1/wp-go/app/pkg/logs" "github.com/fthvgb1/wp-go/app/pkg/models" + "github.com/fthvgb1/wp-go/cache/cachemanager" "github.com/fthvgb1/wp-go/model" "time" ) diff --git a/app/plugins/digest.go b/app/plugins/digest.go index cbcad27..03a8a8c 100644 --- a/app/plugins/digest.go +++ b/app/plugins/digest.go @@ -3,10 +3,10 @@ package plugins import ( "context" "fmt" - "github.com/fthvgb1/wp-go/app/cmd/cachemanager" "github.com/fthvgb1/wp-go/app/pkg/config" "github.com/fthvgb1/wp-go/app/pkg/models" "github.com/fthvgb1/wp-go/cache" + "github.com/fthvgb1/wp-go/cache/cachemanager" "github.com/fthvgb1/wp-go/helper" "github.com/fthvgb1/wp-go/plugin/digest" "regexp" diff --git a/app/cmd/cachemanager/manger_test.go b/cache/cachemanager/manger_test.go similarity index 61% rename from app/cmd/cachemanager/manger_test.go rename to cache/cachemanager/manger_test.go index c84f8a2..8c09935 100644 --- a/app/cmd/cachemanager/manger_test.go +++ b/cache/cachemanager/manger_test.go @@ -4,7 +4,10 @@ import ( "context" "fmt" "github.com/fthvgb1/wp-go/helper/number" + "github.com/fthvgb1/wp-go/helper/slice" + str "github.com/fthvgb1/wp-go/helper/strings" "github.com/fthvgb1/wp-go/taskPools" + "reflect" "testing" "time" ) @@ -51,12 +54,17 @@ func TestFlushMapVal(t *testing.T) { FlushAnyVal("test") fmt.Println(vv.Get(ctx, 5)) fmt.Println(vv.Get(ctx, 6)) + //fmt.Println(GetVarCache("test")) }) } func TestSetExpireTime(t *testing.T) { t.Run("t1", func(t *testing.T) { - c := NewMemoryMapCache[string, string](nil, nil, time.Second, "xx") + c := NewMemoryMapCache[string, string](func(ctx2 context.Context, strings []string, a ...any) (map[string]string, error) { + return slice.ToMap(strings, func(v string) (string, string) { + return v, str.Join(v, "__", v) + }, false), nil + }, nil, time.Second, "xx") c.Set(ctx, "xx", "yy") fmt.Println(c.Get(ctx, "xx")) time.Sleep(time.Second) @@ -67,5 +75,18 @@ func TestSetExpireTime(t *testing.T) { fmt.Println(c.Get(ctx, "xx")) time.Sleep(3 * time.Second) fmt.Println(c.Get(ctx, "xx")) + cc, _ := GetMapCache[string, string]("xx") + fmt.Println(reflect.DeepEqual(c, cc)) + cc.Set(ctx, "fff", "xxxx") + cc.Set(ctx, "ffx", "eex") + cc.Set(ctx, "ww", "vv") + m, err := cc.GetBatchToMap(ctx, []string{"fff", "ffx", "ww", "kkkk"}, time.Second) + fmt.Println(m, err) + fmt.Println(GetMultipleToMap[string]("xx", ctx, []string{"fff", "ffx", "ww", "kkkk"}, time.Second)) + v := NewVarMemoryCache(func(ct context.Context, a ...any) (string, error) { + return "ssss", nil + }, 3*time.Second, "ff") + vv, _ := GetVarCache[string]("ff") + fmt.Println(reflect.DeepEqual(v, vv)) }) } diff --git a/cache/map.go b/cache/map.go index 1f35427..6322cd2 100644 --- a/cache/map.go +++ b/cache/map.go @@ -11,10 +11,11 @@ import ( type MapCache[K comparable, V any] struct { Cache[K, V] - mux sync.Mutex - cacheFunc MapSingleFn[K, V] - batchCacheFn MapBatchFn[K, V] - getCacheBatch func(c context.Context, key []K, timeout time.Duration, params ...any) ([]V, error) + mux sync.Mutex + cacheFunc MapSingleFn[K, V] + batchCacheFn MapBatchFn[K, V] + getCacheBatch func(c context.Context, key []K, timeout time.Duration, params ...any) ([]V, error) + getCacheBatchToMap func(c context.Context, key []K, timeout time.Duration, params ...any) (map[K]V, error) } type MapSingleFn[K, V any] func(context.Context, K, ...any) (V, error) @@ -35,8 +36,10 @@ func NewMapCache[K comparable, V any](data Cache[K, V], cacheFunc MapSingleFn[K, ex, ok := any(data).(Expend[K, V]) if !ok { r.getCacheBatch = r.getCacheBatchs + r.getCacheBatchToMap = r.getBatchToMapes } else { r.getCacheBatch = r.getBatches(ex) + r.getCacheBatchToMap = r.getBatchToMap(ex) } return r } @@ -139,6 +142,153 @@ func (m *MapCache[K, V]) GetCacheBatch(c context.Context, key []K, timeout time. return m.getCacheBatch(c, key, timeout, params...) } +func (m *MapCache[K, V]) GetBatchToMap(c context.Context, key []K, timeout time.Duration, params ...any) (map[K]V, error) { + return m.getCacheBatchToMap(c, key, timeout, params...) +} +func (m *MapCache[K, V]) getBatchToMap(e Expend[K, V]) func(c context.Context, key []K, timeout time.Duration, params ...any) (map[K]V, error) { + return func(ctx context.Context, key []K, timeout time.Duration, params ...any) (map[K]V, error) { + var res = make(map[K]V) + var needIndex = make(map[K]int) + var err error + mm, err := e.Gets(ctx, key) + if err != nil { + return nil, err + } + var flushKeys []K + for i, k := range key { + v, ok := mm[k] + if !ok { + flushKeys = append(flushKeys, k) + needIndex[k] = i + } else { + res[k] = v + } + } + if len(needIndex) < 1 { + return res, nil + } + + call := func() { + m.mux.Lock() + defer m.mux.Unlock() + mmm, er := e.Gets(ctx, maps.FilterToSlice(needIndex, func(k K, v int) (K, bool) { + return k, true + })) + if er != nil { + err = er + return + } + for k, v := range mmm { + res[k] = v + delete(needIndex, k) + } + + if len(needIndex) < 1 { + return + } + + r, er := m.batchCacheFn(ctx, maps.FilterToSlice(needIndex, func(k K, v int) (K, bool) { + return k, true + }), params...) + if err != nil { + err = er + return + } + e.Sets(ctx, r) + + for k := range needIndex { + v, ok := r[k] + if ok { + res[k] = v + } + } + } + if timeout > 0 { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + done := make(chan struct{}, 1) + go func() { + call() + done <- struct{}{} + }() + select { + case <-ctx.Done(): + err = errors.New(fmt.Sprintf("get cache %v %s", key, ctx.Err().Error())) + return nil, err + case <-done: + } + } else { + call() + } + + return res, err + } +} +func (m *MapCache[K, V]) getBatchToMapes(c context.Context, key []K, timeout time.Duration, params ...any) (r map[K]V, err error) { + r = make(map[K]V) + var needIndex = make(map[K]int) + for i, k := range key { + v, ok := m.Get(c, k) + if !ok { + needIndex[k] = i + } else { + r[k] = v + } + } + if len(needIndex) < 1 { + return + } + + call := func() { + m.mux.Lock() + defer m.mux.Unlock() + needFlushs := maps.FilterToSlice(needIndex, func(k K, v int) (K, bool) { + vv, ok := m.Get(c, k) + if ok { + r[k] = vv + delete(needIndex, k) + return k, false + } + return k, true + }) + + if len(needFlushs) < 1 { + return + } + + rr, er := m.batchCacheFn(c, needFlushs, params...) + if err != nil { + err = er + return + } + for k := range needIndex { + v, ok := rr[k] + if ok { + r[k] = v + } + m.Set(c, k, v) + } + } + if timeout > 0 { + ctx, cancel := context.WithTimeout(c, timeout) + defer cancel() + done := make(chan struct{}, 1) + go func() { + call() + done <- struct{}{} + }() + select { + case <-ctx.Done(): + err = errors.New(fmt.Sprintf("get cache %v %s", key, ctx.Err().Error())) + return nil, err + case <-done: + } + } else { + call() + } + return +} + func (m *MapCache[K, V]) getCacheBatchs(c context.Context, key []K, timeout time.Duration, params ...any) ([]V, error) { var res = make([]V, 0, len(key)) var needIndex = make(map[K]int)