From 768f3df70e2e28c301fed599feec7d0524420371 Mon Sep 17 00:00:00 2001 From: HuangJiaLuo <1820799930@qq.com> Date: Mon, 27 Sep 2021 11:29:47 +0800 Subject: [PATCH] lru driver-test --- conf/public_conf_test.go | 32 ++++-------- conf/wheat-cache.yaml | 7 +++ doc/LRU设计.md | 2 +- pkg/event/define.go | 1 + pkg/lru/define.go | 9 ++++ pkg/lru/lru.go | 104 +++++++++++++++++++++++++++++++++++++++ pkg/lru/lru_test.go | 20 ++++++++ pkg/lru/worker.go | 47 ++++++++++++++++++ pkg/util/memory.go | 29 +++++++++++ pkg/util/memory_test.go | 10 ++++ 10 files changed, 238 insertions(+), 23 deletions(-) create mode 100644 pkg/lru/lru.go create mode 100644 pkg/lru/lru_test.go create mode 100644 pkg/lru/worker.go create mode 100644 pkg/util/memory.go create mode 100644 pkg/util/memory_test.go diff --git a/conf/public_conf_test.go b/conf/public_conf_test.go index addb4b2..2a4fbfc 100644 --- a/conf/public_conf_test.go +++ b/conf/public_conf_test.go @@ -7,30 +7,18 @@ import ( "github.com/stretchr/testify/require" ) -func TestLoadConf(t *testing.T) { - type args struct { - path string - } - tests := []struct { - name string - args args - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := LoadConf(tt.args.path); (err != nil) != tt.wantErr { - t.Errorf("LoadConf() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } - -} - func TestConf(t *testing.T) { + // 外部导入 conf.yaml 需要导入 conf 包 + + // get 使用, 读取 public_conf 配置文件 + h := viper.Get("storage.host") + require.Equal(t, h, "127.0.0.1") + + h = viper.Get("env") + require.Equal(t, h, "dev") + + // set 使用 viper.Set("host", "1222") host := viper.GetString("host") - require.Equal(t, host, "1222") } diff --git a/conf/wheat-cache.yaml b/conf/wheat-cache.yaml index f204eb2..e650b6d 100644 --- a/conf/wheat-cache.yaml +++ b/conf/wheat-cache.yaml @@ -6,3 +6,10 @@ storage: host: '127.0.0.1' port: 5890 timeOut: 2 # second + +# clearSize and maxSize must be Int +lruCache: + clearSize: "512MB" + maxSize: "1GB" + eventDriverSize: 2000 + workTime: 1 diff --git a/doc/LRU设计.md b/doc/LRU设计.md index 5bc3f26..0cfd567 100644 --- a/doc/LRU设计.md +++ b/doc/LRU设计.md @@ -2,6 +2,6 @@ EventP : 生产事件 -EventQ : 队列事件 +EventQ : 事件队列 Event CP : 清理事件 \ No newline at end of file diff --git a/pkg/event/define.go b/pkg/event/define.go index 1208c3a..529d7e3 100644 --- a/pkg/event/define.go +++ b/pkg/event/define.go @@ -27,3 +27,4 @@ type ProduceInterface interface { type ConsumerInterface interface { Receive(ctx context.Context) *Event } + diff --git a/pkg/lru/define.go b/pkg/lru/define.go index 1c26239..2e7653a 100644 --- a/pkg/lru/define.go +++ b/pkg/lru/define.go @@ -1 +1,10 @@ package lru + +type SingleWorkFunc func() interface{} + +const ( + OpEventName = "operateEvent" + CleEventName = "clearEvent" + + WorkFuncEventCtxKey = "workFunc" +) diff --git a/pkg/lru/lru.go b/pkg/lru/lru.go new file mode 100644 index 0000000..6f4004d --- /dev/null +++ b/pkg/lru/lru.go @@ -0,0 +1,104 @@ +package lru + +import ( + "container/list" + _ "gitee.com/timedb/wheatCache/conf" + "gitee.com/timedb/wheatCache/pkg/errorx" + "gitee.com/timedb/wheatCache/pkg/event" + "gitee.com/timedb/wheatCache/pkg/structure" + "gitee.com/timedb/wheatCache/pkg/util" + "github.com/spf13/viper" + "sync/atomic" +) + +// feat +/* +1. cleanProduce +2. 定义 LRUSingle Work 函数 + +*/ + +type keyBaseValue struct { + key string + val structure.KeyBaseInterface +} + +type singleCache struct { + maxsize int64 //最大的长度 + clearSize int64 // 清理长度 + nowSize int64 // 现在的长度 + li *list.List + lruMap map[string]*list.Element + + lruDriver event.DriverInterface + lruConsumer event.ConsumerInterface + lruProduce event.ProduceInterface // 发送清理事件 +} + +func (lru *singleCache) UpdateLruLength(length int64) { + atomic.AddInt64(&lru.nowSize, length) +} + +// NewLRUCache lru初始化 +func NewLRUCache() *singleCache { + maxSize := viper.GetString("lruCache.maxSize") + clearSize := viper.GetString("lruCache.clearSize") + maxDriver := viper.GetInt("lruCache.eventDriverSize") + lruDriver := event.NewDriver(maxDriver) + + return &singleCache{ + maxsize: util.ParseSizeToBit(maxSize), + clearSize: util.ParseSizeToBit(clearSize), + nowSize: 0, + li: list.New(), + lruMap: make(map[string]*list.Element), + lruDriver: lruDriver, + lruConsumer: event.NewConsumer(lruDriver), + lruProduce: event.NewProduce(lruDriver), + } +} + +// RetDriver 获取驱动 +func (lru *singleCache) RetDriver() event.DriverInterface { + return lru.lruDriver +} + +//Add 增加 +func (lru *singleCache) Add(key string, val structure.KeyBaseInterface) { + + keyBaseVal := &keyBaseValue{ + key: key, + val: val, + } + if elVal, ok := lru.lruMap[key]; ok { + lru.li.MoveToFront(elVal) + elVal.Value = keyBaseVal + return + } + valEl := lru.li.PushFront(keyBaseVal) + lru.lruMap[key] = valEl +} + +// Get 查找key对应的value +func (lru *singleCache) Get(key string) (structure.KeyBaseInterface, bool) { + + if lru.lruMap == nil { + return nil, false + } + if elVal, ok := lru.lruMap[key]; ok { + lru.li.MoveToFront(elVal) + return elVal.Value.(*keyBaseValue).val, true + } + return nil, false +} + +//Del 删除机制 +func (lru *singleCache) Del() error { + if lru.lruMap == nil { + return errorx.New("lru is nil") + } + data := lru.li.Back() + delete(lru.lruMap, data.Value.(*keyBaseValue).key) + lru.li.Remove(data) + return nil +} diff --git a/pkg/lru/lru_test.go b/pkg/lru/lru_test.go new file mode 100644 index 0000000..8529eb6 --- /dev/null +++ b/pkg/lru/lru_test.go @@ -0,0 +1,20 @@ +package lru + +import ( + "fmt" + "gitee.com/timedb/wheatCache/pkg/structure/stringx" + "testing" +) + +func TestNewLRUCache(t *testing.T) { + cache := NewLRUCache() + v1 := stringx.NewStringSingle() + v2 := stringx.NewStringSingle() + v3 := stringx.NewStringSingle() + cache.Add("1", v1) + cache.Add("2", v2) + cache.Add("3", v3) + cache.Add("1", v1) + cache.Del() + fmt.Println(cache.Get("1")) +} \ No newline at end of file diff --git a/pkg/lru/worker.go b/pkg/lru/worker.go new file mode 100644 index 0000000..b2f2f2b --- /dev/null +++ b/pkg/lru/worker.go @@ -0,0 +1,47 @@ +package lru + +import ( + "context" + "gitee.com/timedb/wheatCache/pkg/errorx" + "gitee.com/timedb/wheatCache/pkg/event" + "log" + "time" +) + +func lruCleanWork() { + +} + +func funcWork(work SingleWorkFunc, ttl time.Duration) interface{} { + t := time.NewTimer(ttl) + + resultCh := make(chan interface{}) + go func() { + resultCh <- work() + }() + + select { + case <-t.C: + return errorx.TimeOutErr() + case res := <-resultCh: + return res + } +} + +func (lru *singleCache) lruSingleWork() interface{} { + ctx := context.Background() + for { + event := lru.lruConsumer.Receive(ctx) + + worFunc, ok := event.GetValue(WorkFuncEventCtxKey) + if !ok { + continue + } + + switch worFunc.(type) { + case SingleWorkFunc: + _, err := event.ExecWorkAndSendResult(worFunc.(*event)) + log.Println(err) + } + } +} diff --git a/pkg/util/memory.go b/pkg/util/memory.go new file mode 100644 index 0000000..d2b2504 --- /dev/null +++ b/pkg/util/memory.go @@ -0,0 +1,29 @@ +package util + +import ( + "regexp" + "strconv" + "strings" +) + +// ParseSizeToBit +// 支持MB, GB, KB, 格式 "5KB" 或者 "5kb" +func ParseSizeToBit(size string) int64 { + + sizes := regexp.MustCompile("^\\d+") + sizeRes := sizes.FindAllString(size, 1) + unit := strings.Split(size, sizeRes[0]) + if unit[1] == "KB"|| unit[1] == "kb"{ + Res, _ := strconv.ParseInt(sizeRes[0], 10, 64) + return Res * 1024 + } else if unit[1] == "MB"|| unit[1] == "mb"{ + Res, _ := strconv.ParseInt(sizeRes[0], 10, 64) + return Res * 1024 * 1024 + } else if unit[1] == "GB"|| unit[1] == "fb"{ + Res, _ := strconv.ParseInt(sizeRes[0], 10, 64) + return Res * 1024 *1024 * 1024 + } + return 0 +} + + diff --git a/pkg/util/memory_test.go b/pkg/util/memory_test.go new file mode 100644 index 0000000..4b6adae --- /dev/null +++ b/pkg/util/memory_test.go @@ -0,0 +1,10 @@ +package util + +import ( + "fmt" + "testing" +) + +func TestParseSizeToBit(t *testing.T) { + fmt.Print(ParseSizeToBit("18KB")) +}