From 768f3df70e2e28c301fed599feec7d0524420371 Mon Sep 17 00:00:00 2001 From: HuangJiaLuo <1820799930@qq.com> Date: Mon, 27 Sep 2021 11:29:47 +0800 Subject: [PATCH 01/12] lru driver-test --- conf/public_conf_test.go | 32 ++++-------- conf/wheat-cache.yaml | 7 +++ doc/LRU设计.md | 2 +- pkg/event/define.go | 1 + pkg/lru/define.go | 9 ++++ pkg/lru/lru.go | 104 +++++++++++++++++++++++++++++++++++++++ pkg/lru/lru_test.go | 20 ++++++++ pkg/lru/worker.go | 47 ++++++++++++++++++ pkg/util/memory.go | 29 +++++++++++ pkg/util/memory_test.go | 10 ++++ 10 files changed, 238 insertions(+), 23 deletions(-) create mode 100644 pkg/lru/lru.go create mode 100644 pkg/lru/lru_test.go create mode 100644 pkg/lru/worker.go create mode 100644 pkg/util/memory.go create mode 100644 pkg/util/memory_test.go diff --git a/conf/public_conf_test.go b/conf/public_conf_test.go index addb4b2..2a4fbfc 100644 --- a/conf/public_conf_test.go +++ b/conf/public_conf_test.go @@ -7,30 +7,18 @@ import ( "github.com/stretchr/testify/require" ) -func TestLoadConf(t *testing.T) { - type args struct { - path string - } - tests := []struct { - name string - args args - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := LoadConf(tt.args.path); (err != nil) != tt.wantErr { - t.Errorf("LoadConf() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } - -} - func TestConf(t *testing.T) { + // 外部导入 conf.yaml 需要导入 conf 包 + + // get 使用, 读取 public_conf 配置文件 + h := viper.Get("storage.host") + require.Equal(t, h, "127.0.0.1") + + h = viper.Get("env") + require.Equal(t, h, "dev") + + // set 使用 viper.Set("host", "1222") host := viper.GetString("host") - require.Equal(t, host, "1222") } diff --git a/conf/wheat-cache.yaml b/conf/wheat-cache.yaml index f204eb2..e650b6d 100644 --- a/conf/wheat-cache.yaml +++ b/conf/wheat-cache.yaml @@ -6,3 +6,10 @@ storage: host: '127.0.0.1' port: 5890 timeOut: 2 # second + +# clearSize and maxSize must be Int +lruCache: + clearSize: "512MB" + maxSize: "1GB" + eventDriverSize: 2000 + workTime: 1 diff --git a/doc/LRU设计.md b/doc/LRU设计.md index 5bc3f26..0cfd567 100644 --- a/doc/LRU设计.md +++ b/doc/LRU设计.md @@ -2,6 +2,6 @@ EventP : 生产事件 -EventQ : 队列事件 +EventQ : 事件队列 Event CP : 清理事件 \ No newline at end of file diff --git a/pkg/event/define.go b/pkg/event/define.go index 1208c3a..529d7e3 100644 --- a/pkg/event/define.go +++ b/pkg/event/define.go @@ -27,3 +27,4 @@ type ProduceInterface interface { type ConsumerInterface interface { Receive(ctx context.Context) *Event } + diff --git a/pkg/lru/define.go b/pkg/lru/define.go index 1c26239..2e7653a 100644 --- a/pkg/lru/define.go +++ b/pkg/lru/define.go @@ -1 +1,10 @@ package lru + +type SingleWorkFunc func() interface{} + +const ( + OpEventName = "operateEvent" + CleEventName = "clearEvent" + + WorkFuncEventCtxKey = "workFunc" +) diff --git a/pkg/lru/lru.go b/pkg/lru/lru.go new file mode 100644 index 0000000..6f4004d --- /dev/null +++ b/pkg/lru/lru.go @@ -0,0 +1,104 @@ +package lru + +import ( + "container/list" + _ "gitee.com/timedb/wheatCache/conf" + "gitee.com/timedb/wheatCache/pkg/errorx" + "gitee.com/timedb/wheatCache/pkg/event" + "gitee.com/timedb/wheatCache/pkg/structure" + "gitee.com/timedb/wheatCache/pkg/util" + "github.com/spf13/viper" + "sync/atomic" +) + +// feat +/* +1. cleanProduce +2. 定义 LRUSingle Work 函数 + +*/ + +type keyBaseValue struct { + key string + val structure.KeyBaseInterface +} + +type singleCache struct { + maxsize int64 //最大的长度 + clearSize int64 // 清理长度 + nowSize int64 // 现在的长度 + li *list.List + lruMap map[string]*list.Element + + lruDriver event.DriverInterface + lruConsumer event.ConsumerInterface + lruProduce event.ProduceInterface // 发送清理事件 +} + +func (lru *singleCache) UpdateLruLength(length int64) { + atomic.AddInt64(&lru.nowSize, length) +} + +// NewLRUCache lru初始化 +func NewLRUCache() *singleCache { + maxSize := viper.GetString("lruCache.maxSize") + clearSize := viper.GetString("lruCache.clearSize") + maxDriver := viper.GetInt("lruCache.eventDriverSize") + lruDriver := event.NewDriver(maxDriver) + + return &singleCache{ + maxsize: util.ParseSizeToBit(maxSize), + clearSize: util.ParseSizeToBit(clearSize), + nowSize: 0, + li: list.New(), + lruMap: make(map[string]*list.Element), + lruDriver: lruDriver, + lruConsumer: event.NewConsumer(lruDriver), + lruProduce: event.NewProduce(lruDriver), + } +} + +// RetDriver 获取驱动 +func (lru *singleCache) RetDriver() event.DriverInterface { + return lru.lruDriver +} + +//Add 增加 +func (lru *singleCache) Add(key string, val structure.KeyBaseInterface) { + + keyBaseVal := &keyBaseValue{ + key: key, + val: val, + } + if elVal, ok := lru.lruMap[key]; ok { + lru.li.MoveToFront(elVal) + elVal.Value = keyBaseVal + return + } + valEl := lru.li.PushFront(keyBaseVal) + lru.lruMap[key] = valEl +} + +// Get 查找key对应的value +func (lru *singleCache) Get(key string) (structure.KeyBaseInterface, bool) { + + if lru.lruMap == nil { + return nil, false + } + if elVal, ok := lru.lruMap[key]; ok { + lru.li.MoveToFront(elVal) + return elVal.Value.(*keyBaseValue).val, true + } + return nil, false +} + +//Del 删除机制 +func (lru *singleCache) Del() error { + if lru.lruMap == nil { + return errorx.New("lru is nil") + } + data := lru.li.Back() + delete(lru.lruMap, data.Value.(*keyBaseValue).key) + lru.li.Remove(data) + return nil +} diff --git a/pkg/lru/lru_test.go b/pkg/lru/lru_test.go new file mode 100644 index 0000000..8529eb6 --- /dev/null +++ b/pkg/lru/lru_test.go @@ -0,0 +1,20 @@ +package lru + +import ( + "fmt" + "gitee.com/timedb/wheatCache/pkg/structure/stringx" + "testing" +) + +func TestNewLRUCache(t *testing.T) { + cache := NewLRUCache() + v1 := stringx.NewStringSingle() + v2 := stringx.NewStringSingle() + v3 := stringx.NewStringSingle() + cache.Add("1", v1) + cache.Add("2", v2) + cache.Add("3", v3) + cache.Add("1", v1) + cache.Del() + fmt.Println(cache.Get("1")) +} \ No newline at end of file diff --git a/pkg/lru/worker.go b/pkg/lru/worker.go new file mode 100644 index 0000000..b2f2f2b --- /dev/null +++ b/pkg/lru/worker.go @@ -0,0 +1,47 @@ +package lru + +import ( + "context" + "gitee.com/timedb/wheatCache/pkg/errorx" + "gitee.com/timedb/wheatCache/pkg/event" + "log" + "time" +) + +func lruCleanWork() { + +} + +func funcWork(work SingleWorkFunc, ttl time.Duration) interface{} { + t := time.NewTimer(ttl) + + resultCh := make(chan interface{}) + go func() { + resultCh <- work() + }() + + select { + case <-t.C: + return errorx.TimeOutErr() + case res := <-resultCh: + return res + } +} + +func (lru *singleCache) lruSingleWork() interface{} { + ctx := context.Background() + for { + event := lru.lruConsumer.Receive(ctx) + + worFunc, ok := event.GetValue(WorkFuncEventCtxKey) + if !ok { + continue + } + + switch worFunc.(type) { + case SingleWorkFunc: + _, err := event.ExecWorkAndSendResult(worFunc.(*event)) + log.Println(err) + } + } +} diff --git a/pkg/util/memory.go b/pkg/util/memory.go new file mode 100644 index 0000000..d2b2504 --- /dev/null +++ b/pkg/util/memory.go @@ -0,0 +1,29 @@ +package util + +import ( + "regexp" + "strconv" + "strings" +) + +// ParseSizeToBit +// 支持MB, GB, KB, 格式 "5KB" 或者 "5kb" +func ParseSizeToBit(size string) int64 { + + sizes := regexp.MustCompile("^\\d+") + sizeRes := sizes.FindAllString(size, 1) + unit := strings.Split(size, sizeRes[0]) + if unit[1] == "KB"|| unit[1] == "kb"{ + Res, _ := strconv.ParseInt(sizeRes[0], 10, 64) + return Res * 1024 + } else if unit[1] == "MB"|| unit[1] == "mb"{ + Res, _ := strconv.ParseInt(sizeRes[0], 10, 64) + return Res * 1024 * 1024 + } else if unit[1] == "GB"|| unit[1] == "fb"{ + Res, _ := strconv.ParseInt(sizeRes[0], 10, 64) + return Res * 1024 *1024 * 1024 + } + return 0 +} + + diff --git a/pkg/util/memory_test.go b/pkg/util/memory_test.go new file mode 100644 index 0000000..4b6adae --- /dev/null +++ b/pkg/util/memory_test.go @@ -0,0 +1,10 @@ +package util + +import ( + "fmt" + "testing" +) + +func TestParseSizeToBit(t *testing.T) { + fmt.Print(ParseSizeToBit("18KB")) +} From 47a49b1fdb7a890034c975527f3013a002c79588 Mon Sep 17 00:00:00 2001 From: HuangJiaLuo <1820799930@qq.com> Date: Sun, 3 Oct 2021 09:47:04 +0800 Subject: [PATCH 02/12] feat(lru): update lru function --- pkg/lru/lru.go | 6 ------ pkg/lru/woker_test.go | 9 +++++++++ pkg/lru/worker.go | 29 ++++++----------------------- 3 files changed, 15 insertions(+), 29 deletions(-) create mode 100644 pkg/lru/woker_test.go diff --git a/pkg/lru/lru.go b/pkg/lru/lru.go index 6f4004d..9a213ec 100644 --- a/pkg/lru/lru.go +++ b/pkg/lru/lru.go @@ -11,12 +11,6 @@ import ( "sync/atomic" ) -// feat -/* -1. cleanProduce -2. 定义 LRUSingle Work 函数 - -*/ type keyBaseValue struct { key string diff --git a/pkg/lru/woker_test.go b/pkg/lru/woker_test.go new file mode 100644 index 0000000..7fa16b0 --- /dev/null +++ b/pkg/lru/woker_test.go @@ -0,0 +1,9 @@ +package lru + +import ( + "testing" +) + +func TestSingleCache_RetDriver(t *testing.T) { + +} diff --git a/pkg/lru/worker.go b/pkg/lru/worker.go index b2f2f2b..285bbcf 100644 --- a/pkg/lru/worker.go +++ b/pkg/lru/worker.go @@ -2,46 +2,29 @@ package lru import ( "context" - "gitee.com/timedb/wheatCache/pkg/errorx" "gitee.com/timedb/wheatCache/pkg/event" "log" - "time" ) func lruCleanWork() { } -func funcWork(work SingleWorkFunc, ttl time.Duration) interface{} { - t := time.NewTimer(ttl) - - resultCh := make(chan interface{}) - go func() { - resultCh <- work() - }() - - select { - case <-t.C: - return errorx.TimeOutErr() - case res := <-resultCh: - return res - } -} - func (lru *singleCache) lruSingleWork() interface{} { ctx := context.Background() for { - event := lru.lruConsumer.Receive(ctx) + eventKon := lru.lruConsumer.Receive(ctx) - worFunc, ok := event.GetValue(WorkFuncEventCtxKey) + worFunc, ok := eventKon.GetValue(WorkFuncEventCtxKey) if !ok { continue } switch worFunc.(type) { - case SingleWorkFunc: - _, err := event.ExecWorkAndSendResult(worFunc.(*event)) - log.Println(err) + case event.EventWorkFunc: + eventKon.ExecWorkAndSendResult(worFunc.(event.EventWorkFunc)) + default: + log.Print("this is debug ") } } } From 96d5a9acc90a366fb7552fc28056deea4ad563fc Mon Sep 17 00:00:00 2001 From: HuangJiaLuo <1820799930@qq.com> Date: Sun, 3 Oct 2021 16:22:33 +0800 Subject: [PATCH 03/12] feat(lru): fix lru function and memory function --- pkg/lru/lru.go | 8 ++++++-- pkg/util/memory.go | 22 +++++++++++----------- pkg/util/memory_test.go | 12 ++++++++++-- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/pkg/lru/lru.go b/pkg/lru/lru.go index 9a213ec..d8703f5 100644 --- a/pkg/lru/lru.go +++ b/pkg/lru/lru.go @@ -33,13 +33,17 @@ func (lru *singleCache) UpdateLruLength(length int64) { atomic.AddInt64(&lru.nowSize, length) } -// NewLRUCache lru初始化 -func NewLRUCache() *singleCache { +func cacheInit() (string, string, event.DriverInterface) { maxSize := viper.GetString("lruCache.maxSize") clearSize := viper.GetString("lruCache.clearSize") maxDriver := viper.GetInt("lruCache.eventDriverSize") lruDriver := event.NewDriver(maxDriver) + return maxSize, clearSize, lruDriver +} +// NewLRUCache lru初始化 +func NewLRUCache() *singleCache { + maxSize, clearSize, lruDriver := cacheInit() return &singleCache{ maxsize: util.ParseSizeToBit(maxSize), clearSize: util.ParseSizeToBit(clearSize), diff --git a/pkg/util/memory.go b/pkg/util/memory.go index d2b2504..c4c32aa 100644 --- a/pkg/util/memory.go +++ b/pkg/util/memory.go @@ -7,23 +7,23 @@ import ( ) // ParseSizeToBit -// 支持MB, GB, KB, 格式 "5KB" 或者 "5kb" +// 支持MB, GB, KB, 格式 "5KB" 或者 "5kb"等等 func ParseSizeToBit(size string) int64 { sizes := regexp.MustCompile("^\\d+") sizeRes := sizes.FindAllString(size, 1) unit := strings.Split(size, sizeRes[0]) - if unit[1] == "KB"|| unit[1] == "kb"{ - Res, _ := strconv.ParseInt(sizeRes[0], 10, 64) - return Res * 1024 - } else if unit[1] == "MB"|| unit[1] == "mb"{ - Res, _ := strconv.ParseInt(sizeRes[0], 10, 64) - return Res * 1024 * 1024 - } else if unit[1] == "GB"|| unit[1] == "fb"{ - Res, _ := strconv.ParseInt(sizeRes[0], 10, 64) - return Res * 1024 *1024 * 1024 + Res, _ := strconv.ParseInt(sizeRes[0], 10, 64) + if strings.ToUpper(unit[1]) == "BIT" || strings.ToUpper(unit[1]) == "B"{ + return Res * 8 + }else if strings.ToUpper(unit[1]) == "KB"{ + return Res * 1024 * 8 + } else if strings.ToUpper(unit[1])=="MB"{ + return Res * 1024 * 1024 * 8 + } else if strings.ToUpper(unit[1]) == "GB"{ + return Res * 1024 *1024 * 1024 * 8 } - return 0 + return -1 } diff --git a/pkg/util/memory_test.go b/pkg/util/memory_test.go index 4b6adae..d7422d0 100644 --- a/pkg/util/memory_test.go +++ b/pkg/util/memory_test.go @@ -1,10 +1,18 @@ package util import ( - "fmt" + "github.com/stretchr/testify/require" "testing" ) func TestParseSizeToBit(t *testing.T) { - fmt.Print(ParseSizeToBit("18KB")) + require.Equal(t, ParseSizeToBit("18Kb"), int64(18*1024*8)) + require.Equal(t, ParseSizeToBit("18KB"), int64(18*1024*8)) + require.Equal(t, ParseSizeToBit("18mB"), int64(18*1024*1024*8)) + require.Equal(t, ParseSizeToBit("18gb"), int64(18*1024*1024*1024*8)) + + require.Equal(t, ParseSizeToBit("18b"), int64(18*8)) + require.Equal(t, ParseSizeToBit("18B"), int64(18*8)) + require.Equal(t, ParseSizeToBit("18bit"), int64(18*8)) + require.Equal(t, ParseSizeToBit("18BIt"), int64(18*8)) } From 99dd878bb398bfcff73c09fe938f4a7557843dbe Mon Sep 17 00:00:00 2001 From: HuangJiaLuo <1820799930@qq.com> Date: Mon, 4 Oct 2021 11:21:02 +0800 Subject: [PATCH 04/12] feat(shell): add init_conf and update makefile --- makefile | 4 ++++ shell/init_conf.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 shell/init_conf.py diff --git a/makefile b/makefile index b61609a..3eda314 100644 --- a/makefile +++ b/makefile @@ -31,3 +31,7 @@ gen-protobuf: .PHONY: gen-middleware gen-middleware: @python3 ./shell/gen_middleware.py + +.PHONY: init-conf +init-conf: + @sudo python3 ./shell/init_conf.py diff --git a/shell/init_conf.py b/shell/init_conf.py new file mode 100644 index 0000000..9058dc2 --- /dev/null +++ b/shell/init_conf.py @@ -0,0 +1,22 @@ +import os +from pathlib import Path +import shutil + +sysPath = os.getcwd() +confInPath = f"{sysPath}/conf/wheat-cache.yaml" +confOutPath = "/etc/wheat-cache" + + +def check_and_make_conf_dir(): + conf_dir = Path(confOutPath) + if not conf_dir.is_dir(): + os.makedirs(confOutPath) + + +def copy_conf(): + shutil.copy(confInPath, confOutPath) + + +if __name__ == '__main__': + check_and_make_conf_dir() + copy_conf() From 29c5aca40d2a91260b5ab4d6d0f4fc32842a949c Mon Sep 17 00:00:00 2001 From: HuangJiaLuo <1820799930@qq.com> Date: Mon, 4 Oct 2021 11:21:55 +0800 Subject: [PATCH 05/12] test(lru): update test --- pkg/lru/lru_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/lru/lru_test.go b/pkg/lru/lru_test.go index 8529eb6..f266192 100644 --- a/pkg/lru/lru_test.go +++ b/pkg/lru/lru_test.go @@ -1,8 +1,8 @@ package lru import ( - "fmt" "gitee.com/timedb/wheatCache/pkg/structure/stringx" + "github.com/stretchr/testify/require" "testing" ) @@ -16,5 +16,6 @@ func TestNewLRUCache(t *testing.T) { cache.Add("3", v3) cache.Add("1", v1) cache.Del() - fmt.Println(cache.Get("1")) + _, isTrue := cache.Get("1") + require.Equal(t, isTrue, true) } \ No newline at end of file From dbc0467de4132b17c45b94b75fde7ccc4b6327f3 Mon Sep 17 00:00:00 2001 From: HuangJiaLuo <1820799930@qq.com> Date: Mon, 4 Oct 2021 16:21:48 +0800 Subject: [PATCH 06/12] fix(util): fix the bug of ParseSizeToBit --- pkg/util/memory.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/util/memory.go b/pkg/util/memory.go index c4c32aa..6b7f64d 100644 --- a/pkg/util/memory.go +++ b/pkg/util/memory.go @@ -1,6 +1,7 @@ package util import ( + "gitee.com/timedb/wheatCache/pkg/errorx" "regexp" "strconv" "strings" @@ -8,7 +9,7 @@ import ( // ParseSizeToBit // 支持MB, GB, KB, 格式 "5KB" 或者 "5kb"等等 -func ParseSizeToBit(size string) int64 { +func ParseSizeToBit(size string) interface{} { sizes := regexp.MustCompile("^\\d+") sizeRes := sizes.FindAllString(size, 1) @@ -23,7 +24,7 @@ func ParseSizeToBit(size string) int64 { } else if strings.ToUpper(unit[1]) == "GB"{ return Res * 1024 *1024 * 1024 * 8 } - return -1 + return errorx.New("your size is wrong") } From 75e1251d2221a9ad307d6c9315fed648d94ae779 Mon Sep 17 00:00:00 2001 From: HuangJiaLuo <1820799930@qq.com> Date: Mon, 4 Oct 2021 16:23:37 +0800 Subject: [PATCH 07/12] feat(lru): feat the init cache --- pkg/lru/lru.go | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/pkg/lru/lru.go b/pkg/lru/lru.go index d8703f5..d2b8d57 100644 --- a/pkg/lru/lru.go +++ b/pkg/lru/lru.go @@ -2,6 +2,7 @@ package lru import ( "container/list" + "fmt" _ "gitee.com/timedb/wheatCache/conf" "gitee.com/timedb/wheatCache/pkg/errorx" "gitee.com/timedb/wheatCache/pkg/event" @@ -11,7 +12,6 @@ import ( "sync/atomic" ) - type keyBaseValue struct { key string val structure.KeyBaseInterface @@ -29,35 +29,46 @@ type singleCache struct { lruProduce event.ProduceInterface // 发送清理事件 } +// UpdateLruLength 更新现在的长度 func (lru *singleCache) UpdateLruLength(length int64) { atomic.AddInt64(&lru.nowSize, length) } func cacheInit() (string, string, event.DriverInterface) { + viper.Set("lruCache.maxSize", "1GB") + viper.Set("lruCache.clearSize", "512MB") + viper.Set("lruCache.eventDriverSize", 2000) maxSize := viper.GetString("lruCache.maxSize") clearSize := viper.GetString("lruCache.clearSize") maxDriver := viper.GetInt("lruCache.eventDriverSize") lruDriver := event.NewDriver(maxDriver) + fmt.Println(clearSize) return maxSize, clearSize, lruDriver } // NewLRUCache lru初始化 func NewLRUCache() *singleCache { - maxSize, clearSize, lruDriver := cacheInit() - return &singleCache{ - maxsize: util.ParseSizeToBit(maxSize), - clearSize: util.ParseSizeToBit(clearSize), - nowSize: 0, - li: list.New(), - lruMap: make(map[string]*list.Element), - lruDriver: lruDriver, - lruConsumer: event.NewConsumer(lruDriver), - lruProduce: event.NewProduce(lruDriver), - } + + lruCacheOnce.Do(func() { + maxSize, clearSize, lruDriver := cacheInit() + lru := &singleCache{ + maxsize: util.ParseSizeToBit(maxSize).(int64), + clearSize: util.ParseSizeToBit(clearSize).(int64), + nowSize: 0, + li: list.New(), + lruMap: make(map[string]*list.Element), + lruDriver: lruDriver, + lruConsumer: event.NewConsumer(lruDriver), + lruProduce: event.NewProduce(lruDriver), + } + lruCache = lru + go lru.lruSingleWork() + }) + return lruCache } -// RetDriver 获取驱动 -func (lru *singleCache) RetDriver() event.DriverInterface { +// GetDriver 获取驱动 +func (lru *singleCache) GetDriver() event.DriverInterface { return lru.lruDriver } From 71ddeb477272854043b45203a89dfdc7c7c53b46 Mon Sep 17 00:00:00 2001 From: HuangJiaLuo <1820799930@qq.com> Date: Mon, 4 Oct 2021 16:25:09 +0800 Subject: [PATCH 08/12] perf(lru): add the sync.Once to define --- pkg/lru/define.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/lru/define.go b/pkg/lru/define.go index 2e7653a..a2da432 100644 --- a/pkg/lru/define.go +++ b/pkg/lru/define.go @@ -1,10 +1,16 @@ package lru +import "sync" + type SingleWorkFunc func() interface{} const ( OpEventName = "operateEvent" CleEventName = "clearEvent" - - WorkFuncEventCtxKey = "workFunc" + WorkFuncEventKey = "workFunc" +) + +var ( + lruCacheOnce sync.Once + lruCache *singleCache ) From ca04ba1686d8e5e5b18336372fbee9fe1ac23063 Mon Sep 17 00:00:00 2001 From: HuangJiaLuo <1820799930@qq.com> Date: Mon, 4 Oct 2021 16:25:52 +0800 Subject: [PATCH 09/12] test(lru): add the test of worker --- pkg/lru/woker_test.go | 80 +++++++++++++++++++++++++++++++++++++++++-- pkg/lru/worker.go | 8 ++--- 2 files changed, 82 insertions(+), 6 deletions(-) diff --git a/pkg/lru/woker_test.go b/pkg/lru/woker_test.go index 7fa16b0..d412b43 100644 --- a/pkg/lru/woker_test.go +++ b/pkg/lru/woker_test.go @@ -1,9 +1,85 @@ package lru import ( + "context" + "gitee.com/timedb/wheatCache/pkg/errorx" + "gitee.com/timedb/wheatCache/pkg/event" + "gitee.com/timedb/wheatCache/pkg/proto" + "gitee.com/timedb/wheatCache/pkg/structure" + "gitee.com/timedb/wheatCache/pkg/structure/stringx" + "github.com/stretchr/testify/require" "testing" + "time" ) -func TestSingleCache_RetDriver(t *testing.T) { - +func TestWorker(t *testing.T) { + ctx := context.Background() + lru := NewLRUCache() + produce := event.NewProduce(lru.GetDriver()) + workEvent := event.NewEvent(OpEventName) + workEvent.SetValue(WorkFuncEventKey, event.EventWorkFunc(func() (interface{}, error) { + v1 := stringx.NewStringSingle() + key := "v1" + res, err := v1.Set(&proto.SetRequest{ + Val: "123", + }) + if err != nil { + return nil, err + } + lru.Add(key, v1) + return res.Result, nil + })) + workEvent.InitWaitEvent() + produce.Call(ctx,workEvent) + res, err := workEvent.StartWaitEvent(2 * time.Second) + require.NoError(t, err) + require.Equal(t, res, "123") + + workEvent.InitWaitEvent() + workEvent.SetValue(WorkFuncEventKey, event.EventWorkFunc(func() (interface{}, error) { + v2, ok := lru.Get("v1") + if !ok{ + return nil, errorx.New("no this key") + } + switch v2.(type) { + case structure.StringXInterface: + res, err := v2.(structure.StringXInterface).Get(&proto.GetRequest{ + }) + if err != nil { + return nil, err + } + return res.Result, nil + default: + return nil, errorx.New("no this type") + } + })) + produce.Call(ctx, workEvent) + res, err = workEvent.StartWaitEvent(2 * time.Second) + require.NoError(t, err) + require.Equal(t, res, "123") + + workEvent.InitWaitEvent() + workEvent.SetValue(WorkFuncEventKey, event.EventWorkFunc(func() (interface{}, error) { + lru.Del() + v2, ok := lru.Get("v1") + if !ok{ + return nil, nil + } + switch v2.(type) { + case structure.StringXInterface: + res, err := v2.(structure.StringXInterface).Get(&proto.GetRequest{ + }) + if err != nil { + return nil, err + } + return res.Result, nil + default: + return nil, errorx.New("no this type") + } + })) + produce.Call(ctx, workEvent) + res, err = workEvent.StartWaitEvent(2 * time.Second) + require.Equal(t, err, nil) + require.Equal(t, res, nil) + } diff --git a/pkg/lru/worker.go b/pkg/lru/worker.go index 285bbcf..2da7e4e 100644 --- a/pkg/lru/worker.go +++ b/pkg/lru/worker.go @@ -13,16 +13,16 @@ func lruCleanWork() { func (lru *singleCache) lruSingleWork() interface{} { ctx := context.Background() for { - eventKon := lru.lruConsumer.Receive(ctx) + workEvent := lru.lruConsumer.Receive(ctx) - worFunc, ok := eventKon.GetValue(WorkFuncEventCtxKey) + workFunc, ok := workEvent.GetValue(WorkFuncEventKey) if !ok { continue } - switch worFunc.(type) { + switch workFunc.(type) { case event.EventWorkFunc: - eventKon.ExecWorkAndSendResult(worFunc.(event.EventWorkFunc)) + workEvent.ExecWorkAndSendResult(workFunc.(event.EventWorkFunc)) default: log.Print("this is debug ") } From 477adec1bb48edb93d0a7f2df00f387c4a5b7907 Mon Sep 17 00:00:00 2001 From: HuangJiaLuo <1820799930@qq.com> Date: Mon, 4 Oct 2021 20:32:20 +0800 Subject: [PATCH 10/12] fix(lru): fix the bug of lru --- pkg/lru/define.go | 10 +++++++-- pkg/lru/lru.go | 50 ++++++++++++++++++++++++++++++--------------- pkg/lru/lru_test.go | 3 +++ 3 files changed, 45 insertions(+), 18 deletions(-) diff --git a/pkg/lru/define.go b/pkg/lru/define.go index a2da432..84d56d1 100644 --- a/pkg/lru/define.go +++ b/pkg/lru/define.go @@ -5,8 +5,8 @@ import "sync" type SingleWorkFunc func() interface{} const ( - OpEventName = "operateEvent" - CleEventName = "clearEvent" + OptionEventName = "operateEvent" + CleanEventName = "clearEvent" WorkFuncEventKey = "workFunc" ) @@ -14,3 +14,9 @@ var ( lruCacheOnce sync.Once lruCache *singleCache ) + +const ( + lruMaxSize = 1*1024*1024*1024*8 + lruClearSize = 0.5*1024*1024*1024*8 + lruEventDriver = 2000 +) diff --git a/pkg/lru/lru.go b/pkg/lru/lru.go index d2b8d57..24522d2 100644 --- a/pkg/lru/lru.go +++ b/pkg/lru/lru.go @@ -2,7 +2,6 @@ package lru import ( "container/list" - "fmt" _ "gitee.com/timedb/wheatCache/conf" "gitee.com/timedb/wheatCache/pkg/errorx" "gitee.com/timedb/wheatCache/pkg/event" @@ -26,40 +25,55 @@ type singleCache struct { lruDriver event.DriverInterface lruConsumer event.ConsumerInterface - lruProduce event.ProduceInterface // 发送清理事件 + lruCleanProduce event.ProduceInterface // 发送清理事件 } -// UpdateLruLength 更新现在的长度 -func (lru *singleCache) UpdateLruLength(length int64) { +// UpdateLruSize 更新现在的长度 +func (lru *singleCache) UpdateLruSize(length int64) { atomic.AddInt64(&lru.nowSize, length) } -func cacheInit() (string, string, event.DriverInterface) { - viper.Set("lruCache.maxSize", "1GB") - viper.Set("lruCache.clearSize", "512MB") - viper.Set("lruCache.eventDriverSize", 2000) +func cacheInit() (int64, int64, event.DriverInterface) { maxSize := viper.GetString("lruCache.maxSize") + retMaxSize, maxErr:= util.ParseSizeToBit(maxSize) + if maxErr != nil{ + return 0, 0, nil + } + if retMaxSize == 0{ + retMaxSize = lruMaxSize + } + clearSize := viper.GetString("lruCache.clearSize") + retClearSize, clearErr := util.ParseSizeToBit(clearSize) + if clearErr != nil{ + return 0, 0, nil + } + if retClearSize == 0{ + retClearSize = lruClearSize + } + maxDriver := viper.GetInt("lruCache.eventDriverSize") + if maxDriver == 0{ + maxDriver = lruEventDriver + } lruDriver := event.NewDriver(maxDriver) - fmt.Println(clearSize) - return maxSize, clearSize, lruDriver + return retMaxSize, retClearSize, lruDriver } // NewLRUCache lru初始化 func NewLRUCache() *singleCache { - + maxSize, clearSize, lruDrivers := cacheInit() lruCacheOnce.Do(func() { - maxSize, clearSize, lruDriver := cacheInit() + _, _, lruDriver := cacheInit() lru := &singleCache{ - maxsize: util.ParseSizeToBit(maxSize).(int64), - clearSize: util.ParseSizeToBit(clearSize).(int64), + maxsize: maxSize, + clearSize: clearSize, nowSize: 0, li: list.New(), lruMap: make(map[string]*list.Element), lruDriver: lruDriver, - lruConsumer: event.NewConsumer(lruDriver), - lruProduce: event.NewProduce(lruDriver), + lruConsumer: event.NewConsumer(lruDrivers), + lruCleanProduce: event.NewProduce(lruDrivers), } lruCache = lru go lru.lruSingleWork() @@ -86,6 +100,8 @@ func (lru *singleCache) Add(key string, val structure.KeyBaseInterface) { } valEl := lru.li.PushFront(keyBaseVal) lru.lruMap[key] = valEl + //增加大小 + lru.UpdateLruSize(valEl.Value.(*keyBaseValue).val.SizeByte()) } // Get 查找key对应的value @@ -108,6 +124,8 @@ func (lru *singleCache) Del() error { } data := lru.li.Back() delete(lru.lruMap, data.Value.(*keyBaseValue).key) + //删除大小 + lru.UpdateLruSize(-1 * data.Value.(*keyBaseValue).val.SizeByte()) lru.li.Remove(data) return nil } diff --git a/pkg/lru/lru_test.go b/pkg/lru/lru_test.go index f266192..7cba185 100644 --- a/pkg/lru/lru_test.go +++ b/pkg/lru/lru_test.go @@ -1,6 +1,7 @@ package lru import ( + "fmt" "gitee.com/timedb/wheatCache/pkg/structure/stringx" "github.com/stretchr/testify/require" "testing" @@ -15,7 +16,9 @@ func TestNewLRUCache(t *testing.T) { cache.Add("2", v2) cache.Add("3", v3) cache.Add("1", v1) + fmt.Println(cache.nowSize) cache.Del() + fmt.Println(cache.nowSize) _, isTrue := cache.Get("1") require.Equal(t, isTrue, true) } \ No newline at end of file From 1774c9f230f18b7544116b0f98e37ea6c2c00cf8 Mon Sep 17 00:00:00 2001 From: HuangJiaLuo <1820799930@qq.com> Date: Mon, 4 Oct 2021 20:32:55 +0800 Subject: [PATCH 11/12] feat(makefile): add init-conf --- makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile b/makefile index 3eda314..7558082 100644 --- a/makefile +++ b/makefile @@ -34,4 +34,4 @@ gen-middleware: .PHONY: init-conf init-conf: - @sudo python3 ./shell/init_conf.py + @python3 ./shell/init_conf.py From b95fb85ad6729a29665f599cf63301ba146c456c Mon Sep 17 00:00:00 2001 From: HuangJiaLuo <1820799930@qq.com> Date: Mon, 4 Oct 2021 20:33:38 +0800 Subject: [PATCH 12/12] perf(util): use switch to change size --- pkg/lru/woker_test.go | 2 +- pkg/util/memory.go | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pkg/lru/woker_test.go b/pkg/lru/woker_test.go index d412b43..b278f20 100644 --- a/pkg/lru/woker_test.go +++ b/pkg/lru/woker_test.go @@ -16,7 +16,7 @@ func TestWorker(t *testing.T) { ctx := context.Background() lru := NewLRUCache() produce := event.NewProduce(lru.GetDriver()) - workEvent := event.NewEvent(OpEventName) + workEvent := event.NewEvent(OptionEventName) workEvent.SetValue(WorkFuncEventKey, event.EventWorkFunc(func() (interface{}, error) { v1 := stringx.NewStringSingle() key := "v1" diff --git a/pkg/util/memory.go b/pkg/util/memory.go index 6b7f64d..c0f5ef3 100644 --- a/pkg/util/memory.go +++ b/pkg/util/memory.go @@ -9,22 +9,24 @@ import ( // ParseSizeToBit // 支持MB, GB, KB, 格式 "5KB" 或者 "5kb"等等 -func ParseSizeToBit(size string) interface{} { - +func ParseSizeToBit(size string) (int64, error) { sizes := regexp.MustCompile("^\\d+") sizeRes := sizes.FindAllString(size, 1) unit := strings.Split(size, sizeRes[0]) Res, _ := strconv.ParseInt(sizeRes[0], 10, 64) - if strings.ToUpper(unit[1]) == "BIT" || strings.ToUpper(unit[1]) == "B"{ - return Res * 8 - }else if strings.ToUpper(unit[1]) == "KB"{ - return Res * 1024 * 8 - } else if strings.ToUpper(unit[1])=="MB"{ - return Res * 1024 * 1024 * 8 - } else if strings.ToUpper(unit[1]) == "GB"{ - return Res * 1024 *1024 * 1024 * 8 + sizeType := strings.ToUpper(unit[1]) + switch { + case sizeType == "BIT" || sizeType == "B": + return Res * 8, nil + case sizeType == "KB": + return Res * 1024 * 8, nil + case sizeType =="MB": + return Res * 1024 * 1024 * 8, nil + case sizeType == "GB": + return Res * 1024 *1024 * 1024 * 8, nil + default: + return 0, errorx.New("your size is wrong") } - return errorx.New("your size is wrong") }