From ac8f7483566168ca8bcbb28632f25f9c8c1f0189 Mon Sep 17 00:00:00 2001 From: bandl <1658002533@qq.com> Date: Mon, 18 Oct 2021 23:58:11 +0800 Subject: [PATCH 1/4] perf(skiplist): perf list pop left, pop range --- pkg/util/skiplist/skiplist.go | 122 ++++++++++++++++++++++++++++------ 1 file changed, 101 insertions(+), 21 deletions(-) diff --git a/pkg/util/skiplist/skiplist.go b/pkg/util/skiplist/skiplist.go index 391f559..34aaeb2 100644 --- a/pkg/util/skiplist/skiplist.go +++ b/pkg/util/skiplist/skiplist.go @@ -166,40 +166,120 @@ func (s *SkipList) RemoveAll(score float64) { s.delSkipListNode(delNode...) } -// ClearLeft 删除 小等于 score 的全部节点 -func (s *SkipList) ClearLeft(score float64) *skipListNode { +func (s *SkipList) PopLeft(score float64) []interface{} { + endPoint := s.tail + head := s.head - for head.down != nil { - p := head - for p.score <= score { - p = p.next + for endPoint.score > score { + endPoint = endPoint.pre + } + + var valuePoint *skipListNode + for endPoint != nil { + // 检查 end 后的节点情况 + for endPoint.next.score <= score { + endPoint = endPoint.next } - head.next = p - p.pre = head + head.next = endPoint.next + endPoint.next.pre = head + valuePoint = endPoint + + endPoint = endPoint.down head = head.down } - node := s.searchNode(score) - for node.score <= score { - node = node.next + values := make([]interface{}, 0, 100) + for valuePoint != nil && valuePoint.score > s.head.score { + values = append(values, valuePoint.val) + valuePoint = valuePoint.pre } - oldNode := node.pre - head.next = node + return values - node.pre = head - return oldNode } -func (s *SkipList) PopLeft(score float64) []interface{} { - node := s.ClearLeft(score) - values := make([]interface{}, 0, 10) - for node.score != math.MinInt64 { - values = append(values, node.val) - node = node.pre +func (s *SkipList) PopRight(score float64) []interface{} { + startPoint := s.head + + tail := s.tail + + for startPoint.score < score { + startPoint = startPoint.next } + + var valuePoint *skipListNode + for startPoint != nil { + // 检查 end 后的节点情况 + for startPoint.pre.score >= score { + startPoint = startPoint.pre + } + + tail.pre = startPoint.pre + startPoint.pre.next = tail + valuePoint = startPoint + + startPoint = startPoint.down + tail = tail.down + } + + values := make([]interface{}, 0, 100) + for valuePoint != nil && valuePoint.score < s.tail.score { + values = append(values, valuePoint.val) + valuePoint = valuePoint.next + } + + return values + +} + +// PopRangeByScore 删除包括 start 到 end 的 score 的全部 value +func (s *SkipList) PopRangeByScore(startScore float64, endScore float64) []interface{} { + if startScore >= endScore { + return nil + } + + startPoint := s.head + endPoint := s.tail + + // 定位删除索引最高级 + for startPoint.score < startScore { + startPoint = startPoint.next + } + for endPoint.score > endScore { + endPoint = endPoint.pre + } + + var valuePoint *skipListNode + + for startPoint != nil { + // 检查 start 前的节点情况 + for startPoint.pre.score >= startScore { + startPoint = startPoint.pre + } + + // 检查 end 后的节点情况 + for endPoint.next.score <= endScore { + endPoint = endPoint.next + } + + // 删除节点 + valuePoint = startPoint + startPoint.pre.next = endPoint.next + endPoint.next.pre = startPoint.pre + + // 下一级处理 + startPoint = startPoint.down + endPoint = endPoint.down + } + + values := make([]interface{}, 0, 100) + for valuePoint != nil && valuePoint.score <= endScore { + values = append(values, valuePoint.val) + valuePoint = valuePoint.next + } + return values } From 95d3fdbd171208e0837bc7e280de9b942ea81f6c Mon Sep 17 00:00:00 2001 From: bandl <1658002533@qq.com> Date: Mon, 18 Oct 2021 23:58:46 +0800 Subject: [PATCH 2/4] test(skiplist, ttl): update ttl --- pkg/lru/ttl_test.go | 5 ++-- pkg/lru/woker_test.go | 7 ++--- pkg/util/skiplist/skiplist_test.go | 41 ++++++++++++++++++++++++++++-- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/pkg/lru/ttl_test.go b/pkg/lru/ttl_test.go index eecfcd6..76b8d15 100644 --- a/pkg/lru/ttl_test.go +++ b/pkg/lru/ttl_test.go @@ -2,11 +2,12 @@ package lru import ( "fmt" + "testing" + "time" + "gitee.com/timedb/wheatCache/pkg/proto" "gitee.com/timedb/wheatCache/pkg/structure/stringx" "github.com/stretchr/testify/require" - "testing" - "time" ) func TestTTlCup(t *testing.T) { diff --git a/pkg/lru/woker_test.go b/pkg/lru/woker_test.go index 52ad96d..6009258 100644 --- a/pkg/lru/woker_test.go +++ b/pkg/lru/woker_test.go @@ -2,13 +2,14 @@ package lru import ( "context" + "testing" + "time" + "gitee.com/timedb/wheatCache/pkg/event" "gitee.com/timedb/wheatCache/pkg/logx" "gitee.com/timedb/wheatCache/pkg/proto" "gitee.com/timedb/wheatCache/pkg/structure/stringx" "github.com/stretchr/testify/require" - "testing" - "time" ) func TestWorker(t *testing.T) { @@ -53,6 +54,6 @@ func TestSingleCache_DelToClearSize(t *testing.T) { workEvent.StartWaitEvent(2 * time.Second) } - time.Sleep(5<<10 * time.Second) + time.Sleep(5 * time.Second) logx.Info("end size is %d", lru.nowSize) } diff --git a/pkg/util/skiplist/skiplist_test.go b/pkg/util/skiplist/skiplist_test.go index 7e28e7b..a200363 100644 --- a/pkg/util/skiplist/skiplist_test.go +++ b/pkg/util/skiplist/skiplist_test.go @@ -2,11 +2,12 @@ package skiplist import ( "fmt" - "gitee.com/timedb/wheatCache/pkg/logx" - "github.com/stretchr/testify/require" "math/rand" "testing" "time" + + "gitee.com/timedb/wheatCache/pkg/logx" + "github.com/stretchr/testify/require" ) // 时间测试 @@ -66,3 +67,39 @@ func Test_skipList_ClearLeft(t *testing.T) { require.Equal(t, s.Get(20), nil) } + +func TestSkipList_PopRangeByScore(t *testing.T) { + startNum := 5 + endNum := 79 + result := make([]interface{}, 0) + for i := startNum; i <= endNum; i++ { + result = append(result, i) + } + + for i := 0; i < 100; i++ { + s := NewSkipList(14) + for i := 0; i < 100; i++ { + s.Insert(float64(i), i) + } + res := s.PopRangeByScore(float64(startNum), float64(endNum)) + require.Equal(t, res, result) + } +} + +func TestSkipList_PopRight(t *testing.T) { + score := 33 + result := make([]interface{}, 0) + for i := score; i < 100; i++ { + result = append(result, i) + } + + for i := 0; i < 100; i++ { + s := NewSkipList(14) + for i := 0; i < 100; i++ { + s.Insert(float64(i), i) + } + res := s.PopRight(float64(score)) + require.Equal(t, res, result) + s.debugPrint() + } +} From d17f8e243c6e66fe464c5e206aaaf8ae476ca24b Mon Sep 17 00:00:00 2001 From: bandl <1658002533@qq.com> Date: Tue, 19 Oct 2021 00:24:16 +0800 Subject: [PATCH 3/4] perf(lru): perf lru keybase err --- pkg/errorx/lru.go | 4 ++++ pkg/lru/define.go | 2 +- pkg/lru/lru.go | 22 ++++++++++++++++++++-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/pkg/errorx/lru.go b/pkg/errorx/lru.go index 569e563..95952eb 100644 --- a/pkg/errorx/lru.go +++ b/pkg/errorx/lru.go @@ -3,3 +3,7 @@ package errorx func LruNotWorkFuncEventErr() error { return New("the event haven't work of function") } + +func KeyBaseIsNilErr() error { + return New("key base not is nil") +} diff --git a/pkg/lru/define.go b/pkg/lru/define.go index 6c8ede4..c5cedb2 100644 --- a/pkg/lru/define.go +++ b/pkg/lru/define.go @@ -33,7 +33,7 @@ const ( type CacheInterface interface { Del() error Get(key *proto.BaseKey) (structure.KeyBaseInterface, bool) - Add(key *proto.BaseKey, val structure.KeyBaseInterface) + Add(key *proto.BaseKey, val structure.KeyBaseInterface) error UpdateLruSize(length structure.UpdateLength) DelByKey(key *proto.BaseKey) error DelToClearSize() error diff --git a/pkg/lru/lru.go b/pkg/lru/lru.go index a8682ad..cb5c9b0 100644 --- a/pkg/lru/lru.go +++ b/pkg/lru/lru.go @@ -108,7 +108,11 @@ func (lru *SingleCache) GetDriver() event.DriverInterface { } //Add 增加 -func (lru *SingleCache) Add(key *proto.BaseKey, val structure.KeyBaseInterface) { +func (lru *SingleCache) Add(key *proto.BaseKey, val structure.KeyBaseInterface) error { + + if key == nil { + return errorx.KeyBaseIsNilErr() + } exp := lru.lruTtlManage.setKeys(key) keyBaseVal := &keyBaseValue{ @@ -119,17 +123,22 @@ func (lru *SingleCache) Add(key *proto.BaseKey, val structure.KeyBaseInterface) if elVal, ok := lru.lruMap[key.Key]; ok { lru.li.MoveToFront(elVal) elVal.Value = keyBaseVal - return + return nil } valEl := lru.li.PushFront(keyBaseVal) lru.lruMap[key.Key] = valEl //增加大小 lru.UpdateLruSize(structure.UpdateLength(valEl.Value.(*keyBaseValue).val.SizeByte())) + return nil } // Get 查找key对应的value func (lru *SingleCache) Get(key *proto.BaseKey) (structure.KeyBaseInterface, bool) { + if key == nil { + return nil, false + } + if elVal, ok := lru.lruMap[key.Key]; ok { lru.li.MoveToFront(elVal) return elVal.Value.(*keyBaseValue).val, true @@ -152,6 +161,11 @@ func (lru *SingleCache) Del() error { //DelByKey 根据key删除 func (lru *SingleCache) DelByKey(key *proto.BaseKey) error { + + if key == nil { + return errorx.KeyBaseIsNilErr() + } + if lru.lruMap == nil { return errorx.New("lru is nil") } @@ -194,6 +208,10 @@ func (lru *SingleCache) DelToClearSize() error { // 更新过期时间 func (lru *SingleCache) UpdateTTl(key *proto.BaseKey) error { + if key == nil { + return errorx.KeyBaseIsNilErr() + } + if elVal, ok := lru.lruMap[key.Key]; ok { expire := lru.lruTtlManage.setKeys(key) elVal.Value.(*keyBaseValue).expire = expire From 21cac4639cad38b2eecadd4537c5a9ad3280a57d Mon Sep 17 00:00:00 2001 From: bandl <1658002533@qq.com> Date: Tue, 19 Oct 2021 00:24:36 +0800 Subject: [PATCH 4/4] fear(dao): update dao api --- pkg/structure/interface.gen.go | 10 +++++----- storage/dao/dao.go | 8 +++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/structure/interface.gen.go b/pkg/structure/interface.gen.go index 6ed2c1e..b9b806a 100644 --- a/pkg/structure/interface.gen.go +++ b/pkg/structure/interface.gen.go @@ -15,10 +15,10 @@ type KeyBaseInterface interface { type StringXInterface interface { KeyBaseInterface - Set(val string) (string, UpdateLength) + Set(string) (string, UpdateLength) Get() string - Add(renewal int32) (string, error) - Reduce(renewal int32) (string, error) - Setbit(offer int32, val bool) UpdateLength - Getbit(offer int32) (bool, error) + Add(int32) (string, error) + Reduce(int32) (string, error) + Setbit(int32, bool) UpdateLength + Getbit(int32) (bool, error) } diff --git a/storage/dao/dao.go b/storage/dao/dao.go index 8cd9b04..d061e5e 100644 --- a/storage/dao/dao.go +++ b/storage/dao/dao.go @@ -32,9 +32,11 @@ func (d *Dao) Set(key *proto.BaseKey, strVal string) (string, error) { // 不存在新建 strValue := stringx.NewStringSingle() - result, length := strValue.Set(strVal) - d.lru.Add(key, strValue) - d.lru.UpdateLruSize(length) + result, _ := strValue.Set(strVal) + err := d.lru.Add(key, strValue) + if err != nil { + return "", err + } return result, nil }