!70 优化 lru

Merge pull request !70 from bandl/perf-lru

pkg/errorx/lru.go

@@ -3,3 +3,7 @@ package errorx
 func LruNotWorkFuncEventErr() error {
 	return New("the event haven't work of function")
 }
+
+func KeyBaseIsNilErr() error {
+	return New("key base not is nil")
+}

pkg/lru/define.go

@@ -33,7 +33,7 @@ const (
 type CacheInterface interface {
 	Del() error
 	Get(key *proto.BaseKey) (structure.KeyBaseInterface, bool)
-	Add(key *proto.BaseKey, val structure.KeyBaseInterface)
+	Add(key *proto.BaseKey, val structure.KeyBaseInterface) error
 	UpdateLruSize(length structure.UpdateLength)
 	DelByKey(key *proto.BaseKey) error
 	DelToClearSize() error

pkg/lru/lru.go

@@ -108,7 +108,11 @@ func (lru *SingleCache) GetDriver() event.DriverInterface {
 }
 
 //Add 增加
-func (lru *SingleCache) Add(key *proto.BaseKey, val structure.KeyBaseInterface) {
+func (lru *SingleCache) Add(key *proto.BaseKey, val structure.KeyBaseInterface) error {
+
+	if key == nil {
+		return errorx.KeyBaseIsNilErr()
+	}
 
 	exp := lru.lruTtlManage.setKeys(key)
 	keyBaseVal := &keyBaseValue{
@@ -119,17 +123,22 @@ func (lru *SingleCache) Add(key *proto.BaseKey, val structure.KeyBaseInterface)
 	if elVal, ok := lru.lruMap[key.Key]; ok {
 		lru.li.MoveToFront(elVal)
 		elVal.Value = keyBaseVal
-		return
+		return nil
 	}
 	valEl := lru.li.PushFront(keyBaseVal)
 	lru.lruMap[key.Key] = valEl
 	//增加大小
 	lru.UpdateLruSize(structure.UpdateLength(valEl.Value.(*keyBaseValue).val.SizeByte()))
+	return nil
 }
 
 // Get 查找key对应的value
 func (lru *SingleCache) Get(key *proto.BaseKey) (structure.KeyBaseInterface, bool) {
 
+	if key == nil {
+		return nil, false
+	}
+
 	if elVal, ok := lru.lruMap[key.Key]; ok {
 		lru.li.MoveToFront(elVal)
 		return elVal.Value.(*keyBaseValue).val, true
@@ -152,6 +161,11 @@ func (lru *SingleCache) Del() error {
 
 //DelByKey 根据key删除
 func (lru *SingleCache) DelByKey(key *proto.BaseKey) error {
+
+	if key == nil {
+		return errorx.KeyBaseIsNilErr()
+	}
+
 	if lru.lruMap == nil {
 		return errorx.New("lru is nil")
 	}
@@ -194,6 +208,10 @@ func (lru *SingleCache) DelToClearSize() error {
 // 更新过期时间
 func (lru *SingleCache) UpdateTTl(key *proto.BaseKey) error {
 
+	if key == nil {
+		return errorx.KeyBaseIsNilErr()
+	}
+
 	if elVal, ok := lru.lruMap[key.Key]; ok {
 		expire := lru.lruTtlManage.setKeys(key)
 		elVal.Value.(*keyBaseValue).expire = expire

pkg/lru/ttl_test.go

@@ -2,11 +2,12 @@ package lru
 
 import (
 	"fmt"
+	"testing"
+	"time"
+
 	"gitee.com/timedb/wheatCache/pkg/proto"
 	"gitee.com/timedb/wheatCache/pkg/structure/stringx"
 	"github.com/stretchr/testify/require"
-	"testing"
-	"time"
 )
 
 func TestTTlCup(t *testing.T) {

pkg/lru/woker_test.go

@@ -2,13 +2,14 @@ package lru
 
 import (
 	"context"
+	"testing"
+	"time"
+
 	"gitee.com/timedb/wheatCache/pkg/event"
 	"gitee.com/timedb/wheatCache/pkg/logx"
 	"gitee.com/timedb/wheatCache/pkg/proto"
 	"gitee.com/timedb/wheatCache/pkg/structure/stringx"
 	"github.com/stretchr/testify/require"
-	"testing"
-	"time"
 )
 
 func TestWorker(t *testing.T) {
@@ -53,6 +54,6 @@ func TestSingleCache_DelToClearSize(t *testing.T) {
 		workEvent.StartWaitEvent(2 * time.Second)
 	}
 
-	time.Sleep(5<<10 * time.Second)
+	time.Sleep(5 * time.Second)
 	logx.Info("end size is %d", lru.nowSize)
 }

pkg/structure/interface.gen.go

@@ -15,10 +15,10 @@ type KeyBaseInterface interface {
 
 type StringXInterface interface {
 	KeyBaseInterface
-	Set(val string) (string, UpdateLength)
+	Set(string) (string, UpdateLength)
 	Get() string
-	Add(renewal int32) (string, error)
+	Add(int32) (string, error)
-	Reduce(renewal int32) (string, error)
+	Reduce(int32) (string, error)
-	Setbit(offer int32, val bool) UpdateLength
+	Setbit(int32, bool) UpdateLength
-	Getbit(offer int32) (bool, error)
+	Getbit(int32) (bool, error)
 }

pkg/util/skiplist/skiplist.go

@@ -166,40 +166,120 @@ func (s *SkipList) RemoveAll(score float64) {
 	s.delSkipListNode(delNode...)
 }
 
-// ClearLeft 删除 小等于 score 的全部节点
+func (s *SkipList) PopLeft(score float64) []interface{} {
-func (s *SkipList) ClearLeft(score float64) *skipListNode {
+	endPoint := s.tail
+
 	head := s.head
 
-	for head.down != nil {
+	for endPoint.score > score {
-		p := head
+		endPoint = endPoint.pre
-		for p.score <= score {
-			p = p.next
 	}
 
-		head.next = p
+	var valuePoint *skipListNode
-		p.pre = head
+	for endPoint != nil {
+		// 检查 end 后的节点情况
+		for endPoint.next.score <= score {
+			endPoint = endPoint.next
+		}
+
+		head.next = endPoint.next
+		endPoint.next.pre = head
+		valuePoint = endPoint
+
+		endPoint = endPoint.down
 		head = head.down
 	}
 
-	node := s.searchNode(score)
+	values := make([]interface{}, 0, 100)
-	for node.score <= score {
+	for valuePoint != nil && valuePoint.score > s.head.score {
-		node = node.next
+		values = append(values, valuePoint.val)
+		valuePoint = valuePoint.pre
 	}
 
-	oldNode := node.pre
+	return values
-	head.next = node
 
-	node.pre = head
-	return oldNode
 }
 
-func (s *SkipList) PopLeft(score float64) []interface{} {
+func (s *SkipList) PopRight(score float64) []interface{} {
-	node := s.ClearLeft(score)
+	startPoint := s.head
-	values := make([]interface{}, 0, 10)
+
-	for node.score != math.MinInt64 {
+	tail := s.tail
-		values = append(values, node.val)
+
-		node = node.pre
+	for startPoint.score < score {
+		startPoint = startPoint.next
 	}
+
+	var valuePoint *skipListNode
+	for startPoint != nil {
+		// 检查 end 后的节点情况
+		for startPoint.pre.score >= score {
+			startPoint = startPoint.pre
+		}
+
+		tail.pre = startPoint.pre
+		startPoint.pre.next = tail
+		valuePoint = startPoint
+
+		startPoint = startPoint.down
+		tail = tail.down
+	}
+
+	values := make([]interface{}, 0, 100)
+	for valuePoint != nil && valuePoint.score < s.tail.score {
+		values = append(values, valuePoint.val)
+		valuePoint = valuePoint.next
+	}
+
+	return values
+
+}
+
+// PopRangeByScore 删除包括 start 到 end 的 score 的全部 value
+func (s *SkipList) PopRangeByScore(startScore float64, endScore float64) []interface{} {
+	if startScore >= endScore {
+		return nil
+	}
+
+	startPoint := s.head
+	endPoint := s.tail
+
+	// 定位删除索引最高级
+	for startPoint.score < startScore {
+		startPoint = startPoint.next
+	}
+	for endPoint.score > endScore {
+		endPoint = endPoint.pre
+	}
+
+	var valuePoint *skipListNode
+
+	for startPoint != nil {
+		// 检查 start 前的节点情况
+		for startPoint.pre.score >= startScore {
+			startPoint = startPoint.pre
+		}
+
+		// 检查 end 后的节点情况
+		for endPoint.next.score <= endScore {
+			endPoint = endPoint.next
+		}
+
+		// 删除节点
+		valuePoint = startPoint
+		startPoint.pre.next = endPoint.next
+		endPoint.next.pre = startPoint.pre
+
+		// 下一级处理
+		startPoint = startPoint.down
+		endPoint = endPoint.down
+	}
+
+	values := make([]interface{}, 0, 100)
+	for valuePoint != nil && valuePoint.score <= endScore {
+		values = append(values, valuePoint.val)
+		valuePoint = valuePoint.next
+	}
+
 	return values
 }
 

pkg/util/skiplist/skiplist_test.go

@@ -2,11 +2,12 @@ package skiplist
 
 import (
 	"fmt"
-	"gitee.com/timedb/wheatCache/pkg/logx"
-	"github.com/stretchr/testify/require"
 	"math/rand"
 	"testing"
 	"time"
+
+	"gitee.com/timedb/wheatCache/pkg/logx"
+	"github.com/stretchr/testify/require"
 )
 
 // 时间测试
@@ -66,3 +67,39 @@ func Test_skipList_ClearLeft(t *testing.T) {
 	require.Equal(t, s.Get(20), nil)
 
 }
+
+func TestSkipList_PopRangeByScore(t *testing.T) {
+	startNum := 5
+	endNum := 79
+	result := make([]interface{}, 0)
+	for i := startNum; i <= endNum; i++ {
+		result = append(result, i)
+	}
+
+	for i := 0; i < 100; i++ {
+		s := NewSkipList(14)
+		for i := 0; i < 100; i++ {
+			s.Insert(float64(i), i)
+		}
+		res := s.PopRangeByScore(float64(startNum), float64(endNum))
+		require.Equal(t, res, result)
+	}
+}
+
+func TestSkipList_PopRight(t *testing.T) {
+	score := 33
+	result := make([]interface{}, 0)
+	for i := score; i < 100; i++ {
+		result = append(result, i)
+	}
+
+	for i := 0; i < 100; i++ {
+		s := NewSkipList(14)
+		for i := 0; i < 100; i++ {
+			s.Insert(float64(i), i)
+		}
+		res := s.PopRight(float64(score))
+		require.Equal(t, res, result)
+		s.debugPrint()
+	}
+}

storage/dao/dao.go

@@ -32,9 +32,11 @@ func (d *Dao) Set(key *proto.BaseKey, strVal string) (string, error) {
 
 	// 不存在新建
 	strValue := stringx.NewStringSingle()
-	result, length := strValue.Set(strVal)
+	result, _ := strValue.Set(strVal)
-	d.lru.Add(key, strValue)
+	err := d.lru.Add(key, strValue)
-	d.lru.UpdateLruSize(length)
+	if err != nil {
+		return "", err
+	}
 	return result, nil
 }