Compare commits

..

2 Commits

Author SHA1 Message Date
bandl fbc6edc60d request update @hjl 2021-09-25 13:37:41 +08:00
HuangJiaLuo 2ad86909d1 update(pkg): commit lru 2021-09-25 12:59:43 +08:00
157 changed files with 898 additions and 21765 deletions

3
.gitignore vendored
View File

@ -17,5 +17,4 @@
# build file
/bin/storage
/bin/gateway
/bin/storage

View File

@ -1,7 +0,0 @@
FROM ubuntu:18.04
WORKDIR /home/src/gitee.com/wheat-os/wheat-cache
ADD . /home/src/gitee.com/wheat-os/wheat-cache
RUN mkdir /etc/wheat-cache
RUN mv /home/src/gitee.com/wheat-os/wheat-cache/conf/wheat-cache.yaml /etc/wheat-cache/

Binary file not shown.

147
README.md

File diff suppressed because one or more lines are too long

View File

@ -1,22 +0,0 @@
package client
import (
"gitee.com/wheat-os/wheatCache/client/middle"
"gitee.com/wheat-os/wheatCache/pkg/proto"
"google.golang.org/grpc"
)
func newWheatClient(targer string, opt ...middle.ClientMiddle) (proto.CommServerClient, error) {
interceptor := middle.GetUnaryInterceptor(opt...)
comm, err := grpc.Dial(targer, grpc.WithInsecure(), grpc.WithUnaryInterceptor(interceptor))
if err != nil {
return nil, err
}
return proto.NewCommServerClient(comm), nil
}
func NewWheatClient(targer string, opt ...middle.ClientMiddle) (proto.CommServerClient, error) {
return newWheatClient(targer, opt...)
}

View File

@ -1,47 +0,0 @@
package client
import (
"context"
"testing"
"gitee.com/wheat-os/wheatCache/client/middle"
"gitee.com/wheat-os/wheatCache/pkg/proto"
"github.com/stretchr/testify/require"
)
func TestClient(t *testing.T) {
cli, err := NewWheatClient("127.0.0.1:5891", middle.WithUnaryColonyClient)
require.NoError(t, err)
ctx := context.Background()
bKey := proto.NewBaseKey("apple")
resp, err := cli.Set(ctx, &proto.SetRequest{
Key: bKey,
Val: "yyyy",
})
require.NoError(t, err)
require.Equal(t, resp.Result, "yyyy")
getResp, err := cli.Get(ctx, &proto.GetRequest{
Key: bKey,
})
require.NoError(t, err)
require.Equal(t, getResp.Result, "yyyy")
}
func TestClientGet(t *testing.T) {
cli, err := NewWheatClient("127.0.0.1:5891", middle.WithUnaryColonyClient)
require.NoError(t, err)
ctx := context.Background()
bKey := &proto.BaseKey{
Key: "apple",
}
getResp, err := cli.Get(ctx, &proto.GetRequest{
Key: bKey,
})
require.NoError(t, err)
require.Equal(t, getResp.Result, "yyyy")
}

View File

@ -1,6 +0,0 @@
package middle
import "context"
// type UnaryClientInterceptor func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error
type ClientMiddle func(ctx context.Context, method string, req, reply interface{}, header map[string]string) error

View File

@ -1,53 +0,0 @@
package middle
import (
"context"
"gitee.com/wheat-os/wheatCache/pkg/proto"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
)
func WithUnaryColonyClient(ctx context.Context, method string, req, reply interface{}, header map[string]string) error {
key, ok := req.(proto.GetKeyBaseInterface)
if !ok {
return status.Errorf(codes.Unknown, "key base err")
}
if header == nil {
return nil
}
// meta 解析会出现 全部小写问题
header[proto.BaseKeyMethodKey] = key.GetKey().Key
return nil
}
func getKeyByKeyMapvalue(m map[string]string) []string {
l := make([]string, 0)
for key, value := range m {
l = append(l, key, value)
}
return l
}
func GetUnaryInterceptor(middleOpts ...ClientMiddle) grpc.UnaryClientInterceptor {
return func(ctx context.Context, method string, req, reply interface{},
cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
// 加载中间件
header := make(map[string]string)
for _, mid := range middleOpts {
err := mid(ctx, method, req, reply, header)
if err != nil {
return err
}
}
lm := getKeyByKeyMapvalue(header)
headerData := metadata.Pairs(lm...)
ctxH := metadata.NewOutgoingContext(ctx, headerData)
return invoker(ctxH, method, req, reply, cc, opts...)
}
}

View File

@ -2,38 +2,34 @@ package conf
import (
"log"
"sync"
"github.com/spf13/viper"
)
const (
linuxPath = "/etc/wheat-cache/"
devPath = "./conf"
devPathBin = "../conf"
)
var confLock sync.Once
func init() {
confLock.Do(func() {
setDefaultConfValue()
err := LoadConf("")
switch err.(type) {
case nil:
case viper.ConfigFileNotFoundError:
formatPath := []string{linuxPath}
log.Fatalf("the profile could not be read, read path:%v", formatPath)
default:
log.Fatalf("the resolution of the profile failed, err: %v", err)
setDefaultConfValue()
err := LoadConf("")
switch err.(type) {
case nil:
case viper.ConfigFileNotFoundError:
formatPath := []string{linuxPath, devPath, devPath}
log.Fatalf("the profile could not be read, read path:%v", formatPath)
default:
log.Fatalf("the resolution of the profile failed, err: %v", err)
}
},
)
}
}
func setDefaultConfValue() {
// 设置一些默认值
viper.SetDefault("version", "base-01")
defaultStorage()
}
func LoadConf(path string) error {
@ -46,6 +42,10 @@ func LoadConf(path string) error {
// linux
viper.AddConfigPath(linuxPath)
// 开发环境
viper.AddConfigPath(devPath)
viper.AddConfigPath(devPathBin)
viper.SetConfigType("yaml")
err := viper.ReadInConfig()

View File

@ -7,29 +7,30 @@ import (
"github.com/stretchr/testify/require"
)
func TestLoadConf(t *testing.T) {
type args struct {
path string
}
tests := []struct {
name string
args args
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := LoadConf(tt.args.path); (err != nil) != tt.wantErr {
t.Errorf("LoadConf() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestConf(t *testing.T) {
// 外部导入 conf.yaml 需要导入 conf 包
// 每次迁移文件时, 使用 sudo make init-conf来将yam文件迁移到指定的文件夹下
// get 使用, 读取 public_conf 配置文件
h := viper.Get("storage.host")
require.Equal(t, h, "127.0.0.1")
h = viper.Get("env")
require.Equal(t, h, "dev")
// set 使用
viper.Set("host", "1222")
host := viper.GetString("host")
require.Equal(t, host, "1222")
}
func TestMiddleConf(t *testing.T) {
ct := viper.GetStringSlice("plugins-control.logcontext")
require.Equal(t, ct, []string{"logMiddle"})
d := viper.GetInt("middleware-driver.driverCount")
require.Equal(t, d, 1000)
c := viper.GetInt("middleware-driver.middleConsumerCount")
require.Equal(t, c, 5)
}

View File

@ -1,11 +0,0 @@
package conf
import "github.com/spf13/viper"
func defaultStorage() {
// aof
viper.SetDefault("storage.aof-path", "/etc/wheat-cache/wheat.aof")
viper.SetDefault("storage.aof-flush-time", 5)
viper.SetDefault("storage.aof-check-time", 1)
viper.SetDefault("storage.aof-check-freq", 20)
}

View File

@ -3,58 +3,10 @@ version: 'v1.0'
env: 'dev'
storage:
host: '0.0.0.0'
host: '127.0.0.1'
port: 5890
timeOut: 2 # second
lru-cache:
mode: "single" # thread
aof-codec: "b16" # 目前只实现了 b16 编码方案。
aof-path: "/etc/wheat-cache/wheat.aof"
aof-flush-time: 5 # second , 每 5 秒刷新缓冲区的内容到磁盘。
aof-check-time: 1 # 每 1 second 执行一次 io 检查
aof-check-freq: 20 # 在一个 aof-check-time 周期内,出现超过 aof-check-freq 的 IO 操作会刷新磁盘
# clearSize and maxSize must be Int
lruCache:
clearSize: "512mb"
maxSize: "1GB"
eventDriverSize: 2000
workTime: 1
detachNum: 300
logPrint:
stath: ["error"]
middleware-driver:
driverCount: 1000
middleConsumerCount: 5
# Register the message push type
# 在这里注册消息推送类型,
plugins-control:
# log-context: Logs generated by storage or gateway are pushed through this message
# log-context: storage 或者 gateway 产生的日志通过这个消息推送
log-context: [ "mock-plugins" ]
# lru-clean-context: Lru is pushed through this message when data cleansing occurs
# lru-clean-context: Lru 发生数据清理时通过这个消息推送
lru-clean-context: ["mock-plugins"]
# lru-ttl-context: Lru is pushed through this message when data expires
# lru-ttl-context: Lru 发生数据过期时通过这个消息推送
lru-ttl-context: ["mock-plugins"]
# plugins-info-contextAll plugins information for the current project
# plugins-info-context: 当前项目全部的插件信息
plugins-infos-context: ["mock-plugins"]
gateway:
host: '0.0.0.0'
port: 5891
target: ["127.0.0.1:5890"]
mock-plugin:
pprof-addr: "127.0.0.1:8000"

View File

@ -2,6 +2,6 @@
EventP : 生产事件
EventQ : 事件队列
EventQ : 队列事件
Event CP : 清理事件

View File

@ -1,5 +1,5 @@
### RDB + AOF 的事务解决方案
### RDB + AOF 的事务解决方案
![事务RDB方案](https://gitee.com/timedb/img/raw/master/images/事务RDB方案.svg)

View File

@ -1,2 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="94" height="20" role="img" aria-label="license: AFL3.0"><title>license: AFL3.0</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="94" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="47" height="20" fill="#555"/><rect x="47" width="47" height="20" fill="#97ca00"/><rect width="94" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="245" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="370">license</text><text x="245" y="140" transform="scale(.1)" fill="#fff" textLength="370">license</text><text aria-hidden="true" x="695" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="370">AFL3.0</text><text x="695" y="140" transform="scale(.1)" fill="#fff" textLength="370">AFL3.0</text></g></svg>

Before

Width:  |  Height:  |  Size: 1.1 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 37 KiB

View File

@ -1,2 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="88" height="20" role="img" aria-label="Wheat: Cache"><title>Wheat: Cache</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="88" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="45" height="20" fill="#555"/><rect x="45" width="43" height="20" fill="#a4a61d"/><rect width="88" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="235" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="350">Wheat</text><text x="235" y="140" transform="scale(.1)" fill="#fff" textLength="350">Wheat</text><text aria-hidden="true" x="655" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="330">Cache</text><text x="655" y="140" transform="scale(.1)" fill="#fff" textLength="330">Cache</text></g></svg>

Before

Width:  |  Height:  |  Size: 1.1 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 13 KiB

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 8.9 KiB

View File

@ -1,2 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="86" height="20" role="img" aria-label="version: v1.1"><title>version: v1.1</title><linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="r"><rect width="86" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#r)"><rect width="51" height="20" fill="#555"/><rect x="51" width="35" height="20" fill="#007ec6"/><rect width="86" height="20" fill="url(#s)"/></g><g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" text-rendering="geometricPrecision" font-size="110"><text aria-hidden="true" x="265" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="410">version</text><text x="265" y="140" transform="scale(.1)" fill="#fff" textLength="410">version</text><text aria-hidden="true" x="675" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="250">v1.1</text><text x="675" y="140" transform="scale(.1)" fill="#fff" textLength="250">v1.1</text></g></svg>

Before

Width:  |  Height:  |  Size: 1.1 KiB

View File

@ -1,11 +0,0 @@
### Cache 分布式方案-Getway
![getway方案](https://gitee.com/timedb/img/raw/master/images/getway方案.svg)
1. single 集群分布式方案中,使用 getway 方向代理客户端的 grpc 请求, 通过 hash 环实现 分布式。
2. 集群模式中, 通过主从来 实现 cache 的备份问题,提高容灾性。

View File

@ -1,32 +0,0 @@
### 构建工具文档
#### dcgen
1. 根据结构体接口模板生成 proto 文件
2. 迁移 proto 到 pkg/proto 下
3. 更新结构体常量
>PS : 开发一个 storage 的新接口时一般有以下步骤
>1. 修改 storage 接口配置文件
>2. make dcgen
>3. 修改生成的 proto 文件
>4. make dcgen
>5. 添加 storage 的操作接口
#### build-storage
编译并且生成 /bin/storage
#### build-gateway
编译并且生成 /bin/gateway
#### install
1. 安装项目,需要 sudo
#### storage
根据配置文件启动 storage
#### gateway
根据配置文件启动 gateway
#### init-conf
根据配置文件文档初始化配置文件到 /etc/wheat-cache/wheat-cache.yaml

17
doc/make/make.md Normal file
View File

@ -0,0 +1,17 @@
## 构建工具文档
### 构建 proto grpc-go
```shell
make dcgen
```
### 编译全部的 go 项目
```shell
make build
```
### 启动 storage 服务
```shell
make dev
```
### 根据配置生成 结构体命令

View File

@ -1,13 +0,0 @@
### 事件驱动 2.0
### event 1.0 存在的问题
事件驱动 1.0 在 相互关联访问时,会发生 死锁问题, 导致一个事件执行周期失败
### event 2.0 新特性
- 异步事件支持
- 挂起操作
### event 2.0 设计图
![](../../_icon/event.svg)

View File

@ -1,150 +0,0 @@
### 事件驱动使用文档
- 包目录: pkg/event
- 使用场景, 异步推送数据
#### 事件驱动基本概念
- event
event 是事件驱动的事件部分, 主要负责一些信息的传递, 一共分为,有等待和无等待的事件, 无等待事件 consumer 消费事件后,不会给 produce 回复, 有等待事件, 支持事件消费后, 向 produce 发送一个回复。
- driver
driver 是事件的驱动, 主要复制连接 produce 以及 consumer 其中维护了一个 event 的队列。
- produce
produce 主要负责 推送 事件, 每个 produce 都需要维护一个 事件驱动通过 driver 来进行数据的推送。
- consumer
consumer 负责接收事件, 每个 consumer 都需要注册一个 事件驱动 来获取 event。
####
#### 应用场景
事件驱动主要用于 异步消息的推送(目前的实现也支持 同步的消息实现), 一般来说 produce 发送 event 后 event 被消费过程会完全于 produce 无关。
#### 无返回 event 使用例子
我们使用 借书为例子使用, 假设 A 要把 书 S 交还给 B 他们约定, A 把书 S 放到图书馆, B 在空闲时去图书馆取出书 S。
```go
func TestNewConsumer(t *testing.T) {
// 定义一个图书馆
type Library struct {
driver DriverInterface
}
library := &Library{
driver: NewDriver(100),
}
ctx := context.Background()
// 定义 A
type A struct {
produce ProduceInterface
}
a := &A{
produce: NewProduce(library.driver),
}
// 定义 B
type B struct {
consumer ConsumerInterface
}
b := &B{
consumer: NewConsumer(library.driver),
}
// 定义书 S 并且添加一些描述
book := NewEvent("S")
book.SetMsg("title", "hello world")
book.SetCtxValue("pages", 120)
// A 把书 S 放到图书馆
go func() {
a.produce.Call(ctx, book)
}()
// 模拟 B 去图书馆拿书
book = b.consumer.Receive(ctx)
fmt.Println(book.GetMsg("title"))
}
```
#### 有返回event的使用例子
在上述流程的情景下, 我们添加一些条件, B 拿到书以后检查书是否损坏,如果损坏, 需要告诉 A 书损坏情况。
```go
func TestNewConsumer(t *testing.T) {
// 定义一个图书馆
type Library struct {
driver DriverInterface
}
library := &Library{
driver: NewDriver(100),
}
ctx := context.Background()
// 定义 A
type A struct {
produce ProduceInterface
}
a := &A{
produce: NewProduce(library.driver),
}
// 定义 B
type B struct {
consumer ConsumerInterface
}
b := &B{
consumer: NewConsumer(library.driver),
}
// 定义书 S 并且添加一些描述
book := NewEvent("S")
book.SetMsg("title", "hello world")
book.SetValue("pages", 120)
// A 把书 S 放到图书馆
go func() {
book.InitWaitEvent()
a.produce.Call(ctx, book)
// A 等待 B 的回复, 但是他最多只会等待 B 2个小时
res, err := book.StartWaitEvent(2 * time.Hour)
require.NoError(t, err)
fmt.Println(res)
}()
// 模拟 B 去图书馆拿书
book = b.consumer.Receive(ctx)
fmt.Println(book.GetMsg("title"))
// 书完好
book.ExecWorkAndSendResult(func() (interface{}, error) {
// b 检查书
return "OK", nil
})
time.Sleep(50 * time.Millisecond)
}
```

View File

@ -1,47 +0,0 @@
### 中间件调用
### 创建事件
event := event.NewEvent("logcontext")
### 创建驱动
middleware := NewMiddleWare()
### 将事件推入驱动
middleware.eventProduce.Call(ctx, event)
### 获取驱动的事件
middleware.eventConsumer.Reciver(ctx)
### 插件接口
type MiddleToolsInterface interface {
Init() // 初始化
Exec(interface{}) (interface{}, error) // 处理用户发送事件
Name() string // 获取中间件名称
Describe() string // 描述
}
### 插件的New方法规定为 NewMiddleWare()
每个插件都要定义 NewMiddleWare()
### 将插件名 “logMiddle” 注册到配置文件wheat-cache.yaml,其他插件注册
plugins-control:
logcontext: ["logMiddle"]

View File

@ -1,60 +0,0 @@
### 结构体开发文档
#### 基础结构体开发
- 基础结构体放到 /pkg/structure。
- 每个结构 类型都以 x 结尾,如 listx stringx。
- 在 pkg/structure/define.go 文件中添加对应结构体的接口,主要为 了保证项目可以扩展线程安全结构体, 以及结构体接口的实现, 目前 lru 采用 single 模式, 可以只开发 single 的结构体,如 stringxSingle。
- 请在结构体包中 补充单元测试。
#### 目前实现结构体接口说明
> ps: 所有的详细用法都可以查看单元测试文件。
```go
// stringx
type StringXInterface interface {
KeyBaseInterface
// 重新设置一个 值
Set(string) (string, UpdateLength)
// 获取值的 string 形式
Get() string
// 值自动增加 一个值,只对 float 和 string 有效
Add(int32) (string, error)
// 值自动增加 减少值,只对 float 和 string 有效
Reduce(int32) (string, error)
// 使用位图类型
Setbit(int32, bool) UpdateLength
Getbit(int32) (bool, error)
// 获取字符串的切片
Getrange(start, end int32) (string, error)
GetLength() int
}
// listx
type ListXInterface interface {
KeyBaseInterface
LPush(...string) UpdateLength
RPush(...string) UpdateLength
LPop(int) ([]string, UpdateLength)
RPop(int) ([]string, UpdateLength)
Index(int) (string, error)
// 插入一组数据, bool 类型表示是否右插尾插false 时采用左插(头插)
Insert(int, bool, ...string) (UpdateLength, error)
Length() int
// 切片, O(n)复杂度
Slice(start, end int) (UpdateLength, error)
}
```
### structure.Value 类型
- 结构体的数据区域全部使用 structure.Value 类型存储。
- structure.Value 主要实现了 sting, int64, float64 的存储接口。
- structure.Value 非常容易计算内存占用。
- structure.UpdateLength 指的是在进行某一个操作以后,内存大小的变化。
- 为了保证 lru 的存储效率lru 不会去遍历全部的 key 来重新计算大小,而是根据 UpdateLength 来动态更新 lru 的大小,具体实现在 dao 中。
- structure.Value 类型的使用方法可以在 pkg/structure/value_test.go 中获取。

View File

@ -1,46 +0,0 @@
### 快速进行 storage 开发
#### 开发环境
- ubuntu18, 可以使用 wsl
- go1.15+, python3
- jinja2
- go mod
- protobuf 3.17.3
- protoc-gen-go v1.26.0
#### storage 执行流程
![](../_icon/storage-dao.svg)
#### 分层,简介
```sh
.
├── cmd # storage 启动函数
│ └── root.go
├── dao # 实际处理层,接口实现全部再 dao 层里实现
│ ├── dao.go
│ ├── dao_test.go
│ ├── interface.gen.go
│ ├── listx.go # listx 相关功能
│ └── stringx.go
├── main.go
├── service # 接口层,由 gen-service 自动生成
│ ├── define.go
│ ├── single.go
│ └── single_service.gen.go
└── temp # 开发模板层
├── const.gen.go
├── const.template
├── dao.template
├── service.template
└── tem.yaml
```
#### 快速开发接口
> [快速开发视频 blibli](https://www.bilibili.com/video/BV1HL4y1v7ps)
1. 修改 temp/tem.yaml 文件,添加新接口
2. 在项目根目录执行 `make dcgen` 生成 proto 原始结构
3. 修改对应新添加接口的 proto 文件,再次执行 `make dcgen` 完成 proto 迁移
4. 执行 `make gen-service` 生成 dao 接口
5. 完成 新 dao 层接口, 根据需要添加单元测试。
6. 使用 make install 编译并且安装项目

0
doc/test.md Normal file
View File

View File

@ -1,54 +0,0 @@
### 单元测试文档
#### 样例
```go
package dao
import (
"testing"
_ "gitee.com/wheat-os/wheatCache/conf"
"gitee.com/wheat-os/wheatCache/pkg/lru"
"gitee.com/wheat-os/wheatCache/pkg/proto"
"github.com/stretchr/testify/require"
)
// 规范1. 每个 package 应该都至少有一个单测
// 规范2. 包里有公用的 mock 数据应该分离出来。
func mockData(t *testing.T, d *Dao) {
values := []string{"1", "1.3", "abcdefg"}
for _, val := range values {
key := &proto.BaseKey{
Key: val,
}
_, err := d.Set(key, val)
// 规范3. 使用 require 包来完成单测
require.NoError(t, err)
}
}
// 规范4. 单元测试应该尽可能覆盖全部情况,不要使用依赖注入。
func TestDao_Reduce(t *testing.T) {
lruCache := lru.NewLRUCache()
dao := NewDao(lruCache)
mockData(t, dao)
resp, err := dao.Reduce(&proto.BaseKey{Key: "1"}, 2)
require.NoError(t, err)
require.Equal(t, resp, "-1")
resp, err = dao.Reduce(&proto.BaseKey{Key: "1.3"}, 2)
require.NoError(t, err)
require.Equal(t, resp, "-0.70")
_, err = dao.Reduce(&proto.BaseKey{Key: "abcdefg"}, 2)
require.Error(t, err)
}
```

View File

@ -1,69 +0,0 @@
package cmd
import (
"fmt"
"net"
_ "gitee.com/wheat-os/wheatCache/conf"
wheatCodec "gitee.com/wheat-os/wheatCache/gateway/codec"
"gitee.com/wheat-os/wheatCache/gateway/endpoint"
"gitee.com/wheat-os/wheatCache/gateway/proxy"
"gitee.com/wheat-os/wheatCache/pkg/logx"
"gitee.com/wheat-os/wheatCache/pkg/util/server"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"google.golang.org/grpc"
)
// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
Use: "getway",
Short: "getway",
Long: `start getway server`,
Run: func(cmd *cobra.Command, args []string) {
host := viper.GetString("gateway.host")
port := viper.GetInt("gateway.port")
tcpAddr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", host, port))
if err != nil {
logx.Panic("get gateway addr err:%v", err)
}
listen, err := net.ListenTCP("tcp", tcpAddr)
if err != nil {
logx.Panic("get gateway tcp conn err:%v", err)
}
gatewayServer := GetGatewayServer()
server.ElegantExitServer(gatewayServer)
logx.Info("start gateway in addr: %s", tcpAddr.String())
if err := gatewayServer.Serve(listen); err != nil {
logx.Errorln(err)
}
},
}
// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
cobra.CheckErr(rootCmd.Execute())
}
func GetGatewayServer() *grpc.Server {
targets := viper.GetStringSlice("gateway.target")
logx.Debug("service target in %v", targets)
stream := proxy.GetDirectorByServiceHash()
endpoint := endpoint.NewHashEndpoint(endpoint.HashReplicasDefault, nil, targets...)
opts := make([]grpc.ServerOption, 0)
opts = append(
opts,
grpc.ForceServerCodec(wheatCodec.Codec()),
grpc.UnknownServiceHandler(proxy.TransparentHandler(stream, endpoint)),
)
return grpc.NewServer(opts...)
}

View File

@ -1,60 +0,0 @@
package codec
import (
"google.golang.org/grpc/encoding"
"google.golang.org/protobuf/proto"
)
// protoCodec 用于 gateway 解析全部的 grpc 类型的消息
type protoCodec struct{}
func (protoCodec) Name() string {
return "wheat-cache-proto"
}
func (protoCodec) Marshal(v interface{}) ([]byte, error) {
return proto.Marshal(v.(proto.Message))
}
func (protoCodec) Unmarshal(data []byte, v interface{}) error {
return proto.Unmarshal(data, v.(proto.Message))
}
type Frame struct {
payload []byte
}
type proxyCodec struct {
baseCodec encoding.Codec
}
func (p *proxyCodec) Name() string {
return "wheat-cache-proxy"
}
func (p *proxyCodec) Marshal(v interface{}) ([]byte, error) {
out, ok := v.(*Frame)
if !ok {
return p.Marshal(v)
}
return out.payload, nil
}
func (p *proxyCodec) Unmarshal(data []byte, v interface{}) error {
dst, ok := v.(*Frame)
if !ok {
return p.Unmarshal(data, v)
}
dst.payload = data
return nil
}
// CodeWithParent 生成基于 proto 的解码器
func CodeWithParent(parent encoding.Codec) encoding.Codec {
return &proxyCodec{parent}
}
func Codec() encoding.Codec {
return CodeWithParent(protoCodec{})
}

View File

@ -1,11 +0,0 @@
package endpoint
type EndpointInterface interface {
GetTargetAddr(...string) (string, error)
IsEmpty() bool
AddTarget(targets ...string)
}
const (
HashReplicasDefault = 3
)

View File

@ -1,85 +0,0 @@
package endpoint
import (
"hash/crc32"
"sort"
"strconv"
"gitee.com/wheat-os/wheatCache/pkg/errorx"
)
type HashFunc func(data []byte) uint32
// 实现 sort
type UInt32Slice []uint32
func (s UInt32Slice) Len() int {
return len(s)
}
func (s UInt32Slice) Less(i, j int) bool {
return s[i] < s[j]
}
func (s UInt32Slice) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
type HashEndpoint struct {
hash HashFunc
replicas int // 复制因子
keys UInt32Slice
hashMap map[uint32]string // taraget 隐射
}
func NewHashEndpoint(replicas int, fn HashFunc, target ...string) EndpointInterface {
endpoint := &HashEndpoint{
replicas: replicas,
hash: fn,
hashMap: make(map[uint32]string, len(target)),
}
if endpoint.hash == nil {
endpoint.hash = crc32.ChecksumIEEE // 默认使用 CRC32 算法
}
endpoint.AddTarget(target...)
return endpoint
}
func (h *HashEndpoint) IsEmpty() bool {
return len(h.keys) == 0
}
func (h *HashEndpoint) AddTarget(targets ...string) {
for _, tar := range targets {
for i := 0; i < h.replicas; i++ {
hash := h.hash([]byte(strconv.Itoa(i) + tar))
h.keys = append(h.keys, hash)
h.hashMap[hash] = tar
}
}
// 虚拟值排序,方便查找
sort.Sort(h.keys)
}
func (h *HashEndpoint) GetTargetAddr(str ...string) (string, error) {
if h.IsEmpty() {
return "", errorx.New("gateway not register transport")
}
if len(str) != 1 {
return "", errorx.New("must give key")
}
hash := h.hash([]byte(str[0]))
idx := sort.Search(len(h.keys), func(i int) bool { return h.keys[i] >= hash })
if idx == len(h.keys) {
return h.hashMap[h.keys[0]], nil
}
return h.hashMap[h.keys[idx]], nil
}

View File

@ -1,17 +0,0 @@
package endpoint
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestHashTransport_GetTargetAddr(t *testing.T) {
tran := NewHashEndpoint(3, nil, "127.0.0.1:5581", "127.0.0.1:5582", "127.0.0.1:5583")
key := "test"
target, err := tran.GetTargetAddr(key)
require.NoError(t, err)
require.Equal(t, target, "127.0.0.1:5582")
}

View File

@ -1,7 +0,0 @@
package main
import "gitee.com/wheat-os/wheatCache/gateway/cmd"
func main() {
cmd.Execute()
}

View File

@ -1,17 +0,0 @@
package proxy
import (
"context"
"gitee.com/wheat-os/wheatCache/gateway/endpoint"
"google.golang.org/grpc"
)
type StreamDirector func(ctx context.Context, fullMethodName string, endpoint endpoint.EndpointInterface) (context.Context, *grpc.ClientConn, error)
var (
clientStreamDescForProxying = &grpc.StreamDesc{
ServerStreams: true,
ClientStreams: true,
}
)

View File

@ -1,37 +0,0 @@
package proxy
import (
"context"
"gitee.com/wheat-os/wheatCache/gateway/codec"
"gitee.com/wheat-os/wheatCache/gateway/endpoint"
"gitee.com/wheat-os/wheatCache/pkg/proto"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
)
func GetDirectorByServiceHash() StreamDirector {
return func(ctx context.Context, fullMethodName string, endpoint endpoint.EndpointInterface) (context.Context, *grpc.ClientConn, error) {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return nil, nil, status.Errorf(codes.Unknown, "from FromIncomingContext err")
}
baseKey, ok := md[proto.BaseKeyMethodKey]
if !ok {
return nil, nil, status.Errorf(codes.Unknown,
"grpc header is not found %s, please check the client interceptor", proto.BaseKeyMethodKey)
}
target, err := endpoint.GetTargetAddr(baseKey...)
if err != nil {
return nil, nil, status.Errorf(codes.Unknown, "get transport err, err:%v", err)
}
cli, err := grpc.DialContext(ctx, target, grpc.WithInsecure(), grpc.WithDefaultCallOptions(grpc.ForceCodec(codec.Codec())))
return ctx, cli, err
}
}

View File

@ -1,131 +0,0 @@
package proxy
import (
"context"
"io"
wheatCodec "gitee.com/wheat-os/wheatCache/gateway/codec"
"gitee.com/wheat-os/wheatCache/gateway/endpoint"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// TransparentHandler returns a handler that attempts to proxy all requests that are not registered in the server.
// The indented use here is as a transparent proxy, where the server doesn't know about the services implemented by the
// backends. It should be used as a `grpc.UnknownServiceHandler`.
//
// This can *only* be used if the `server` also uses grpcproxy.CodecForServer() ServerOption.
func TransparentHandler(director StreamDirector, endpoint endpoint.EndpointInterface) grpc.StreamHandler {
streamer := &handler{
director,
endpoint,
}
return streamer.handler
}
type handler struct {
director StreamDirector
endpoint endpoint.EndpointInterface
}
// handler is where the real magic of proxying happens.
// It is invoked like any gRPC server stream and uses the gRPC server framing to get and receive bytes from the wire,
// forwarding it to a ClientStream established against the relevant ClientConn.
func (s *handler) handler(srv interface{}, serverStream grpc.ServerStream) error {
fullMethodName, ok := grpc.MethodFromServerStream(serverStream)
if !ok {
return status.Errorf(codes.Internal, "lowLevelServerStream not exists in context")
}
outgoingCtx, backendConn, err := s.director(serverStream.Context(), fullMethodName, s.endpoint)
if err != nil {
return err
}
clientCtx, clientCancel := context.WithCancel(outgoingCtx)
defer clientCancel()
clientStream, err := grpc.NewClientStream(clientCtx, clientStreamDescForProxying, backendConn, fullMethodName)
if err != nil {
return err
}
s2cErrChan := s.forwardServerToClient(serverStream, clientStream)
c2sErrChan := s.forwardClientToServer(clientStream, serverStream)
for i := 0; i < 2; i++ {
select {
case s2cErr := <-s2cErrChan:
if s2cErr == io.EOF {
// 客户端流发送完毕正常关闭结束, Proxy 关闭对 Backend 的连接
clientStream.CloseSend()
break
}
clientCancel()
return status.Errorf(codes.Internal, "failed proxying s2c: %v", s2cErr)
case c2sErr := <-c2sErrChan:
// 服务的没用在提供数据触发这个分支
serverStream.SetTrailer(clientStream.Trailer())
if c2sErr != io.EOF {
return c2sErr
}
return nil
}
}
return status.Errorf(codes.Internal, "gRPC proxying should never reach this stage.")
}
func (s *handler) forwardClientToServer(src grpc.ClientStream, dst grpc.ServerStream) chan error {
ret := make(chan error, 1)
go func() {
f := &wheatCodec.Frame{}
for i := 0; ; i++ {
if err := src.RecvMsg(f); err != nil {
ret <- err // this can be io.EOF which is happy case
break
}
if i == 0 {
// This is a bit of a hack, but client to server headers are only readable after first client msg is
// received but must be written to server stream before the first msg is flushed.
// This is the only place to do it nicely.
md, err := src.Header()
if err != nil {
ret <- err
break
}
if err := dst.SendHeader(md); err != nil {
ret <- err
break
}
}
if err := dst.SendMsg(f); err != nil {
ret <- err
break
}
}
}()
return ret
}
func (s *handler) forwardServerToClient(src grpc.ServerStream, dst grpc.ClientStream) chan error {
ret := make(chan error, 1)
go func() {
f := &wheatCodec.Frame{}
for i := 0; ; i++ {
if err := src.RecvMsg(f); err != nil {
ret <- err // this can be io.EOF which is happy case
break
}
if err := dst.SendMsg(f); err != nil {
ret <- err
break
}
}
}()
return ret
}

5
go.mod
View File

@ -1,12 +1,11 @@
module gitee.com/wheat-os/wheatCache
module gitee.com/timedb/wheatCache
go 1.16
require (
github.com/golang/mock v1.5.0
github.com/spf13/cobra v1.2.1
github.com/spf13/viper v1.8.1
github.com/stretchr/testify v1.7.0
google.golang.org/grpc v1.41.0
google.golang.org/grpc v1.38.0
google.golang.org/protobuf v1.26.0
)

8
go.sum
View File

@ -46,7 +46,6 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@ -54,7 +53,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@ -67,7 +65,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
@ -89,7 +86,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g=
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@ -256,7 +252,6 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
@ -562,9 +557,8 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E=
google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

View File

@ -2,36 +2,23 @@ BASE_PATH = $(shell pwd)
STORAGE_PATH = $(BASE_PATH)/storage
BASE_OUT = $(BASE_PATH)/bin
MAKEFLAGS+= --no-print-directory
dcgen:
@python3 ./shell/gen_protobuf.py
@python3 ./shell/proto.py
@python3 ./shell/make-struct.py
.PHONY: build-storage
build-storage:
.PHONY : build
build:
@cd storage && go build -o $(BASE_OUT)/storage
.PHONY: build-gateway
build-gateway:
@cd gateway && go build -o $(BASE_OUT)/gateway
.PHONY: install
install:
@make gen-middleware
@make build-storage
@make build-gateway
@sudo python3 ./shell/init_conf.py
@make build
.PHONY: storage
storage:
.PHONY: dev
dev:
@./bin/storage storage
.PHONY: gateway
gateway:
@./bin/gateway gateway
.PHONY: gen-struct
gen-struct:
@python3 ./shell/make-struct.py
@ -39,20 +26,3 @@ gen-struct:
.PHONY: gen-protobuf
gen-protobuf:
@python3 ./shell/gen_protobuf.py
.PHONY: gen-middleware
gen-middleware:
@python3 ./shell/gen_middleware.py
.PHONY: init-conf
init-conf:
@python3 ./shell/init_conf.py
.PHONY: gen-service
gen-service:
@python3 ./shell/make_service.py
.PHONY: gen-mock
gen-mock:
@mockgen -source=./pkg/proto/storage.pb.go CommServerClient > ./mock/storage/mock_client.gen.go

File diff suppressed because it is too large Load Diff

View File

@ -1,9 +0,0 @@
package errorx
func DaoTypeErr(typ string) error {
return New("the type is not: %s", typ)
}
func NotKeyErr(key string) error {
return New("the key is not exist, key:%s", key)
}

View File

@ -6,7 +6,7 @@ import (
)
// New TODO 添加链路追踪等 @bandl @lgq
func New(msg string, format ...interface{}) error {
msg = fmt.Sprintf(msg, format...)
func New(msg string, opt ...interface{}) error {
msg = fmt.Sprintf(msg, opt...)
return errors.New(msg)
}

View File

@ -1,5 +0,0 @@
package errorx
func EventRecoveryErr() error {
return New("this event has been recycled")
}

View File

@ -1,9 +0,0 @@
package errorx
func LruNotWorkFuncEventErr() error {
return New("the event haven't work of function")
}
func KeyBaseIsNilErr() error {
return New("key base can't be nil")
}

View File

@ -1,5 +0,0 @@
package errorx
func TimeOutErr() error {
return New("time out err")
}

View File

@ -6,18 +6,10 @@ type Consumer struct {
driver DriverInterface
}
func (c *Consumer) Receive(ctx context.Context) *event {
func (c *Consumer) Receive(ctx context.Context) *Event {
return c.driver.Get()
}
func (c *Consumer) NewEvent(name string) *event {
return c.driver.NewEvent(name)
}
func (c *Consumer) Recovery(e *event) {
c.driver.Recovery(e)
}
func NewConsumer(driver DriverInterface) ConsumerInterface {
return &Consumer{
driver: driver,

View File

@ -4,30 +4,22 @@ import (
"context"
)
type eventType int8
const (
defaultEventState = int32(iota) //默认情况下的状态
waitEventState // 等待状态
workEventState //工作状态
closeEventState //事件关闭状态
normalEvent = eventType(iota)
waitEvent
)
type EventWorkFunc func() (interface{}, error)
type DriverInterface interface {
Get() *event
Put(*event)
GetLength() int
NewEvent(string) *event
Recovery(*event)
Get() *Event
Put(event *Event)
}
type ProduceInterface interface {
Call(context.Context, *event)
NewEvent(string) *event
Recovery(*event)
Call(ctx context.Context, event *Event)
}
type ConsumerInterface interface {
Receive(ctx context.Context) *event
Recovery(*event)
Receive(ctx context.Context) *Event
}

View File

@ -1,189 +1,121 @@
package event
import (
"sync/atomic"
"gitee.com/timedb/wheatCache/pkg/errorx"
"sync"
"time"
"gitee.com/wheat-os/wheatCache/pkg/errorx"
)
// 事件 poll 降低 new 对象的频率
type eventPoll struct {
poll chan *event
maxSize int32
nowSize *int32
}
type Active func() ([]string, error) // 事件带函数
func (e *eventPoll) getEvent() *event {
issSize := atomic.LoadInt32(e.nowSize)
if issSize < e.maxSize {
atomic.AddInt32(e.nowSize, 1)
return newEvent()
}
return <-e.poll
}
func (e *eventPoll) recovery(rEvent *event) {
rEvent.Reset()
e.poll <- rEvent
}
func newEventPoll(maxSize int) *eventPoll {
return &eventPoll{
poll: make(chan *event, maxSize),
maxSize: int32(maxSize),
nowSize: new(int32),
}
}
type event struct {
type Event struct {
msgCtx map[string]interface{}
eventName string
WorkTime time.Duration // 工作时间
msg map[string]string // 消息
waitResult chan interface{} // 等待返回
err error
eventStatus *int32
ttlManage *time.Timer
ru sync.RWMutex
eventOnType eventType
}
func newEvent() *event {
status := defaultEventState
return &event{
eventStatus: &status,
}
}
func (e *event) Reset() {
if e.ttlManage != nil {
e.ttlManage.Stop()
}
e.err = nil
atomic.SwapInt32(e.eventStatus, defaultEventState)
}
func (e *event) SetMsg(key string, val string) {
func (e *Event) SetMsg(key string, val string) {
e.ru.Lock()
defer e.ru.Unlock()
if e.msg == nil {
e.msg = make(map[string]string)
}
e.msg[key] = val
}
func (e *event) GetMsg(key string) string {
func (e *Event) GetMsg(key string) string {
e.ru.RLock()
defer e.ru.RUnlock()
return e.msg[key]
}
func (e *event) GetEventName() string {
return e.eventName
}
// SetValue 写入 ctx 传递用参数
func (e *event) SetValue(key string, value interface{}) {
// SetCtxValue 写入 ctx 传递用参数
func (e *Event) SetCtxValue(key string, value interface{}) {
e.ru.Lock()
defer e.ru.Unlock()
if e.msgCtx == nil {
e.msgCtx = make(map[string]interface{})
}
e.msgCtx[key] = value
}
func (e *event) GetValue(key string) (interface{}, bool) {
func (e *Event) GetCtxValue(key string) (interface{}, bool) {
e.ru.RLock()
defer e.ru.RUnlock()
val, ok := e.msgCtx[key]
return val, ok
}
// InitWaitEvent 初始化 wait event 必须调用才拥有等待特性
func (e *event) InitWaitEvent() {
if e.waitResult == nil || len(e.waitResult) > 0 {
e.waitResult = make(chan interface{})
}
// 清理残留
if e.ttlManage == nil {
e.ttlManage = time.NewTimer(0)
}
e.ttlManage.Stop()
if len(e.ttlManage.C) > 0 {
<-e.ttlManage.C
}
atomic.CompareAndSwapInt32(e.eventStatus, defaultEventState, waitEventState)
}
// StartWaitEvent 开始一个等待任务
func (e *event) StartWaitEvent(ttl time.Duration) (interface{}, error) {
e.ttlManage.Reset(ttl)
for {
select {
case <-e.ttlManage.C:
if atomic.CompareAndSwapInt32(e.eventStatus, waitEventState, closeEventState) {
return nil, errorx.TimeOutErr()
}
continue
case result := <-e.waitResult:
atomic.CompareAndSwapInt32(e.eventStatus, workEventState, closeEventState)
return result, e.err
func (e *Event) SetWaitResult(val interface{}) (err error) {
defer func() {
if errChan := recover(); errChan != nil {
err = errorx.New("channel err:%v", errChan)
}
}()
e.waitResult <- val
return nil
}
// UpdateWaitEvent 升级到 wait Event
func (e *Event) UpdateWaitEvent(ttl time.Duration) (<-chan interface{}, error) {
if e.eventOnType == waitEvent {
return nil, errorx.New("the upgrade cannot be repeated")
}
ttl = 1 * time.Second
if ttl > 0 {
e.WorkTime = ttl
}
e.waitResult = make(chan interface{})
e.eventOnType = waitEvent
return e.waitResult, nil
}
// GetTtlTimer 只对 wait Event 有效
func (e *Event) GetTtlTimer() (*time.Timer, error) {
if e.eventOnType != waitEvent {
return nil, errorx.New("cannot be called in normalEvent")
}
timer := time.NewTimer(e.WorkTime)
return timer, nil
}
func NewEvent(eventName string) *Event {
return &Event{
eventName: eventName,
eventOnType: normalEvent,
}
}
func (e *event) ExecWorkAndSendResult(work EventWorkFunc) (interface{}, error) {
if !atomic.CompareAndSwapInt32(e.eventStatus, waitEventState, workEventState) {
return nil, errorx.New("not wait status, exec err")
}
res, err := work()
e.err = err
e.waitResult <- res
return res, err
}
func (e *event) SetResultErr(err error) {
if !atomic.CompareAndSwapInt32(e.eventStatus, waitEventState, workEventState) {
return
}
e.err = err
e.waitResult <- nil
// Close 关闭结束事件
func (e *Event) Close() {
close(e.waitResult)
}
type Driver struct {
maxQueueSize int
queue chan *event
poll *eventPoll
queue chan *Event
}
// Get 获取驱动
func (d *Driver) Get() *event {
func (d *Driver) Get() *Event {
return <-d.queue
}
func (d *Driver) Put(event *event) {
func (d *Driver) Put(event *Event) {
d.queue <- event
}
func (d *Driver) GetLength() int {
return len(d.queue)
}
func (d *Driver) NewEvent(name string) *event {
event := d.poll.getEvent()
event.eventName = name
return event
}
// 任何时候回收事件都应该由 最后使用者回收
func (d *Driver) Recovery(e *event) {
d.poll.recovery(e)
}
// NewDriver 新建 Driver
func NewDriver(maxSize int) DriverInterface {
return &Driver{
maxQueueSize: maxSize,
queue: make(chan *event, maxSize),
poll: newEventPoll(maxSize),
queue: make(chan *Event, maxSize),
}
}

View File

@ -3,87 +3,110 @@ package event
import (
"context"
"fmt"
"strconv"
"sync"
"github.com/stretchr/testify/require"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestEvent_SetWaitResult(t *testing.T) {
event := &Event{
waitResult: make(chan interface{}),
}
c, err := event.UpdateWaitEvent(2 * time.Second)
require.NoError(t, err)
go getValueByChannel(c)
err = event.SetWaitResult(1)
require.NoError(t, err)
event.Close()
err = event.SetWaitResult(1)
require.Error(t, err)
}
func getValueByChannel(c <-chan interface{}) {
for {
if i, isClose := <-c; !isClose {
break
} else {
fmt.Println(i)
}
}
}
const testEvent = "1001"
const waitTestEvent = "1002"
// 简单的 单向 event 使用
func Test_EventDriver(t *testing.T) {
driver := NewDriver(2000)
// 简单 非等待响应模式, 使用 event driver
func TestEvent_DriverEventTest(t *testing.T) {
ctx := context.Background()
driver := NewDriver(500)
produce := NewProduce(driver)
consumer := NewConsumer(driver)
ctx := context.Background()
wait := sync.WaitGroup{}
wait.Add(30000)
go func() {
for i := 0; i < 30000; i++ {
event := produce.NewEvent(testEvent)
event.SetMsg("k", strconv.Itoa(i))
produce.Call(ctx, event)
}
}()
go func() {
for {
event := consumer.Receive(ctx)
fmt.Println(event.GetMsg("k"))
consumer.Recovery(event)
wait.Done()
}
}()
wait.Wait()
fmt.Println(*driver.(*Driver).poll.nowSize)
go produceEvent(t, ctx, produce)
consumerEvent(t, ctx, consumer)
}
// 双向 event
func Test_WaitEventDriver(t *testing.T) {
driver := NewDriver(200)
func produceEvent(t *testing.T, ctx context.Context, v ProduceInterface) {
for i := 0; i < 100; i++ {
event := NewEvent(testEvent)
event.SetCtxValue("test", i)
v.Call(ctx, event)
}
}
func consumerEvent(t *testing.T, ctx context.Context, v ConsumerInterface) {
for i := 0; i < 100; i++ {
event := v.Receive(ctx)
res, ok := event.GetCtxValue("test")
require.True(t, ok)
fmt.Println(res)
require.Equal(t, res, i)
}
}
// 响应等待用法
func TestEvent_SpanWaitEvent(t *testing.T) {
ctx := context.Background()
driver := NewDriver(500)
produce := NewProduce(driver)
consumer := NewConsumer(driver)
ctx := context.Background()
go waitConsumer(t, ctx, consumer)
wait := sync.WaitGroup{}
wait.Add(300000)
go func() {
for i := 0; i < 300000; i++ {
event := produce.NewEvent(testEvent)
event.SetMsg("k", strconv.Itoa(i))
event.InitWaitEvent()
produce.Call(ctx, event)
val, err := event.StartWaitEvent(2 * time.Second)
require.NoError(t, err)
fmt.Println(val)
produce.Recovery(event)
wait.Done()
}
}()
go func() {
for {
event := consumer.Receive(ctx)
event.ExecWorkAndSendResult(func() (interface{}, error) {
msg := event.GetMsg("k")
return "hello: " + msg, nil
})
}
}()
wait.Wait()
fmt.Println(*driver.(*Driver).poll.nowSize)
waitProduce(t, ctx, produce)
}
func waitProduce(t *testing.T, ctx context.Context, v ProduceInterface) {
for i := 0; i < 100; i++ {
event := NewEvent(waitTestEvent)
waitChan, err := event.UpdateWaitEvent(2 * time.Second) // 设置事件过期时间, 获取结果等待对象
require.NoError(t, err)
event.SetCtxValue("test", i)
v.Call(ctx, event) // 推送给 consumer
timer, err := event.GetTtlTimer()
require.NoError(t, err)
select {
case result := <-waitChan:
require.Equal(t, result, fmt.Sprintf("test:%v", i))
fmt.Println(result)
case <-timer.C:
panic("time out")
}
}
}
func waitConsumer(t *testing.T, ctx context.Context, v ConsumerInterface) {
for i := 0; i < 100; i++ {
event := v.Receive(ctx) // 接受 produce 的 event
res, ok := event.GetCtxValue("test")
require.True(t, ok)
require.Equal(t, res, i)
err := event.SetWaitResult(fmt.Sprintf("test:%v", res)) // 发送返回值给 produce
require.NoError(t, err)
}
}

View File

@ -6,16 +6,8 @@ type Produce struct {
driver DriverInterface
}
func (p *Produce) NewEvent(name string) *event {
return p.driver.NewEvent(name)
}
func (p *Produce) Recovery(e *event) {
p.driver.Recovery(e)
}
func (p *Produce) Call(ctx context.Context, e *event) {
p.driver.Put(e)
func (p *Produce) Call(ctx context.Context, event *Event) {
p.driver.Put(event)
}
func NewProduce(driver DriverInterface) ProduceInterface {

View File

@ -1,21 +0,0 @@
package event2
import "context"
type Consumer struct {
driver DriverInterface
}
func (c *Consumer) Receive(ctx context.Context) *event {
return c.driver.Get()
}
func (c *Consumer) NewEvent(name string) *event {
return c.driver.NewEvent(name)
}
func NewConsumer(driver DriverInterface) ConsumerInterface {
return &Consumer{
driver: driver,
}
}

View File

@ -1,49 +0,0 @@
package event2
import (
"context"
)
const (
initEventState = int32(iota) // 初始化状态
waitEventState // 等待状态
workEventState // 工作状态
closeEventState // 事件关闭状态
recoveryEventState // 事件回收状态
)
const (
awaitThread = 3
)
const (
WorkFuncEventKey = "workFunc"
)
// 线程安全
type EventWorkFunc func() (interface{}, error)
// 挂起事件, 线程不安全
type EventAwaitFunc func() (interface{}, error)
// 实际操作
type awaitFunc func() (*event, interface{}, error)
type DriverInterface interface {
Get() *event
Put(*event)
GetLength() int
NewEvent(string) *event
await(awaitFunc)
recovery(e *event)
}
type ProduceInterface interface {
Call(context.Context, *event)
NewEvent(string) *event
}
type ConsumerInterface interface {
Receive(ctx context.Context) *event
}

View File

@ -1,221 +0,0 @@
package event2
import (
"sync/atomic"
"time"
"gitee.com/wheat-os/wheatCache/pkg/errorx"
)
type event struct {
msgCtx map[string]interface{}
eventName string
msg map[string]string // 消息
waitResult chan interface{} // 等待返回
err error
eventStatus int32
ttlManage *time.Timer
parentDriver DriverInterface
}
func (e *event) reset() {
if e.ttlManage != nil {
e.ttlManage.Stop()
if len(e.ttlManage.C) > 0 {
<-e.ttlManage.C
}
}
e.err = nil
// 清空结果
if len(e.waitResult) != 0 {
<-e.waitResult
}
}
func (e *event) Recovery() {
e.parentDriver.recovery(e)
}
func (e *event) SetMsg(key string, val string) {
if e.msg == nil {
e.msg = make(map[string]string)
}
e.msg[key] = val
}
func (e *event) GetMsg(key string) string {
if e.msg == nil {
return ""
}
return e.msg[key]
}
func (e *event) GetEventName() string {
return e.eventName
}
// SetValue 写入 ctx 传递用参数
func (e *event) SetValue(key string, value interface{}) {
if e.msgCtx == nil {
e.msgCtx = make(map[string]interface{})
}
e.msgCtx[key] = value
}
func (e *event) GetValue(key string) (interface{}, bool) {
if e.msgCtx == nil {
return nil, false
}
val, ok := e.msgCtx[key]
return val, ok
}
func (e *event) InitWaitEvent() {
e.reset()
if e.waitResult == nil {
e.waitResult = make(chan interface{})
}
atomic.SwapInt32(&e.eventStatus, waitEventState)
}
func (e *event) SetResultErr(err error) {
if !atomic.CompareAndSwapInt32(&e.eventStatus, waitEventState, workEventState) {
return
}
e.err = err
e.waitResult <- nil
}
// StartWaitEvent 开始一个等待任务
func (e *event) StartWaitEvent(ttl time.Duration) (interface{}, error) {
if e.ttlManage == nil {
e.ttlManage = time.NewTimer(ttl)
} else {
e.ttlManage.Reset(ttl)
}
for {
select {
case <-e.ttlManage.C:
if atomic.CompareAndSwapInt32(&e.eventStatus, waitEventState, closeEventState) {
return nil, errorx.TimeOutErr()
}
continue
case result := <-e.waitResult:
atomic.SwapInt32(&e.eventStatus, closeEventState)
return result, e.err
}
}
}
// 实际执行推送
func (e *event) execWorker(res interface{}, err error) {
switch work := res.(type) {
case EventAwaitFunc:
await := func() (*event, interface{}, error) {
result, err := work()
return e, result, err
}
e.parentDriver.await(await)
case EventWorkFunc:
e.InitWaitEvent()
e.SetValue(WorkFuncEventKey, work)
e.parentDriver.Put(e)
default:
e.err = err
e.waitResult <- res
}
}
func (e *event) ExecWorkAndSendResult(work EventWorkFunc) (interface{}, error) {
if !atomic.CompareAndSwapInt32(&e.eventStatus, waitEventState, workEventState) {
return nil, errorx.New("not wait status, exec err")
}
res, err := work()
e.execWorker(res, err)
return res, err
}
type driver struct {
waitQueue chan awaitFunc
eventQueue chan *event
levelQueue chan *event
// event 池的实现
poll chan *event
maxPoolSize int32
nowPoolSize int32
}
func NewDriver(maxSize int) DriverInterface {
d := &driver{
// pool
maxPoolSize: int32(maxSize),
nowPoolSize: 0,
poll: make(chan *event, maxSize),
// waitQueue 1/3 的挂起指标
waitQueue: make(chan awaitFunc, maxSize/3),
levelQueue: make(chan *event, maxSize/3),
eventQueue: make(chan *event, maxSize),
}
d.awaitWorker()
return d
}
func (d *driver) NewEvent(name string) *event {
issSize := atomic.LoadInt32(&d.nowPoolSize)
if issSize < d.maxPoolSize {
atomic.AddInt32(&d.nowPoolSize, 1)
return d.newEvent(name)
}
e := <-d.poll
e.eventName = name
return e
}
func (d *driver) newEvent(name string) *event {
status := initEventState
return &event{
eventStatus: status,
parentDriver: d,
eventName: name,
}
}
// 先尝试 level
func (d *driver) Get() *event {
if len(d.levelQueue) > 0 {
return <-d.levelQueue
}
return <-d.eventQueue
}
func (d *driver) Put(e *event) {
d.eventQueue <- e
}
func (d *driver) GetLength() int {
return len(d.eventQueue) + len(d.levelQueue)
}
func (d *driver) recovery(e *event) {
atomic.SwapInt32(&e.eventStatus, recoveryEventState)
e.reset()
d.poll <- e
}
// 挂起操作相关
func (d *driver) await(a awaitFunc) {
d.waitQueue <- a
}

View File

@ -1,152 +0,0 @@
package event2
import (
"context"
"fmt"
"strconv"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
)
const testEvent = "1001"
const waitTestEvent = "1002"
// 简单的 单向 event 使用
func Test_EventDriver(t *testing.T) {
driver := NewDriver(2000)
produce := NewProduce(driver)
consumer := NewConsumer(driver)
ctx := context.Background()
wait := sync.WaitGroup{}
wait.Add(30000)
go func() {
for i := 0; i < 30000; i++ {
event := produce.NewEvent(testEvent)
event.SetMsg("k", strconv.Itoa(i))
produce.Call(ctx, event)
}
}()
go func() {
for {
event := consumer.Receive(ctx)
fmt.Println(event.GetMsg("k"))
event.Recovery()
wait.Done()
}
}()
wait.Wait()
}
// 双向 event
func Test_EventDriver_Tow_way(t *testing.T) {
ctx := context.Background()
driver := NewDriver(2000)
produce := NewProduce(driver)
consumer := NewConsumer(driver)
go func() {
for {
event := consumer.Receive(ctx)
work, ok := event.GetValue(WorkFuncEventKey)
if !ok {
panic("get work key err")
}
workFunc, ok := work.(EventWorkFunc)
if !ok {
panic("work func err")
}
_, err := event.ExecWorkAndSendResult(workFunc)
require.NoError(t, err)
}
}()
// 一般的 two-way 模式
for i := 0; i < 10000; i++ {
event := produce.NewEvent(waitTestEvent)
event.InitWaitEvent()
event.SetValue(WorkFuncEventKey, EventWorkFunc(func() (interface{}, error) {
return i + 1, nil
}))
produce.Call(ctx, event)
res, err := event.StartWaitEvent(2 * time.Second)
require.NoError(t, err)
require.Equal(t, res, i+1)
event.Recovery()
}
// 挂起模式2 秒左右的执行时间
group := sync.WaitGroup{}
group.Add(5)
for i := 0; i < 5; i++ {
go func(i int) {
event := produce.NewEvent(waitTestEvent)
event.InitWaitEvent()
event.SetValue(WorkFuncEventKey, EventWorkFunc(func() (interface{}, error) {
// 访问 await Work 来发起一个 异步请求操作
return EventAwaitFunc(func() (interface{}, error) {
time.Sleep(time.Second)
return i + 1, nil
}), nil
}))
produce.Call(ctx, event)
res, err := event.StartWaitEvent(2 * time.Second)
require.NoError(t, err)
require.Equal(t, res, i+1)
event.Recovery()
group.Done()
}(i)
}
// 挂起成功不发生超时
for i := 0; i < 10000; i++ {
event := produce.NewEvent(waitTestEvent)
event.InitWaitEvent()
event.SetValue(WorkFuncEventKey, EventWorkFunc(func() (interface{}, error) {
return i + 1, nil
}))
produce.Call(ctx, event)
res, err := event.StartWaitEvent(500 * time.Millisecond)
require.NoError(t, err)
require.Equal(t, res, i+1)
event.Recovery()
}
group.Wait()
// 挂起一个高延迟操作, 保证局部操作还在事件中
group = sync.WaitGroup{}
group.Add(5)
for i := 0; i < 5; i++ {
event := produce.NewEvent(waitTestEvent)
event.InitWaitEvent()
event.SetValue(WorkFuncEventKey, EventWorkFunc(func() (interface{}, error) {
return EventAwaitFunc(func() (interface{}, error) {
// 返回值为 EventWorkFunc 时, 会重新加入末端队列
return EventWorkFunc(func() (interface{}, error) {
return i + 1, nil
}), nil
}), nil
}))
produce.Call(ctx, event)
res, err := event.StartWaitEvent(2 * time.Second)
require.NoError(t, err)
require.Equal(t, res, i+1)
event.Recovery()
group.Done()
fmt.Println(i)
}
group.Wait()
}

View File

@ -1,21 +0,0 @@
package event2
import "context"
type Produce struct {
driver DriverInterface
}
func (p *Produce) NewEvent(name string) *event {
return p.driver.NewEvent(name)
}
func (p *Produce) Call(ctx context.Context, e *event) {
p.driver.Put(e)
}
func NewProduce(driver DriverInterface) ProduceInterface {
return &Produce{
driver: driver,
}
}

View File

@ -1,13 +0,0 @@
package event2
func (d *driver) awaitWorker() {
for i := 0; i < awaitThread; i++ {
go func() {
for {
awaitFunc := <-d.waitQueue
e, res, err := awaitFunc()
e.execWorker(res, err)
}
}()
}
}

1
pkg/log/define.go Normal file
View File

@ -0,0 +1 @@
package log

View File

@ -1,43 +0,0 @@
package logx
import (
"context"
"sync"
"gitee.com/wheat-os/wheatCache/pkg/event"
"github.com/spf13/viper"
)
type LogLevelState int8
var (
once sync.Once
stath []string
)
type upLogger struct {
ctx context.Context
produce event.ProduceInterface
}
func init() {
once.Do(func() {
stath = viper.GetStringSlice("logPrint.stath")
})
}
type logInterface interface {
Debug(format string, msg ...interface{})
Info(format string, msg ...interface{})
Warn(format string, msg ...interface{})
Error(format string, msg ...interface{})
Panic(format string, msg ...interface{})
Debugln(msg ...interface{})
Infoln(msg ...interface{})
Warnln(msg ...interface{})
Errorln(msg ...interface{})
Panicln(msg ...interface{})
Print(level string, format string, msg ...interface{})
}

View File

@ -1,161 +0,0 @@
package logx
import (
"context"
"fmt"
"os"
"runtime"
"strings"
"time"
"gitee.com/wheat-os/wheatCache/pkg/event"
middleMsg "gitee.com/wheat-os/wheatCache/pkg/middle-msg"
)
func With(ctx context.Context, p event.ProduceInterface) *upLogger {
return &upLogger{
ctx: ctx,
produce: p,
}
}
func (l *upLogger) Debug(format string, msg ...interface{}) {
l.Print("DEBUG", format, msg...)
}
func (l *upLogger) Info(format string, msg ...interface{}) {
l.Print("INFO", format, msg...)
}
func (l *upLogger) Warn(format string, msg ...interface{}) {
l.Print("WARN", format, msg...)
}
func (l *upLogger) Error(format string, msg ...interface{}) {
l.Print("ERROR", format, msg...)
}
func (l *upLogger) Panic(format string, msg ...interface{}) {
l.Print("ERROR", format, msg...)
os.Exit(-1)
}
func (l *upLogger) Print(level string, format string, msg ...interface{}) {
logPrint(4, level, format, msg...)
sendMsg := &middleMsg.LogContext{
Level: level,
Data: time.Now(),
Msg: fmt.Sprintf(format, msg...),
Route: findPlace(4),
}
middleMsg.SendMiddleMsg(l.ctx, l.produce, sendMsg)
}
func (l *upLogger) Debugln(msg ...interface{}) {
l.Print("DEBUG", "%s", format(msg...))
}
func (l *upLogger) Infoln(msg ...interface{}) {
l.Print("INFO", "%s", format(msg...))
}
func (l *upLogger) Warnln(msg ...interface{}) {
l.Print("WARN", "%s", format(msg...))
}
func (l *upLogger) Errorln(msg ...interface{}) {
l.Print("ERROR", "%s", format(msg...))
}
func (l *upLogger) Panicln(msg ...interface{}) {
l.Print("ERROR", "%s", format(msg...))
os.Exit(-1)
}
func Debug(format string, msg ...interface{}) {
logPrint(3, "DEBUG", format, msg...)
}
func Info(format string, msg ...interface{}) {
logPrint(3, "INFO", format, msg...)
}
func Warn(format string, msg ...interface{}) {
logPrint(3, "WARN", format, msg...)
}
func Error(format string, msg ...interface{}) {
logPrint(3, "ERROR", format, msg...)
}
func Panic(format string, msg ...interface{}) {
logPrint(3, "PANIC", format, msg...)
os.Exit(-1)
}
func Debugln(msg ...interface{}) {
logPrint(3, "DEBUG", "%s", format(msg...))
}
func Infoln(msg ...interface{}) {
logPrint(3, "INFO", "%s", format(msg...))
}
func Warnln(msg ...interface{}) {
logPrint(3, "WARN", "%s", format(msg...))
}
func Errorln(msg ...interface{}) {
logPrint(3, "ERROR", "%s", format(msg...))
}
func Panicln(msg ...interface{}) {
logPrint(3, "PANIC", "%s", format(msg...))
os.Exit(-1)
}
func logPrint(floor int, level string, format string, msg ...interface{}) {
place := findPlace(floor)
datetime := fmt.Sprintf("%s", time.Now())[0:19]
switch level {
case "DEBUG":
fmt.Printf("\033[1;37;40m")
case "INFO":
fmt.Printf("\033[1;32;40m")
case "WARN":
fmt.Printf("\033[1;33;40m")
default:
fmt.Printf("\033[1;31;40m")
}
//fmt.Println(level, datetime, fmt.Sprintf(format, msg...))
fmt.Printf("%s\t%v\t%s\n", level, datetime, fmt.Sprintf(format, msg...))
for _, lv := range stath {
if strings.ToUpper(lv) == strings.ToUpper(level) {
fmt.Println(place)
break
}
}
}
func findPlace(floor int) string {
var (
place string
i = floor
)
for {
_, file, line, _ := runtime.Caller(i)
if line == 0 {
break
}
i++
place = fmt.Sprintf("%s:%d\n%s", file, line, place)
}
return place
}
func format(message ...interface{}) (context string) {
for _, msg := range message {
context = fmt.Sprintf("%s\t%v", context, msg)
}
return context
}

View File

@ -1,38 +0,0 @@
package logx
import (
"context"
"testing"
_ "gitee.com/wheat-os/wheatCache/conf"
"gitee.com/wheat-os/wheatCache/pkg/event"
)
func TestStd(t *testing.T) {
Info("%d%s", 11, "Info")
Debug("%d%s", 11, "Debug")
Warn("%d%s", 11, "Warn")
Error("%d%s", 11, "Error")
Infoln(1321, "dwad", 0x9812933)
Debugln(1321, "dwad", 0x9812933)
Warnln(1321, "dwad", 0x9812933)
Errorln(1321, "dwad", 0x9812933)
//Panic("%d%s", 11, "Panic")
logger := With(context.Background(),
event.NewProduce(event.NewDriver(100)))
logger.Info("%d%s", 11, "Info")
logger.Debug("%d%s", 11, "Debug")
logger.Warn("%d%s", 11, "Warn")
logger.Error("%d%s", 11, "Error")
//logger.Panic("%d%s", 11, "Panic")
logger.Infoln(11, "Info")
logger.Debugln(11, "Debug")
logger.Warnln(11, "Warn")
logger.Errorln(11, "Error")
}

101
pkg/lru/Lru.go Normal file
View File

@ -0,0 +1,101 @@
package lru
import (
"container/list"
"errors"
"gitee.com/timedb/wheatCache/pkg/structure"
)
// feat
/*
1. cleanProduce
2. 定义 LRUSingle Work 函数
*/
type Node struct {
Key interface{}
value structure.KeyBaseInterface
}
func (*Node) NewCacheNode(k interface{}, v structure.KeyBaseInterface) *Node {
return &Node{Key: k, value: v}
}
type LRUCache struct {
maxsize int64 //最大的长度
clearsize int64 // 清理长度
nowsize int64 // 现在的长度
li *list.List
LruMap map[interface{}]*list.Element // TODO LRU 对外暴露
}
// NewLRUCache lru初始化 TODO 单例模式
func NewLRUCache(msize, csize int64) *LRUCache {
return &LRUCache{
maxsize: msize,
clearsize: csize,
nowsize: 0,
li: list.New(),
LruMap: make(map[interface{}]*list.Element)}
}
// Add 增 TODO 大问题
func (lru *LRUCache) Add(k interface{}, v structure.KeyBaseInterface) error {
if lru.li == nil {
return nil // TODO err
}
// TODO 封装 Key 迁移方法
if PreEle, ok := lru.LruMap[k]; ok {
lru.li.MoveToFront(PreEle)
PreEle.Value.(*Node).value = v
lru.nowsize += PreEle.Value.(*Node).value.SizeByte()
return nil
}
// TODO 增加的情况下, 应该只维护 maxSize
if lru.nowsize >= lru.maxsize*1/3 {
lru.nowsize -= lru.maxsize
if PreEle, ok := lru.LruMap[k]; ok {
cacheNode := PreEle.Value.(*Node).value
delete(lru.LruMap, cacheNode.SizeByte())
lru.li.Remove(PreEle)
}
}
newEle := lru.li.PushFront(&Node{k, v})
lru.LruMap[k] = newEle // TODO nil 指针
return nil
}
// Get 查 // TODO err
func (lru *LRUCache) Get(k interface{}) (v structure.KeyBaseInterface, ret bool) {
if lru.LruMap == nil {
return v, false
}
if PreEle, ok := lru.LruMap[k]; ok {
// 移到最前面
lru.li.MoveToFront(PreEle)
return PreEle.Value.(*Node).value, true
}
return v, false
}
// Clear 删除 // TODO 方案不对
func (lru *LRUCache) Clear(k interface{}) error {
if lru.LruMap == nil {
return errors.New("cache 为空")
}
for lru.nowsize >= lru.clearsize*1/3 {
if PreEle, ok := lru.LruMap[k]; ok {
cacheNode := PreEle.Value.(*Node).value
delete(lru.LruMap, cacheNode)
lru.li.Remove(PreEle)
return nil
}
}
return nil
}

View File

@ -1,46 +1,3 @@
package lru
import (
"sync"
"time"
"gitee.com/wheat-os/wheatCache/pkg/proto"
"gitee.com/wheat-os/wheatCache/pkg/structure"
)
type SingleWorkFunc func() interface{}
const (
OptionEventName = "operateEvent"
CleanEventName = "clearEvent"
TtlEventName = "ttlEvent"
)
var (
lruCacheOnce sync.Once
lruCache *SingleCache
)
const (
defaultLruMaxSize = 1 * 1024 * 1024 * 1024
defaultLruClearSize = 0.5 * 1024 * 1024 * 1024
defaultLruEventDriver = 2000
)
const (
defaultWaitTime = 20 * time.Minute
)
type CacheInterface interface {
Del() error
Get(key *proto.BaseKey) (structure.KeyBaseInterface, bool)
Add(key *proto.BaseKey, val structure.KeyBaseInterface) error
UpdateLruSize(length structure.UpdateLength)
DelByKey(key *proto.BaseKey) error
DelToClearSize() error
}
// TTL
const (
defaultDetachNum = 300
defaultTtlMaxLevel = 18
)
type SingleWorkFunc func()

View File

@ -1,225 +0,0 @@
package lru
import (
"container/list"
"sync/atomic"
_ "gitee.com/wheat-os/wheatCache/conf"
"gitee.com/wheat-os/wheatCache/pkg/errorx"
"gitee.com/wheat-os/wheatCache/pkg/event"
"gitee.com/wheat-os/wheatCache/pkg/event2"
"gitee.com/wheat-os/wheatCache/pkg/middle"
"gitee.com/wheat-os/wheatCache/pkg/proto"
"gitee.com/wheat-os/wheatCache/pkg/structure"
"gitee.com/wheat-os/wheatCache/pkg/util"
"github.com/spf13/viper"
)
type keyBaseValue struct {
key string
val structure.KeyBaseInterface
expire int64 // 过期时间戳
}
type SingleCache struct {
maxsize int64 //最大的长度
clearSize int64 // 清理长度
nowSize int64 // 现在的长度
li *list.List
lruMap map[string]*list.Element
lruMaxDiverSize int
lruTtlManage *lruTTl // 定时清理器
lruDriver event2.DriverInterface
lruConsumer event2.ConsumerInterface
lruCleanProduce event2.ProduceInterface // 发送清理事件
middleProduce event.ProduceInterface // 中间件驱动
}
// UpdateLruSize 更新现在的长度
func (lru *SingleCache) UpdateLruSize(length structure.UpdateLength) {
atomic.AddInt64(&lru.nowSize, int64(length))
}
func cacheInit() (int64, int64, int, int) {
maxSize := viper.GetString("lruCache.maxSize")
retMaxSize, maxErr := util.ParseSizeToBit(maxSize)
if maxErr != nil {
return 0, 0, 0, 0
}
if retMaxSize == 0 {
retMaxSize = defaultLruMaxSize
}
clearSize := viper.GetString("lruCache.clearSize")
retClearSize, clearErr := util.ParseSizeToBit(clearSize)
if clearErr != nil {
return 0, 0, 0, 0
}
if retClearSize == 0 {
retClearSize = defaultLruClearSize
}
maxDriver := viper.GetInt("lruCache.eventDriverSize")
if maxDriver == 0 {
maxDriver = defaultLruEventDriver
}
detachNum := viper.GetInt("lruCache.detachNum")
if detachNum == 0 {
detachNum = defaultDetachNum
}
return retMaxSize, retClearSize, maxDriver, detachNum
}
// NewLRUCache lru初始化
func NewLRUCache() *SingleCache {
maxSize, clearSize, maxDriverSize, detachNum := cacheInit()
lruDriver := event2.NewDriver(maxDriverSize)
lruCacheOnce.Do(func() {
lru := &SingleCache{
maxsize: maxSize,
clearSize: clearSize,
nowSize: 0,
li: list.New(),
lruMap: make(map[string]*list.Element),
lruMaxDiverSize: maxDriverSize,
lruDriver: lruDriver,
lruConsumer: event2.NewConsumer(lruDriver),
lruCleanProduce: event2.NewProduce(lruDriver),
middleProduce: event.NewProduce(middle.NewMiddleWare().GetEventDriver()),
lruTtlManage: newLruTTl(detachNum),
}
lruCache = lru
// 启动 lru 事件驱动
go lru.lruSingleWork()
go lru.lruTtlWork()
go lru.cleanWork()
})
return lruCache
}
// GetDriver 获取驱动
func (lru *SingleCache) GetDriver() event2.DriverInterface {
return lru.lruDriver
}
//Add 增加
func (lru *SingleCache) Add(key *proto.BaseKey, val structure.KeyBaseInterface) error {
if key == nil {
return errorx.KeyBaseIsNilErr()
}
exp := lru.lruTtlManage.setKeys(key)
keyBaseVal := &keyBaseValue{
key: key.Key,
val: val,
expire: exp,
}
if elVal, ok := lru.lruMap[key.Key]; ok {
lru.li.MoveToFront(elVal)
oldSize := elVal.Value.(structure.KeyBaseInterface).SizeByte()
lru.UpdateLruSize(structure.UpdateLength(val.SizeByte() - oldSize))
elVal.Value = keyBaseVal
return nil
}
valEl := lru.li.PushFront(keyBaseVal)
lru.lruMap[key.Key] = valEl
//增加大小
lru.UpdateLruSize(structure.UpdateLength(valEl.Value.(*keyBaseValue).val.SizeByte()))
return nil
}
// Get 查找key对应的value
func (lru *SingleCache) Get(key *proto.BaseKey) (structure.KeyBaseInterface, bool) {
if key == nil {
return nil, false
}
if elVal, ok := lru.lruMap[key.Key]; ok {
lru.li.MoveToFront(elVal)
return elVal.Value.(*keyBaseValue).val, true
}
return nil, false
}
//Del 删除机制
func (lru *SingleCache) Del() error {
if lru.lruMap == nil {
return errorx.New("lru is nil")
}
data := lru.li.Back()
delete(lru.lruMap, data.Value.(*keyBaseValue).key)
//删除大小
lru.UpdateLruSize(structure.UpdateLength(-1 * data.Value.(*keyBaseValue).val.SizeByte()))
lru.li.Remove(data)
return nil
}
//DelByKey 根据key删除
func (lru *SingleCache) DelByKey(key *proto.BaseKey) error {
if key == nil {
return errorx.KeyBaseIsNilErr()
}
if lru.lruMap == nil {
return errorx.New("lru is nil")
}
if el, ok := lru.lruMap[key.Key]; ok {
delete(lru.lruMap, key.Key)
lru.li.Remove(el)
lru.UpdateLruSize(structure.UpdateLength(-1 * el.Value.(*keyBaseValue).val.SizeByte()))
return nil
}
return errorx.New("lru no this key")
}
//DelByKeyAndExTtl 根据key(string)删除已经过期的 key
func (lru *SingleCache) delByKeyAndExTtl(key string, beforeTime int64) {
if elVal, ok := lru.lruMap[key]; ok {
exp := elVal.Value.(*keyBaseValue).expire
if exp <= beforeTime {
delete(lru.lruMap, key)
lru.li.Remove(elVal)
lru.UpdateLruSize(structure.UpdateLength(-1 * elVal.Value.(*keyBaseValue).val.SizeByte()))
}
}
}
func (lru *SingleCache) DelToClearSize() error {
if lru.lruMap == nil {
return errorx.New("lru is nil")
}
for lru.nowSize > lru.clearSize {
//del自动给nowSize进行大小的改变
err := lru.Del()
if err != nil {
return err
}
}
return nil
}
// 更新过期时间
func (lru *SingleCache) UpdateTTl(key *proto.BaseKey) error {
if key == nil {
return errorx.KeyBaseIsNilErr()
}
if elVal, ok := lru.lruMap[key.Key]; ok {
expire := lru.lruTtlManage.setKeys(key)
elVal.Value.(*keyBaseValue).expire = expire
}
return errorx.New("the key is not in lru cache, key:%s", key.Key)
}

View File

@ -1,103 +0,0 @@
package lru
import (
"fmt"
"testing"
"time"
"gitee.com/wheat-os/wheatCache/pkg/proto"
"gitee.com/wheat-os/wheatCache/pkg/structure/stringx"
"github.com/stretchr/testify/require"
)
func TestNewLRUCache(t *testing.T) {
cache := NewLRUCache()
v1 := stringx.NewStringSingle()
v2 := stringx.NewStringSingle()
v3 := stringx.NewStringSingle()
key1 := proto.BaseKey{
Key: "1",
}
key2 := proto.BaseKey{
Key: "2",
}
key3 := proto.BaseKey{
Key: "3",
}
cache.Add(&key1, v1)
cache.Add(&key2, v2)
cache.Add(&key3, v3)
cache.Add(&key1, v1)
fmt.Println(cache.nowSize)
cache.Del()
fmt.Println(cache.nowSize)
_, isTrue := cache.Get(&key1)
require.Equal(t, isTrue, true)
}
func TestNewLRUCache2(t *testing.T) {
//根据key删除
cache := NewLRUCache()
v1 := stringx.NewStringSingle()
v2 := stringx.NewStringSingle()
v3 := stringx.NewStringSingle()
key1 := proto.BaseKey{
Key: "1",
}
key2 := proto.BaseKey{
Key: "2",
}
key3 := proto.BaseKey{
Key: "3",
}
cache.Add(&key1, v1)
cache.Add(&key2, v2)
cache.Add(&key3, v3)
cache.DelByKey(&key1)
_, ok := cache.Get(&key1)
require.Equal(t, ok, false)
require.Error(t, cache.DelByKey(&key1))
}
func TestLruProcess(t *testing.T) {
lru := NewLRUCache()
lru.clearSize = 3600
for i := 100; i < 200; i++ {
lru.Add(&proto.BaseKey{
Key: fmt.Sprint(i),
Ttl: 20 << 2,
}, stringx.NewStringSingle())
}
// mock LruKey
for i := 0; i < 100; i++ {
lru.Add(&proto.BaseKey{
Key: fmt.Sprint(i),
Ttl: 4,
}, stringx.NewStringSingle())
}
require.Equal(t, lru.nowSize, int64(200*24))
// 自动清理测试
fmt.Println(lru.clearSize)
require.Equal(t, lru.li.Len(), 200)
time.Sleep(3 * time.Second)
require.Less(t, lru.nowSize, lru.clearSize+1)
// TTL 测试, 100-200 key 发生自动清理 留下 50-100 共 1000-100 + 20个 key5s 后,前 0-100的 key 过期,剩下
time.Sleep(2 * time.Second)
require.Equal(t, lru.li.Len(), 50)
// 过期全部的 Key
for i := 100; i < 200; i++ {
lru.UpdateTTl(&proto.BaseKey{
Key: fmt.Sprint(i),
Ttl: -1,
})
}
time.Sleep(2 * time.Second)
require.Equal(t, lru.nowSize, int64(0))
}

View File

@ -1,56 +0,0 @@
package lru
import (
"sync"
"time"
"gitee.com/wheat-os/wheatCache/pkg/proto"
"gitee.com/wheat-os/wheatCache/pkg/util/skiplist"
)
// lru 的 ttl 管理器
type lruTTl struct {
sk *skiplist.SkipList
memoryKey chan string // 缓存过期的 key
detachNum int // 每次移除的数量
mu sync.Mutex
}
func (l *lruTTl) setKeys(key *proto.BaseKey) int64 {
l.mu.Lock()
defer l.mu.Unlock()
// 永久存储
if key.Expire == nil && key.Ttl == 0 {
return 0
}
ttlTime := time.Now().Unix()
if key.Expire != nil {
ttlTime = key.Expire.GetSeconds()
}
ttlTime += key.GetTtl()
l.sk.Insert(float64(ttlTime), key.GetKey())
return ttlTime
}
// 加载过期的 Key 到 Memory
func (l *lruTTl) ttlKeyToMemoryBySecond() {
t := time.Now()
values := l.sk.PopLeft(float64(t.Unix()))
for _, val := range values {
l.memoryKey <- val.(string)
}
}
func newLruTTl(detachNum int) *lruTTl {
return &lruTTl{
sk: skiplist.NewSkipList(defaultTtlMaxLevel),
// 默认 10000 个 Key
memoryKey: make(chan string, 10000),
detachNum: detachNum,
}
}

View File

@ -1,36 +0,0 @@
package lru
import (
"fmt"
"testing"
"time"
"gitee.com/wheat-os/wheatCache/pkg/proto"
"gitee.com/wheat-os/wheatCache/pkg/structure/stringx"
"github.com/stretchr/testify/require"
)
func TestTTlCup(t *testing.T) {
k := make([]string, 100, 3000)
fmt.Println(cap(k))
p := k[:50]
fmt.Println(cap(p))
}
func Test_LruTTl(t *testing.T) {
lru := NewLRUCache()
s := stringx.NewStringSingle()
lru.Add(&proto.BaseKey{
Key: "k8s",
Ttl: 1,
}, s)
lru.Add(&proto.BaseKey{
Key: "990",
Ttl: 10,
}, s)
require.Equal(t, lru.nowSize, int64(48))
time.Sleep(4 * time.Second)
require.Equal(t, lru.nowSize, int64(24))
}

View File

@ -1,62 +0,0 @@
package lru
import (
"context"
"testing"
"time"
"gitee.com/wheat-os/wheatCache/pkg/event2"
"gitee.com/wheat-os/wheatCache/pkg/logx"
"gitee.com/wheat-os/wheatCache/pkg/proto"
"gitee.com/wheat-os/wheatCache/pkg/structure/stringx"
"github.com/stretchr/testify/require"
)
func TestWorker(t *testing.T) {
ctx := context.Background()
lru := NewLRUCache()
produce := event2.NewProduce(lru.GetDriver())
workEvent := produce.NewEvent(OptionEventName)
workEvent.SetValue(event2.WorkFuncEventKey, event2.EventWorkFunc(func() (interface{}, error) {
v1 := stringx.NewStringSingle()
key := proto.BaseKey{
Key: "v1",
}
res, _ := v1.Set("123")
lru.Add(&key, v1)
return res, nil
}))
workEvent.InitWaitEvent()
produce.Call(ctx, workEvent)
res, err := workEvent.StartWaitEvent(2 * time.Second)
require.NoError(t, err)
require.Equal(t, res, "123")
}
func TestSingleCache_DelToClearSize(t *testing.T) {
ctx := context.Background()
lru := NewLRUCache()
produce := event2.NewProduce(lru.GetDriver())
for i := int32(20000); i > 0; i-- {
workEvent := produce.NewEvent(OptionEventName)
workEvent.SetValue(event2.WorkFuncEventKey, event2.EventWorkFunc(func() (interface{}, error) {
v1 := stringx.NewStringSingle()
key := proto.BaseKey{
Key: string(i),
Ttl: 1,
}
u := v1.Setbit(i, true)
lru.Add(&key, v1)
return u, nil
}))
workEvent.InitWaitEvent()
produce.Call(ctx, workEvent)
workEvent.StartWaitEvent(2 * time.Second)
workEvent.Recovery()
}
logx.Info("start size is %d", lru.nowSize)
time.Sleep(10 * time.Second)
logx.Info("end size is %d", lru.nowSize)
}

View File

@ -1,135 +0,0 @@
package lru
import (
"context"
"time"
"gitee.com/wheat-os/wheatCache/pkg/errorx"
"gitee.com/wheat-os/wheatCache/pkg/event"
"gitee.com/wheat-os/wheatCache/pkg/event2"
"gitee.com/wheat-os/wheatCache/pkg/logx"
mMsg "gitee.com/wheat-os/wheatCache/pkg/middle-msg"
)
func (lru *SingleCache) lruSingleWork() {
ctx := context.Background()
for {
workEvent := lru.lruConsumer.Receive(ctx)
workFunc, ok := workEvent.GetValue(event2.WorkFuncEventKey)
if !ok {
workEvent.SetResultErr(errorx.LruNotWorkFuncEventErr())
continue
}
switch workEvent.GetEventName() {
case OptionEventName:
if work, ok := workFunc.(event2.EventWorkFunc); ok {
workEvent.ExecWorkAndSendResult(work)
}
case CleanEventName:
// 对当前的io数量进行判断
ioNum := lru.GetDriver().GetLength()
if ioNum > lru.lruMaxDiverSize/2 {
lru.lruCleanProduce.Call(ctx, workEvent)
continue
}
if work, ok := workFunc.(event2.EventWorkFunc); ok {
workEvent.ExecWorkAndSendResult(work)
}
case TtlEventName:
if work, ok := workFunc.(event2.EventWorkFunc); ok {
workEvent.ExecWorkAndSendResult(work)
}
}
}
}
// 执行过期事件
func (lru *SingleCache) lruTtlWork() {
ctx := context.Background()
// 清理事件
go func() {
work := event2.EventWorkFunc(func() (interface{}, error) {
beforeTime := time.Now().Unix()
cle := lru.lruTtlManage.detachNum
if cle > len(lru.lruTtlManage.memoryKey) {
cle = len(lru.lruTtlManage.memoryKey)
}
keys := make([]string, 0)
for i := 0; i < cle; i++ {
key := <-lru.lruTtlManage.memoryKey
keys = append(keys, key)
lru.delByKeyAndExTtl(key, beforeTime)
}
return keys, nil
})
cleanTTlTicker := time.NewTicker(500 * time.Millisecond)
defer cleanTTlTicker.Stop()
for {
// 清理事件
<-cleanTTlTicker.C
if len(lru.lruTtlManage.memoryKey) == 0 {
continue
}
ttlEvent := lru.lruCleanProduce.NewEvent(TtlEventName)
ttlEvent.SetValue(event2.WorkFuncEventKey, work)
ttlEvent.InitWaitEvent()
lru.lruCleanProduce.Call(ctx, ttlEvent)
keys, err := ttlEvent.StartWaitEvent(time.Second * 2)
ttlEvent.Recovery()
mMsg.SendMiddleMsg(ctx, lru.middleProduce, mMsg.LruTTlContext{
Keys: keys.([]string),
CleanTime: time.Now(),
})
if err != nil {
logx.With(ctx, lru.middleProduce).Errorln(err)
}
}
}()
// 收集事件
for {
time.Sleep(1 * time.Second)
lru.lruTtlManage.ttlKeyToMemoryBySecond()
}
}
func (lru *SingleCache) cleanWork() {
cxt := context.Background()
work := event.EventWorkFunc(func() (interface{}, error) {
err := lru.DelToClearSize()
return nil, err
})
for {
time.Sleep(2 * time.Second)
if lru.clearSize < lru.nowSize {
lruCleanEvent := lru.lruCleanProduce.NewEvent(CleanEventName)
lruCleanEvent.SetValue(event2.WorkFuncEventKey, work)
lruCleanEvent.InitWaitEvent()
lru.lruCleanProduce.Call(cxt, lruCleanEvent)
_, err := lruCleanEvent.StartWaitEvent(defaultWaitTime)
if err != nil {
logx.With(cxt, lru.middleProduce).Errorln(err)
}
// 归还
lruCleanEvent.Recovery()
}
}
}

View File

@ -1,40 +0,0 @@
package middlemsg
import (
"context"
"gitee.com/wheat-os/wheatCache/pkg/errorx"
"gitee.com/wheat-os/wheatCache/pkg/event"
)
const (
MiddleMsgKey = "middleMsgKey"
)
func SendMiddleMsg(
ctx context.Context,
middleProduce event.ProduceInterface,
val interface{},
) error {
if middleProduce == nil {
return errorx.New("middleProduce not is nil")
}
var eventName string
switch val.(type) {
case *LogContext:
eventName = LogContextName
case *LruCleanContext:
eventName = LruCleanContextName
case *LruTTlContext:
eventName = LruTTlContextName
case *PulginsInfos:
eventName = PulginsInfosName
}
msgEvent := middleProduce.NewEvent(eventName)
msgEvent.SetValue(MiddleMsgKey, val)
middleProduce.Call(ctx, msgEvent)
return nil
}

View File

@ -1,14 +0,0 @@
package middlemsg
import "time"
var (
LogContextName = "log-context"
)
type LogContext struct {
Level string
Data time.Time
Msg string
Route string
}

View File

@ -1,21 +0,0 @@
package middlemsg
import "time"
const LruCleanContextName = "lru-clean-context"
type LruCleanContext struct {
Keys []string
BeforeCleanSize int64
BehindCleanSize int64
StartTime time.Time
EndTime time.Time
}
const LruTTlContextName = "lru-ttl-context"
type LruTTlContext struct {
Keys []string
CleanTime time.Time
}

View File

@ -1,21 +0,0 @@
package middlemsg
import (
"time"
)
const (
PulginsInfosName = "plugins-infos-context"
)
type PulginsInfo struct {
Version string
Desc string
Name string
Statux string
Time time.Duration
}
type PulginsInfos struct {
Infos []*PulginsInfo
}

View File

@ -1,15 +1 @@
package middle
import (
"sync"
)
var (
oneMiddle sync.Once
middleWareDriver *MiddleWare
)
const (
defaultConsumerCount = 5
defaultDriverCount = 1000
)

View File

@ -1,74 +0,0 @@
package middle
import (
_ "gitee.com/wheat-os/wheatCache/conf"
"gitee.com/wheat-os/wheatCache/pkg/event"
"gitee.com/wheat-os/wheatCache/plugins"
"gitee.com/wheat-os/wheatCache/plugins/config"
"github.com/spf13/viper"
)
type MiddleWare struct {
eventDriver event.DriverInterface
eventConsumer event.ConsumerInterface
eventProduce event.ProduceInterface
plugins map[string][]plugins.PluginInterface
consumerCount int
driverCount int
}
func NewMiddleWare() *MiddleWare {
oneMiddle.Do(func() {
consumerCount, driverCount := loadConfigAndDefault()
driver := event.NewDriver(driverCount)
middleWareDriver = &MiddleWare{
eventDriver: driver,
eventConsumer: event.NewConsumer(driver),
eventProduce: event.NewProduce(driver),
driverCount: driverCount,
consumerCount: consumerCount,
}
middleWareDriver.loadPlugins()
// 多消费 middle
middleWareDriver.startWork()
})
return middleWareDriver
}
func (m *MiddleWare) GetEventDriver() event.DriverInterface {
return m.eventDriver
}
func (m *MiddleWare) loadPlugins() {
plug := viper.GetStringMapStringSlice("plugins-control")
pluginsMap := config.GetMiddlewareMap()
pluginsContext := make(map[string][]plugins.PluginInterface)
for msg, pluNames := range plug {
pulgSingle := make([]plugins.PluginInterface, 0)
for _, name := range pluNames {
pulgSingle = append(pulgSingle, pluginsMap[name])
}
pluginsContext[msg] = pulgSingle
}
m.plugins = pluginsContext
}
func loadConfigAndDefault() (int, int) {
// 加载 consumerCount
consumerCount := viper.GetInt("middle-driver.middleConsumerCount")
if consumerCount == 0 {
consumerCount = defaultConsumerCount
}
driverCount := viper.GetInt("middle-driver.driverCount")
if driverCount == 0 {
driverCount = defaultDriverCount
}
return consumerCount, driverCount
}

View File

@ -1,36 +0,0 @@
package middle
import (
"context"
"gitee.com/wheat-os/wheatCache/pkg/logx"
middleMsg "gitee.com/wheat-os/wheatCache/pkg/middle-msg"
)
func (m *MiddleWare) startWork() {
for i := 0; i < m.consumerCount; i++ {
go func() {
ctx := context.Background()
for {
workEvent := m.eventConsumer.Receive(ctx)
plugs := m.plugins[workEvent.GetEventName()]
msg, ok := workEvent.GetValue(middleMsg.MiddleMsgKey)
m.eventConsumer.Recovery(workEvent)
if !ok {
logx.With(ctx, m.eventProduce).Error("get event value errnot key:%s", middleMsg.MiddleMsgKey)
continue
}
// 发送事件到 全部的 plugs 里
for _, val := range plugs {
_, err := val.Exec(msg)
if err != nil {
logx.With(ctx, m.eventProduce).Errorln(err)
}
}
}
}()
}
}

View File

@ -1,42 +0,0 @@
package middle
import (
"context"
"fmt"
"testing"
"time"
"gitee.com/wheat-os/wheatCache/pkg/event"
middleMsg "gitee.com/wheat-os/wheatCache/pkg/middle-msg"
)
func Test_middleware_loadPlugins(t *testing.T) {
m := NewMiddleWare()
m.loadPlugins()
fmt.Println(m.plugins)
}
func TestWorker(t *testing.T) {
ctx := context.Background()
m := NewMiddleWare()
product := event.NewProduce(m.GetEventDriver())
middleMsg.SendMiddleMsg(ctx, product, &middleMsg.LogContext{
Msg: "debug msg",
})
middleMsg.SendMiddleMsg(ctx, product, &middleMsg.PulginsInfos{
Infos: []*middleMsg.PulginsInfo{
{
Desc: "miss",
},
},
})
middleMsg.SendMiddleMsg(ctx, product, &middleMsg.LruTTlContext{
Keys: []string{"1", "2", "3"},
})
time.Sleep(1 * time.Second)
}

View File

@ -84,44 +84,6 @@ func (x *BaseKey) GetExpire() *timestamppb.Timestamp {
return nil
}
type External struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
}
func (x *External) Reset() {
*x = External{}
if protoimpl.UnsafeEnabled {
mi := &file_base_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *External) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*External) ProtoMessage() {}
func (x *External) ProtoReflect() protoreflect.Message {
mi := &file_base_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use External.ProtoReflect.Descriptor instead.
func (*External) Descriptor() ([]byte, []int) {
return file_base_proto_rawDescGZIP(), []int{1}
}
var File_base_proto protoreflect.FileDescriptor
var file_base_proto_rawDesc = []byte{
@ -134,9 +96,8 @@ var file_base_proto_rawDesc = []byte{
0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67,
0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54,
0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x06, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65,
0x22, 0x0a, 0x0a, 0x08, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x42, 0x0b, 0x5a, 0x09,
0x70, 0x6b, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x33,
0x42, 0x0b, 0x5a, 0x09, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@ -151,14 +112,13 @@ func file_base_proto_rawDescGZIP() []byte {
return file_base_proto_rawDescData
}
var file_base_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
var file_base_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
var file_base_proto_goTypes = []interface{}{
(*BaseKey)(nil), // 0: BaseKey
(*External)(nil), // 1: External
(*timestamppb.Timestamp)(nil), // 2: google.protobuf.Timestamp
(*timestamppb.Timestamp)(nil), // 1: google.protobuf.Timestamp
}
var file_base_proto_depIdxs = []int32{
2, // 0: BaseKey.expire:type_name -> google.protobuf.Timestamp
1, // 0: BaseKey.expire:type_name -> google.protobuf.Timestamp
1, // [1:1] is the sub-list for method output_type
1, // [1:1] is the sub-list for method input_type
1, // [1:1] is the sub-list for extension type_name
@ -184,18 +144,6 @@ func file_base_proto_init() {
return nil
}
}
file_base_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*External); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
@ -203,7 +151,7 @@ func file_base_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_base_proto_rawDesc,
NumEnums: 0,
NumMessages: 2,
NumMessages: 1,
NumExtensions: 0,
NumServices: 0,
},

View File

@ -1,733 +0,0 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.17.3
// source: channelx.proto
package proto
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type CPushRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Key *BaseKey `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
Value []string `protobuf:"bytes,2,rep,name=value,proto3" json:"value,omitempty"`
}
func (x *CPushRequest) Reset() {
*x = CPushRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_channelx_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *CPushRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*CPushRequest) ProtoMessage() {}
func (x *CPushRequest) ProtoReflect() protoreflect.Message {
mi := &file_channelx_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use CPushRequest.ProtoReflect.Descriptor instead.
func (*CPushRequest) Descriptor() ([]byte, []int) {
return file_channelx_proto_rawDescGZIP(), []int{0}
}
func (x *CPushRequest) GetKey() *BaseKey {
if x != nil {
return x.Key
}
return nil
}
func (x *CPushRequest) GetValue() []string {
if x != nil {
return x.Value
}
return nil
}
type CPushResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
E *External `protobuf:"bytes,1,opt,name=e,proto3" json:"e,omitempty"`
}
func (x *CPushResponse) Reset() {
*x = CPushResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_channelx_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *CPushResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*CPushResponse) ProtoMessage() {}
func (x *CPushResponse) ProtoReflect() protoreflect.Message {
mi := &file_channelx_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use CPushResponse.ProtoReflect.Descriptor instead.
func (*CPushResponse) Descriptor() ([]byte, []int) {
return file_channelx_proto_rawDescGZIP(), []int{1}
}
func (x *CPushResponse) GetE() *External {
if x != nil {
return x.E
}
return nil
}
type CPopRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Key *BaseKey `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
Count int32 `protobuf:"varint,2,opt,name=count,proto3" json:"count,omitempty"`
}
func (x *CPopRequest) Reset() {
*x = CPopRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_channelx_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *CPopRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*CPopRequest) ProtoMessage() {}
func (x *CPopRequest) ProtoReflect() protoreflect.Message {
mi := &file_channelx_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use CPopRequest.ProtoReflect.Descriptor instead.
func (*CPopRequest) Descriptor() ([]byte, []int) {
return file_channelx_proto_rawDescGZIP(), []int{2}
}
func (x *CPopRequest) GetKey() *BaseKey {
if x != nil {
return x.Key
}
return nil
}
func (x *CPopRequest) GetCount() int32 {
if x != nil {
return x.Count
}
return 0
}
type CPopResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
E *External `protobuf:"bytes,1,opt,name=e,proto3" json:"e,omitempty"`
Result []string `protobuf:"bytes,2,rep,name=result,proto3" json:"result,omitempty"`
}
func (x *CPopResponse) Reset() {
*x = CPopResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_channelx_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *CPopResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*CPopResponse) ProtoMessage() {}
func (x *CPopResponse) ProtoReflect() protoreflect.Message {
mi := &file_channelx_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use CPopResponse.ProtoReflect.Descriptor instead.
func (*CPopResponse) Descriptor() ([]byte, []int) {
return file_channelx_proto_rawDescGZIP(), []int{3}
}
func (x *CPopResponse) GetE() *External {
if x != nil {
return x.E
}
return nil
}
func (x *CPopResponse) GetResult() []string {
if x != nil {
return x.Result
}
return nil
}
type CMakeRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Key *BaseKey `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
Length int32 `protobuf:"varint,2,opt,name=length,proto3" json:"length,omitempty"`
}
func (x *CMakeRequest) Reset() {
*x = CMakeRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_channelx_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *CMakeRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*CMakeRequest) ProtoMessage() {}
func (x *CMakeRequest) ProtoReflect() protoreflect.Message {
mi := &file_channelx_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use CMakeRequest.ProtoReflect.Descriptor instead.
func (*CMakeRequest) Descriptor() ([]byte, []int) {
return file_channelx_proto_rawDescGZIP(), []int{4}
}
func (x *CMakeRequest) GetKey() *BaseKey {
if x != nil {
return x.Key
}
return nil
}
func (x *CMakeRequest) GetLength() int32 {
if x != nil {
return x.Length
}
return 0
}
type CMakeResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
}
func (x *CMakeResponse) Reset() {
*x = CMakeResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_channelx_proto_msgTypes[5]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *CMakeResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*CMakeResponse) ProtoMessage() {}
func (x *CMakeResponse) ProtoReflect() protoreflect.Message {
mi := &file_channelx_proto_msgTypes[5]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use CMakeResponse.ProtoReflect.Descriptor instead.
func (*CMakeResponse) Descriptor() ([]byte, []int) {
return file_channelx_proto_rawDescGZIP(), []int{5}
}
type CLenRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Key *BaseKey `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
}
func (x *CLenRequest) Reset() {
*x = CLenRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_channelx_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *CLenRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*CLenRequest) ProtoMessage() {}
func (x *CLenRequest) ProtoReflect() protoreflect.Message {
mi := &file_channelx_proto_msgTypes[6]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use CLenRequest.ProtoReflect.Descriptor instead.
func (*CLenRequest) Descriptor() ([]byte, []int) {
return file_channelx_proto_rawDescGZIP(), []int{6}
}
func (x *CLenRequest) GetKey() *BaseKey {
if x != nil {
return x.Key
}
return nil
}
type CLenResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Length int32 `protobuf:"varint,2,opt,name=length,proto3" json:"length,omitempty"`
}
func (x *CLenResponse) Reset() {
*x = CLenResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_channelx_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *CLenResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*CLenResponse) ProtoMessage() {}
func (x *CLenResponse) ProtoReflect() protoreflect.Message {
mi := &file_channelx_proto_msgTypes[7]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use CLenResponse.ProtoReflect.Descriptor instead.
func (*CLenResponse) Descriptor() ([]byte, []int) {
return file_channelx_proto_rawDescGZIP(), []int{7}
}
func (x *CLenResponse) GetLength() int32 {
if x != nil {
return x.Length
}
return 0
}
type CCleanRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Key *BaseKey `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
}
func (x *CCleanRequest) Reset() {
*x = CCleanRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_channelx_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *CCleanRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*CCleanRequest) ProtoMessage() {}
func (x *CCleanRequest) ProtoReflect() protoreflect.Message {
mi := &file_channelx_proto_msgTypes[8]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use CCleanRequest.ProtoReflect.Descriptor instead.
func (*CCleanRequest) Descriptor() ([]byte, []int) {
return file_channelx_proto_rawDescGZIP(), []int{8}
}
func (x *CCleanRequest) GetKey() *BaseKey {
if x != nil {
return x.Key
}
return nil
}
type CCleanResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
}
func (x *CCleanResponse) Reset() {
*x = CCleanResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_channelx_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *CCleanResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*CCleanResponse) ProtoMessage() {}
func (x *CCleanResponse) ProtoReflect() protoreflect.Message {
mi := &file_channelx_proto_msgTypes[9]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use CCleanResponse.ProtoReflect.Descriptor instead.
func (*CCleanResponse) Descriptor() ([]byte, []int) {
return file_channelx_proto_rawDescGZIP(), []int{9}
}
var File_channelx_proto protoreflect.FileDescriptor
var file_channelx_proto_rawDesc = []byte{
0x0a, 0x0e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x78, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x1a, 0x0a, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x40, 0x0a, 0x0c,
0x43, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x03,
0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x42, 0x61, 0x73, 0x65,
0x4b, 0x65, 0x79, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75,
0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x28,
0x0a, 0x0d, 0x43, 0x50, 0x75, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
0x17, 0x0a, 0x01, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x45, 0x78, 0x74,
0x65, 0x72, 0x6e, 0x61, 0x6c, 0x52, 0x01, 0x65, 0x22, 0x3f, 0x0a, 0x0b, 0x43, 0x50, 0x6f, 0x70,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x42, 0x61, 0x73, 0x65, 0x4b, 0x65, 0x79, 0x52, 0x03,
0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01,
0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x3f, 0x0a, 0x0c, 0x43, 0x50, 0x6f,
0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x17, 0x0a, 0x01, 0x65, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x52,
0x01, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x02, 0x20, 0x03,
0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, 0x42, 0x0a, 0x0c, 0x43, 0x4d,
0x61, 0x6b, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x03, 0x6b, 0x65,
0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x42, 0x61, 0x73, 0x65, 0x4b, 0x65,
0x79, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68,
0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0x0f,
0x0a, 0x0d, 0x43, 0x4d, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
0x29, 0x0a, 0x0b, 0x43, 0x4c, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a,
0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x42, 0x61,
0x73, 0x65, 0x4b, 0x65, 0x79, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x26, 0x0a, 0x0c, 0x43, 0x4c,
0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65,
0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x6c, 0x65, 0x6e, 0x67,
0x74, 0x68, 0x22, 0x2b, 0x0a, 0x0d, 0x43, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x08, 0x2e, 0x42, 0x61, 0x73, 0x65, 0x4b, 0x65, 0x79, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22,
0x10, 0x0a, 0x0e, 0x43, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x42, 0x0b, 0x5a, 0x09, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
file_channelx_proto_rawDescOnce sync.Once
file_channelx_proto_rawDescData = file_channelx_proto_rawDesc
)
func file_channelx_proto_rawDescGZIP() []byte {
file_channelx_proto_rawDescOnce.Do(func() {
file_channelx_proto_rawDescData = protoimpl.X.CompressGZIP(file_channelx_proto_rawDescData)
})
return file_channelx_proto_rawDescData
}
var file_channelx_proto_msgTypes = make([]protoimpl.MessageInfo, 10)
var file_channelx_proto_goTypes = []interface{}{
(*CPushRequest)(nil), // 0: CPushRequest
(*CPushResponse)(nil), // 1: CPushResponse
(*CPopRequest)(nil), // 2: CPopRequest
(*CPopResponse)(nil), // 3: CPopResponse
(*CMakeRequest)(nil), // 4: CMakeRequest
(*CMakeResponse)(nil), // 5: CMakeResponse
(*CLenRequest)(nil), // 6: CLenRequest
(*CLenResponse)(nil), // 7: CLenResponse
(*CCleanRequest)(nil), // 8: CCleanRequest
(*CCleanResponse)(nil), // 9: CCleanResponse
(*BaseKey)(nil), // 10: BaseKey
(*External)(nil), // 11: External
}
var file_channelx_proto_depIdxs = []int32{
10, // 0: CPushRequest.key:type_name -> BaseKey
11, // 1: CPushResponse.e:type_name -> External
10, // 2: CPopRequest.key:type_name -> BaseKey
11, // 3: CPopResponse.e:type_name -> External
10, // 4: CMakeRequest.key:type_name -> BaseKey
10, // 5: CLenRequest.key:type_name -> BaseKey
10, // 6: CCleanRequest.key:type_name -> BaseKey
7, // [7:7] is the sub-list for method output_type
7, // [7:7] is the sub-list for method input_type
7, // [7:7] is the sub-list for extension type_name
7, // [7:7] is the sub-list for extension extendee
0, // [0:7] is the sub-list for field type_name
}
func init() { file_channelx_proto_init() }
func file_channelx_proto_init() {
if File_channelx_proto != nil {
return
}
file_base_proto_init()
if !protoimpl.UnsafeEnabled {
file_channelx_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CPushRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_channelx_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CPushResponse); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_channelx_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CPopRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_channelx_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CPopResponse); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_channelx_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CMakeRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_channelx_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CMakeResponse); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_channelx_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CLenRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_channelx_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CLenResponse); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_channelx_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CCleanRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_channelx_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CCleanResponse); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_channelx_proto_rawDesc,
NumEnums: 0,
NumMessages: 10,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_channelx_proto_goTypes,
DependencyIndexes: file_channelx_proto_depIdxs,
MessageInfos: file_channelx_proto_msgTypes,
}.Build()
File_channelx_proto = out.File
file_channelx_proto_rawDesc = nil
file_channelx_proto_goTypes = nil
file_channelx_proto_depIdxs = nil
}

View File

@ -1,35 +0,0 @@
package proto
import (
"google.golang.org/protobuf/types/known/timestamppb"
)
type GetKeyBaseInterface interface {
GetKey() *BaseKey
}
const (
BaseKeyMethodKey = "basekey"
)
// NewBaseKey
// keyttlexpire
func NewBaseKey(key string, t ...int64) *BaseKey {
var expire *timestamppb.Timestamp = nil
var ttl int64
if len(t) > 1 {
expire = &timestamppb.Timestamp{
Seconds: t[1],
}
ttl = t[0]
} else if len(t) == 1 {
ttl = t[0]
}
return &BaseKey{
Key: key,
Expire: expire,
Ttl: ttl,
}
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,66 +0,0 @@
package channelx
import (
"sync/atomic"
"gitee.com/wheat-os/wheatCache/pkg/structure"
)
type ChannelX struct {
channel chan *structure.Value
sizeByte int64
}
func MakeChannelX(length int) structure.ChannelXInterface {
return &ChannelX{
channel: make(chan *structure.Value, length),
sizeByte: 0,
}
}
func (c *ChannelX) SizeByte() int64 {
return c.sizeByte
}
// RollBack TODO 事务相关, V2 实现
func (c *ChannelX) RollBack() error {
panic("not implemented") // TODO: Implement
}
// Begin 事务相关, V2 实现
func (c *ChannelX) Begin() error {
panic("not implemented") // TODO: Implement
}
// Comment 事务相关, V2 实现
func (c *ChannelX) Comment() error {
panic("not implemented") // TODO: Implement
}
func (c *ChannelX) Encode() ([]byte, error) {
panic("not implemented") // TODO: Implement
}
func (c *ChannelX) Push(value string) structure.UpdateLength {
val := structure.NewValue(value)
up := val.GetSize()
c.channel <- val
atomic.AddInt64(&c.sizeByte, int64(up))
return structure.UpdateLength(up)
}
func (c *ChannelX) Pop() (string, structure.UpdateLength) {
val := <-c.channel
return val.ToString(), structure.UpdateLength(val.GetSize()) * -1
}
func (c *ChannelX) Length() int {
return len(c.channel)
}
func (c *ChannelX) Clean() structure.UpdateLength {
c.channel = make(chan *structure.Value, cap(c.channel))
up := c.sizeByte
c.sizeByte = 0
return structure.UpdateLength(up) * -1
}

View File

@ -1,24 +0,0 @@
package channelx
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestChannelX_Push(t *testing.T) {
c := MakeChannelX(10)
require.Equal(t, c.Length(), 0)
up := c.Push("111")
require.Equal(t, 24, int(up))
res, up := c.Pop()
require.Equal(t, -24, int(up))
require.Equal(t, res, "111")
up = c.Push("111")
c.Clean()
require.Equal(t, c.Length(), 0)
}

View File

@ -1,77 +1,3 @@
package structure
const (
defaultLen = 8 // 默认创建的 value 大小
)
type DynamicType int8
type UpdateLength int64
const (
DynamicNull = DynamicType(iota)
DynamicInt
DynamicFloat
DynamicString
)
type KeyBaseInterface interface {
SizeByte() int64
// RollBack TODO 事务相关, V2 实现
RollBack() error
// Begin 事务相关, V2 实现
Begin() error
// Comment 事务相关, V2 实现
Comment() error
Encode() ([]byte, error)
}
type StringXInterface interface {
KeyBaseInterface
Set(string) (string, UpdateLength)
Get() string
Add(int32) (string, error)
Reduce(int32) (string, error)
Setbit(int32, bool) UpdateLength
Getbit(int32) (bool, error)
Getrange(start, end int32) (string, error)
GetLength() int
}
type ListXInterface interface {
KeyBaseInterface
LPush(...string) UpdateLength
RPush(...string) UpdateLength
LPop(int) ([]string, UpdateLength)
RPop(int) ([]string, UpdateLength)
Index(int) (string, error)
Insert(int, bool, ...string) (UpdateLength, error)
Length() int
Slice(start, end int) (UpdateLength, error) // 切片, O(n)复杂度
Range(start, end int) ([]string, error)
Remove(value string, count int) (int, UpdateLength)
}
type HashXInterface interface {
KeyBaseInterface
Set(key string, val string) UpdateLength
Get(key string) (string, error)
Del(key string) (UpdateLength, error)
Key() []string
Value() []string
Item() map[string]string
Add(renewal int, key ...string) (int, []string, error) // 访问影响成功的结果
SetX(key string, val string) (bool, UpdateLength) // 不存在才插入
Length() int
Range(consur, count int, regex string) []string
}
type ChannelXInterface interface {
KeyBaseInterface
Push(value string) UpdateLength
Pop() (string, UpdateLength)
Length() int
Clean() UpdateLength
}

View File

@ -1,11 +1,11 @@
// Code generated by gen-struct. DO NOT EDIT.
// Code generated by gen-struct-const. DO NOT EDIT.
// make gen-struct generated
package structure
package generate
const (
DEFAULT_KEY = iota
{% for key in keys %}
{% for key in keys -%}
{{key}}
{%- endfor %}
)
@ -18,7 +18,7 @@ const (
)
var CommKeyString = map[string]int {
{%- for kmp in key_maps %}
{%- for kmp in key_maps -%}
{% for comm in kmp.val -%}
"{{comm}}": {{kmp.key}},
{% endfor -%}
@ -26,7 +26,7 @@ var CommKeyString = map[string]int {
}
var CommKey = map[int]int {
{%- for kmp in key_maps %}
{%- for kmp in key_maps -%}
{% for comm in kmp.upper -%}
{{comm}}: {{kmp.key}},
{% endfor -%}

View File

@ -0,0 +1,29 @@
// Code generated by gen-struct-const. DO NOT EDIT.
// make gen-struct generated
package structure
import "gitee.com/timedb/wheatCache/pkg/proto"
type KeyBaseInterface interface {
SizeByte() int64
// TODO RollBack 事务相关, V2 实现
RollBack() error
// Begin 事务相关, V2 实现
Begin() error
// Comment 事务相关, V2 实现
Comment() error
Encode() ([]byte, error)
}
{% for dict_item in Data %}
{% for key, value in dict_item.items() %}
type {{key}}Interface interface{
KeyBaseInterface
{% for val in value -%}
{{val}}(*proto.{{val}}Request) ([]string, error)
{% endfor %}
}
{% endfor -%}
{%- endfor %}

View File

@ -0,0 +1,35 @@
// Code generated by gen-struct-const. DO NOT EDIT.
// make gen-struct generated
package generate
const (
DEFAULT_KEY = iota
STRING_X
)
const (
DEFAULT_COMM = iota
SET
GET
ADD
REDUCE
SETBIT
GETBIT
)
var CommKeyString = map[string]int{"set": STRING_X,
"get": STRING_X,
"add": STRING_X,
"reduce": STRING_X,
"setbit": STRING_X,
"getbit": STRING_X,
}
var CommKey = map[int]int{SET: STRING_X,
GET: STRING_X,
ADD: STRING_X,
REDUCE: STRING_X,
SETBIT: STRING_X,
GETBIT: STRING_X,
}

View File

@ -0,0 +1,9 @@
# 这里定义结构体的支持的命令, 以后也许会添加结构体的命令验证
STRING_X:
- set
- get
- add
- reduce
- setbit
- getbit

View File

@ -1,157 +0,0 @@
package hashx
import (
"regexp"
"gitee.com/wheat-os/wheatCache/pkg/errorx"
"gitee.com/wheat-os/wheatCache/pkg/structure"
)
type HashX map[string]*structure.Value
func NewHashXSingle() structure.HashXInterface {
return make(HashX)
}
func (h HashX) SizeByte() int64 {
var size int
for _, val := range h {
size += val.GetSize()
}
return int64(size)
}
// RollBack TODO 事务相关, V2 实现
func (h HashX) RollBack() error {
panic("not implemented") // TODO: Implement
}
// Begin 事务相关, V2 实现
func (h HashX) Begin() error {
panic("not implemented") // TODO: Implement
}
// Comment 事务相关, V2 实现
func (h HashX) Comment() error {
panic("not implemented") // TODO: Implement
}
func (h HashX) Encode() ([]byte, error) {
panic("not implemented") // TODO: Implement
}
func (h HashX) Set(key string, val string) structure.UpdateLength {
var Length structure.UpdateLength
if v, ok := h[key]; ok {
Length -= structure.UpdateLength(v.GetSize())
}
strVal := structure.NewValue(val)
h[key] = strVal
return Length + structure.UpdateLength(strVal.GetSize())
}
func (h HashX) Get(key string) (string, error) {
if v, ok := h[key]; ok {
return v.ToString(), nil
}
return "", errorx.New("this key does not exist in hashx, key:%s", key)
}
func (h HashX) Del(key string) (structure.UpdateLength, error) {
if v, ok := h[key]; ok {
delete(h, key)
return structure.UpdateLength(v.GetSize()), nil
}
return 0, errorx.New("this key does not exist in hashx, key:%s", key)
}
func (h HashX) Key() []string {
result := make([]string, 0, len(h))
for key := range h {
result = append(result, key)
}
return result
}
func (h HashX) Value() []string {
result := make([]string, 0, len(h))
for _, val := range h {
result = append(result, val.ToString())
}
return result
}
func (h HashX) Item() map[string]string {
result := make(map[string]string, len(h))
for key, val := range h {
result[key] = val.ToString()
}
return result
}
func (h HashX) Add(renewal int, keys ...string) (count int, result []string, err error) {
for _, key := range keys {
if v, ok := h[key]; ok {
res, err := v.Incr(int32(renewal))
if err != nil {
continue
}
count += 1
result = append(result, res)
}
}
return count, result, nil
}
func (h HashX) SetX(key string, val string) (bool, structure.UpdateLength) {
if _, ok := h[key]; ok {
return false, 0
}
strVal := structure.NewValue(val)
h[key] = strVal
return true, structure.UpdateLength(strVal.GetSize())
}
func (h HashX) Length() int {
return len(h)
}
func (h HashX) Range(consur, count int, regex string) []string {
var reComp *regexp.Regexp
if regex == "" {
reComp = nil
} else {
reComp = regexp.MustCompile(regex)
}
result := make([]string, 0)
for _, val := range h {
if consur > 0 {
consur--
continue
}
if count == 0 && count != -1 {
break
}
s := val.ToString()
if reComp == nil {
count--
result = append(result, s)
continue
}
if reComp.MatchString(s) {
count--
result = append(result, s)
}
}
return result
}

View File

@ -1,113 +0,0 @@
package hashx
import (
"fmt"
"reflect"
"regexp"
"testing"
"unsafe"
"github.com/stretchr/testify/require"
)
func TestHashX_Set_SetX(t *testing.T) {
h := NewHashXSingle()
h.Set("key", "opq")
res, err := h.Get("key")
require.NoError(t, err)
require.Equal(t, res, "opq")
b, _ := h.SetX("key", "opq")
require.Equal(t, b, false)
b, _ = h.SetX("key1", "opq")
require.Equal(t, b, true)
res, err = h.Get("key1")
require.NoError(t, err)
require.Equal(t, res, "opq")
}
func TestHashX_Del(t *testing.T) {
h := NewHashXSingle()
up := h.Set("key", "opq")
upu, err := h.Del("key")
require.NoError(t, err)
require.Equal(t, up, upu)
}
func TestHashX_Key(t *testing.T) {
h := NewHashXSingle()
h.Set("key", "opq")
h.Set("key1", "opq")
h.Set("key2", "opq")
require.Equal(t, h.Key(), []string{"key", "key1", "key2"})
}
func TestHashX_Value(t *testing.T) {
h := NewHashXSingle()
h.Set("key", "opq")
h.Set("key1", "opq")
h.Set("key2", "opq")
require.Equal(t, h.Value(), []string{"opq", "opq", "opq"})
}
func TestHashX_Add(t *testing.T) {
h := NewHashXSingle()
h.Set("1", "1")
c, res, err := h.Add(1, "1")
require.NoError(t, err)
require.Equal(t, c, 1)
require.Equal(t, res, []string{"2"})
s, err := h.Get("1")
require.NoError(t, err)
require.Equal(t, s, "2")
}
func Test_Pointer(t *testing.T) {
s := make([]int, 9, 20)
lens := *(*int)(unsafe.Pointer(uintptr(unsafe.Pointer(&s)) + uintptr(8)))
fmt.Println(lens, len(s))
mp := make(map[string]int)
mp["qcrao"] = 100
mp["stefno"] = 18
count := **(**uint8)(unsafe.Pointer(uintptr(unsafe.Pointer(&mp)) + uintptr(16)))
fmt.Println(count, len(mp)) // 2
}
func string2bytes(s string) []byte {
stringHeader := (*reflect.StringHeader)(unsafe.Pointer(&s))
result := reflect.SliceHeader{
Data: stringHeader.Data,
Len: stringHeader.Len,
Cap: stringHeader.Len,
}
return *(*[]byte)(unsafe.Pointer(&result))
}
func TestHashX_Range(t *testing.T) {
reComp := regexp.MustCompile("a.+")
require.True(t, reComp.MatchString("abbs"))
h := NewHashXSingle()
h.Set("abbs", "abbs")
h.Set("ppp", "ppp")
h.Set("accs", "accs")
result := h.Range(0, 3, "")
require.Len(t, result, 3)
result = h.Range(0, -1, "")
require.Len(t, result, 3)
result = h.Range(0, -1, "a.+")
require.Len(t, result, 2)
result = h.Range(1, -1, "a.+")
require.Len(t, result, 1)
}

View File

@ -0,0 +1,29 @@
// Code generated by gen-struct-const. DO NOT EDIT.
// make gen-struct generated
package structure
import "gitee.com/timedb/wheatCache/pkg/proto"
type KeyBaseInterface interface {
SizeByte() int64
// TODO RollBack 事务相关, V2 实现
RollBack() error
// Begin 事务相关, V2 实现
Begin() error
// Comment 事务相关, V2 实现
Comment() error
Encode() ([]byte, error)
}
type StringXInterface interface {
KeyBaseInterface
Set(*proto.SetRequest) ([]string, error)
Get(*proto.GetRequest) ([]string, error)
Add(*proto.AddRequest) ([]string, error)
Reduce(*proto.ReduceRequest) ([]string, error)
Setbit(*proto.SetbitRequest) ([]string, error)
Getbit(*proto.GetbitRequest) ([]string, error)
}

View File

@ -1,428 +0,0 @@
package listx
import (
"gitee.com/wheat-os/wheatCache/pkg/errorx"
"gitee.com/wheat-os/wheatCache/pkg/structure"
)
/*
1. 双向链表
2. 支持头尾操作
3. 支持索引
4. 支持切片
*/
type ListxNode struct {
next *ListxNode
pre *ListxNode
val *structure.Value
}
type Listx struct {
head *ListxNode
tail *ListxNode
length int
}
func NewListXSingle() structure.ListXInterface {
return &Listx{
head: nil,
tail: nil,
length: 0,
}
}
func (l *Listx) initByValue(val string) int {
if l.head == nil && l.length == 0 {
node := &ListxNode{
val: structure.NewValue(val),
}
l.head = node
l.tail = node
l.length = 1
return node.val.GetSize()
}
return 0
}
// 定位到 list 的元素, 支持反向索引
func (l *Listx) location(index int) (*ListxNode, error) {
// 正定位
if index >= 0 {
node := l.head
for ; index != 0 && node != nil; index -= 1 {
node = node.next
}
if node == nil {
return nil, errorx.New("index crosses the line")
}
return node, nil
}
node := l.tail
for index = (-index) - 1; index != 0 && node != nil; {
node = node.pre
index -= 1
}
if node == nil {
return nil, errorx.New("index crosses the line")
}
return node, nil
}
// 转换为左索引,负数索引
func (l *Listx) leftIndex(index int) (int, error) {
if index < 0 && l.length+index > 0 {
return index, nil
}
if index >= 0 && index < l.length {
return index - l.length, nil
}
return 0, errorx.New("the index is not valid, index:%d", index)
}
// 转换为右索引,正数索引
func (l *Listx) rightIndex(index int) (int, error) {
if index >= 0 && index < l.length {
return index, nil
}
if index < 0 && l.length+index >= 0 {
return l.length + index, nil
}
return 0, errorx.New("the index is not valid, index:%d", index)
}
func (l *Listx) remove(node *ListxNode) {
if node == nil {
return
}
l.length -= 1
if node == l.head {
l.head = node.next
node.next.pre = nil
return
}
if node == l.tail {
l.tail = node.pre
node.pre.next = nil
return
}
node.pre.next = node.next
node.next.pre = node.pre
}
func (l *Listx) SizeByte() int64 {
bytes := 0
r := l.head
for r != nil {
bytes += 16 + r.val.GetSize()
r = r.next
}
return int64(bytes)
}
// RollBack TODO 事务相关, V2 实现
func (l *Listx) RollBack() error {
panic("not implemented") // TODO: Implement
}
// Begin 事务相关, V2 实现
func (l *Listx) Begin() error {
panic("not implemented") // TODO: Implement
}
// Comment 事务相关, V2 实现
func (l *Listx) Comment() error {
panic("not implemented") // TODO: Implement
}
func (l *Listx) Encode() ([]byte, error) {
panic("not implemented") // TODO: Implement
}
func (l *Listx) LPush(valueStr ...string) structure.UpdateLength {
if len(valueStr) == 0 {
return 0
}
// 使用第一个元素尝试初始化列表
updateLength := l.initByValue(valueStr[0])
for i := 1; i < len(valueStr); i++ {
head := l.head
val := structure.NewValue(valueStr[i])
node := &ListxNode{val: val}
node.next = head
head.pre = node
l.head = node
l.length += 1
updateLength += val.GetSize()
}
return structure.UpdateLength(updateLength)
}
func (l *Listx) RPush(valueStr ...string) structure.UpdateLength {
if len(valueStr) == 0 {
return 0
}
updateLength := l.initByValue(valueStr[0])
for i := 1; i < len(valueStr); i++ {
tail := l.tail
val := structure.NewValue(valueStr[i])
node := &ListxNode{val: val}
tail.next = node
node.pre = tail
l.tail = node
l.length += 1
updateLength += val.GetSize()
}
return structure.UpdateLength(updateLength)
}
func (l *Listx) LPop(count int) ([]string, structure.UpdateLength) {
values := make([]string, 0, count)
head := l.head
// 动态变化长度
updateLength := 0
for nodePoint := 0; head != nil && nodePoint < count; nodePoint++ {
values = append(values, head.val.ToString())
updateLength += head.val.GetSize()
head = head.next
l.length -= 1
}
if head != nil {
head.pre = nil
} else {
l.tail = nil
}
l.head = head
return values, structure.UpdateLength(updateLength)
}
func (l *Listx) RPop(count int) ([]string, structure.UpdateLength) {
values := make([]string, 0, count)
tail := l.tail
updateLength := 0
for nodePoint := 0; tail != nil && nodePoint < count; nodePoint++ {
values = append(values, tail.val.ToString())
updateLength += tail.val.GetSize()
tail = tail.pre
l.length -= 1
}
if tail != nil {
tail.next = nil
} else {
l.head = nil
}
l.tail = tail
return values, structure.UpdateLength(updateLength)
}
func (l *Listx) Index(index int) (string, error) {
node, err := l.location(index)
if err != nil {
return "", nil
}
return node.val.ToString(), nil
}
// Insert, right 为 true 右添加,否则左添加
func (l *Listx) Insert(index int, right bool, valueStr ...string) (structure.UpdateLength, error) {
targetNode, err := l.location(index)
if err != nil {
return 0, err
}
updateLength := 0
for _, valStr := range valueStr {
val := structure.NewValue(valStr)
node := &ListxNode{val: val}
// 右插
if right {
node.pre = targetNode
node.next = targetNode.next
targetNode.next = node
targetNode.next.pre = node
// 更新尾部
if targetNode == l.tail {
l.tail = node
}
targetNode = node
} else {
// 左插
node.pre = targetNode.pre
targetNode.pre.next = node
node.next = targetNode
targetNode.pre = node
// 更新头部
if targetNode == l.head {
l.head = node
}
targetNode = node
}
updateLength += val.GetSize()
l.length += 1
}
return structure.UpdateLength(updateLength), nil
}
func (l *Listx) Length() int {
return l.length
}
// Slice 切片
func (l *Listx) Slice(start int, end int) (structure.UpdateLength, error) {
startOffset, err := l.rightIndex(start)
if err != nil {
return 0, err
}
endRightOffset, err := l.rightIndex(end)
if err != nil && end != l.length {
return 0, errorx.New("index overstep the boundary, index: %d", end)
}
if startOffset >= endRightOffset && endRightOffset != 0 {
return 0, errorx.New("the start index must be larger than the end index")
}
// 计算左偏移
var endOffset int
if end == l.length {
endOffset = 0
} else {
endOffset, err = l.leftIndex(end)
if err != nil {
return 0, err
}
}
updateLength := 0
// 右切片
head := l.head
for nodePoint := 0; head != nil && nodePoint < startOffset; nodePoint++ {
updateLength += head.val.GetSize()
head = head.next
l.length -= 1
}
l.head = head
head.pre = nil
tail := l.tail
for nodePoint := 0; tail != nil && nodePoint < -endOffset; nodePoint++ {
updateLength += tail.val.GetSize()
tail = tail.pre
l.length -= 1
}
l.tail = tail
tail.next = nil
return structure.UpdateLength(updateLength), nil
}
// Range 遍历
func (l *Listx) Range(start, end int) ([]string, error) {
startOffset, err := l.rightIndex(start)
if err != nil {
return nil, err
}
endRightOffset, err := l.rightIndex(end)
if err != nil && end != l.length {
return nil, errorx.New("index overstep the boundary, index: %d", end)
}
if startOffset >= endRightOffset && endRightOffset != 0 {
return nil, errorx.New("the start index must be larger than the end index")
}
head := l.head
for nodePoint := 0; head != nil && nodePoint < startOffset; nodePoint++ {
head = head.next
}
values := make([]string, 0)
for i := 0; i < endRightOffset-startOffset && head != nil; i++ {
values = append(values, head.val.ToString())
head = head.next
}
return values, nil
}
func (l *Listx) Remove(value string, count int) (int, structure.UpdateLength) {
if count == 0 {
return 0, 0
}
updateLength := 0
remCount := count
// 头删除
if count > 0 {
node := l.head
for node != nil && remCount > 0 {
if node.val.ToString() == value {
l.remove(node)
remCount--
updateLength += node.val.GetSize()
}
node = node.next
}
return count - remCount, structure.UpdateLength(updateLength)
}
// 尾删除
node := l.tail
for node != nil && remCount < 0 {
if node.val.ToString() == value {
l.remove(node)
remCount++
updateLength += node.val.GetSize()
}
node = node.pre
}
return remCount - count, structure.UpdateLength(updateLength)
}

View File

@ -1,228 +0,0 @@
package listx
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestListx_LPush_And_Pop(t *testing.T) {
list := NewListXSingle()
up := list.LPush("1", "2", "3")
require.Equal(t, list.Length(), 3)
values, updateLength := list.LPop(3)
require.Equal(t, values, []string{"3", "2", "1"})
require.Equal(t, up, updateLength)
list.LPush("1", "2", "3")
values, updateLength = list.LPop(1)
require.Equal(t, values, []string{"3"})
require.Equal(t, int(updateLength), 24)
}
func TestListx_RPush_Pop(t *testing.T) {
list := NewListXSingle()
up := list.RPush("1", "2", "3")
require.Equal(t, list.Length(), 3)
values, updateLength := list.LPop(3)
require.Equal(t, values, []string{"1", "2", "3"})
require.Equal(t, up, updateLength)
list.RPush("1", "2", "3")
values, updateLength = list.RPop(2)
require.Equal(t, values, []string{"3", "2"})
require.Equal(t, int(updateLength), 48)
}
func TestListx_location(t *testing.T) {
list := &Listx{
head: nil,
tail: nil,
length: 0,
}
list.RPush("1", "2", "3")
node, err := list.location(1)
require.NoError(t, err)
require.Equal(t, node.val.ToString(), "2")
node, err = list.location(0)
require.NoError(t, err)
require.Equal(t, node.val.ToString(), "1")
node, err = list.location(2)
require.NoError(t, err)
require.Equal(t, node.val.ToString(), "3")
_, err = list.location(3)
require.Error(t, err)
node, err = list.location(-1)
require.NoError(t, err)
require.Equal(t, node.val.ToString(), "3")
node, err = list.location(-2)
require.NoError(t, err)
require.Equal(t, node.val.ToString(), "2")
node, err = list.location(-3)
require.NoError(t, err)
require.Equal(t, node.val.ToString(), "1")
_, err = list.location(-4)
require.Error(t, err)
}
func TestListx_Insert(t *testing.T) {
list := NewListXSingle()
list.LPush("a", "b", "c", "d", "e")
val, _ := list.LPop(1)
require.Equal(t, val, []string{"e"})
res, err := list.Index(1)
require.NoError(t, err)
require.Equal(t, res, "c")
_, err = list.Insert(1, false, "1", "2", "3")
require.NoError(t, err)
// 全部取出
val, _ = list.LPop(list.Length())
require.Equal(t, val, []string{"d", "3", "2", "1", "c", "b", "a"})
list.RPush("1", "2", "3", "4", "5")
res, err = list.Index(-2)
require.NoError(t, err)
require.Equal(t, res, "4")
_, err = list.Insert(-1, true, "6", "7")
require.NoError(t, err)
val, _ = list.LPop(list.Length())
require.Equal(t, val, []string{"1", "2", "3", "4", "5", "6", "7"})
}
func TestListx_Index(t *testing.T) {
list := &Listx{
head: nil,
tail: nil,
length: 0,
}
list.RPush("a", "b", "c", "d", "e")
index := 2
leftIndex, err := list.leftIndex(index)
require.NoError(t, err)
rightIndex, err := list.rightIndex(index)
require.NoError(t, err)
v1, err := list.Index(leftIndex)
require.NoError(t, err)
v2, err := list.Index(rightIndex)
require.NoError(t, err)
require.Equal(t, v1, v2)
}
func TestListx_Slice(t *testing.T) {
list := NewListXSingle()
list.RPush("a", "b", "c", "d", "e")
// 主流程测试
list.Slice(1, 2)
values, _ := list.LPop(list.Length())
require.Equal(t, values, []string{"b"})
list2 := NewListXSingle()
list2.RPush("1", "2", "3", "4", "5")
list2.Slice(0, 4)
values, _ = list2.LPop(list2.Length())
require.Equal(t, values, []string{"1", "2", "3", "4"})
list3 := NewListXSingle()
list3.RPush("1", "2", "3", "4", "5")
list3.Slice(2, list3.Length())
values, _ = list3.LPop(list3.Length())
require.Equal(t, values, []string{"3", "4", "5"})
// 测试负数索引
list3 = NewListXSingle()
list3.RPush("1", "2", "3", "4", "5")
_, err := list3.Slice(0, -2)
require.NoError(t, err)
values, _ = list3.LPop(list3.Length())
require.Equal(t, values, []string{"1", "2", "3"})
// 测试负数双边际
list3 = NewListXSingle()
list3.RPush("1", "2", "3", "4", "5")
_, err = list3.Slice(-3, -2)
require.NoError(t, err)
values, _ = list3.LPop(list3.Length())
require.Equal(t, values, []string{"3"})
// 测试负数边际
list3 = NewListXSingle()
list3.RPush("1", "2", "3", "4", "5")
_, err = list3.Slice(-5, 4)
require.NoError(t, err)
values, _ = list3.LPop(list3.Length())
require.Equal(t, values, []string{"1", "2", "3", "4"})
}
func TestListx_Range(t *testing.T) {
list := NewListXSingle()
list.RPush("a", "b", "c", "d", "e")
val, err := list.Range(0, 2)
require.NoError(t, err)
require.Equal(t, val, []string{"a", "b"})
val, err = list.Range(0, -1)
require.NoError(t, err)
require.Equal(t, val, []string{"a", "b", "c", "d"})
val, err = list.Range(-3, 3)
require.NoError(t, err)
require.Equal(t, val, []string{"c"})
val, err = list.Range(-1, list.Length())
require.NoError(t, err)
require.Equal(t, val, []string{})
_, err = list.Range(6, -1)
require.Error(t, err)
}
func TestListx_Remove(t *testing.T) {
list := NewListXSingle()
list.RPush("a", "b", "c", "c", "e")
count, up := list.Remove("c", -2)
require.Equal(t, count, 2)
require.Equal(t, int(up), 34)
res, _ := list.LPop(list.Length())
require.Equal(t, res, []string{"a", "b", "e"})
list = NewListXSingle()
list.RPush("a", "b", "c", "c", "e")
count, up = list.Remove("b", 1)
require.Equal(t, count, 1)
require.Equal(t, int(up), 17)
res, _ = list.LPop(list.Length())
require.Equal(t, res, []string{"a", "c", "c", "e"})
}

Some files were not shown because too many files have changed in this diff Show More