diff --git a/README.md b/README.md new file mode 100644 index 0000000..83cb08a --- /dev/null +++ b/README.md @@ -0,0 +1,236 @@ +# craw + +- 数据缓存包 + +## 1. 功能 + +实现了数据的二级缓存，使用K-V结构存储，用于缓存网络访问后的数据以便于二次访问，可用于redis，mysql等数据的二级缓存: + +
+ +
+ + +## 2. 使用 + +```go + import ( + "github.com/wonderivan/craw" + ) + + // 使用者需要实现自定义的CrawInterface包含的四个方法才可以进行craw初始化 + type benchCraw struct { + } + + func (this *benchCraw) Init() error { + // 创建缓存时，在创建完成后会执行用户自定义的初始化函数，如果不需要初始化其他项，可以直接返回nil + return nil + } + + func (this *benchCraw) CustomGet(key string) (data interface{}, expired time.Duration, err error) { + // 当调用craw获取远端数据时,内部会调用该方式实现,用户需要指定缓存数据过期时间，0为马上过期 + return key, -1, nil + } + + func (this *benchCraw) CustomSet(key string, data interface{}) error { + // 当调用craw设置远端数据时,内部会调用该方式实现，不需要设置，可以直接返回nil + return nil + } + + func (this *benchCraw) Destroy() { + // 销毁缓存时，会执行用户自定义的销毁方法，如果不需要销毁其他项，该方法可以为空 + } + + + // 创建缓存，使用默认配置 + craw := NewCraw("mytest1", new(benchCraw)) + // 使用完后销毁缓存 + defer craw.Destroy() + + // 设置缓存数据 key:"name",value:"Lily", 过期时间-1，不过期 + handler.SetCraw("name","Lily", -1) + + // 获取缓存数据 + handler.GetData("name") + + // 获取缓存数据命中率 + handler.HitRate() + +``` + +## 3. 方法列表 +```go +// 创建一个craw +// 参数 crawName:缓存名称，config缓存配置， dispose缓存数据处理方法 +func NewCraw(crawName string, dispose CrawInterface, config ...string) *Craw + +// config格式 +{ + "low": 858993459, // 缓存压缩最低阈值限制，默认为800 MB + "high": 1073741824, // 缓存触发清理最高阈值，默认为1 GB，当缓存数据大于1G时，开始清理，详见LruCache说明 + "interval": 3600 // 缓存有效数据检查间隔，默认为1天 +} + +// 销毁craw +func (dc *Craw) Destroy() + +// 获取缓存数据，如果不存在则从远端获取数据并更新到craw然后返回 +// +// 参数 key:要查找的craw数据的key +// 返回值 interface{}:找到的数据 error:成功为nil +func (dc *Craw) GetData(key string) (interface{}, error) + +// 强制从远端获取数据，并更新到craw +// +// 参数 key:要查找的远端数据的key +// 返回值 interface{}:更新到craw的数据 +// 返回值 成功为nil +func (dc *Craw) UpdateData(key string) (data interface{}, err error) + +// 删除craw缓存数据 +// +// 参数 key:要查找的远端数据的key +// 参数可选 delay:延迟删除数据的时间 单位(s) +// 返回值 成功为nil +func (dc *Craw) DeleteData(key string, delay ...time.Duration) (err error) + +// 保存数据到远端，并删除craw中已有的缓存值 +// +// 参数 key:要保存到远端的数据的key data:要保存到远端的数据 +// 返回值 error:成功为nil +func (dc *Craw) SetData(key string, data interface{}) (err error) + +// 清空craw的所有数据 +func (dc *Craw) ClearAll() error + +// 清除craw中所有包含前缀prefix的key的数据 +func (dc *Craw) ClearPrefixKeys(Prefix string) error + +// 获取当前craw缓存命中率 +// +// 返回值 float64:计算的结果，XX.XXXXX% +func (dc *Craw) HitRate() + +// 获取当前craw缓存命中率并重置命中率为0 +// +// 返回值 float64:计算的结果，XX.XXXXX% +func (dc *Craw) ResetHitRate() float64 + +// 设置craw缓存数据，不更新远端 +// +// 参数 key:要保存的数据的key data:要保存的数据,expired要保存的数据的过期时间，<0不过期 +// 返回值 error:成功为nil +func (dc *Craw) SetCraw(key string, data interface{}, expired time.Duration) error + +// 查询craw中指定的key是否存在 +func (dc *Craw) IsExist(key string) (bool, error) +``` + +## 4. 性能测试 + +针对当前设计的二级缓存进行精简的K-V读写压力测试 +测试分为1W，5W,10W,50W,100W,200W， +分别设置命中率为0%,10%,30%,50%,70%,100%进行读写测试， +然后统计总用时，每条读写耗时，每秒读写条数 + +### 4.1 性能图示 + +
+读取缓存性能测试曲线图 + +
+ +
+写入缓存性能测试曲线图 + +
+ +
+读取缓存性能测试柱状图 + +
+ +
+写入缓存性能测试柱状图 + +
+ +### 4.2 实测数据 + +测试量|读取命中率|总用时|每条读取耗时|每秒读取条数 +|-|-|-|-|-| +|10000|0.00%|8.681537ms|868.15ns/op|1151869.77op| +|10000|10.00%|6.025206ms|602.52ns/op|1659694.29op| +|10000|30.00%|5.374784ms|537.48ns/op|1860539.88op| +|10000|50.00%|3.643583ms|364.36ns/op|2744551.17op| +|10000|70.00%|2.648619ms|264.86ns/op|3775552.47op| +|10000|100.00%|1.210079ms|121.01ns/op|8263923.26op| +|50000|0.00%|45.364929ms|907.30ns/op|1102173.00op| +|50000|10.00%|38.790983ms|775.82ns/op|1288959.34op| +|50000|30.00%|29.191049ms|583.82ns/op|1712853.83op| +|50000|50.00%|24.245481ms|484.91ns/op|2062239.97op| +|50000|70.00%|13.359667ms|267.19ns/op|3742608.26op| +|50000|100.00%|6.47295ms|129.46ns/op|7724453.30op| +|100000|0.00%|96.2213ms|962.21ns/op|1039270.93op| +|100000|10.00%|81.877599ms|818.78ns/op|1221335.28op| +|100000|30.00%|68.601832ms|686.02ns/op|1457687.02op| +|100000|50.00%|52.831959ms|528.32ns/op|1892793.72op| +|100000|70.00%|28.859204ms|288.59ns/op|3465099.04op| +|100000|100.00%|14.512613ms|145.13ns/op|6890557.89op| +|500000|0.00%|580.435456ms|1160.87ns/op|861422.22op| +|500000|10.00%|550.468097ms|1100.94ns/op|908317.85op| +|500000|30.00%|419.168104ms|838.34ns/op|1192838.85op| +|500000|50.00%|312.806598ms|625.61ns/op|1598431.76op| +|500000|70.00%|259.572573ms|519.15ns/op|1926243.57op| +|500000|100.00%|89.384522ms|178.77ns/op|5593809.63op| +|1000000|0.00%|1.163256084s|1163.26ns/op|859655.94op| +|1000000|10.00%|1.070822941s|1070.82ns/op|933861.20op| +|1000000|30.00%|851.79008ms|851.79ns/op|1173998.18op| +|1000000|50.00%|647.7622ms|647.76ns/op|1543776.40op| +|1000000|70.00%|578.194375ms|578.19ns/op|1729522.19op| +|1000000|100.00%|195.846202ms|195.85ns/op|5106047.45op| +|2000000|0.00%|2.290312946s|1145.16ns/op|873243.11op| +|2000000|10.00%|2.440874727s|1220.44ns/op|819378.39op| +|2000000|30.00%|1.938333138s|969.17ns/op|1031814.38op| +|2000000|50.00%|1.50874304s|754.37ns/op|1325606.78op| +|2000000|70.00%|1.248193013s|624.10ns/op|1602316.29op| +|2000000|100.00%|424.174077ms|212.09ns/op|4715045.33op| + +测试量|写入命中率|总用时|每条写入耗时|每秒写入条数 +|-|-|-|-|-| +|10000|0.00%|2.84051ms|284.05ns/op|3520494.56op| +|10000|10.00%|2.72975ms|272.98ns/op|3663339.13op| +|10000|30.00%|2.409758ms|240.98ns/op|4149794.29op| +|10000|50.00%|2.157198ms|215.72ns/op|4635643.09op| +|10000|70.00%|1.676161ms|167.62ns/op|5966014.00op| +|10000|100.00%|1.272334ms|127.23ns/op|7859571.46op| +|50000|0.00%|19.780616ms|395.61ns/op|2527727.14op| +|50000|10.00%|21.111112ms|422.22ns/op|2368420.95op| +|50000|30.00%|18.702363ms|374.05ns/op|2673458.96op| +|50000|50.00%|15.233516ms|304.67ns/op|3282236.35op| +|50000|70.00%|10.959622ms|219.19ns/op|4562201.14op| +|50000|100.00%|8.132663ms|162.65ns/op|6148047.69op| +|100000|0.00%|40.368559ms|403.69ns/op|2477175.37op| +|100000|10.00%|41.886854ms|418.87ns/op|2387383.88op| +|100000|30.00%|39.782839ms|397.83ns/op|2513646.65op| +|100000|50.00%|39.339891ms|393.40ns/op|2541949.09op| +|100000|70.00%|36.249307ms|362.49ns/op|2758673.43op| +|100000|100.00%|16.909974ms|169.10ns/op|5913669.65op| +|500000|0.00%|290.534367ms|581.07ns/op|1720966.80op| +|500000|10.00%|294.119801ms|588.24ns/op|1699987.55op| +|500000|30.00%|301.406549ms|602.81ns/op|1658888.97op| +|500000|50.00%|240.751022ms|481.50ns/op|2076834.38op| +|500000|70.00%|232.707867ms|465.42ns/op|2148616.66op| +|500000|100.00%|103.425529ms|206.85ns/op|4834396.35op| +|1000000|0.00%|751.04895ms|751.05ns/op|1331471.14op| +|1000000|10.00%|763.607503ms|763.61ns/op|1309573.30op| +|1000000|30.00%|706.018802ms|706.02ns/op|1416392.87op| +|1000000|50.00%|466.665309ms|466.67ns/op|2142863.38op| +|1000000|70.00%|494.240314ms|494.24ns/op|2023307.23op| +|1000000|100.00%|215.737465ms|215.74ns/op|4635263.51op| +|2000000|0.00%|1.378425505s|689.21ns/op|1450930.78op| +|2000000|10.00%|1.321529037s|660.76ns/op|1513398.45op| +|2000000|30.00%|1.314644183s|657.32ns/op|1521324.19op| +|2000000|50.00%|882.820403ms|441.41ns/op|2265466.45op| +|2000000|70.00%|1.010007853s|505.00ns/op|1980182.62op| +|2000000|100.00%|468.810806ms|234.41ns/op|4266113.27op| diff --git a/batchget_test.go b/batchget_test.go new file mode 100644 index 0000000..43dbd4e --- /dev/null +++ b/batchget_test.go @@ -0,0 +1,213 @@ +package craw + +import ( + "fmt" + "log" + "testing" + "time" + + "github.com/wonderivan/logger" +) + +func init() { + logger.SetLogger(`{"Console": {"level": "WARN"}}`) +} + +type benchCraw struct { +} + +func (this *benchCraw) Init() error { + return nil +} + +func (this *benchCraw) CustomGet(key string) (data interface{}, expired time.Duration, err error) { + return key, -1, nil +} + +func (this *benchCraw) CustomSet(key string, data interface{}) error { + return nil +} + +func (this *benchCraw) Destroy() { + +} + +func TestGetByOrder(t *testing.T) { + handler := NewCraw("mytest", new(benchCraw)) + defer handler.Destroy() + + total := 2000000 + keys := make([]string, total) + fmt.Printf("fill cache use %d test data\n", total) + for i := 0; i < total; i++ { + keys[i] = fmt.Sprintf("key%d", i) + handler.SetCraw(keys[i], keys[i], -1) + } + + start := time.Now() + for i := 0; i < total; i++ { + _, err := handler.GetData(keys[i]) + if err != nil { + log.Fatalf("GetData err:%s", err) + } + } + interval := time.Since(start) + + fmt.Printf("test %d use %v, average:%.2fns/op,%.2fop/s \n", + total, interval, float64(interval)/float64(total), float64(total)/float64(interval)*1000000000) +} + +var outputType int +var outputGraph map[string][]string + +func TestStatisticsTable(t *testing.T) { + outputType = 1 + hitRate := []int{0, 1, 3, 5, 7, 10} + fmt.Println("测试量|读取命中率|总用时|每条读取耗时|每秒读取条数") + fmt.Println("|-|-|-|-|-|") + ExecStatisticsGet(10000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsGet(50000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsGet(100000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsGet(500000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsGet(1000000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsGet(2000000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + fmt.Println() + + fmt.Println("测试量|写入命中率|总用时|每条写入耗时|每秒写入条数") + fmt.Println("|-|-|-|-|-|") + ExecStatisticsSet(10000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsSet(50000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsSet(100000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsSet(500000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsSet(1000000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsSet(2000000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + fmt.Println() +} + +func TestStatisticsGraph(t *testing.T) { + outputType = 2 + outputGraph = make(map[string][]string) + hitRate := []int{0, 1, 3, 5, 7, 10} + ExecStatisticsGet(10000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsGet(50000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsGet(100000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsGet(500000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsGet(1000000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsGet(2000000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + + fmt.Println("读取命中率\t1W\t5W\t10W\t50W\t100W\t200W") + for _, hit := range hitRate { + k := fmt.Sprintf("%.2f%%", float32(hit)/10*100) + v := outputGraph[k] + fmt.Printf("%s", k) + for _, v1 := range v { + fmt.Printf("\t%s", v1) + } + fmt.Println() + } + fmt.Println() + + outputGraph = make(map[string][]string) + + ExecStatisticsSet(10000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsSet(50000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsSet(100000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsSet(500000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsSet(1000000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + ExecStatisticsSet(2000000, hitRate) // 0%,10%,30%,50%,70%,100%命中率 + fmt.Println("写入命中率\t1W\t5W\t10W\t50W\t100W\t200W") + for _, hit := range hitRate { + k := fmt.Sprintf("%.2f%%", float32(hit)/10*100) + v := outputGraph[k] + fmt.Printf("%s", k) + for _, v1 := range v { + fmt.Printf("\t%s", v1) + } + fmt.Println() + } + fmt.Println() +} + +func ExecStatisticsGet(total int, hits []int) { + for _, hit := range hits { + ExecStatisticsTestGet(total, hit) + } +} + +func ExecStatisticsSet(total int, hits []int) { + for _, hit := range hits { + ExecStatisticsTestSet(total, hit) + } +} + +// hit : 0-10 => 0%-100% +func ExecStatisticsTestGet(total, hit int) { + handler := NewCraw("mytest1", new(benchCraw)) + defer handler.Destroy() + + keys := make([]string, total) + for i := 0; i < total; i++ { + keys[i] = fmt.Sprintf("key%d", i) + if i%10 < hit { + handler.SetCraw(keys[i], keys[i], -1) + } + } + + start := time.Now() + for i := 0; i < total; i++ { + _, err := handler.GetData(keys[i]) + if err != nil { + log.Fatalf("GetData err:%s", err) + } + } + interval := time.Since(start) + hitRate := float32(hit) / 10 * 100 + if int(hitRate) != int(handler.HitRate()) { + log.Fatalf("Unexpect hitRate:%v,%v", hitRate, handler.HitRate()) + } + switch outputType { + case 1: + fmt.Printf("|%d|%.2f%%|%v|%.2fns/op|%.2fop|\n", + total, hitRate, + interval, float64(interval)/float64(total), float64(total)/float64(interval)*1000000000) + case 2: + outputGraph[fmt.Sprintf("%.2f%%", hitRate)] = append(outputGraph[fmt.Sprintf("%.2f%%", hitRate)], fmt.Sprintf("%.2f", float64(total)/float64(interval)*100000)) + fmt.Printf("|%d|%.2f%%|%.2fW|\n", + total, hitRate, float64(total)/float64(interval)*100000) + } + +} + +// hit : 0-10 => 0%-100% +func ExecStatisticsTestSet(total, hit int) { + handler := NewCraw("mytest1", new(benchCraw)) + defer handler.Destroy() + + keys := make([]string, total) + for i := 0; i < total; i++ { + keys[i] = fmt.Sprintf("key%d", i) + if i%10 < hit { + handler.SetCraw(keys[i], keys[i], -1) + } + } + + start := time.Now() + for i := 0; i < total; i++ { + err := handler.SetCraw(keys[i], keys[i], -1) + if err != nil { + log.Fatalf("SetData err:%s", err) + } + } + interval := time.Since(start) + hitRate := float32(hit) / 10 * 100 + switch outputType { + case 1: + fmt.Printf("|%d|%.2f%%|%v|%.2fns/op|%.2fop|\n", + total, hitRate, + interval, float64(interval)/float64(total), float64(total)/float64(interval)*1000000000) + case 2: + outputGraph[fmt.Sprintf("%.2f%%", hitRate)] = append(outputGraph[fmt.Sprintf("%.2f%%", hitRate)], fmt.Sprintf("%.2f", float64(total)/float64(interval)*100000)) + fmt.Printf("|%d|%.2f%%|%.2fW|\n", + total, hitRate, float64(total)/float64(interval)*100000) + } +} diff --git a/craw.go b/craw.go new file mode 100755 index 0000000..efe210d --- /dev/null +++ b/craw.go @@ -0,0 +1,227 @@ +package craw + +import ( + "errors" + "sync" + "sync/atomic" + "time" + + logger "github.com/wonderivan/logger" + lrucache "github.com/wonderivan/lrucache" +) + +// 缓存数据操作接口 +type CrawInterface interface { + // 初始化 + Init() error + // 获取远端数据的方式 + CustomGet(key string) (data interface{}, expired time.Duration, err error) + // 设置远端数据的方式，设置后会清除对应的缓存 + CustomSet(key string, data interface{}) (err error) + // 销毁 + Destroy() +} + +type customValue struct { + data interface{} + err error + wait chan bool +} + +// 获取远端数据模块 +type Craw struct { + dispose CrawInterface + keyMap map[string]*customValue + craw *lrucache.LruCache + enable bool + sync.RWMutex + + // 统计命中率 + totalAccess *uint64 // 总访问量 + staggerAccess *uint64 // 未命中的访问数 +} + +// 创建一个craw +// +// 参数 crawName:缓存名称，config缓存配置， dispose缓存数据处理方法 +func NewCraw(crawName string, dispose CrawInterface, config ...string) *Craw { + logger.Info("craw(%s) use setting:%s", crawName, config) + c := new(Craw) + c.dispose = dispose + c.keyMap = make(map[string]*customValue) + c.craw = lrucache.NewLruCache(crawName, config...) + c.totalAccess = new(uint64) + c.staggerAccess = new(uint64) + err := c.dispose.Init() + if err == nil { + c.enable = true + } + + return c +} + +// 销毁Craw +func (c *Craw) Destroy() { + c.enable = false + c.craw.Destroy() + c.dispose.Destroy() +} + +// 获取远端数据更新到缓存并返回 +// +// 参数 key:要查找的远端数据的key +// 返回值 interface{}:找到的数据 error:成功为nil +func (c *Craw) GetData(key string) (interface{}, error) { + if !c.enable { + return nil, errors.New("this craw has been destroyed.") + } + + atomic.AddUint64(c.totalAccess, 1) + + // 先查找craw + data, exist := c.craw.GetEx(key) + if exist { + return data, nil + } + + c.Lock() + value, ok := c.keyMap[key] + + if !ok { + atomic.AddUint64(c.staggerAccess, 1) + + valueOpt := &customValue{nil, nil, make(chan bool)} + c.keyMap[key] = valueOpt + c.Unlock() + // 此key没找过 + // 开始找 + var expired time.Duration + valueOpt.data, expired, valueOpt.err = c.dispose.CustomGet(key) + if valueOpt.err == nil { + if valueOpt.err == nil { + c.craw.Put(key, valueOpt.data, expired) + } + } + c.Lock() + delete(c.keyMap, key) + c.Unlock() + close(valueOpt.wait) + return valueOpt.data, valueOpt.err + } else { + c.Unlock() + // 其他线程在找同个key，阻塞 + <-value.wait + // 其他线程执行完，从map的data取 + return value.data, value.err + } +} + +// 强制从远端获取数据，并更新craw +// +// 参数 key:要查找的远端数据的key +// 返回值 interface 更新到cache的数据 +// 返回值 成功为nil +func (c *Craw) UpdateData(key string) (data interface{}, err error) { + if !c.enable { + return nil, errors.New("this craw has been destroyed.") + } + data, expired, err := c.dispose.CustomGet(key) + if err == nil && data != nil { + c.craw.Put(key, data, expired) + } + return data, err +} + +//删除craw指定数据 +// +// 参数 key:要查找的远端数据的key +// 参数可选 delay:延迟删除数据的时间 单位(s) +// 返回值 成功为nil +func (c *Craw) DeleteData(key string, delay ...time.Duration) (err error) { + if !c.enable { + return errors.New("this craw has been destroyed.") + } + if len(delay) > 0 { + return c.craw.DelayDelete(key, delay[0]) + } + return c.craw.Delete(key) + +} + +// 保存数据到远端，并删除Craw中已有的缓存值 +// +// 参数 key:要保存到远端的数据的key data:要保存到远端的数据 +// 返回值 error:成功为nil +func (c *Craw) SetData(key string, data interface{}) (err error) { + if !c.enable { + return errors.New("this craw has been destroyed.") + } + + err = c.dispose.CustomSet(key, data) + if err != nil { + c.craw.Delete(key) + } + return +} + +//清除craw的所有数据 +func (c *Craw) ClearAll() error { + if !c.enable { + return errors.New("this craw has been destroyed.") + } + c.craw.ClearAll() + return nil +} + +// 清除craw中指定的包含前缀prefix的key的数据 +func (c *Craw) ClearPrefixKeys(Prefix string) error { + if !c.enable { + return errors.New("this craw has been destroyed.") + } + c.craw.ClearPrefixKeys(Prefix) + return nil +} + +// 获取craw的缓存命中率 +// +// 返回值 float64:计算的结果，XX.XXXXX% +func (c *Craw) HitRate() float64 { + custom := atomic.LoadUint64(c.staggerAccess) + total := atomic.LoadUint64(c.totalAccess) + if total == 0 { + return 0 + } + return float64(total-custom) / float64(total) * 100 +} + +// 重置craw命中率并返回重置之前命中率 +// +// 返回值 float64:计算的结果，XX.XXXXX% +func (c *Craw) ResetHitRate() float64 { + custom := atomic.SwapUint64(c.staggerAccess, 0) + total := atomic.SwapUint64(c.totalAccess, 0) + if total == 0 { + return 0 + } + return float64(total-custom) / float64(total) * 100 +} + +// 设置craw数据，不更新远端 +// +// 参数 key:要保存的数据的key data:要保存的数据,expired要保存的数据的过期时间，<0不过期 +// 返回值 error:成功为nil +func (c *Craw) SetCraw(key string, data interface{}, expired time.Duration) error { + if !c.enable { + return errors.New("this craw has been destroyed.") + } + c.craw.Put(key, data, expired) + return nil +} + +// 查询craw中指定的key是否存在 +func (c *Craw) IsExist(key string) (bool, error) { + if !c.enable { + return false, errors.New("this craw has been destroyed.") + } + return c.craw.IsExist(key), nil +} diff --git a/craw_test.go b/craw_test.go new file mode 100755 index 0000000..9ea2bfc --- /dev/null +++ b/craw_test.go @@ -0,0 +1,113 @@ +package craw + +import ( + "fmt" + "log" + "testing" + "time" +) + +type cacheGet struct { + testV int +} + +func (this *cacheGet) Init() error { + this.testV = 0 + return nil +} + +func (this *cacheGet) CustomGet(key string) (data interface{}, expired time.Duration, err error) { + this.testV++ + return this.testV, 0, nil +} + +func (this *cacheGet) CustomSet(key string, data interface{}) error { + return nil +} + +func (this *cacheGet) Destroy() { + +} + +func TestCrawGetPut(t *testing.T) { + handler := NewCraw("mytest", new(cacheGet)) + defer handler.Destroy() + count := 0 + for count < 10 { + v, err := handler.GetData("k1") + if err != nil { + log.Fatalf("GetData err:%s", err) + } + log.Printf("GetData type(%T), value:%v", v, v) + + count = v.(int) + + err = handler.SetCraw("k1", count+1, 1) + if err != nil { + log.Fatalf("GetData err:%s", err) + } + } +} + +func TestCrawHitRate(t *testing.T) { + handler := NewCraw("mytest", new(cacheGet)) + defer handler.Destroy() + count := 0 + for count < 100000 { + key := fmt.Sprintf("key%d", count) + _, err := handler.GetData(key) + if err != nil { + log.Fatalf("GetData err:%s", err) + } + //log.Printf("GetData %s type(%T), value:%v", key, v, v) + + if count%3 == 0 { + key = fmt.Sprintf("key%d", count+1) + err = handler.SetCraw(key, count+1, 10) + if err != nil { + log.Fatalf("GetData err:%s", err) + } + //log.Printf("SetData %s value:%v", key, v) + } + count++ + } + + log.Printf("hit rate:%v", handler.HitRate()) // output: 1/3 = 33.3333% +} + +// go test -benchmem -run=^$ -bench ^BenchmarkPut$ +func BenchmarkPut(b *testing.B) { + handler := NewCraw("mytest2", new(cacheGet)) + defer handler.Destroy() + + b.ResetTimer() + b.N = 10000000 + keys := make([]string, b.N) + for i := 0; i < b.N; i++ { + keys[i] = fmt.Sprintf("key%d", i) + } + for i := 0; i < b.N; i++ { + handler.SetCraw(keys[i], keys[i], -1) + } +} + +// go test -benchmem -run=^$ -bench ^BenchmarkGet$ +func BenchmarkGet(b *testing.B) { + handler := NewCraw("mytest", new(cacheGet)) + defer handler.Destroy() + + b.N = 10000000 + keys := make([]string, b.N) + for i := 0; i < b.N; i++ { + keys[i] = fmt.Sprintf("key%d", i) + handler.SetCraw(keys[i], keys[i], -1) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := handler.GetData(keys[i]) + if err != nil { + log.Fatalf("GetData err:%s", err) + } + } +} diff --git a/images/craw-graph-get.png b/images/craw-graph-get.png new file mode 100644 index 0000000..7014ee7 Binary files /dev/null and b/images/craw-graph-get.png differ diff --git a/images/craw-graph-set.png b/images/craw-graph-set.png new file mode 100644 index 0000000..8f31ddc Binary files /dev/null and b/images/craw-graph-set.png differ diff --git a/images/craw-histogram-get.png b/images/craw-histogram-get.png new file mode 100644 index 0000000..0a1208e Binary files /dev/null and b/images/craw-histogram-get.png differ diff --git a/images/craw-histogram-set.png b/images/craw-histogram-set.png new file mode 100644 index 0000000..ddc263c Binary files /dev/null and b/images/craw-histogram-set.png differ diff --git a/images/logic.png b/images/logic.png new file mode 100644 index 0000000..b64036f Binary files /dev/null and b/images/logic.png differ