go/kratos
1
0
Fork 0
mirror of https://github.com/go-kratos/kratos.git synced 2025-01-10 00:29:01 +02:00
Code Issues Releases Activity

Merge remote-tracking branch 'github/master'

Browse Source
This commit is contained in:
wuxingzhong 2019-11-04 11:14:08 +08:00
commit 26b90c83e8
122 changed files with 6540 additions and 745 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.travis.yml
View File

@ -2,6 +2,10 @@ language: go
go:
- 1.12.x
- 1.13.x
services:
- docker
# Only clone the most recent commit.
git:
@ -16,6 +20,13 @@ env:
- DEPLOY_ENV=dev
- DISCOVERY_NODES=127.0.0.1:7171
- HTTP_PERF=tcp://0.0.0.0:0
- DOCKER_COMPOSE_VERSION=1.24.1
before_install:
- sudo rm /usr/local/bin/docker-compose
- curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
- chmod +x docker-compose
- sudo mv docker-compose /usr/local/bin
# Skip the install step. Don't `go get` dependencies. Only build with the code
# in vendor/

BIN
doc/img/zipkin.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 64 KiB

2
doc/wiki-cn/FAQ.md
View File

@ -9,7 +9,7 @@ go get -u github.com/bilibili/kratos/tool/kratos
go: github.com/prometheus/client_model@v0.0.0-20190220174349-fd36f4220a90: parsing go.mod: missing module line
go: github.com/remyoudompheng/bigfft@v0.0.0-20190806203942-babf20351dd7e3ac320adedbbe5eb311aec8763c: parsing go.mod: missing module line
```
如果你使用了https://goproxy.io/ 代理,那你要使用其他代理来替换它，然后删除GOPATH目录下的mod缓存文件夹,然后重新执行安装命令
如果你使用了https://goproxy.io/ 代理,那你要使用其他代理来替换它，然后删除GOPATH目录下的mod缓存文件夹（`go clean --modcache`）,然后重新执行安装命令
代理列表

49
doc/wiki-cn/breaker.md
View File

@ -0,0 +1,49 @@
## 熔断器/Breaker
熔断器是为了当依赖的服务已经出现故障时，主动阻止对依赖服务的请求。保证自身服务的正常运行不受依赖服务影响，防止雪崩效应。
## kratos内置breaker的组件
一般情况下直接使用kratos的组件时都自带了熔断逻辑，并且在提供了对应的breaker配置项。
目前在kratos内集成熔断器的组件有:
- RPC client: pkg/net/rpc/warden/client
- Mysql client：pkg/database/sql
- Tidb client：pkg/database/tidb
- Http client：pkg/net/http/blademaster
## 使用说明
```go
//初始化熔断器组
//一组熔断器公用同一个配置项，可从分组内取出单个熔断器使用。可用在比如mysql主从分离等场景。
brkGroup := breaker.NewGroup(&breaker.Config{})
//为每一个连接指定一个brekaker
//此处假设一个客户端连接对象实例为conn
//breakName定义熔断器名称 一般可以使用连接地址
breakName = conn.Addr
conn.breaker = brkGroup.Get(breakName)
//在连接发出请求前判断熔断器状态
if err = conn.breaker.Allow(); err != nil {
return
}
//连接执行成功或失败将结果告知braker
if(respErr != nil){
conn.breaker.MarkFailed()
}else{
conn.breaker.MarkSuccess()
}
```
## 配置说明
```go
type Config struct {
SwitchOff bool // 熔断器开关,默认关 false.
K float64 //触发熔断的错误率（K = 1 - 1/错误率）
Window xtime.Duration //统计桶窗口时间
Bucket int //统计桶大小
Request int64 //触发熔断的最少请求数量（请求少于该值时不会触发熔断）
}
```

75
doc/wiki-cn/config-paladin.md
View File

@ -46,9 +46,10 @@ func TestMain(t *testing.M) {
### example main
```go
# main.go
// main.go
func main() {
# 初始化paladin
flag.Parse()
// 初始化paladin
if err := paladin.Init(); err != nil {
panic(err)
}
@ -58,28 +59,31 @@ func main() {
```
### example HTTP/gRPC
```go
```toml
# http.toml
[server]
addr = "0.0.0.0:9000"
timeout = "1s"
# server.go
```
```go
// server.go
func NewServer() {
# 默认配置用nil，这时读取HTTP/gRPC构架中的flag或者环境变量（可能是docker注入的环境变量，默认端口：8000/9000）
engine := bm.DefaultServer(nil)
# 除非自己要替换了配置，用http.toml
var bc struct {
Server *bm.ServerConfig
}
if err = paladin.Get("http.toml").UnmarshalTOML("server", &bc); err != nil {
// 不存在时，将会为nil使用默认配置
if err ！= paladin.ErrNotExist {
panic(err)
}
}
engine := bm.DefaultServer(conf)
// 默认配置用nil，这时读取HTTP/gRPC构架中的flag或者环境变量（可能是docker注入的环境变量，默认端口：8000/9000）
engine := bm.DefaultServer(nil)
// 除非自己要替换了配置，用http.toml
var bc struct {
Server *bm.ServerConfig
}
if err := paladin.Get("http.toml").UnmarshalTOML(&bc); err != nil {
// 不存在时，将会为nil使用默认配置
if err != paladin.ErrNotExist {
panic(err)
}
}
engine := bm.DefaultServer(bc.Server)
}
```
@ -87,25 +91,28 @@ func NewServer() {
```go
# service.go
type Service struct {
ac *paladin.Map
ac *paladin.Map
}
func New() *Service {
# paladin.Map 通过atomic.Value支持自动热加载
var ac = new(paladin.TOML)
if err := paladin.Watch("application.toml", ac); err != nil {
panic(err)
}
s := &Service{
ac : ac;
}
return s
// paladin.Map 通过atomic.Value支持自动热加载
var ac = new(paladin.TOML)
if err := paladin.Watch("application.toml", ac); err != nil {
panic(err)
}
s := &Service{
ac: ac,
}
return s
}
func (s *Service) Test() {
switch, err := s.ac.Bool("switch")
if err != nil {
// TODO
}
# or use default value
switch := paladin.Bool(s.ac.Value("switch"), false)
sw, err := s.ac.Get("switch").Bool()
if err != nil {
// TODO
}
// or use default value
sw := paladin.Bool(s.ac.Get("switch"), false)
}
```

0
doc/wiki-cn/dapper.md
View File

102
doc/wiki-cn/ecode.md Normal file
View File

@ -0,0 +1,102 @@
# ecode
## 背景
错误码一般被用来进行异常传递，且需要具有携带`message`文案信息的能力。
## 错误码之Codes
在`kratos`里，错误码被设计成`Codes`接口，声明如下[代码位置](https://github.com/bilibili/kratos/blob/master/pkg/ecode/ecode.go)：
```go
// Codes ecode error interface which has a code & message.
type Codes interface {
// sometimes Error return Code in string form
// NOTE: don't use Error in monitor report even it also work for now
Error() string
// Code get error code.
Code() int
// Message get code message.
Message() string
//Detail get error detail,it may be nil.
Details() []interface{}
}
// A Code is an int error code spec.
type Code int
```
可以看到该接口一共有四个方法，且`type Code int`结构体实现了该接口。
### 注册message
一个`Code`错误码可以对应一个`message`，默认实现会从全局变量`_messages`中获取，业务可以将自定义`Code`对应的`message`通过调用`Register`方法的方式传递进去，如：
```go
cms := map[int]string{
0: "很好很强大！",
-304: "啥都没变啊~",
-404: "啥都没有啊~",
}
ecode.Register(cms)
fmt.Println(ecode.OK.Message()) // 输出：很好很强大！
```
注意：`map[int]string`类型并不是绝对，比如有业务要支持多语言的场景就可以扩展为类似`map[int]LangStruct`的结构，因为全局变量`_messages`是`atomic.Value`类型，只需要修改对应的`Message`方法实现即可。
### Details
`Details`接口为`gRPC`预留，`gRPC`传递异常会将服务端的错误码pb序列化之后赋值给`Details`，客户端拿到之后反序列化得到，具体可阅读`status`的实现：
1. `ecode`包内的`Status`结构体实现了`Codes`接口[代码位置](https://github.com/bilibili/kratos/blob/master/pkg/ecode/status.go)
2. `warden/internal/status`包内包装了`ecode.Status`和`grpc.Status`进行互相转换的方法[代码位置](https://github.com/bilibili/kratos/blob/master/pkg/net/rpc/warden/internal/status/status.go)
3. `warden`的`client`和`server`则使用转换方法将`gRPC`底层返回的`error`最终转换为`ecode.Status` [代码位置](https://github.com/bilibili/kratos/blob/master/pkg/net/rpc/warden/client.go#L162)
## 转换为ecode
错误码转换有以下两种情况：
1. 因为框架传递错误是靠`ecode`错误码，比如bm框架返回的`code`字段默认就是数字，那么客户端接收到如`{"code":-404}`的话，可以使用`ec := ecode.Int(-404)`或`ec := ecode.String("-404")`来进行转换。
2. 在项目中`dao`层返回一个错误码，往往返回参数类型建议为`error`而不是`ecode.Codes`，因为`error`更通用，那么上层`service`就可以使用`ec := ecode.Cause(err)`进行转换。
## 判断
错误码判断是否相等：
1. `ecode`与`ecode`判断使用：`ecode.Equal(ec1, ec2)`
2. `ecode`与`error`判断使用：`ecode.EqualError(ec, err)`
## 使用工具生成
使用proto协议定义错误码，格式如下：
```proto
// user.proto
syntax = "proto3";
package ecode;
enum UserErrCode {
UserUndefined = 0; // 因protobuf协议限制必须存在！！！无意义的0，工具生成代码时会忽略该参数
UserNotLogin = 123; // 正式错误码
}
```
需要注意以下几点：
1. 必须是enum类型，且名字规范必须以"ErrCode"结尾，如：UserErrCode
2. 因为protobuf协议限制，第一个enum值必须为无意义的0
使用`kratos tool protoc --ecode user.proto`进行生成，生成如下代码：
```go
package ecode
import (
"github.com/bilibili/kratos/pkg/ecode"
)
var _ ecode.Codes
// UserErrCode
var (
UserNotLogin = ecode.New(123);
)
```

7
doc/wiki-cn/summary.md
View File

@ -16,7 +16,8 @@
* [warden protobuf](warden-pb.md)
* [config](config.md)
* [paladin](config-paladin.md)
* [dapper trace](dapper.md)
* [ecode](ecode.md)
* [trace](dapper.md)
* [log](logger.md)
* [log-agent](log-agent.md)
* [database](database.md)
@ -34,3 +35,7 @@
* [genbts](kratos-genbts.md)
* [限流bbr](ratelimit.md)
* [熔断breaker](breaker.md)
* [UT单元测试](ut.md)
* [testcli UT运行环境构建工具](ut-testcli.md)
* [testgen UT代码自动生成器](ut-testgen.md)
* [support UT周边辅助工具](ut-support.md)

42
doc/wiki-cn/trace.md Normal file
View File

@ -0,0 +1,42 @@
# 背景
当代的互联网的服务，通常都是用复杂的、大规模分布式集群来实现的。互联网应用构建在不同的软件模块集上，这些软件模块，有可能是由不同的团队开发、可能使用不同的编程语言来实现、有可能布在了几千台服务器，横跨多个不同的数据中心。因此，就需要一些可以帮助理解系统行为、用于分析性能问题的工具。
# 概览
* kratos内部的trace基于opentracing语义
* 使用protobuf协议描述trace结构
* 全链路支持（gRPC/HTTP/MySQL/Redis/Memcached等）
## 参考文档
[opentracing](https://github.com/opentracing-contrib/opentracing-specification-zh/blob/master/specification.md)
[dapper](https://bigbully.github.io/Dapper-translation/)
# 使用
kratos本身不提供整套`trace`数据方案，但在`net/trace/report.go`内声明了`repoter`接口，可以简单的集成现有开源系统，比如：`zipkin`和`jaeger`。
### zipkin使用
可以看[zipkin](https://github.com/bilibili/kratos/tree/master/pkg/net/trace/zipkin)的协议上报实现，具体使用方式如下：
1. 前提是需要有一套自己搭建的`zipkin`集群
2. 在业务代码的`main`函数内进行初始化，代码如下：
```go
// 忽略其他代码
import "github.com/bilibili/kratos/pkg/net/trace/zipkin"
// 忽略其他代码
func main(){
// 忽略其他代码
zipkin.Init(&zipkin.Config{
Endpoint: "http://localhost:9411/api/v2/spans",
})
// 忽略其他代码
}
```
### zipkin效果图
![zipkin](/doc/img/zipkin.jpg)

497
doc/wiki-cn/ut-support.md Normal file
View File

@ -0,0 +1,497 @@
## 单元测试辅助工具
在单元测试中，我们希望每个测试用例都是独立的。这时候就需要Stub, Mock, Fakes等工具来帮助我们进行用例和依赖之间的隔离。
同时通过对错误情况的 Mock 也可以帮我们检查代码多个分支结果，从而提高覆盖率。
以下工具已加入到 Kratos 框架 go modules，可以借助 testgen 代码生成器自动生成部分工具代码，请放心食用。更多使用方法还欢迎大家多多探索。
### GoConvey
GoConvey是一套针对golang语言的BDD类型的测试框架。提供了良好的管理和执行测试用例的方式，包含丰富的断言函数，而且同时有测试执行和报告Web界面的支持。
#### 使用特性
为了更好的使用 GoConvey 来编写和组织测试用例，需要注意以下几点特性：
1. Convey方法和So方法的使用
> - Convey方法声明了一种规格的组织，每个组织内包含一句描述和一个方法。在方法内也可以嵌套其他Convey语句和So语句。
```Go
// 顶层Convey方法，需引入*testing.T对象
Convey(description string, t *testing.T, action func())
// 其他嵌套Convey方法，无需引入*testing.T对象
Convey(description string, action func())
```
注：同一Scope下的Convey语句描述不可以相同！
> - So方法是断言方法，用于对执行结果进行比对。GoConvey官方提供了大量断言，同时也可以自定义自己的断言（[戳这里了解官方文档](https://github.com/smartystreets/goconvey/wiki/Assertions)）
```Go
// A=B断言
So(A, ShouldEqual, B)
// A不为空断言
So(A, ShouldNotBeNil)
```
2. 执行次序
> 假设有以下Convey伪代码，执行次序将为A1B2A1C3。将Convey方法类比树的结点的话，整体执行类似树的遍历操作。
> 所以Convey A部分可在组织测试用例时，充当“Setup”的方法。用于初始化等一些操作。
```Go
Convey伪代码
Convey A
So 1
Convey B
So 2
Convey C
So 3
```
3. Reset方法
> GoConvey提供了Reset方法来进行“Teardown”的操作。用于执行完测试用例后一些状态的回收，连接关闭等操作。Reset方法不可与顶层Convey语句在同层。
```Go
// Reset
Reset func(action func())
```
假设有以下带有Reset方法的伪代码，同层Convey语句执行完后均会执行同层的Reset方法。执行次序为A1B2C3EA1D4E。
```Go
Convey A
So 1
Convey B
So 2
Convey C
So 3
Convey D
So 4
Reset E
```
4. 自然语言逻辑到测试用例的转换
> 在了解了Convey方法的特性和执行次序后，我们可以通过这些性质把对一个方法的测试用例按照日常逻辑组织起来。尤其建议使用Given-When-Then的形式来组织
> - 比较直观的组织示例
```Go
Convey("Top-level", t, func() {
// Setup 工作，在本层内每个Convey方法执行前都会执行的部分：
db.Open()
db.Initialize()
Convey("Test a query", func() {
db.Query()
// TODO: assertions here
})
Convey("Test inserts", func() {
db.Insert()
// TODO: assertions here
})
Reset(func() {
// Teardown工作，在本层内每个Convey方法执行完后都会执行的部分：
db.Close()
})
})
```
> - 定义单独的包含Setup和Teardown的帮助方法
```Go
package main
import (
"database/sql"
"testing"
_ "github.com/lib/pq"
. "github.com/smartystreets/goconvey/convey"
)
// 帮助方法，将原先所需的处理方法以参数（f）形式传入
func WithTransaction(db *sql.DB, f func(tx *sql.Tx)) func() {
return func() {
// Setup工作
tx, err := db.Begin()
So(err, ShouldBeNil)
Reset(func() {
// Teardown工作
/* Verify that the transaction is alive by executing a command */
_, err := tx.Exec("SELECT 1")
So(err, ShouldBeNil)
tx.Rollback()
})
// 调用传入的闭包做实际的事务处理
f(tx)
}
}
func TestUsers(t *testing.T) {
db, err := sql.Open("postgres", "postgres://localhost?sslmode=disable")
if err != nil {
panic(err)
}
Convey("Given a user in the database", t, WithTransaction(db, func(tx *sql.Tx) {
_, err := tx.Exec(`INSERT INTO "Users" ("id", "name") VALUES (1, 'Test User')`)
So(err, ShouldBeNil)
Convey("Attempting to retrieve the user should return the user", func() {
var name string
data := tx.QueryRow(`SELECT "name" FROM "Users" WHERE "id" = 1`)
err = data.Scan(&name)
So(err, ShouldBeNil)
So(name, ShouldEqual, "Test User")
})
}))
}
```
#### 使用建议
强烈建议使用 [testgen](https://github.com/bilibili/kratos/blob/master/doc/wiki-cn/ut-testgen.md) 进行测试用例的生成，生成后每个方法将包含一个符合以下规范的正向用例。
用例规范：
1. 每个方法至少包含一个测试方法（命名为Test[PackageName][FunctionName]）
2. 每个测试方法包含一个顶层Convey语句，仅在此引入admin *testing.T类型的对象，在该层进行变量声明。
3. 每个测试方法不同的用例用Convey方法组织
4. 每个测试用例的一组断言用一个Convey方法组织
5. 使用convey.C保持上下文一致
### MonkeyPatching
#### 特性和使用条件
1. Patch()对任何无接收者的方法均有效
2. PatchInstanceMethod()对有接收者的包内/私有方法无法工作（因使用到了反射机制）。可以采用给私有方法的下一级打补丁，或改为无接收者的方法，或将方法转为公有
#### 适用场景（建议）
项目代码中上层对下层包依赖时，下层包方法Mock（例如service层对dao层方法依赖时）
基础库（MySql, Memcache, Redis）错误Mock
其他标准库，基础库以及第三方包方法Mock
#### 使用示例
1. 上层包对下层包依赖示例
Service层对Dao层依赖：
```GO
// 原方法
func (s *Service) realnameAlipayApply(c context.Context, mid int64) (info *model.RealnameAlipayApply, err error) {
if info, err = s.mbDao.RealnameAlipayApply(c, mid); err != nil {
return
}
...
return
}
// 测试方法
func TestServicerealnameAlipayApply(t *testing.T) {
convey.Convey("realnameAlipayApply", t, func(ctx convey.C) {
...
ctx.Convey("When everything goes positive", func(ctx convey.C) {
guard := monkey.PatchInstanceMethod(reflect.TypeOf(s.mbDao), "RealnameAlipayApply", func(_ *dao.Dao, _ context.Context, _ int64) (*model.RealnameAlipayApply, error) {
return nil, nil
})
defer guard.Unpatch()
info, err := s.realnameAlipayApply(c, mid)
ctx.Convey("Then err should be nil,info should not be nil", func(ctx convey.C) {
ctx.So(info, convey.ShouldNotBeNil)
ctx.So(err, convey.ShouldBeNil)
})
})
})
}
```
2. 基础库错误Mock示例
```Go
// 原方法（部分）
func (d *Dao) BaseInfoCache(c context.Context, mid int64) (info *model.BaseInfo, err error) {
...
conn := d.mc.Get(c)
defer conn.Close()
item, err := conn.Get(key)
if err != nil {
log.Error("conn.Get(%s) error(%v)", key, err)
return
}
...
return
}
// 测试方法（错误Mock部分）
func TestDaoBaseInfoCache(t *testing.T) {
convey.Convey("BaseInfoCache", t, func(ctx convey.C) {
...
Convey("When conn.Get gets error", func(ctx convey.C) {
guard := monkey.PatchInstanceMethod(reflect.TypeOf(d.mc), "Get", func(_ *memcache.Pool, _ context.Context) memcache.Conn {
return memcache.MockWith(memcache.ErrItemObject)
})
defer guard.Unpatch()
_, err := d.BaseInfoCache(c, mid)
ctx.Convey("Error should be equal to memcache.ErrItemObject", func(ctx convey.C) {
ctx.So(err, convey.ShouldEqual, memcache.ErrItemObject)
})
})
})
}
```
#### 注意事项
- Monkey非线程安全
- Monkey无法针对Inline方法打补丁，在测试时可以使用go test -gcflags=-l来关闭inline编译的模式（一些简单的go inline介绍戳这里）
- Monkey在一些面向安全不允许内存页写和执行同时进行的操作系统上无法工作
- 更多详情请戳：https://github.com/bouk/monkey
### Gock——HTTP请求Mock工具
#### 特性和使用条件
#### 工作原理
1. 截获任意通过 http.DefaultTransport或者自定义http.Transport对外的http.Client请求
2. 以“先进先出”原则将对外需求和预定义好的HTTP Mock池中进行匹配
3. 如果至少一个Mock被匹配，将按照2中顺序原则组成Mock的HTTP返回
4. 如果没有Mock被匹配，若实际的网络可用，将进行实际的HTTP请求。否则将返回错误
#### 特性
- 内建帮助工具实现JSON/XML简单Mock
- 支持持久的、易失的和TTL限制的Mock
- 支持HTTP Mock请求完整的正则表达式匹配
- 可通过HTTP方法，URL参数，请求头和请求体匹配
- 可扩展和可插件化的HTTP匹配规则
- 具备在Mock和实际网络模式之间切换的能力
- 具备过滤和映射HTTP请求到正确的Mock匹配的能力
- 支持映射和过滤可以更简单的掌控Mock
- 通过使用http.RoundTripper接口广泛兼容HTTP拦截器
- 可以在任意net/http兼容的Client上工作
- 网络延迟模拟（beta版本）
- 无其他依赖
#### 适用场景（建议）
任何需要进行HTTP请求的操作，建议全部用Gock进行Mock，以减少对环境的依赖。
使用示例：
1. net/http 标准库 HTTP 请求Mock
```Go
import gock "gopkg.in/h2non/gock.v1"
// 原方法
func (d *Dao) Upload(c context.Context, fileName, fileType string, expire int64, body io.Reader) (location string, err error) {
...
resp, err = d.bfsClient.Do(req) //d.bfsClient类型为*http.client
...
if resp.StatusCode != http.StatusOK {
...
}
header = resp.Header
code = header.Get("Code")
if code != strconv.Itoa(http.StatusOK) {
...
}
...
return
}
// 测试方法
func TestDaoUpload(t *testing.T) {
convey.Convey("Upload", t, func(ctx convey.C) {
...
// d.client 类型为 *http.client 根据Gock包描述需要设置http.Client的Transport情况。也可在TestMain中全局设置，则所有的HTTP请求均通过Gock来解决
d.client.Transport = gock.DefaultTransport // ！注意：进行httpMock前需要对http 请求进行拦截，否则Mock失败
// HTTP请求状态和Header都正确的Mock
ctx.Convey("When everything is correct", func(ctx convey.C) {
httpMock("PUT", url).Reply(200).SetHeaders(map[string]string{
"Code": "200",
"Location": "SomePlace",
})
location, err := d.Upload(c, fileName, fileType, expire, body)
ctx.Convey("Then err should be nil.location should not be nil.", func(ctx convey.C) {
ctx.So(err, convey.ShouldBeNil)
ctx.So(location, convey.ShouldNotBeNil)
})
})
...
// HTTP请求状态错误Mock
ctx.Convey("When http request status != 200", func(ctx convey.C) {
d.client.Transport = gock.DefaultTransport
httpMock("PUT", url).Reply(404)
_, err := d.Upload(c, fileName, fileType, expire, body)
ctx.Convey("Then err should not be nil", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
})
})
// HTTP请求Header中Code值错误Mock
ctx.Convey("When http request Code in header != 200", func(ctx convey.C) {
d.client.Transport = gock.DefaultTransport
httpMock("PUT", url).Reply(404).SetHeaders(map[string]string{
"Code": "404",
"Location": "SomePlace",
})
_, err := d.Upload(c, fileName, fileType, expire, body)
ctx.Convey("Then err should not be nil", func(ctx convey.C) {
ctx.So(err, convey.ShouldNotBeNil)
})
})
// 由于同包内有其他进行实际HTTP请求的测试。所以再每次用例结束后，进行现场恢复（关闭Gock设置默认的Transport）
ctx.Reset(func() {
gock.OffAll()
d.client.Transport = http.DefaultClient.Transport
})
})
}
func httpMock(method, url string) *gock.Request {
r := gock.New(url)
r.Method = strings.ToUpper(method)
return r
}
```
2. blademaster库HTTP请求Mock
```Go
// 原方法
func (d *Dao) SendWechatToGroup(c context.Context, chatid, msg string) (err error) {
...
if err = d.client.Do(c, req, &res); err != nil {
...
}
if res.Code != 0 {
...
}
return
}
// 测试方法
func TestDaoSendWechatToGroup(t *testing.T) {
convey.Convey("SendWechatToGroup", t, func(ctx convey.C) {
...
// 根据Gock包描述需要设置bm.Client的Transport情况。也可在TestMain中全局设置，则所有的HTTP请求均通过Gock来解决。
// d.client 类型为 *bm.client
d.client.SetTransport(gock.DefaultTransport) // ！注意：进行httpMock前需要对http 请求进行拦截，否则Mock失败
// HTTP请求状态和返回内容正常Mock
ctx.Convey("When everything gose postive", func(ctx convey.C) {
httpMock("POST", _sagaWechatURL+"/appchat/send").Reply(200).JSON(`{"code":0,"message":"0"}`)
err := d.SendWechatToGroup(c, d.c.WeChat.ChatID, msg)
...
})
// HTTP请求状态错误Mock
ctx.Convey("When http status != 200", func(ctx convey.C) {
httpMock("POST", _sagaWechatURL+"/appchat/send").Reply(404)
err := d.SendWechatToGroup(c, d.c.WeChat.ChatID, msg)
...
})
// HTTP请求返回值错误Mock
ctx.Convey("When http response code != 0", func(ctx convey.C) {
httpMock("POST", _sagaWechatURL+"/appchat/send").Reply(200).JSON(`{"code":-401,"message":"0"}`)
err := d.SendWechatToGroup(c, d.c.WeChat.ChatID, msg)
...
})
// 由于同包内有其他进行实际HTTP请求的测试。所以再每次用例结束后，进行现场恢复（关闭Gock设置默认的Transport）。
ctx.Reset(func() {
gock.OffAll()
d.client.SetTransport(http.DefaultClient.Transport)
})
})
}
func httpMock(method, url string) *gock.Request {
r := gock.New(url)
r.Method = strings.ToUpper(method)
return r
}
```
#### 注意事项
- Gock不是完全线程安全的
- 如果执行并发代码，在配置Gock和解释定制的HTTP clients时，要确保Mock已经事先声明好了来避免不需要的竞争机制
- 更多详情请戳：https://github.com/h2non/gock
### GoMock
#### 使用条件
只能对公有接口（interface）定义的代码进行Mock，并仅能在测试过程中进行
#### 使用方法
- 官方安装使用步骤
```shell
## 获取GoMock包和自动生成Mock代码工具mockgen
go get github.com/golang/mock/gomock
go install github.com/golang/mock/mockgen
## 生成mock文件
## 方法1：生成对应文件下所有interface
mockgen -source=path/to/your/interface/file.go
## 方法2：生成对应包内指定多个interface，并用逗号隔开
mockgen database/sql/driver Conn,Driver
## 示例：
mockgen -destination=$GOPATH/kratos/app/xxx/dao/dao_mock.go -package=dao kratos/app/xxx/dao DaoInterface
```
- testgen 使用步骤（GoMock生成功能已集成在Creater工具中，无需额外安装步骤即可直接使用）
```shell
## 直接给出含有接口类型定义的包路径，生成Mock文件将放在包目录下一级mock/pkgName_mock.go中
./creater --m mock absolute/path/to/your/pkg
```
- 测试代码内使用方法
```Go
// 测试用例内直接使用
// 需引入的包
import (
...
"github.com/otokaze/mock/gomock"
...
)
func TestPkgFoo(t *testing.T) {
convey.Convey("Foo", t, func(ctx convey.C) {
...
ctx.Convey("Mock Interface to test", func(ctx convey.C) {
// 1. 使用gomock.NewController新增一个控制器
mockCtrl := gomock.NewController(t)
// 2. 测试完成后关闭控制器
defer mockCtrl.Finish()
// 3. 以控制器为参数生成Mock对象
yourMock := mock.NewMockYourClient(mockCtrl)
// 4. 使用Mock对象替代原代码中的对象
yourClient = yourMock
// 5. 使用EXPECT().方法名(方法参数).Return(返回值)来构造所需输入/输出
yourMock.EXPECT().YourMethod(gomock.Any()).Return(nil)
res:= Foo(params)
...
})
...
})
}
// 可以利用Convey执行顺序方式适当调整以简化代码
func TestPkgFoo(t *testing.T) {
convey.Convey("Foo", t, func(ctx convey.C) {
...
mockCtrl := gomock.NewController(t)
yourMock := mock.NewMockYourClient(mockCtrl)
ctx.Convey("Mock Interface to test1", func(ctx convey.C) {
yourMock.EXPECT().YourMethod(gomock.Any()).Return(nil)
...
})
ctx.Convey("Mock Interface to test2", func(ctx convey.C) {
yourMock.EXPECT().YourMethod(args).Return(res)
...
})
...
ctx.Reset(func(){
mockCtrl.Finish()
})
})
}
```
#### 适用场景（建议）
1. gRPC中的Client接口
2. 也可改造现有代码构造Interface后使用（具体可配合Creater的功能进行Interface和Mock的生成）
3. 任何对接口中定义方法依赖的场景
#### 注意事项
- 如有Mock文件在包内，在执行单元测试时Mock代码会被识别进行测试。请注意Mock文件的放置。
- 更多详情请戳：https://github.com/golang/mock

154
doc/wiki-cn/ut-testcli.md Normal file
View File

@ -0,0 +1,154 @@
## testcli UT运行环境构建工具
基于 docker-compose 实现跨平台跨语言环境的容器依赖管理方案，以解决运行ut场景下的 (mysql, redis, mc)容器依赖问题。
*这个是testing/lich的二进制工具版本（Go请直接使用库版本：github.com/bilibili/kratos/pkg/testing/lich)*
### 功能和特性
- 自动读取 test 目录下的 yaml 并启动依赖
- 自动导入 test 目录下的 DB 初始化 SQL
- 提供特定容器内的 healthcheck (mysql, mc, redis)
- 提供一站式解决 UT 服务依赖的工具版本 (testcli)
### 编译安装
*使用本工具/库需要前置安装好 docker & docker-compose@v1.24.1^*
#### Method 1. With go get
```shell
go get -u github.com/bilibili/kratos/tool/testcli
$GOPATH/bin/testcli -h
```
#### Method 2. Build with Go
```shell
cd github.com/bilibili/kratos/tool/testcli
go build -o $GOPATH/bin/testcli
$GOPATH/bin/testcli -h
```
#### Method 3. Import with Kratos pkg
```Go
import "github.com/bilibili/kratos/pkg/testing/lich"
```
### 构建数据
#### Step 1. create docker-compose.yml
创建依赖服务的 docker-compose.yml，并把它放在项目路径下的 test 文件夹下面。例如：
```shell
mkdir -p $YOUR_PROJECT/test
```
```yaml
version: "3.7"
services:
db:
image: mysql:5.6
ports:
- 3306:3306
environment:
- MYSQL_ROOT_PASSWORD=root
volumes:
- .:/docker-entrypoint-initdb.d
command: [
'--character-set-server=utf8',
'--collation-server=utf8_unicode_ci'
]
redis:
image: redis
ports:
- 6379:6379
```
一般来讲，我们推荐在项目根目录创建 test 目录，里面存放描述服务的yml，以及需要初始化的数据（database.sql等）。
同时也需要注意，正确的对容器内服务进行健康检测，testcli会在容器的health状态执行UT，其实我们也内置了针对几个较为通用镜像（mysql mariadb mc redis）的健康检测，也就是不写也没事(^^;;
#### Step 2. export database.sql
构造初始化的数据（database.sql等），当然也把它也在 test 文件夹里。
```sql
CREATE DATABASE IF NOT EXISTS `YOUR_DATABASE_NAME`;
SET NAMES 'utf8';
USE `YOUR_DATABASE_NAME`;
CREATE TABLE IF NOT EXISTS `YOUR_TABLE_NAME` (
`id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键'，
PRIMARY KEY (`id`),
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='YOUR_TABLE_NAME';
```
这里需要注意，在创建库/表的时候尽量加上 IF NOT EXISTS，以给予一定程度的容错，以及 SET NAMES 'utf8'; 用于解决客户端连接乱码问题。
#### Step 3. change your project mysql config
```toml
[mysql]
addr = "127.0.0.1:3306"
dsn = "root:root@tcp(127.0.0.1:3306)/YOUR_DATABASE?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8mb4,utf8"
active = 20
idle = 10
idleTimeout ="1s"
queryTimeout = "1s"
execTimeout = "1s"
tranTimeout = "1s"
```
在 *Step 1* 我们已经指定了服务对外暴露的端口为3306（这当然也可以是你指定的任何值），那理所应当的我们也要修改项目连接数据库的配置～
Great! 至此你已经完成了运行所需要用到的数据配置，接下来就来运行它。
### 运行
开头也说过本工具支持两种运行方式：testcli 二进制工具版本和 go package 源码包，业务方可以根据需求场景进行选择。
#### Method 1. With testcli tool
*已支持的 flag： -f，--nodown，down，run*
- -f，指定 docker-compose.yaml 文件路径，默认为当前目录下。
- --nodown，指定是否在UT执行完成后保留容器，以供下次复用。
- down，teardown 销毁当前项目下这个 compose 文件产生的容器。
- run，运行你当前语言的单测执行命令（如：golang为 go test -v ./）
example:
```shell
testcli -f ../../test/docker-compose.yaml run go test -v ./
```
#### Method 2. Import with Kratos pkg
- Step1. 在 Dao|Service 层中的 TestMain 单测主入口中，import "github.com/bilibili/kratos/pkg/testing/lich" 引入testcli工具的go库版本。
- Step2. 使用 flag.Set("f", "../../test/docker-compose.yaml") 指定 docker-compose.yaml 文件的路径。
- Step3. 在 flag.Parse() 后即可使用 lich.Setup() 安装依赖&初始化数据（注意测试用例执行结束后 lich.Teardown() 回收下～）
- Step4. 运行 `go test -v ./ `看看效果吧～
example:
```Go
package dao
import (
"flag"
"os"
"strings"
"testing"
"github.com/bilibili/kratos/pkg/conf/paladin"
"github.com/bilibili/kratos/pkg/testing/lich"
)
var (
d *Dao
)
func TestMain(m *testing.M) {
flag.Set("conf", "../../configs")
flag.Set("f", "../../test/docker-compose.yaml")
flag.Parse()
if err := paladin.Init(); err != nil {
panic(err)
}
if err := lich.Setup(); err != nil {
panic(err)
}
defer lich.Teardown()
d = New()
if code := m.Run(); code != 0 {
panic(code)
}
}
```
## 注意
因为启动mysql容器较为缓慢，健康检测的机制会重试3次，每次暂留5秒钟，基本在10s内mysql就能从creating到服务正常启动！
当然你也可以在使用 testcli 时加上 --nodown，使其不用每次跑都新建容器，只在第一次跑的时候会初始化容器，后面都进行复用，这样速度会快很多。
成功启动后就欢乐奔放的玩耍吧～ Good Lucky!

52
doc/wiki-cn/ut-testgen.md Normal file
View File

@ -0,0 +1,52 @@
## testgen UT代码自动生成器
解放你的双手，让你的UT一步到位！
### 功能和特性
- 支持生成 Dao|Service 层UT代码功能（每个方法包含一个正向用例）
- 支持生成 Dao|Service 层测试入口文件dao_test.go, service_test.go（用于控制初始化，控制测试流程等）
- 支持生成Mock代码（使用GoMock框架）
- 支持选择不同模式生成不同代码（使用"–m mode"指定）
- 生成单元测试代码时，同时支持传入目录或文件
- 支持指定方法追加生成测试用例（使用"–func funcName"指定）
### 编译安装
#### Method 1. With go get
```shell
go get -u github.com/bilibili/kratos/tool/testgen
$GOPATH/bin/testgen -h
```
#### Method 2. Build with Go
```shell
cd github.com/bilibili/kratos/tool/testgen
go build -o $GOPATH/bin/testgen
$GOPATH/bin/testgen -h
```
### 运行
#### 生成Dao/Service层单元UT
```shell
$GOPATH/bin/testgen YOUR_PROJECT/dao # default mode
$GOPATH/bin/testgen --m test path/to/your/pkg
$GOPATH/bin/testgen --func functionName path/to/your/pkg
```
#### 生成接口类型
```shell
$GOPATH/bin/testgen --m interface YOUR_PROJECT/dao #当前仅支持传目录，如目录包含子目录也会做处理
```
#### 生成Mock代码
```shell
$GOPATH/bin/testgen --m mock YOUR_PROJECT/dao #仅传入包路径即可
```
#### 生成Monkey代码
```shell
$GOPATH/bin/testgen --m monkey yourCodeDirPath #仅传入包路径即可
```
### 赋诗一首
```
莫生气 莫生气
代码辣鸡非我意
自己动手分田地
谈笑风生活长命
```

38
doc/wiki-cn/ut.md Normal file
View File

@ -0,0 +1,38 @@
# 背景
单元测试即对最小可测试单元进行检查和验证，它可以很好的让你的代码在上测试环境之前自己就能前置的发现问题，解决问题。当然每个语言都有原生支持的 UT 框架，不过在 Kratos 里面我们需要有一些配套设施以及周边工具来辅助我们构筑整个 UT 生态。
# 工具链
- testgen UT代码自动生成器（README: tool/testgen/README.md）
- testcli UT运行环境构建工具（README: tool/testcli/README.md）
# 测试框架选型
golang 的单元测试，既可以用官方自带的 testing 包，也有开源的如 testify、goconvey 业内知名，使用非常多也很好用的框架。
根据一番调研和内部使用经验，我们确定：
> - testing 作为基础库测试框架（非常精简不过够用）
> - goconvey 作为业务程序的单元测试框架（因为涉及比较多的业务场景和流程控制判断，比如更丰富的res值判断、上下文嵌套支持、还有webUI等）
# 单元测试标准
1. 覆盖率，当前标准：60%（所有包均需达到）
尽量达到70%以上。当然覆盖率并不能完全说明单元测试的质量，开发者需要考虑关键的条件判断和预期的结果。复杂的代码是需要好好设计测试用例的。
2. 通过率，当前标准：100%（所有用例中的断言必须通过）
# 书写建议
1. 结果验证
> - 校验err是否为nil. err是go函数的标配了，也是最基础的判断，如果err不为nil，基本上函数返回值或者处理肯定是有问题了。
> - 检验res值是否正确。res值的校验是非常重要的，也是很容易忽略的地方。比如返回结构体对象，要对结构体的成员进行判断，而有可能里面是0值。goconvey对res值的判断支持是非常友好的。
2. 逻辑验证
> 业务代码经常是流程比较复杂的，而函数的执行结果也是有上下文的，比如有不同条件分支。goconvey就非常优雅的支持了这种情况，可以嵌套执行。单元测试要结合业务代码逻辑，才能尽量的减少线上bug。
3. 如何mock
主要分以下3块：
> - 基础组件，如mc、redis、mysql等，由 testcli(testing/lich) 起基础镜像支持（需要提供建表、INSERT语句）与本地开发环境一致，也保证了结果的一致性。
> - rpc server，如 xxxx-service 需要定义 interface 供业务依赖方使用。所有rpc server 都必须要提供一个interface+mock代码(gomock)。
> - http server则直接写mock代码gock。
# 注意
单元测试极其重要，良好的单元测试习惯能很大程度上避免代码变更引起的bug！
单元测试极其重要，良好的单元测试习惯能很大程度上避免代码变更引起的bug！
单元测试极其重要，良好的单元测试习惯能很大程度上避免代码变更引起的bug！
以为很重要所以重复 3 遍～

2
example/blademaster/middleware/auth/example_test.go
View File

@ -3,8 +3,8 @@ package auth_test
import (
"fmt"
bm "github.com/bilibili/kratos/pkg/net/http/blademaster"
"github.com/bilibili/kratos/example/blademaster/middleware/auth"
bm "github.com/bilibili/kratos/pkg/net/http/blademaster"
"github.com/bilibili/kratos/pkg/net/metadata"
)

14
example/protobuf/api.bm.go
View File

@ -9,6 +9,7 @@ import (
bm "github.com/bilibili/kratos/pkg/net/http/blademaster"
"github.com/bilibili/kratos/pkg/net/http/blademaster/binding"
)
import google_protobuf1 "github.com/golang/protobuf/ptypes/empty"
// to suppressed 'imported but not used warning'
var _ *bm.Context
@ -16,10 +17,13 @@ var _ context.Context
var _ binding.StructValidator
var PathUserInfo = "/user.api.User/Info"
var PathUserCard = "/user.api.User/Card"
// UserBMServer is the server API for User service.
type UserBMServer interface {
Info(ctx context.Context, req *UserReq) (resp *InfoReply, err error)
Card(ctx context.Context, req *UserReq) (resp *google_protobuf1.Empty, err error)
}
var UserSvc UserBMServer
@ -33,8 +37,18 @@ func userInfo(c *bm.Context) {
c.JSON(resp, err)
}
func userCard(c *bm.Context) {
p := new(UserReq)
if err := c.BindWith(p, binding.Default(c.Request.Method, c.Request.Header.Get("Content-Type"))); err != nil {
return
}
resp, err := UserSvc.Card(c, p)
c.JSON(resp, err)
}
// RegisterUserBMServer Register the blademaster route
func RegisterUserBMServer(e *bm.Engine, server UserBMServer) {
UserSvc = server
e.GET("/user.api.User/Info", userInfo)
e.GET("/user.api.User/Card", userCard)
}

2
example/protobuf/api.proto
View File

@ -3,6 +3,7 @@ syntax = "proto3";
package user.api;
import "github.com/gogo/protobuf/gogoproto/gogo.proto";
import "google/protobuf/empty.proto";
option go_package = "api";
@ -30,4 +31,5 @@ message InfoReply {
service User {
rpc Info(UserReq) returns (InfoReply);
rpc Card(UserReq) returns (google.protobuf.Empty);
}

36
go.mod
View File

@ -4,8 +4,9 @@ go 1.12
require (
github.com/BurntSushi/toml v0.3.1
github.com/StackExchange/wmi v0.0.0-20190523213609-cbe66965904d // indirect
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 // indirect
github.com/aristanetworks/goarista v0.0.0-20190912214011-b54698eaaca6 // indirect
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect
@ -16,24 +17,23 @@ require (
github.com/go-playground/locales v0.12.1 // indirect
github.com/go-playground/universal-translator v0.16.0 // indirect
github.com/go-sql-driver/mysql v1.4.1
github.com/gogo/protobuf v1.2.1
github.com/gogo/protobuf v1.3.0
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
github.com/golang/mock v1.3.1 // indirect
github.com/golang/protobuf v1.3.2
github.com/google/btree v1.0.0 // indirect
github.com/google/uuid v1.1.1 // indirect
github.com/gorilla/websocket v1.4.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.9.4 // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/leodido/go-urn v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/montanaflynn/stats v0.5.0
github.com/openzipkin/zipkin-go v0.2.0
github.com/openzipkin/zipkin-go v0.2.1
github.com/otokaze/mock v0.0.0-20190125081256-8282b7a7c7c3
github.com/philchia/agollo v0.0.0-20190728085453-a95533fccea3
github.com/pkg/errors v0.8.1
github.com/prometheus/client_golang v1.0.0
github.com/prometheus/client_model v0.0.0-20190220174349-fd36f4220a90 // indirect
github.com/prometheus/common v0.6.0 // indirect
github.com/prometheus/procfs v0.0.3 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20190806203942-babf20351dd7e3ac320adedbbe5eb311aec8763c // indirect
github.com/prometheus/client_golang v1.1.0
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec // indirect
github.com/shirou/gopsutil v2.19.6+incompatible
github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726
@ -42,16 +42,16 @@ require (
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
github.com/tsuna/gohbase v0.0.0-20190502052937-24ffed0537aa
github.com/urfave/cli v1.20.0
go.etcd.io/etcd v0.0.0-20190720005121-fe86a786a4c3
go.etcd.io/etcd v0.0.0-20190917205325-a14579fbfb1a
go.uber.org/atomic v1.4.0 // indirect
go.uber.org/zap v1.10.0 // indirect
go.uber.org/multierr v1.2.0 // indirect
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect
golang.org/x/net v0.0.0-20190628185345-da137c7871d7
golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 // indirect
golang.org/x/time v0.0.0-20190513212739-9d24e82272b4 // indirect
golang.org/x/net v0.0.0-20191011234655-491137f69257
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect
golang.org/x/tools v0.0.0-20190912185636-87d9f09c5d89
google.golang.org/appengine v1.6.1 // indirect
google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610
google.golang.org/grpc v1.22.0
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03
google.golang.org/grpc v1.24.0
gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
gopkg.in/go-playground/validator.v9 v9.29.1
gopkg.in/yaml.v2 v2.2.2

158
go.sum
View File

@ -1,36 +1,44 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8=
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/StackExchange/wmi v0.0.0-20190523213609-cbe66965904d h1:VWP4o43LuzNbykZJzMUv5b9DWLgn0sn3GUj3RUyWMMQ=
github.com/StackExchange/wmi v0.0.0-20190523213609-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015 h1:7ABPr1+uJdqESAdlVevnc/2FJGiC/K3uMg1JiELeF+0=
github.com/aristanetworks/goarista v0.0.0-20190712234253-ed1100a1c015/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
github.com/aristanetworks/fsnotify v1.4.2/go.mod h1:D/rtu7LpjYM8tRJphJ0hUBYpjai8SfX+aSNsWDTq/Ks=
github.com/aristanetworks/glog v0.0.0-20180419172825-c15b03b3054f/go.mod h1:KASm+qXFKs/xjSoWn30NrWBBvdTTQq+UjkhjEJHfSFA=
github.com/aristanetworks/goarista v0.0.0-20190912214011-b54698eaaca6 h1:6bZNnQcA2fkzH9AhZXbp2nDqbWa4bBqFeUb70Zq1HBM=
github.com/aristanetworks/goarista v0.0.0-20190912214011-b54698eaaca6/go.mod h1:Z4RTxGAuYhPzcq8+EdRM+R8M48Ssle2TsWtwRKa+vns=
github.com/aristanetworks/splunk-hec-go v0.3.3/go.mod h1:1VHO9r17b0K7WmOlLb9nTk/2YanvOEnLMUgsFrxBROc=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cockroachdb/datadriven v0.0.0-20190531201743-edce55837238 h1:uNljlOxtOHrPnRoPPx+JanqjAGZpNiqAGVBfGskd/pg=
github.com/cockroachdb/datadriven v0.0.0-20190531201743-edce55837238/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5tgDm3YN7+9dYrpK96E5wFilTFWIDZOM=
github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c=
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf h1:CAKfRE2YtTUIjjh1bkBtyYFaUT/WmOqsJjgtihT0vMI=
github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8=
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 h1:MZRmHqDBd0vxNwenEbKSQqRVT24d3C05ft8kduSwlqM=
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -48,7 +56,7 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
@ -64,35 +72,35 @@ github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZp
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
@ -102,30 +110,34 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
github.com/grpc-ecosystem/grpc-gateway v1.9.4 h1:5xLhQjsk4zqPf9EHCrja2qFZMx+yBqkO3XgJ14bNnU0=
github.com/grpc-ecosystem/grpc-gateway v1.9.4/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI=
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/influxdb1-client v0.0.0-20190809212627-fc22c7df067e/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/reedsolomon v1.9.2/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
@ -136,11 +148,12 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk=
@ -149,10 +162,20 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/openzipkin/zipkin-go v0.2.0 h1:33/f6xXB6YlOQ9tgTsXVOkdLCJsHTcZJnMy4DnSd6FU=
github.com/openzipkin/zipkin-go v0.2.0/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/openconfig/gnmi v0.0.0-20190823184014-89b2bf29312c/go.mod h1:t+O9It+LKzfOAhKTT5O0ehDix+MTqbtT0T9t+7zzOvc=
github.com/openconfig/reference v0.0.0-20190727015836-8dfd928c9696/go.mod h1:ym2A+zigScwkSEb/cVQB0/ZMpU3rqiH6X7WRRsxgOGw=
github.com/openzipkin/zipkin-go v0.2.1 h1:noL5/5Uf1HpVl3wNsfkZhIKbSWCVi5jgqkONNx8PXcA=
github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
github.com/otokaze/mock v0.0.0-20190125081256-8282b7a7c7c3 h1:zjmNboC3QFuMdJSaZJ7Qvi3HUxWXPdj7wb3rc4jH5HI=
github.com/otokaze/mock v0.0.0-20190125081256-8282b7a7c7c3/go.mod h1:pLR8n2aimFxvvDJ6n8JuQWthMGezCYMjuhlaTjPTZf0=
github.com/philchia/agollo v0.0.0-20190728085453-a95533fccea3 h1:e/WwwXpp+h9CtbiwSdDxrgq6ymvrvLH/P+kS+8qq4v8=
github.com/philchia/agollo v0.0.0-20190728085453-a95533fccea3/go.mod h1:EXNdWdQkS+QBi0nb/Xm+sBBuQ1PM7/NIPr1JDzOlt8A=
github.com/philchia/agollo v2.3.1+incompatible h1:C4zDDuOcP1Qynikz2rSJQSMjwexv4GfDpwBHJRinhPc=
github.com/philchia/agollo v2.3.1+incompatible/go.mod h1:EXNdWdQkS+QBi0nb/Xm+sBBuQ1PM7/NIPr1JDzOlt8A=
github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
@ -160,37 +183,32 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8=
github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190220174349-fd36f4220a90 h1:Cov9QkEXNhh8QeXoICvORjJ4RrpyvXmSf7rHSpS+ZfI=
github.com/prometheus/client_model v0.0.0-20190220174349-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo=
github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/remyoudompheng/bigfft v0.0.0-20190806203942-babf20351dd7e3ac320adedbbe5eb311aec8763c h1:eED6LswgZ3TfAl9fb+L2TfdSlXpYdg21iWZMdHuoSks=
github.com/remyoudompheng/bigfft v0.0.0-20190806203942-babf20351dd7e3ac320adedbbe5eb311aec8763c/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook=
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec h1:6ncX5ko6B9LntYM0YBRXkiSaZMmLYeZ/NWcmeB43mMY=
github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/shirou/gopsutil v2.19.6+incompatible h1:49/Gru26Lne9Cl3IoAVDZVM09hvkSrUodgIIsCVRwbs=
github.com/shirou/gopsutil v2.19.6+incompatible/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM=
github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw=
github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@ -205,6 +223,9 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEvslTq+iOEAMDLSzhVuOt5BRZ05WirO+b09GHQU=
github.com/templexxx/xor v0.0.0-20181023030647-4e92f724b73b/go.mod h1:5XA7W9S6mni3h5uvOC75dA3m9CCCaS83lltmc0ukdi4=
github.com/tjfoc/gmsm v1.0.1/go.mod h1:XxO4hdhhrzAd+G4CjDqaOkd0hUzmtPR/d3EiBBMn/wc=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@ -212,23 +233,28 @@ github.com/tsuna/gohbase v0.0.0-20190502052937-24ffed0537aa h1:V/ABqiqsgqmpoIcLD
github.com/tsuna/gohbase v0.0.0-20190502052937-24ffed0537aa/go.mod h1:3HfLQly3YNLGxNv/2YOfmz30vcjG9hbuME1GpxoLlGs=
github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xtaci/kcp-go v5.4.5+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE=
github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/etcd v0.0.0-20190720005121-fe86a786a4c3 h1:LlCFU/KJ9P/8QKB73kkd1z/zbm7ZJ2V4HEgEHI3N7gk=
go.etcd.io/etcd v0.0.0-20190720005121-fe86a786a4c3/go.mod h1:N0RPWo9FXJYZQI4BTkDtQylrstIigYHeR18ONnyTufk=
go.etcd.io/etcd v0.0.0-20190917205325-a14579fbfb1a h1:OpCyFK9+wUB3g4o1guENLYOUZhG4hswKiFbE+jC12Cc=
go.etcd.io/etcd v0.0.0-20190917205325-a14579fbfb1a/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/multierr v1.2.0 h1:6I+W7f5VwC5SV9dNrZ3qXrDB9mD0dyGOi/ZJmYw03T4=
go.uber.org/multierr v1.2.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@ -246,8 +272,12 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2 h1:4dVFTC832rPn4pomLSz1vA+are2+dU19w1H8OngV7nc=
golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191011234655-491137f69257 h1:ry8e2D+cwaV6hk7lb3aRTjjZo24shrbK0e11QEOkTIg=
golang.org/x/net v0.0.0-20191011234655-491137f69257/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -260,51 +290,65 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI=
golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8 h1:41hwlulw1prEMBxLQSlMSux1zxJf07B3WPsdjJlKZxE=
golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190513212739-9d24e82272b4 h1:RMGusaKverhgGR5KBERIKiTyWoWHRd84GCtsNlvLvIo=
golang.org/x/time v0.0.0-20190513212739-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b h1:mSUCVIwDx4hfXJfWsOPfdzEHxzb2Xjl6BQ8YgPnazQA=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190912185636-87d9f09c5d89 h1:WiVZGyzQN7gPNLRkkpsNX3jC0Jx5j9GxadCZW/8eXw0=
golang.org/x/tools v0.0.0-20190912185636-87d9f09c5d89/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/genproto v0.0.0-20180608181217-32ee49c4dd80/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610 h1:Ygq9/SRJX9+dU0WCIICM8RkWvDw03lvB77hrhJnpxfU=
google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 h1:4HYDjxeNXAOTv3o1N2tjo8UUSlhQgAD52FVkwxnWgM8=
google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
google.golang.org/grpc v1.22.0 h1:J0UbZOIrCAl+fpTOf8YLs4dJo8L/owV4LYVtAXQoPkw=
google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk=
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a/go.mod h1:KF9sEfUPAXdG8Oev9e99iLGnl2uJMjc5B+4y3O7x610=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvRQyEAKbw1xc=
gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q=
gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4=
gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
gopkg.in/redis.v4 v4.2.4/go.mod h1:8KREHdypkCEojGKQcjMqAODMICIVwZAONWq8RowTITA=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=

48
pkg/cache/memcache/test/BUILD.bazel vendored
View File

@ -1,48 +0,0 @@
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
load(
"@io_bazel_rules_go//proto:def.bzl",
"go_proto_library",
)
go_library(
name = "go_default_library",
srcs = [],
embed = [":proto_go_proto"],
importpath = "go-common/library/cache/memcache/test",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = ["@com_github_golang_protobuf//proto:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
proto_library(
name = "test_proto",
srcs = ["test.proto"],
import_prefix = "go-common/library/cache/memcache/test",
strip_import_prefix = "",
tags = ["automanaged"],
)
go_proto_library(
name = "proto_go_proto",
compilers = ["@io_bazel_rules_go//proto:go_proto"],
importpath = "go-common/library/cache/memcache/test",
proto = ":test_proto",
tags = ["automanaged"],
)

9
pkg/cache/memcache/test/docker-compose.yaml vendored Executable file
View File

@ -0,0 +1,9 @@
version: "3.7"
services:
mc:
image: memcached:1
ports:
- 11211:11211

1
pkg/cache/metrics.go vendored
View File

@ -4,6 +4,7 @@ import "github.com/bilibili/kratos/pkg/stat/metric"
const _metricNamespace = "cache"
// be used in tool/kratos-gen-bts
var (
MetricHits = metric.NewCounterVec(&metric.CounterVecOpts{
Namespace: _metricNamespace,

27
pkg/cache/redis/commandinfo_test.go vendored Normal file
View File

@ -0,0 +1,27 @@
package redis
import "testing"
func TestLookupCommandInfo(t *testing.T) {
for _, n := range []string{"watch", "WATCH", "wAtch"} {
if LookupCommandInfo(n) == (CommandInfo{}) {
t.Errorf("LookupCommandInfo(%q) = CommandInfo{}, expected non-zero value", n)
}
}
}
func benchmarkLookupCommandInfo(b *testing.B, names ...string) {
for i := 0; i < b.N; i++ {
for _, c := range names {
LookupCommandInfo(c)
}
}
}
func BenchmarkLookupCommandInfoCorrectCase(b *testing.B) {
benchmarkLookupCommandInfo(b, "watch", "WATCH", "monitor", "MONITOR")
}
func BenchmarkLookupCommandInfoMixedCase(b *testing.B) {
benchmarkLookupCommandInfo(b, "wAtch", "WeTCH", "monItor", "MONiTOR")
}

62
pkg/cache/redis/conn.go vendored
View File

@ -30,6 +30,33 @@ import (
"github.com/pkg/errors"
)
// Conn represents a connection to a Redis server.
type Conn interface {
// Close closes the connection.
Close() error
// Err returns a non-nil value if the connection is broken. The returned
// value is either the first non-nil value returned from the underlying
// network connection or a protocol parsing error. Applications should
// close broken connections.
Err() error
// Do sends a command to the server and returns the received reply.
Do(commandName string, args ...interface{}) (reply interface{}, err error)
// Send writes the command to the client's output buffer.
Send(commandName string, args ...interface{}) error
// Flush flushes the output buffer to the Redis server.
Flush() error
// Receive receives a single reply from the Redis server
Receive() (reply interface{}, err error)
// WithContext returns Conn with the input ctx.
WithContext(ctx context.Context) Conn
}
// conn is the low-level implementation of Conn
type conn struct {
// Shared
@ -38,6 +65,8 @@ type conn struct {
err error
conn net.Conn
ctx context.Context
// Read
readTimeout time.Duration
br *bufio.Reader
@ -226,6 +255,7 @@ func NewConn(c *Config) (cn Conn, err error) {
func (c *conn) Close() error {
c.mu.Lock()
c.ctx = nil
err := c.err
if c.err == nil {
c.err = errors.New("redigo: closed")
@ -295,7 +325,7 @@ func (c *conn) writeFloat64(n float64) error {
func (c *conn) writeCommand(cmd string, args []interface{}) (err error) {
if c.writeTimeout != 0 {
c.conn.SetWriteDeadline(time.Now().Add(c.writeTimeout))
c.conn.SetWriteDeadline(shrinkDeadline(c.ctx, c.writeTimeout))
}
c.writeLen('*', 1+len(args))
err = c.writeString(cmd)
@ -478,7 +508,7 @@ func (c *conn) Send(cmd string, args ...interface{}) (err error) {
func (c *conn) Flush() (err error) {
if c.writeTimeout != 0 {
c.conn.SetWriteDeadline(time.Now().Add(c.writeTimeout))
c.conn.SetWriteDeadline(shrinkDeadline(c.ctx, c.writeTimeout))
}
if err = c.bw.Flush(); err != nil {
c.fatal(err)
@ -488,7 +518,7 @@ func (c *conn) Flush() (err error) {
func (c *conn) Receive() (reply interface{}, err error) {
if c.readTimeout != 0 {
c.conn.SetReadDeadline(time.Now().Add(c.readTimeout))
c.conn.SetReadDeadline(shrinkDeadline(c.ctx, c.readTimeout))
}
if reply, err = c.readReply(); err != nil {
return nil, c.fatal(err)
@ -511,7 +541,7 @@ func (c *conn) Receive() (reply interface{}, err error) {
return
}
func (c *conn) Do(cmd string, args ...interface{}) (interface{}, error) {
func (c *conn) Do(cmd string, args ...interface{}) (reply interface{}, err error) {
c.mu.Lock()
pending := c.pending
c.pending = 0
@ -519,7 +549,7 @@ func (c *conn) Do(cmd string, args ...interface{}) (interface{}, error) {
if cmd == "" && pending == 0 {
return nil, nil
}
var err error
if cmd != "" {
err = c.writeCommand(cmd, args)
}
@ -530,7 +560,7 @@ func (c *conn) Do(cmd string, args ...interface{}) (interface{}, error) {
return nil, c.fatal(err)
}
if c.readTimeout != 0 {
c.conn.SetReadDeadline(time.Now().Add(c.readTimeout))
c.conn.SetReadDeadline(shrinkDeadline(c.ctx, c.readTimeout))
}
if cmd == "" {
reply := make([]interface{}, pending)
@ -548,7 +578,6 @@ func (c *conn) Do(cmd string, args ...interface{}) (interface{}, error) {
return reply, nil
}
var reply interface{}
for i := 0; i <= pending; i++ {
var e error
if reply, e = c.readReply(); e != nil {
@ -561,5 +590,20 @@ func (c *conn) Do(cmd string, args ...interface{}) (interface{}, error) {
return reply, err
}
// WithContext FIXME: implement WithContext
func (c *conn) WithContext(ctx context.Context) Conn { return c }
func (c *conn) copy() *conn {
return &conn{
pending: c.pending,
err: c.err,
conn: c.conn,
bw: c.bw,
br: c.br,
readTimeout: c.readTimeout,
writeTimeout: c.writeTimeout,
}
}
func (c *conn) WithContext(ctx context.Context) Conn {
c2 := c.copy()
c2.ctx = ctx
return c2
}

670
pkg/cache/redis/conn_test.go vendored Normal file
View File

@ -0,0 +1,670 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis
import (
"bytes"
"context"
"io"
"math"
"net"
"os"
"reflect"
"strings"
"testing"
"time"
)
type tConn struct {
io.Reader
io.Writer
}
func (*tConn) Close() error { return nil }
func (*tConn) LocalAddr() net.Addr { return nil }
func (*tConn) RemoteAddr() net.Addr { return nil }
func (*tConn) SetDeadline(t time.Time) error { return nil }
func (*tConn) SetReadDeadline(t time.Time) error { return nil }
func (*tConn) SetWriteDeadline(t time.Time) error { return nil }
func dialTestConn(r io.Reader, w io.Writer) DialOption {
return DialNetDial(func(net, addr string) (net.Conn, error) {
return &tConn{Reader: r, Writer: w}, nil
})
}
var writeTests = []struct {
args []interface{}
expected string
}{
{
[]interface{}{"SET", "key", "value"},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$5\r\nvalue\r\n",
},
{
[]interface{}{"SET", "key", "value"},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$5\r\nvalue\r\n",
},
{
[]interface{}{"SET", "key", byte(100)},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$3\r\n100\r\n",
},
{
[]interface{}{"SET", "key", 100},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$3\r\n100\r\n",
},
{
[]interface{}{"SET", "key", int64(math.MinInt64)},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$20\r\n-9223372036854775808\r\n",
},
{
[]interface{}{"SET", "key", float64(1349673917.939762)},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$21\r\n1.349673917939762e+09\r\n",
},
{
[]interface{}{"SET", "key", ""},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$0\r\n\r\n",
},
{
[]interface{}{"SET", "key", nil},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$0\r\n\r\n",
},
{
[]interface{}{"ECHO", true, false},
"*3\r\n$4\r\nECHO\r\n$1\r\n1\r\n$1\r\n0\r\n",
},
}
func TestWrite(t *testing.T) {
for _, tt := range writeTests {
var buf bytes.Buffer
c, _ := Dial("", "", dialTestConn(nil, &buf))
err := c.Send(tt.args[0].(string), tt.args[1:]...)
if err != nil {
t.Errorf("Send(%v) returned error %v", tt.args, err)
continue
}
c.Flush()
actual := buf.String()
if actual != tt.expected {
t.Errorf("Send(%v) = %q, want %q", tt.args, actual, tt.expected)
}
}
}
var errorSentinel = &struct{}{}
var readTests = []struct {
reply string
expected interface{}
}{
{
"+OK\r\n",
"OK",
},
{
"+PONG\r\n",
"PONG",
},
{
"@OK\r\n",
errorSentinel,
},
{
"$6\r\nfoobar\r\n",
[]byte("foobar"),
},
{
"$-1\r\n",
nil,
},
{
":1\r\n",
int64(1),
},
{
":-2\r\n",
int64(-2),
},
{
"*0\r\n",
[]interface{}{},
},
{
"*-1\r\n",
nil,
},
{
"*4\r\n$3\r\nfoo\r\n$3\r\nbar\r\n$5\r\nHello\r\n$5\r\nWorld\r\n",
[]interface{}{[]byte("foo"), []byte("bar"), []byte("Hello"), []byte("World")},
},
{
"*3\r\n$3\r\nfoo\r\n$-1\r\n$3\r\nbar\r\n",
[]interface{}{[]byte("foo"), nil, []byte("bar")},
},
{
// "x" is not a valid length
"$x\r\nfoobar\r\n",
errorSentinel,
},
{
// -2 is not a valid length
"$-2\r\n",
errorSentinel,
},
{
// "x" is not a valid integer
":x\r\n",
errorSentinel,
},
{
// missing \r\n following value
"$6\r\nfoobar",
errorSentinel,
},
{
// short value
"$6\r\nxx",
errorSentinel,
},
{
// long value
"$6\r\nfoobarx\r\n",
errorSentinel,
},
}
func TestRead(t *testing.T) {
for _, tt := range readTests {
c, _ := Dial("", "", dialTestConn(strings.NewReader(tt.reply), nil))
actual, err := c.Receive()
if tt.expected == errorSentinel {
if err == nil {
t.Errorf("Receive(%q) did not return expected error", tt.reply)
}
} else {
if err != nil {
t.Errorf("Receive(%q) returned error %v", tt.reply, err)
continue
}
if !reflect.DeepEqual(actual, tt.expected) {
t.Errorf("Receive(%q) = %v, want %v", tt.reply, actual, tt.expected)
}
}
}
}
var testCommands = []struct {
args []interface{}
expected interface{}
}{
{
[]interface{}{"PING"},
"PONG",
},
{
[]interface{}{"SET", "foo", "bar"},
"OK",
},
{
[]interface{}{"GET", "foo"},
[]byte("bar"),
},
{
[]interface{}{"GET", "nokey"},
nil,
},
{
[]interface{}{"MGET", "nokey", "foo"},
[]interface{}{nil, []byte("bar")},
},
{
[]interface{}{"INCR", "mycounter"},
int64(1),
},
{
[]interface{}{"LPUSH", "mylist", "foo"},
int64(1),
},
{
[]interface{}{"LPUSH", "mylist", "bar"},
int64(2),
},
{
[]interface{}{"LRANGE", "mylist", 0, -1},
[]interface{}{[]byte("bar"), []byte("foo")},
},
{
[]interface{}{"MULTI"},
"OK",
},
{
[]interface{}{"LRANGE", "mylist", 0, -1},
"QUEUED",
},
{
[]interface{}{"PING"},
"QUEUED",
},
{
[]interface{}{"EXEC"},
[]interface{}{
[]interface{}{[]byte("bar"), []byte("foo")},
"PONG",
},
},
}
func TestDoCommands(t *testing.T) {
c, err := DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer c.Close()
for _, cmd := range testCommands {
actual, err := c.Do(cmd.args[0].(string), cmd.args[1:]...)
if err != nil {
t.Errorf("Do(%v) returned error %v", cmd.args, err)
continue
}
if !reflect.DeepEqual(actual, cmd.expected) {
t.Errorf("Do(%v) = %v, want %v", cmd.args, actual, cmd.expected)
}
}
}
func TestPipelineCommands(t *testing.T) {
c, err := DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer c.Close()
for _, cmd := range testCommands {
if err := c.Send(cmd.args[0].(string), cmd.args[1:]...); err != nil {
t.Fatalf("Send(%v) returned error %v", cmd.args, err)
}
}
if err := c.Flush(); err != nil {
t.Errorf("Flush() returned error %v", err)
}
for _, cmd := range testCommands {
actual, err := c.Receive()
if err != nil {
t.Fatalf("Receive(%v) returned error %v", cmd.args, err)
}
if !reflect.DeepEqual(actual, cmd.expected) {
t.Errorf("Receive(%v) = %v, want %v", cmd.args, actual, cmd.expected)
}
}
}
func TestBlankCommmand(t *testing.T) {
c, err := DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer c.Close()
for _, cmd := range testCommands {
if err = c.Send(cmd.args[0].(string), cmd.args[1:]...); err != nil {
t.Fatalf("Send(%v) returned error %v", cmd.args, err)
}
}
reply, err := Values(c.Do(""))
if err != nil {
t.Fatalf("Do() returned error %v", err)
}
if len(reply) != len(testCommands) {
t.Fatalf("len(reply)=%d, want %d", len(reply), len(testCommands))
}
for i, cmd := range testCommands {
actual := reply[i]
if !reflect.DeepEqual(actual, cmd.expected) {
t.Errorf("Receive(%v) = %v, want %v", cmd.args, actual, cmd.expected)
}
}
}
func TestRecvBeforeSend(t *testing.T) {
c, err := DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer c.Close()
done := make(chan struct{})
go func() {
c.Receive()
close(done)
}()
time.Sleep(time.Millisecond)
c.Send("PING")
c.Flush()
<-done
_, err = c.Do("")
if err != nil {
t.Fatalf("error=%v", err)
}
}
func TestError(t *testing.T) {
c, err := DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer c.Close()
c.Do("SET", "key", "val")
_, err = c.Do("HSET", "key", "fld", "val")
if err == nil {
t.Errorf("Expected err for HSET on string key.")
}
if c.Err() != nil {
t.Errorf("Conn has Err()=%v, expect nil", c.Err())
}
_, err = c.Do("SET", "key", "val")
if err != nil {
t.Errorf("Do(SET, key, val) returned error %v, expected nil.", err)
}
}
func TestReadTimeout(t *testing.T) {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("net.Listen returned %v", err)
}
defer l.Close()
go func() {
for {
c, err1 := l.Accept()
if err1 != nil {
return
}
go func() {
time.Sleep(time.Second)
c.Write([]byte("+OK\r\n"))
c.Close()
}()
}
}()
// Do
c1, err := Dial(l.Addr().Network(), l.Addr().String(), DialReadTimeout(time.Millisecond))
if err != nil {
t.Fatalf("Dial returned %v", err)
}
defer c1.Close()
_, err = c1.Do("PING")
if err == nil {
t.Fatalf("c1.Do() returned nil, expect error")
}
if c1.Err() == nil {
t.Fatalf("c1.Err() = nil, expect error")
}
// Send/Flush/Receive
c2, err := Dial(l.Addr().Network(), l.Addr().String(), DialReadTimeout(time.Millisecond))
if err != nil {
t.Fatalf("Dial returned %v", err)
}
defer c2.Close()
c2.Send("PING")
c2.Flush()
_, err = c2.Receive()
if err == nil {
t.Fatalf("c2.Receive() returned nil, expect error")
}
if c2.Err() == nil {
t.Fatalf("c2.Err() = nil, expect error")
}
}
var dialErrors = []struct {
rawurl string
expectedError string
}{
{
"localhost",
"invalid redis URL scheme",
},
// The error message for invalid hosts is diffferent in different
// versions of Go, so just check that there is an error message.
{
"redis://weird url",
"",
},
{
"redis://foo:bar:baz",
"",
},
{
"http://www.google.com",
"invalid redis URL scheme: http",
},
{
"redis://localhost:6379/abc123",
"invalid database: abc123",
},
}
func TestDialURLErrors(t *testing.T) {
for _, d := range dialErrors {
_, err := DialURL(d.rawurl)
if err == nil || !strings.Contains(err.Error(), d.expectedError) {
t.Errorf("DialURL did not return expected error (expected %v to contain %s)", err, d.expectedError)
}
}
}
func TestDialURLPort(t *testing.T) {
checkPort := func(network, address string) (net.Conn, error) {
if address != "localhost:6379" {
t.Errorf("DialURL did not set port to 6379 by default (got %v)", address)
}
return nil, nil
}
_, err := DialURL("redis://localhost", DialNetDial(checkPort))
if err != nil {
t.Error("dial error:", err)
}
}
func TestDialURLHost(t *testing.T) {
checkHost := func(network, address string) (net.Conn, error) {
if address != "localhost:6379" {
t.Errorf("DialURL did not set host to localhost by default (got %v)", address)
}
return nil, nil
}
_, err := DialURL("redis://:6379", DialNetDial(checkHost))
if err != nil {
t.Error("dial error:", err)
}
}
func TestDialURLPassword(t *testing.T) {
var buf bytes.Buffer
_, err := DialURL("redis://x:abc123@localhost", dialTestConn(strings.NewReader("+OK\r\n"), &buf))
if err != nil {
t.Error("dial error:", err)
}
expected := "*2\r\n$4\r\nAUTH\r\n$6\r\nabc123\r\n"
actual := buf.String()
if actual != expected {
t.Errorf("commands = %q, want %q", actual, expected)
}
}
func TestDialURLDatabase(t *testing.T) {
var buf bytes.Buffer
_, err := DialURL("redis://localhost/3", dialTestConn(strings.NewReader("+OK\r\n"), &buf))
if err != nil {
t.Error("dial error:", err)
}
expected := "*2\r\n$6\r\nSELECT\r\n$1\r\n3\r\n"
actual := buf.String()
if actual != expected {
t.Errorf("commands = %q, want %q", actual, expected)
}
}
// Connect to local instance of Redis running on the default port.
func ExampleDial() {
c, err := Dial("tcp", ":6379")
if err != nil {
// handle error
}
defer c.Close()
}
// Connect to remote instance of Redis using a URL.
func ExampleDialURL() {
c, err := DialURL(os.Getenv("REDIS_URL"))
if err != nil {
// handle connection error
}
defer c.Close()
}
// TextExecError tests handling of errors in a transaction. See
// http://io/topics/transactions for information on how Redis handles
// errors in a transaction.
func TestExecError(t *testing.T) {
c, err := DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer c.Close()
// Execute commands that fail before EXEC is called.
c.Do("DEL", "k0")
c.Do("ZADD", "k0", 0, 0)
c.Send("MULTI")
c.Send("NOTACOMMAND", "k0", 0, 0)
c.Send("ZINCRBY", "k0", 0, 0)
v, err := c.Do("EXEC")
if err == nil {
t.Fatalf("EXEC returned values %v, expected error", v)
}
// Execute commands that fail after EXEC is called. The first command
// returns an error.
c.Do("DEL", "k1")
c.Do("ZADD", "k1", 0, 0)
c.Send("MULTI")
c.Send("HSET", "k1", 0, 0)
c.Send("ZINCRBY", "k1", 0, 0)
v, err = c.Do("EXEC")
if err != nil {
t.Fatalf("EXEC returned error %v", err)
}
vs, err := Values(v, nil)
if err != nil {
t.Fatalf("Values(v) returned error %v", err)
}
if len(vs) != 2 {
t.Fatalf("len(vs) == %d, want 2", len(vs))
}
if _, ok := vs[0].(error); !ok {
t.Fatalf("first result is type %T, expected error", vs[0])
}
if _, ok := vs[1].([]byte); !ok {
t.Fatalf("second result is type %T, expected []byte", vs[1])
}
// Execute commands that fail after EXEC is called. The second command
// returns an error.
c.Do("ZADD", "k2", 0, 0)
c.Send("MULTI")
c.Send("ZINCRBY", "k2", 0, 0)
c.Send("HSET", "k2", 0, 0)
v, err = c.Do("EXEC")
if err != nil {
t.Fatalf("EXEC returned error %v", err)
}
vs, err = Values(v, nil)
if err != nil {
t.Fatalf("Values(v) returned error %v", err)
}
if len(vs) != 2 {
t.Fatalf("len(vs) == %d, want 2", len(vs))
}
if _, ok := vs[0].([]byte); !ok {
t.Fatalf("first result is type %T, expected []byte", vs[0])
}
if _, ok := vs[1].(error); !ok {
t.Fatalf("second result is type %T, expected error", vs[2])
}
}
func BenchmarkDoEmpty(b *testing.B) {
c, err := DialDefaultServer()
if err != nil {
b.Fatal(err)
}
defer c.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := c.Do(""); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkDoPing(b *testing.B) {
c, err := DialDefaultServer()
if err != nil {
b.Fatal(err)
}
defer c.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := c.Do("PING"); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkConn(b *testing.B) {
for i := 0; i < b.N; i++ {
c, err := DialDefaultServer()
if err != nil {
b.Fatal(err)
}
c2 := c.WithContext(context.TODO())
if _, err := c2.Do("PING"); err != nil {
b.Fatal(err)
}
c2.Close()
}
}

18
pkg/cache/redis/log.go vendored
View File

@ -16,16 +16,18 @@ package redis
import (
"bytes"
"context"
"fmt"
"log"
)
// NewLoggingConn returns a logging wrapper around a connection.
// ATTENTION: ONLY use loggingConn in developing, DO NOT use this in production.
func NewLoggingConn(conn Conn, logger *log.Logger, prefix string) Conn {
if prefix != "" {
prefix = prefix + "."
}
return &loggingConn{conn, logger, prefix}
return &loggingConn{Conn: conn, logger: logger, prefix: prefix}
}
type loggingConn struct {
@ -98,16 +100,16 @@ func (c *loggingConn) print(method, commandName string, args []interface{}, repl
c.logger.Output(3, buf.String())
}
func (c *loggingConn) Do(commandName string, args ...interface{}) (interface{}, error) {
reply, err := c.Conn.Do(commandName, args...)
func (c *loggingConn) Do(commandName string, args ...interface{}) (reply interface{}, err error) {
reply, err = c.Conn.Do(commandName, args...)
c.print("Do", commandName, args, reply, err)
return reply, err
}
func (c *loggingConn) Send(commandName string, args ...interface{}) error {
err := c.Conn.Send(commandName, args...)
func (c *loggingConn) Send(commandName string, args ...interface{}) (err error) {
err = c.Conn.Send(commandName, args...)
c.print("Send", commandName, args, nil, err)
return err
return
}
func (c *loggingConn) Receive() (interface{}, error) {
@ -115,3 +117,7 @@ func (c *loggingConn) Receive() (interface{}, error) {
c.print("Receive", "", nil, reply, err)
return reply, err
}
func (c *loggingConn) WithContext(ctx context.Context) Conn {
return c
}

67
pkg/cache/redis/main_test.go vendored Normal file
View File

@ -0,0 +1,67 @@
package redis
import (
"flag"
"os"
"testing"
"time"
"github.com/bilibili/kratos/pkg/container/pool"
"github.com/bilibili/kratos/pkg/testing/lich"
xtime "github.com/bilibili/kratos/pkg/time"
)
var (
testRedisAddr string
testPool *Pool
testConfig *Config
)
func setupTestConfig(addr string) {
c := getTestConfig(addr)
c.Config = &pool.Config{
Active: 20,
Idle: 2,
IdleTimeout: xtime.Duration(90 * time.Second),
}
testConfig = c
}
func getTestConfig(addr string) *Config {
return &Config{
Name: "test",
Proto: "tcp",
Addr: addr,
DialTimeout: xtime.Duration(time.Second),
ReadTimeout: xtime.Duration(time.Second),
WriteTimeout: xtime.Duration(time.Second),
}
}
func setupTestPool() {
testPool = NewPool(testConfig)
}
// DialDefaultServer starts the test server if not already started and dials a
// connection to the server.
func DialDefaultServer() (Conn, error) {
c, err := Dial("tcp", testRedisAddr, DialReadTimeout(1*time.Second), DialWriteTimeout(1*time.Second))
if err != nil {
return nil, err
}
c.Do("FLUSHDB")
return c, nil
}
func TestMain(m *testing.M) {
flag.Set("f", "./test/docker-compose.yaml")
if err := lich.Setup(); err != nil {
panic(err)
}
defer lich.Teardown()
testRedisAddr = "localhost:6379"
setupTestConfig(testRedisAddr)
setupTestPool()
ret := m.Run()
os.Exit(ret)
}

4
pkg/cache/redis/metrics.go vendored
View File

@ -1,6 +1,8 @@
package redis
import "github.com/bilibili/kratos/pkg/stat/metric"
import (
"github.com/bilibili/kratos/pkg/stat/metric"
)
const namespace = "redis_client"

4
pkg/cache/redis/mock.go vendored
View File

@ -1,8 +1,6 @@
package redis
import (
"context"
)
import "context"
// MockErr for unit test.
type MockErr struct {

85
pkg/cache/redis/pipeline.go vendored Normal file
View File

@ -0,0 +1,85 @@
package redis
import (
"context"
"errors"
)
type Pipeliner interface {
// Send writes the command to the client's output buffer.
Send(commandName string, args ...interface{})
// Exec executes all commands and get replies.
Exec(ctx context.Context) (rs *Replies, err error)
}
var (
ErrNoReply = errors.New("redis: no reply in result set")
)
type pipeliner struct {
pool *Pool
cmds []*cmd
}
type Replies struct {
replies []*reply
}
type reply struct {
reply interface{}
err error
}
func (rs *Replies) Next() bool {
return len(rs.replies) > 0
}
func (rs *Replies) Scan() (reply interface{}, err error) {
if !rs.Next() {
return nil, ErrNoReply
}
reply, err = rs.replies[0].reply, rs.replies[0].err
rs.replies = rs.replies[1:]
return
}
type cmd struct {
commandName string
args []interface{}
}
func (p *pipeliner) Send(commandName string, args ...interface{}) {
p.cmds = append(p.cmds, &cmd{commandName: commandName, args: args})
return
}
func (p *pipeliner) Exec(ctx context.Context) (rs *Replies, err error) {
n := len(p.cmds)
if n == 0 {
return &Replies{}, nil
}
c := p.pool.Get(ctx)
defer c.Close()
for len(p.cmds) > 0 {
cmd := p.cmds[0]
p.cmds = p.cmds[1:]
if err := c.Send(cmd.commandName, cmd.args...); err != nil {
p.cmds = p.cmds[:0]
return nil, err
}
}
if err = c.Flush(); err != nil {
p.cmds = p.cmds[:0]
return nil, err
}
rps := make([]*reply, 0, n)
for i := 0; i < n; i++ {
rp, err := c.Receive()
rps = append(rps, &reply{reply: rp, err: err})
}
rs = &Replies{
replies: rps,
}
return
}

96
pkg/cache/redis/pipeline_test.go vendored Normal file
View File

@ -0,0 +1,96 @@
package redis
import (
"context"
"fmt"
"reflect"
"testing"
"time"
"github.com/bilibili/kratos/pkg/container/pool"
xtime "github.com/bilibili/kratos/pkg/time"
)
func TestRedis_Pipeline(t *testing.T) {
conf := &Config{
Name: "test",
Proto: "tcp",
Addr: testRedisAddr,
DialTimeout: xtime.Duration(1 * time.Second),
ReadTimeout: xtime.Duration(1 * time.Second),
WriteTimeout: xtime.Duration(1 * time.Second),
}
conf.Config = &pool.Config{
Active: 10,
Idle: 2,
IdleTimeout: xtime.Duration(90 * time.Second),
}
r := NewRedis(conf)
r.Do(context.TODO(), "FLUSHDB")
p := r.Pipeline()
for _, cmd := range testCommands {
p.Send(cmd.args[0].(string), cmd.args[1:]...)
}
replies, err := p.Exec(context.TODO())
i := 0
for replies.Next() {
cmd := testCommands[i]
actual, err := replies.Scan()
if err != nil {
t.Fatalf("Receive(%v) returned error %v", cmd.args, err)
}
if !reflect.DeepEqual(actual, cmd.expected) {
t.Errorf("Receive(%v) = %v, want %v", cmd.args, actual, cmd.expected)
}
i++
}
err = r.Close()
if err != nil {
t.Errorf("Close() error %v", err)
}
}
func ExamplePipeliner() {
r := NewRedis(testConfig)
defer r.Close()
pip := r.Pipeline()
pip.Send("SET", "hello", "world")
pip.Send("GET", "hello")
replies, err := pip.Exec(context.TODO())
if err != nil {
fmt.Printf("%#v\n", err)
}
for replies.Next() {
s, err := String(replies.Scan())
if err != nil {
fmt.Printf("err %#v\n", err)
}
fmt.Printf("%#v\n", s)
}
// Output:
// "OK"
// "world"
}
func BenchmarkRedisPipelineExec(b *testing.B) {
r := NewRedis(testConfig)
defer r.Close()
r.Do(context.TODO(), "SET", "abcde", "fghiasdfasdf")
b.ResetTimer()
for i := 0; i < b.N; i++ {
p := r.Pipeline()
p.Send("GET", "abcde")
_, err := p.Exec(context.TODO())
if err != nil {
b.Fatal(err)
}
}
}

39
pkg/cache/redis/pool.go vendored
View File

@ -45,24 +45,14 @@ type Pool struct {
statfunc func(name, addr, cmd string, t time.Time, err error) func()
}
// Config client settings.
type Config struct {
*pool.Config
Name string // redis name, for trace
Proto string
Addr string
Auth string
DialTimeout xtime.Duration
ReadTimeout xtime.Duration
WriteTimeout xtime.Duration
}
// NewPool creates a new pool.
func NewPool(c *Config, options ...DialOption) (p *Pool) {
if c.DialTimeout <= 0 || c.ReadTimeout <= 0 || c.WriteTimeout <= 0 {
panic("must config redis timeout")
}
if c.SlowLog <= 0 {
c.SlowLog = xtime.Duration(250 * time.Millisecond)
}
ops := []DialOption{
DialConnectTimeout(time.Duration(c.DialTimeout)),
DialReadTimeout(time.Duration(c.ReadTimeout)),
@ -71,12 +61,18 @@ func NewPool(c *Config, options ...DialOption) (p *Pool) {
}
ops = append(ops, options...)
p1 := pool.NewSlice(c.Config)
// new pool
p1.New = func(ctx context.Context) (io.Closer, error) {
conn, err := Dial(c.Proto, c.Addr, ops...)
if err != nil {
return nil, err
}
return &traceConn{Conn: conn, connTags: []trace.Tag{trace.TagString(trace.TagPeerAddress, c.Addr)}}, nil
return &traceConn{
Conn: conn,
connTags: []trace.Tag{trace.TagString(trace.TagPeerAddress, c.Addr)},
slowLogThreshold: time.Duration(c.SlowLog),
}, nil
}
p = &Pool{Slice: p1, c: c, statfunc: pstat}
return
@ -93,7 +89,7 @@ func (p *Pool) Get(ctx context.Context) Conn {
return errorConnection{err}
}
c1, _ := c.(Conn)
return &pooledConnection{p: p, c: c1.WithContext(ctx), ctx: ctx, now: beginTime}
return &pooledConnection{p: p, c: c1.WithContext(ctx), rc: c1, now: beginTime}
}
// Close releases the resources used by the pool.
@ -103,12 +99,12 @@ func (p *Pool) Close() error {
type pooledConnection struct {
p *Pool
rc Conn
c Conn
state int
now time.Time
cmds []string
ctx context.Context
}
var (
@ -180,7 +176,7 @@ func (pc *pooledConnection) Close() error {
}
}
_, err := c.Do("")
pc.p.Slice.Put(context.Background(), c, pc.state != 0 || c.Err() != nil)
pc.p.Slice.Put(context.Background(), pc.rc, pc.state != 0 || c.Err() != nil)
return err
}
@ -193,7 +189,9 @@ func (pc *pooledConnection) Do(commandName string, args ...interface{}) (reply i
ci := LookupCommandInfo(commandName)
pc.state = (pc.state | ci.Set) &^ ci.Clear
reply, err = pc.c.Do(commandName, args...)
pc.p.statfunc(pc.p.c.Name, pc.p.c.Addr, commandName, now, err)()
if pc.p.statfunc != nil {
pc.p.statfunc(pc.p.c.Name, pc.p.c.Addr, commandName, now, err)()
}
return
}
@ -217,13 +215,14 @@ func (pc *pooledConnection) Receive() (reply interface{}, err error) {
if len(pc.cmds) > 0 {
cmd := pc.cmds[0]
pc.cmds = pc.cmds[1:]
pc.p.statfunc(pc.p.c.Name, pc.p.c.Addr, cmd, pc.now, err)()
if pc.p.statfunc != nil {
pc.p.statfunc(pc.p.c.Name, pc.p.c.Addr, cmd, pc.now, err)()
}
}
return
}
func (pc *pooledConnection) WithContext(ctx context.Context) Conn {
pc.ctx = ctx
return pc
}

540
pkg/cache/redis/pool_test.go vendored Normal file
View File

@ -0,0 +1,540 @@
// Copyright 2011 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis
import (
"context"
"errors"
"io"
"reflect"
"sync"
"testing"
"time"
"github.com/bilibili/kratos/pkg/container/pool"
)
type poolTestConn struct {
d *poolDialer
err error
c Conn
ctx context.Context
}
func (c *poolTestConn) Flush() error {
return c.c.Flush()
}
func (c *poolTestConn) Receive() (reply interface{}, err error) {
return c.c.Receive()
}
func (c *poolTestConn) WithContext(ctx context.Context) Conn {
c.c.WithContext(ctx)
c.ctx = ctx
return c
}
func (c *poolTestConn) Close() error {
c.d.mu.Lock()
c.d.open--
c.d.mu.Unlock()
return c.c.Close()
}
func (c *poolTestConn) Err() error { return c.err }
func (c *poolTestConn) Do(commandName string, args ...interface{}) (reply interface{}, err error) {
if commandName == "ERR" {
c.err = args[0].(error)
commandName = "PING"
}
if commandName != "" {
c.d.commands = append(c.d.commands, commandName)
}
return c.c.Do(commandName, args...)
}
func (c *poolTestConn) Send(commandName string, args ...interface{}) error {
c.d.commands = append(c.d.commands, commandName)
return c.c.Send(commandName, args...)
}
type poolDialer struct {
mu sync.Mutex
t *testing.T
dialed int
open int
commands []string
dialErr error
}
func (d *poolDialer) dial() (Conn, error) {
d.mu.Lock()
d.dialed += 1
dialErr := d.dialErr
d.mu.Unlock()
if dialErr != nil {
return nil, d.dialErr
}
c, err := DialDefaultServer()
if err != nil {
return nil, err
}
d.mu.Lock()
d.open += 1
d.mu.Unlock()
return &poolTestConn{d: d, c: c}, nil
}
func (d *poolDialer) check(message string, p *Pool, dialed, open int) {
d.mu.Lock()
if d.dialed != dialed {
d.t.Errorf("%s: dialed=%d, want %d", message, d.dialed, dialed)
}
if d.open != open {
d.t.Errorf("%s: open=%d, want %d", message, d.open, open)
}
// if active := p.ActiveCount(); active != open {
// d.t.Errorf("%s: active=%d, want %d", message, active, open)
// }
d.mu.Unlock()
}
func TestPoolReuse(t *testing.T) {
d := poolDialer{t: t}
p := NewPool(testConfig)
p.Slice.New = func(ctx context.Context) (io.Closer, error) {
return d.dial()
}
var err error
for i := 0; i < 10; i++ {
c1 := p.Get(context.TODO())
c1.Do("PING")
c2 := p.Get(context.TODO())
c2.Do("PING")
c1.Close()
c2.Close()
}
d.check("before close", p, 2, 2)
err = p.Close()
if err != nil {
t.Fatal(err)
}
d.check("after close", p, 2, 0)
}
func TestPoolMaxIdle(t *testing.T) {
d := poolDialer{t: t}
p := NewPool(testConfig)
p.Slice.New = func(ctx context.Context) (io.Closer, error) {
return d.dial()
}
defer p.Close()
for i := 0; i < 10; i++ {
c1 := p.Get(context.TODO())
c1.Do("PING")
c2 := p.Get(context.TODO())
c2.Do("PING")
c3 := p.Get(context.TODO())
c3.Do("PING")
c1.Close()
c2.Close()
c3.Close()
}
d.check("before close", p, 12, 2)
p.Close()
d.check("after close", p, 12, 0)
}
func TestPoolError(t *testing.T) {
d := poolDialer{t: t}
p := NewPool(testConfig)
p.Slice.New = func(ctx context.Context) (io.Closer, error) {
return d.dial()
}
defer p.Close()
c := p.Get(context.TODO())
c.Do("ERR", io.EOF)
if c.Err() == nil {
t.Errorf("expected c.Err() != nil")
}
c.Close()
c = p.Get(context.TODO())
c.Do("ERR", io.EOF)
c.Close()
d.check(".", p, 2, 0)
}
func TestPoolClose(t *testing.T) {
d := poolDialer{t: t}
p := NewPool(testConfig)
p.Slice.New = func(ctx context.Context) (io.Closer, error) {
return d.dial()
}
defer p.Close()
c1 := p.Get(context.TODO())
c1.Do("PING")
c2 := p.Get(context.TODO())
c2.Do("PING")
c3 := p.Get(context.TODO())
c3.Do("PING")
c1.Close()
if _, err := c1.Do("PING"); err == nil {
t.Errorf("expected error after connection closed")
}
c2.Close()
c2.Close()
p.Close()
d.check("after pool close", p, 3, 1)
if _, err := c1.Do("PING"); err == nil {
t.Errorf("expected error after connection and pool closed")
}
c3.Close()
d.check("after conn close", p, 3, 0)
c1 = p.Get(context.TODO())
if _, err := c1.Do("PING"); err == nil {
t.Errorf("expected error after pool closed")
}
}
func TestPoolConcurrenSendReceive(t *testing.T) {
p := NewPool(testConfig)
p.Slice.New = func(ctx context.Context) (io.Closer, error) {
return DialDefaultServer()
}
defer p.Close()
c := p.Get(context.TODO())
done := make(chan error, 1)
go func() {
_, err := c.Receive()
done <- err
}()
c.Send("PING")
c.Flush()
err := <-done
if err != nil {
t.Fatalf("Receive() returned error %v", err)
}
_, err = c.Do("")
if err != nil {
t.Fatalf("Do() returned error %v", err)
}
c.Close()
}
func TestPoolMaxActive(t *testing.T) {
d := poolDialer{t: t}
conf := getTestConfig(testRedisAddr)
conf.Config = &pool.Config{
Active: 2,
Idle: 2,
}
p := NewPool(conf)
p.Slice.New = func(ctx context.Context) (io.Closer, error) {
return d.dial()
}
defer p.Close()
c1 := p.Get(context.TODO())
c1.Do("PING")
c2 := p.Get(context.TODO())
c2.Do("PING")
d.check("1", p, 2, 2)
c3 := p.Get(context.TODO())
if _, err := c3.Do("PING"); err != pool.ErrPoolExhausted {
t.Errorf("expected pool exhausted")
}
c3.Close()
d.check("2", p, 2, 2)
c2.Close()
d.check("3", p, 2, 2)
c3 = p.Get(context.TODO())
if _, err := c3.Do("PING"); err != nil {
t.Errorf("expected good channel, err=%v", err)
}
c3.Close()
d.check("4", p, 2, 2)
}
func TestPoolMonitorCleanup(t *testing.T) {
d := poolDialer{t: t}
p := NewPool(testConfig)
p.Slice.New = func(ctx context.Context) (io.Closer, error) {
return d.dial()
}
defer p.Close()
c := p.Get(context.TODO())
c.Send("MONITOR")
c.Close()
d.check("", p, 1, 0)
}
func TestPoolPubSubCleanup(t *testing.T) {
d := poolDialer{t: t}
p := NewPool(testConfig)
p.Slice.New = func(ctx context.Context) (io.Closer, error) {
return d.dial()
}
defer p.Close()
c := p.Get(context.TODO())
c.Send("SUBSCRIBE", "x")
c.Close()
want := []string{"SUBSCRIBE", "UNSUBSCRIBE", "PUNSUBSCRIBE", "ECHO"}
if !reflect.DeepEqual(d.commands, want) {
t.Errorf("got commands %v, want %v", d.commands, want)
}
d.commands = nil
c = p.Get(context.TODO())
c.Send("PSUBSCRIBE", "x*")
c.Close()
want = []string{"PSUBSCRIBE", "UNSUBSCRIBE", "PUNSUBSCRIBE", "ECHO"}
if !reflect.DeepEqual(d.commands, want) {
t.Errorf("got commands %v, want %v", d.commands, want)
}
d.commands = nil
}
func TestPoolTransactionCleanup(t *testing.T) {
d := poolDialer{t: t}
p := NewPool(testConfig)
p.Slice.New = func(ctx context.Context) (io.Closer, error) {
return d.dial()
}
defer p.Close()
c := p.Get(context.TODO())
c.Do("WATCH", "key")
c.Do("PING")
c.Close()
want := []string{"WATCH", "PING", "UNWATCH"}
if !reflect.DeepEqual(d.commands, want) {
t.Errorf("got commands %v, want %v", d.commands, want)
}
d.commands = nil
c = p.Get(context.TODO())
c.Do("WATCH", "key")
c.Do("UNWATCH")
c.Do("PING")
c.Close()
want = []string{"WATCH", "UNWATCH", "PING"}
if !reflect.DeepEqual(d.commands, want) {
t.Errorf("got commands %v, want %v", d.commands, want)
}
d.commands = nil
c = p.Get(context.TODO())
c.Do("WATCH", "key")
c.Do("MULTI")
c.Do("PING")
c.Close()
want = []string{"WATCH", "MULTI", "PING", "DISCARD"}
if !reflect.DeepEqual(d.commands, want) {
t.Errorf("got commands %v, want %v", d.commands, want)
}
d.commands = nil
c = p.Get(context.TODO())
c.Do("WATCH", "key")
c.Do("MULTI")
c.Do("DISCARD")
c.Do("PING")
c.Close()
want = []string{"WATCH", "MULTI", "DISCARD", "PING"}
if !reflect.DeepEqual(d.commands, want) {
t.Errorf("got commands %v, want %v", d.commands, want)
}
d.commands = nil
c = p.Get(context.TODO())
c.Do("WATCH", "key")
c.Do("MULTI")
c.Do("EXEC")
c.Do("PING")
c.Close()
want = []string{"WATCH", "MULTI", "EXEC", "PING"}
if !reflect.DeepEqual(d.commands, want) {
t.Errorf("got commands %v, want %v", d.commands, want)
}
d.commands = nil
}
func startGoroutines(p *Pool, cmd string, args ...interface{}) chan error {
errs := make(chan error, 10)
for i := 0; i < cap(errs); i++ {
go func() {
c := p.Get(context.TODO())
_, err := c.Do(cmd, args...)
errs <- err
c.Close()
}()
}
// Wait for goroutines to block.
time.Sleep(time.Second / 4)
return errs
}
func TestWaitPoolDialError(t *testing.T) {
testErr := errors.New("test")
d := poolDialer{t: t}
config1 := testConfig
config1.Config = &pool.Config{
Active: 1,
Idle: 1,
Wait: true,
}
p := NewPool(config1)
p.Slice.New = func(ctx context.Context) (io.Closer, error) {
return d.dial()
}
defer p.Close()
c := p.Get(context.TODO())
errs := startGoroutines(p, "ERR", testErr)
d.check("before close", p, 1, 1)
d.dialErr = errors.New("dial")
c.Close()
nilCount := 0
errCount := 0
timeout := time.After(2 * time.Second)
for i := 0; i < cap(errs); i++ {
select {
case err := <-errs:
switch err {
case nil:
nilCount++
case d.dialErr:
errCount++
default:
t.Fatalf("expected dial error or nil, got %v", err)
}
case <-timeout:
t.Logf("Wait all the time and timeout %d", i)
return
}
}
if nilCount != 1 {
t.Errorf("expected one nil error, got %d", nilCount)
}
if errCount != cap(errs)-1 {
t.Errorf("expected %d dial erors, got %d", cap(errs)-1, errCount)
}
d.check("done", p, cap(errs), 0)
}
func BenchmarkPoolGet(b *testing.B) {
b.StopTimer()
p := NewPool(testConfig)
c := p.Get(context.Background())
if err := c.Err(); err != nil {
b.Fatal(err)
}
c.Close()
defer p.Close()
b.StartTimer()
for i := 0; i < b.N; i++ {
c := p.Get(context.Background())
c.Close()
}
}
func BenchmarkPoolGetErr(b *testing.B) {
b.StopTimer()
p := NewPool(testConfig)
c := p.Get(context.Background())
if err := c.Err(); err != nil {
b.Fatal(err)
}
c.Close()
defer p.Close()
b.StartTimer()
for i := 0; i < b.N; i++ {
c = p.Get(context.Background())
if err := c.Err(); err != nil {
b.Fatal(err)
}
c.Close()
}
}
func BenchmarkPoolGetPing(b *testing.B) {
b.StopTimer()
p := NewPool(testConfig)
c := p.Get(context.Background())
if err := c.Err(); err != nil {
b.Fatal(err)
}
c.Close()
defer p.Close()
b.StartTimer()
for i := 0; i < b.N; i++ {
c := p.Get(context.Background())
if _, err := c.Do("PING"); err != nil {
b.Fatal(err)
}
c.Close()
}
}
func BenchmarkPooledConn(b *testing.B) {
p := NewPool(testConfig)
defer p.Close()
for i := 0; i < b.N; i++ {
ctx := context.TODO()
c := p.Get(ctx)
c2 := c.WithContext(context.TODO())
if _, err := c2.Do("PING"); err != nil {
b.Fatal(err)
}
c2.Close()
}
}

146
pkg/cache/redis/pubsub_test.go vendored Normal file
View File

@ -0,0 +1,146 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis
import (
"fmt"
"reflect"
"sync"
"testing"
)
func publish(channel, value interface{}) {
c, err := dial()
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
c.Do("PUBLISH", channel, value)
}
// Applications can receive pushed messages from one goroutine and manage subscriptions from another goroutine.
func ExamplePubSubConn() {
c, err := dial()
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
var wg sync.WaitGroup
wg.Add(2)
psc := PubSubConn{Conn: c}
// This goroutine receives and prints pushed notifications from the server.
// The goroutine exits when the connection is unsubscribed from all
// channels or there is an error.
go func() {
defer wg.Done()
for {
switch n := psc.Receive().(type) {
case Message:
fmt.Printf("Message: %s %s\n", n.Channel, n.Data)
case PMessage:
fmt.Printf("PMessage: %s %s %s\n", n.Pattern, n.Channel, n.Data)
case Subscription:
fmt.Printf("Subscription: %s %s %d\n", n.Kind, n.Channel, n.Count)
if n.Count == 0 {
return
}
case error:
fmt.Printf("error: %v\n", n)
return
}
}
}()
// This goroutine manages subscriptions for the connection.
go func() {
defer wg.Done()
psc.Subscribe("example")
psc.PSubscribe("p*")
// The following function calls publish a message using another
// connection to the Redis server.
publish("example", "hello")
publish("example", "world")
publish("pexample", "foo")
publish("pexample", "bar")
// Unsubscribe from all connections. This will cause the receiving
// goroutine to exit.
psc.Unsubscribe()
psc.PUnsubscribe()
}()
wg.Wait()
// Output:
// Subscription: subscribe example 1
// Subscription: psubscribe p* 2
// Message: example hello
// Message: example world
// PMessage: p* pexample foo
// PMessage: p* pexample bar
// Subscription: unsubscribe example 1
// Subscription: punsubscribe p* 0
}
func expectPushed(t *testing.T, c PubSubConn, message string, expected interface{}) {
actual := c.Receive()
if !reflect.DeepEqual(actual, expected) {
t.Errorf("%s = %v, want %v", message, actual, expected)
}
}
func TestPushed(t *testing.T) {
pc, err := DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer pc.Close()
sc, err := DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer sc.Close()
c := PubSubConn{Conn: sc}
c.Subscribe("c1")
expectPushed(t, c, "Subscribe(c1)", Subscription{Kind: "subscribe", Channel: "c1", Count: 1})
c.Subscribe("c2")
expectPushed(t, c, "Subscribe(c2)", Subscription{Kind: "subscribe", Channel: "c2", Count: 2})
c.PSubscribe("p1")
expectPushed(t, c, "PSubscribe(p1)", Subscription{Kind: "psubscribe", Channel: "p1", Count: 3})
c.PSubscribe("p2")
expectPushed(t, c, "PSubscribe(p2)", Subscription{Kind: "psubscribe", Channel: "p2", Count: 4})
c.PUnsubscribe()
expectPushed(t, c, "Punsubscribe(p1)", Subscription{Kind: "punsubscribe", Channel: "p1", Count: 3})
expectPushed(t, c, "Punsubscribe()", Subscription{Kind: "punsubscribe", Channel: "p2", Count: 2})
pc.Do("PUBLISH", "c1", "hello")
expectPushed(t, c, "PUBLISH c1 hello", Message{Channel: "c1", Data: []byte("hello")})
c.Ping("hello")
expectPushed(t, c, `Ping("hello")`, Pong{"hello"})
c.Conn.Send("PING")
c.Conn.Flush()
expectPushed(t, c, `Send("PING")`, Pong{})
}

75
pkg/cache/redis/redis.go vendored
View File

@ -16,6 +16,9 @@ package redis
import (
"context"
"github.com/bilibili/kratos/pkg/container/pool"
xtime "github.com/bilibili/kratos/pkg/time"
)
// Error represents an error returned in a command reply.
@ -23,29 +26,53 @@ type Error string
func (err Error) Error() string { return string(err) }
// Conn represents a connection to a Redis server.
type Conn interface {
// Close closes the connection.
Close() error
// Config client settings.
type Config struct {
*pool.Config
// Err returns a non-nil value if the connection is broken. The returned
// value is either the first non-nil value returned from the underlying
// network connection or a protocol parsing error. Applications should
// close broken connections.
Err() error
// Do sends a command to the server and returns the received reply.
Do(commandName string, args ...interface{}) (reply interface{}, err error)
// Send writes the command to the client's output buffer.
Send(commandName string, args ...interface{}) error
// Flush flushes the output buffer to the Redis server.
Flush() error
// Receive receives a single reply from the Redis server
Receive() (reply interface{}, err error)
// WithContext
WithContext(ctx context.Context) Conn
Name string // redis name, for trace
Proto string
Addr string
Auth string
DialTimeout xtime.Duration
ReadTimeout xtime.Duration
WriteTimeout xtime.Duration
SlowLog xtime.Duration
}
type Redis struct {
pool *Pool
conf *Config
}
func NewRedis(c *Config, options ...DialOption) *Redis {
return &Redis{
pool: NewPool(c, options...),
conf: c,
}
}
// Do gets a new conn from pool, then execute Do with this conn, finally close this conn.
// ATTENTION: Don't use this method with transaction command like MULTI etc. Because every Do will close conn automatically, use r.Conn to get a raw conn for this situation.
func (r *Redis) Do(ctx context.Context, commandName string, args ...interface{}) (reply interface{}, err error) {
conn := r.pool.Get(ctx)
defer conn.Close()
reply, err = conn.Do(commandName, args...)
return
}
// Close closes connection pool
func (r *Redis) Close() error {
return r.pool.Close()
}
// Conn direct gets a connection
func (r *Redis) Conn(ctx context.Context) Conn {
return r.pool.Get(ctx)
}
func (r *Redis) Pipeline() (p Pipeliner) {
return &pipeliner{
pool: r.pool,
}
}

324
pkg/cache/redis/redis_test.go vendored Normal file
View File

@ -0,0 +1,324 @@
package redis
import (
"context"
"reflect"
"testing"
"time"
"github.com/bilibili/kratos/pkg/container/pool"
xtime "github.com/bilibili/kratos/pkg/time"
)
func TestRedis(t *testing.T) {
testSet(t, testPool)
testSend(t, testPool)
testGet(t, testPool)
testErr(t, testPool)
if err := testPool.Close(); err != nil {
t.Errorf("redis: close error(%v)", err)
}
conn, err := NewConn(testConfig)
if err != nil {
t.Errorf("redis: new conn error(%v)", err)
}
if err := conn.Close(); err != nil {
t.Errorf("redis: close error(%v)", err)
}
}
func testSet(t *testing.T, p *Pool) {
var (
key = "test"
value = "test"
conn = p.Get(context.TODO())
)
defer conn.Close()
if reply, err := conn.Do("set", key, value); err != nil {
t.Errorf("redis: conn.Do(SET, %s, %s) error(%v)", key, value, err)
} else {
t.Logf("redis: set status: %s", reply)
}
}
func testSend(t *testing.T, p *Pool) {
var (
key = "test"
value = "test"
expire = 1000
conn = p.Get(context.TODO())
)
defer conn.Close()
if err := conn.Send("SET", key, value); err != nil {
t.Errorf("redis: conn.Send(SET, %s, %s) error(%v)", key, value, err)
}
if err := conn.Send("EXPIRE", key, expire); err != nil {
t.Errorf("redis: conn.Send(EXPIRE key(%s) expire(%d)) error(%v)", key, expire, err)
}
if err := conn.Flush(); err != nil {
t.Errorf("redis: conn.Flush error(%v)", err)
}
for i := 0; i < 2; i++ {
if _, err := conn.Receive(); err != nil {
t.Errorf("redis: conn.Receive error(%v)", err)
return
}
}
t.Logf("redis: set value: %s", value)
}
func testGet(t *testing.T, p *Pool) {
var (
key = "test"
conn = p.Get(context.TODO())
)
defer conn.Close()
if reply, err := conn.Do("GET", key); err != nil {
t.Errorf("redis: conn.Do(GET, %s) error(%v)", key, err)
} else {
t.Logf("redis: get value: %s", reply)
}
}
func testErr(t *testing.T, p *Pool) {
conn := p.Get(context.TODO())
if err := conn.Close(); err != nil {
t.Errorf("redis: close error(%v)", err)
}
if err := conn.Err(); err == nil {
t.Errorf("redis: err not nil")
} else {
t.Logf("redis: err: %v", err)
}
}
func BenchmarkRedis(b *testing.B) {
conf := &Config{
Name: "test",
Proto: "tcp",
Addr: testRedisAddr,
DialTimeout: xtime.Duration(time.Second),
ReadTimeout: xtime.Duration(time.Second),
WriteTimeout: xtime.Duration(time.Second),
}
conf.Config = &pool.Config{
Active: 10,
Idle: 5,
IdleTimeout: xtime.Duration(90 * time.Second),
}
benchmarkPool := NewPool(conf)
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
conn := benchmarkPool.Get(context.TODO())
if err := conn.Close(); err != nil {
b.Errorf("redis: close error(%v)", err)
}
}
})
if err := benchmarkPool.Close(); err != nil {
b.Errorf("redis: close error(%v)", err)
}
}
var testRedisCommands = []struct {
args []interface{}
expected interface{}
}{
{
[]interface{}{"PING"},
"PONG",
},
{
[]interface{}{"SET", "foo", "bar"},
"OK",
},
{
[]interface{}{"GET", "foo"},
[]byte("bar"),
},
{
[]interface{}{"GET", "nokey"},
nil,
},
{
[]interface{}{"MGET", "nokey", "foo"},
[]interface{}{nil, []byte("bar")},
},
{
[]interface{}{"INCR", "mycounter"},
int64(1),
},
{
[]interface{}{"LPUSH", "mylist", "foo"},
int64(1),
},
{
[]interface{}{"LPUSH", "mylist", "bar"},
int64(2),
},
{
[]interface{}{"LRANGE", "mylist", 0, -1},
[]interface{}{[]byte("bar"), []byte("foo")},
},
}
func TestNewRedis(t *testing.T) {
type args struct {
c *Config
options []DialOption
}
tests := []struct {
name string
args args
wantErr bool
}{
{
"new_redis",
args{
testConfig,
make([]DialOption, 0),
},
false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := NewRedis(tt.args.c, tt.args.options...)
if r == nil {
t.Errorf("NewRedis() error, got nil")
return
}
err := r.Close()
if err != nil {
t.Errorf("Close() error %v", err)
}
})
}
}
func TestRedis_Do(t *testing.T) {
r := NewRedis(testConfig)
r.Do(context.TODO(), "FLUSHDB")
for _, cmd := range testRedisCommands {
actual, err := r.Do(context.TODO(), cmd.args[0].(string), cmd.args[1:]...)
if err != nil {
t.Errorf("Do(%v) returned error %v", cmd.args, err)
continue
}
if !reflect.DeepEqual(actual, cmd.expected) {
t.Errorf("Do(%v) = %v, want %v", cmd.args, actual, cmd.expected)
}
}
err := r.Close()
if err != nil {
t.Errorf("Close() error %v", err)
}
}
func TestRedis_Conn(t *testing.T) {
type args struct {
ctx context.Context
}
tests := []struct {
name string
p *Redis
args args
wantErr bool
g int
c int
}{
{
"Close",
NewRedis(&Config{
Config: &pool.Config{
Active: 1,
Idle: 1,
},
Name: "test_get",
Proto: "tcp",
Addr: testRedisAddr,
DialTimeout: xtime.Duration(time.Second),
ReadTimeout: xtime.Duration(time.Second),
WriteTimeout: xtime.Duration(time.Second),
}),
args{context.TODO()},
false,
3,
3,
},
{
"CloseExceededPoolSize",
NewRedis(&Config{
Config: &pool.Config{
Active: 1,
Idle: 1,
},
Name: "test_get_out",
Proto: "tcp",
Addr: testRedisAddr,
DialTimeout: xtime.Duration(time.Second),
ReadTimeout: xtime.Duration(time.Second),
WriteTimeout: xtime.Duration(time.Second),
}),
args{context.TODO()},
true,
5,
3,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
for i := 1; i <= tt.g; i++ {
got := tt.p.Conn(tt.args.ctx)
if err := got.Close(); err != nil {
if !tt.wantErr {
t.Error(err)
}
}
if i <= tt.c {
if err := got.Close(); err != nil {
t.Error(err)
}
}
}
})
}
}
func BenchmarkRedisDoPing(b *testing.B) {
r := NewRedis(testConfig)
defer r.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := r.Do(context.Background(), "PING"); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkRedisDoSET(b *testing.B) {
r := NewRedis(testConfig)
defer r.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := r.Do(context.Background(), "SET", "a", "b"); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkRedisDoGET(b *testing.B) {
r := NewRedis(testConfig)
defer r.Close()
r.Do(context.Background(), "SET", "a", "b")
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := r.Do(context.Background(), "GET", "b"); err != nil {
b.Fatal(err)
}
}
}

179
pkg/cache/redis/reply_test.go vendored Normal file
View File

@ -0,0 +1,179 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis
import (
"fmt"
"reflect"
"testing"
"github.com/pkg/errors"
)
type valueError struct {
v interface{}
err error
}
func ve(v interface{}, err error) valueError {
return valueError{v, err}
}
var replyTests = []struct {
name interface{}
actual valueError
expected valueError
}{
{
"ints([v1, v2])",
ve(Ints([]interface{}{[]byte("4"), []byte("5")}, nil)),
ve([]int{4, 5}, nil),
},
{
"ints(nil)",
ve(Ints(nil, nil)),
ve([]int(nil), ErrNil),
},
{
"strings([v1, v2])",
ve(Strings([]interface{}{[]byte("v1"), []byte("v2")}, nil)),
ve([]string{"v1", "v2"}, nil),
},
{
"strings(nil)",
ve(Strings(nil, nil)),
ve([]string(nil), ErrNil),
},
{
"byteslices([v1, v2])",
ve(ByteSlices([]interface{}{[]byte("v1"), []byte("v2")}, nil)),
ve([][]byte{[]byte("v1"), []byte("v2")}, nil),
},
{
"byteslices(nil)",
ve(ByteSlices(nil, nil)),
ve([][]byte(nil), ErrNil),
},
{
"values([v1, v2])",
ve(Values([]interface{}{[]byte("v1"), []byte("v2")}, nil)),
ve([]interface{}{[]byte("v1"), []byte("v2")}, nil),
},
{
"values(nil)",
ve(Values(nil, nil)),
ve([]interface{}(nil), ErrNil),
},
{
"float64(1.0)",
ve(Float64([]byte("1.0"), nil)),
ve(float64(1.0), nil),
},
{
"float64(nil)",
ve(Float64(nil, nil)),
ve(float64(0.0), ErrNil),
},
{
"uint64(1)",
ve(Uint64(int64(1), nil)),
ve(uint64(1), nil),
},
{
"uint64(-1)",
ve(Uint64(int64(-1), nil)),
ve(uint64(0), errNegativeInt),
},
}
func TestReply(t *testing.T) {
for _, rt := range replyTests {
if errors.Cause(rt.actual.err) != rt.expected.err {
t.Errorf("%s returned err %v, want %v", rt.name, rt.actual.err, rt.expected.err)
continue
}
if !reflect.DeepEqual(rt.actual.v, rt.expected.v) {
t.Errorf("%s=%+v, want %+v", rt.name, rt.actual.v, rt.expected.v)
}
}
}
// dial wraps DialDefaultServer() with a more suitable function name for examples.
func dial() (Conn, error) {
return DialDefaultServer()
}
func ExampleBool() {
c, err := dial()
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
c.Do("SET", "foo", 1)
exists, _ := Bool(c.Do("EXISTS", "foo"))
fmt.Printf("%#v\n", exists)
// Output:
// true
}
func ExampleInt() {
c, err := dial()
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
c.Do("SET", "k1", 1)
n, _ := Int(c.Do("GET", "k1"))
fmt.Printf("%#v\n", n)
n, _ = Int(c.Do("INCR", "k1"))
fmt.Printf("%#v\n", n)
// Output:
// 1
// 2
}
func ExampleInts() {
c, err := dial()
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
c.Do("SADD", "set_with_integers", 4, 5, 6)
ints, _ := Ints(c.Do("SMEMBERS", "set_with_integers"))
fmt.Printf("%#v\n", ints)
// Output:
// []int{4, 5, 6}
}
func ExampleString() {
c, err := dial()
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
c.Do("SET", "hello", "world")
s, _ := String(c.Do("GET", "hello"))
fmt.Printf("%#v", s)
// Output:
// "world"
}

438
pkg/cache/redis/scan_test.go vendored Normal file
View File

@ -0,0 +1,438 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis
import (
"fmt"
"math"
"reflect"
"testing"
)
var scanConversionTests = []struct {
src interface{}
dest interface{}
}{
{[]byte("-inf"), math.Inf(-1)},
{[]byte("+inf"), math.Inf(1)},
{[]byte("0"), float64(0)},
{[]byte("3.14159"), float64(3.14159)},
{[]byte("3.14"), float32(3.14)},
{[]byte("-100"), int(-100)},
{[]byte("101"), int(101)},
{int64(102), int(102)},
{[]byte("103"), uint(103)},
{int64(104), uint(104)},
{[]byte("105"), int8(105)},
{int64(106), int8(106)},
{[]byte("107"), uint8(107)},
{int64(108), uint8(108)},
{[]byte("0"), false},
{int64(0), false},
{[]byte("f"), false},
{[]byte("1"), true},
{int64(1), true},
{[]byte("t"), true},
{"hello", "hello"},
{[]byte("hello"), "hello"},
{[]byte("world"), []byte("world")},
{[]interface{}{[]byte("foo")}, []interface{}{[]byte("foo")}},
{[]interface{}{[]byte("foo")}, []string{"foo"}},
{[]interface{}{[]byte("hello"), []byte("world")}, []string{"hello", "world"}},
{[]interface{}{[]byte("bar")}, [][]byte{[]byte("bar")}},
{[]interface{}{[]byte("1")}, []int{1}},
{[]interface{}{[]byte("1"), []byte("2")}, []int{1, 2}},
{[]interface{}{[]byte("1"), []byte("2")}, []float64{1, 2}},
{[]interface{}{[]byte("1")}, []byte{1}},
{[]interface{}{[]byte("1")}, []bool{true}},
}
func TestScanConversion(t *testing.T) {
for _, tt := range scanConversionTests {
values := []interface{}{tt.src}
dest := reflect.New(reflect.TypeOf(tt.dest))
values, err := Scan(values, dest.Interface())
if err != nil {
t.Errorf("Scan(%v) returned error %v", tt, err)
continue
}
if !reflect.DeepEqual(tt.dest, dest.Elem().Interface()) {
t.Errorf("Scan(%v) returned %v values: %v, want %v", tt, dest.Elem().Interface(), values, tt.dest)
}
}
}
var scanConversionErrorTests = []struct {
src interface{}
dest interface{}
}{
{[]byte("1234"), byte(0)},
{int64(1234), byte(0)},
{[]byte("-1"), byte(0)},
{int64(-1), byte(0)},
{[]byte("junk"), false},
{Error("blah"), false},
}
func TestScanConversionError(t *testing.T) {
for _, tt := range scanConversionErrorTests {
values := []interface{}{tt.src}
dest := reflect.New(reflect.TypeOf(tt.dest))
values, err := Scan(values, dest.Interface())
if err == nil {
t.Errorf("Scan(%v) did not return error values: %v", tt, values)
}
}
}
func ExampleScan() {
c, err := dial()
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
c.Send("HMSET", "album:1", "title", "Red", "rating", 5)
c.Send("HMSET", "album:2", "title", "Earthbound", "rating", 1)
c.Send("HMSET", "album:3", "title", "Beat")
c.Send("LPUSH", "albums", "1")
c.Send("LPUSH", "albums", "2")
c.Send("LPUSH", "albums", "3")
values, err := Values(c.Do("SORT", "albums",
"BY", "album:*->rating",
"GET", "album:*->title",
"GET", "album:*->rating"))
if err != nil {
fmt.Println(err)
return
}
for len(values) > 0 {
var title string
rating := -1 // initialize to illegal value to detect nil.
values, err = Scan(values, &title, &rating)
if err != nil {
fmt.Println(err)
return
}
if rating == -1 {
fmt.Println(title, "not-rated")
} else {
fmt.Println(title, rating)
}
}
// Output:
// Beat not-rated
// Earthbound 1
// Red 5
}
type s0 struct {
X int
Y int `redis:"y"`
Bt bool
}
type s1 struct {
X int `redis:"-"`
I int `redis:"i"`
U uint `redis:"u"`
S string `redis:"s"`
P []byte `redis:"p"`
B bool `redis:"b"`
Bt bool
Bf bool
s0
}
var scanStructTests = []struct {
title string
reply []string
value interface{}
}{
{"basic",
[]string{"i", "-1234", "u", "5678", "s", "hello", "p", "world", "b", "t", "Bt", "1", "Bf", "0", "X", "123", "y", "456"},
&s1{I: -1234, U: 5678, S: "hello", P: []byte("world"), B: true, Bt: true, Bf: false, s0: s0{X: 123, Y: 456}},
},
}
func TestScanStruct(t *testing.T) {
for _, tt := range scanStructTests {
var reply []interface{}
for _, v := range tt.reply {
reply = append(reply, []byte(v))
}
value := reflect.New(reflect.ValueOf(tt.value).Type().Elem())
if err := ScanStruct(reply, value.Interface()); err != nil {
t.Fatalf("ScanStruct(%s) returned error %v", tt.title, err)
}
if !reflect.DeepEqual(value.Interface(), tt.value) {
t.Fatalf("ScanStruct(%s) returned %v, want %v", tt.title, value.Interface(), tt.value)
}
}
}
func TestBadScanStructArgs(t *testing.T) {
x := []interface{}{"A", "b"}
test := func(v interface{}) {
if err := ScanStruct(x, v); err == nil {
t.Errorf("Expect error for ScanStruct(%T, %T)", x, v)
}
}
test(nil)
var v0 *struct{}
test(v0)
var v1 int
test(&v1)
x = x[:1]
v2 := struct{ A string }{}
test(&v2)
}
var scanSliceTests = []struct {
src []interface{}
fieldNames []string
ok bool
dest interface{}
}{
{
[]interface{}{[]byte("1"), nil, []byte("-1")},
nil,
true,
[]int{1, 0, -1},
},
{
[]interface{}{[]byte("1"), nil, []byte("2")},
nil,
true,
[]uint{1, 0, 2},
},
{
[]interface{}{[]byte("-1")},
nil,
false,
[]uint{1},
},
{
[]interface{}{[]byte("hello"), nil, []byte("world")},
nil,
true,
[][]byte{[]byte("hello"), nil, []byte("world")},
},
{
[]interface{}{[]byte("hello"), nil, []byte("world")},
nil,
true,
[]string{"hello", "", "world"},
},
{
[]interface{}{[]byte("a1"), []byte("b1"), []byte("a2"), []byte("b2")},
nil,
true,
[]struct{ A, B string }{{"a1", "b1"}, {"a2", "b2"}},
},
{
[]interface{}{[]byte("a1"), []byte("b1")},
nil,
false,
[]struct{ A, B, C string }{{"a1", "b1", ""}},
},
{
[]interface{}{[]byte("a1"), []byte("b1"), []byte("a2"), []byte("b2")},
nil,
true,
[]*struct{ A, B string }{{"a1", "b1"}, {"a2", "b2"}},
},
{
[]interface{}{[]byte("a1"), []byte("b1"), []byte("a2"), []byte("b2")},
[]string{"A", "B"},
true,
[]struct{ A, C, B string }{{"a1", "", "b1"}, {"a2", "", "b2"}},
},
{
[]interface{}{[]byte("a1"), []byte("b1"), []byte("a2"), []byte("b2")},
nil,
false,
[]struct{}{},
},
}
func TestScanSlice(t *testing.T) {
for _, tt := range scanSliceTests {
typ := reflect.ValueOf(tt.dest).Type()
dest := reflect.New(typ)
err := ScanSlice(tt.src, dest.Interface(), tt.fieldNames...)
if tt.ok != (err == nil) {
t.Errorf("ScanSlice(%v, []%s, %v) returned error %v", tt.src, typ, tt.fieldNames, err)
continue
}
if tt.ok && !reflect.DeepEqual(dest.Elem().Interface(), tt.dest) {
t.Errorf("ScanSlice(src, []%s) returned %#v, want %#v", typ, dest.Elem().Interface(), tt.dest)
}
}
}
func ExampleScanSlice() {
c, err := dial()
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
c.Send("HMSET", "album:1", "title", "Red", "rating", 5)
c.Send("HMSET", "album:2", "title", "Earthbound", "rating", 1)
c.Send("HMSET", "album:3", "title", "Beat", "rating", 4)
c.Send("LPUSH", "albums", "1")
c.Send("LPUSH", "albums", "2")
c.Send("LPUSH", "albums", "3")
values, err := Values(c.Do("SORT", "albums",
"BY", "album:*->rating",
"GET", "album:*->title",
"GET", "album:*->rating"))
if err != nil {
fmt.Println(err)
return
}
var albums []struct {
Title string
Rating int
}
if err := ScanSlice(values, &albums); err != nil {
fmt.Println(err)
return
}
fmt.Printf("%v\n", albums)
// Output:
// [{Earthbound 1} {Beat 4} {Red 5}]
}
var argsTests = []struct {
title string
actual Args
expected Args
}{
{"struct ptr",
Args{}.AddFlat(&struct {
I int `redis:"i"`
U uint `redis:"u"`
S string `redis:"s"`
P []byte `redis:"p"`
M map[string]string `redis:"m"`
Bt bool
Bf bool
}{
-1234, 5678, "hello", []byte("world"), map[string]string{"hello": "world"}, true, false,
}),
Args{"i", int(-1234), "u", uint(5678), "s", "hello", "p", []byte("world"), "m", map[string]string{"hello": "world"}, "Bt", true, "Bf", false},
},
{"struct",
Args{}.AddFlat(struct{ I int }{123}),
Args{"I", 123},
},
{"slice",
Args{}.Add(1).AddFlat([]string{"a", "b", "c"}).Add(2),
Args{1, "a", "b", "c", 2},
},
{"struct omitempty",
Args{}.AddFlat(&struct {
I int `redis:"i,omitempty"`
U uint `redis:"u,omitempty"`
S string `redis:"s,omitempty"`
P []byte `redis:"p,omitempty"`
M map[string]string `redis:"m,omitempty"`
Bt bool `redis:"Bt,omitempty"`
Bf bool `redis:"Bf,omitempty"`
}{
0, 0, "", []byte{}, map[string]string{}, true, false,
}),
Args{"Bt", true},
},
}
func TestArgs(t *testing.T) {
for _, tt := range argsTests {
if !reflect.DeepEqual(tt.actual, tt.expected) {
t.Fatalf("%s is %v, want %v", tt.title, tt.actual, tt.expected)
}
}
}
func ExampleArgs() {
c, err := dial()
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
var p1, p2 struct {
Title string `redis:"title"`
Author string `redis:"author"`
Body string `redis:"body"`
}
p1.Title = "Example"
p1.Author = "Gary"
p1.Body = "Hello"
if _, err := c.Do("HMSET", Args{}.Add("id1").AddFlat(&p1)...); err != nil {
fmt.Println(err)
return
}
m := map[string]string{
"title": "Example2",
"author": "Steve",
"body": "Map",
}
if _, err := c.Do("HMSET", Args{}.Add("id2").AddFlat(m)...); err != nil {
fmt.Println(err)
return
}
for _, id := range []string{"id1", "id2"} {
v, err := Values(c.Do("HGETALL", id))
if err != nil {
fmt.Println(err)
return
}
if err := ScanStruct(v, &p2); err != nil {
fmt.Println(err)
return
}
fmt.Printf("%+v\n", p2)
}
// Output:
// {Title:Example Author:Gary Body:Hello}
// {Title:Example2 Author:Steve Body:Map}
}

103
pkg/cache/redis/script_test.go vendored Normal file
View File

@ -0,0 +1,103 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis
import (
"fmt"
"reflect"
"testing"
"time"
)
func ExampleScript() {
c, err := Dial("tcp", ":6379")
if err != nil {
// handle error
}
defer c.Close()
// Initialize a package-level variable with a script.
var getScript = NewScript(1, `return call('get', KEYS[1])`)
// In a function, use the script Do method to evaluate the script. The Do
// method optimistically uses the EVALSHA command. If the script is not
// loaded, then the Do method falls back to the EVAL command.
if _, err = getScript.Do(c, "foo"); err != nil {
// handle error
}
}
func TestScript(t *testing.T) {
c, err := DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer c.Close()
// To test fall back in Do, we make script unique by adding comment with current time.
script := fmt.Sprintf("--%d\nreturn {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}", time.Now().UnixNano())
s := NewScript(2, script)
reply := []interface{}{[]byte("key1"), []byte("key2"), []byte("arg1"), []byte("arg2")}
v, err := s.Do(c, "key1", "key2", "arg1", "arg2")
if err != nil {
t.Errorf("s.Do(c, ...) returned %v", err)
}
if !reflect.DeepEqual(v, reply) {
t.Errorf("s.Do(c, ..); = %v, want %v", v, reply)
}
err = s.Load(c)
if err != nil {
t.Errorf("s.Load(c) returned %v", err)
}
err = s.SendHash(c, "key1", "key2", "arg1", "arg2")
if err != nil {
t.Errorf("s.SendHash(c, ...) returned %v", err)
}
err = c.Flush()
if err != nil {
t.Errorf("c.Flush() returned %v", err)
}
v, err = c.Receive()
if err != nil {
t.Errorf("c.Receive() returned %v", err)
}
if !reflect.DeepEqual(v, reply) {
t.Errorf("s.SendHash(c, ..); c.Receive() = %v, want %v", v, reply)
}
err = s.Send(c, "key1", "key2", "arg1", "arg2")
if err != nil {
t.Errorf("s.Send(c, ...) returned %v", err)
}
err = c.Flush()
if err != nil {
t.Errorf("c.Flush() returned %v", err)
}
v, err = c.Receive()
if err != nil {
t.Errorf("c.Receive() returned %v", err)
}
if !reflect.DeepEqual(v, reply) {
t.Errorf("s.Send(c, ..); c.Receive() = %v, want %v", v, reply)
}
}

12
pkg/cache/redis/test/docker-compose.yaml vendored Normal file
View File

@ -0,0 +1,12 @@
version: "3.7"
services:
redis:
image: redis
ports:
- 6379:6379
healthcheck:
test: ["CMD", "redis-cli","ping"]
interval: 20s
timeout: 1s
retries: 20

42
pkg/cache/redis/trace.go vendored
View File

@ -10,10 +10,9 @@ import (
)
const (
_traceComponentName = "pkg/cache/redis"
_traceComponentName = "library/cache/redis"
_tracePeerService = "redis"
_traceSpanKind = "client"
_slowLogDuration = time.Millisecond * 250
)
var _internalTags = []trace.Tag{
@ -24,26 +23,28 @@ var _internalTags = []trace.Tag{
type traceConn struct {
// tr for pipeline, if tr != nil meaning on pipeline
tr trace.Trace
ctx context.Context
tr trace.Trace
// connTag include e.g. ip,port
connTags []trace.Tag
ctx context.Context
// origin redis conn
Conn
pending int
// TODO: split slow log from trace.
slowLogThreshold time.Duration
}
func (t *traceConn) Do(commandName string, args ...interface{}) (reply interface{}, err error) {
statement := getStatement(commandName, args...)
defer slowLog(statement, time.Now())
root, ok := trace.FromContext(t.ctx)
defer t.slowLog(statement, time.Now())
// NOTE: ignored empty commandName
// current sdk will Do empty command after pipeline finished
if !ok || commandName == "" {
if t.tr == nil || commandName == "" {
return t.Conn.Do(commandName, args...)
}
tr := root.Fork("", "Redis:"+commandName)
tr := t.tr.Fork("", "Redis:"+commandName)
tr.SetTag(_internalTags...)
tr.SetTag(t.connTags...)
tr.SetTag(trace.TagString(trace.TagDBStatement, statement))
@ -52,16 +53,15 @@ func (t *traceConn) Do(commandName string, args ...interface{}) (reply interface
return
}
func (t *traceConn) Send(commandName string, args ...interface{}) error {
func (t *traceConn) Send(commandName string, args ...interface{}) (err error) {
statement := getStatement(commandName, args...)
defer slowLog(statement, time.Now())
defer t.slowLog(statement, time.Now())
t.pending++
root, ok := trace.FromContext(t.ctx)
if !ok {
if t.tr == nil {
return t.Conn.Send(commandName, args...)
}
if t.tr == nil {
t.tr = root.Fork("", "Redis:Pipeline")
if t.pending == 1 {
t.tr = t.tr.Fork("", "Redis:Pipeline")
t.tr.SetTag(_internalTags...)
t.tr.SetTag(t.connTags...)
}
@ -69,8 +69,7 @@ func (t *traceConn) Send(commandName string, args ...interface{}) error {
trace.Log(trace.LogEvent, "Send"),
trace.Log("db.statement", statement),
)
err := t.Conn.Send(commandName, args...)
if err != nil {
if err = t.Conn.Send(commandName, args...); err != nil {
t.tr.SetTag(trace.TagBool(trace.TagError, true))
t.tr.SetLog(
trace.Log(trace.LogEvent, "Send Fail"),
@ -81,7 +80,7 @@ func (t *traceConn) Send(commandName string, args ...interface{}) error {
}
func (t *traceConn) Flush() error {
defer slowLog("Flush", time.Now())
defer t.slowLog("Flush", time.Now())
if t.tr == nil {
return t.Conn.Flush()
}
@ -98,7 +97,7 @@ func (t *traceConn) Flush() error {
}
func (t *traceConn) Receive() (reply interface{}, err error) {
defer slowLog("Receive", time.Now())
defer t.slowLog("Receive", time.Now())
if t.tr == nil {
return t.Conn.Receive()
}
@ -122,13 +121,14 @@ func (t *traceConn) Receive() (reply interface{}, err error) {
}
func (t *traceConn) WithContext(ctx context.Context) Conn {
t.ctx = ctx
t.Conn = t.Conn.WithContext(ctx)
t.tr, _ = trace.FromContext(ctx)
return t
}
func slowLog(statement string, now time.Time) {
func (t *traceConn) slowLog(statement string, now time.Time) {
du := time.Since(now)
if du > _slowLogDuration {
if du > t.slowLogThreshold {
log.Warn("%s slow log statement: %s time: %v", _tracePeerService, statement, du)
}
}

192
pkg/cache/redis/trace_test.go vendored Normal file
View File

@ -0,0 +1,192 @@
package redis
import (
"context"
"fmt"
"testing"
"time"
"github.com/bilibili/kratos/pkg/net/trace"
"github.com/stretchr/testify/assert"
)
const testTraceSlowLogThreshold = time.Duration(250 * time.Millisecond)
type mockTrace struct {
tags []trace.Tag
logs []trace.LogField
perr *error
operationName string
finished bool
}
func (m *mockTrace) Fork(serviceName string, operationName string) trace.Trace {
m.operationName = operationName
return m
}
func (m *mockTrace) Follow(serviceName string, operationName string) trace.Trace {
panic("not implemented")
}
func (m *mockTrace) Finish(err *error) {
m.perr = err
m.finished = true
}
func (m *mockTrace) SetTag(tags ...trace.Tag) trace.Trace {
m.tags = append(m.tags, tags...)
return m
}
func (m *mockTrace) SetLog(logs ...trace.LogField) trace.Trace {
m.logs = append(m.logs, logs...)
return m
}
func (m *mockTrace) Visit(fn func(k, v string)) {}
func (m *mockTrace) SetTitle(title string) {}
func (m *mockTrace) TraceID() string { return "" }
type mockConn struct{}
func (c *mockConn) Close() error { return nil }
func (c *mockConn) Err() error { return nil }
func (c *mockConn) Do(commandName string, args ...interface{}) (reply interface{}, err error) {
return nil, nil
}
func (c *mockConn) Send(commandName string, args ...interface{}) error { return nil }
func (c *mockConn) Flush() error { return nil }
func (c *mockConn) Receive() (reply interface{}, err error) { return nil, nil }
func (c *mockConn) WithContext(context.Context) Conn { return c }
func TestTraceDo(t *testing.T) {
tr := &mockTrace{}
ctx := trace.NewContext(context.Background(), tr)
tc := &traceConn{Conn: &mockConn{}, slowLogThreshold: testTraceSlowLogThreshold}
conn := tc.WithContext(ctx)
conn.Do("GET", "test")
assert.Equal(t, "Redis:GET", tr.operationName)
assert.NotEmpty(t, tr.tags)
assert.True(t, tr.finished)
}
func TestTraceDoErr(t *testing.T) {
tr := &mockTrace{}
ctx := trace.NewContext(context.Background(), tr)
tc := &traceConn{Conn: MockErr{Error: fmt.Errorf("hhhhhhh")},
slowLogThreshold: testTraceSlowLogThreshold}
conn := tc.WithContext(ctx)
conn.Do("GET", "test")
assert.Equal(t, "Redis:GET", tr.operationName)
assert.True(t, tr.finished)
assert.NotNil(t, *tr.perr)
}
func TestTracePipeline(t *testing.T) {
tr := &mockTrace{}
ctx := trace.NewContext(context.Background(), tr)
tc := &traceConn{Conn: &mockConn{}, slowLogThreshold: testTraceSlowLogThreshold}
conn := tc.WithContext(ctx)
N := 2
for i := 0; i < N; i++ {
conn.Send("GET", "hello, world")
}
conn.Flush()
for i := 0; i < N; i++ {
conn.Receive()
}
assert.Equal(t, "Redis:Pipeline", tr.operationName)
assert.NotEmpty(t, tr.tags)
assert.NotEmpty(t, tr.logs)
assert.True(t, tr.finished)
}
func TestTracePipelineErr(t *testing.T) {
tr := &mockTrace{}
ctx := trace.NewContext(context.Background(), tr)
tc := &traceConn{Conn: MockErr{Error: fmt.Errorf("hahah")},
slowLogThreshold: testTraceSlowLogThreshold}
conn := tc.WithContext(ctx)
N := 2
for i := 0; i < N; i++ {
conn.Send("GET", "hello, world")
}
conn.Flush()
for i := 0; i < N; i++ {
conn.Receive()
}
assert.Equal(t, "Redis:Pipeline", tr.operationName)
assert.NotEmpty(t, tr.tags)
assert.NotEmpty(t, tr.logs)
assert.True(t, tr.finished)
var isError bool
for _, tag := range tr.tags {
if tag.Key == "error" {
isError = true
}
}
assert.True(t, isError)
}
func TestSendStatement(t *testing.T) {
tr := &mockTrace{}
ctx := trace.NewContext(context.Background(), tr)
tc := &traceConn{Conn: MockErr{Error: fmt.Errorf("hahah")},
slowLogThreshold: testTraceSlowLogThreshold}
conn := tc.WithContext(ctx)
conn.Send("SET", "hello", "test")
conn.Flush()
conn.Receive()
assert.Equal(t, "Redis:Pipeline", tr.operationName)
assert.NotEmpty(t, tr.tags)
assert.NotEmpty(t, tr.logs)
assert.Equal(t, "event", tr.logs[0].Key)
assert.Equal(t, "Send", tr.logs[0].Value)
assert.Equal(t, "db.statement", tr.logs[1].Key)
assert.Equal(t, "SET hello", tr.logs[1].Value)
assert.True(t, tr.finished)
var isError bool
for _, tag := range tr.tags {
if tag.Key == "error" {
isError = true
}
}
assert.True(t, isError)
}
func TestDoStatement(t *testing.T) {
tr := &mockTrace{}
ctx := trace.NewContext(context.Background(), tr)
tc := &traceConn{Conn: MockErr{Error: fmt.Errorf("hahah")},
slowLogThreshold: testTraceSlowLogThreshold}
conn := tc.WithContext(ctx)
conn.Do("SET", "hello", "test")
assert.Equal(t, "Redis:SET", tr.operationName)
assert.Equal(t, "SET hello", tr.tags[len(tr.tags)-1].Value)
assert.True(t, tr.finished)
}
func BenchmarkTraceConn(b *testing.B) {
for i := 0; i < b.N; i++ {
c, err := DialDefaultServer()
if err != nil {
b.Fatal(err)
}
t := &traceConn{
Conn: c,
connTags: []trace.Tag{trace.TagString(trace.TagPeerAddress, "abc")},
slowLogThreshold: time.Duration(1 * time.Second),
}
c2 := t.WithContext(context.TODO())
if _, err := c2.Do("PING"); err != nil {
b.Fatal(err)
}
c2.Close()
}
}

17
pkg/cache/redis/util.go vendored Normal file
View File

@ -0,0 +1,17 @@
package redis
import (
"context"
"time"
)
func shrinkDeadline(ctx context.Context, timeout time.Duration) time.Time {
var timeoutTime = time.Now().Add(timeout)
if ctx == nil {
return timeoutTime
}
if deadline, ok := ctx.Deadline(); ok && timeoutTime.After(deadline) {
return deadline
}
return timeoutTime
}

37
pkg/cache/redis/util_test.go vendored Normal file
View File

@ -0,0 +1,37 @@
package redis
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestShrinkDeadline(t *testing.T) {
t.Run("test not deadline", func(t *testing.T) {
timeout := time.Second
timeoutTime := time.Now().Add(timeout)
tm := shrinkDeadline(context.Background(), timeout)
assert.True(t, tm.After(timeoutTime))
})
t.Run("test big deadline", func(t *testing.T) {
timeout := time.Second
timeoutTime := time.Now().Add(timeout)
deadlineTime := time.Now().Add(2 * time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
tm := shrinkDeadline(ctx, timeout)
assert.True(t, tm.After(timeoutTime) && tm.Before(deadlineTime))
})
t.Run("test small deadline", func(t *testing.T) {
timeout := time.Second
deadlineTime := time.Now().Add(500 * time.Millisecond)
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
tm := shrinkDeadline(ctx, timeout)
assert.True(t, tm.After(deadlineTime) && tm.Before(time.Now().Add(timeout)))
})
}

74
pkg/conf/paladin/README.md
View File

@ -2,17 +2,18 @@
##### 项目简介
paladin 是一个config SDK客户端，包括了file、mock几个抽象功能，方便使用本地文件或者sven配置中心，并且集成了对象自动reload功能。
paladin 是一个config SDK客户端，包括了file、mock几个抽象功能，方便使用本地文件或者sven\apollo配置中心，并且集成了对象自动reload功能。
local files:
```
demo -conf=/data/conf/app/msm-servie.toml
// or dir
demo -conf=/data/conf/app/
```
example:
*注：使用远程配置中心的用户在执行应用，如这里的`demo`时务必**不要**带上`-conf`参数，具体见下文远程配置中心的例子*
local file example:
```
type exampleConf struct {
Bool bool
@ -65,6 +66,71 @@ func ExampleClient() {
}
```
remote config center example:
```
type exampleConf struct {
Bool bool
Int int64
Float float64
String string
}
func (e *exampleConf) Set(text string) error {
var ec exampleConf
if err := yaml.Unmarshal([]byte(text), &ec); err != nil {
return err
}
*e = ec
return nil
}
func ExampleApolloClient() {
/*
pass flags or set envs that apollo needs, for example:
```
export APOLLO_APP_ID=SampleApp
export APOLLO_CLUSTER=default
export APOLLO_CACHE_DIR=/tmp
export APOLLO_META_ADDR=localhost:8080
export APOLLO_NAMESPACES=example.yml
```
*/
if err := paladin.Init(apollo.PaladinDriverApollo); err != nil {
panic(err)
}
var (
ec exampleConf
eo exampleConf
m paladin.Map
strs []string
)
// config unmarshal
if err := paladin.Get("example.yml").UnmarshalYAML(&ec); err != nil {
panic(err)
}
// config setter
if err := paladin.Watch("example.yml", &ec); err != nil {
panic(err)
}
// paladin map
if err := paladin.Watch("example.yml", &m); err != nil {
panic(err)
}
s, err := m.Value("key").String()
b, err := m.Value("key").Bool()
i, err := m.Value("key").Int64()
f, err := m.Value("key").Float64()
// value slice
err = m.Value("strings").Slice(&strs)
// watch key
for event := range paladin.WatchEvent(context.TODO(), "key") {
fmt.Println(event)
}
}
```
##### 编译环境
- **请只用 Golang v1.12.x 以上版本编译执行**

273
pkg/conf/paladin/apollo/apollo.go Normal file
View File

@ -0,0 +1,273 @@
package apollo
import (
"context"
"errors"
"flag"
"log"
"os"
"strings"
"sync"
"time"
"github.com/philchia/agollo"
"github.com/bilibili/kratos/pkg/conf/paladin"
)
var (
_ paladin.Client = &apollo{}
defaultValue = ""
)
type apolloWatcher struct {
keys []string // in apollo, they're called namespaces
C chan paladin.Event
}
func newApolloWatcher(keys []string) *apolloWatcher {
return &apolloWatcher{keys: keys, C: make(chan paladin.Event, 5)}
}
func (aw *apolloWatcher) HasKey(key string) bool {
if len(aw.keys) == 0 {
return true
}
for _, k := range aw.keys {
if k == key {
return true
}
}
return false
}
func (aw *apolloWatcher) Handle(event paladin.Event) {
select {
case aw.C <- event:
default:
log.Printf("paladin: event channel full discard ns %s update event", event.Key)
}
}
// apollo is apollo config client.
type apollo struct {
client *agollo.Client
values *paladin.Map
wmu sync.RWMutex
watchers map[*apolloWatcher]struct{}
}
// Config is apollo config client config.
type Config struct {
AppID string `json:"app_id"`
Cluster string `json:"cluster"`
CacheDir string `json:"cache_dir"`
MetaAddr string `json:"meta_addr"`
Namespaces []string `json:"namespaces"`
}
type apolloDriver struct{}
var (
confAppID, confCluster, confCacheDir, confMetaAddr, confNamespaces string
)
func init() {
addApolloFlags()
paladin.Register(PaladinDriverApollo, &apolloDriver{})
}
func addApolloFlags() {
flag.StringVar(&confAppID, "apollo.appid", "", "apollo app id")
flag.StringVar(&confCluster, "apollo.cluster", "", "apollo cluster")
flag.StringVar(&confCacheDir, "apollo.cachedir", "/tmp", "apollo cache dir")
flag.StringVar(&confMetaAddr, "apollo.metaaddr", "", "apollo meta server addr, e.g. localhost:8080")
flag.StringVar(&confNamespaces, "apollo.namespaces", "", "subscribed apollo namespaces, comma separated, e.g. app.yml,mysql.yml")
}
func buildConfigForApollo() (c *Config, err error) {
if appidFromEnv := os.Getenv("APOLLO_APP_ID"); appidFromEnv != "" {
confAppID = appidFromEnv
}
if confAppID == "" {
err = errors.New("invalid apollo appid, pass it via APOLLO_APP_ID=xxx with env or --apollo.appid=xxx with flag")
return
}
if clusterFromEnv := os.Getenv("APOLLO_CLUSTER"); clusterFromEnv != "" {
confCluster = clusterFromEnv
}
if confAppID == "" {
err = errors.New("invalid apollo cluster, pass it via APOLLO_CLUSTER=xxx with env or --apollo.cluster=xxx with flag")
return
}
if cacheDirFromEnv := os.Getenv("APOLLO_CACHE_DIR"); cacheDirFromEnv != "" {
confCacheDir = cacheDirFromEnv
}
if metaAddrFromEnv := os.Getenv("APOLLO_META_ADDR"); metaAddrFromEnv != "" {
confMetaAddr = metaAddrFromEnv
}
if confMetaAddr == "" {
err = errors.New("invalid apollo meta addr, pass it via APOLLO_META_ADDR=xxx with env or --apollo.metaaddr=xxx with flag")
return
}
if namespacesFromEnv := os.Getenv("APOLLO_NAMESPACES"); namespacesFromEnv != "" {
confNamespaces = namespacesFromEnv
}
namespaceNames := strings.Split(confNamespaces, ",")
if len(namespaceNames) == 0 {
err = errors.New("invalid apollo namespaces, pass it via APOLLO_NAMESPACES=xxx with env or --apollo.namespaces=xxx with flag")
return
}
c = &Config{
AppID: confAppID,
Cluster: confCluster,
CacheDir: confCacheDir,
MetaAddr: confMetaAddr,
Namespaces: namespaceNames,
}
return
}
// New new an apollo config client.
// it watches apollo namespaces changes and updates local cache.
// BTW, in our context, namespaces in apollo means keys in paladin.
func (ad *apolloDriver) New() (paladin.Client, error) {
c, err := buildConfigForApollo()
if err != nil {
return nil, err
}
return ad.new(c)
}
func (ad *apolloDriver) new(conf *Config) (paladin.Client, error) {
if conf == nil {
err := errors.New("invalid apollo conf")
return nil, err
}
client := agollo.NewClient(&agollo.Conf{
AppID: conf.AppID,
Cluster: conf.Cluster,
NameSpaceNames: conf.Namespaces, // these namespaces will be subscribed at init
CacheDir: conf.CacheDir,
IP: conf.MetaAddr,
})
err := client.Start()
if err != nil {
return nil, err
}
a := &apollo{
client: client,
values: new(paladin.Map),
watchers: make(map[*apolloWatcher]struct{}),
}
raws, err := a.loadValues(conf.Namespaces)
if err != nil {
return nil, err
}
a.values.Store(raws)
// watch namespaces by default.
a.WatchEvent(context.TODO(), conf.Namespaces...)
go a.watchproc(conf.Namespaces)
return a, nil
}
// loadValues load values from apollo namespaces to values
func (a *apollo) loadValues(keys []string) (values map[string]*paladin.Value, err error) {
values = make(map[string]*paladin.Value, len(keys))
for _, k := range keys {
if values[k], err = a.loadValue(k); err != nil {
return
}
}
return
}
// loadValue load value from apollo namespace content to value
func (a *apollo) loadValue(key string) (*paladin.Value, error) {
content := a.client.GetNameSpaceContent(key, defaultValue)
return paladin.NewValue(content, content), nil
}
// reloadValue reload value by key and send event
func (a *apollo) reloadValue(key string) (err error) {
// NOTE: in some case immediately read content from client after receive event
// will get old content due to cache, sleep 100ms make sure get correct content.
time.Sleep(100 * time.Millisecond)
var (
value *paladin.Value
rawValue string
)
value, err = a.loadValue(key)
if err != nil {
return
}
rawValue, err = value.Raw()
if err != nil {
return
}
raws := a.values.Load()
raws[key] = value
a.values.Store(raws)
a.wmu.RLock()
n := 0
for w := range a.watchers {
if w.HasKey(key) {
n++
// FIXME(Colstuwjx): check change event and send detail type like EventAdd\Update\Delete.
w.Handle(paladin.Event{Event: paladin.EventUpdate, Key: key, Value: rawValue})
}
}
a.wmu.RUnlock()
log.Printf("paladin: reload config: %s events: %d\n", key, n)
return
}
// apollo config daemon to watch remote apollo notifications
func (a *apollo) watchproc(keys []string) {
events := a.client.WatchUpdate()
for {
select {
case event := <-events:
if err := a.reloadValue(event.Namespace); err != nil {
log.Printf("paladin: load key: %s error: %s, skipped", event.Namespace, err)
}
}
}
}
// Get return value by key.
func (a *apollo) Get(key string) *paladin.Value {
return a.values.Get(key)
}
// GetAll return value map.
func (a *apollo) GetAll() *paladin.Map {
return a.values
}
// WatchEvent watch with the specified keys.
func (a *apollo) WatchEvent(ctx context.Context, keys ...string) <-chan paladin.Event {
aw := newApolloWatcher(keys)
err := a.client.SubscribeToNamespaces(keys...)
if err != nil {
log.Printf("subscribe namespaces %v failed, %v", keys, err)
return aw.C
}
a.wmu.Lock()
a.watchers[aw] = struct{}{}
a.wmu.Unlock()
return aw.C
}
// Close close watcher.
func (a *apollo) Close() (err error) {
if err = a.client.Stop(); err != nil {
return
}
a.wmu.RLock()
for w := range a.watchers {
close(w.C)
}
a.wmu.RUnlock()
return
}

73
pkg/conf/paladin/apollo/apollo_test.go Normal file
View File

@ -0,0 +1,73 @@
package apollo
import (
"context"
"fmt"
"log"
"os"
"testing"
"time"
"github.com/bilibili/kratos/pkg/conf/paladin/apollo/internal/mockserver"
)
func TestMain(m *testing.M) {
setup()
code := m.Run()
teardown()
os.Exit(code)
}
func setup() {
go func() {
if err := mockserver.Run(); err != nil {
log.Fatal(err)
}
}()
// wait for mock server to run
time.Sleep(time.Millisecond * 500)
}
func teardown() {
mockserver.Close()
}
func TestApollo(t *testing.T) {
var (
testAppYAML = "app.yml"
testAppYAMLContent1 = "test: test12234\ntest2: test333"
testAppYAMLContent2 = "test: 1111"
testClientJSON = "client.json"
testClientJSONContent = `{"name":"agollo"}`
)
os.Setenv("APOLLO_APP_ID", "SampleApp")
os.Setenv("APOLLO_CLUSTER", "default")
os.Setenv("APOLLO_CACHE_DIR", "/tmp")
os.Setenv("APOLLO_META_ADDR", "localhost:8080")
os.Setenv("APOLLO_NAMESPACES", fmt.Sprintf("%s,%s", testAppYAML, testClientJSON))
mockserver.Set(testAppYAML, "content", testAppYAMLContent1)
mockserver.Set(testClientJSON, "content", testClientJSONContent)
ad := &apolloDriver{}
apollo, err := ad.New()
if err != nil {
t.Fatalf("new apollo error, %v", err)
}
value := apollo.Get(testAppYAML)
if content, _ := value.String(); content != testAppYAMLContent1 {
t.Fatalf("got app.yml unexpected value %s", content)
}
value = apollo.Get(testClientJSON)
if content, _ := value.String(); content != testClientJSONContent {
t.Fatalf("got app.yml unexpected value %s", content)
}
mockserver.Set(testAppYAML, "content", testAppYAMLContent2)
updates := apollo.WatchEvent(context.TODO(), testAppYAML)
select {
case <-updates:
case <-time.After(time.Millisecond * 30000):
}
value = apollo.Get(testAppYAML)
if content, _ := value.String(); content != testAppYAMLContent2 {
t.Fatalf("got app.yml unexpected updated value %s", content)
}
}

6
pkg/conf/paladin/apollo/const.go Normal file
View File

@ -0,0 +1,6 @@
package apollo
const (
// PaladinDriverApollo ...
PaladinDriverApollo = "apollo"
)

149
pkg/conf/paladin/apollo/internal/mockserver/mockserver.go Normal file
View File

@ -0,0 +1,149 @@
package mockserver
import (
"context"
"encoding/json"
"net/http"
"strings"
"sync"
"time"
)
type notification struct {
NamespaceName string `json:"namespaceName,omitempty"`
NotificationID int `json:"notificationId,omitempty"`
}
type result struct {
// AppID string `json:"appId"`
// Cluster string `json:"cluster"`
NamespaceName string `json:"namespaceName"`
Configurations map[string]string `json:"configurations"`
ReleaseKey string `json:"releaseKey"`
}
type mockServer struct {
server http.Server
lock sync.Mutex
notifications map[string]int
config map[string]map[string]string
}
func (s *mockServer) NotificationHandler(rw http.ResponseWriter, req *http.Request) {
s.lock.Lock()
defer s.lock.Unlock()
req.ParseForm()
var notifications []notification
if err := json.Unmarshal([]byte(req.FormValue("notifications")), &notifications); err != nil {
rw.WriteHeader(http.StatusInternalServerError)
return
}
var changes []notification
for _, noti := range notifications {
if currentID := s.notifications[noti.NamespaceName]; currentID != noti.NotificationID {
changes = append(changes, notification{NamespaceName: noti.NamespaceName, NotificationID: currentID})
}
}
if len(changes) == 0 {
rw.WriteHeader(http.StatusNotModified)
return
}
bts, err := json.Marshal(&changes)
if err != nil {
rw.WriteHeader(http.StatusInternalServerError)
return
}
rw.Write(bts)
}
func (s *mockServer) ConfigHandler(rw http.ResponseWriter, req *http.Request) {
req.ParseForm()
strs := strings.Split(req.RequestURI, "/")
var namespace, releaseKey = strings.Split(strs[4], "?")[0], req.FormValue("releaseKey")
config := s.Get(namespace)
var result = result{NamespaceName: namespace, Configurations: config, ReleaseKey: releaseKey}
bts, err := json.Marshal(&result)
if err != nil {
rw.WriteHeader(http.StatusInternalServerError)
return
}
rw.Write(bts)
}
var server *mockServer
func (s *mockServer) Set(namespace, key, value string) {
server.lock.Lock()
defer server.lock.Unlock()
notificationID := s.notifications[namespace]
notificationID++
s.notifications[namespace] = notificationID
if kv, ok := s.config[namespace]; ok {
kv[key] = value
return
}
kv := map[string]string{key: value}
s.config[namespace] = kv
}
func (s *mockServer) Get(namespace string) map[string]string {
server.lock.Lock()
defer server.lock.Unlock()
return s.config[namespace]
}
func (s *mockServer) Delete(namespace, key string) {
server.lock.Lock()
defer server.lock.Unlock()
if kv, ok := s.config[namespace]; ok {
delete(kv, key)
}
notificationID := s.notifications[namespace]
notificationID++
s.notifications[namespace] = notificationID
}
// Set namespace's key value
func Set(namespace, key, value string) {
server.Set(namespace, key, value)
}
// Delete namespace's key
func Delete(namespace, key string) {
server.Delete(namespace, key)
}
// Run mock server
func Run() error {
initServer()
return server.server.ListenAndServe()
}
func initServer() {
server = &mockServer{
notifications: map[string]int{},
config: map[string]map[string]string{},
}
mux := http.NewServeMux()
mux.Handle("/notifications/", http.HandlerFunc(server.NotificationHandler))
mux.Handle("/configs/", http.HandlerFunc(server.ConfigHandler))
server.server.Handler = mux
server.server.Addr = ":8080"
}
// Close mock server
func Close() error {
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second))
defer cancel()
return server.server.Shutdown(ctx)
}

25
pkg/conf/paladin/default.go
View File

@ -2,6 +2,7 @@ package paladin
import (
"context"
"errors"
"flag"
)
@ -16,12 +17,30 @@ func init() {
}
// Init init config client.
func Init() (err error) {
// If confPath is set, it inits file client by default
// Otherwise we could pass args to init remote client
// args[0]: driver name, string type
func Init(args ...interface{}) (err error) {
if confPath != "" {
DefaultClient, err = NewFile(confPath)
} else {
// TODO: Get the configuration from the remote service
panic("Please specify a file or dir name by -conf flag.")
var (
driver Driver
)
argsLackErr := errors.New("lack of remote config center args")
if len(args) == 0 {
panic(argsLackErr.Error())
}
argsInvalidErr := errors.New("invalid remote config center args")
driverName, ok := args[0].(string)
if !ok {
panic(argsInvalidErr.Error())
}
driver, err = GetDriver(driverName)
if err != nil {
return
}
DefaultClient, err = driver.New()
}
if err != nil {
return

9
pkg/conf/paladin/driver.go Normal file
View File

@ -0,0 +1,9 @@
package paladin
// Driver defined paladin remote client impl
// each remote config center driver must do
// 1. implements `New` method
// 2. call `Register` to register itself
type Driver interface {
New() (Client, error)
}

39
pkg/conf/paladin/example_test.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"github.com/bilibili/kratos/pkg/conf/paladin"
"github.com/bilibili/kratos/pkg/conf/paladin/apollo"
"github.com/BurntSushi/toml"
)
@ -26,7 +27,7 @@ func (e *exampleConf) Set(text string) error {
return nil
}
// ExampleClient is a example client usage.
// ExampleClient is an example client usage.
// exmaple.toml:
/*
bool = true
@ -56,7 +57,41 @@ func ExampleClient() {
}()
}
// ExampleMap is a example map usage.
// ExampleApolloClient is an example client for apollo driver usage.
func ExampleApolloClient() {
/*
pass flags or set envs that apollo needs, for example:
```
export APOLLO_APP_ID=SampleApp
export APOLLO_CLUSTER=default
export APOLLO_CACHE_DIR=/tmp
export APOLLO_META_ADDR=localhost:8080
export APOLLO_NAMESPACES=example.yml
```
*/
if err := paladin.Init(apollo.PaladinDriverApollo); err != nil {
panic(err)
}
var ec exampleConf
// var setter
if err := paladin.Watch("example.yml", &ec); err != nil {
panic(err)
}
if err := paladin.Get("example.yml").UnmarshalYAML(&ec); err != nil {
panic(err)
}
// use exampleConf
// watch event key
go func() {
for event := range paladin.WatchEvent(context.TODO(), "key") {
fmt.Println(event)
}
}()
}
// ExampleMap is an example map usage.
// exmaple.toml:
/*
bool = true

6
pkg/conf/paladin/file.go
View File

@ -31,7 +31,7 @@ func (w *watcher) HasKey(key string) bool {
return true
}
for _, k := range w.keys {
if keyNamed(k) == key {
if KeyNamed(k) == key {
return true
}
}
@ -138,7 +138,7 @@ func (f *file) reloadFile(fpath string) (err error) {
if err != nil {
return
}
key := keyNamed(path.Base(fpath))
key := KeyNamed(path.Base(fpath))
raws := f.values.Load()
raws[key] = value
f.values.Store(raws)
@ -167,7 +167,7 @@ func loadValues(base string) (map[string]*Value, error) {
return nil, fmt.Errorf("paladin: read dir %s error: %s", base, err)
}
for _, file := range files {
if !file.IsDir() {
if !file.IsDir() && (file.Mode()&os.ModeSymlink) != os.ModeSymlink {
paths = append(paths, path.Join(base, file.Name()))
}
}

10
pkg/conf/paladin/map.go
View File

@ -5,8 +5,8 @@ import (
"sync/atomic"
)
// keyNamed key naming to lower case.
func keyNamed(key string) string {
// KeyNamed key naming to lower case.
func KeyNamed(key string) string {
return strings.ToLower(key)
}
@ -19,7 +19,7 @@ type Map struct {
func (m *Map) Store(values map[string]*Value) {
dst := make(map[string]*Value, len(values))
for k, v := range values {
dst[keyNamed(k)] = v
dst[KeyNamed(k)] = v
}
m.values.Store(dst)
}
@ -36,13 +36,13 @@ func (m *Map) Load() map[string]*Value {
// Exist check if values map exist a key.
func (m *Map) Exist(key string) bool {
_, ok := m.Load()[keyNamed(key)]
_, ok := m.Load()[KeyNamed(key)]
return ok
}
// Get return get value by key.
func (m *Map) Get(key string) *Value {
v, ok := m.Load()[keyNamed(key)]
v, ok := m.Load()[KeyNamed(key)]
if ok {
return v
}

2
pkg/conf/paladin/mock.go
View File

@ -13,7 +13,7 @@ type Mock struct {
}
// NewMock new a config mock client.
func NewMock(vs map[string]string) *Mock {
func NewMock(vs map[string]string) Client {
values := make(map[string]*Value, len(vs))
for k, v := range vs {
values[k] = &Value{val: v, raw: v}

55
pkg/conf/paladin/register.go Normal file
View File

@ -0,0 +1,55 @@
package paladin
import (
"fmt"
"sort"
"sync"
)
var (
driversMu sync.RWMutex
drivers = make(map[string]Driver)
)
// Register makes a paladin driver available by the provided name.
// If Register is called twice with the same name or if driver is nil,
// it panics.
func Register(name string, driver Driver) {
driversMu.Lock()
defer driversMu.Unlock()
if driver == nil {
panic("paladin: driver is nil")
}
if _, dup := drivers[name]; dup {
panic("paladin: Register called twice for driver " + name)
}
drivers[name] = driver
}
// Drivers returns a sorted list of the names of the registered paladin driver.
func Drivers() []string {
driversMu.RLock()
defer driversMu.RUnlock()
var list []string
for name := range drivers {
list = append(list, name)
}
sort.Strings(list)
return list
}
// GetDriver returns a driver implement by name.
func GetDriver(name string) (Driver, error) {
driversMu.RLock()
driveri, ok := drivers[name]
driversMu.RUnlock()
if !ok {
return nil, fmt.Errorf("paladin: unknown driver %q (forgotten import?)", name)
}
return driveri, nil
}

2
pkg/conf/paladin/toml.go
View File

@ -28,7 +28,7 @@ func (m *TOML) UnmarshalText(text []byte) error {
}
values := map[string]*Value{}
for k, v := range raws {
k = keyNamed(k)
k = KeyNamed(k)
rv := reflect.ValueOf(v)
switch rv.Kind() {
case reflect.Map:

11
pkg/conf/paladin/value.go
View File

@ -25,6 +25,14 @@ type Value struct {
raw string
}
// NewValue new a value
func NewValue(val interface{}, raw string) *Value {
return &Value{
val: val,
raw: raw,
}
}
// Bool return bool value.
func (v *Value) Bool() (bool, error) {
if v.val == nil {
@ -112,7 +120,7 @@ func (v *Value) Raw() (string, error) {
return v.raw, nil
}
// Slice scan a slcie interface, if slice has element it will be discard.
// Slice scan a slice interface, if slice has element it will be discard.
func (v *Value) Slice(dst interface{}) error {
// NOTE: val is []interface{}, slice is []type
if v.val == nil {
@ -167,6 +175,7 @@ func (v *Value) UnmarshalJSON(dst interface{}) error {
return json.Unmarshal([]byte(text), dst)
}
// UnmarshalYAML unmarshal yaml to struct.
func (v *Value) UnmarshalYAML(dst interface{}) error {
text, err := v.Raw()
if err != nil {

4
pkg/database/hbase/metrics.go
View File

@ -41,8 +41,8 @@ func codeFromErr(err error) string {
code = "connot_find_region"
case gohbase.TableNotFound:
code = "table_not_found"
//case gohbase.ErrRegionUnavailable:
// code = "region_unavailable"
//case gohbase.ErrRegionUnavailable:
// code = "region_unavailable"
}
return code
}

5
pkg/ecode/status.go
View File

@ -5,6 +5,7 @@ import (
"strconv"
"github.com/bilibili/kratos/pkg/ecode/types"
"github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes"
)
@ -81,13 +82,13 @@ func (s *Status) Proto() *types.Status {
// FromCode create status from ecode
func FromCode(code Code) *Status {
return &Status{s: &types.Status{Code: int32(code)}}
return &Status{s: &types.Status{Code: int32(code), Message: code.Message()}}
}
// FromProto new status from grpc detail
func FromProto(pbMsg proto.Message) Codes {
if msg, ok := pbMsg.(*types.Status); ok {
if msg.Message == "" {
if msg.Message == "" || msg.Message == strconv.FormatInt(int64(msg.Code), 10) {
// NOTE: if message is empty convert to pure Code, will get message from config center.
return Code(msg.Code)
}

14
pkg/naming/discovery/discovery.go
View File

@ -102,7 +102,7 @@ type appInfo struct {
}
func fixConfig(c *Config) error {
if len(c.Nodes) == 0 {
if len(c.Nodes) == 0 && env.DiscoveryNodes != "" {
c.Nodes = strings.Split(env.DiscoveryNodes, ",")
}
if c.Region == "" {
@ -370,9 +370,15 @@ func (d *Discovery) register(ctx context.Context, ins *naming.Instance) (err err
uri := fmt.Sprintf(_registerURL, d.pickNode())
params := d.newParams(c)
params.Set("appid", ins.AppID)
params.Set("addrs", strings.Join(ins.Addrs, ","))
for _, addr := range ins.Addrs {
params.Add("addrs", addr)
}
params.Set("version", ins.Version)
params.Set("status", _statusUP)
if ins.Status == 0 {
params.Set("status", _statusUP)
} else {
params.Set("status", strconv.FormatInt(ins.Status, 10))
}
params.Set("metadata", string(metadata))
if err = d.httpClient.Post(ctx, uri, "", params, &res); err != nil {
d.switchNode()
@ -469,7 +475,7 @@ func (d *Discovery) set(ctx context.Context, ins *naming.Instance) (err error) {
params := d.newParams(conf)
params.Set("appid", ins.AppID)
params.Set("version", ins.Version)
params.Set("status", _statusUP)
params.Set("status", strconv.FormatInt(ins.Status, 10))
if ins.Metadata != nil {
var metadata []byte
if metadata, err = json.Marshal(ins.Metadata); err != nil {

4
pkg/naming/etcd/etcd_test.go
View File

@ -3,11 +3,11 @@ package etcd
import (
"context"
"fmt"
"testing"
"time"
"github.com/bilibili/kratos/pkg/naming"
"go.etcd.io/etcd/clientv3"
"google.golang.org/grpc"
"testing"
"time"
)
func TestNew(t *testing.T) {

2
pkg/naming/naming.go
View File

@ -35,6 +35,8 @@ type Instance struct {
// Metadata is the information associated with Addr, which may be used
// to make load balancing decision.
Metadata map[string]string `json:"metadata"`
// Status instance status, eg: 1UP 2Waiting
Status int64 `json:"status"`
}
// Resolver resolve naming service

14
pkg/net/http/blademaster/context.go
View File

@ -49,7 +49,6 @@ type Context struct {
RoutePath string
Params Params
}
/************************************/
@ -67,7 +66,6 @@ func (c *Context) Next() {
}
}
// Abort prevents pending handlers from being called. Note that this will not stop the current handler.
// Let's say you have an authorization middleware that validates that the current request is authorized.
// If the authorization fails (ex: the password does not match), call Abort to ensure the remaining handlers
@ -276,9 +274,17 @@ func (c *Context) BindWith(obj interface{}, b binding.Binding) error {
return c.mustBindWith(obj, b)
}
// Bind bind req arg with defult form binding.
// Bind checks the Content-Type to select a binding engine automatically,
// Depending the "Content-Type" header different bindings are used:
// "application/json" --> JSON binding
// "application/xml" --> XML binding
// otherwise --> returns an error.
// It parses the request's body as JSON if Content-Type == "application/json" using JSON or XML as a JSON input.
// It decodes the json payload into the struct specified as a pointer.
// It writes a 400 error and sets Content-Type header "text/plain" in the response if input is not valid.
func (c *Context) Bind(obj interface{}) error {
return c.mustBindWith(obj, binding.Form)
b := binding.Default(c.Request.Method, c.Request.Header.Get("Content-Type"))
return c.mustBindWith(obj, b)
}
// mustBindWith binds the passed struct pointer using the specified binding engine.

6
pkg/net/http/blademaster/logger.go
View File

@ -34,8 +34,10 @@ func Logger() HandlerFunc {
caller = noUser
}
_metricServerReqCodeTotal.Inc(c.RoutePath[1:], caller, strconv.FormatInt(int64(cerr.Code()), 10))
_metricServerReqDur.Observe(int64(dt/time.Millisecond), c.RoutePath[1:], caller)
if len(c.RoutePath) > 0 {
_metricServerReqCodeTotal.Inc(c.RoutePath[1:], caller, strconv.FormatInt(int64(cerr.Code()), 10))
_metricServerReqDur.Observe(int64(dt/time.Millisecond), c.RoutePath[1:], caller)
}
lf := log.Infov
errmsg := ""

37
pkg/net/http/blademaster/perf.go
View File

@ -19,28 +19,45 @@ var (
func init() {
v := os.Getenv("HTTP_PERF")
if v == "" {
v = "tcp://0.0.0.0:2333"
}
flag.StringVar(&_perfDSN, "http.perf", v, "listen http perf dsn, or use HTTP_PERF env variable.")
}
func startPerf() {
func startPerf(engine *Engine) {
_perfOnce.Do(func() {
mux := http.NewServeMux()
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
if os.Getenv("HTTP_PERF") == "" {
prefixRouter := engine.Group("/debug/pprof")
{
prefixRouter.GET("/", pprofHandler(pprof.Index))
prefixRouter.GET("/cmdline", pprofHandler(pprof.Cmdline))
prefixRouter.GET("/profile", pprofHandler(pprof.Profile))
prefixRouter.POST("/symbol", pprofHandler(pprof.Symbol))
prefixRouter.GET("/symbol", pprofHandler(pprof.Symbol))
prefixRouter.GET("/trace", pprofHandler(pprof.Trace))
prefixRouter.GET("/allocs", pprofHandler(pprof.Handler("allocs").ServeHTTP))
prefixRouter.GET("/block", pprofHandler(pprof.Handler("block").ServeHTTP))
prefixRouter.GET("/goroutine", pprofHandler(pprof.Handler("goroutine").ServeHTTP))
prefixRouter.GET("/heap", pprofHandler(pprof.Handler("heap").ServeHTTP))
prefixRouter.GET("/mutex", pprofHandler(pprof.Handler("mutex").ServeHTTP))
prefixRouter.GET("/threadcreate", pprofHandler(pprof.Handler("threadcreate").ServeHTTP))
}
return
}
go func() {
d, err := dsn.Parse(_perfDSN)
if err != nil {
panic(errors.Errorf("blademaster: http perf dsn must be tcp://$host:port, %s:error(%v)", _perfDSN, err))
}
if err := http.ListenAndServe(d.Host, mux); err != nil {
if err := http.ListenAndServe(d.Host, nil); err != nil {
panic(errors.Errorf("blademaster: listen %s: error(%v)", d.Host, err))
}
}()
})
}
func pprofHandler(h http.HandlerFunc) HandlerFunc {
handler := http.HandlerFunc(h)
return func(c *Context) {
handler.ServeHTTP(c.Writer, c.Request)
}
}

2
pkg/net/http/blademaster/server.go
View File

@ -195,7 +195,7 @@ func NewServer(conf *ServerConfig) *Engine {
c.Bytes(405, "text/plain", []byte(http.StatusText(405)))
c.Abort()
})
startPerf()
startPerf(engine)
return engine
}

4
pkg/net/ip/ip.go
View File

@ -68,7 +68,7 @@ func InternalIP() string {
return ""
}
// isUp Interface is up
// isUp Interface is up
func isUp(v net.Flags) bool {
return v&net.FlagUp == net.FlagUp
return v&net.FlagUp == net.FlagUp
}

3
pkg/net/netutil/breaker/README.md
View File

@ -11,10 +11,9 @@
> 4. 默认配置如下所示：
_conf = &Config{
Window: xtime.Duration(3 * time.Second),
Sleep: xtime.Duration(100 * time.Millisecond),
Bucket: 10,
Ratio: 0.5,
Request: 100,
K:1.5,
}
##### 测试

12
pkg/net/netutil/breaker/breaker.go
View File

@ -11,10 +11,6 @@ import (
type Config struct {
SwitchOff bool // breaker switch,default off.
// Hystrix
Ratio float32
Sleep xtime.Duration
// Google
K float64
@ -30,12 +26,6 @@ func (conf *Config) fix() {
if conf.Request == 0 {
conf.Request = 100
}
if conf.Ratio == 0 {
conf.Ratio = 0.5
}
if conf.Sleep == 0 {
conf.Sleep = xtime.Duration(500 * time.Millisecond)
}
if conf.Bucket == 0 {
conf.Bucket = 10
}
@ -84,8 +74,6 @@ var (
Bucket: 10,
Request: 100,
Sleep: xtime.Duration(500 * time.Millisecond),
Ratio: 0.5,
// Percentage of failures must be lower than 33.33%
K: 1.5,

6
pkg/net/netutil/breaker/breaker_test.go
View File

@ -28,9 +28,7 @@ func TestGroup(t *testing.T) {
g := NewGroup(_conf)
c := &Config{
Window: xtime.Duration(1 * time.Second),
Sleep: xtime.Duration(100 * time.Millisecond),
Bucket: 10,
Ratio: 0.5,
Request: 100,
SwitchOff: !_conf.SwitchOff,
}
@ -44,9 +42,7 @@ func TestInit(t *testing.T) {
switchOff := _conf.SwitchOff
c := &Config{
Window: xtime.Duration(3 * time.Second),
Sleep: xtime.Duration(100 * time.Millisecond),
Bucket: 10,
Ratio: 0.5,
Request: 100,
SwitchOff: !switchOff,
}
@ -69,9 +65,7 @@ func TestGo(t *testing.T) {
_group.Reload(&Config{
Window: xtime.Duration(3 * time.Second),
Sleep: xtime.Duration(100 * time.Millisecond),
Bucket: 10,
Ratio: 0.5,
Request: 100,
SwitchOff: true,
})

3
pkg/net/netutil/breaker/example_test.go
View File

@ -12,9 +12,8 @@ import (
func ExampleGroup() {
c := &breaker.Config{
Window: xtime.Duration(3 * time.Second),
Sleep: xtime.Duration(100 * time.Millisecond),
K: 1.5,
Bucket: 10,
Ratio: 0.5,
Request: 100,
}
// init default config

91
pkg/net/rpc/warden/CHANGELOG.md
View File

@ -1,91 +0,0 @@
### net/rpc/warden
##### Version 1.1.21
1. fix resolver bug
##### Version 1.1.20
1. client增加timeoutCallOpt强制覆盖每次请求的timeout
##### Version 1.1.19
1. 升级grpc至1.22.0
2. client增加keepAlive选项
##### Version 1.1.18
1. 修复resolver过滤导致的子集bug
##### Version 1.1.17
1. 移除 bbr feature flag，默认开启自适应限流
##### Version 1.1.16
1. 使用 flag(grpc.bbr) 绑定 BBR 限流
##### Version 1.1.15
1. warden使用 metadata.Range 方法
##### Version 1.1.14
1. 为 server log 添加选项
##### Version 1.1.13
1. 为 client log 添加选项
##### Version 1.1.12
1. 设置 caller 为 no_user 如果 user 不存在
##### Version 1.1.12
1. warden支持mirror传递
##### Version 1.1.11
1. Validate RequestErr支持详细报错信息
##### Version 1.1.10
1. 默认读取环境中的color
##### Version 1.1.9
1. 增加NonBlock模式
##### Version 1.1.8
1. 新增appid mock
##### Version 1.1.7
1. 兼容cpu为0和wrr dt为0的情况
##### Version 1.1.6
1. 修改caller传递和获取方式
2. 添加error detail example
##### Version 1.1.5
1. 增加server端json格式支持
##### Version 1.1.4
1. 判断reosvler.builder为nil之后再注册
##### Version 1.1.3
1. 支持zone和clusters
##### Version 1.1.2
1. 业务错误日志记为 WARN
##### Version 1.1.1
1. server实现了返回cpu信息
##### Version 1.1.0
1. 增加ErrorDetail
2. 修复日志打印error信息丢失问题
##### Version 1.0.3
1. 给server增加keepalive参数
##### Version 1.0.2
1. 替代默认的timoue，使用durtaion.Shrink()来传递context
2. 修复peer.Addr为nil时会panic的问题
##### Version 1.0.1
1. 去除timeout的手动传递，改为使用grpc默认自带的grpc-timeout
2. 获取server address改为使用call option的方式，去除对balancer的依赖
##### Version 1.0.0
1. 使用NewClient来新建一个RPC客户端，并默认集成trace、log、recovery、moniter拦截器
2. 使用NewServer来新建一个RPC服务端，并默认集成trace、log、recovery、moniter拦截器

4
pkg/net/rpc/warden/balancer/wrr/wrr_test.go
View File

@ -137,8 +137,8 @@ func TestBalancerDone(t *testing.T) {
latency, count := picker.(*wrrPicker).subConns[0].latencySummary()
expectLatency := float64(100*time.Millisecond) / 1e5
if !(expectLatency < latency && latency < (expectLatency+100)) {
t.Fatalf("latency is less than 100ms or greter than 100ms, %f", latency)
if latency < expectLatency || latency > (expectLatency+500) {
t.Fatalf("latency is less than 100ms or greater than 150ms, %f", latency)
}
assert.Equal(t, int64(1), count)

3
pkg/net/rpc/warden/exapmle_test.go
View File

@ -60,9 +60,8 @@ func ExampleClient() {
Timeout: xtime.Duration(time.Second * 10),
Breaker: &breaker.Config{
Window: xtime.Duration(3 * time.Second),
Sleep: xtime.Duration(3 * time.Second),
Bucket: 10,
Ratio: 0.3,
K: 1.5,
Request: 20,
},
})

3
pkg/net/rpc/warden/internal/benchmark/bench/client/client.go
View File

@ -39,10 +39,9 @@ func wardenCli() proto.HelloClient {
Timeout: xtime.Duration(time.Second * 10),
Breaker: &breaker.Config{
Window: xtime.Duration(3 * time.Second),
Sleep: xtime.Duration(3 * time.Second),
Bucket: 10,
Ratio: 0.3,
Request: 20,
K: 1.5,
},
},
grpc.WithInitialWindowSize(iws),

3
pkg/net/rpc/warden/internal/benchmark/helloworld/client/greeter_client.go
View File

@ -21,10 +21,9 @@ var (
Timeout: xtime.Duration(time.Second * 10),
Breaker: &breaker.Config{
Window: xtime.Duration(3 * time.Second),
Sleep: xtime.Duration(3 * time.Second),
Bucket: 10,
Ratio: 0.3,
Request: 20,
K: 1.5,
},
}
cli pb.GreeterClient

48
pkg/net/rpc/warden/internal/pb/ecode.go
View File

@ -1,48 +0,0 @@
package pb
import (
"strconv"
"github.com/bilibili/kratos/pkg/ecode"
any "github.com/golang/protobuf/ptypes/any"
)
func (e *Error) Error() string {
return strconv.FormatInt(int64(e.GetErrCode()), 10)
}
// Code is the code of error.
func (e *Error) Code() int {
return int(e.GetErrCode())
}
// Message is error message.
func (e *Error) Message() string {
return e.GetErrMessage()
}
// Equal compare whether two errors are equal.
func (e *Error) Equal(ec error) bool {
return ecode.Cause(ec).Code() == e.Code()
}
// Details return error details.
func (e *Error) Details() []interface{} {
return []interface{}{e.GetErrDetail()}
}
// From will convert ecode.Codes to pb.Error.
//
// Deprecated: please use ecode.Error
func From(ec ecode.Codes) *Error {
var detail *any.Any
if details := ec.Details(); len(details) > 0 {
detail, _ = details[0].(*any.Any)
}
return &Error{
ErrCode: int32(ec.Code()),
ErrMessage: ec.Message(),
ErrDetail: detail,
}
}

96
pkg/net/rpc/warden/internal/pb/ecode.pb.go
View File

@ -1,96 +0,0 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: error.proto
package pb
import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
import any "github.com/golang/protobuf/ptypes/any"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
// Deprecated: please use ecode.Error
type Error struct {
ErrCode int32 `protobuf:"varint,1,opt,name=err_code,json=errCode,proto3" json:"err_code,omitempty"`
ErrMessage string `protobuf:"bytes,2,opt,name=err_message,json=errMessage,proto3" json:"err_message,omitempty"`
ErrDetail *any.Any `protobuf:"bytes,3,opt,name=err_detail,json=errDetail,proto3" json:"err_detail,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Error) Reset() { *m = Error{} }
func (m *Error) String() string { return proto.CompactTextString(m) }
func (*Error) ProtoMessage() {}
func (*Error) Descriptor() ([]byte, []int) {
return fileDescriptor_error_28aad86a4e53115b, []int{0}
}
func (m *Error) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Error.Unmarshal(m, b)
}
func (m *Error) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Error.Marshal(b, m, deterministic)
}
func (dst *Error) XXX_Merge(src proto.Message) {
xxx_messageInfo_Error.Merge(dst, src)
}
func (m *Error) XXX_Size() int {
return xxx_messageInfo_Error.Size(m)
}
func (m *Error) XXX_DiscardUnknown() {
xxx_messageInfo_Error.DiscardUnknown(m)
}
var xxx_messageInfo_Error proto.InternalMessageInfo
func (m *Error) GetErrCode() int32 {
if m != nil {
return m.ErrCode
}
return 0
}
func (m *Error) GetErrMessage() string {
if m != nil {
return m.ErrMessage
}
return ""
}
func (m *Error) GetErrDetail() *any.Any {
if m != nil {
return m.ErrDetail
}
return nil
}
func init() {
proto.RegisterType((*Error)(nil), "err.Error")
}
func init() { proto.RegisterFile("error.proto", fileDescriptor_error_28aad86a4e53115b) }
var fileDescriptor_error_28aad86a4e53115b = []byte{
// 164 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x34, 0x8d, 0xc1, 0xca, 0x82, 0x40,
0x14, 0x85, 0x99, 0x5f, 0xfc, 0xcb, 0x71, 0x37, 0xb4, 0xd0, 0x36, 0x49, 0x2b, 0x57, 0x23, 0xe4,
0x13, 0x44, 0xb5, 0x6c, 0xe3, 0x0b, 0x88, 0xe6, 0x49, 0x02, 0xf3, 0xc6, 0xd1, 0x20, 0xdf, 0x3e,
0x1c, 0x69, 0x79, 0xcf, 0xf7, 0x71, 0x3f, 0x1d, 0x82, 0x14, 0xda, 0x17, 0x65, 0x14, 0xe3, 0x81,
0xdc, 0xc6, 0xad, 0x48, 0xdb, 0x21, 0x73, 0x53, 0xfd, 0xbe, 0x67, 0x55, 0x3f, 0x2d, 0x7c, 0xff,
0xd1, 0xfe, 0x65, 0xd6, 0x4d, 0xac, 0xd7, 0x20, 0xcb, 0x9b, 0x34, 0x88, 0x54, 0xa2, 0x52, 0xbf,
0x58, 0x81, 0x3c, 0x49, 0x03, 0xb3, 0x73, 0x2f, 0xcb, 0x27, 0x86, 0xa1, 0x6a, 0x11, 0xfd, 0x25,
0x2a, 0x0d, 0x0a, 0x0d, 0xf2, 0xba, 0x2c, 0x26, 0xd7, 0xf3, 0x55, 0x36, 0x18, 0xab, 0x47, 0x17,
0x79, 0x89, 0x4a, 0xc3, 0xc3, 0xc6, 0x2e, 0x51, 0xfb, 0x8b, 0xda, 0x63, 0x3f, 0x15, 0x01, 0xc8,
0xb3, 0xd3, 0xea, 0x7f, 0x07, 0xf2, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xf7, 0x41, 0x22, 0xfd,
0xaf, 0x00, 0x00, 0x00,
}

13
pkg/net/rpc/warden/internal/pb/ecode.proto
View File

@ -1,13 +0,0 @@
syntax = "proto3";
package pb;
import "google/protobuf/any.proto";
option go_package = "github.com/bilibili/kratos/pkg/net/rpc/warden/internal/pb";
message Error {
int32 err_code = 1;
string err_message = 2;
google.protobuf.Any err_detail = 3;
}

39
pkg/net/rpc/warden/internal/status/status.go
View File

@ -4,14 +4,12 @@ import (
"context"
"strconv"
"github.com/bilibili/kratos/pkg/ecode"
"github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes"
"github.com/pkg/errors"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/bilibili/kratos/pkg/ecode"
"github.com/bilibili/kratos/pkg/net/rpc/warden/internal/pb"
)
// togRPCCode convert ecode.Codo to gRPC code
@ -93,9 +91,6 @@ func FromError(svrErr error) (gst *status.Status) {
func gRPCStatusFromEcode(code ecode.Codes) (*status.Status, error) {
var st *ecode.Status
switch v := code.(type) {
// compatible old pb.Error remove it after nobody use pb.Error.
case *pb.Error:
return status.New(codes.Unknown, v.Error()).WithDetails(v)
case *ecode.Status:
st = v
case ecode.Code:
@ -108,40 +103,14 @@ func gRPCStatusFromEcode(code ecode.Codes) (*status.Status, error) {
}
}
}
// gst := status.New(togRPCCode(st), st.Message())
// NOTE: compatible with PHP swoole gRPC put code in status message as string.
// gst := status.New(togRPCCode(st), strconv.Itoa(st.Code()))
gst := status.New(codes.Unknown, strconv.Itoa(st.Code()))
pbe := &pb.Error{ErrCode: int32(st.Code()), ErrMessage: gst.Message()}
// NOTE: server return ecode.Status will be covert to pb.Error details will be ignored
// and put it at details[0] for compatible old client
return gst.WithDetails(pbe, st.Proto())
return gst.WithDetails(st.Proto())
}
// ToEcode convert grpc.status to ecode.Codes
func ToEcode(gst *status.Status) ecode.Codes {
details := gst.Details()
// reverse range details, details may contain three case,
// if details contain pb.Error and ecode.Status use eocde.Status first.
//
// Details layout:
// pb.Error [0: pb.Error]
// both pb.Error and ecode.Status [0: pb.Error, 1: ecode.Status]
// ecode.Status [0: ecode.Status]
for i := len(details) - 1; i >= 0; i-- {
detail := details[i]
// compatible with old pb.Error.
if pe, ok := detail.(*pb.Error); ok {
st := ecode.Error(ecode.Code(pe.ErrCode), pe.ErrMessage)
if pe.ErrDetail != nil {
dynMsg := new(ptypes.DynamicAny)
// TODO deal with unmarshalAny error.
if err := ptypes.UnmarshalAny(pe.ErrDetail, dynMsg); err == nil {
st, _ = st.WithDetails(dynMsg.Message)
}
}
return st
}
for _, detail := range details {
// convert detail to status only use first detail
if pb, ok := detail.(proto.Message); ok {
return ecode.FromProto(pb)

43
pkg/net/rpc/warden/internal/status/status_test.go
View File

@ -7,7 +7,6 @@ import (
"testing"
"time"
"github.com/golang/protobuf/ptypes"
"github.com/golang/protobuf/ptypes/timestamp"
pkgerr "github.com/pkg/errors"
"github.com/stretchr/testify/assert"
@ -15,7 +14,6 @@ import (
"google.golang.org/grpc/status"
"github.com/bilibili/kratos/pkg/ecode"
"github.com/bilibili/kratos/pkg/net/rpc/warden/internal/pb"
)
func TestCodeConvert(t *testing.T) {
@ -89,16 +87,6 @@ func TestFromError(t *testing.T) {
assert.Equal(t, codes.Unknown, gst.Code())
assert.Equal(t, "-504", gst.Message())
})
t.Run("input pb.Error", func(t *testing.T) {
m := &timestamp.Timestamp{Seconds: time.Now().Unix()}
detail, _ := ptypes.MarshalAny(m)
err := &pb.Error{ErrCode: 2233, ErrMessage: "message", ErrDetail: detail}
gst := FromError(err)
assert.Equal(t, codes.Unknown, gst.Code())
assert.Len(t, gst.Details(), 1)
assert.Equal(t, "2233", gst.Message())
})
t.Run("input ecode.Status", func(t *testing.T) {
m := &timestamp.Timestamp{Seconds: time.Now().Unix()}
err, _ := ecode.Error(ecode.Unauthorized, "unauthorized").WithDetails(m)
@ -107,10 +95,9 @@ func TestFromError(t *testing.T) {
//assert.Equal(t, codes.Unauthenticated, gst.Code())
// NOTE: set all grpc.status as Unkown when error is ecode.Codes for compatible
assert.Equal(t, codes.Unknown, gst.Code())
assert.Len(t, gst.Details(), 2)
assert.Len(t, gst.Details(), 1)
details := gst.Details()
assert.IsType(t, &pb.Error{}, details[0])
assert.IsType(t, err.Proto(), details[1])
assert.IsType(t, err.Proto(), details[0])
})
}
@ -123,32 +110,6 @@ func TestToEcode(t *testing.T) {
assert.Equal(t, "-500", ec.Message())
assert.Len(t, ec.Details(), 0)
})
t.Run("input pb.Error", func(t *testing.T) {
m := &timestamp.Timestamp{Seconds: time.Now().Unix()}
detail, _ := ptypes.MarshalAny(m)
gst := status.New(codes.InvalidArgument, "requesterr")
gst, _ = gst.WithDetails(&pb.Error{ErrCode: 1122, ErrMessage: "message", ErrDetail: detail})
ec := ToEcode(gst)
assert.Equal(t, 1122, ec.Code())
assert.Equal(t, "message", ec.Message())
assert.Len(t, ec.Details(), 1)
assert.IsType(t, m, ec.Details()[0])
})
t.Run("input pb.Error and ecode.Status", func(t *testing.T) {
gst := status.New(codes.InvalidArgument, "requesterr")
gst, _ = gst.WithDetails(
&pb.Error{ErrCode: 401, ErrMessage: "message"},
ecode.Errorf(ecode.Unauthorized, "Unauthorized").Proto(),
)
ec := ToEcode(gst)
assert.Equal(t, int(ecode.Unauthorized), ec.Code())
assert.Equal(t, "Unauthorized", ec.Message())
})
t.Run("input encode.Status", func(t *testing.T) {
m := &timestamp.Timestamp{Seconds: time.Now().Unix()}
st, _ := ecode.Errorf(ecode.Unauthorized, "Unauthorized").WithDetails(m)

3
pkg/net/rpc/warden/resolver/direct/test/direct_test.go
View File

@ -55,10 +55,9 @@ func createTestClient(t *testing.T, connStr string) pb.GreeterClient {
Timeout: xtime.Duration(time.Second * 10),
Breaker: &breaker.Config{
Window: xtime.Duration(3 * time.Second),
Sleep: xtime.Duration(3 * time.Second),
Bucket: 10,
Ratio: 0.3,
Request: 20,
K: 1.5,
},
})
conn, err := client.Dial(context.TODO(), connStr)

3
pkg/net/rpc/warden/resolver/test/resovler_test.go
View File

@ -74,10 +74,9 @@ func createTestClient(t *testing.T) pb.GreeterClient {
Timeout: xtime.Duration(time.Second * 10),
Breaker: &breaker.Config{
Window: xtime.Duration(3 * time.Second),
Sleep: xtime.Duration(3 * time.Second),
Bucket: 10,
Ratio: 0.3,
Request: 20,
K: 1.5,
},
})
conn, err := client.Dial(context.TODO(), "mockdiscovery://authority/main.test")

24
pkg/net/rpc/warden/server.go
View File

@ -23,6 +23,7 @@ import (
"github.com/pkg/errors"
"google.golang.org/grpc"
_ "google.golang.org/grpc/encoding/gzip" // NOTE: use grpc gzip by header grpc-accept-encoding
"google.golang.org/grpc/keepalive"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/peer"
@ -110,6 +111,7 @@ func (s *Server) handle() grpc.UnaryServerInterceptor {
var t trace.Trace
cmd := nmd.MD{}
if gmd, ok := metadata.FromIncomingContext(ctx); ok {
t, _ = trace.Extract(trace.GRPCFormat, gmd)
for key, vals := range gmd {
if nmd.IsIncomingKey(key) {
cmd[key] = vals[0]
@ -300,6 +302,26 @@ func (s *Server) RunUnix(file string) error {
// will panic if any error happend
// return server itself
func (s *Server) Start() (*Server, error) {
_, err := s.startWithAddr()
if err != nil {
return nil, err
}
return s, nil
}
// StartWithAddr create a new goroutine run server with configured listen addr
// will panic if any error happend
// return server itself and the actually listened address (if configured listen
// port is zero, the os will allocate an unused port)
func (s *Server) StartWithAddr() (*Server, net.Addr, error) {
addr, err := s.startWithAddr()
if err != nil {
return nil, nil, err
}
return s, addr, nil
}
func (s *Server) startWithAddr() (net.Addr, error) {
lis, err := net.Listen(s.conf.Network, s.conf.Addr)
if err != nil {
return nil, err
@ -311,7 +333,7 @@ func (s *Server) Start() (*Server, error) {
panic(err)
}
}()
return s, nil
return lis.Addr(), nil
}
// Serve accepts incoming connections on the listener lis, creating a new

25
pkg/net/rpc/warden/server_test.go
View File

@ -41,11 +41,9 @@ var (
Dial: xtime.Duration(time.Second * 10),
Timeout: xtime.Duration(time.Second * 10),
Breaker: &breaker.Config{
Window: xtime.Duration(3 * time.Second),
Sleep: xtime.Duration(3 * time.Second),
Bucket: 10,
Ratio: 0.3,
Request: 20,
Window: xtime.Duration(3 * time.Second),
Bucket: 10,
K: 1.5,
},
}
clientConfig2 = ClientConfig{
@ -53,10 +51,9 @@ var (
Timeout: xtime.Duration(time.Second * 10),
Breaker: &breaker.Config{
Window: xtime.Duration(3 * time.Second),
Sleep: xtime.Duration(3 * time.Second),
Bucket: 10,
Ratio: 0.3,
Request: 20,
K: 1.5,
},
Method: map[string]*ClientConfig{`/testproto.Greeter/SayHello`: {Timeout: xtime.Duration(time.Millisecond * 200)}},
}
@ -293,7 +290,7 @@ func testBreaker(t *testing.T) {
}
defer conn.Close()
c := pb.NewGreeterClient(conn)
for i := 0; i < 50; i++ {
for i := 0; i < 1000; i++ {
_, err := c.SayHello(context.Background(), &pb.HelloRequest{Name: "breaker_test"})
if err != nil {
if ecode.EqualError(ecode.ServiceUnavailable, err) {
@ -594,3 +591,15 @@ func TestMetadata(t *testing.T) {
_, err := cli.SayHello(ctx, &pb.HelloRequest{Name: "test"})
assert.Nil(t, err)
}
func TestStartWithAddr(t *testing.T) {
configuredAddr := "127.0.0.1:0"
server = NewServer(&ServerConfig{Addr: configuredAddr, Timeout: xtime.Duration(time.Second)})
if _, realAddr, err := server.StartWithAddr(); err == nil && realAddr != nil {
assert.NotEqual(t, realAddr.String(), configuredAddr)
} else {
assert.NotNil(t, realAddr)
assert.Nil(t, err)
}
}

4
pkg/net/trace/sample_test.go
View File

@ -21,8 +21,8 @@ func TestProbabilitySampling(t *testing.T) {
count++
}
}
if count < 60 || count > 120 {
t.Errorf("expect count between 60~120 get %d", count)
if count < 60 || count > 150 {
t.Errorf("expect count between 60~150 get %d", count)
}
})
}

2
pkg/net/trace/zipkin/zipkin_test.go
View File

@ -37,7 +37,7 @@ func TestZipkin(t *testing.T) {
t2 := trace.NewTracer("service2", report, true)
sp1 := t1.New("option_1")
sp2 := sp1.Fork("service3", "opt_client")
sp2.SetLog(trace.Log("log_k","log_v"))
sp2.SetLog(trace.Log("log_k", "log_v"))
// inject
header := make(http.Header)
t1.Inject(sp2, trace.HTTPFormat, header)

2
pkg/ratelimit/bbr/bbr.go
View File

@ -32,7 +32,7 @@ func init() {
go cpuproc()
}
// cpu = cpuᵗ⁻¹ * decay + cpuᵗ * (1 - decay)
// cpu = cpuᵗ⁻¹ * decay + cpuᵗ * (1 - decay)
func cpuproc() {
ticker := time.NewTicker(time.Millisecond * 250)
defer func() {

5
pkg/stat/sys/cpu/cgroup.go
View File

@ -56,7 +56,10 @@ func (c *cgroup) CPUAcctUsagePerCPU() ([]uint64, error) {
if u, err = parseUint(v); err != nil {
return nil, err
}
usage = append(usage, u)
// fix possible_cpu:https://www.ibm.com/support/knowledgecenter/en/linuxonibm/com.ibm.linux.z.lgdd/lgdd_r_posscpusparm.html
if u != 0 {
usage = append(usage, u)
}
}
return usage, nil
}

18
pkg/stat/sys/cpu/cgroupCPU.go
View File

@ -8,6 +8,7 @@ import (
"strings"
"github.com/pkg/errors"
pscpu "github.com/shirou/gopsutil/cpu"
)
type cgroupCPU struct {
@ -21,12 +22,17 @@ type cgroupCPU struct {
}
func newCgroupCPU() (cpu *cgroupCPU, err error) {
cpus, err := perCPUUsage()
if err != nil {
err = errors.Errorf("perCPUUsage() failed!err:=%v", err)
return
var cores int
cores, err = pscpu.Counts(true)
if err != nil || cores == 0 {
var cpus []uint64
cpus, err = perCPUUsage()
if err != nil {
err = errors.Errorf("perCPUUsage() failed!err:=%v", err)
return
}
cores = len(cpus)
}
cores := uint64(len(cpus))
sets, err := cpuSets()
if err != nil {
@ -61,7 +67,7 @@ func newCgroupCPU() (cpu *cgroupCPU, err error) {
cpu = &cgroupCPU{
frequency: maxFreq,
quota: quota,
cores: cores,
cores: uint64(cores),
preSystem: preSystem,
preTotal: preTotal,
}

9
pkg/sync/pipeline/CHANGELOG.md Executable file
View File

@ -0,0 +1,9 @@
### pipeline
#### Version 1.2.0
> 1. 默认为平滑触发事件
> 2. 增加metric上报
#### Version 1.1.0
> 1. 增加平滑时间的支持
#### Version 1.0.0
> 1. 提供聚合方法 内部区分压测流量

6
pkg/sync/pipeline/fanout/CHANGELOG.md Executable file
View File

@ -0,0 +1,6 @@
### pipeline/fanout
#### Version 1.1.0
> 1. 增加处理速度metric上报
#### Version 1.0.0
> 1. library/cache包改为fanout

7
pkg/sync/pipeline/fanout/fanout.go
View File

@ -15,8 +15,8 @@ var (
// ErrFull chan full.
ErrFull = errors.New("fanout: chan full")
traceTags = []trace.Tag{
trace.Tag{Key: trace.TagSpanKind, Value: "background"},
trace.Tag{Key: trace.TagComponent, Value: "sync/pipeline/fanout"},
{Key: trace.TagSpanKind, Value: "background"},
{Key: trace.TagComponent, Value: "sync/pipeline/fanout"},
}
)
@ -67,7 +67,7 @@ type Fanout struct {
// New new a fanout struct.
func New(name string, opts ...Option) *Fanout {
if name == "" {
name = "fanout"
name = "anonymous"
}
o := &options{
worker: 1,
@ -96,6 +96,7 @@ func (c *Fanout) proc() {
case t := <-c.ch:
wrapFunc(t.f)(t.ctx)
_metricChanSize.Set(float64(len(c.ch)), c.name)
_metricCount.Inc(c.name)
case <-c.ctx.Done():
return
}

22
pkg/sync/pipeline/fanout/metrics.go
View File

@ -1,15 +1,27 @@
package fanout
import "github.com/bilibili/kratos/pkg/stat/metric"
import (
"github.com/bilibili/kratos/pkg/stat/metric"
)
const namespace = "sync"
const (
_metricNamespace = "sync"
_metricSubSystem = "pipeline_fanout"
)
var (
_metricChanSize = metric.NewGaugeVec(&metric.GaugeVecOpts{
Namespace: namespace,
Subsystem: "pipeline_fanout",
Name: "current",
Namespace: _metricNamespace,
Subsystem: _metricSubSystem,
Name: "chan_len",
Help: "sync pipeline fanout current channel size.",
Labels: []string{"name"},
})
_metricCount = metric.NewCounterVec(&metric.CounterVecOpts{
Namespace: _metricNamespace,
Subsystem: _metricSubSystem,
Name: "process_count",
Help: "process count",
Labels: []string{"name"},
})
)

40
pkg/sync/pipeline/pipeline.go
View File

@ -3,16 +3,38 @@ package pipeline
import (
"context"
"errors"
"strconv"
"sync"
"time"
"github.com/bilibili/kratos/pkg/net/metadata"
"github.com/bilibili/kratos/pkg/stat/metric"
xtime "github.com/bilibili/kratos/pkg/time"
)
// ErrFull channel full error
var ErrFull = errors.New("channel full")
const _metricNamespace = "sync"
const _metricSubSystem = "pipeline"
var (
_metricCount = metric.NewCounterVec(&metric.CounterVecOpts{
Namespace: _metricNamespace,
Subsystem: _metricSubSystem,
Name: "process_count",
Help: "process count",
Labels: []string{"name", "chan"},
})
_metricChanLen = metric.NewGaugeVec(&metric.GaugeVecOpts{
Namespace: _metricNamespace,
Subsystem: _metricSubSystem,
Name: "chan_len",
Help: "channel length",
Labels: []string{"name", "chan"},
})
)
type message struct {
key string
value interface{}
@ -26,6 +48,7 @@ type Pipeline struct {
mirrorChans []chan *message
config *Config
wait sync.WaitGroup
name string
}
// Config Pipeline config
@ -38,8 +61,8 @@ type Config struct {
Buffer int
// Worker channel number
Worker int
// Smooth smoothing interval
Smooth bool
// Name use for metrics
Name string
}
func (c *Config) fix() {
@ -55,6 +78,9 @@ func (c *Config) fix() {
if c.Worker <= 0 {
c.Worker = 10
}
if c.Name == "" {
c.Name = "anonymous"
}
}
// NewPipeline new pipline
@ -67,6 +93,7 @@ func NewPipeline(config *Config) (res *Pipeline) {
chans: make([]chan *message, config.Worker),
mirrorChans: make([]chan *message, config.Worker),
config: config,
name: config.Name,
}
for i := 0; i < config.Worker; i++ {
res.chans[i] = make(chan *message, config.Buffer)
@ -144,7 +171,7 @@ func (p *Pipeline) mergeproc(mirror bool, index int, ch <-chan *message) {
inteval = p.config.Interval
oldTicker = true
)
if p.config.Smooth && index > 0 {
if index > 0 {
inteval = xtime.Duration(int64(index) * (int64(p.config.Interval) / int64(p.config.Worker)))
}
ticker := time.NewTicker(time.Duration(inteval))
@ -162,21 +189,26 @@ func (p *Pipeline) mergeproc(mirror bool, index int, ch <-chan *message) {
}
continue
case <-ticker.C:
if p.config.Smooth && oldTicker {
if oldTicker {
ticker.Stop()
ticker = time.NewTicker(time.Duration(p.config.Interval))
oldTicker = false
}
}
name := p.name
process := count
if len(vals) > 0 {
ctx := context.Background()
if mirror {
ctx = metadata.NewContext(ctx, metadata.MD{metadata.Mirror: "1"})
name = "mirror_" + name
}
p.Do(ctx, index, vals)
vals = make(map[string][]interface{}, p.config.MaxSize)
count = 0
}
_metricChanLen.Set(float64(len(ch)), name, strconv.Itoa(index))
_metricCount.Add(float64(process), name, strconv.Itoa(index))
if closed {
ticker.Stop()
return

1
pkg/sync/pipeline/pipeline_test.go
View File

@ -92,7 +92,6 @@ func TestPipelineSmooth(t *testing.T) {
Interval: xtime.Duration(time.Second),
Buffer: 100,
Worker: 10,
Smooth: true,
}
type result struct {
index int

4
pkg/testing/lich/README.md Normal file
View File

@ -0,0 +1,4 @@
## testing/lich 运行环境构建
基于 docker-compose 实现跨平台跨语言环境的容器依赖管理方案，以解决运行ut场景下的 (mysql, redis, mc)容器依赖问题。
使用说明参见：https://github.com/bilibili/kratos/tree/master/tool/testcli/README.md

125
pkg/testing/lich/composer.go Normal file
View File

@ -0,0 +1,125 @@
package lich
import (
"bytes"
"crypto/md5"
"encoding/json"
"flag"
"fmt"
"os"
"os/exec"
"path/filepath"
"time"
"github.com/bilibili/kratos/pkg/log"
)
var (
retry int
noDown bool
yamlPath string
pathHash string
services map[string]*Container
)
func init() {
flag.StringVar(&yamlPath, "f", "docker-compose.yaml", "composer yaml path.")
flag.BoolVar(&noDown, "nodown", false, "containers are not recycled.")
}
func runCompose(args ...string) (output []byte, err error) {
if _, err = os.Stat(yamlPath); os.IsNotExist(err) {
log.Error("os.Stat(%s) composer yaml is not exist!", yamlPath)
return
}
if yamlPath, err = filepath.Abs(yamlPath); err != nil {
log.Error("filepath.Abs(%s) error(%v)", yamlPath, err)
return
}
pathHash = fmt.Sprintf("%x", md5.Sum([]byte(yamlPath)))[:9]
args = append([]string{"-f", yamlPath, "-p", pathHash}, args...)
if output, err = exec.Command("docker-compose", args...).CombinedOutput(); err != nil {
log.Error("exec.Command(docker-compose) args(%v) stdout(%s) error(%v)", args, string(output), err)
return
}
return
}
// Setup setup UT related environment dependence for everything.
func Setup() (err error) {
if _, err = runCompose("up", "-d"); err != nil {
return
}
defer func() {
if err != nil {
Teardown()
}
}()
if _, err = getServices(); err != nil {
return
}
_, err = checkServices()
return
}
// Teardown unsetup all environment dependence.
func Teardown() (err error) {
if !noDown {
_, err = runCompose("down")
}
return
}
func getServices() (output []byte, err error) {
if output, err = runCompose("config", "--services"); err != nil {
return
}
services = make(map[string]*Container)
output = bytes.TrimSpace(output)
for _, svr := range bytes.Split(output, []byte("\n")) {
if output, err = runCompose("ps", "-a", "-q", string(svr)); err != nil {
return
}
var (
id = string(bytes.TrimSpace(output))
args = []string{"inspect", id, "--format", "'{{json .}}'"}
)
if output, err = exec.Command("docker", args...).CombinedOutput(); err != nil {
log.Error("exec.Command(docker) args(%v) stdout(%s) error(%v)", args, string(output), err)
return
}
if output = bytes.TrimSpace(output); bytes.Equal(output, []byte("")) {
err = fmt.Errorf("service: %s | container: %s fails to launch", svr, id)
log.Error("exec.Command(docker) args(%v) error(%v)", args, err)
return
}
var c = &Container{}
if err = json.Unmarshal(bytes.Trim(output, "'"), c); err != nil {
log.Error("json.Unmarshal(%s) error(%v)", string(output), err)
return
}
services[string(svr)] = c
}
return
}
func checkServices() (output []byte, err error) {
defer func() {
if err != nil && retry < 4 {
retry++
getServices()
time.Sleep(time.Second * 5)
output, err = checkServices()
return
}
retry = 0
}()
for svr, c := range services {
if err = c.Healthcheck(); err != nil {
log.Error("healthcheck(%s) error(%v) retrying %d times...", svr, err, 5-retry)
return
}
// TODO About container check and more...
}
return
}

85
pkg/testing/lich/healthcheck.go Normal file
View File

@ -0,0 +1,85 @@
package lich
import (
"database/sql"
"fmt"
"net"
"strconv"
"strings"
"github.com/bilibili/kratos/pkg/log"
// Register go-sql-driver stuff
_ "github.com/go-sql-driver/mysql"
)
var healthchecks = map[string]func(*Container) error{"mysql": checkMysql, "mariadb": checkMysql}
// Healthcheck check container health.
func (c *Container) Healthcheck() (err error) {
if status, health := c.State.Status, c.State.Health.Status; !c.State.Running || (health != "" && health != "healthy") {
err = fmt.Errorf("service: %s | container: %s not running", c.GetImage(), c.GetID())
log.Error("docker status(%s) health(%s) error(%v)", status, health, err)
return
}
if check, ok := healthchecks[c.GetImage()]; ok {
err = check(c)
return
}
for proto, ports := range c.NetworkSettings.Ports {
if id := c.GetID(); !strings.Contains(proto, "tcp") {
log.Error("container: %s proto(%s) unsupported.", id, proto)
continue
}
for _, publish := range ports {
var (
ip = net.ParseIP(publish.HostIP)
port, _ = strconv.Atoi(publish.HostPort)
tcpAddr = &net.TCPAddr{IP: ip, Port: port}
tcpConn *net.TCPConn
)
if tcpConn, err = net.DialTCP("tcp", nil, tcpAddr); err != nil {
log.Error("net.DialTCP(%s:%s) error(%v)", publish.HostIP, publish.HostPort, err)
return
}
tcpConn.Close()
}
}
return
}
func checkMysql(c *Container) (err error) {
var ip, port, user, passwd string
for _, env := range c.Config.Env {
splits := strings.Split(env, "=")
if strings.Contains(splits[0], "MYSQL_ROOT_PASSWORD") {
user, passwd = "root", splits[1]
continue
}
if strings.Contains(splits[0], "MYSQL_ALLOW_EMPTY_PASSWORD") {
user, passwd = "root", ""
continue
}
if strings.Contains(splits[0], "MYSQL_USER") {
user = splits[1]
continue
}
if strings.Contains(splits[0], "MYSQL_PASSWORD") {
passwd = splits[1]
continue
}
}
var db *sql.DB
if ports, ok := c.NetworkSettings.Ports["3306/tcp"]; ok {
ip, port = ports[0].HostIP, ports[0].HostPort
}
var dsn = fmt.Sprintf("%s:%s@tcp(%s:%s)/", user, passwd, ip, port)
if db, err = sql.Open("mysql", dsn); err != nil {
log.Error("sql.Open(mysql) dsn(%s) error(%v)", dsn, err)
return
}
if err = db.Ping(); err != nil {
log.Error("ping(db) dsn(%s) error(%v)", dsn, err)
}
defer db.Close()
return
}

88
pkg/testing/lich/model.go Normal file
View File

@ -0,0 +1,88 @@
package lich
import (
"strings"
"time"
)
// Container docker inspect resp.
type Container struct {
ID string `json:"Id"`
Created time.Time `json:"Created"`
Path string `json:"Path"`
Args []string `json:"Args"`
State struct {
Status string `json:"Status"`
Running bool `json:"Running"`
Paused bool `json:"Paused"`
Restarting bool `json:"Restarting"`
OOMKilled bool `json:"OOMKilled"`
Dead bool `json:"Dead"`
Pid int `json:"Pid"`
ExitCode int `json:"ExitCode"`
Error string `json:"Error"`
StartedAt time.Time `json:"StartedAt"`
FinishedAt time.Time `json:"FinishedAt"`
Health struct {
Status string `json:"Status"`
FailingStreak int `json:"FailingStreak"`
Log []struct {
Start time.Time `json:"Start"`
End time.Time `json:"End"`
ExitCode int `json:"ExitCode"`
Output string `json:"Output"`
} `json:"Log"`
} `json:"Health"`
} `json:"State"`
Config struct {
Hostname string `json:"Hostname"`
Domainname string `json:"Domainname"`
User string `json:"User"`
Tty bool `json:"Tty"`
OpenStdin bool `json:"OpenStdin"`
StdinOnce bool `json:"StdinOnce"`
Env []string `json:"Env"`
Cmd []string `json:"Cmd"`
Image string `json:"Image"`
WorkingDir string `json:"WorkingDir"`
Entrypoint []string `json:"Entrypoint"`
} `json:"Config"`
Image string `json:"Image"`
ResolvConfPath string `json:"ResolvConfPath"`
HostnamePath string `json:"HostnamePath"`
HostsPath string `json:"HostsPath"`
LogPath string `json:"LogPath"`
Name string `json:"Name"`
RestartCount int `json:"RestartCount"`
Driver string `json:"Driver"`
Platform string `json:"Platform"`
MountLabel string `json:"MountLabel"`
ProcessLabel string `json:"ProcessLabel"`
AppArmorProfile string `json:"AppArmorProfile"`
NetworkSettings struct {
Bridge string `json:"Bridge"`
SandboxID string `json:"SandboxID"`
HairpinMode bool `json:"HairpinMode"`
Ports map[string][]struct {
HostIP string `json:"HostIp"`
HostPort string `json:"HostPort"`
} `json:"Ports"`
} `json:"NetworkSettings"`
}
// GetImage get image name at container
func (c *Container) GetImage() (image string) {
image = c.Config.Image
if images := strings.Split(image, ":"); len(images) > 0 {
image = images[0]
}
return
}
// GetID get id at container
func (c *Container) GetID() (id string) {
if id = c.ID; len(id) > 9 {
id = id[0:9]
}
return
}

Some files were not shown because too many files have changed in this diff Show More