go/kratos
1
0
Fork 0
mirror of https://github.com/go-kratos/kratos.git synced 2025-02-13 13:48:51 +02:00
Code Issues Releases Activity

Merge pull request #233 from bilibili/library/bts_prom

Browse Source 
Library/bts fix bts prometheus
This commit is contained in:
Tony 2019-07-23 22:25:08 +08:00 committed by GitHub
commit 040c045951
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 48 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
pkg/cache/metrics.go vendored Normal file
View File

@ -0,0 +1,22 @@
package cache
import "github.com/bilibili/kratos/pkg/stat/metric"
const _metricNamespace = "cache"
var (
MetricHits = metric.NewCounterVec(&metric.CounterVecOpts{
Namespace: _metricNamespace,
Subsystem: "",
Name: "hits_total",
Help: "cache hits total.",
Labels: []string{"name"},
})
MetricMisses = metric.NewCounterVec(&metric.CounterVecOpts{
Namespace: _metricNamespace,
Subsystem: "",
Name: "misses_total",
Help: "cache misses total.",
Labels: []string{"name"},
})
)

4
tool/kratos-gen-bts/header_template.go
View File

@ -17,7 +17,7 @@ import (
"context"
{{if .EnableBatch }}"sync"{{end}}
NEWLINE
"github.com/bilibili/kratos/pkg/stat/metric"
"github.com/bilibili/kratos/pkg/cache"
{{if .EnableBatch }}"github.com/bilibili/kratos/pkg/sync/errgroup"{{end}}
{{.ImportPackage}}
NEWLINE
@ -26,8 +26,6 @@ NEWLINE
var (
_ _bts
_metricHits = metric.NewBusinessMetricCount("hits_total", "name")
_metricMisses = metric.NewBusinessMetricCount("misses_total", "name")
)
{{if .EnableSingleFlight}}
var cacheSingleFlights = [SFCOUNT]*singleflight.Group{SFINIT}

8
tool/kratos-gen-bts/multi_template.go
View File

@ -26,7 +26,7 @@ func (d *Dao) NAME(c context.Context, {{.IDName}} []KEY{{.ExtraArgsType}}) (res
miss = append(miss, key)
}
}
_metricHits.Add(float64(len({{.IDName}}) - len(miss)), "NAME")
cache.MetricHits.Add(float64(len({{.IDName}}) - len(miss)), "bts:NAME")
{{if .EnableNullCache}}
for k, v := range res {
{{if .SimpleValue}} if v == {{.NullCache}} { {{else}} if {{.CheckNullCode}} { {{end}}
@ -47,14 +47,14 @@ func (d *Dao) NAME(c context.Context, {{.IDName}} []KEY{{.ExtraArgsType}}) (res
var rr interface{}
sf := d.cacheSFNAME({{.IDName}} {{.ExtraArgs}})
rr, err, _ = cacheSingleFlights[SFNUM].Do(sf, func() (r interface{}, e error) {
_metricMisses.Add(float64(len(miss)), "NAME")
cache.MetricMisses.Add(float64(len(miss)), "bts:NAME")
r, e = RAWFUNC(c, miss {{.ExtraRawArgs}})
return
})
missData = rr.(map[KEY]VALUE)
{{else}}
{{if .EnableBatch}}
_metricMisses.Add(int64(missLen), "NAME")
cache.MetricMisses.Add(float64(missLen), "bts:NAME")
var mutex sync.Mutex
{{if .BatchErrBreak}}
group := errgroup.WithCancel(c)
@ -87,7 +87,7 @@ func (d *Dao) NAME(c context.Context, {{.IDName}} []KEY{{.ExtraArgsType}}) (res
}
err = group.Wait()
{{else}}
_metricMisses.Add(int64(len(miss)), "NAME")
cache.MetricMisses.Add(float64(len(miss)), "bts:NAME")
missData, err = RAWFUNC(c, miss {{.ExtraRawArgs}})
{{end}}
{{end}}

6
tool/kratos-gen-bts/none_template.go
View File

@ -21,20 +21,20 @@ func (d *Dao) NAME(c context.Context) (res VALUE, err error) {
{{else}}
if res != {{.ZeroValue}} {
{{end}}
_metricHits.Incr("NAME")
cache.MetricHits.Inc("bts:NAME")
return
}
{{if .EnableSingleFlight}}
var rr interface{}
sf := d.cacheSFNAME()
rr, err, _ = cacheSingleFlights[SFNUM].Do(sf, func() (r interface{}, e error) {
_metricMisses.Incr("NAME")
cache.MetricMisses.Inc("bts:NAME")
r, e = RAWFUNC(c)
return
})
res = rr.(VALUE)
{{else}}
_metricMisses.Incr("NAME")
cache.MetricMisses.Inc("bts:NAME")
res, err = RAWFUNC(c)
{{end}}
if err != nil {

6
tool/kratos-gen-bts/single_template.go
View File

@ -21,7 +21,7 @@ func (d *Dao) NAME(c context.Context, {{.IDName}} KEY{{.ExtraArgsType}}) (res VA
{{else}}
if res != {{.ZeroValue}} {
{{end}}
prom.CacheHit.Incr("NAME")
cache.MetricHits.Inc("bts:NAME")
return
}
{{if .EnablePaging}}
@ -31,7 +31,7 @@ func (d *Dao) NAME(c context.Context, {{.IDName}} KEY{{.ExtraArgsType}}) (res VA
var rr interface{}
sf := d.cacheSFNAME({{.IDName}} {{.ExtraArgs}})
rr, err, _ = cacheSingleFlights[SFNUM].Do(sf, func() (r interface{}, e error) {
prom.CacheMiss.Incr("NAME")
cache.MetricMisses.Inc("bts:NAME")
{{if .EnablePaging}}
var rrs [2]interface{}
rrs[0], rrs[1], e = RAWFUNC(c, {{.IDName}} {{.ExtraRawArgs}})
@ -48,7 +48,7 @@ func (d *Dao) NAME(c context.Context, {{.IDName}} KEY{{.ExtraArgsType}}) (res VA
res = rr.(VALUE)
{{end}}
{{else}}
prom.CacheMiss.Incr("NAME")
cache.MetricMisses.Inc("bts:NAME")
{{if .EnablePaging}}
res, miss, err = RAWFUNC(c, {{.IDName}} {{.ExtraRawArgs}})
{{else}}

26
tool/kratos-gen-bts/testdata/dao.bts.go vendored
View File

@ -23,14 +23,12 @@ import (
"context"
"sync"
"github.com/bilibili/kratos/pkg/stat/metric"
"github.com/bilibili/kratos/pkg/cache"
"github.com/bilibili/kratos/pkg/sync/errgroup"
)
var (
_ _bts
_metricHits = metric.NewBusinessMetricCount("hits_total", "NAME")
_metricMisses = metric.NewBusinessMetricCount("misses_total", "NAME")
_ _bts
)
// Demos get data from cache if miss will call source method, then add to cache.
@ -50,7 +48,7 @@ func (d *Dao) Demos(c context.Context, keys []int64) (res map[int64]*Demo, err e
miss = append(miss, key)
}
}
_metricHits.Add(float64(len(keys)-len(miss)), "Demos")
cache.MetricHits.Add(float64(len(keys)-len(miss)), "bts:Demos")
for k, v := range res {
if v.ID == -1 {
delete(res, k)
@ -61,7 +59,7 @@ func (d *Dao) Demos(c context.Context, keys []int64) (res map[int64]*Demo, err e
return
}
missData := make(map[int64]*Demo, missLen)
_metricMisses.Add(int64(missLen), "Demos")
cache.MetricMisses.Add(float64(missLen), "bts:Demos")
var mutex sync.Mutex
group := errgroup.WithCancel(c)
if missLen > 20 {
@ -129,7 +127,7 @@ func (d *Dao) Demos1(c context.Context, keys []int64) (res map[int64]*Demo, err
miss = append(miss, key)
}
}
_metricHits.Add(float64(len(keys)-len(miss)), "Demos1")
cache.MetricHits.Add(float64(len(keys)-len(miss)), "bts:Demos1")
for k, v := range res {
if v.ID == -1 {
delete(res, k)
@ -140,7 +138,7 @@ func (d *Dao) Demos1(c context.Context, keys []int64) (res map[int64]*Demo, err
return
}
missData := make(map[int64]*Demo, missLen)
_metricMisses.Add(int64(missLen), "Demos1")
cache.MetricMisses.Add(float64(missLen), "bts:Demos1")
var mutex sync.Mutex
group := errgroup.WithContext(c)
if missLen > 20 {
@ -205,10 +203,10 @@ func (d *Dao) Demo(c context.Context, key int64) (res *Demo, err error) {
}
}()
if res != nil {
prom.CacheHit.Incr("Demo")
cache.MetricHits.Inc("bts:Demo")
return
}
prom.CacheMiss.Incr("Demo")
cache.MetricMisses.Inc("bts:Demo")
res, err = d.RawDemo(c, key)
if err != nil {
return
@ -233,11 +231,11 @@ func (d *Dao) Demo1(c context.Context, key int64, pn int, ps int) (res *Demo, er
err = nil
}
if res != nil {
prom.CacheHit.Incr("Demo1")
cache.MetricHits.Inc("bts:Demo1")
return
}
var miss *Demo
prom.CacheMiss.Incr("Demo1")
cache.MetricMisses.Inc("bts:Demo1")
res, miss, err = d.RawDemo1(c, key, pn, ps)
if err != nil {
return
@ -265,10 +263,10 @@ func (d *Dao) None(c context.Context) (res *Demo, err error) {
}
}()
if res != nil {
_metricHits.Incr("None")
cache.MetricHits.Inc("bts:None")
return
}
_metricMisses.Incr("None")
cache.MetricMisses.Inc("bts:None")
res, err = d.RawNone(c)
if err != nil {
return

2
tool/kratos-gen-mc/header_template.go
View File

@ -19,7 +19,6 @@ import (
{{if .UseStrConv}}"strconv"{{end}}
{{if .EnableBatch }}"sync"{{end}}
NEWLINE
"github.com/bilibili/kratos/pkg/stat/metric"
{{if .UseMemcached }}"github.com/bilibili/kratos/pkg/cache/memcache"{{end}}
{{if .EnableBatch }}"github.com/bilibili/kratos/pkg/sync/errgroup"{{end}}
"github.com/bilibili/kratos/pkg/log"
@ -28,6 +27,5 @@ NEWLINE
var (
_ _mc
_metricErrCount = metric.NewBusinessMetricCount("mc_error_total", "NAME")
)
`

8
tool/kratos-gen-mc/multi_template.go
View File

@ -44,7 +44,6 @@ func (d *{{.StructName}}) NAME(c context.Context, ids []KEY {{.ExtraArgsType}})
}
replies, err := d.mc.GetMulti(c, keys)
if err != nil {
_metricErrCount.Inc("NAME")
log.Errorv(ctx, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("keys", keys))
return
}
@ -67,14 +66,12 @@ func (d *{{.StructName}}) NAME(c context.Context, ids []KEY {{.ExtraArgsType}})
{{end}}
{{end}}
if err != nil {
_metricErrCount.Inc("NAME")
log.Errorv(ctx, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
{{if .GetSimpleValue}}
r, err := {{.ConvertBytes2Value}}
if err != nil {
_metricErrCount.Inc("NAME")
log.Errorv(ctx, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("key", key))
return res, err
}
@ -114,7 +111,6 @@ func (d *{{.StructName}}) NAME(c context.Context, ids []KEY {{.ExtraArgsType}})
}
replies, err := d.mc.GetMulti(c, keys)
if err != nil {
_metricErrCount.Inc("NAME")
log.Errorv(c, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("keys", keys))
return
}
@ -132,14 +128,12 @@ func (d *{{.StructName}}) NAME(c context.Context, ids []KEY {{.ExtraArgsType}})
{{end}}
{{end}}
if err != nil {
_metricErrCount.Inc("NAME")
log.Errorv(c, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
{{if .GetSimpleValue}}
r, err := {{.ConvertBytes2Value}}
if err != nil {
_metricErrCount.Inc("NAME")
log.Errorv(c, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("key", key))
return res, err
}
@ -174,7 +168,6 @@ func (d *{{.StructName}}) NAME(c context.Context, values map[KEY]VALUE {{.ExtraA
item := &memcache.Item{Key: key, Object: val, Expiration: {{.ExpireCode}}, Flags: {{.Encode}}}
{{end}}
if err = d.mc.Set(c, item); err != nil {
_metricErrCount.Inc("NAME")
log.Errorv(c, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -198,7 +191,6 @@ func (d *{{.StructName}}) NAME(c context.Context, ids []KEY {{.ExtraArgsType}})
err = nil
continue
}
_metricErrCount.Inc("NAME")
log.Errorv(c, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}

4
tool/kratos-gen-mc/none_template.go
View File

@ -37,14 +37,12 @@ func (d *{{.StructName}}) NAME(c context.Context) (res VALUE, err error) {
return
}
{{end}}
_metricErrCount.Inc("NAME")
log.Errorv(c, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
{{if .GetSimpleValue}}
r, err := {{.ConvertBytes2Value}}
if err != nil {
_metricErrCount.Inc("NAME")
log.Errorv(c, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -75,7 +73,6 @@ func (d *{{.StructName}}) NAME(c context.Context, val VALUE) (err error) {
item := &memcache.Item{Key: key, Object: val, Expiration: {{.ExpireCode}}, Flags: {{.Encode}}}
{{end}}
if err = d.mc.Set(c, item); err != nil {
_metricErrCount.Inc("NAME")
log.Errorv(c, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -94,7 +91,6 @@ func (d *{{.StructName}}) NAME(c context.Context) (err error) {
err = nil
return
}
_metricErrCount.Inc("NAME")
log.Errorv(c, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}

4
tool/kratos-gen-mc/single_template.go
View File

@ -36,14 +36,12 @@ func (d *{{.StructName}}) NAME(c context.Context, id KEY {{.ExtraArgsType}}) (re
return
}
{{end}}
_metricErrCount.Inc("NAME")
log.Errorv(c, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
{{if .GetSimpleValue}}
r, err := {{.ConvertBytes2Value}}
if err != nil {
_metricErrCount.Incr("NAME")
log.Errorv(c, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -74,7 +72,6 @@ func (d *{{.StructName}}) NAME(c context.Context, id KEY, val VALUE {{.ExtraArgs
item := &memcache.Item{Key: key, Object: val, Expiration: {{.ExpireCode}}, Flags: {{.Encode}}}
{{end}}
if err = d.mc.Set(c, item); err != nil {
_metricErrCount.Incr("NAME")
log.Errorv(c, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -93,7 +90,6 @@ func (d *{{.StructName}}) NAME(c context.Context, id KEY {{.ExtraArgsType}}) (er
err = nil
return
}
_metricErrCount.Incr("NAME")
log.Errorv(c, log.KV("NAME", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}

20
tool/kratos-gen-mc/testdata/mc.cache.go vendored
View File

@ -48,12 +48,10 @@ import (
"github.com/bilibili/kratos/pkg/cache/memcache"
"github.com/bilibili/kratos/pkg/log"
"github.com/bilibili/kratos/pkg/stat/metric"
)
var (
_ _mc
_metricErrCount = metric.NewBusinessMetricCount("mc_error_total", "NAME")
_ _mc
)
// CacheDemos get data from mc
@ -71,7 +69,6 @@ func (d *Dao) CacheDemos(c context.Context, ids []int64) (res map[int64]*Demo, e
}
replies, err := d.mc.GetMulti(c, keys)
if err != nil {
_metricErrCount.Inc("CacheDemos")
log.Errorv(c, log.KV("CacheDemos", fmt.Sprintf("%+v", err)), log.KV("keys", keys))
return
}
@ -79,7 +76,6 @@ func (d *Dao) CacheDemos(c context.Context, ids []int64) (res map[int64]*Demo, e
v := &Demo{}
err = replies.Scan(key, v)
if err != nil {
_metricErrCount.Inc("CacheDemos")
log.Errorv(c, log.KV("CacheDemos", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -102,7 +98,6 @@ func (d *Dao) CacheDemo(c context.Context, id int64) (res *Demo, err error) {
}
}
if err != nil {
_metricErrCount.Inc("CacheDemo")
log.Errorv(c, log.KV("CacheDemo", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -120,7 +115,6 @@ func (d *Dao) CacheDemo1(c context.Context, id int64, mid int64) (res *Demo, err
}
}
if err != nil {
_metricErrCount.Inc("CacheDemo1")
log.Errorv(c, log.KV("CacheDemo1", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -139,7 +133,6 @@ func (d *Dao) CacheNone(c context.Context) (res *Demo, err error) {
}
}
if err != nil {
_metricErrCount.Inc("CacheNone")
log.Errorv(c, log.KV("CacheNone", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -155,7 +148,6 @@ func (d *Dao) CacheString(c context.Context, id int64) (res string, err error) {
err = nil
return
}
_metricErrCount.Inc("CacheString")
log.Errorv(c, log.KV("CacheString", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -171,7 +163,6 @@ func (d *Dao) AddCacheDemos(c context.Context, values map[int64]*Demo) (err erro
key := demoKey(id)
item := &memcache.Item{Key: key, Object: val, Expiration: d.demoExpire, Flags: memcache.FlagJSON}
if err = d.mc.Set(c, item); err != nil {
_metricErrCount.Inc("AddCacheDemos")
log.Errorv(c, log.KV("AddCacheDemos", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -188,7 +179,6 @@ func (d *Dao) AddCacheDemos2(c context.Context, values map[int64]*Demo, tp int64
key := demo2Key(id, tp)
item := &memcache.Item{Key: key, Object: val, Expiration: d.demoExpire, Flags: memcache.FlagJSON}
if err = d.mc.Set(c, item); err != nil {
_metricErrCount.Inc("AddCacheDemos2")
log.Errorv(c, log.KV("AddCacheDemos2", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -204,7 +194,6 @@ func (d *Dao) AddCacheDemo(c context.Context, id int64, val *Demo) (err error) {
key := demoKey(id)
item := &memcache.Item{Key: key, Object: val, Expiration: d.demoExpire, Flags: memcache.FlagJSON | memcache.FlagGzip}
if err = d.mc.Set(c, item); err != nil {
_metricErrCount.Incr("AddCacheDemo")
log.Errorv(c, log.KV("AddCacheDemo", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -219,7 +208,6 @@ func (d *Dao) AddCacheDemo1(c context.Context, id int64, val *Demo, mid int64) (
key := keyMid(id, mid)
item := &memcache.Item{Key: key, Object: val, Expiration: d.demoExpire, Flags: memcache.FlagGOB}
if err = d.mc.Set(c, item); err != nil {
_metricErrCount.Incr("AddCacheDemo1")
log.Errorv(c, log.KV("AddCacheDemo1", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -234,7 +222,6 @@ func (d *Dao) AddCacheNone(c context.Context, val *Demo) (err error) {
key := noneKey()
item := &memcache.Item{Key: key, Object: val, Expiration: d.demoExpire, Flags: memcache.FlagJSON}
if err = d.mc.Set(c, item); err != nil {
_metricErrCount.Inc("AddCacheNone")
log.Errorv(c, log.KV("AddCacheNone", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -250,7 +237,6 @@ func (d *Dao) AddCacheString(c context.Context, id int64, val string) (err error
bs := []byte(val)
item := &memcache.Item{Key: key, Value: bs, Expiration: d.demoExpire, Flags: memcache.FlagRAW}
if err = d.mc.Set(c, item); err != nil {
_metricErrCount.Incr("AddCacheString")
log.Errorv(c, log.KV("AddCacheString", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -269,7 +255,6 @@ func (d *Dao) DelCacheDemos(c context.Context, ids []int64) (err error) {
err = nil
continue
}
_metricErrCount.Inc("DelCacheDemos")
log.Errorv(c, log.KV("DelCacheDemos", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -285,7 +270,6 @@ func (d *Dao) DelCacheDemo(c context.Context, id int64) (err error) {
err = nil
return
}
_metricErrCount.Incr("DelCacheDemo")
log.Errorv(c, log.KV("DelCacheDemo", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -300,7 +284,6 @@ func (d *Dao) DelCacheDemo1(c context.Context, id int64, mid int64) (err error)
err = nil
return
}
_metricErrCount.Incr("DelCacheDemo1")
log.Errorv(c, log.KV("DelCacheDemo1", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}
@ -315,7 +298,6 @@ func (d *Dao) DelCacheNone(c context.Context) (err error) {
err = nil
return
}
_metricErrCount.Inc("DelCacheNone")
log.Errorv(c, log.KV("DelCacheNone", fmt.Sprintf("%+v", err)), log.KV("key", key))
return
}

4
tool/kratos/tool_index.go
View File

@ -33,7 +33,7 @@ var toolIndexs = []*Tool{
&Tool{
Name: "genbts",
Alias: "kratos-gen-bts",
BuildTime: time.Date(2019, 7, 20, 0, 0, 0, 0, time.Local),
BuildTime: time.Date(2019, 7, 23, 0, 0, 0, 0, time.Local),
Install: "go get -u github.com/bilibili/kratos/tool/kratos-gen-bts",
Summary: "缓存回源逻辑代码生成器",
Platform: []string{"darwin", "linux", "windows"},
@ -42,7 +42,7 @@ var toolIndexs = []*Tool{
&Tool{
Name: "genmc",
Alias: "kratos-gen-mc",
BuildTime: time.Date(2019, 7, 20, 0, 0, 0, 0, time.Local),
BuildTime: time.Date(2019, 7, 23, 0, 0, 0, 0, time.Local),
Install: "go get -u github.com/bilibili/kratos/tool/kratos-gen-mc",
Summary: "mc缓存代码生成",
Platform: []string{"darwin", "linux", "windows"},