go/go-micro
1
0
Fork 0
mirror of https://github.com/go-micro/go-micro.git synced 2024-11-24 08:02:32 +02:00
Code Issues Releases Activity

Merge master into registry-namespace

Browse Source
This commit is contained in:
Ben Toogood 2020-04-14 09:15:13 +01:00
commit 0c75a0306b
95 changed files with 4442 additions and 5556 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -1,6 +1,6 @@
# Go Micro [![License](https://img.shields.io/:license-apache-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Go.Dev reference](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/micro/go-micro?tab=doc) [![Travis CI](https://api.travis-ci.org/micro/go-micro.svg?branch=master)](https://travis-ci.org/micro/go-micro) [![Go Report Card](https://goreportcard.com/badge/micro/go-micro)](https://goreportcard.com/report/github.com/micro/go-micro)
Go Micro is a framework for microservice development.
Go Micro is a framework for distributed systems development.
## Overview
@ -35,8 +35,7 @@ communication. A request made to a service will be automatically resolved, load
transport is [gRPC](https://grpc.io/).
- **Async Messaging** - PubSub is built in as a first class citizen for asynchronous communication and event driven architectures.
Event notifications are a core pattern in micro service development. The default messaging system is an embedded [NATS](https://nats.io/)
server.
Event notifications are a core pattern in micro service development. The default messaging system is a HTTP event message broker.
- **Pluggable Interfaces** - Go Micro makes use of Go interfaces for each distributed system abstraction. Because of this these interfaces
are pluggable and allows Go Micro to be runtime agnostic. You can plugin any underlying technology. Find plugins in
@ -45,4 +44,3 @@ are pluggable and allows Go Micro to be runtime agnostic. You can plugin any und
## Getting Started
See the [docs](https://micro.mu/docs/framework.html) for detailed information on the architecture, installation and use of go-micro.

60
api/grpc_test.go
View File

@ -9,20 +9,16 @@ import (
"testing"
"time"
"github.com/micro/go-micro/v2"
"github.com/micro/go-micro/v2/api"
ahandler "github.com/micro/go-micro/v2/api/handler"
apirpc "github.com/micro/go-micro/v2/api/handler/rpc"
"github.com/micro/go-micro/v2/api/handler"
"github.com/micro/go-micro/v2/api/handler/rpc"
"github.com/micro/go-micro/v2/api/router"
rstatic "github.com/micro/go-micro/v2/api/router/static"
bmemory "github.com/micro/go-micro/v2/broker/memory"
"github.com/micro/go-micro/v2/client"
gcli "github.com/micro/go-micro/v2/client/grpc"
rmemory "github.com/micro/go-micro/v2/registry/memory"
"github.com/micro/go-micro/v2/server"
gsrv "github.com/micro/go-micro/v2/server/grpc"
tgrpc "github.com/micro/go-micro/v2/transport/grpc"
pb "github.com/micro/go-micro/v2/server/grpc/proto"
)
@ -39,49 +35,33 @@ func (s *testServer) Call(ctx context.Context, req *pb.Request, rsp *pb.Response
func TestApiAndGRPC(t *testing.T) {
r := rmemory.NewRegistry()
b := bmemory.NewBroker()
tr := tgrpc.NewTransport()
// create a new client
s := gsrv.NewServer(
server.Broker(b),
server.Name("foo"),
server.Registry(r),
server.Transport(tr),
)
// create a new server
c := gcli.NewClient(
client.Registry(r),
client.Broker(b),
client.Transport(tr),
)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
svc := micro.NewService(
micro.Server(s),
micro.Client(c),
micro.Broker(b),
micro.Registry(r),
micro.Transport(tr),
micro.Context(ctx))
h := &testServer{}
pb.RegisterTestHandler(s, h)
go func() {
if err := svc.Run(); err != nil {
t.Fatalf("failed to start: %v", err)
}
}()
time.Sleep(1 * time.Second)
// check registration
services, err := r.GetService("foo")
if err != nil || len(services) == 0 {
t.Fatalf("failed to get service: %v # %d", err, len(services))
if err := s.Start(); err != nil {
t.Fatalf("failed to start: %v", err)
}
defer s.Stop()
// create a new router
router := rstatic.NewRouter(
router.WithHandler(apirpc.Handler),
router.WithRegistry(svc.Server().Options().Registry),
router.WithHandler(rpc.Handler),
router.WithRegistry(r),
)
err = router.Register(&api.Endpoint{
err := router.Register(&api.Endpoint{
Name: "foo.Test.Call",
Method: []string{"GET"},
Path: []string{"/api/v0/test/call/{name}"},
@ -91,9 +71,9 @@ func TestApiAndGRPC(t *testing.T) {
t.Fatal(err)
}
hrpc := apirpc.NewHandler(
ahandler.WithService(svc),
ahandler.WithRouter(router),
hrpc := rpc.NewHandler(
handler.WithClient(c),
handler.WithRouter(router),
)
hsrv := &http.Server{
@ -115,6 +95,7 @@ func TestApiAndGRPC(t *testing.T) {
t.Fatalf("Failed to created http.Request: %v", err)
}
defer rsp.Body.Close()
buf, err := ioutil.ReadAll(rsp.Body)
if err != nil {
t.Fatal(err)
@ -124,9 +105,4 @@ func TestApiAndGRPC(t *testing.T) {
if string(buf) != jsonMsg {
t.Fatalf("invalid message received, parsing error %s != %s", buf, jsonMsg)
}
select {
case <-ctx.Done():
return
}
}

2
api/handler/api/api.go
View File

@ -65,7 +65,7 @@ func (a *apiHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
// create request and response
c := a.opts.Service.Client()
c := a.opts.Client
req := c.NewRequest(service.Name, service.Endpoint.Name, request)
rsp := &api.Response{}

2
api/handler/event/event.go
View File

@ -118,7 +118,7 @@ func (e *event) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
// get client
c := e.opts.Service.Client()
c := e.opts.Client
// create publication
p := c.NewMessage(topic, ev)

15
api/handler/options.go
View File

@ -1,8 +1,9 @@
package handler
import (
"github.com/micro/go-micro/v2"
"github.com/micro/go-micro/v2/api/router"
"github.com/micro/go-micro/v2/client"
"github.com/micro/go-micro/v2/client/grpc"
)
var (
@ -13,7 +14,7 @@ type Options struct {
MaxRecvSize int64
Namespace string
Router router.Router
Service micro.Service
Client client.Client
}
type Option func(o *Options)
@ -25,9 +26,8 @@ func NewOptions(opts ...Option) Options {
o(&options)
}
// create service if its blank
if options.Service == nil {
WithService(micro.NewService())(&options)
if options.Client == nil {
WithClient(grpc.NewClient())(&options)
}
// set namespace if blank
@ -56,10 +56,9 @@ func WithRouter(r router.Router) Option {
}
}
// WithService specifies a micro.Service
func WithService(s micro.Service) Option {
func WithClient(c client.Client) Option {
return func(o *Options) {
o.Service = s
o.Client = c
}
}

16
api/handler/rpc/rpc.go
View File

@ -100,12 +100,6 @@ func (h *rpcHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
// only allow post when we have the router
if r.Method != "GET" && (h.opts.Router != nil && r.Method != "POST") {
writeError(w, r, errors.MethodNotAllowed("go.micro.api", "method not allowed"))
return
}
ct := r.Header.Get("Content-Type")
// Strip charset from Content-Type (like `application/json; charset=UTF-8`)
@ -114,16 +108,17 @@ func (h *rpcHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
// micro client
c := h.opts.Service.Client()
c := h.opts.Client
// create context
cx := ctx.FromRequest(r)
// get context from http handler wrappers
md, ok := r.Context().Value(metadata.MetadataKey{}).(metadata.Metadata)
md, ok := metadata.FromContext(r.Context())
if !ok {
md = make(metadata.Metadata)
}
// fill contex with http headers
md["Host"] = r.Host
// merge context with overwrite
cx = metadata.MergeContext(cx, md, true)
@ -293,7 +288,7 @@ func requestPayload(r *http.Request) ([]byte, error) {
// otherwise as per usual
ctx := r.Context()
// dont user meadata.FromContext as it mangles names
md, ok := ctx.Value(metadata.MetadataKey{}).(metadata.Metadata)
md, ok := metadata.FromContext(ctx)
if !ok {
md = make(map[string]string)
}
@ -304,6 +299,7 @@ func requestPayload(r *http.Request) ([]byte, error) {
// get fields from url path
for k, v := range md {
k = strings.ToLower(k)
// filter own keys
if strings.HasPrefix(k, "x-api-field-") {
matches[strings.TrimPrefix(k, "x-api-field-")] = v

14
api/router/static/static.go
View File

@ -1,7 +1,6 @@
package static
import (
"context"
"errors"
"fmt"
"net/http"
@ -16,6 +15,7 @@ import (
"github.com/micro/go-micro/v2/logger"
"github.com/micro/go-micro/v2/metadata"
"github.com/micro/go-micro/v2/registry"
util "github.com/micro/go-micro/v2/util/registry"
)
type endpoint struct {
@ -164,7 +164,7 @@ func (r *staticRouter) Endpoint(req *http.Request) (*api.Service, error) {
// hack for stream endpoint
if ep.apiep.Stream {
svcs := registry.Copy(services)
svcs := util.Copy(services)
for _, svc := range svcs {
if len(svc.Endpoints) == 0 {
e := &registry.Endpoint{}
@ -263,12 +263,14 @@ func (r *staticRouter) endpoint(req *http.Request) (*endpoint, error) {
for _, pathreg := range ep.pathregs {
matches, err := pathreg.Match(path, "")
if err != nil {
// TODO: log error
if logger.V(logger.DebugLevel, logger.DefaultLogger) {
logger.Debugf("api path not match %s != %v", path, pathreg)
}
continue
}
pMatch = true
ctx := req.Context()
md, ok := ctx.Value(metadata.MetadataKey{}).(metadata.Metadata)
md, ok := metadata.FromContext(ctx)
if !ok {
md = make(metadata.Metadata)
}
@ -276,7 +278,7 @@ func (r *staticRouter) endpoint(req *http.Request) (*endpoint, error) {
md[fmt.Sprintf("x-api-field-%s", k)] = v
}
md["x-api-body"] = ep.apiep.Body
*req = *req.Clone(context.WithValue(ctx, metadata.MetadataKey{}, md))
*req = *req.Clone(metadata.NewContext(ctx, md))
break pathLoop
}
if !pMatch {
@ -289,7 +291,7 @@ func (r *staticRouter) endpoint(req *http.Request) (*endpoint, error) {
}
// no match
return nil, fmt.Errorf("endpoint not found for %v", req)
return nil, fmt.Errorf("endpoint not found for %v", req.URL)
}
func (r *staticRouter) Route(req *http.Request) (*api.Service, error) {

236
api/server/acme/certmagic/certmagic_test.go
View File

@ -1,236 +0,0 @@
package certmagic
import (
"net"
"net/http"
"os"
"reflect"
"sort"
"testing"
"time"
"github.com/go-acme/lego/v3/providers/dns/cloudflare"
"github.com/mholt/certmagic"
"github.com/micro/go-micro/v2/api/server/acme"
cfstore "github.com/micro/go-micro/v2/store/cloudflare"
"github.com/micro/go-micro/v2/sync/lock/memory"
)
func TestCertMagic(t *testing.T) {
if len(os.Getenv("IN_TRAVIS_CI")) != 0 {
t.Skip()
}
l, err := NewProvider().Listen()
if err != nil {
if _, ok := err.(*net.OpError); ok {
t.Skip("Run under non privileged user")
}
t.Fatal(err.Error())
}
l.Close()
c := cloudflare.NewDefaultConfig()
c.AuthEmail = ""
c.AuthKey = ""
c.AuthToken = "test"
c.ZoneToken = "test"
p, err := cloudflare.NewDNSProviderConfig(c)
if err != nil {
t.Fatal(err.Error())
}
l, err = NewProvider(acme.AcceptToS(true),
acme.CA(acme.LetsEncryptStagingCA),
acme.ChallengeProvider(p),
).Listen()
if err != nil {
t.Fatal(err.Error())
}
l.Close()
}
func TestStorageImplementation(t *testing.T) {
if len(os.Getenv("IN_TRAVIS_CI")) != 0 {
t.Skip()
}
apiToken, accountID := os.Getenv("CF_API_TOKEN"), os.Getenv("CF_ACCOUNT_ID")
kvID := os.Getenv("KV_NAMESPACE_ID")
if len(apiToken) == 0 || len(accountID) == 0 || len(kvID) == 0 {
t.Skip("No Cloudflare API keys available, skipping test")
}
var s certmagic.Storage
st := cfstore.NewStore(
cfstore.Token(apiToken),
cfstore.Account(accountID),
cfstore.Namespace(kvID),
)
s = &storage{
lock: memory.NewLock(),
store: st,
}
// Test Lock
if err := s.Lock("test"); err != nil {
t.Fatal(err)
}
// Test Unlock
if err := s.Unlock("test"); err != nil {
t.Fatal(err)
}
// Test data
testdata := []struct {
key string
value []byte
}{
{key: "/foo/a", value: []byte("lorem")},
{key: "/foo/b", value: []byte("ipsum")},
{key: "/foo/c", value: []byte("dolor")},
{key: "/foo/d", value: []byte("sit")},
{key: "/bar/a", value: []byte("amet")},
{key: "/bar/b", value: []byte("consectetur")},
{key: "/bar/c", value: []byte("adipiscing")},
{key: "/bar/d", value: []byte("elit")},
{key: "/foo/bar/a", value: []byte("sed")},
{key: "/foo/bar/b", value: []byte("do")},
{key: "/foo/bar/c", value: []byte("eiusmod")},
{key: "/foo/bar/d", value: []byte("tempor")},
{key: "/foo/bar/baz/a", value: []byte("incididunt")},
{key: "/foo/bar/baz/b", value: []byte("ut")},
{key: "/foo/bar/baz/c", value: []byte("labore")},
{key: "/foo/bar/baz/d", value: []byte("et")},
// a duplicate just in case there's any edge cases
{key: "/foo/a", value: []byte("lorem")},
}
// Test Store
for _, d := range testdata {
if err := s.Store(d.key, d.value); err != nil {
t.Fatal(err.Error())
}
}
// Test Load
for _, d := range testdata {
if value, err := s.Load(d.key); err != nil {
t.Fatal(err.Error())
} else {
if !reflect.DeepEqual(value, d.value) {
t.Fatalf("Load %s: expected %v, got %v", d.key, d.value, value)
}
}
}
// Test Exists
for _, d := range testdata {
if !s.Exists(d.key) {
t.Fatalf("%s should exist, but doesn't\n", d.key)
}
}
// Test List
if list, err := s.List("/", true); err != nil {
t.Fatal(err.Error())
} else {
var expected []string
for i, d := range testdata {
if i != len(testdata)-1 {
// Don't store the intentionally duplicated key
expected = append(expected, d.key)
}
}
sort.Strings(expected)
sort.Strings(list)
if !reflect.DeepEqual(expected, list) {
t.Fatalf("List: Expected %v, got %v\n", expected, list)
}
}
if list, err := s.List("/foo", false); err != nil {
t.Fatal(err.Error())
} else {
sort.Strings(list)
expected := []string{"/foo/a", "/foo/b", "/foo/bar", "/foo/c", "/foo/d"}
if !reflect.DeepEqual(expected, list) {
t.Fatalf("List: expected %s, got %s\n", expected, list)
}
}
// Test Stat
for _, d := range testdata {
info, err := s.Stat(d.key)
if err != nil {
t.Fatal(err.Error())
} else {
if info.Key != d.key {
t.Fatalf("Stat().Key: expected %s, got %s\n", d.key, info.Key)
}
if info.Size != int64(len(d.value)) {
t.Fatalf("Stat().Size: expected %d, got %d\n", len(d.value), info.Size)
}
if time.Since(info.Modified) > time.Minute {
t.Fatalf("Stat().Modified: expected time since last modified to be < 1 minute, got %v\n", time.Since(info.Modified))
}
}
}
// Test Delete
for _, d := range testdata {
if err := s.Delete(d.key); err != nil {
t.Fatal(err.Error())
}
}
// New interface doesn't return an error, so call it in case any log.Fatal
// happens
NewProvider(acme.Cache(s))
}
// Full test with a real zone, with against LE staging
func TestE2e(t *testing.T) {
if len(os.Getenv("IN_TRAVIS_CI")) != 0 {
t.Skip()
}
apiToken, accountID := os.Getenv("CF_API_TOKEN"), os.Getenv("CF_ACCOUNT_ID")
kvID := os.Getenv("KV_NAMESPACE_ID")
if len(apiToken) == 0 || len(accountID) == 0 || len(kvID) == 0 {
t.Skip("No Cloudflare API keys available, skipping test")
}
testLock := memory.NewLock()
testStore := cfstore.NewStore(
cfstore.Token(apiToken),
cfstore.Account(accountID),
cfstore.Namespace(kvID),
)
testStorage := NewStorage(testLock, testStore)
conf := cloudflare.NewDefaultConfig()
conf.AuthToken = apiToken
conf.ZoneToken = apiToken
testChallengeProvider, err := cloudflare.NewDNSProviderConfig(conf)
if err != nil {
t.Fatal(err.Error())
}
testProvider := NewProvider(
acme.AcceptToS(true),
acme.Cache(testStorage),
acme.CA(acme.LetsEncryptStagingCA),
acme.ChallengeProvider(testChallengeProvider),
acme.OnDemand(false),
)
listener, err := testProvider.Listen("*.micro.mu", "micro.mu")
if err != nil {
t.Fatal(err.Error())
}
go http.Serve(listener, http.NotFoundHandler())
time.Sleep(10 * time.Minute)
}

10
api/server/acme/certmagic/storage.go
View File

@ -11,7 +11,7 @@ import (
"github.com/mholt/certmagic"
"github.com/micro/go-micro/v2/store"
"github.com/micro/go-micro/v2/sync/lock"
"github.com/micro/go-micro/v2/sync"
)
// File represents a "File" that will be stored in store.Store - the contents and last modified time
@ -26,16 +26,16 @@ type File struct {
// As certmagic storage expects a filesystem (with stat() abilities) we have to implement
// the bare minimum of metadata.
type storage struct {
lock lock.Lock
lock sync.Sync
store store.Store
}
func (s *storage) Lock(key string) error {
return s.lock.Acquire(key, lock.TTL(10*time.Minute))
return s.lock.Lock(key, sync.LockTTL(10*time.Minute))
}
func (s *storage) Unlock(key string) error {
return s.lock.Release(key)
return s.lock.Unlock(key)
}
func (s *storage) Store(key string, value []byte) error {
@ -139,7 +139,7 @@ func (s *storage) Stat(key string) (certmagic.KeyInfo, error) {
}
// NewStorage returns a certmagic.Storage backed by a go-micro/lock and go-micro/store
func NewStorage(lock lock.Lock, store store.Store) certmagic.Storage {
func NewStorage(lock sync.Sync, store store.Store) certmagic.Storage {
return &storage{
lock: lock,
store: store,

13
auth/default.go
View File

@ -2,6 +2,7 @@ package auth
import (
"github.com/google/uuid"
"github.com/micro/go-micro/v2/auth/provider/basic"
)
var (
@ -9,7 +10,17 @@ var (
)
func NewAuth(opts ...Option) Auth {
return &noop{opts: NewOptions(opts...)}
options := Options{
Provider: basic.NewProvider(),
}
for _, o := range opts {
o(&options)
}
return &noop{
opts: options,
}
}
type noop struct {

504
broker/default.go
View File

@ -1,504 +0,0 @@
package broker
import (
"context"
"errors"
"net"
"net/url"
"strconv"
"strings"
"sync"
"time"
"github.com/micro/go-micro/v2/codec/json"
"github.com/micro/go-micro/v2/logger"
"github.com/micro/go-micro/v2/registry"
"github.com/micro/go-micro/v2/util/addr"
"github.com/nats-io/nats-server/v2/server"
nats "github.com/nats-io/nats.go"
)
type natsBroker struct {
sync.Once
sync.RWMutex
// indicate if we're connected
connected bool
// address to bind routes to
addrs []string
// servers for the client
servers []string
// client connection and nats opts
conn *nats.Conn
opts Options
nopts nats.Options
// should we drain the connection
drain bool
closeCh chan (error)
// embedded server
server *server.Server
// configure to use local server
local bool
// server exit channel
exit chan bool
}
type subscriber struct {
s *nats.Subscription
opts SubscribeOptions
}
type publication struct {
t string
err error
m *Message
}
func (p *publication) Topic() string {
return p.t
}
func (p *publication) Message() *Message {
return p.m
}
func (p *publication) Ack() error {
// nats does not support acking
return nil
}
func (p *publication) Error() error {
return p.err
}
func (s *subscriber) Options() SubscribeOptions {
return s.opts
}
func (s *subscriber) Topic() string {
return s.s.Subject
}
func (s *subscriber) Unsubscribe() error {
return s.s.Unsubscribe()
}
func (n *natsBroker) Address() string {
n.RLock()
defer n.RUnlock()
if n.server != nil {
return n.server.ClusterAddr().String()
}
if n.conn != nil && n.conn.IsConnected() {
return n.conn.ConnectedUrl()
}
if len(n.addrs) > 0 {
return n.addrs[0]
}
return "127.0.0.1:-1"
}
func (n *natsBroker) setAddrs(addrs []string) []string {
//nolint:prealloc
var cAddrs []string
for _, addr := range addrs {
if len(addr) == 0 {
continue
}
if !strings.HasPrefix(addr, "nats://") {
addr = "nats://" + addr
}
cAddrs = append(cAddrs, addr)
}
// if there's no address and we weren't told to
// embed a local server then use the default url
if len(cAddrs) == 0 && !n.local {
cAddrs = []string{nats.DefaultURL}
}
return cAddrs
}
// serve stats a local nats server if needed
func (n *natsBroker) serve(exit chan bool) error {
// local server address
host := "127.0.0.1"
port := -1
// cluster address
caddr := "0.0.0.0"
cport := -1
// with no address we just default it
// this is a local client address
if len(n.addrs) > 0 {
address := n.addrs[0]
if strings.HasPrefix(address, "nats://") {
address = strings.TrimPrefix(address, "nats://")
}
// parse out the address
h, p, err := net.SplitHostPort(address)
if err == nil {
caddr = h
cport, _ = strconv.Atoi(p)
}
}
// 1. create new server
// 2. register the server
// 3. connect to other servers
// set cluster opts
cOpts := server.ClusterOpts{
Host: caddr,
Port: cport,
}
// get the routes for other nodes
var routes []*url.URL
// get existing nats servers to connect to
services, err := n.opts.Registry.GetService("go.micro.nats.broker")
if err == nil {
for _, service := range services {
for _, node := range service.Nodes {
u, err := url.Parse("nats://" + node.Address)
if err != nil {
if logger.V(logger.InfoLevel, logger.DefaultLogger) {
logger.Info(err)
}
continue
}
// append to the cluster routes
routes = append(routes, u)
}
}
}
// try get existing server
s := n.server
if s != nil {
// stop the existing server
s.Shutdown()
}
s, err = server.NewServer(&server.Options{
// Specify the host
Host: host,
// Use a random port
Port: port,
// Set the cluster ops
Cluster: cOpts,
// Set the routes
Routes: routes,
NoLog: true,
NoSigs: true,
MaxControlLine: 2048,
TLSConfig: n.opts.TLSConfig,
})
if err != nil {
return err
}
// save the server
n.server = s
// start the server
go s.Start()
var ready bool
// wait till its ready for connections
for i := 0; i < 3; i++ {
if s.ReadyForConnections(time.Second) {
ready = true
break
}
}
if !ready {
return errors.New("server not ready")
}
// set the client address
n.servers = []string{s.ClientURL()}
go func() {
var advertise string
// parse out the address
_, port, err := net.SplitHostPort(s.ClusterAddr().String())
if err == nil {
addr, _ := addr.Extract("")
advertise = net.JoinHostPort(addr, port)
} else {
s.ClusterAddr().String()
}
// register the cluster address
for {
select {
case err := <-n.closeCh:
if err != nil {
if logger.V(logger.InfoLevel, logger.DefaultLogger) {
logger.Info(err)
}
}
case <-exit:
// deregister on exit
n.opts.Registry.Deregister(&registry.Service{
Name: "go.micro.nats.broker",
Version: "v2",
Nodes: []*registry.Node{
{Id: s.ID(), Address: advertise},
},
})
s.Shutdown()
return
default:
// register the broker
n.opts.Registry.Register(&registry.Service{
Name: "go.micro.nats.broker",
Version: "v2",
Nodes: []*registry.Node{
{Id: s.ID(), Address: advertise},
},
}, registry.RegisterTTL(time.Minute))
time.Sleep(time.Minute)
}
}
}()
return nil
}
func (n *natsBroker) Connect() error {
n.Lock()
defer n.Unlock()
if !n.connected {
// create exit chan
n.exit = make(chan bool)
// start the local server
if err := n.serve(n.exit); err != nil {
return err
}
// set to connected
}
status := nats.CLOSED
if n.conn != nil {
status = n.conn.Status()
}
switch status {
case nats.CONNECTED, nats.RECONNECTING, nats.CONNECTING:
return nil
default: // DISCONNECTED or CLOSED or DRAINING
opts := n.nopts
opts.DrainTimeout = 1 * time.Second
opts.AsyncErrorCB = n.onAsyncError
opts.DisconnectedErrCB = n.onDisconnectedError
opts.ClosedCB = n.onClose
opts.Servers = n.servers
opts.Secure = n.opts.Secure
opts.TLSConfig = n.opts.TLSConfig
// secure might not be set
if n.opts.TLSConfig != nil {
opts.Secure = true
}
c, err := opts.Connect()
if err != nil {
return err
}
n.conn = c
n.connected = true
return nil
}
}
func (n *natsBroker) Disconnect() error {
n.RLock()
defer n.RUnlock()
if !n.connected {
return nil
}
// drain the connection if specified
if n.drain {
n.conn.Drain()
}
// close the client connection
n.conn.Close()
// shutdown the local server
// and deregister
if n.server != nil {
select {
case <-n.exit:
default:
close(n.exit)
}
}
// set not connected
n.connected = false
return nil
}
func (n *natsBroker) Init(opts ...Option) error {
n.setOption(opts...)
return nil
}
func (n *natsBroker) Options() Options {
return n.opts
}
func (n *natsBroker) Publish(topic string, msg *Message, opts ...PublishOption) error {
b, err := n.opts.Codec.Marshal(msg)
if err != nil {
return err
}
n.RLock()
defer n.RUnlock()
return n.conn.Publish(topic, b)
}
func (n *natsBroker) Subscribe(topic string, handler Handler, opts ...SubscribeOption) (Subscriber, error) {
if n.conn == nil {
return nil, errors.New("not connected")
}
opt := SubscribeOptions{
AutoAck: true,
Context: context.Background(),
}
for _, o := range opts {
o(&opt)
}
fn := func(msg *nats.Msg) {
var m Message
pub := &publication{t: msg.Subject}
eh := n.opts.ErrorHandler
err := n.opts.Codec.Unmarshal(msg.Data, &m)
pub.err = err
pub.m = &m
if err != nil {
m.Body = msg.Data
if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
logger.Error(err)
}
if eh != nil {
eh(pub)
}
return
}
if err := handler(pub); err != nil {
pub.err = err
if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
logger.Error(err)
}
if eh != nil {
eh(pub)
}
}
}
var sub *nats.Subscription
var err error
n.RLock()
if len(opt.Queue) > 0 {
sub, err = n.conn.QueueSubscribe(topic, opt.Queue, fn)
} else {
sub, err = n.conn.Subscribe(topic, fn)
}
n.RUnlock()
if err != nil {
return nil, err
}
return &subscriber{s: sub, opts: opt}, nil
}
func (n *natsBroker) String() string {
return "eats"
}
func (n *natsBroker) setOption(opts ...Option) {
for _, o := range opts {
o(&n.opts)
}
n.Once.Do(func() {
n.nopts = nats.GetDefaultOptions()
})
// local embedded server
n.local = true
// set to drain
n.drain = true
if !n.opts.Secure {
n.opts.Secure = n.nopts.Secure
}
if n.opts.TLSConfig == nil {
n.opts.TLSConfig = n.nopts.TLSConfig
}
n.addrs = n.setAddrs(n.opts.Addrs)
}
func (n *natsBroker) onClose(conn *nats.Conn) {
n.closeCh <- nil
}
func (n *natsBroker) onDisconnectedError(conn *nats.Conn, err error) {
n.closeCh <- err
}
func (n *natsBroker) onAsyncError(conn *nats.Conn, sub *nats.Subscription, err error) {
// There are kinds of different async error nats might callback, but we are interested
// in ErrDrainTimeout only here.
if err == nats.ErrDrainTimeout {
n.closeCh <- err
}
}
func NewBroker(opts ...Option) Broker {
options := Options{
// Default codec
Codec: json.Marshaler{},
Context: context.Background(),
Registry: registry.DefaultRegistry,
}
n := &natsBroker{
opts: options,
closeCh: make(chan error),
}
n.setOption(opts...)
return n
}

711
broker/http.go Normal file
View File

@ -0,0 +1,711 @@
// Package http provides a http based message broker
package broker
import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net"
"net/http"
"net/url"
"runtime"
"sync"
"time"
"github.com/google/uuid"
"github.com/micro/go-micro/v2/codec/json"
merr "github.com/micro/go-micro/v2/errors"
"github.com/micro/go-micro/v2/registry"
"github.com/micro/go-micro/v2/registry/cache"
maddr "github.com/micro/go-micro/v2/util/addr"
mnet "github.com/micro/go-micro/v2/util/net"
mls "github.com/micro/go-micro/v2/util/tls"
"golang.org/x/net/http2"
)
// HTTP Broker is a point to point async broker
type httpBroker struct {
id string
address string
opts Options
mux *http.ServeMux
c *http.Client
r registry.Registry
sync.RWMutex
subscribers map[string][]*httpSubscriber
running bool
exit chan chan error
// offline message inbox
mtx sync.RWMutex
inbox map[string][][]byte
}
type httpSubscriber struct {
opts SubscribeOptions
id string
topic string
fn Handler
svc *registry.Service
hb *httpBroker
}
type httpEvent struct {
m *Message
t string
err error
}
var (
DefaultPath = "/"
DefaultAddress = "127.0.0.1:0"
serviceName = "micro.http.broker"
broadcastVersion = "ff.http.broadcast"
registerTTL = time.Minute
registerInterval = time.Second * 30
)
func init() {
rand.Seed(time.Now().Unix())
}
func newTransport(config *tls.Config) *http.Transport {
if config == nil {
config = &tls.Config{
InsecureSkipVerify: true,
}
}
dialTLS := func(network string, addr string) (net.Conn, error) {
return tls.Dial(network, addr, config)
}
t := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
DialTLS: dialTLS,
}
runtime.SetFinalizer(&t, func(tr **http.Transport) {
(*tr).CloseIdleConnections()
})
// setup http2
http2.ConfigureTransport(t)
return t
}
func newHttpBroker(opts ...Option) Broker {
options := Options{
Codec: json.Marshaler{},
Context: context.TODO(),
Registry: registry.DefaultRegistry,
}
for _, o := range opts {
o(&options)
}
// set address
addr := DefaultAddress
if len(options.Addrs) > 0 && len(options.Addrs[0]) > 0 {
addr = options.Addrs[0]
}
h := &httpBroker{
id: uuid.New().String(),
address: addr,
opts: options,
r: options.Registry,
c: &http.Client{Transport: newTransport(options.TLSConfig)},
subscribers: make(map[string][]*httpSubscriber),
exit: make(chan chan error),
mux: http.NewServeMux(),
inbox: make(map[string][][]byte),
}
// specify the message handler
h.mux.Handle(DefaultPath, h)
// get optional handlers
if h.opts.Context != nil {
handlers, ok := h.opts.Context.Value("http_handlers").(map[string]http.Handler)
if ok {
for pattern, handler := range handlers {
h.mux.Handle(pattern, handler)
}
}
}
return h
}
func (h *httpEvent) Ack() error {
return nil
}
func (h *httpEvent) Error() error {
return h.err
}
func (h *httpEvent) Message() *Message {
return h.m
}
func (h *httpEvent) Topic() string {
return h.t
}
func (h *httpSubscriber) Options() SubscribeOptions {
return h.opts
}
func (h *httpSubscriber) Topic() string {
return h.topic
}
func (h *httpSubscriber) Unsubscribe() error {
return h.hb.unsubscribe(h)
}
func (h *httpBroker) saveMessage(topic string, msg []byte) {
h.mtx.Lock()
defer h.mtx.Unlock()
// get messages
c := h.inbox[topic]
// save message
c = append(c, msg)
// max length 64
if len(c) > 64 {
c = c[:64]
}
// save inbox
h.inbox[topic] = c
}
func (h *httpBroker) getMessage(topic string, num int) [][]byte {
h.mtx.Lock()
defer h.mtx.Unlock()
// get messages
c, ok := h.inbox[topic]
if !ok {
return nil
}
// more message than requests
if len(c) >= num {
msg := c[:num]
h.inbox[topic] = c[num:]
return msg
}
// reset inbox
h.inbox[topic] = nil
// return all messages
return c
}
func (h *httpBroker) subscribe(s *httpSubscriber) error {
h.Lock()
defer h.Unlock()
if err := h.r.Register(s.svc, registry.RegisterTTL(registerTTL)); err != nil {
return err
}
h.subscribers[s.topic] = append(h.subscribers[s.topic], s)
return nil
}
func (h *httpBroker) unsubscribe(s *httpSubscriber) error {
h.Lock()
defer h.Unlock()
//nolint:prealloc
var subscribers []*httpSubscriber
// look for subscriber
for _, sub := range h.subscribers[s.topic] {
// deregister and skip forward
if sub == s {
_ = h.r.Deregister(sub.svc)
continue
}
// keep subscriber
subscribers = append(subscribers, sub)
}
// set subscribers
h.subscribers[s.topic] = subscribers
return nil
}
func (h *httpBroker) run(l net.Listener) {
t := time.NewTicker(registerInterval)
defer t.Stop()
for {
select {
// heartbeat for each subscriber
case <-t.C:
h.RLock()
for _, subs := range h.subscribers {
for _, sub := range subs {
_ = h.r.Register(sub.svc, registry.RegisterTTL(registerTTL))
}
}
h.RUnlock()
// received exit signal
case ch := <-h.exit:
ch <- l.Close()
h.RLock()
for _, subs := range h.subscribers {
for _, sub := range subs {
_ = h.r.Deregister(sub.svc)
}
}
h.RUnlock()
return
}
}
}
func (h *httpBroker) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" {
err := merr.BadRequest("go.micro.broker", "Method not allowed")
http.Error(w, err.Error(), http.StatusMethodNotAllowed)
return
}
defer req.Body.Close()
req.ParseForm()
b, err := ioutil.ReadAll(req.Body)
if err != nil {
errr := merr.InternalServerError("go.micro.broker", "Error reading request body: %v", err)
w.WriteHeader(500)
w.Write([]byte(errr.Error()))
return
}
var m *Message
if err = h.opts.Codec.Unmarshal(b, &m); err != nil {
errr := merr.InternalServerError("go.micro.broker", "Error parsing request body: %v", err)
w.WriteHeader(500)
w.Write([]byte(errr.Error()))
return
}
topic := m.Header["Micro-Topic"]
//delete(m.Header, ":topic")
if len(topic) == 0 {
errr := merr.InternalServerError("go.micro.broker", "Topic not found")
w.WriteHeader(500)
w.Write([]byte(errr.Error()))
return
}
p := &httpEvent{m: m, t: topic}
id := req.Form.Get("id")
//nolint:prealloc
var subs []Handler
h.RLock()
for _, subscriber := range h.subscribers[topic] {
if id != subscriber.id {
continue
}
subs = append(subs, subscriber.fn)
}
h.RUnlock()
// execute the handler
for _, fn := range subs {
p.err = fn(p)
}
}
func (h *httpBroker) Address() string {
h.RLock()
defer h.RUnlock()
return h.address
}
func (h *httpBroker) Connect() error {
h.RLock()
if h.running {
h.RUnlock()
return nil
}
h.RUnlock()
h.Lock()
defer h.Unlock()
var l net.Listener
var err error
if h.opts.Secure || h.opts.TLSConfig != nil {
config := h.opts.TLSConfig
fn := func(addr string) (net.Listener, error) {
if config == nil {
hosts := []string{addr}
// check if its a valid host:port
if host, _, err := net.SplitHostPort(addr); err == nil {
if len(host) == 0 {
hosts = maddr.IPs()
} else {
hosts = []string{host}
}
}
// generate a certificate
cert, err := mls.Certificate(hosts...)
if err != nil {
return nil, err
}
config = &tls.Config{Certificates: []tls.Certificate{cert}}
}
return tls.Listen("tcp", addr, config)
}
l, err = mnet.Listen(h.address, fn)
} else {
fn := func(addr string) (net.Listener, error) {
return net.Listen("tcp", addr)
}
l, err = mnet.Listen(h.address, fn)
}
if err != nil {
return err
}
addr := h.address
h.address = l.Addr().String()
go http.Serve(l, h.mux)
go func() {
h.run(l)
h.Lock()
h.opts.Addrs = []string{addr}
h.address = addr
h.Unlock()
}()
// get registry
reg := h.opts.Registry
if reg == nil {
reg = registry.DefaultRegistry
}
// set cache
h.r = cache.New(reg)
// set running
h.running = true
return nil
}
func (h *httpBroker) Disconnect() error {
h.RLock()
if !h.running {
h.RUnlock()
return nil
}
h.RUnlock()
h.Lock()
defer h.Unlock()
// stop cache
rc, ok := h.r.(cache.Cache)
if ok {
rc.Stop()
}
// exit and return err
ch := make(chan error)
h.exit <- ch
err := <-ch
// set not running
h.running = false
return err
}
func (h *httpBroker) Init(opts ...Option) error {
h.RLock()
if h.running {
h.RUnlock()
return errors.New("cannot init while connected")
}
h.RUnlock()
h.Lock()
defer h.Unlock()
for _, o := range opts {
o(&h.opts)
}
if len(h.opts.Addrs) > 0 && len(h.opts.Addrs[0]) > 0 {
h.address = h.opts.Addrs[0]
}
if len(h.id) == 0 {
h.id = "go.micro.http.broker-" + uuid.New().String()
}
// get registry
reg := h.opts.Registry
if reg == nil {
reg = registry.DefaultRegistry
}
// get cache
if rc, ok := h.r.(cache.Cache); ok {
rc.Stop()
}
// set registry
h.r = cache.New(reg)
// reconfigure tls config
if c := h.opts.TLSConfig; c != nil {
h.c = &http.Client{
Transport: newTransport(c),
}
}
return nil
}
func (h *httpBroker) Options() Options {
return h.opts
}
func (h *httpBroker) Publish(topic string, msg *Message, opts ...PublishOption) error {
// create the message first
m := &Message{
Header: make(map[string]string),
Body: msg.Body,
}
for k, v := range msg.Header {
m.Header[k] = v
}
m.Header["Micro-Topic"] = topic
// encode the message
b, err := h.opts.Codec.Marshal(m)
if err != nil {
return err
}
// save the message
h.saveMessage(topic, b)
// now attempt to get the service
h.RLock()
s, err := h.r.GetService(serviceName)
if err != nil {
h.RUnlock()
return err
}
h.RUnlock()
pub := func(node *registry.Node, t string, b []byte) error {
scheme := "http"
// check if secure is added in metadata
if node.Metadata["secure"] == "true" {
scheme = "https"
}
vals := url.Values{}
vals.Add("id", node.Id)
uri := fmt.Sprintf("%s://%s%s?%s", scheme, node.Address, DefaultPath, vals.Encode())
r, err := h.c.Post(uri, "application/json", bytes.NewReader(b))
if err != nil {
return err
}
// discard response body
io.Copy(ioutil.Discard, r.Body)
r.Body.Close()
return nil
}
srv := func(s []*registry.Service, b []byte) {
for _, service := range s {
var nodes []*registry.Node
for _, node := range service.Nodes {
// only use nodes tagged with broker http
if node.Metadata["broker"] != "http" {
continue
}
// look for nodes for the topic
if node.Metadata["topic"] != topic {
continue
}
nodes = append(nodes, node)
}
// only process if we have nodes
if len(nodes) == 0 {
continue
}
switch service.Version {
// broadcast version means broadcast to all nodes
case broadcastVersion:
var success bool
// publish to all nodes
for _, node := range nodes {
// publish async
if err := pub(node, topic, b); err == nil {
success = true
}
}
// save if it failed to publish at least once
if !success {
h.saveMessage(topic, b)
}
default:
// select node to publish to
node := nodes[rand.Int()%len(nodes)]
// publish async to one node
if err := pub(node, topic, b); err != nil {
// if failed save it
h.saveMessage(topic, b)
}
}
}
}
// do the rest async
go func() {
// get a third of the backlog
messages := h.getMessage(topic, 8)
delay := (len(messages) > 1)
// publish all the messages
for _, msg := range messages {
// serialize here
srv(s, msg)
// sending a backlog of messages
if delay {
time.Sleep(time.Millisecond * 100)
}
}
}()
return nil
}
func (h *httpBroker) Subscribe(topic string, handler Handler, opts ...SubscribeOption) (Subscriber, error) {
var err error
var host, port string
options := NewSubscribeOptions(opts...)
// parse address for host, port
host, port, err = net.SplitHostPort(h.Address())
if err != nil {
return nil, err
}
addr, err := maddr.Extract(host)
if err != nil {
return nil, err
}
var secure bool
if h.opts.Secure || h.opts.TLSConfig != nil {
secure = true
}
// register service
node := &registry.Node{
Id: topic + "-" + h.id,
Address: mnet.HostPort(addr, port),
Metadata: map[string]string{
"secure": fmt.Sprintf("%t", secure),
"broker": "http",
"topic": topic,
},
}
// check for queue group or broadcast queue
version := options.Queue
if len(version) == 0 {
version = broadcastVersion
}
service := &registry.Service{
Name: serviceName,
Version: version,
Nodes: []*registry.Node{node},
}
// generate subscriber
subscriber := &httpSubscriber{
opts: options,
hb: h,
id: node.Id,
topic: topic,
fn: handler,
svc: service,
}
// subscribe now
if err := h.subscribe(subscriber); err != nil {
return nil, err
}
// return the subscriber
return subscriber, nil
}
func (h *httpBroker) String() string {
return "http"
}
// NewBroker returns a new http broker
func NewBroker(opts ...Option) Broker {
return newHttpBroker(opts...)
}

11
broker/http/http.go Normal file
View File

@ -0,0 +1,11 @@
// Package http provides a http based message broker
package http
import (
"github.com/micro/go-micro/v2/broker"
)
// NewBroker returns a new http broker
func NewBroker(opts ...broker.Option) broker.Broker {
return broker.NewBroker(opts...)
}

23
broker/http/options.go Normal file
View File

@ -0,0 +1,23 @@
package http
import (
"context"
"net/http"
"github.com/micro/go-micro/v2/broker"
)
// Handle registers the handler for the given pattern.
func Handle(pattern string, handler http.Handler) broker.Option {
return func(o *broker.Options) {
if o.Context == nil {
o.Context = context.Background()
}
handlers, ok := o.Context.Value("http_handlers").(map[string]http.Handler)
if !ok {
handlers = make(map[string]http.Handler)
}
handlers[pattern] = handler
o.Context = context.WithValue(o.Context, "http_handlers", handlers)
}
}

384
broker/http_test.go Normal file
View File

@ -0,0 +1,384 @@
package broker_test
import (
"sync"
"testing"
"time"
"github.com/google/uuid"
"github.com/micro/go-micro/v2/broker"
"github.com/micro/go-micro/v2/registry"
"github.com/micro/go-micro/v2/registry/memory"
)
var (
// mock data
testData = map[string][]*registry.Service{
"foo": {
{
Name: "foo",
Version: "1.0.0",
Nodes: []*registry.Node{
{
Id: "foo-1.0.0-123",
Address: "localhost:9999",
},
{
Id: "foo-1.0.0-321",
Address: "localhost:9999",
},
},
},
{
Name: "foo",
Version: "1.0.1",
Nodes: []*registry.Node{
{
Id: "foo-1.0.1-321",
Address: "localhost:6666",
},
},
},
{
Name: "foo",
Version: "1.0.3",
Nodes: []*registry.Node{
{
Id: "foo-1.0.3-345",
Address: "localhost:8888",
},
},
},
},
}
)
func newTestRegistry() registry.Registry {
return memory.NewRegistry(memory.Services(testData))
}
func sub(be *testing.B, c int) {
be.StopTimer()
m := newTestRegistry()
b := broker.NewBroker(broker.Registry(m))
topic := uuid.New().String()
if err := b.Init(); err != nil {
be.Fatalf("Unexpected init error: %v", err)
}
if err := b.Connect(); err != nil {
be.Fatalf("Unexpected connect error: %v", err)
}
msg := &broker.Message{
Header: map[string]string{
"Content-Type": "application/json",
},
Body: []byte(`{"message": "Hello World"}`),
}
var subs []broker.Subscriber
done := make(chan bool, c)
for i := 0; i < c; i++ {
sub, err := b.Subscribe(topic, func(p broker.Event) error {
done <- true
m := p.Message()
if string(m.Body) != string(msg.Body) {
be.Fatalf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body))
}
return nil
}, broker.Queue("shared"))
if err != nil {
be.Fatalf("Unexpected subscribe error: %v", err)
}
subs = append(subs, sub)
}
for i := 0; i < be.N; i++ {
be.StartTimer()
if err := b.Publish(topic, msg); err != nil {
be.Fatalf("Unexpected publish error: %v", err)
}
<-done
be.StopTimer()
}
for _, sub := range subs {
sub.Unsubscribe()
}
if err := b.Disconnect(); err != nil {
be.Fatalf("Unexpected disconnect error: %v", err)
}
}
func pub(be *testing.B, c int) {
be.StopTimer()
m := newTestRegistry()
b := broker.NewBroker(broker.Registry(m))
topic := uuid.New().String()
if err := b.Init(); err != nil {
be.Fatalf("Unexpected init error: %v", err)
}
if err := b.Connect(); err != nil {
be.Fatalf("Unexpected connect error: %v", err)
}
msg := &broker.Message{
Header: map[string]string{
"Content-Type": "application/json",
},
Body: []byte(`{"message": "Hello World"}`),
}
done := make(chan bool, c*4)
sub, err := b.Subscribe(topic, func(p broker.Event) error {
done <- true
m := p.Message()
if string(m.Body) != string(msg.Body) {
be.Fatalf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body))
}
return nil
}, broker.Queue("shared"))
if err != nil {
be.Fatalf("Unexpected subscribe error: %v", err)
}
var wg sync.WaitGroup
ch := make(chan int, c*4)
be.StartTimer()
for i := 0; i < c; i++ {
go func() {
for range ch {
if err := b.Publish(topic, msg); err != nil {
be.Fatalf("Unexpected publish error: %v", err)
}
select {
case <-done:
case <-time.After(time.Second):
}
wg.Done()
}
}()
}
for i := 0; i < be.N; i++ {
wg.Add(1)
ch <- i
}
wg.Wait()
be.StopTimer()
sub.Unsubscribe()
close(ch)
close(done)
if err := b.Disconnect(); err != nil {
be.Fatalf("Unexpected disconnect error: %v", err)
}
}
func TestBroker(t *testing.T) {
m := newTestRegistry()
b := broker.NewBroker(broker.Registry(m))
if err := b.Init(); err != nil {
t.Fatalf("Unexpected init error: %v", err)
}
if err := b.Connect(); err != nil {
t.Fatalf("Unexpected connect error: %v", err)
}
msg := &broker.Message{
Header: map[string]string{
"Content-Type": "application/json",
},
Body: []byte(`{"message": "Hello World"}`),
}
done := make(chan bool)
sub, err := b.Subscribe("test", func(p broker.Event) error {
m := p.Message()
if string(m.Body) != string(msg.Body) {
t.Fatalf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body))
}
close(done)
return nil
})
if err != nil {
t.Fatalf("Unexpected subscribe error: %v", err)
}
if err := b.Publish("test", msg); err != nil {
t.Fatalf("Unexpected publish error: %v", err)
}
<-done
sub.Unsubscribe()
if err := b.Disconnect(); err != nil {
t.Fatalf("Unexpected disconnect error: %v", err)
}
}
func TestConcurrentSubBroker(t *testing.T) {
m := newTestRegistry()
b := broker.NewBroker(broker.Registry(m))
if err := b.Init(); err != nil {
t.Fatalf("Unexpected init error: %v", err)
}
if err := b.Connect(); err != nil {
t.Fatalf("Unexpected connect error: %v", err)
}
msg := &broker.Message{
Header: map[string]string{
"Content-Type": "application/json",
},
Body: []byte(`{"message": "Hello World"}`),
}
var subs []broker.Subscriber
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
sub, err := b.Subscribe("test", func(p broker.Event) error {
defer wg.Done()
m := p.Message()
if string(m.Body) != string(msg.Body) {
t.Fatalf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body))
}
return nil
})
if err != nil {
t.Fatalf("Unexpected subscribe error: %v", err)
}
wg.Add(1)
subs = append(subs, sub)
}
if err := b.Publish("test", msg); err != nil {
t.Fatalf("Unexpected publish error: %v", err)
}
wg.Wait()
for _, sub := range subs {
sub.Unsubscribe()
}
if err := b.Disconnect(); err != nil {
t.Fatalf("Unexpected disconnect error: %v", err)
}
}
func TestConcurrentPubBroker(t *testing.T) {
m := newTestRegistry()
b := broker.NewBroker(broker.Registry(m))
if err := b.Init(); err != nil {
t.Fatalf("Unexpected init error: %v", err)
}
if err := b.Connect(); err != nil {
t.Fatalf("Unexpected connect error: %v", err)
}
msg := &broker.Message{
Header: map[string]string{
"Content-Type": "application/json",
},
Body: []byte(`{"message": "Hello World"}`),
}
var wg sync.WaitGroup
sub, err := b.Subscribe("test", func(p broker.Event) error {
defer wg.Done()
m := p.Message()
if string(m.Body) != string(msg.Body) {
t.Fatalf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body))
}
return nil
})
if err != nil {
t.Fatalf("Unexpected subscribe error: %v", err)
}
for i := 0; i < 10; i++ {
wg.Add(1)
if err := b.Publish("test", msg); err != nil {
t.Fatalf("Unexpected publish error: %v", err)
}
}
wg.Wait()
sub.Unsubscribe()
if err := b.Disconnect(); err != nil {
t.Fatalf("Unexpected disconnect error: %v", err)
}
}
func BenchmarkSub1(b *testing.B) {
sub(b, 1)
}
func BenchmarkSub8(b *testing.B) {
sub(b, 8)
}
func BenchmarkSub32(b *testing.B) {
sub(b, 32)
}
func BenchmarkSub64(b *testing.B) {
sub(b, 64)
}
func BenchmarkSub128(b *testing.B) {
sub(b, 128)
}
func BenchmarkPub1(b *testing.B) {
pub(b, 1)
}
func BenchmarkPub8(b *testing.B) {
pub(b, 8)
}
func BenchmarkPub32(b *testing.B) {
pub(b, 32)
}
func BenchmarkPub64(b *testing.B) {
pub(b, 64)
}
func BenchmarkPub128(b *testing.B) {
pub(b, 128)
}

222
broker/nats/nats.go
View File

@ -4,19 +4,13 @@ package nats
import (
"context"
"errors"
"net"
"net/url"
"strconv"
"strings"
"sync"
"time"
"github.com/micro/go-micro/v2/broker"
"github.com/micro/go-micro/v2/codec/json"
"github.com/micro/go-micro/v2/logger"
"github.com/micro/go-micro/v2/registry"
"github.com/micro/go-micro/v2/util/addr"
"github.com/nats-io/nats-server/v2/server"
nats "github.com/nats-io/nats.go"
)
@ -35,13 +29,6 @@ type natsBroker struct {
// should we drain the connection
drain bool
closeCh chan (error)
// embedded server
server *server.Server
// configure to use local server
local bool
// server exit channel
exit chan bool
}
type subscriber struct {
@ -108,186 +95,18 @@ func (n *natsBroker) setAddrs(addrs []string) []string {
}
cAddrs = append(cAddrs, addr)
}
// if there's no address and we weren't told to
// embed a local server then use the default url
if len(cAddrs) == 0 && !n.local {
if len(cAddrs) == 0 {
cAddrs = []string{nats.DefaultURL}
}
return cAddrs
}
// serve stats a local nats server if needed
func (n *natsBroker) serve(exit chan bool) error {
var host string
var port int
var local bool
// with no address we just default it
// this is a local client address
if len(n.addrs) == 0 {
// find an advertiseable ip
if h, err := addr.Extract(""); err != nil {
host = "127.0.0.1"
} else {
host = h
}
port = -1
local = true
} else {
address := n.addrs[0]
if strings.HasPrefix(address, "nats://") {
address = strings.TrimPrefix(address, "nats://")
}
// check if its a local address and only then embed
if addr.IsLocal(address) {
h, p, err := net.SplitHostPort(address)
if err == nil {
host = h
port, _ = strconv.Atoi(p)
local = true
}
}
}
// we only setup a server for local things
if !local {
return nil
}
// 1. create new server
// 2. register the server
// 3. connect to other servers
var cOpts server.ClusterOpts
var routes []*url.URL
// get existing nats servers to connect to
services, err := n.opts.Registry.GetService("go.micro.nats.broker")
if err == nil {
for _, service := range services {
for _, node := range service.Nodes {
u, err := url.Parse("nats://" + node.Address)
if err != nil {
if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
logger.Error(err)
}
continue
}
// append to the cluster routes
routes = append(routes, u)
}
}
}
// try get existing server
s := n.server
// get a host address
caddr, err := addr.Extract("")
if err != nil {
caddr = "0.0.0.0"
}
// set cluster opts
cOpts = server.ClusterOpts{
Host: caddr,
Port: -1,
}
if s == nil {
var err error
s, err = server.NewServer(&server.Options{
// Specify the host
Host: host,
// Use a random port
Port: port,
// Set the cluster ops
Cluster: cOpts,
// Set the routes
Routes: routes,
NoLog: true,
NoSigs: true,
MaxControlLine: 2048,
TLSConfig: n.opts.TLSConfig,
})
if err != nil {
return err
}
// save the server
n.server = s
}
// start the server
go s.Start()
var ready bool
// wait till its ready for connections
for i := 0; i < 3; i++ {
if s.ReadyForConnections(time.Second) {
ready = true
break
}
}
if !ready {
return errors.New("server not ready")
}
// set the client address
n.addrs = []string{s.ClientURL()}
go func() {
// register the cluster address
for {
select {
case <-exit:
// deregister on exit
n.opts.Registry.Deregister(&registry.Service{
Name: "go.micro.nats.broker",
Version: "v2",
Nodes: []*registry.Node{
{Id: s.ID(), Address: s.ClusterAddr().String()},
},
})
s.Shutdown()
return
default:
// register the broker
n.opts.Registry.Register(&registry.Service{
Name: "go.micro.nats.broker",
Version: "v2",
Nodes: []*registry.Node{
{Id: s.ID(), Address: s.ClusterAddr().String()},
},
}, registry.RegisterTTL(time.Minute))
time.Sleep(time.Minute)
}
}
}()
return nil
}
func (n *natsBroker) Connect() error {
n.Lock()
defer n.Unlock()
if !n.connected {
// create exit chan
n.exit = make(chan bool)
// start embedded server if asked to
if n.local {
if err := n.serve(n.exit); err != nil {
return err
}
}
// set to connected
n.connected = true
if n.connected {
return nil
}
status := nats.CLOSED
@ -297,6 +116,7 @@ func (n *natsBroker) Connect() error {
switch status {
case nats.CONNECTED, nats.RECONNECTING, nats.CONNECTING:
n.connected = true
return nil
default: // DISCONNECTED or CLOSED or DRAINING
opts := n.nopts
@ -314,13 +134,14 @@ func (n *natsBroker) Connect() error {
return err
}
n.conn = c
n.connected = true
return nil
}
}
func (n *natsBroker) Disconnect() error {
n.RLock()
defer n.RUnlock()
n.Lock()
defer n.Unlock()
// drain the connection if specified
if n.drain {
@ -331,16 +152,6 @@ func (n *natsBroker) Disconnect() error {
// close the client connection
n.conn.Close()
// shutdown the local server
// and deregister
if n.server != nil {
select {
case <-n.exit:
default:
close(n.exit)
}
}
// set not connected
n.connected = false
@ -357,19 +168,27 @@ func (n *natsBroker) Options() broker.Options {
}
func (n *natsBroker) Publish(topic string, msg *broker.Message, opts ...broker.PublishOption) error {
n.RLock()
defer n.RUnlock()
if n.conn == nil {
return errors.New("not connected")
}
b, err := n.opts.Codec.Marshal(msg)
if err != nil {
return err
}
n.RLock()
defer n.RUnlock()
return n.conn.Publish(topic, b)
}
func (n *natsBroker) Subscribe(topic string, handler broker.Handler, opts ...broker.SubscribeOption) (broker.Subscriber, error) {
n.RLock()
if n.conn == nil {
n.RUnlock()
return nil, errors.New("not connected")
}
n.RUnlock()
opt := broker.SubscribeOptions{
AutoAck: true,
@ -441,15 +260,10 @@ func (n *natsBroker) setOption(opts ...broker.Option) {
n.nopts = nopts
}
local, ok := n.opts.Context.Value(localServerKey{}).(bool)
if ok {
n.local = local
}
// broker.Options have higher priority than nats.Options
// only if Addrs, Secure or TLSConfig were not set through a broker.Option
// we read them from nats.Option
if len(n.opts.Addrs) == 0 && !n.local {
if len(n.opts.Addrs) == 0 {
n.opts.Addrs = n.nopts.Servers
}

6
broker/nats/options.go
View File

@ -7,18 +7,12 @@ import (
type optionsKey struct{}
type drainConnectionKey struct{}
type localServerKey struct{}
// Options accepts nats.Options
func Options(opts nats.Options) broker.Option {
return setBrokerOption(optionsKey{}, opts)
}
// LocalServer embeds a local server rather than connecting to one
func LocalServer() broker.Option {
return setBrokerOption(localServerKey{}, true)
}
// DrainConnection will drain subscription on close
func DrainConnection() broker.Option {
return setBrokerOption(drainConnectionKey{}, struct{}{})

12
config/cmd/cmd.go
View File

@ -36,13 +36,13 @@ import (
smucp "github.com/micro/go-micro/v2/server/mucp"
// brokers
brokerHttp "github.com/micro/go-micro/v2/broker/http"
"github.com/micro/go-micro/v2/broker/memory"
"github.com/micro/go-micro/v2/broker/nats"
brokerSrv "github.com/micro/go-micro/v2/broker/service"
// registries
"github.com/micro/go-micro/v2/registry/etcd"
kreg "github.com/micro/go-micro/v2/registry/kubernetes"
"github.com/micro/go-micro/v2/registry/mdns"
rmem "github.com/micro/go-micro/v2/registry/memory"
regSrv "github.com/micro/go-micro/v2/registry/service"
@ -320,6 +320,7 @@ var (
"service": brokerSrv.NewBroker,
"memory": memory.NewBroker,
"nats": nats.NewBroker,
"http": brokerHttp.NewBroker,
}
DefaultClients = map[string]func(...client.Option) client.Client{
@ -328,11 +329,10 @@ var (
}
DefaultRegistries = map[string]func(...registry.Option) registry.Registry{
"service": regSrv.NewRegistry,
"etcd": etcd.NewRegistry,
"mdns": mdns.NewRegistry,
"memory": rmem.NewRegistry,
"kubernetes": kreg.NewRegistry,
"service": regSrv.NewRegistry,
"etcd": etcd.NewRegistry,
"mdns": mdns.NewRegistry,
"memory": rmem.NewRegistry,
}
DefaultSelectors = map[string]func(...selector.Option) selector.Selector{

33
go.mod
View File

@ -4,10 +4,15 @@ go 1.13
require (
github.com/BurntSushi/toml v0.3.1
github.com/beevik/ntp v0.2.0
github.com/bitly/go-simplejson v0.5.0
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
github.com/bwmarrin/discordgo v0.20.2
github.com/coreos/bbolt v1.3.3 // indirect
github.com/coreos/etcd v3.3.18+incompatible
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/davecgh/go-spew v1.1.1
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/ef-ds/deque v1.0.4-0.20190904040645-54cb57c252a1
@ -17,40 +22,50 @@ require (
github.com/fsouza/go-dockerclient v1.6.0
github.com/ghodss/yaml v1.0.0
github.com/go-acme/lego/v3 v3.3.0
github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee
github.com/gobwas/pool v0.2.0 // indirect
github.com/gobwas/ws v1.0.3
github.com/golang/protobuf v1.3.2
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/protobuf v1.3.5
github.com/google/go-cmp v0.4.0 // indirect
github.com/google/uuid v1.1.1
github.com/gorilla/handlers v1.4.2
github.com/gorilla/websocket v1.4.1
github.com/gorilla/websocket v1.4.1 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.9.5
github.com/hashicorp/hcl v1.0.0
github.com/hpcloud/tail v1.0.0
github.com/imdario/mergo v0.3.8
github.com/jonboulle/clockwork v0.1.0 // indirect
github.com/joncalhoun/qson v0.0.0-20170526102502-8a9cab3a62b1
github.com/json-iterator/go v1.1.9 // indirect
github.com/kr/pretty v0.1.0
github.com/lib/pq v1.3.0
github.com/lucas-clemente/quic-go v0.14.1
github.com/mholt/certmagic v0.9.3
github.com/micro/cli/v2 v2.1.2
github.com/micro/mdns v0.3.0
github.com/micro/micro/v2 v2.4.0
github.com/miekg/dns v1.1.27
github.com/mitchellh/hashstructure v1.0.0
github.com/nats-io/nats-server/v2 v2.1.4
github.com/nats-io/nats.go v1.9.1
github.com/nats-io/nats-server/v2 v2.1.6 // indirect
github.com/nats-io/nats.go v1.9.2
github.com/nlopes/slack v0.6.1-0.20191106133607-d06c2a2b3249
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pkg/errors v0.9.1
github.com/soheilhy/cmux v0.1.4 // indirect
github.com/stretchr/testify v1.4.0
github.com/technoweenie/multipartstreamer v1.0.1 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
go.etcd.io/bbolt v1.3.4
go.uber.org/zap v1.13.0
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59
golang.org/x/net v0.0.0-20200222125558-5a598a2470a0
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
google.golang.org/grpc v1.26.0
gopkg.in/go-playground/validator.v9 v9.31.0
gopkg.in/src-d/go-git.v4 v4.13.1
gopkg.in/telegram-bot-api.v4 v4.6.4
sigs.k8s.io/yaml v1.1.0 // indirect
)

76
go.sum
View File

@ -53,8 +53,6 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/aws/aws-sdk-go v1.23.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
github.com/beevik/ntp v0.2.0 h1:sGsd+kAXzT0bfVfzJfce04g+dSRfrs+tbQW8lweuYgw=
github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@ -64,9 +62,6 @@ github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngE
github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/bwmarrin/discordgo v0.20.2 h1:nA7jiTtqUA9lT93WL2jPjUp8ZTEInRujBdx1C9gkr20=
github.com/bwmarrin/discordgo v0.20.2/go.mod h1:O9S4p+ofTFwB02em7jkpkV8M3R0/PUVOwN61zSZ0r4Q=
github.com/cenkalti/backoff/v3 v3.0.0 h1:ske+9nBpD9qZsTBoF41nW5L+AIuFBKMeze18XQ3eG1c=
@ -75,14 +70,9 @@ github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/cloudflare-go v0.10.2 h1:VBodKICVPnwmDxstcW3biKcDSpFIfS/RELUXsZSBYK4=
github.com/cloudflare/cloudflare-go v0.10.2/go.mod h1:qhVI5MKwBGhdNU89ZRz2plgYutcJ5PCekLxXn56w6SY=
github.com/cloudflare/cloudflare-go v0.10.9 h1:d8KOgLpYiC+Xq3T4tuO+/goM+RZvuO+T4pojuv8giL8=
github.com/cloudflare/cloudflare-go v0.10.9/go.mod h1:5TrsWH+3f4NV6WjtS5QFp+DifH81rph40gU374Sh0dQ=
github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
@ -94,6 +84,7 @@ github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv
github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY=
github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.18+incompatible h1:Zz1aXgDrFFi1nadh58tA9ktt06cmPTwNNP3dXwIq1lE=
github.com/coreos/etcd v3.3.18+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@ -131,7 +122,6 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/ef-ds/deque v1.0.4-0.20190904040645-54cb57c252a1 h1:jFGzikHboUMRXmMBtwD/PbxoTHPs2919Irp/3rxMbvM=
github.com/ef-ds/deque v1.0.4-0.20190904040645-54cb57c252a1/go.mod h1:HvODWzv6Y6kBf3Ah2WzN1bHjDUezGLaAhwuWVwfpEJs=
github.com/eknkc/basex v1.0.0/go.mod h1:k/F/exNEHFdbs3ZHuasoP2E7zeWwZblG84Y7Z59vQRo=
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@ -144,7 +134,6 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjr
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/forestgiant/sliceutil v0.0.0-20160425183142-94783f95db6c h1:pBgVXWDXju1m8W4lnEeIqTHPOzhTUO81a7yknM/xQR4=
github.com/forestgiant/sliceutil v0.0.0-20160425183142-94783f95db6c/go.mod h1:pFdJbAhRf7rh6YYMUdIQGyzne6zYL1tCUW8QV2B3UfY=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsouza/go-dockerclient v1.6.0 h1:f7j+AX94143JL1H3TiqSMkM4EcLDI0De1qD4GGn3Hig=
@ -162,10 +151,6 @@ github.com/go-ini/ini v1.44.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible h1:2cauKuaELYAEARXRkq2LrJ0yDDv1rW7+wrTEdVL3uaU=
github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible/go.mod h1:qf9acutJ8cwBUhm1bqgz6Bei9/C/c93FPDljKWwsOgM=
@ -195,6 +180,8 @@ github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
@ -204,14 +191,12 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@ -235,7 +220,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI=
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
github.com/hako/branca v0.0.0-20180808000428-10b799466ada/go.mod h1:tOPn4gvKEUWqIJNE+zpTeTALaRAXnrRqqSnPlO3VpEo=
github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@ -283,26 +267,20 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/labbsr0x/bindman-dns-webhook v1.0.2/go.mod h1:p6b+VCXIR8NYKpDr8/dg1HKfQoRHCdcsROXKvmoehKA=
github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c027w=
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/linode/linodego v0.10.0/go.mod h1:cziNP7pbvE3mXIPneHj0oRY8L1WtGEIKlZ8LANE4eXA=
github.com/liquidweb/liquidweb-go v1.6.0/go.mod h1:UDcVnAMDkZxpw4Y7NOHkqoeiGacVLEIG/i5J9cyixzQ=
github.com/lucas-clemente/quic-go v0.14.1 h1:c1aKoBZKOPA+49q96B1wGkibyPP0AxYh45WuAoq+87E=
github.com/lucas-clemente/quic-go v0.14.1/go.mod h1:Vn3/Fb0/77b02SGhQk36KzOUmXgVpFfizUfW5WMaqyU=
github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/marten-seemann/chacha20 v0.2.0 h1:f40vqzzx+3GdOmzQoItkLX5WLvHgPgyYqFFIO5Gh4hQ=
github.com/marten-seemann/chacha20 v0.2.0/go.mod h1:HSdjFau7GzYRj+ahFNwsO3ouVJr1HFkWoEwNDb4TMtE=
github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
github.com/marten-seemann/qtls v0.4.1 h1:YlT8QP3WCCvvok7MGEZkMldXbyqgr8oFg5/n8Gtbkks=
github.com/marten-seemann/qtls v0.4.1/go.mod h1:pxVXcHHw1pNIt8Qo0pwSYQEoZ8yYOOPXTCZLQQunvRc=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@ -310,13 +288,6 @@ github.com/mholt/certmagic v0.9.3 h1:RmzuNJ5mpFplDbyS41z+gGgE/py24IX6m0nHZ0yNTQU
github.com/mholt/certmagic v0.9.3/go.mod h1:nu8jbsbtwK4205EDH/ZUMTKsfYpJA1Q7MKXHfgTihNw=
github.com/micro/cli/v2 v2.1.2 h1:43J1lChg/rZCC1rvdqZNFSQDrGT7qfMrtp6/ztpIkEM=
github.com/micro/cli/v2 v2.1.2/go.mod h1:EguNh6DAoWKm9nmk+k/Rg0H3lQnDxqzu5x5srOtGtYg=
github.com/micro/go-micro v1.18.0 h1:gP70EZVHpJuUIT0YWth192JmlIci+qMOEByHm83XE9E=
github.com/micro/go-micro/v2 v2.3.1-0.20200331090613-76ade7efd9b8/go.mod h1:lYuHYFPjY3QE9fdiy3F2awXcsXTdB68AwoY3RQ3dPN4=
github.com/micro/mdns v0.3.0 h1:bYycYe+98AXR3s8Nq5qvt6C573uFTDPIYzJemWON0QE=
github.com/micro/mdns v0.3.0/go.mod h1:KJ0dW7KmicXU2BV++qkLlmHYcVv7/hHnbtguSWt9Aoc=
github.com/micro/micro/v2 v2.4.0 h1:GlbLaD/50KaSFym7GCQZ/2I4fuTTX9U4Zftni4ImJ40=
github.com/micro/micro/v2 v2.4.0/go.mod h1:/7lxBaU/Isx3USObggNVw6x6pdIJzTDexee7EsARD+A=
github.com/miekg/dns v1.1.3/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM=
github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
@ -336,20 +307,18 @@ github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8d
github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04/go.mod h1:5sN+Lt1CaY4wsPvgQH/jsuJi4XO2ssZbdsIizr4CVC8=
github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI=
github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
github.com/nats-io/nats-server/v2 v2.1.4 h1:BILRnsJ2Yb/fefiFbBWADpViGF69uh4sxe8poVDQ06g=
github.com/nats-io/nats-server/v2 v2.1.4/go.mod h1:Jw1Z28soD/QasIA2uWjXyM9El1jly3YwyFOuR8tH1rg=
github.com/nats-io/nats.go v1.9.1 h1:ik3HbLhZ0YABLto7iX80pZLPw/6dx3T+++MZJwLnMrQ=
github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nkeys v0.1.3 h1:6JrEfig+HzTH85yxzhSVbjHRJv9cn0p6n3IngIcM5/k=
github.com/nats-io/nats-server/v2 v2.1.6 h1:qAaHZaS8pRRNQLFaiBA1rq5WynyEGp9DFgmMfoaiXGY=
github.com/nats-io/nats-server/v2 v2.1.6/go.mod h1:BL1NOtaBQ5/y97djERRVWNouMW7GT3gxnmbE/eC8u8A=
github.com/nats-io/nats.go v1.9.2 h1:oDeERm3NcZVrPpdR/JpGdWHMv3oJ8yY30YwxKq+DU2s=
github.com/nats-io/nats.go v1.9.2/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE=
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nkeys v0.1.4 h1:aEsHIssIk6ETN5m2/MD8Y4B2X7FfXrBAUdkyRvbVYzA=
github.com/nats-io/nkeys v0.1.4/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
github.com/netdata/go-orchestrator v0.0.0-20190905093727-c793edba0e8f/go.mod h1:ECF8anFVCt/TfTIWVPgPrNaYJXtAtpAOF62ugDbw41A=
github.com/nlopes/slack v0.6.1-0.20191106133607-d06c2a2b3249 h1:Pr5gZa2VcmktVwq0lyC39MsN5tz356vC/pQHKvq+QBo=
github.com/nlopes/slack v0.6.1-0.20191106133607-d06c2a2b3249/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
github.com/nrdcg/auroradns v1.0.0/go.mod h1:6JPXKzIRzZzMqtTDgueIhTi6rFf1QvYE/HzqidhOhjw=
@ -357,8 +326,6 @@ github.com/nrdcg/dnspod-go v0.3.0/go.mod h1:vZSoFSFeQVm2gWLMkyX61LZ8HI3BaqtHZWgP
github.com/nrdcg/goinwx v0.6.1/go.mod h1:XPiut7enlbEdntAqalBIqcYcTEVhpv/dKWgDCX2SwKQ=
github.com/nrdcg/namesilo v0.2.1/go.mod h1:lwMvfQTyYq+BbjJd30ylEG4GPSS6PII0Tia4rRpRiyw=
github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/olekukonko/tablewriter v0.0.3/go.mod h1:YZeBtGzYYEsCHp2LST/u/0NDwGkRoBtmn1cIWCJiS6M=
github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@ -382,7 +349,6 @@ github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgF
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -391,7 +357,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pquerna/otp v1.2.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
@ -420,7 +385,6 @@ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sacloud/libsacloud v1.26.1/go.mod h1:79ZwATmHLIFZIMd7sxA3LwzVy/B77uj3LDoToVTxDoQ=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516/go.mod h1:Yow6lPLSAXx2ifx470yD/nUe22Dv5vBvxK/UK9UUTVs=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
@ -431,7 +395,6 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@ -454,7 +417,6 @@ github.com/transip/gotransip v0.0.0-20190812104329-6d8d9179b66f/go.mod h1:i0f4R4
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/vultr/govultr v0.1.4/go.mod h1:9H008Uxr/C4vFNGLqKx232C206GL0PBHzOP0809bGNA=
github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
@ -464,8 +426,6 @@ github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf
github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
@ -488,7 +448,6 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190130090550-b01c7a725664/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@ -499,8 +458,8 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw=
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM=
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -543,14 +502,11 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190930134127-c5a3c61f89f3/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200222125558-5a598a2470a0 h1:MsuvTghUPjX762sGLnGsxC3HM0B5r83wEtYcYR8/vRs=
golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -565,11 +521,9 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -585,7 +539,6 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -658,23 +611,16 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M=
gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ns1/ns1-go.v2 v2.0.0-20190730140822-b51389932cbc/go.mod h1:VV+3haRsgDiVLxyifmMBrBIuCWFBPYKbRssXB9z67Hw=
gopkg.in/olivere/elastic.v5 v5.0.83/go.mod h1:LXF6q9XNBxpMqrcgax95C6xyARXWbbCXUrtTxrNrxJI=
gopkg.in/resty.v1 v1.9.1/go.mod h1:vo52Hzryw9PnPHcJfPsBiFW62XhNx5OczbV9y+IMpgc=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4=

16
metadata/metadata.go
View File

@ -6,7 +6,7 @@ import (
"strings"
)
type MetadataKey struct{}
type metadataKey struct{}
// Metadata is our way of representing request headers internally.
// They're used at the RPC level and translate back and forth
@ -25,6 +25,10 @@ func (md Metadata) Get(key string) (string, bool) {
return val, ok
}
func (md Metadata) Set(key, val string) {
md[key] = val
}
func (md Metadata) Delete(key string) {
// delete key as-is
delete(md, key)
@ -57,7 +61,7 @@ func Set(ctx context.Context, k, v string) context.Context {
} else {
md[k] = v
}
return context.WithValue(ctx, MetadataKey{}, md)
return context.WithValue(ctx, metadataKey{}, md)
}
// Get returns a single value from metadata in the context
@ -80,7 +84,7 @@ func Get(ctx context.Context, key string) (string, bool) {
// FromContext returns metadata from the given context
func FromContext(ctx context.Context) (Metadata, bool) {
md, ok := ctx.Value(MetadataKey{}).(Metadata)
md, ok := ctx.Value(metadataKey{}).(Metadata)
if !ok {
return nil, ok
}
@ -96,7 +100,7 @@ func FromContext(ctx context.Context) (Metadata, bool) {
// NewContext creates a new context with the given metadata
func NewContext(ctx context.Context, md Metadata) context.Context {
return context.WithValue(ctx, MetadataKey{}, md)
return context.WithValue(ctx, metadataKey{}, md)
}
// MergeContext merges metadata to existing metadata, overwriting if specified
@ -104,7 +108,7 @@ func MergeContext(ctx context.Context, patchMd Metadata, overwrite bool) context
if ctx == nil {
ctx = context.Background()
}
md, _ := ctx.Value(MetadataKey{}).(Metadata)
md, _ := ctx.Value(metadataKey{}).(Metadata)
cmd := make(Metadata, len(md))
for k, v := range md {
cmd[k] = v
@ -118,5 +122,5 @@ func MergeContext(ctx context.Context, patchMd Metadata, overwrite bool) context
delete(cmd, k)
}
}
return context.WithValue(ctx, MetadataKey{}, cmd)
return context.WithValue(ctx, metadataKey{}, cmd)
}

2
proxy/mucp/mucp.go
View File

@ -206,7 +206,7 @@ func (p *Proxy) cacheRoutes(service string) ([]router.Route, error) {
results, err := p.Router.Lookup(router.QueryService(service))
if err != nil {
// assumption that we're ok with stale routes
logger.Debugf("Failed to lookup route for %s: %v", service, err)
// otherwise return the error
return nil, err
}

5
registry/cache/cache.go vendored
View File

@ -9,6 +9,7 @@ import (
"github.com/micro/go-micro/v2/logger"
"github.com/micro/go-micro/v2/registry"
util "github.com/micro/go-micro/v2/util/registry"
)
// Cache is the registry cache interface
@ -119,7 +120,7 @@ func (c *cache) get(service string) ([]*registry.Service, error) {
// get cache ttl
ttl := c.ttls[service]
// make a copy
cp := registry.Copy(services)
cp := util.Copy(services)
// got services && within ttl so return cache
if c.isValid(cp, ttl) {
@ -152,7 +153,7 @@ func (c *cache) get(service string) ([]*registry.Service, error) {
// cache results
c.Lock()
c.set(service, registry.Copy(services))
c.set(service, util.Copy(services))
c.Unlock()
return services, nil

73
registry/encoding.go
View File

@ -1,73 +0,0 @@
package registry
import (
"bytes"
"compress/zlib"
"encoding/hex"
"encoding/json"
"io/ioutil"
"strings"
)
func encode(txt *mdnsTxt) ([]string, error) {
b, err := json.Marshal(txt)
if err != nil {
return nil, err
}
var buf bytes.Buffer
defer buf.Reset()
w := zlib.NewWriter(&buf)
if _, err := w.Write(b); err != nil {
return nil, err
}
w.Close()
encoded := hex.EncodeToString(buf.Bytes())
// individual txt limit
if len(encoded) <= 255 {
return []string{encoded}, nil
}
// split encoded string
var record []string
for len(encoded) > 255 {
record = append(record, encoded[:255])
encoded = encoded[255:]
}
record = append(record, encoded)
return record, nil
}
func decode(record []string) (*mdnsTxt, error) {
encoded := strings.Join(record, "")
hr, err := hex.DecodeString(encoded)
if err != nil {
return nil, err
}
br := bytes.NewReader(hr)
zr, err := zlib.NewReader(br)
if err != nil {
return nil, err
}
rbuf, err := ioutil.ReadAll(zr)
if err != nil {
return nil, err
}
var txt *mdnsTxt
if err := json.Unmarshal(rbuf, &txt); err != nil {
return nil, err
}
return txt, nil
}

65
registry/encoding_test.go
View File

@ -1,65 +0,0 @@
package registry
import (
"testing"
)
func TestEncoding(t *testing.T) {
testData := []*mdnsTxt{
{
Version: "1.0.0",
Metadata: map[string]string{
"foo": "bar",
},
Endpoints: []*Endpoint{
{
Name: "endpoint1",
Request: &Value{
Name: "request",
Type: "request",
},
Response: &Value{
Name: "response",
Type: "response",
},
Metadata: map[string]string{
"foo1": "bar1",
},
},
},
},
}
for _, d := range testData {
encoded, err := encode(d)
if err != nil {
t.Fatal(err)
}
for _, txt := range encoded {
if len(txt) > 255 {
t.Fatalf("One of parts for txt is %d characters", len(txt))
}
}
decoded, err := decode(encoded)
if err != nil {
t.Fatal(err)
}
if decoded.Version != d.Version {
t.Fatalf("Expected version %s got %s", d.Version, decoded.Version)
}
if len(decoded.Endpoints) != len(d.Endpoints) {
t.Fatalf("Expected %d endpoints, got %d", len(d.Endpoints), len(decoded.Endpoints))
}
for k, v := range d.Metadata {
if val := decoded.Metadata[k]; val != v {
t.Fatalf("Expected %s=%s got %s=%s", k, v, k, val)
}
}
}
}

66
registry/kubernetes/README.md
View File

@ -1,66 +0,0 @@
# Kubernetes Registry Plugin for micro
This is a plugin for go-micro that allows you to use Kubernetes as a registry.
## Overview
This registry plugin makes use of Annotations and Labels on a Kubernetes pod
to build a service discovery mechanism.
## RBAC
If your Kubernetes cluster has RBAC enabled, a role and role binding
will need to be created to allow this plugin to `list` and `patch` pods.
A cluster role can be used to specify the `list` and `patch`
requirements, while a role binding per namespace can be used to apply
the cluster role. The example RBAC configs below assume your Micro-based
services are running in the `test` namespace, and the pods that contain
the services are using the `micro-services` service account.
```
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: micro-registry
rules:
- apiGroups:
- ""
resources:
- pods
verbs:
- list
- patch
- watch
```
```
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: micro-registry
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: micro-registry
subjects:
- kind: ServiceAccount
name: micro-services
namespace: test
```
## Gotchas
* Registering/Deregistering relies on the HOSTNAME Environment Variable, which inside a pod
is the place where it can be retrieved from. (This needs improving)
## Connecting to the Kubernetes API
### Within a pod
If the `--registry_address` flag is omitted, the plugin will securely connect to
the Kubernetes API using the pods "Service Account". No extra configuration is necessary.
Find out more about service accounts here. http://kubernetes.io/docs/user-guide/accessing-the-cluster/
### Outside of Kubernetes
Some functions of the plugin should work, but its not been heavily tested.
Currently no TLS support.

289
registry/kubernetes/kubernetes.go
View File

@ -1,289 +0,0 @@
// Package kubernetes provides a kubernetes registry
package kubernetes
import (
"encoding/json"
"errors"
"fmt"
"os"
"regexp"
"strings"
"time"
"github.com/micro/go-micro/v2/registry"
"github.com/micro/go-micro/v2/util/kubernetes/client"
)
type kregistry struct {
client client.Client
timeout time.Duration
options registry.Options
}
var (
// used on pods as labels & services to select
// eg: svcSelectorPrefix+"svc.name"
servicePrefix = "go.micro/"
serviceValue = "service"
labelTypeKey = "micro"
labelTypeValue = "service"
// used on k8s services to scope a serialised
// micro service by pod name
annotationPrefix = "go.micro/"
// Pod status
podRunning = "Running"
// label name regex
labelRe = regexp.MustCompilePOSIX("[-A-Za-z0-9_.]")
)
// podSelector
var podSelector = map[string]string{
labelTypeKey: labelTypeValue,
}
func configure(k *kregistry, opts ...registry.Option) error {
for _, o := range opts {
o(&k.options)
}
// get first host
var host string
if len(k.options.Addrs) > 0 && len(k.options.Addrs[0]) > 0 {
host = k.options.Addrs[0]
}
if k.options.Timeout == 0 {
k.options.Timeout = time.Second * 1
}
// if no hosts setup, assume InCluster
var c client.Client
if len(host) > 0 {
c = client.NewLocalClient(host)
} else {
c = client.NewClusterClient()
}
k.client = c
k.timeout = k.options.Timeout
return nil
}
// serviceName generates a valid service name for k8s labels
func serviceName(name string) string {
aname := make([]byte, len(name))
for i, r := range []byte(name) {
if !labelRe.Match([]byte{r}) {
aname[i] = '_'
continue
}
aname[i] = r
}
return string(aname)
}
// Init allows reconfig of options
func (c *kregistry) Init(opts ...registry.Option) error {
return configure(c, opts...)
}
// Options returns the registry Options
func (c *kregistry) Options() registry.Options {
return c.options
}
// Register sets a service selector label and an annotation with a
// serialised version of the service passed in.
func (c *kregistry) Register(s *registry.Service, opts ...registry.RegisterOption) error {
if len(s.Nodes) == 0 {
return errors.New("no nodes")
}
// TODO: grab podname from somewhere better than this.
podName := os.Getenv("HOSTNAME")
svcName := s.Name
// encode micro service
b, err := json.Marshal(s)
if err != nil {
return err
}
/// marshalled service
svc := string(b)
pod := &client.Pod{
Metadata: &client.Metadata{
Labels: map[string]string{
// micro: service
labelTypeKey: labelTypeValue,
// micro/service/name: service
servicePrefix + serviceName(svcName): serviceValue,
},
Annotations: map[string]string{
// micro/service/name: definition
annotationPrefix + serviceName(svcName): svc,
},
},
}
return c.client.Update(&client.Resource{
Name: podName,
Kind: "pod",
Value: pod,
})
}
// Deregister nils out any things set in Register
func (c *kregistry) Deregister(s *registry.Service) error {
if len(s.Nodes) == 0 {
return errors.New("you must deregister at least one node")
}
// TODO: grab podname from somewhere better than this.
podName := os.Getenv("HOSTNAME")
svcName := s.Name
pod := &client.Pod{
Metadata: &client.Metadata{
Labels: map[string]string{
servicePrefix + serviceName(svcName): "",
},
Annotations: map[string]string{
annotationPrefix + serviceName(svcName): "",
},
},
}
return c.client.Update(&client.Resource{
Name: podName,
Kind: "pod",
Value: pod,
})
}
// GetService will get all the pods with the given service selector,
// and build services from the annotations.
func (c *kregistry) GetService(name string) ([]*registry.Service, error) {
var pods client.PodList
if err := c.client.Get(&client.Resource{
Kind: "pod",
Value: &pods,
}, map[string]string{
servicePrefix + serviceName(name): serviceValue,
}); err != nil {
return nil, err
}
if len(pods.Items) == 0 {
return nil, registry.ErrNotFound
}
// svcs mapped by version
svcs := make(map[string]*registry.Service)
// loop through items
for _, pod := range pods.Items {
if pod.Status.Phase != podRunning {
continue
}
// get serialised service from annotation
svcStr, ok := pod.Metadata.Annotations[annotationPrefix+serviceName(name)]
if !ok {
continue
}
// unmarshal service string
var svc registry.Service
if err := json.Unmarshal([]byte(svcStr), &svc); err != nil {
return nil, fmt.Errorf("could not unmarshal service '%s' from pod annotation", name)
}
// merge up pod service & ip with versioned service.
vs, ok := svcs[svc.Version]
if !ok {
svcs[svc.Version] = &svc
continue
}
vs.Nodes = append(vs.Nodes, svc.Nodes...)
}
list := make([]*registry.Service, 0, len(svcs))
for _, val := range svcs {
list = append(list, val)
}
return list, nil
}
// ListServices will list all the service names
func (c *kregistry) ListServices() ([]*registry.Service, error) {
var pods client.PodList
if err := c.client.Get(&client.Resource{
Kind: "pod",
Value: &pods,
}, podSelector); err != nil {
return nil, err
}
// svcs mapped by name
svcs := make(map[string]bool)
for _, pod := range pods.Items {
if pod.Status.Phase != podRunning {
continue
}
for k, v := range pod.Metadata.Annotations {
if !strings.HasPrefix(k, annotationPrefix) {
continue
}
// we have to unmarshal the annotation itself since the
// key is encoded to match the regex restriction.
var svc registry.Service
if err := json.Unmarshal([]byte(v), &svc); err != nil {
continue
}
svcs[svc.Name] = true
}
}
var list []*registry.Service
for val := range svcs {
list = append(list, &registry.Service{Name: val})
}
return list, nil
}
// Watch returns a kubernetes watcher
func (c *kregistry) Watch(opts ...registry.WatchOption) (registry.Watcher, error) {
return newWatcher(c, opts...)
}
func (c *kregistry) String() string {
return "kubernetes"
}
// NewRegistry creates a kubernetes registry
func NewRegistry(opts ...registry.Option) registry.Registry {
k := &kregistry{
options: registry.Options{},
}
configure(k, opts...)
return k
}

263
registry/kubernetes/watcher.go
View File

@ -1,263 +0,0 @@
package kubernetes
import (
"encoding/json"
"errors"
"strings"
"sync"
"github.com/micro/go-micro/v2/logger"
"github.com/micro/go-micro/v2/registry"
"github.com/micro/go-micro/v2/util/kubernetes/client"
)
type k8sWatcher struct {
registry *kregistry
watcher client.Watcher
next chan *registry.Result
stop chan bool
sync.RWMutex
pods map[string]*client.Pod
}
// build a cache of pods when the watcher starts.
func (k *k8sWatcher) updateCache() ([]*registry.Result, error) {
var pods client.PodList
if err := k.registry.client.Get(&client.Resource{
Kind: "pod",
Value: &pods,
}, podSelector); err != nil {
return nil, err
}
var results []*registry.Result
for _, pod := range pods.Items {
rslts := k.buildPodResults(&pod, nil)
for _, r := range rslts {
results = append(results, r)
}
k.Lock()
k.pods[pod.Metadata.Name] = &pod
k.Unlock()
}
return results, nil
}
// look through pod annotations, compare against cache if present
// and return a list of results to send down the wire.
func (k *k8sWatcher) buildPodResults(pod *client.Pod, cache *client.Pod) []*registry.Result {
var results []*registry.Result
ignore := make(map[string]bool)
if pod.Metadata != nil {
for ak, av := range pod.Metadata.Annotations {
// check this annotation kv is a service notation
if !strings.HasPrefix(ak, annotationPrefix) {
continue
}
if len(av) == 0 {
continue
}
// ignore when we check the cached annotations
// as we take care of it here
ignore[ak] = true
// compare aginst cache.
var cacheExists bool
var cav string
if cache != nil && cache.Metadata != nil {
cav, cacheExists = cache.Metadata.Annotations[ak]
if cacheExists && len(cav) > 0 && cav == av {
// service notation exists and is identical -
// no change result required.
continue
}
}
rslt := &registry.Result{}
if cacheExists {
rslt.Action = "update"
} else {
rslt.Action = "create"
}
// unmarshal service notation from annotation value
err := json.Unmarshal([]byte(av), &rslt.Service)
if err != nil {
continue
}
results = append(results, rslt)
}
}
// loop through cache annotations to find services
// not accounted for above, and "delete" them.
if cache != nil && cache.Metadata != nil {
for ak, av := range cache.Metadata.Annotations {
if ignore[ak] {
continue
}
// check this annotation kv is a service notation
if !strings.HasPrefix(ak, annotationPrefix) {
continue
}
rslt := &registry.Result{Action: "delete"}
// unmarshal service notation from annotation value
err := json.Unmarshal([]byte(av), &rslt.Service)
if err != nil {
continue
}
results = append(results, rslt)
}
}
return results
}
// handleEvent will taken an event from the k8s pods API and do the correct
// things with the result, based on the local cache.
func (k *k8sWatcher) handleEvent(event client.Event) {
var pod client.Pod
if err := json.Unmarshal([]byte(event.Object), &pod); err != nil {
if logger.V(logger.InfoLevel, logger.DefaultLogger) {
logger.Info("K8s Watcher: Couldnt unmarshal event object from pod")
}
return
}
switch event.Type {
case client.Modified:
// Pod was modified
k.RLock()
cache := k.pods[pod.Metadata.Name]
k.RUnlock()
// service could have been added, edited or removed.
var results []*registry.Result
if pod.Status.Phase == podRunning {
results = k.buildPodResults(&pod, cache)
} else {
// passing in cache might not return all results
results = k.buildPodResults(&pod, nil)
}
for _, result := range results {
// pod isnt running
if pod.Status.Phase != podRunning {
result.Action = "delete"
}
select {
case k.next <- result:
case <-k.stop:
return
}
}
k.Lock()
k.pods[pod.Metadata.Name] = &pod
k.Unlock()
return
case client.Deleted:
// Pod was deleted
// passing in cache might not return all results
results := k.buildPodResults(&pod, nil)
for _, result := range results {
result.Action = "delete"
select {
case k.next <- result:
case <-k.stop:
return
}
}
k.Lock()
delete(k.pods, pod.Metadata.Name)
k.Unlock()
return
}
}
// Next will block until a new result comes in
func (k *k8sWatcher) Next() (*registry.Result, error) {
select {
case r := <-k.next:
return r, nil
case <-k.stop:
return nil, errors.New("watcher stopped")
}
}
// Stop will cancel any requests, and close channels
func (k *k8sWatcher) Stop() {
select {
case <-k.stop:
return
default:
k.watcher.Stop()
close(k.stop)
}
}
func newWatcher(kr *kregistry, opts ...registry.WatchOption) (registry.Watcher, error) {
var wo registry.WatchOptions
for _, o := range opts {
o(&wo)
}
selector := podSelector
if len(wo.Service) > 0 {
selector = map[string]string{
servicePrefix + serviceName(wo.Service): serviceValue,
}
}
// Create watch request
watcher, err := kr.client.Watch(&client.Resource{
Kind: "pod",
}, client.WatchParams(selector))
if err != nil {
return nil, err
}
k := &k8sWatcher{
registry: kr,
watcher: watcher,
next: make(chan *registry.Result),
stop: make(chan bool),
pods: make(map[string]*client.Pod),
}
// update cache, but dont emit changes
if _, err := k.updateCache(); err != nil {
return nil, err
}
// range over watch request changes, and invoke
// the update event
go func() {
for event := range watcher.Chan() {
k.handleEvent(event)
}
}()
return k, nil
}

148
registry/mdns_registry.go
View File

@ -2,8 +2,13 @@
package registry
import (
"bytes"
"compress/zlib"
"context"
"encoding/hex"
"encoding/json"
"fmt"
"io/ioutil"
"net"
"strconv"
"strings"
@ -12,7 +17,7 @@ import (
"github.com/google/uuid"
"github.com/micro/go-micro/v2/logger"
"github.com/micro/mdns"
"github.com/micro/go-micro/v2/util/mdns"
)
var (
@ -49,6 +54,79 @@ type mdnsRegistry struct {
listener chan *mdns.ServiceEntry
}
type mdnsWatcher struct {
id string
wo WatchOptions
ch chan *mdns.ServiceEntry
exit chan struct{}
// the mdns domain
domain string
// the registry
registry *mdnsRegistry
}
func encode(txt *mdnsTxt) ([]string, error) {
b, err := json.Marshal(txt)
if err != nil {
return nil, err
}
var buf bytes.Buffer
defer buf.Reset()
w := zlib.NewWriter(&buf)
if _, err := w.Write(b); err != nil {
return nil, err
}
w.Close()
encoded := hex.EncodeToString(buf.Bytes())
// individual txt limit
if len(encoded) <= 255 {
return []string{encoded}, nil
}
// split encoded string
var record []string
for len(encoded) > 255 {
record = append(record, encoded[:255])
encoded = encoded[255:]
}
record = append(record, encoded)
return record, nil
}
func decode(record []string) (*mdnsTxt, error) {
encoded := strings.Join(record, "")
hr, err := hex.DecodeString(encoded)
if err != nil {
return nil, err
}
br := bytes.NewReader(hr)
zr, err := zlib.NewReader(br)
if err != nil {
return nil, err
}
rbuf, err := ioutil.ReadAll(zr)
if err != nil {
return nil, err
}
var txt *mdnsTxt
if err := json.Unmarshal(rbuf, &txt); err != nil {
return nil, err
}
return txt, nil
}
func newRegistry(opts ...Option) Registry {
options := Options{
Context: context.Background(),
@ -467,6 +545,74 @@ func (m *mdnsRegistry) String() string {
return "mdns"
}
func (m *mdnsWatcher) Next() (*Result, error) {
for {
select {
case e := <-m.ch:
txt, err := decode(e.InfoFields)
if err != nil {
continue
}
if len(txt.Service) == 0 || len(txt.Version) == 0 {
continue
}
// Filter watch options
// wo.Service: Only keep services we care about
if len(m.wo.Service) > 0 && txt.Service != m.wo.Service {
continue
}
var action string
if e.TTL == 0 {
action = "delete"
} else {
action = "create"
}
service := &Service{
Name: txt.Service,
Version: txt.Version,
Endpoints: txt.Endpoints,
}
// skip anything without the domain we care about
suffix := fmt.Sprintf(".%s.%s.", service.Name, m.domain)
if !strings.HasSuffix(e.Name, suffix) {
continue
}
service.Nodes = append(service.Nodes, &Node{
Id: strings.TrimSuffix(e.Name, suffix),
Address: fmt.Sprintf("%s:%d", e.AddrV4.String(), e.Port),
Metadata: txt.Metadata,
})
return &Result{
Action: action,
Service: service,
}, nil
case <-m.exit:
return nil, ErrWatcherStopped
}
}
}
func (m *mdnsWatcher) Stop() {
select {
case <-m.exit:
return
default:
close(m.exit)
// remove self from the registry
m.registry.mtx.Lock()
delete(m.registry.watchers, m.id)
m.registry.mtx.Unlock()
}
}
// NewRegistry returns a new default registry which is mdns
func NewRegistry(opts ...Option) Registry {
return newRegistry(opts...)

202
registry/mdns_test.go
View File

@ -137,3 +137,205 @@ func TestMDNS(t *testing.T) {
}
}
func TestEncoding(t *testing.T) {
testData := []*mdnsTxt{
{
Version: "1.0.0",
Metadata: map[string]string{
"foo": "bar",
},
Endpoints: []*Endpoint{
{
Name: "endpoint1",
Request: &Value{
Name: "request",
Type: "request",
},
Response: &Value{
Name: "response",
Type: "response",
},
Metadata: map[string]string{
"foo1": "bar1",
},
},
},
},
}
for _, d := range testData {
encoded, err := encode(d)
if err != nil {
t.Fatal(err)
}
for _, txt := range encoded {
if len(txt) > 255 {
t.Fatalf("One of parts for txt is %d characters", len(txt))
}
}
decoded, err := decode(encoded)
if err != nil {
t.Fatal(err)
}
if decoded.Version != d.Version {
t.Fatalf("Expected version %s got %s", d.Version, decoded.Version)
}
if len(decoded.Endpoints) != len(d.Endpoints) {
t.Fatalf("Expected %d endpoints, got %d", len(d.Endpoints), len(decoded.Endpoints))
}
for k, v := range d.Metadata {
if val := decoded.Metadata[k]; val != v {
t.Fatalf("Expected %s=%s got %s=%s", k, v, k, val)
}
}
}
}
func TestWatcher(t *testing.T) {
if travis := os.Getenv("TRAVIS"); travis == "true" {
t.Skip()
}
testData := []*Service{
{
Name: "test1",
Version: "1.0.1",
Nodes: []*Node{
{
Id: "test1-1",
Address: "10.0.0.1:10001",
Metadata: map[string]string{
"foo": "bar",
},
},
},
},
{
Name: "test2",
Version: "1.0.2",
Nodes: []*Node{
{
Id: "test2-1",
Address: "10.0.0.2:10002",
Metadata: map[string]string{
"foo2": "bar2",
},
},
},
},
{
Name: "test3",
Version: "1.0.3",
Nodes: []*Node{
{
Id: "test3-1",
Address: "10.0.0.3:10003",
Metadata: map[string]string{
"foo3": "bar3",
},
},
},
},
}
testFn := func(service, s *Service) {
if s == nil {
t.Fatalf("Expected one result for %s got nil", service.Name)
}
if s.Name != service.Name {
t.Fatalf("Expected name %s got %s", service.Name, s.Name)
}
if s.Version != service.Version {
t.Fatalf("Expected version %s got %s", service.Version, s.Version)
}
if len(s.Nodes) != 1 {
t.Fatalf("Expected 1 node, got %d", len(s.Nodes))
}
node := s.Nodes[0]
if node.Id != service.Nodes[0].Id {
t.Fatalf("Expected node id %s got %s", service.Nodes[0].Id, node.Id)
}
if node.Address != service.Nodes[0].Address {
t.Fatalf("Expected node address %s got %s", service.Nodes[0].Address, node.Address)
}
}
travis := os.Getenv("TRAVIS")
var opts []Option
if travis == "true" {
opts = append(opts, Timeout(time.Millisecond*100))
}
// new registry
r := NewRegistry(opts...)
w, err := r.Watch()
if err != nil {
t.Fatal(err)
}
defer w.Stop()
for _, service := range testData {
// register service
if err := r.Register(service); err != nil {
t.Fatal(err)
}
for {
res, err := w.Next()
if err != nil {
t.Fatal(err)
}
if res.Service.Name != service.Name {
continue
}
if res.Action != "create" {
t.Fatalf("Expected create event got %s for %s", res.Action, res.Service.Name)
}
testFn(service, res.Service)
break
}
// deregister
if err := r.Deregister(service); err != nil {
t.Fatal(err)
}
for {
res, err := w.Next()
if err != nil {
t.Fatal(err)
}
if res.Service.Name != service.Name {
continue
}
if res.Action != "delete" {
continue
}
testFn(service, res.Service)
break
}
}
}

87
registry/mdns_watcher.go
View File

@ -1,87 +0,0 @@
package registry
import (
"fmt"
"strings"
"github.com/micro/mdns"
)
type mdnsWatcher struct {
id string
wo WatchOptions
ch chan *mdns.ServiceEntry
exit chan struct{}
// the mdns domain
domain string
// the registry
registry *mdnsRegistry
}
func (m *mdnsWatcher) Next() (*Result, error) {
for {
select {
case e := <-m.ch:
txt, err := decode(e.InfoFields)
if err != nil {
continue
}
if len(txt.Service) == 0 || len(txt.Version) == 0 {
continue
}
// Filter watch options
// wo.Service: Only keep services we care about
if len(m.wo.Service) > 0 && txt.Service != m.wo.Service {
continue
}
var action string
if e.TTL == 0 {
action = "delete"
} else {
action = "create"
}
service := &Service{
Name: txt.Service,
Version: txt.Version,
Endpoints: txt.Endpoints,
}
// skip anything without the domain we care about
suffix := fmt.Sprintf(".%s.%s.", service.Name, m.domain)
if !strings.HasSuffix(e.Name, suffix) {
continue
}
service.Nodes = append(service.Nodes, &Node{
Id: strings.TrimSuffix(e.Name, suffix),
Address: fmt.Sprintf("%s:%d", e.AddrV4.String(), e.Port),
Metadata: txt.Metadata,
})
return &Result{
Action: action,
Service: service,
}, nil
case <-m.exit:
return nil, ErrWatcherStopped
}
}
}
func (m *mdnsWatcher) Stop() {
select {
case <-m.exit:
return
default:
close(m.exit)
// remove self from the registry
m.registry.mtx.Lock()
delete(m.registry.watchers, m.id)
m.registry.mtx.Unlock()
}
}

27
registry/registry.go
View File

@ -28,6 +28,33 @@ type Registry interface {
String() string
}
type Service struct {
Name string `json:"name"`
Version string `json:"version"`
Metadata map[string]string `json:"metadata"`
Endpoints []*Endpoint `json:"endpoints"`
Nodes []*Node `json:"nodes"`
}
type Node struct {
Id string `json:"id"`
Address string `json:"address"`
Metadata map[string]string `json:"metadata"`
}
type Endpoint struct {
Name string `json:"name"`
Request *Value `json:"request"`
Response *Value `json:"response"`
Metadata map[string]string `json:"metadata"`
}
type Value struct {
Name string `json:"name"`
Type string `json:"type"`
Values []*Value `json:"values"`
}
type Option func(*Options)
type RegisterOption func(*RegisterOptions)

28
registry/service.go
View File

@ -1,28 +0,0 @@
package registry
type Service struct {
Name string `json:"name"`
Version string `json:"version"`
Metadata map[string]string `json:"metadata"`
Endpoints []*Endpoint `json:"endpoints"`
Nodes []*Node `json:"nodes"`
}
type Node struct {
Id string `json:"id"`
Address string `json:"address"`
Metadata map[string]string `json:"metadata"`
}
type Endpoint struct {
Name string `json:"name"`
Request *Value `json:"request"`
Response *Value `json:"response"`
Metadata map[string]string `json:"metadata"`
}
type Value struct {
Name string `json:"name"`
Type string `json:"type"`
Values []*Value `json:"values"`
}

149
registry/watcher_test.go
View File

@ -1,149 +0,0 @@
package registry
import (
"os"
"testing"
"time"
)
func TestWatcher(t *testing.T) {
if travis := os.Getenv("TRAVIS"); travis == "true" {
t.Skip()
}
testData := []*Service{
{
Name: "test1",
Version: "1.0.1",
Nodes: []*Node{
{
Id: "test1-1",
Address: "10.0.0.1:10001",
Metadata: map[string]string{
"foo": "bar",
},
},
},
},
{
Name: "test2",
Version: "1.0.2",
Nodes: []*Node{
{
Id: "test2-1",
Address: "10.0.0.2:10002",
Metadata: map[string]string{
"foo2": "bar2",
},
},
},
},
{
Name: "test3",
Version: "1.0.3",
Nodes: []*Node{
{
Id: "test3-1",
Address: "10.0.0.3:10003",
Metadata: map[string]string{
"foo3": "bar3",
},
},
},
},
}
testFn := func(service, s *Service) {
if s == nil {
t.Fatalf("Expected one result for %s got nil", service.Name)
}
if s.Name != service.Name {
t.Fatalf("Expected name %s got %s", service.Name, s.Name)
}
if s.Version != service.Version {
t.Fatalf("Expected version %s got %s", service.Version, s.Version)
}
if len(s.Nodes) != 1 {
t.Fatalf("Expected 1 node, got %d", len(s.Nodes))
}
node := s.Nodes[0]
if node.Id != service.Nodes[0].Id {
t.Fatalf("Expected node id %s got %s", service.Nodes[0].Id, node.Id)
}
if node.Address != service.Nodes[0].Address {
t.Fatalf("Expected node address %s got %s", service.Nodes[0].Address, node.Address)
}
}
travis := os.Getenv("TRAVIS")
var opts []Option
if travis == "true" {
opts = append(opts, Timeout(time.Millisecond*100))
}
// new registry
r := NewRegistry(opts...)
w, err := r.Watch()
if err != nil {
t.Fatal(err)
}
defer w.Stop()
for _, service := range testData {
// register service
if err := r.Register(service); err != nil {
t.Fatal(err)
}
for {
res, err := w.Next()
if err != nil {
t.Fatal(err)
}
if res.Service.Name != service.Name {
continue
}
if res.Action != "create" {
t.Fatalf("Expected create event got %s for %s", res.Action, res.Service.Name)
}
testFn(service, res.Service)
break
}
// deregister
if err := r.Deregister(service); err != nil {
t.Fatal(err)
}
for {
res, err := w.Next()
if err != nil {
t.Fatal(err)
}
if res.Service.Name != service.Name {
continue
}
if res.Action != "delete" {
continue
}
testFn(service, res.Service)
break
}
}
}

28
runtime/README.md
View File

@ -1,28 +0,0 @@
# Runtime
A runtime for self governing services.
## Overview
In recent years we've started to develop complex architectures for the pipeline between writing code and running it. This
philosophy of build, run, manage or however many variations, has created a number of layers of abstraction that make it
all the more difficult to run code.
Runtime manages the lifecycle of a service from source to running process. If the source is the *source of truth* then
everything in between running is wasted breath. Applications should be self governing and self sustaining.
To enable that we need libraries which make it possible.
Runtime will fetch source code, build a binary and execute it. Any Go program that uses this library should be able
to run dependencies or itself with ease, with the ability to update itself as the source is updated.
## Features
- **Source** - Fetches source whether it be git, go, docker, etc
- **Package** - Compiles the source into a binary which can be executed
- **Process** - Executes a binary and creates a running process
## Usage
TODO

8
runtime/default.go
View File

@ -40,6 +40,10 @@ func NewRuntime(opts ...Option) Runtime {
o(&options)
}
// make the logs directory
path := filepath.Join(os.TempDir(), "micro", "logs")
_ = os.MkdirAll(path, 0755)
return &runtime{
options: options,
closed: make(chan bool),
@ -177,8 +181,10 @@ func (r *runtime) run(events <-chan Event) {
}
func logFile(serviceName string) string {
// make the directory
name := strings.Replace(serviceName, "/", "-", -1)
return filepath.Join(os.TempDir(), fmt.Sprintf("%v.log", name))
path := filepath.Join(os.TempDir(), "micro", "logs")
return filepath.Join(path, fmt.Sprintf("%v.log", name))
}
// Create creates a new service which is then started by runtime

4
runtime/local/process/os/os.go
View File

@ -15,13 +15,17 @@ import (
func (p *Process) Exec(exe *process.Executable) error {
cmd := exec.Command(exe.Package.Path)
cmd.Dir = exe.Dir
return cmd.Run()
}
func (p *Process) Fork(exe *process.Executable) (*process.PID, error) {
// create command
cmd := exec.Command(exe.Package.Path, exe.Args...)
cmd.Dir = exe.Dir
// set env vars
cmd.Env = append(cmd.Env, os.Environ()...)
cmd.Env = append(cmd.Env, exe.Env...)
// create process group

2
runtime/local/process/process.go
View File

@ -26,6 +26,8 @@ type Executable struct {
Env []string
// Args to pass
Args []string
// Initial working directory
Dir string
}
// PID is the running process

10
runtime/runtime.go
View File

@ -17,8 +17,6 @@ var (
// Runtime is a service runtime manager
type Runtime interface {
// String describes runtime
String() string
// Init initializes runtime
Init(...Option) error
// Create registers a service
@ -29,14 +27,14 @@ type Runtime interface {
Update(*Service) error
// Remove a service
Delete(*Service) error
// List the managed services
List() ([]*Service, error)
// Logs returns the logs for a service
Logs(*Service, ...LogsOption) (LogStream, error)
// Start starts the runtime
Start() error
// Stop shuts down the runtime
Stop() error
// Logs
Logs(*Service, ...LogsOption) (LogStream, error)
// String describes runtime
String() string
}
// Stream returns a log stream

1
runtime/service.go
View File

@ -55,6 +55,7 @@ func newService(s *Service, c CreateOptions) *service {
},
Env: c.Env,
Args: args,
Dir: s.Source,
},
closed: make(chan bool),
output: c.Output,

124
runtime/service/proto/runtime.pb.go
View File

@ -1,5 +1,5 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: micro/go-micro/runtime/service/proto/runtime.proto
// source: github.com/micro/go-micro/runtime/service/proto/runtime.proto
package go_micro_runtime
@ -38,7 +38,7 @@ func (m *Service) Reset() { *m = Service{} }
func (m *Service) String() string { return proto.CompactTextString(m) }
func (*Service) ProtoMessage() {}
func (*Service) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{0}
return fileDescriptor_976fccef828ab1f0, []int{0}
}
func (m *Service) XXX_Unmarshal(b []byte) error {
@ -101,7 +101,7 @@ func (m *Event) Reset() { *m = Event{} }
func (m *Event) String() string { return proto.CompactTextString(m) }
func (*Event) ProtoMessage() {}
func (*Event) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{1}
return fileDescriptor_976fccef828ab1f0, []int{1}
}
func (m *Event) XXX_Unmarshal(b []byte) error {
@ -172,7 +172,7 @@ func (m *CreateOptions) Reset() { *m = CreateOptions{} }
func (m *CreateOptions) String() string { return proto.CompactTextString(m) }
func (*CreateOptions) ProtoMessage() {}
func (*CreateOptions) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{2}
return fileDescriptor_976fccef828ab1f0, []int{2}
}
func (m *CreateOptions) XXX_Unmarshal(b []byte) error {
@ -247,7 +247,7 @@ func (m *CreateRequest) Reset() { *m = CreateRequest{} }
func (m *CreateRequest) String() string { return proto.CompactTextString(m) }
func (*CreateRequest) ProtoMessage() {}
func (*CreateRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{3}
return fileDescriptor_976fccef828ab1f0, []int{3}
}
func (m *CreateRequest) XXX_Unmarshal(b []byte) error {
@ -292,7 +292,7 @@ func (m *CreateResponse) Reset() { *m = CreateResponse{} }
func (m *CreateResponse) String() string { return proto.CompactTextString(m) }
func (*CreateResponse) ProtoMessage() {}
func (*CreateResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{4}
return fileDescriptor_976fccef828ab1f0, []int{4}
}
func (m *CreateResponse) XXX_Unmarshal(b []byte) error {
@ -329,7 +329,7 @@ func (m *ReadOptions) Reset() { *m = ReadOptions{} }
func (m *ReadOptions) String() string { return proto.CompactTextString(m) }
func (*ReadOptions) ProtoMessage() {}
func (*ReadOptions) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{5}
return fileDescriptor_976fccef828ab1f0, []int{5}
}
func (m *ReadOptions) XXX_Unmarshal(b []byte) error {
@ -382,7 +382,7 @@ func (m *ReadRequest) Reset() { *m = ReadRequest{} }
func (m *ReadRequest) String() string { return proto.CompactTextString(m) }
func (*ReadRequest) ProtoMessage() {}
func (*ReadRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{6}
return fileDescriptor_976fccef828ab1f0, []int{6}
}
func (m *ReadRequest) XXX_Unmarshal(b []byte) error {
@ -421,7 +421,7 @@ func (m *ReadResponse) Reset() { *m = ReadResponse{} }
func (m *ReadResponse) String() string { return proto.CompactTextString(m) }
func (*ReadResponse) ProtoMessage() {}
func (*ReadResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{7}
return fileDescriptor_976fccef828ab1f0, []int{7}
}
func (m *ReadResponse) XXX_Unmarshal(b []byte) error {
@ -460,7 +460,7 @@ func (m *DeleteRequest) Reset() { *m = DeleteRequest{} }
func (m *DeleteRequest) String() string { return proto.CompactTextString(m) }
func (*DeleteRequest) ProtoMessage() {}
func (*DeleteRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{8}
return fileDescriptor_976fccef828ab1f0, []int{8}
}
func (m *DeleteRequest) XXX_Unmarshal(b []byte) error {
@ -498,7 +498,7 @@ func (m *DeleteResponse) Reset() { *m = DeleteResponse{} }
func (m *DeleteResponse) String() string { return proto.CompactTextString(m) }
func (*DeleteResponse) ProtoMessage() {}
func (*DeleteResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{9}
return fileDescriptor_976fccef828ab1f0, []int{9}
}
func (m *DeleteResponse) XXX_Unmarshal(b []byte) error {
@ -530,7 +530,7 @@ func (m *UpdateRequest) Reset() { *m = UpdateRequest{} }
func (m *UpdateRequest) String() string { return proto.CompactTextString(m) }
func (*UpdateRequest) ProtoMessage() {}
func (*UpdateRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{10}
return fileDescriptor_976fccef828ab1f0, []int{10}
}
func (m *UpdateRequest) XXX_Unmarshal(b []byte) error {
@ -568,7 +568,7 @@ func (m *UpdateResponse) Reset() { *m = UpdateResponse{} }
func (m *UpdateResponse) String() string { return proto.CompactTextString(m) }
func (*UpdateResponse) ProtoMessage() {}
func (*UpdateResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{11}
return fileDescriptor_976fccef828ab1f0, []int{11}
}
func (m *UpdateResponse) XXX_Unmarshal(b []byte) error {
@ -599,7 +599,7 @@ func (m *ListRequest) Reset() { *m = ListRequest{} }
func (m *ListRequest) String() string { return proto.CompactTextString(m) }
func (*ListRequest) ProtoMessage() {}
func (*ListRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{12}
return fileDescriptor_976fccef828ab1f0, []int{12}
}
func (m *ListRequest) XXX_Unmarshal(b []byte) error {
@ -631,7 +631,7 @@ func (m *ListResponse) Reset() { *m = ListResponse{} }
func (m *ListResponse) String() string { return proto.CompactTextString(m) }
func (*ListResponse) ProtoMessage() {}
func (*ListResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{13}
return fileDescriptor_976fccef828ab1f0, []int{13}
}
func (m *ListResponse) XXX_Unmarshal(b []byte) error {
@ -679,7 +679,7 @@ func (m *LogsRequest) Reset() { *m = LogsRequest{} }
func (m *LogsRequest) String() string { return proto.CompactTextString(m) }
func (*LogsRequest) ProtoMessage() {}
func (*LogsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{14}
return fileDescriptor_976fccef828ab1f0, []int{14}
}
func (m *LogsRequest) XXX_Unmarshal(b []byte) error {
@ -744,7 +744,7 @@ func (m *LogRecord) Reset() { *m = LogRecord{} }
func (m *LogRecord) String() string { return proto.CompactTextString(m) }
func (*LogRecord) ProtoMessage() {}
func (*LogRecord) Descriptor() ([]byte, []int) {
return fileDescriptor_4bc91a8efec81434, []int{15}
return fileDescriptor_976fccef828ab1f0, []int{15}
}
func (m *LogRecord) XXX_Unmarshal(b []byte) error {
@ -808,51 +808,51 @@ func init() {
}
func init() {
proto.RegisterFile("micro/go-micro/runtime/service/proto/runtime.proto", fileDescriptor_4bc91a8efec81434)
proto.RegisterFile("github.com/micro/go-micro/runtime/service/proto/runtime.proto", fileDescriptor_976fccef828ab1f0)
}
var fileDescriptor_4bc91a8efec81434 = []byte{
// 663 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0xcb, 0x6e, 0xd3, 0x40,
0x14, 0xad, 0xe3, 0x3c, 0xda, 0x6b, 0x82, 0xaa, 0x51, 0x85, 0x4c, 0x79, 0x45, 0xde, 0x50, 0x16,
0xb8, 0x28, 0x15, 0xe2, 0xb5, 0x6c, 0x53, 0x36, 0x8d, 0x90, 0x8c, 0xfa, 0x01, 0x83, 0x73, 0x65,
0x59, 0xad, 0x3d, 0xc6, 0x33, 0x8e, 0x94, 0x15, 0xdf, 0xc0, 0x57, 0xb1, 0x85, 0x3f, 0x42, 0xf3,
0xf0, 0x2b, 0xb1, 0xbb, 0xc9, 0x6e, 0xee, 0xe4, 0xce, 0xf1, 0x39, 0x67, 0xce, 0x9d, 0xc0, 0x3c,
0x89, 0xc3, 0x9c, 0x9d, 0x47, 0xec, 0xad, 0x5e, 0xe4, 0x45, 0x2a, 0xe2, 0x04, 0xcf, 0x39, 0xe6,
0xeb, 0x38, 0xc4, 0xf3, 0x2c, 0x67, 0xa2, 0xda, 0xf5, 0x55, 0x45, 0x8e, 0x23, 0xe6, 0xab, 0x6e,
0xdf, 0xec, 0x7b, 0xff, 0x2c, 0x98, 0x7c, 0xd7, 0x27, 0x08, 0x81, 0x61, 0x4a, 0x13, 0x74, 0xad,
0x99, 0x75, 0x76, 0x14, 0xa8, 0x35, 0x71, 0x61, 0xb2, 0xc6, 0x9c, 0xc7, 0x2c, 0x75, 0x07, 0x6a,
0xbb, 0x2c, 0xc9, 0x13, 0x18, 0x73, 0x56, 0xe4, 0x21, 0xba, 0xb6, 0xfa, 0xc1, 0x54, 0xe4, 0x12,
0x0e, 0x13, 0x14, 0x74, 0x45, 0x05, 0x75, 0x87, 0x33, 0xfb, 0xcc, 0x99, 0xbf, 0xf6, 0xb7, 0x3f,
0xeb, 0x9b, 0x4f, 0xfa, 0x4b, 0xd3, 0xb9, 0x48, 0x45, 0xbe, 0x09, 0xaa, 0x83, 0xa7, 0x5f, 0x60,
0xda, 0xfa, 0x89, 0x1c, 0x83, 0x7d, 0x87, 0x1b, 0x43, 0x4d, 0x2e, 0xc9, 0x09, 0x8c, 0xd6, 0xf4,
0xbe, 0x40, 0xc3, 0x4b, 0x17, 0x9f, 0x07, 0x1f, 0x2d, 0x2f, 0x81, 0xd1, 0x62, 0x8d, 0xa9, 0x90,
0x82, 0xc4, 0x26, 0xab, 0x04, 0xc9, 0x35, 0x79, 0x0e, 0x47, 0x92, 0x01, 0x17, 0x34, 0xc9, 0xd4,
0x51, 0x3b, 0xa8, 0x37, 0xa4, 0x5c, 0xe3, 0x9f, 0x51, 0x55, 0x96, 0x4d, 0x23, 0x86, 0x2d, 0x23,
0xbc, 0xdf, 0x16, 0x4c, 0x2f, 0x73, 0xa4, 0x02, 0xbf, 0x65, 0x22, 0x66, 0x29, 0x97, 0xbd, 0x21,
0x4b, 0x12, 0x9a, 0xae, 0x5c, 0x6b, 0x66, 0xcb, 0x5e, 0x53, 0x4a, 0x46, 0x34, 0x8f, 0xb8, 0x3b,
0x50, 0xdb, 0x6a, 0x2d, 0xa5, 0x61, 0xba, 0x76, 0x6d, 0xb5, 0x25, 0x97, 0xd2, 0x5a, 0x56, 0x88,
0xac, 0x10, 0xe6, 0x53, 0xa6, 0xaa, 0xf4, 0x8c, 0x1a, 0x7a, 0x4e, 0x60, 0x14, 0x27, 0x34, 0x42,
0x77, 0xac, 0x6d, 0x50, 0x85, 0xf7, 0xab, 0xa4, 0x14, 0xe0, 0xcf, 0x02, 0xb9, 0x20, 0x17, 0xb5,
0x30, 0xe9, 0x86, 0x33, 0x7f, 0xda, 0x7b, 0x29, 0xb5, 0xe6, 0x4f, 0x30, 0x61, 0x5a, 0x92, 0x72,
0xca, 0x99, 0xbf, 0xda, 0x3d, 0xd4, 0x52, 0x1e, 0x94, 0xfd, 0xde, 0x31, 0x3c, 0x2e, 0x09, 0xf0,
0x8c, 0xa5, 0x1c, 0xbd, 0x5b, 0x70, 0x02, 0xa4, 0xab, 0x86, 0x47, 0x4d, 0x42, 0xdd, 0x4e, 0x6f,
0x45, 0xae, 0xd4, 0x6f, 0xd7, 0xfa, 0xbd, 0x6b, 0x0d, 0x5b, 0xea, 0xfc, 0x50, 0x53, 0xd6, 0x3a,
0x5f, 0xec, 0x52, 0x6e, 0xd0, 0xa8, 0x09, 0x2f, 0xe0, 0x91, 0xc6, 0xd1, 0x74, 0xc9, 0x7b, 0x38,
0x34, 0x84, 0xb8, 0xba, 0xc4, 0x07, 0x1d, 0xab, 0x5a, 0xbd, 0x2b, 0x98, 0x5e, 0xe1, 0x3d, 0xee,
0x67, 0xbc, 0x74, 0xaf, 0x44, 0x31, 0xee, 0x5d, 0xc1, 0xf4, 0x36, 0x5b, 0xd1, 0xfd, 0x71, 0x4b,
0x14, 0x83, 0x3b, 0x05, 0xe7, 0x26, 0xe6, 0xc2, 0xa0, 0x4a, 0x17, 0x74, 0xb9, 0x9f, 0x0b, 0x77,
0xe0, 0xdc, 0xb0, 0x88, 0x97, 0x5c, 0xfb, 0xef, 0x5a, 0x3e, 0x22, 0x22, 0x47, 0x9a, 0xa8, 0xab,
0x3e, 0x0c, 0x4c, 0x25, 0x53, 0x1d, 0xb2, 0x22, 0x15, 0xea, 0xaa, 0xed, 0x40, 0x17, 0x72, 0x97,
0xc7, 0x69, 0x88, 0x6a, 0x2c, 0xec, 0x40, 0x17, 0xde, 0x1f, 0x0b, 0x8e, 0x6e, 0x58, 0x14, 0x60,
0xc8, 0xf2, 0x55, 0x7b, 0xbe, 0xad, 0xed, 0xf9, 0x5e, 0x34, 0x1e, 0xa7, 0x81, 0xd2, 0xf3, 0x66,
0x57, 0x4f, 0x05, 0xd6, 0xf7, 0x3c, 0x49, 0x41, 0x09, 0x72, 0x2e, 0xc7, 0xce, 0x3c, 0x13, 0xa6,
0xdc, 0xeb, 0xe1, 0x9a, 0xff, 0xb5, 0x61, 0x12, 0x68, 0x12, 0x64, 0x09, 0x63, 0x3d, 0x40, 0xa4,
0x77, 0xe8, 0x8c, 0xbd, 0xa7, 0xb3, 0xfe, 0x06, 0x73, 0xcb, 0x07, 0xe4, 0x2b, 0x0c, 0x65, 0xbc,
0x49, 0xcf, 0x38, 0x94, 0x50, 0x2f, 0xfb, 0x7e, 0xae, 0x80, 0x96, 0x30, 0xd6, 0xd1, 0xec, 0xe2,
0xd5, 0x8a, 0x7e, 0x17, 0xaf, 0xad, 0x54, 0x2b, 0x38, 0x9d, 0xc8, 0x2e, 0xb8, 0x56, 0xe2, 0xbb,
0xe0, 0xb6, 0xc2, 0xac, 0x64, 0xca, 0xfc, 0x76, 0xc9, 0x6c, 0xc4, 0xbc, 0x4b, 0x66, 0x33, 0xf6,
0xde, 0x01, 0xb9, 0x86, 0xa1, 0x4c, 0x70, 0x27, 0x50, 0x9d, 0xec, 0xd3, 0x67, 0x0f, 0xa4, 0xc7,
0x3b, 0x78, 0x67, 0xfd, 0x18, 0xab, 0x3f, 0xde, 0x8b, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x17,
0xe1, 0xab, 0x77, 0xae, 0x07, 0x00, 0x00,
var fileDescriptor_976fccef828ab1f0 = []byte{
// 662 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0xbb, 0x6e, 0xdb, 0x4a,
0x10, 0x35, 0x45, 0x3d, 0xec, 0xd1, 0xd5, 0x85, 0xb1, 0x30, 0x02, 0xc6, 0x79, 0x09, 0x6c, 0xe2,
0x14, 0xa1, 0x02, 0x19, 0x41, 0x5e, 0x48, 0x65, 0xcb, 0x69, 0x6c, 0x04, 0x60, 0xe0, 0x0f, 0x58,
0x53, 0x03, 0x86, 0xb0, 0x97, 0xcb, 0x70, 0x97, 0x02, 0x5c, 0xa5, 0x4c, 0x9d, 0xaf, 0x4a, 0x9d,
0x3f, 0x0a, 0xf6, 0x41, 0x8a, 0x94, 0x48, 0x37, 0xea, 0x76, 0x46, 0xb3, 0x87, 0xe7, 0x9c, 0x99,
0x59, 0xc1, 0xe7, 0x38, 0x91, 0xdf, 0x8b, 0x9b, 0x20, 0xe2, 0x6c, 0xc6, 0x92, 0x28, 0xe7, 0xb3,
0x98, 0xbf, 0x36, 0x87, 0xbc, 0x48, 0x65, 0xc2, 0x70, 0x26, 0x30, 0x5f, 0x25, 0x11, 0xce, 0xb2,
0x9c, 0xcb, 0x2a, 0x1b, 0xe8, 0x88, 0x1c, 0xc6, 0x3c, 0xd0, 0xd5, 0x81, 0xcd, 0xfb, 0x7f, 0x1d,
0x18, 0x7d, 0x33, 0x37, 0x08, 0x81, 0x7e, 0x4a, 0x19, 0x7a, 0xce, 0xd4, 0x39, 0x39, 0x08, 0xf5,
0x99, 0x78, 0x30, 0x5a, 0x61, 0x2e, 0x12, 0x9e, 0x7a, 0x3d, 0x9d, 0x2e, 0x43, 0xf2, 0x08, 0x86,
0x82, 0x17, 0x79, 0x84, 0x9e, 0xab, 0x7f, 0xb0, 0x11, 0x39, 0x83, 0x7d, 0x86, 0x92, 0x2e, 0xa9,
0xa4, 0x5e, 0x7f, 0xea, 0x9e, 0x8c, 0xe7, 0x2f, 0x83, 0xcd, 0xcf, 0x06, 0xf6, 0x93, 0xc1, 0x95,
0xad, 0x5c, 0xa4, 0x32, 0xbf, 0x0f, 0xab, 0x8b, 0xc7, 0x9f, 0x60, 0xd2, 0xf8, 0x89, 0x1c, 0x82,
0x7b, 0x8b, 0xf7, 0x96, 0x9a, 0x3a, 0x92, 0x23, 0x18, 0xac, 0xe8, 0x5d, 0x81, 0x96, 0x97, 0x09,
0x3e, 0xf6, 0xde, 0x3b, 0x3e, 0x83, 0xc1, 0x62, 0x85, 0xa9, 0x54, 0x82, 0xe4, 0x7d, 0x56, 0x09,
0x52, 0x67, 0xf2, 0x14, 0x0e, 0x14, 0x03, 0x21, 0x29, 0xcb, 0xf4, 0x55, 0x37, 0x5c, 0x27, 0x94,
0x5c, 0xeb, 0x9f, 0x55, 0x55, 0x86, 0x75, 0x23, 0xfa, 0x0d, 0x23, 0xfc, 0xdf, 0x0e, 0x4c, 0xce,
0x72, 0xa4, 0x12, 0xbf, 0x66, 0x32, 0xe1, 0xa9, 0x50, 0xb5, 0x11, 0x67, 0x8c, 0xa6, 0x4b, 0xcf,
0x99, 0xba, 0xaa, 0xd6, 0x86, 0x8a, 0x11, 0xcd, 0x63, 0xe1, 0xf5, 0x74, 0x5a, 0x9f, 0x95, 0x34,
0x4c, 0x57, 0x9e, 0xab, 0x53, 0xea, 0xa8, 0xac, 0xe5, 0x85, 0xcc, 0x0a, 0x69, 0x3f, 0x65, 0xa3,
0x4a, 0xcf, 0xa0, 0xa6, 0xe7, 0x08, 0x06, 0x09, 0xa3, 0x31, 0x7a, 0x43, 0x63, 0x83, 0x0e, 0xfc,
0x9f, 0x25, 0xa5, 0x10, 0x7f, 0x14, 0x28, 0x24, 0x39, 0x5d, 0x0b, 0x53, 0x6e, 0x8c, 0xe7, 0x8f,
0x3b, 0x9b, 0xb2, 0xd6, 0xfc, 0x01, 0x46, 0xdc, 0x48, 0xd2, 0x4e, 0x8d, 0xe7, 0x2f, 0xb6, 0x2f,
0x35, 0x94, 0x87, 0x65, 0xbd, 0x7f, 0x08, 0xff, 0x97, 0x04, 0x44, 0xc6, 0x53, 0x81, 0xfe, 0x35,
0x8c, 0x43, 0xa4, 0xcb, 0x9a, 0x47, 0x75, 0x42, 0xed, 0x4e, 0x6f, 0x8c, 0x5c, 0xa9, 0xdf, 0x5d,
0xeb, 0xf7, 0x2f, 0x0c, 0x6c, 0xa9, 0xf3, 0xdd, 0x9a, 0xb2, 0xd1, 0xf9, 0x6c, 0x9b, 0x72, 0x8d,
0xc6, 0x9a, 0xf0, 0x02, 0xfe, 0x33, 0x38, 0x86, 0x2e, 0x79, 0x0b, 0xfb, 0x96, 0x90, 0xd0, 0x4d,
0x7c, 0xd0, 0xb1, 0xaa, 0xd4, 0x3f, 0x87, 0xc9, 0x39, 0xde, 0xe1, 0x6e, 0xc6, 0x2b, 0xf7, 0x4a,
0x14, 0xeb, 0xde, 0x39, 0x4c, 0xae, 0xb3, 0x25, 0xdd, 0x1d, 0xb7, 0x44, 0xb1, 0xb8, 0x13, 0x18,
0x5f, 0x26, 0x42, 0x5a, 0x54, 0xe5, 0x82, 0x09, 0x77, 0x73, 0xe1, 0x16, 0xc6, 0x97, 0x3c, 0x16,
0x25, 0xd7, 0xee, 0x5e, 0xab, 0x47, 0x44, 0xe6, 0x48, 0x99, 0x6e, 0xf5, 0x7e, 0x68, 0x23, 0x35,
0xd5, 0x11, 0x2f, 0x52, 0xa9, 0x5b, 0xed, 0x86, 0x26, 0x50, 0x59, 0x91, 0xa4, 0x11, 0xea, 0xb5,
0x70, 0x43, 0x13, 0xf8, 0x7f, 0x1c, 0x38, 0xb8, 0xe4, 0x71, 0x88, 0x11, 0xcf, 0x97, 0xcd, 0xfd,
0x76, 0x36, 0xf7, 0x7b, 0x51, 0x7b, 0x9c, 0x7a, 0x5a, 0xcf, 0xab, 0x6d, 0x3d, 0x15, 0x58, 0xd7,
0xf3, 0xa4, 0x04, 0x31, 0x14, 0x42, 0xad, 0x9d, 0x7d, 0x26, 0x6c, 0xb8, 0xd3, 0xc3, 0x35, 0xff,
0xe5, 0xc2, 0x28, 0x34, 0x24, 0xc8, 0x15, 0x0c, 0xcd, 0x02, 0x91, 0xce, 0xa5, 0xb3, 0xf6, 0x1e,
0x4f, 0xbb, 0x0b, 0x6c, 0x97, 0xf7, 0xc8, 0x17, 0xe8, 0xab, 0xf1, 0x26, 0x1d, 0xeb, 0x50, 0x42,
0x3d, 0xef, 0xfa, 0xb9, 0x02, 0xba, 0x82, 0xa1, 0x19, 0xcd, 0x36, 0x5e, 0x8d, 0xd1, 0x6f, 0xe3,
0xb5, 0x31, 0xd5, 0x1a, 0xce, 0x4c, 0x64, 0x1b, 0x5c, 0x63, 0xe2, 0xdb, 0xe0, 0x36, 0x86, 0x79,
0x8f, 0x5c, 0x40, 0x5f, 0x0d, 0x5e, 0x9b, 0xcc, 0xda, 0x40, 0x1e, 0x3f, 0x79, 0xa0, 0xe9, 0xfe,
0xde, 0x1b, 0xe7, 0x66, 0xa8, 0xff, 0x2f, 0x4f, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x40, 0x42,
0xb3, 0x4e, 0x70, 0x07, 0x00, 0x00,
}

19
runtime/service/proto/runtime.pb.micro.go
View File

@ -1,5 +1,5 @@
// Code generated by protoc-gen-micro. DO NOT EDIT.
// source: micro/go-micro/runtime/service/proto/runtime.proto
// source: github.com/micro/go-micro/runtime/service/proto/runtime.proto
package go_micro_runtime
@ -38,7 +38,6 @@ type RuntimeService interface {
Read(ctx context.Context, in *ReadRequest, opts ...client.CallOption) (*ReadResponse, error)
Delete(ctx context.Context, in *DeleteRequest, opts ...client.CallOption) (*DeleteResponse, error)
Update(ctx context.Context, in *UpdateRequest, opts ...client.CallOption) (*UpdateResponse, error)
List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (*ListResponse, error)
Logs(ctx context.Context, in *LogsRequest, opts ...client.CallOption) (Runtime_LogsService, error)
}
@ -94,16 +93,6 @@ func (c *runtimeService) Update(ctx context.Context, in *UpdateRequest, opts ...
return out, nil
}
func (c *runtimeService) List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (*ListResponse, error) {
req := c.c.NewRequest(c.name, "Runtime.List", in)
out := new(ListResponse)
err := c.c.Call(ctx, req, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *runtimeService) Logs(ctx context.Context, in *LogsRequest, opts ...client.CallOption) (Runtime_LogsService, error) {
req := c.c.NewRequest(c.name, "Runtime.Logs", &LogsRequest{})
stream, err := c.c.Stream(ctx, req, opts...)
@ -160,7 +149,6 @@ type RuntimeHandler interface {
Read(context.Context, *ReadRequest, *ReadResponse) error
Delete(context.Context, *DeleteRequest, *DeleteResponse) error
Update(context.Context, *UpdateRequest, *UpdateResponse) error
List(context.Context, *ListRequest, *ListResponse) error
Logs(context.Context, *LogsRequest, Runtime_LogsStream) error
}
@ -170,7 +158,6 @@ func RegisterRuntimeHandler(s server.Server, hdlr RuntimeHandler, opts ...server
Read(ctx context.Context, in *ReadRequest, out *ReadResponse) error
Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error
Update(ctx context.Context, in *UpdateRequest, out *UpdateResponse) error
List(ctx context.Context, in *ListRequest, out *ListResponse) error
Logs(ctx context.Context, stream server.Stream) error
}
type Runtime struct {
@ -200,10 +187,6 @@ func (h *runtimeHandler) Update(ctx context.Context, in *UpdateRequest, out *Upd
return h.RuntimeHandler.Update(ctx, in, out)
}
func (h *runtimeHandler) List(ctx context.Context, in *ListRequest, out *ListResponse) error {
return h.RuntimeHandler.List(ctx, in, out)
}
func (h *runtimeHandler) Logs(ctx context.Context, stream server.Stream) error {
m := new(LogsRequest)
if err := stream.Recv(m); err != nil {

1
runtime/service/proto/runtime.proto
View File

@ -7,7 +7,6 @@ service Runtime {
rpc Read(ReadRequest) returns (ReadResponse) {};
rpc Delete(DeleteRequest) returns (DeleteResponse) {};
rpc Update(UpdateRequest) returns (UpdateResponse) {};
rpc List(ListRequest) returns (ListResponse) {};
rpc Logs(LogsRequest) returns (stream LogRecord) {};
}

22
runtime/service/service.go
View File

@ -194,28 +194,6 @@ func (s *svc) Delete(svc *runtime.Service) error {
return nil
}
// List lists all services managed by the runtime
func (s *svc) List() ([]*runtime.Service, error) {
// list all services managed by the runtime
resp, err := s.runtime.List(context.Background(), &pb.ListRequest{})
if err != nil {
return nil, err
}
services := make([]*runtime.Service, 0, len(resp.Services))
for _, service := range resp.Services {
svc := &runtime.Service{
Name: service.Name,
Version: service.Version,
Source: service.Source,
Metadata: service.Metadata,
}
services = append(services, svc)
}
return services, nil
}
// Start starts the runtime
func (s *svc) Start() error {
// NOTE: nothing to be done here

39
store/cache/cache.go vendored
View File

@ -5,21 +5,34 @@ import (
"fmt"
"github.com/micro/go-micro/v2/store"
"github.com/micro/go-micro/v2/store/memory"
"github.com/pkg/errors"
)
type cache struct {
stores []store.Store
options store.Options
stores []store.Store
}
// Cache is a cpu register style cache for the store.
// It syncs between N stores in a faulting manner.
type Cache interface {
// Implements the store interface
store.Store
}
// NewCache returns a new store using the underlying stores, which must be already Init()ialised
func NewCache(stores ...store.Store) store.Store {
c := &cache{}
c.stores = make([]store.Store, len(stores))
for i, s := range stores {
c.stores[i] = s
func NewCache(stores ...store.Store) Cache {
if len(stores) == 0 {
stores = []store.Store{
memory.NewStore(),
}
}
// TODO: build in an in memory cache
c := &cache{
stores: stores,
}
return c
}
@ -27,15 +40,19 @@ func (c *cache) Close() error {
return nil
}
func (c *cache) Init(...store.Option) error {
if len(c.stores) < 2 {
return errors.New("cache requires at least 2 stores")
func (c *cache) Init(opts ...store.Option) error {
// pass to the stores
for _, store := range c.stores {
if err := store.Init(opts...); err != nil {
return err
}
}
return nil
}
func (c *cache) Options() store.Options {
return c.options
// return from first store
return c.stores[0].Options()
}
func (c *cache) String() string {

7
store/cache/cache_test.go vendored
View File

@ -15,13 +15,12 @@ func TestCache(t *testing.T) {
assert := assert.New(t)
nonCache := NewCache(l0)
assert.NotNil(nonCache.Init(), "Expected a cache initialised with just 1 store to fail")
nonCache := NewCache(nil)
assert.Equal(len(nonCache.(*cache).stores), 1, "Expected a cache initialised with just 1 store to fail")
// Basic functionality
cachedStore := NewCache(l0, l1, l2)
assert.Nil(cachedStore.Init(), "Init should not error")
assert.Equal(cachedStore.Options(), store.Options{}, "Options on store/cache are nonsensical")
assert.Equal(cachedStore.Options(), l0.Options(), "Options on store/cache are nonsensical")
expectedString := "cache [memory memory memory]"
assert.Equal(cachedStore.String(), expectedString, "Cache couldn't describe itself as expected")

411
store/cloudflare/cloudflare.go
View File

@ -1,411 +0,0 @@
// Package cloudflare is a store implementation backed by cloudflare workers kv
// Note that the cloudflare workers KV API is eventually consistent.
package cloudflare
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"math"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"time"
"github.com/micro/go-micro/v2/store"
"github.com/pkg/errors"
"github.com/patrickmn/go-cache"
)
const (
apiBaseURL = "https://api.cloudflare.com/client/v4/"
)
type workersKV struct {
options store.Options
// cf account id
account string
// cf api token
token string
// cf kv namespace
namespace string
// http client to use
httpClient *http.Client
// cache
cache *cache.Cache
}
// apiResponse is a cloudflare v4 api response
type apiResponse struct {
Result []struct {
ID string `json:"id"`
Type string `json:"type"`
Name string `json:"name"`
Expiration int64 `json:"expiration"`
Content string `json:"content"`
Proxiable bool `json:"proxiable"`
Proxied bool `json:"proxied"`
TTL int64 `json:"ttl"`
Priority int64 `json:"priority"`
Locked bool `json:"locked"`
ZoneID string `json:"zone_id"`
ZoneName string `json:"zone_name"`
ModifiedOn time.Time `json:"modified_on"`
CreatedOn time.Time `json:"created_on"`
} `json:"result"`
Success bool `json:"success"`
Errors []apiMessage `json:"errors"`
// not sure Messages is ever populated?
Messages []apiMessage `json:"messages"`
ResultInfo struct {
Page int `json:"page"`
PerPage int `json:"per_page"`
Count int `json:"count"`
TotalCount int `json:"total_count"`
} `json:"result_info"`
}
// apiMessage is a Cloudflare v4 API Error
type apiMessage struct {
Code int `json:"code"`
Message string `json:"message"`
}
// getOptions returns account id, token and namespace
func getOptions() (string, string, string) {
accountID := strings.TrimSpace(os.Getenv("CF_ACCOUNT_ID"))
apiToken := strings.TrimSpace(os.Getenv("CF_API_TOKEN"))
namespace := strings.TrimSpace(os.Getenv("KV_NAMESPACE_ID"))
return accountID, apiToken, namespace
}
func validateOptions(account, token, namespace string) {
if len(account) == 0 {
log.Fatal("Store: CF_ACCOUNT_ID is blank")
}
if len(token) == 0 {
log.Fatal("Store: CF_API_TOKEN is blank")
}
if len(namespace) == 0 {
log.Fatal("Store: KV_NAMESPACE_ID is blank")
}
}
func (w *workersKV) Close() error {
return nil
}
func (w *workersKV) Init(opts ...store.Option) error {
for _, o := range opts {
o(&w.options)
}
if len(w.options.Database) > 0 {
w.namespace = w.options.Database
}
if w.options.Context == nil {
w.options.Context = context.TODO()
}
ttl := w.options.Context.Value("STORE_CACHE_TTL")
if ttl != nil {
ttlduration, ok := ttl.(time.Duration)
if !ok {
log.Fatal("STORE_CACHE_TTL from context must be type int64")
}
w.cache = cache.New(ttlduration, 3*ttlduration)
}
return nil
}
func (w *workersKV) list(prefix string) ([]string, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/keys", w.account, w.namespace)
body := make(map[string]string)
if len(prefix) > 0 {
body["prefix"] = prefix
}
response, _, _, err := w.request(ctx, http.MethodGet, path, body, make(http.Header))
if err != nil {
return nil, err
}
a := &apiResponse{}
if err := json.Unmarshal(response, a); err != nil {
return nil, err
}
if !a.Success {
messages := ""
for _, m := range a.Errors {
messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
}
return nil, errors.New(messages)
}
keys := make([]string, 0, len(a.Result))
for _, r := range a.Result {
keys = append(keys, r.Name)
}
return keys, nil
}
// In the cloudflare workers KV implemention, List() doesn't guarantee
// anything as the workers API is eventually consistent.
func (w *workersKV) List(opts ...store.ListOption) ([]string, error) {
keys, err := w.list("")
if err != nil {
return nil, err
}
return keys, nil
}
func (w *workersKV) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
var options store.ReadOptions
for _, o := range opts {
o(&options)
}
keys := []string{key}
if options.Prefix {
k, err := w.list(key)
if err != nil {
return nil, err
}
keys = k
}
//nolint:prealloc
var records []*store.Record
for _, k := range keys {
if w.cache != nil {
if resp, hit := w.cache.Get(k); hit {
if record, ok := resp.(*store.Record); ok {
records = append(records, record)
continue
}
}
}
path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(k))
response, headers, status, err := w.request(ctx, http.MethodGet, path, nil, make(http.Header))
if err != nil {
return records, err
}
if status < 200 || status >= 300 {
if status == 404 {
return nil, store.ErrNotFound
}
return records, errors.New("Received unexpected Status " + strconv.Itoa(status) + string(response))
}
record := &store.Record{
Key: k,
Value: response,
}
if expiry := headers.Get("Expiration"); len(expiry) != 0 {
expiryUnix, err := strconv.ParseInt(expiry, 10, 64)
if err != nil {
return records, err
}
record.Expiry = time.Until(time.Unix(expiryUnix, 0))
}
if w.cache != nil {
w.cache.Set(record.Key, record, cache.DefaultExpiration)
}
records = append(records, record)
}
return records, nil
}
func (w *workersKV) Write(r *store.Record, opts ...store.WriteOption) error {
// Set it in local cache, with the global TTL from options
if w.cache != nil {
w.cache.Set(r.Key, r, cache.DefaultExpiration)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(r.Key))
if r.Expiry != 0 {
// Minimum cloudflare TTL is 60 Seconds
exp := int(math.Max(60, math.Round(r.Expiry.Seconds())))
path = path + "?expiration_ttl=" + strconv.Itoa(exp)
}
headers := make(http.Header)
resp, _, _, err := w.request(ctx, http.MethodPut, path, r.Value, headers)
if err != nil {
return err
}
a := &apiResponse{}
if err := json.Unmarshal(resp, a); err != nil {
return err
}
if !a.Success {
messages := ""
for _, m := range a.Errors {
messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
}
return errors.New(messages)
}
return nil
}
func (w *workersKV) Delete(key string, opts ...store.DeleteOption) error {
if w.cache != nil {
w.cache.Delete(key)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(key))
resp, _, _, err := w.request(ctx, http.MethodDelete, path, nil, make(http.Header))
if err != nil {
return err
}
a := &apiResponse{}
if err := json.Unmarshal(resp, a); err != nil {
return err
}
if !a.Success {
messages := ""
for _, m := range a.Errors {
messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
}
return errors.New(messages)
}
return nil
}
func (w *workersKV) request(ctx context.Context, method, path string, body interface{}, headers http.Header) ([]byte, http.Header, int, error) {
var jsonBody []byte
var err error
if body != nil {
if paramBytes, ok := body.([]byte); ok {
jsonBody = paramBytes
} else {
jsonBody, err = json.Marshal(body)
if err != nil {
return nil, nil, 0, errors.Wrap(err, "error marshalling params to JSON")
}
}
} else {
jsonBody = nil
}
var reqBody io.Reader
if jsonBody != nil {
reqBody = bytes.NewReader(jsonBody)
}
req, err := http.NewRequestWithContext(ctx, method, apiBaseURL+path, reqBody)
if err != nil {
return nil, nil, 0, errors.Wrap(err, "error creating new request")
}
for key, value := range headers {
req.Header[key] = value
}
// set token if it exists
if len(w.token) > 0 {
req.Header.Set("Authorization", "Bearer "+w.token)
}
// set the user agent to micro
req.Header.Set("User-Agent", "micro/1.0 (https://micro.mu)")
// Official cloudflare client does exponential backoff here
// TODO: retry and use util/backoff
resp, err := w.httpClient.Do(req)
if err != nil {
return nil, nil, 0, err
}
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
return respBody, resp.Header, resp.StatusCode, err
}
return respBody, resp.Header, resp.StatusCode, nil
}
func (w *workersKV) String() string {
return "cloudflare"
}
func (w *workersKV) Options() store.Options {
return w.options
}
// NewStore returns a cloudflare Store implementation.
// Account ID, Token and Namespace must either be passed as options or
// environment variables. If set as env vars we expect the following;
// CF_API_TOKEN to a cloudflare API token scoped to Workers KV.
// CF_ACCOUNT_ID to contain a string with your cloudflare account ID.
// KV_NAMESPACE_ID to contain the namespace UUID for your KV storage.
func NewStore(opts ...store.Option) store.Store {
var options store.Options
for _, o := range opts {
o(&options)
}
// get options from environment
account, token, namespace := getOptions()
if len(account) == 0 {
account = getAccount(options.Context)
}
if len(token) == 0 {
token = getToken(options.Context)
}
if len(namespace) == 0 {
namespace = options.Database
}
// validate options are not blank or log.Fatal
validateOptions(account, token, namespace)
return &workersKV{
account: account,
namespace: namespace,
token: token,
options: options,
httpClient: &http.Client{},
}
}

95
store/cloudflare/cloudflare_test.go
View File

@ -1,95 +0,0 @@
package cloudflare
import (
"math/rand"
"os"
"strconv"
"testing"
"time"
"github.com/micro/go-micro/v2/store"
)
func TestCloudflare(t *testing.T) {
if len(os.Getenv("IN_TRAVIS_CI")) != 0 {
t.Skip()
}
apiToken, accountID := os.Getenv("CF_API_TOKEN"), os.Getenv("CF_ACCOUNT_ID")
kvID := os.Getenv("KV_NAMESPACE_ID")
if len(apiToken) == 0 || len(accountID) == 0 || len(kvID) == 0 {
t.Skip("No Cloudflare API keys available, skipping test")
}
rand.Seed(time.Now().UnixNano())
randomK := strconv.Itoa(rand.Int())
randomV := strconv.Itoa(rand.Int())
wkv := NewStore(
Token(apiToken),
Account(accountID),
Namespace(kvID),
CacheTTL(60000000000),
)
records, err := wkv.List()
if err != nil {
t.Fatalf("List: %s\n", err.Error())
} else {
if len(os.Getenv("IN_TRAVIS_CI")) == 0 {
t.Log("Listed " + strconv.Itoa(len(records)) + " records")
}
}
err = wkv.Write(&store.Record{
Key: randomK,
Value: []byte(randomV),
})
if err != nil {
t.Errorf("Write: %s", err.Error())
}
err = wkv.Write(&store.Record{
Key: "expirationtest",
Value: []byte("This message will self destruct"),
Expiry: 75 * time.Second,
})
if err != nil {
t.Errorf("Write: %s", err.Error())
}
// This might be needed for cloudflare eventual consistency
time.Sleep(1 * time.Minute)
r, err := wkv.Read(randomK)
if err != nil {
t.Errorf("Read: %s\n", err.Error())
}
if len(r) != 1 {
t.Errorf("Expected to read 1 key, got %d keys\n", len(r))
}
if string(r[0].Value) != randomV {
t.Errorf("Read: expected %s, got %s\n", randomK, string(r[0].Value))
}
r, err = wkv.Read("expirationtest")
if err != nil {
t.Errorf("Read: expirationtest should still exist")
}
if r[0].Expiry == 0 {
t.Error("Expected r to have an expiry")
} else {
t.Log(r[0].Expiry)
}
time.Sleep(20 * time.Second)
r, err = wkv.Read("expirationtest")
if err == nil && len(r) != 0 {
t.Error("Read: Managed to read expirationtest, but it should have expired")
t.Log(err, r[0].Key, string(r[0].Value), r[0].Expiry, len(r))
}
err = wkv.Delete(randomK)
if err != nil {
t.Errorf("Delete: %s\n", err.Error())
}
}

64
store/cloudflare/options.go
View File

@ -1,64 +0,0 @@
package cloudflare
import (
"context"
"time"
"github.com/micro/go-micro/v2/store"
)
func getOption(ctx context.Context, key string) string {
if ctx == nil {
return ""
}
val, ok := ctx.Value(key).(string)
if !ok {
return ""
}
return val
}
func getToken(ctx context.Context) string {
return getOption(ctx, "CF_API_TOKEN")
}
func getAccount(ctx context.Context) string {
return getOption(ctx, "CF_ACCOUNT_ID")
}
// Token sets the cloudflare api token
func Token(t string) store.Option {
return func(o *store.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, "CF_API_TOKEN", t)
}
}
// Account sets the cloudflare account id
func Account(id string) store.Option {
return func(o *store.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, "CF_ACCOUNT_ID", id)
}
}
// Namespace sets the KV namespace
func Namespace(ns string) store.Option {
return func(o *store.Options) {
o.Database = ns
}
}
// CacheTTL sets the timeout in nanoseconds of the read/write cache
func CacheTTL(ttl time.Duration) store.Option {
return func(o *store.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, "STORE_CACHE_TTL", ttl)
}
}

178
store/etcd/config.go
View File

@ -1,178 +0,0 @@
package etcd
import (
"context"
cryptotls "crypto/tls"
"time"
"github.com/coreos/etcd/clientv3"
"github.com/micro/go-micro/v2/store"
"google.golang.org/grpc"
)
// Implement all the options from https://pkg.go.dev/github.com/coreos/etcd/clientv3?tab=doc#Config
// Need to use non basic types in context.WithValue
type autoSyncInterval string
type dialTimeout string
type dialKeepAliveTime string
type dialKeepAliveTimeout string
type maxCallSendMsgSize string
type maxCallRecvMsgSize string
type tls string
type username string
type password string
type rejectOldCluster string
type dialOptions string
type clientContext string
type permitWithoutStream string
// AutoSyncInterval is the interval to update endpoints with its latest members.
// 0 disables auto-sync. By default auto-sync is disabled.
func AutoSyncInterval(d time.Duration) store.Option {
return func(o *store.Options) {
o.Context = context.WithValue(o.Context, autoSyncInterval(""), d)
}
}
// DialTimeout is the timeout for failing to establish a connection.
func DialTimeout(d time.Duration) store.Option {
return func(o *store.Options) {
o.Context = context.WithValue(o.Context, dialTimeout(""), d)
}
}
// DialKeepAliveTime is the time after which client pings the server to see if
// transport is alive.
func DialKeepAliveTime(d time.Duration) store.Option {
return func(o *store.Options) {
o.Context = context.WithValue(o.Context, dialKeepAliveTime(""), d)
}
}
// DialKeepAliveTimeout is the time that the client waits for a response for the
// keep-alive probe. If the response is not received in this time, the connection is closed.
func DialKeepAliveTimeout(d time.Duration) store.Option {
return func(o *store.Options) {
o.Context = context.WithValue(o.Context, dialKeepAliveTimeout(""), d)
}
}
// MaxCallSendMsgSize is the client-side request send limit in bytes.
// If 0, it defaults to 2.0 MiB (2 * 1024 * 1024).
// Make sure that "MaxCallSendMsgSize" < server-side default send/recv limit.
// ("--max-request-bytes" flag to etcd or "embed.Config.MaxRequestBytes").
func MaxCallSendMsgSize(size int) store.Option {
return func(o *store.Options) {
o.Context = context.WithValue(o.Context, maxCallSendMsgSize(""), size)
}
}
// MaxCallRecvMsgSize is the client-side response receive limit.
// If 0, it defaults to "math.MaxInt32", because range response can
// easily exceed request send limits.
// Make sure that "MaxCallRecvMsgSize" >= server-side default send/recv limit.
// ("--max-request-bytes" flag to etcd or "embed.Config.MaxRequestBytes").
func MaxCallRecvMsgSize(size int) store.Option {
return func(o *store.Options) {
o.Context = context.WithValue(o.Context, maxCallRecvMsgSize(""), size)
}
}
// TLS holds the client secure credentials, if any.
func TLS(conf *cryptotls.Config) store.Option {
return func(o *store.Options) {
t := conf.Clone()
o.Context = context.WithValue(o.Context, tls(""), t)
}
}
// Username is a user name for authentication.
func Username(u string) store.Option {
return func(o *store.Options) {
o.Context = context.WithValue(o.Context, username(""), u)
}
}
// Password is a password for authentication.
func Password(p string) store.Option {
return func(o *store.Options) {
o.Context = context.WithValue(o.Context, password(""), p)
}
}
// RejectOldCluster when set will refuse to create a client against an outdated cluster.
func RejectOldCluster(b bool) store.Option {
return func(o *store.Options) {
o.Context = context.WithValue(o.Context, rejectOldCluster(""), b)
}
}
// DialOptions is a list of dial options for the grpc client (e.g., for interceptors).
// For example, pass "grpc.WithBlock()" to block until the underlying connection is up.
// Without this, Dial returns immediately and connecting the server happens in background.
func DialOptions(opts []grpc.DialOption) store.Option {
return func(o *store.Options) {
if len(opts) > 0 {
ops := make([]grpc.DialOption, len(opts))
copy(ops, opts)
o.Context = context.WithValue(o.Context, dialOptions(""), ops)
}
}
}
// ClientContext is the default etcd3 client context; it can be used to cancel grpc
// dial out andother operations that do not have an explicit context.
func ClientContext(ctx context.Context) store.Option {
return func(o *store.Options) {
o.Context = context.WithValue(o.Context, clientContext(""), ctx)
}
}
// PermitWithoutStream when set will allow client to send keepalive pings to server without any active streams(RPCs).
func PermitWithoutStream(b bool) store.Option {
return func(o *store.Options) {
o.Context = context.WithValue(o.Context, permitWithoutStream(""), b)
}
}
func (e *etcdStore) applyConfig(cfg *clientv3.Config) {
if v := e.options.Context.Value(autoSyncInterval("")); v != nil {
cfg.AutoSyncInterval = v.(time.Duration)
}
if v := e.options.Context.Value(dialTimeout("")); v != nil {
cfg.DialTimeout = v.(time.Duration)
}
if v := e.options.Context.Value(dialKeepAliveTime("")); v != nil {
cfg.DialKeepAliveTime = v.(time.Duration)
}
if v := e.options.Context.Value(dialKeepAliveTimeout("")); v != nil {
cfg.DialKeepAliveTimeout = v.(time.Duration)
}
if v := e.options.Context.Value(maxCallSendMsgSize("")); v != nil {
cfg.MaxCallSendMsgSize = v.(int)
}
if v := e.options.Context.Value(maxCallRecvMsgSize("")); v != nil {
cfg.MaxCallRecvMsgSize = v.(int)
}
if v := e.options.Context.Value(tls("")); v != nil {
cfg.TLS = v.(*cryptotls.Config)
}
if v := e.options.Context.Value(username("")); v != nil {
cfg.Username = v.(string)
}
if v := e.options.Context.Value(password("")); v != nil {
cfg.Username = v.(string)
}
if v := e.options.Context.Value(rejectOldCluster("")); v != nil {
cfg.RejectOldCluster = v.(bool)
}
if v := e.options.Context.Value(dialOptions("")); v != nil {
cfg.DialOptions = v.([]grpc.DialOption)
}
if v := e.options.Context.Value(clientContext("")); v != nil {
cfg.Context = v.(context.Context)
}
if v := e.options.Context.Value(permitWithoutStream("")); v != nil {
cfg.PermitWithoutStream = v.(bool)
}
}

272
store/etcd/etcd.go
View File

@ -1,272 +0,0 @@
// Package etcd implements a go-micro/v2/store with etcd
package etcd
import (
"bytes"
"context"
"encoding/gob"
"math"
"strings"
"time"
"github.com/coreos/etcd/clientv3"
"github.com/coreos/etcd/clientv3/namespace"
"github.com/micro/go-micro/v2/store"
"github.com/pkg/errors"
)
type etcdStore struct {
options store.Options
client *clientv3.Client
config clientv3.Config
}
// NewStore returns a new etcd store
func NewStore(opts ...store.Option) store.Store {
e := &etcdStore{}
for _, o := range opts {
o(&e.options)
}
e.init()
return e
}
func (e *etcdStore) Close() error {
return e.client.Close()
}
func (e *etcdStore) Init(opts ...store.Option) error {
for _, o := range opts {
o(&e.options)
}
return e.init()
}
func (e *etcdStore) init() error {
// ensure context is non-nil
e.options.Context = context.Background()
// set up config
e.config = clientv3.Config{}
e.applyConfig(&e.config)
if len(e.options.Nodes) == 0 {
e.config.Endpoints = []string{"http://127.0.0.1:2379"}
} else {
e.config.Endpoints = make([]string, len(e.options.Nodes))
copy(e.config.Endpoints, e.options.Nodes)
}
if e.client != nil {
e.client.Close()
}
client, err := clientv3.New(e.config)
if err != nil {
return err
}
e.client = client
ns := ""
if len(e.options.Table) > 0 {
ns = e.options.Table
}
if len(e.options.Database) > 0 {
ns = e.options.Database + "/" + ns
}
if len(ns) > 0 {
e.client.KV = namespace.NewKV(e.client.KV, ns)
e.client.Watcher = namespace.NewWatcher(e.client.Watcher, ns)
e.client.Lease = namespace.NewLease(e.client.Lease, ns)
}
return nil
}
func (e *etcdStore) Options() store.Options {
return e.options
}
func (e *etcdStore) String() string {
return "etcd"
}
func (e *etcdStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) {
readOpts := store.ReadOptions{}
for _, o := range opts {
o(&readOpts)
}
if readOpts.Suffix {
return e.readSuffix(key, readOpts)
}
var etcdOpts []clientv3.OpOption
if readOpts.Prefix {
etcdOpts = append(etcdOpts, clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend))
}
resp, err := e.client.KV.Get(context.Background(), key, etcdOpts...)
if err != nil {
return nil, err
}
if resp.Count == 0 && !(readOpts.Prefix || readOpts.Suffix) {
return nil, store.ErrNotFound
}
var records []*store.Record
for _, kv := range resp.Kvs {
ir := internalRecord{}
if err := gob.NewDecoder(bytes.NewReader(kv.Value)).Decode(&ir); err != nil {
return records, errors.Wrapf(err, "couldn't decode %s into internalRecord", err.Error())
}
r := store.Record{
Key: ir.Key,
Value: ir.Value,
}
if !ir.ExpiresAt.IsZero() {
r.Expiry = time.Until(ir.ExpiresAt)
}
records = append(records, &r)
}
if readOpts.Limit > 0 || readOpts.Offset > 0 {
return records[readOpts.Offset:min(readOpts.Limit, uint(len(records)))], nil
}
return records, nil
}
func (e *etcdStore) readSuffix(key string, readOpts store.ReadOptions) ([]*store.Record, error) {
opts := []store.ListOption{store.ListSuffix(key)}
if readOpts.Prefix {
opts = append(opts, store.ListPrefix(key))
}
keys, err := e.List(opts...)
if err != nil {
return nil, errors.Wrapf(err, "Couldn't list with suffix %s", key)
}
var records []*store.Record
for _, k := range keys {
resp, err := e.client.KV.Get(context.Background(), k)
if err != nil {
return nil, errors.Wrapf(err, "Couldn't get key %s", k)
}
ir := internalRecord{}
if err := gob.NewDecoder(bytes.NewReader(resp.Kvs[0].Value)).Decode(&ir); err != nil {
return records, errors.Wrapf(err, "couldn't decode %s into internalRecord", err.Error())
}
r := store.Record{
Key: ir.Key,
Value: ir.Value,
}
if !ir.ExpiresAt.IsZero() {
r.Expiry = time.Until(ir.ExpiresAt)
}
records = append(records, &r)
}
if readOpts.Limit > 0 || readOpts.Offset > 0 {
return records[readOpts.Offset:min(readOpts.Limit, uint(len(records)))], nil
}
return records, nil
}
func (e *etcdStore) Write(r *store.Record, opts ...store.WriteOption) error {
options := store.WriteOptions{}
for _, o := range opts {
o(&options)
}
if len(opts) > 0 {
// Copy the record before applying options, or the incoming record will be mutated
newRecord := store.Record{}
newRecord.Key = r.Key
newRecord.Value = make([]byte, len(r.Value))
copy(newRecord.Value, r.Value)
newRecord.Expiry = r.Expiry
if !options.Expiry.IsZero() {
newRecord.Expiry = time.Until(options.Expiry)
}
if options.TTL != 0 {
newRecord.Expiry = options.TTL
}
return e.write(&newRecord)
}
return e.write(r)
}
func (e *etcdStore) write(r *store.Record) error {
var putOpts []clientv3.OpOption
ir := &internalRecord{}
ir.Key = r.Key
ir.Value = make([]byte, len(r.Value))
copy(ir.Value, r.Value)
if r.Expiry != 0 {
ir.ExpiresAt = time.Now().Add(r.Expiry)
var leasexpiry int64
if r.Expiry.Seconds() < 5.0 {
// minimum etcd lease is 5 seconds
leasexpiry = 5
} else {
leasexpiry = int64(math.Ceil(r.Expiry.Seconds()))
}
lr, err := e.client.Lease.Grant(context.Background(), leasexpiry)
if err != nil {
return errors.Wrapf(err, "couldn't grant an etcd lease for %s", r.Key)
}
putOpts = append(putOpts, clientv3.WithLease(lr.ID))
}
b := &bytes.Buffer{}
if err := gob.NewEncoder(b).Encode(ir); err != nil {
return errors.Wrapf(err, "couldn't encode %s", r.Key)
}
_, err := e.client.KV.Put(context.Background(), ir.Key, string(b.Bytes()), putOpts...)
return errors.Wrapf(err, "couldn't put key %s in to etcd", err)
}
func (e *etcdStore) Delete(key string, opts ...store.DeleteOption) error {
options := store.DeleteOptions{}
for _, o := range opts {
o(&options)
}
_, err := e.client.KV.Delete(context.Background(), key)
return errors.Wrapf(err, "couldn't delete key %s", key)
}
func (e *etcdStore) List(opts ...store.ListOption) ([]string, error) {
options := store.ListOptions{}
for _, o := range opts {
o(&options)
}
searchPrefix := ""
if len(options.Prefix) > 0 {
searchPrefix = options.Prefix
}
resp, err := e.client.KV.Get(context.Background(), searchPrefix, clientv3.WithPrefix(), clientv3.WithKeysOnly(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend))
if err != nil {
return nil, errors.Wrap(err, "couldn't list, etcd get failed")
}
if len(options.Suffix) == 0 {
keys := make([]string, resp.Count)
for i, kv := range resp.Kvs {
keys[i] = string(kv.Key)
}
return keys, nil
}
keys := []string{}
for _, kv := range resp.Kvs {
if strings.HasSuffix(string(kv.Key), options.Suffix) {
keys = append(keys, string(kv.Key))
}
}
if options.Limit > 0 || options.Offset > 0 {
return keys[options.Offset:min(options.Limit, uint(len(keys)))], nil
}
return keys, nil
}
type internalRecord struct {
Key string
Value []byte
ExpiresAt time.Time
}
func min(i, j uint) uint {
if i < j {
return i
}
return j
}

225
store/etcd/etcd_test.go
View File

@ -1,225 +0,0 @@
package etcd
import (
"fmt"
"testing"
"time"
"github.com/kr/pretty"
"github.com/micro/go-micro/v2/store"
)
func TestEtcd(t *testing.T) {
e := NewStore()
if err := e.Init(); err != nil {
t.Fatal(err)
}
//basictest(e, t)
}
func basictest(s store.Store, t *testing.T) {
t.Logf("Testing store %s, with options %# v\n", s.String(), pretty.Formatter(s.Options()))
// Read and Write an expiring Record
if err := s.Write(&store.Record{
Key: "Hello",
Value: []byte("World"),
Expiry: time.Second * 5,
}); err != nil {
t.Fatal(err)
}
if r, err := s.Read("Hello"); err != nil {
t.Fatal(err)
} else {
if len(r) != 1 {
t.Fatal("Read returned multiple records")
}
if r[0].Key != "Hello" {
t.Fatalf("Expected %s, got %s", "Hello", r[0].Key)
}
if string(r[0].Value) != "World" {
t.Fatalf("Expected %s, got %s", "World", r[0].Value)
}
}
time.Sleep(time.Second * 6)
if records, err := s.Read("Hello"); err != store.ErrNotFound {
t.Fatalf("Expected %# v, got %# v\nResults were %# v", store.ErrNotFound, err, pretty.Formatter(records))
}
// Write 3 records with various expiry and get with prefix
records := []*store.Record{
&store.Record{
Key: "foo",
Value: []byte("foofoo"),
},
&store.Record{
Key: "foobar",
Value: []byte("foobarfoobar"),
Expiry: time.Second * 5,
},
&store.Record{
Key: "foobarbaz",
Value: []byte("foobarbazfoobarbaz"),
Expiry: 2 * time.Second * 5,
},
}
for _, r := range records {
if err := s.Write(r); err != nil {
t.Fatalf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err)
}
}
if results, err := s.Read("foo", store.ReadPrefix()); err != nil {
t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
} else {
if len(results) != 3 {
t.Fatalf("Expected 3 items, got %d", len(results))
}
}
time.Sleep(time.Second * 6)
if results, err := s.Read("foo", store.ReadPrefix()); err != nil {
t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
} else {
if len(results) != 2 {
t.Fatalf("Expected 2 items, got %d", len(results))
}
}
time.Sleep(time.Second * 5)
if results, err := s.Read("foo", store.ReadPrefix()); err != nil {
t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
} else {
if len(results) != 1 {
t.Fatalf("Expected 1 item, got %d", len(results))
}
}
if err := s.Delete("foo", func(d *store.DeleteOptions) {}); err != nil {
t.Fatalf("Delete failed (%v)", err)
}
if results, err := s.Read("foo", store.ReadPrefix()); err != nil {
t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
} else {
if len(results) != 0 {
t.Fatalf("Expected 0 items, got %d (%# v)", len(results), pretty.Formatter(results))
}
}
// Write 3 records with various expiry and get with Suffix
records = []*store.Record{
&store.Record{
Key: "foo",
Value: []byte("foofoo"),
},
&store.Record{
Key: "barfoo",
Value: []byte("barfoobarfoo"),
Expiry: time.Second * 5,
},
&store.Record{
Key: "bazbarfoo",
Value: []byte("bazbarfoobazbarfoo"),
Expiry: 2 * time.Second * 5,
},
}
for _, r := range records {
if err := s.Write(r); err != nil {
t.Fatalf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err)
}
}
if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
} else {
if len(results) != 3 {
t.Fatalf("Expected 3 items, got %d", len(results))
}
}
time.Sleep(time.Second * 6)
if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
} else {
if len(results) != 2 {
t.Fatalf("Expected 2 items, got %d", len(results))
}
t.Logf("Prefix test: %v\n", pretty.Formatter(results))
}
time.Sleep(time.Second * 5)
if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
} else {
if len(results) != 1 {
t.Fatalf("Expected 1 item, got %d", len(results))
}
t.Logf("Prefix test: %# v\n", pretty.Formatter(results))
}
if err := s.Delete("foo"); err != nil {
t.Fatalf("Delete failed (%v)", err)
}
if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
} else {
if len(results) != 0 {
t.Fatalf("Expected 0 items, got %d (%# v)", len(results), pretty.Formatter(results))
}
}
// Test Prefix, Suffix and WriteOptions
if err := s.Write(&store.Record{
Key: "foofoobarbar",
Value: []byte("something"),
}, store.WriteTTL(time.Millisecond*100)); err != nil {
t.Fatal(err)
}
if err := s.Write(&store.Record{
Key: "foofoo",
Value: []byte("something"),
}, store.WriteExpiry(time.Now().Add(time.Millisecond*100))); err != nil {
t.Fatal(err)
}
if err := s.Write(&store.Record{
Key: "barbar",
Value: []byte("something"),
// TTL has higher precedence than expiry
}, store.WriteExpiry(time.Now().Add(time.Hour)), store.WriteTTL(time.Millisecond*100)); err != nil {
t.Fatal(err)
}
if results, err := s.Read("foo", store.ReadPrefix(), store.ReadSuffix()); err != nil {
t.Fatal(err)
} else {
if len(results) != 1 {
t.Fatalf("Expected 1 results, got %d: %# v", len(results), pretty.Formatter(results))
}
}
time.Sleep(time.Second * 6)
if results, err := s.List(); err != nil {
t.Fatalf("List failed: %s", err)
} else {
if len(results) != 0 {
t.Fatal("Expiry options were not effective")
}
}
s.Init()
for i := 0; i < 10; i++ {
s.Write(&store.Record{
Key: fmt.Sprintf("a%d", i),
Value: []byte{},
})
}
if results, err := s.Read("a", store.ReadLimit(5), store.ReadPrefix()); err != nil {
t.Fatal(err)
} else {
if len(results) != 5 {
t.Fatal("Expected 5 results, got ", len(results))
}
if results[0].Key != "a0" {
t.Fatalf("Expected a0, got %s", results[0].Key)
}
if results[4].Key != "a4" {
t.Fatalf("Expected a4, got %s", results[4].Key)
}
}
if results, err := s.Read("a", store.ReadLimit(30), store.ReadOffset(5), store.ReadPrefix()); err != nil {
t.Fatal(err)
} else {
if len(results) != 5 {
t.Fatal("Expected 5 results, got ", len(results))
}
}
}

2
store/file/file.go
View File

@ -21,7 +21,7 @@ var (
// DefaultTable when none is specified
DefaultTable = "micro"
// DefaultDir is the default directory for bbolt files
DefaultDir = os.TempDir()
DefaultDir = filepath.Join(os.TempDir(), "micro", "store")
// bucket used for data storage
dataBucket = "data"

40
store/options.go
View File

@ -54,6 +54,7 @@ func WithContext(c context.Context) Option {
// ReadOptions configures an individual Read operation
type ReadOptions struct {
Database, Table string
// Prefix returns all records that are prefixed with key
Prefix bool
// Suffix returns all records that have the suffix key
@ -67,6 +68,14 @@ type ReadOptions struct {
// ReadOption sets values in ReadOptions
type ReadOption func(r *ReadOptions)
// ReadFrom the database and table
func ReadFrom(database, table string) ReadOption {
return func(r *ReadOptions) {
r.Database = database
r.Table = table
}
}
// ReadPrefix returns all records that are prefixed with key
func ReadPrefix() ReadOption {
return func(r *ReadOptions) {
@ -98,6 +107,7 @@ func ReadOffset(o uint) ReadOption {
// WriteOptions configures an individual Write operation
// If Expiry and TTL are set TTL takes precedence
type WriteOptions struct {
Database, Table string
// Expiry is the time the record expires
Expiry time.Time
// TTL is the time until the record expires
@ -107,6 +117,14 @@ type WriteOptions struct {
// WriteOption sets values in WriteOptions
type WriteOption func(w *WriteOptions)
// WriteTo the database and table
func WriteTo(database, table string) WriteOption {
return func(w *WriteOptions) {
w.Database = database
w.Table = table
}
}
// WriteExpiry is the time the record expires
func WriteExpiry(t time.Time) WriteOption {
return func(w *WriteOptions) {
@ -122,13 +140,25 @@ func WriteTTL(d time.Duration) WriteOption {
}
// DeleteOptions configures an individual Delete operation
type DeleteOptions struct{}
type DeleteOptions struct {
Database, Table string
}
// DeleteOption sets values in DeleteOptions
type DeleteOption func(d *DeleteOptions)
// DeleteFrom the database and table
func DeleteFrom(database, table string) DeleteOption {
return func(d *DeleteOptions) {
d.Database = database
d.Table = table
}
}
// ListOptions configures an individual List operation
type ListOptions struct {
// List from the following
Database, Table string
// Prefix returns all keys that are prefixed with key
Prefix string
// Suffix returns all keys that end with key
@ -142,6 +172,14 @@ type ListOptions struct {
// ListOption sets values in ListOptions
type ListOption func(l *ListOptions)
// ListFrom the database and table
func ListFrom(database, table string) ListOption {
return func(l *ListOptions) {
l.Database = database
l.Table = table
}
}
// ListPrefix returns all keys that are prefixed with key
func ListPrefix(p string) ListOption {
return func(l *ListOptions) {

471
store/service/proto/store.pb.go
View File

@ -1,15 +1,11 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: store/service/proto/store.proto
// source: store.proto
package go_micro_store
import (
context "context"
fmt "fmt"
proto "github.com/golang/protobuf/proto"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
math "math"
)
@ -40,7 +36,7 @@ func (m *Record) Reset() { *m = Record{} }
func (m *Record) String() string { return proto.CompactTextString(m) }
func (*Record) ProtoMessage() {}
func (*Record) Descriptor() ([]byte, []int) {
return fileDescriptor_1ba364858f5c3cdb, []int{0}
return fileDescriptor_98bbca36ef968dfc, []int{0}
}
func (m *Record) XXX_Unmarshal(b []byte) error {
@ -96,7 +92,7 @@ func (m *ReadOptions) Reset() { *m = ReadOptions{} }
func (m *ReadOptions) String() string { return proto.CompactTextString(m) }
func (*ReadOptions) ProtoMessage() {}
func (*ReadOptions) Descriptor() ([]byte, []int) {
return fileDescriptor_1ba364858f5c3cdb, []int{1}
return fileDescriptor_98bbca36ef968dfc, []int{1}
}
func (m *ReadOptions) XXX_Unmarshal(b []byte) error {
@ -157,7 +153,7 @@ func (m *ReadRequest) Reset() { *m = ReadRequest{} }
func (m *ReadRequest) String() string { return proto.CompactTextString(m) }
func (*ReadRequest) ProtoMessage() {}
func (*ReadRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_1ba364858f5c3cdb, []int{2}
return fileDescriptor_98bbca36ef968dfc, []int{2}
}
func (m *ReadRequest) XXX_Unmarshal(b []byte) error {
@ -203,7 +199,7 @@ func (m *ReadResponse) Reset() { *m = ReadResponse{} }
func (m *ReadResponse) String() string { return proto.CompactTextString(m) }
func (*ReadResponse) ProtoMessage() {}
func (*ReadResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_1ba364858f5c3cdb, []int{3}
return fileDescriptor_98bbca36ef968dfc, []int{3}
}
func (m *ReadResponse) XXX_Unmarshal(b []byte) error {
@ -245,7 +241,7 @@ func (m *WriteOptions) Reset() { *m = WriteOptions{} }
func (m *WriteOptions) String() string { return proto.CompactTextString(m) }
func (*WriteOptions) ProtoMessage() {}
func (*WriteOptions) Descriptor() ([]byte, []int) {
return fileDescriptor_1ba364858f5c3cdb, []int{4}
return fileDescriptor_98bbca36ef968dfc, []int{4}
}
func (m *WriteOptions) XXX_Unmarshal(b []byte) error {
@ -292,7 +288,7 @@ func (m *WriteRequest) Reset() { *m = WriteRequest{} }
func (m *WriteRequest) String() string { return proto.CompactTextString(m) }
func (*WriteRequest) ProtoMessage() {}
func (*WriteRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_1ba364858f5c3cdb, []int{5}
return fileDescriptor_98bbca36ef968dfc, []int{5}
}
func (m *WriteRequest) XXX_Unmarshal(b []byte) error {
@ -337,7 +333,7 @@ func (m *WriteResponse) Reset() { *m = WriteResponse{} }
func (m *WriteResponse) String() string { return proto.CompactTextString(m) }
func (*WriteResponse) ProtoMessage() {}
func (*WriteResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_1ba364858f5c3cdb, []int{6}
return fileDescriptor_98bbca36ef968dfc, []int{6}
}
func (m *WriteResponse) XXX_Unmarshal(b []byte) error {
@ -368,7 +364,7 @@ func (m *DeleteOptions) Reset() { *m = DeleteOptions{} }
func (m *DeleteOptions) String() string { return proto.CompactTextString(m) }
func (*DeleteOptions) ProtoMessage() {}
func (*DeleteOptions) Descriptor() ([]byte, []int) {
return fileDescriptor_1ba364858f5c3cdb, []int{7}
return fileDescriptor_98bbca36ef968dfc, []int{7}
}
func (m *DeleteOptions) XXX_Unmarshal(b []byte) error {
@ -401,7 +397,7 @@ func (m *DeleteRequest) Reset() { *m = DeleteRequest{} }
func (m *DeleteRequest) String() string { return proto.CompactTextString(m) }
func (*DeleteRequest) ProtoMessage() {}
func (*DeleteRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_1ba364858f5c3cdb, []int{8}
return fileDescriptor_98bbca36ef968dfc, []int{8}
}
func (m *DeleteRequest) XXX_Unmarshal(b []byte) error {
@ -446,7 +442,7 @@ func (m *DeleteResponse) Reset() { *m = DeleteResponse{} }
func (m *DeleteResponse) String() string { return proto.CompactTextString(m) }
func (*DeleteResponse) ProtoMessage() {}
func (*DeleteResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_1ba364858f5c3cdb, []int{9}
return fileDescriptor_98bbca36ef968dfc, []int{9}
}
func (m *DeleteResponse) XXX_Unmarshal(b []byte) error {
@ -481,7 +477,7 @@ func (m *ListOptions) Reset() { *m = ListOptions{} }
func (m *ListOptions) String() string { return proto.CompactTextString(m) }
func (*ListOptions) ProtoMessage() {}
func (*ListOptions) Descriptor() ([]byte, []int) {
return fileDescriptor_1ba364858f5c3cdb, []int{10}
return fileDescriptor_98bbca36ef968dfc, []int{10}
}
func (m *ListOptions) XXX_Unmarshal(b []byte) error {
@ -541,7 +537,7 @@ func (m *ListRequest) Reset() { *m = ListRequest{} }
func (m *ListRequest) String() string { return proto.CompactTextString(m) }
func (*ListRequest) ProtoMessage() {}
func (*ListRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_1ba364858f5c3cdb, []int{11}
return fileDescriptor_98bbca36ef968dfc, []int{11}
}
func (m *ListRequest) XXX_Unmarshal(b []byte) error {
@ -580,7 +576,7 @@ func (m *ListResponse) Reset() { *m = ListResponse{} }
func (m *ListResponse) String() string { return proto.CompactTextString(m) }
func (*ListResponse) ProtoMessage() {}
func (*ListResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_1ba364858f5c3cdb, []int{12}
return fileDescriptor_98bbca36ef968dfc, []int{12}
}
func (m *ListResponse) XXX_Unmarshal(b []byte) error {
@ -608,6 +604,154 @@ func (m *ListResponse) GetKeys() []string {
return nil
}
type DatabasesRequest struct {
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *DatabasesRequest) Reset() { *m = DatabasesRequest{} }
func (m *DatabasesRequest) String() string { return proto.CompactTextString(m) }
func (*DatabasesRequest) ProtoMessage() {}
func (*DatabasesRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_98bbca36ef968dfc, []int{13}
}
func (m *DatabasesRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_DatabasesRequest.Unmarshal(m, b)
}
func (m *DatabasesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_DatabasesRequest.Marshal(b, m, deterministic)
}
func (m *DatabasesRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_DatabasesRequest.Merge(m, src)
}
func (m *DatabasesRequest) XXX_Size() int {
return xxx_messageInfo_DatabasesRequest.Size(m)
}
func (m *DatabasesRequest) XXX_DiscardUnknown() {
xxx_messageInfo_DatabasesRequest.DiscardUnknown(m)
}
var xxx_messageInfo_DatabasesRequest proto.InternalMessageInfo
type DatabasesResponse struct {
Databases []string `protobuf:"bytes,1,rep,name=databases,proto3" json:"databases,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *DatabasesResponse) Reset() { *m = DatabasesResponse{} }
func (m *DatabasesResponse) String() string { return proto.CompactTextString(m) }
func (*DatabasesResponse) ProtoMessage() {}
func (*DatabasesResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_98bbca36ef968dfc, []int{14}
}
func (m *DatabasesResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_DatabasesResponse.Unmarshal(m, b)
}
func (m *DatabasesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_DatabasesResponse.Marshal(b, m, deterministic)
}
func (m *DatabasesResponse) XXX_Merge(src proto.Message) {
xxx_messageInfo_DatabasesResponse.Merge(m, src)
}
func (m *DatabasesResponse) XXX_Size() int {
return xxx_messageInfo_DatabasesResponse.Size(m)
}
func (m *DatabasesResponse) XXX_DiscardUnknown() {
xxx_messageInfo_DatabasesResponse.DiscardUnknown(m)
}
var xxx_messageInfo_DatabasesResponse proto.InternalMessageInfo
func (m *DatabasesResponse) GetDatabases() []string {
if m != nil {
return m.Databases
}
return nil
}
type TablesRequest struct {
Database string `protobuf:"bytes,1,opt,name=database,proto3" json:"database,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *TablesRequest) Reset() { *m = TablesRequest{} }
func (m *TablesRequest) String() string { return proto.CompactTextString(m) }
func (*TablesRequest) ProtoMessage() {}
func (*TablesRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_98bbca36ef968dfc, []int{15}
}
func (m *TablesRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_TablesRequest.Unmarshal(m, b)
}
func (m *TablesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_TablesRequest.Marshal(b, m, deterministic)
}
func (m *TablesRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_TablesRequest.Merge(m, src)
}
func (m *TablesRequest) XXX_Size() int {
return xxx_messageInfo_TablesRequest.Size(m)
}
func (m *TablesRequest) XXX_DiscardUnknown() {
xxx_messageInfo_TablesRequest.DiscardUnknown(m)
}
var xxx_messageInfo_TablesRequest proto.InternalMessageInfo
func (m *TablesRequest) GetDatabase() string {
if m != nil {
return m.Database
}
return ""
}
type TablesResponse struct {
Tables []string `protobuf:"bytes,1,rep,name=tables,proto3" json:"tables,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *TablesResponse) Reset() { *m = TablesResponse{} }
func (m *TablesResponse) String() string { return proto.CompactTextString(m) }
func (*TablesResponse) ProtoMessage() {}
func (*TablesResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_98bbca36ef968dfc, []int{16}
}
func (m *TablesResponse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_TablesResponse.Unmarshal(m, b)
}
func (m *TablesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_TablesResponse.Marshal(b, m, deterministic)
}
func (m *TablesResponse) XXX_Merge(src proto.Message) {
xxx_messageInfo_TablesResponse.Merge(m, src)
}
func (m *TablesResponse) XXX_Size() int {
return xxx_messageInfo_TablesResponse.Size(m)
}
func (m *TablesResponse) XXX_DiscardUnknown() {
xxx_messageInfo_TablesResponse.DiscardUnknown(m)
}
var xxx_messageInfo_TablesResponse proto.InternalMessageInfo
func (m *TablesResponse) GetTables() []string {
if m != nil {
return m.Tables
}
return nil
}
func init() {
proto.RegisterType((*Record)(nil), "go.micro.store.Record")
proto.RegisterType((*ReadOptions)(nil), "go.micro.store.ReadOptions")
@ -622,256 +766,51 @@ func init() {
proto.RegisterType((*ListOptions)(nil), "go.micro.store.ListOptions")
proto.RegisterType((*ListRequest)(nil), "go.micro.store.ListRequest")
proto.RegisterType((*ListResponse)(nil), "go.micro.store.ListResponse")
proto.RegisterType((*DatabasesRequest)(nil), "go.micro.store.DatabasesRequest")
proto.RegisterType((*DatabasesResponse)(nil), "go.micro.store.DatabasesResponse")
proto.RegisterType((*TablesRequest)(nil), "go.micro.store.TablesRequest")
proto.RegisterType((*TablesResponse)(nil), "go.micro.store.TablesResponse")
}
func init() { proto.RegisterFile("store/service/proto/store.proto", fileDescriptor_1ba364858f5c3cdb) }
var fileDescriptor_1ba364858f5c3cdb = []byte{
// 474 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x54, 0x5d, 0x6f, 0xd3, 0x30,
0x14, 0x9d, 0x9b, 0x34, 0x5b, 0x6f, 0xcb, 0xa8, 0x2c, 0x34, 0x45, 0xb0, 0x41, 0xe5, 0xa7, 0x3c,
0xa5, 0x53, 0x11, 0x1f, 0x8f, 0x48, 0x0c, 0x04, 0x08, 0x09, 0xc9, 0x48, 0x20, 0xf1, 0x36, 0xba,
0x5b, 0x64, 0xb5, 0x9b, 0x83, 0xed, 0x56, 0xeb, 0x1f, 0xe2, 0x77, 0x22, 0x7f, 0xb5, 0x69, 0x48,
0x5e, 0x78, 0xf3, 0xbd, 0xbe, 0x39, 0xe7, 0x9e, 0xe3, 0xa3, 0xc0, 0x33, 0x6d, 0xa4, 0xc2, 0xa9,
0x46, 0xb5, 0x11, 0x73, 0x9c, 0x56, 0x4a, 0x1a, 0x39, 0x75, 0xbd, 0xd2, 0x9d, 0xe9, 0xe9, 0x2f,
0x59, 0xde, 0x8a, 0xb9, 0x92, 0xa5, 0xeb, 0xb2, 0x0f, 0x90, 0x71, 0x9c, 0x4b, 0x75, 0x43, 0xc7,
0x90, 0x2c, 0x71, 0x9b, 0x93, 0x09, 0x29, 0x06, 0xdc, 0x1e, 0xe9, 0x23, 0xe8, 0x6f, 0xae, 0x57,
0x6b, 0xcc, 0x7b, 0x13, 0x52, 0x8c, 0xb8, 0x2f, 0xe8, 0x19, 0x64, 0x78, 0x5f, 0x09, 0xb5, 0xcd,
0x93, 0x09, 0x29, 0x12, 0x1e, 0x2a, 0xb6, 0x84, 0x21, 0xc7, 0xeb, 0x9b, 0x2f, 0x95, 0x11, 0xf2,
0x4e, 0xdb, 0xb1, 0x4a, 0xe1, 0x42, 0xdc, 0x3b, 0xc4, 0x13, 0x1e, 0x2a, 0xdb, 0xd7, 0xeb, 0x85,
0xed, 0xf7, 0x7c, 0xdf, 0x57, 0x96, 0x6c, 0x25, 0x6e, 0x85, 0x71, 0xa8, 0x29, 0xf7, 0x85, 0x9d,
0x96, 0x8b, 0x85, 0x46, 0x93, 0xa7, 0xae, 0x1d, 0x2a, 0xf6, 0xcd, 0x93, 0x71, 0xfc, 0xbd, 0x46,
0x6d, 0x5a, 0x76, 0x7f, 0x01, 0xc7, 0xd2, 0x6f, 0xe2, 0x78, 0x86, 0xb3, 0x27, 0xe5, 0xa1, 0xf2,
0xb2, 0xb6, 0x2c, 0x8f, 0xb3, 0xec, 0x0d, 0x8c, 0x3c, 0xae, 0xae, 0xe4, 0x9d, 0x46, 0x7a, 0x09,
0xc7, 0xca, 0xd9, 0xa3, 0x73, 0x32, 0x49, 0x8a, 0xe1, 0xec, 0xec, 0x5f, 0x18, 0x7b, 0xcd, 0xe3,
0x18, 0x7b, 0x0d, 0xa3, 0xef, 0x4a, 0x18, 0xac, 0xf9, 0x10, 0xec, 0x22, 0x75, 0xbb, 0xec, 0xca,
0xc6, 0xac, 0xdc, 0x72, 0x09, 0xb7, 0x47, 0xb6, 0x09, 0x5f, 0x46, 0x51, 0x25, 0x64, 0x1e, 0xd4,
0x7d, 0xd9, 0x4d, 0x1d, 0xa6, 0xe8, 0xcb, 0xa6, 0xe4, 0xf3, 0xe6, 0x07, 0xf5, 0xc5, 0xf6, 0x9a,
0x1f, 0xc2, 0x83, 0xc0, 0xeb, 0x45, 0xdb, 0xc6, 0x15, 0xae, 0x70, 0x37, 0xca, 0x7e, 0xc4, 0x46,
0xb7, 0xdf, 0xaf, 0x9a, 0xe4, 0x17, 0x4d, 0xf2, 0x03, 0xc8, 0x3d, 0xfb, 0x18, 0x4e, 0x23, 0x76,
0xa0, 0x5f, 0xc2, 0xf0, 0xb3, 0xd0, 0xa6, 0x3d, 0x48, 0x83, 0x8e, 0x20, 0x0d, 0xfe, 0x33, 0x48,
0x57, 0x9e, 0x2c, 0x0a, 0xab, 0xc5, 0x86, 0xb4, 0xc7, 0xa6, 0xb6, 0xda, 0x5e, 0x44, 0x01, 0x23,
0x8f, 0x12, 0x62, 0x43, 0x21, 0x5d, 0xe2, 0xd6, 0x5a, 0x91, 0x14, 0x03, 0xee, 0xce, 0x9f, 0xd2,
0x13, 0x32, 0xee, 0xcd, 0xfe, 0xf4, 0xa0, 0xff, 0xd5, 0x02, 0xd1, 0xb7, 0x90, 0xda, 0xa8, 0xd1,
0xd6, 0x60, 0x86, 0x7d, 0x1e, 0x9f, 0xb7, 0x5f, 0x06, 0xa7, 0x8e, 0xe8, 0x7b, 0xe8, 0xbb, 0xb7,
0xa3, 0xed, 0x6f, 0x1d, 0x61, 0x2e, 0x3a, 0x6e, 0x77, 0x38, 0x1f, 0x21, 0xf3, 0xaf, 0x40, 0x3b,
0xde, 0x2d, 0x22, 0x3d, 0xed, 0xba, 0xde, 0x41, 0xbd, 0x83, 0xd4, 0x7a, 0x41, 0x5b, 0x9d, 0xeb,
0xd4, 0x55, 0xb7, 0x8f, 0x1d, 0x5d, 0x92, 0x9f, 0x99, 0xfb, 0x5f, 0x3d, 0xff, 0x1b, 0x00, 0x00,
0xff, 0xff, 0xdd, 0xdb, 0x9c, 0x15, 0xd2, 0x04, 0x00, 0x00,
func init() {
proto.RegisterFile("store.proto", fileDescriptor_98bbca36ef968dfc)
}
// Reference imports to suppress errors if they are not otherwise used.
var _ context.Context
var _ grpc.ClientConn
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
const _ = grpc.SupportPackageIsVersion4
// StoreClient is the client API for Store service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
type StoreClient interface {
Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error)
Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (*WriteResponse, error)
Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*DeleteResponse, error)
List(ctx context.Context, in *ListRequest, opts ...grpc.CallOption) (Store_ListClient, error)
}
type storeClient struct {
cc *grpc.ClientConn
}
func NewStoreClient(cc *grpc.ClientConn) StoreClient {
return &storeClient{cc}
}
func (c *storeClient) Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
out := new(ReadResponse)
err := c.cc.Invoke(ctx, "/go.micro.store.Store/Read", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *storeClient) Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (*WriteResponse, error) {
out := new(WriteResponse)
err := c.cc.Invoke(ctx, "/go.micro.store.Store/Write", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *storeClient) Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*DeleteResponse, error) {
out := new(DeleteResponse)
err := c.cc.Invoke(ctx, "/go.micro.store.Store/Delete", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *storeClient) List(ctx context.Context, in *ListRequest, opts ...grpc.CallOption) (Store_ListClient, error) {
stream, err := c.cc.NewStream(ctx, &_Store_serviceDesc.Streams[0], "/go.micro.store.Store/List", opts...)
if err != nil {
return nil, err
}
x := &storeListClient{stream}
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
if err := x.ClientStream.CloseSend(); err != nil {
return nil, err
}
return x, nil
}
type Store_ListClient interface {
Recv() (*ListResponse, error)
grpc.ClientStream
}
type storeListClient struct {
grpc.ClientStream
}
func (x *storeListClient) Recv() (*ListResponse, error) {
m := new(ListResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
// StoreServer is the server API for Store service.
type StoreServer interface {
Read(context.Context, *ReadRequest) (*ReadResponse, error)
Write(context.Context, *WriteRequest) (*WriteResponse, error)
Delete(context.Context, *DeleteRequest) (*DeleteResponse, error)
List(*ListRequest, Store_ListServer) error
}
// UnimplementedStoreServer can be embedded to have forward compatible implementations.
type UnimplementedStoreServer struct {
}
func (*UnimplementedStoreServer) Read(ctx context.Context, req *ReadRequest) (*ReadResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Read not implemented")
}
func (*UnimplementedStoreServer) Write(ctx context.Context, req *WriteRequest) (*WriteResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Write not implemented")
}
func (*UnimplementedStoreServer) Delete(ctx context.Context, req *DeleteRequest) (*DeleteResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Delete not implemented")
}
func (*UnimplementedStoreServer) List(req *ListRequest, srv Store_ListServer) error {
return status.Errorf(codes.Unimplemented, "method List not implemented")
}
func RegisterStoreServer(s *grpc.Server, srv StoreServer) {
s.RegisterService(&_Store_serviceDesc, srv)
}
func _Store_Read_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ReadRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(StoreServer).Read(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/go.micro.store.Store/Read",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(StoreServer).Read(ctx, req.(*ReadRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Store_Write_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(WriteRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(StoreServer).Write(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/go.micro.store.Store/Write",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(StoreServer).Write(ctx, req.(*WriteRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Store_Delete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(DeleteRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(StoreServer).Delete(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/go.micro.store.Store/Delete",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(StoreServer).Delete(ctx, req.(*DeleteRequest))
}
return interceptor(ctx, in, info, handler)
}
func _Store_List_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(ListRequest)
if err := stream.RecvMsg(m); err != nil {
return err
}
return srv.(StoreServer).List(m, &storeListServer{stream})
}
type Store_ListServer interface {
Send(*ListResponse) error
grpc.ServerStream
}
type storeListServer struct {
grpc.ServerStream
}
func (x *storeListServer) Send(m *ListResponse) error {
return x.ServerStream.SendMsg(m)
}
var _Store_serviceDesc = grpc.ServiceDesc{
ServiceName: "go.micro.store.Store",
HandlerType: (*StoreServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Read",
Handler: _Store_Read_Handler,
},
{
MethodName: "Write",
Handler: _Store_Write_Handler,
},
{
MethodName: "Delete",
Handler: _Store_Delete_Handler,
},
},
Streams: []grpc.StreamDesc{
{
StreamName: "List",
Handler: _Store_List_Handler,
ServerStreams: true,
},
},
Metadata: "store/service/proto/store.proto",
var fileDescriptor_98bbca36ef968dfc = []byte{
// 552 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x54, 0x5d, 0x8b, 0xd3, 0x40,
0x14, 0x6d, 0x9a, 0x34, 0xdb, 0xdc, 0x76, 0x6b, 0x1d, 0xa4, 0x94, 0xda, 0x95, 0x38, 0x4f, 0x01,
0x21, 0xac, 0x15, 0x3f, 0x1e, 0x05, 0xab, 0xa8, 0x08, 0xc2, 0x28, 0x0a, 0xbe, 0xa5, 0xdb, 0xa9,
0x84, 0x66, 0x77, 0x62, 0x66, 0xba, 0x6c, 0x7f, 0xa0, 0xff, 0x4b, 0xe6, 0x2b, 0x4d, 0xd3, 0xc4,
0x87, 0x7d, 0x9b, 0x7b, 0xe7, 0xce, 0x39, 0xf7, 0xdc, 0x7b, 0x12, 0x18, 0x70, 0xc1, 0x0a, 0x1a,
0xe7, 0x05, 0x13, 0x0c, 0x8d, 0x7e, 0xb3, 0xf8, 0x3a, 0xbd, 0x2a, 0x58, 0xac, 0xb2, 0xf8, 0x23,
0xf8, 0x84, 0x5e, 0xb1, 0x62, 0x8d, 0xc6, 0xe0, 0x6e, 0xe9, 0x7e, 0xea, 0x84, 0x4e, 0x14, 0x10,
0x79, 0x44, 0x8f, 0xa0, 0x77, 0x9b, 0x64, 0x3b, 0x3a, 0xed, 0x86, 0x4e, 0x34, 0x24, 0x3a, 0x40,
0x13, 0xf0, 0xe9, 0x5d, 0x9e, 0x16, 0xfb, 0xa9, 0x1b, 0x3a, 0x91, 0x4b, 0x4c, 0x84, 0xb7, 0x30,
0x20, 0x34, 0x59, 0x7f, 0xcd, 0x45, 0xca, 0x6e, 0xb8, 0x2c, 0xcb, 0x0b, 0xba, 0x49, 0xef, 0x14,
0x62, 0x9f, 0x98, 0x48, 0xe6, 0xf9, 0x6e, 0x23, 0xf3, 0x5d, 0x9d, 0xd7, 0x91, 0x24, 0xcb, 0xd2,
0xeb, 0x54, 0x28, 0x54, 0x8f, 0xe8, 0x40, 0x56, 0xb3, 0xcd, 0x86, 0x53, 0x31, 0xf5, 0x54, 0xda,
0x44, 0xf8, 0x87, 0x26, 0x23, 0xf4, 0xcf, 0x8e, 0x72, 0xd1, 0xd0, 0xfb, 0x4b, 0x38, 0x63, 0xba,
0x13, 0xc5, 0x33, 0x58, 0x3c, 0x8e, 0x8f, 0x95, 0xc7, 0x95, 0x66, 0x89, 0xad, 0xc5, 0x6f, 0x61,
0xa8, 0x71, 0x79, 0xce, 0x6e, 0x38, 0x45, 0x97, 0x70, 0x56, 0xa8, 0xf1, 0xf0, 0xa9, 0x13, 0xba,
0xd1, 0x60, 0x31, 0x39, 0x85, 0x91, 0xd7, 0xc4, 0x96, 0xe1, 0x37, 0x30, 0xfc, 0x59, 0xa4, 0x82,
0x56, 0xe6, 0x60, 0xc6, 0xe5, 0x54, 0xc7, 0x25, 0x5b, 0x16, 0x22, 0x53, 0xcd, 0xb9, 0x44, 0x1e,
0xf1, 0xad, 0x79, 0x69, 0x45, 0xc5, 0xe0, 0x6b, 0x50, 0xf5, 0xb2, 0x9d, 0xda, 0x54, 0xa1, 0x57,
0x75, 0xc9, 0xf3, 0xfa, 0x83, 0x6a, 0x63, 0x07, 0xcd, 0x0f, 0xe0, 0xdc, 0xf0, 0x6a, 0xd1, 0x32,
0xb1, 0xa4, 0x19, 0x2d, 0x4b, 0xf1, 0x2f, 0x9b, 0x68, 0x9f, 0xf7, 0xeb, 0x3a, 0xf9, 0x45, 0x9d,
0xfc, 0x08, 0xf2, 0xc0, 0x3e, 0x86, 0x91, 0xc5, 0x36, 0xf4, 0x5b, 0x18, 0x7c, 0x49, 0xb9, 0x68,
0x36, 0x52, 0xd0, 0x62, 0xa4, 0xe0, 0x9e, 0x46, 0x5a, 0x6a, 0x32, 0x2b, 0xac, 0x62, 0x1b, 0xa7,
0xd9, 0x36, 0x95, 0xd6, 0x0e, 0x22, 0x22, 0x18, 0x6a, 0x14, 0x63, 0x1b, 0x04, 0xde, 0x96, 0xee,
0xe5, 0x28, 0xdc, 0x28, 0x20, 0xea, 0xfc, 0xd9, 0xeb, 0x3b, 0xe3, 0x2e, 0x46, 0x30, 0x5e, 0x26,
0x22, 0x59, 0x25, 0x9c, 0x72, 0x43, 0x8a, 0x9f, 0xc3, 0xc3, 0x4a, 0xce, 0x40, 0xcc, 0x21, 0x58,
0xdb, 0xa4, 0xf2, 0x5e, 0x40, 0x0e, 0x09, 0xfc, 0x0c, 0xce, 0xbf, 0x27, 0xab, 0xac, 0xc4, 0x40,
0x33, 0xe8, 0xdb, 0x5b, 0x33, 0xa7, 0x32, 0xc6, 0x11, 0x8c, 0x6c, 0xb1, 0x01, 0x9f, 0x80, 0x2f,
0x54, 0xc6, 0x20, 0x9b, 0x68, 0xf1, 0xd7, 0x85, 0xde, 0x37, 0x29, 0x13, 0xbd, 0x03, 0x4f, 0x7e,
0x08, 0xa8, 0xf1, 0xb3, 0x31, 0xa4, 0xb3, 0x79, 0xf3, 0xa5, 0xd9, 0x63, 0x07, 0x7d, 0x80, 0x9e,
0x72, 0x16, 0x6a, 0x76, 0xa2, 0x85, 0xb9, 0x68, 0xb9, 0x2d, 0x71, 0x3e, 0x81, 0xaf, 0x3d, 0x82,
0x5a, 0x5c, 0x65, 0x91, 0x9e, 0xb4, 0x5d, 0x97, 0x50, 0xef, 0xc1, 0x93, 0x9b, 0x42, 0x8d, 0x7b,
0x6d, 0xd5, 0x55, 0x5d, 0x2e, 0xee, 0x5c, 0x3a, 0x88, 0x40, 0x50, 0xae, 0x0c, 0x85, 0x27, 0xac,
0xb5, 0x0d, 0xcf, 0x9e, 0xfe, 0xa7, 0xa2, 0xaa, 0x52, 0xaf, 0xe9, 0x54, 0xe5, 0xd1, 0xae, 0x4f,
0x55, 0x1e, 0x6f, 0x17, 0x77, 0x56, 0xbe, 0xfa, 0xd9, 0xbf, 0xf8, 0x17, 0x00, 0x00, 0xff, 0xff,
0x18, 0xa5, 0x9b, 0x82, 0xfb, 0x05, 0x00, 0x00,
}

36
store/service/proto/store.pb.micro.go
View File

@ -1,5 +1,5 @@
// Code generated by protoc-gen-micro. DO NOT EDIT.
// source: store/service/proto/store.proto
// source: store.proto
package go_micro_store
@ -38,6 +38,8 @@ type StoreService interface {
Write(ctx context.Context, in *WriteRequest, opts ...client.CallOption) (*WriteResponse, error)
Delete(ctx context.Context, in *DeleteRequest, opts ...client.CallOption) (*DeleteResponse, error)
List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (Store_ListService, error)
Databases(ctx context.Context, in *DatabasesRequest, opts ...client.CallOption) (*DatabasesResponse, error)
Tables(ctx context.Context, in *TablesRequest, opts ...client.CallOption) (*TablesResponse, error)
}
type storeService struct {
@ -131,6 +133,26 @@ func (x *storeServiceList) Recv() (*ListResponse, error) {
return m, nil
}
func (c *storeService) Databases(ctx context.Context, in *DatabasesRequest, opts ...client.CallOption) (*DatabasesResponse, error) {
req := c.c.NewRequest(c.name, "Store.Databases", in)
out := new(DatabasesResponse)
err := c.c.Call(ctx, req, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *storeService) Tables(ctx context.Context, in *TablesRequest, opts ...client.CallOption) (*TablesResponse, error) {
req := c.c.NewRequest(c.name, "Store.Tables", in)
out := new(TablesResponse)
err := c.c.Call(ctx, req, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// Server API for Store service
type StoreHandler interface {
@ -138,6 +160,8 @@ type StoreHandler interface {
Write(context.Context, *WriteRequest, *WriteResponse) error
Delete(context.Context, *DeleteRequest, *DeleteResponse) error
List(context.Context, *ListRequest, Store_ListStream) error
Databases(context.Context, *DatabasesRequest, *DatabasesResponse) error
Tables(context.Context, *TablesRequest, *TablesResponse) error
}
func RegisterStoreHandler(s server.Server, hdlr StoreHandler, opts ...server.HandlerOption) error {
@ -146,6 +170,8 @@ func RegisterStoreHandler(s server.Server, hdlr StoreHandler, opts ...server.Han
Write(ctx context.Context, in *WriteRequest, out *WriteResponse) error
Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error
List(ctx context.Context, stream server.Stream) error
Databases(ctx context.Context, in *DatabasesRequest, out *DatabasesResponse) error
Tables(ctx context.Context, in *TablesRequest, out *TablesResponse) error
}
type Store struct {
store
@ -209,3 +235,11 @@ func (x *storeListStream) RecvMsg(m interface{}) error {
func (x *storeListStream) Send(m *ListResponse) error {
return x.stream.Send(m)
}
func (h *storeHandler) Databases(ctx context.Context, in *DatabasesRequest, out *DatabasesResponse) error {
return h.StoreHandler.Databases(ctx, in, out)
}
func (h *storeHandler) Tables(ctx context.Context, in *TablesRequest, out *TablesResponse) error {
return h.StoreHandler.Tables(ctx, in, out)
}

16
store/service/proto/store.proto
View File

@ -7,6 +7,8 @@ service Store {
rpc Write(WriteRequest) returns (WriteResponse) {};
rpc Delete(DeleteRequest) returns (DeleteResponse) {};
rpc List(ListRequest) returns (stream ListResponse) {};
rpc Databases(DatabasesRequest) returns (DatabasesResponse) {};
rpc Tables(TablesRequest) returns (TablesResponse) {};
}
message Record {
@ -72,3 +74,17 @@ message ListResponse {
reserved 1; //repeated Record records = 1;
repeated string keys = 2;
}
message DatabasesRequest {}
message DatabasesResponse {
repeated string databases = 1;
}
message TablesRequest {
string database = 1;
}
message TablesResponse {
repeated string tables = 1;
}

6
store/store.go
View File

@ -36,7 +36,7 @@ type Store interface {
// Record is an item stored or retrieved from a Store
type Record struct {
Key string
Value []byte
Expiry time.Duration
Key string `json:"key"`
Value []byte `json:"value"`
Expiry time.Duration `json:"expiry,omitempty"`
}

116
sync/README.md
View File

@ -1,116 +0,0 @@
# Sync
Sync is a synchronization library for distributed systems.
## Overview
Distributed systems by their very nature are decoupled and independent. In most cases they must honour 2 out of 3 letters of the CAP theorem
e.g Availability and Partitional tolerance but sacrificing consistency. In the case of microservices we often offload this concern to
an external database or eventing system. Go Sync provides a framework for synchronization which can be used in the application by the developer.
## Getting Started
- [Leader](#leader) - leadership election for group coordination
- [Lock](#lock) - distributed locking for exclusive resource access
- [Task](#task) - distributed job execution
- [Time](#time) - provides synchronized time
## Lock
The Lock interface provides distributed locking. Multiple instances attempting to lock the same id will block until available.
```go
import "github.com/micro/go-micro/sync/lock/consul"
lock := consul.NewLock()
// acquire lock
err := lock.Acquire("id")
// handle err
// release lock
err = lock.Release("id")
// handle err
```
## Leader
Leader provides leadership election. Useful where one node needs to coordinate some action.
```go
import (
"github.com/micro/go-micro/sync/leader"
"github.com/micro/go-micro/sync/leader/consul"
)
l := consul.NewLeader(
leader.Group("name"),
)
// elect leader
e, err := l.Elect("id")
// handle err
// operate while leader
revoked := e.Revoked()
for {
select {
case <-revoked:
// re-elect
e.Elect("id")
default:
// leader operation
}
}
// resign leadership
e.Resign()
```
## Task
Task provides distributed job execution. It's a simple way to distribute work across a coordinated pool of workers.
```go
import (
"github.com/micro/go-micro/sync/task"
"github.com/micro/go-micro/sync/task/local"
)
t := local.NewTask(
task.WithPool(10),
)
err := t.Run(task.Command{
Name: "atask",
Func: func() error {
// exec some work
return nil
},
})
if err != nil {
// do something
}
```
## Time
Time provides synchronized time. Local machines may have clock skew and time cannot be guaranteed to be the same everywhere.
Synchronized Time allows you to decide how time is defined for your applications.
```go
import (
"github.com/micro/go-micro/sync/time/ntp"
)
t := ntp.NewTime()
time, err := t.Now()
```
## TODO
- Event package - strongly consistent event stream e.g kafka

99
sync/cron.go
View File

@ -1,99 +0,0 @@
package sync
import (
"fmt"
"math"
"time"
"github.com/micro/go-micro/v2/logger"
"github.com/micro/go-micro/v2/sync/leader/etcd"
"github.com/micro/go-micro/v2/sync/task"
"github.com/micro/go-micro/v2/sync/task/local"
)
type syncCron struct {
opts Options
}
func backoff(attempts int) time.Duration {
if attempts == 0 {
return time.Duration(0)
}
return time.Duration(math.Pow(10, float64(attempts))) * time.Millisecond
}
func (c *syncCron) Schedule(s task.Schedule, t task.Command) error {
id := fmt.Sprintf("%s-%s", s.String(), t.String())
go func() {
// run the scheduler
tc := s.Run()
var i int
for {
// leader election
e, err := c.opts.Leader.Elect(id)
if err != nil {
if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
logger.Errorf("[cron] leader election error: %v", err)
}
time.Sleep(backoff(i))
i++
continue
}
i = 0
r := e.Revoked()
// execute the task
Tick:
for {
select {
// schedule tick
case _, ok := <-tc:
// ticked once
if !ok {
break Tick
}
if logger.V(logger.InfoLevel, logger.DefaultLogger) {
logger.Infof("[cron] executing command %s", t.Name)
}
if err := c.opts.Task.Run(t); err != nil {
if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
logger.Errorf("[cron] error executing command %s: %v", t.Name, err)
}
}
// leader revoked
case <-r:
break Tick
}
}
// resign
e.Resign()
}
}()
return nil
}
func NewCron(opts ...Option) Cron {
var options Options
for _, o := range opts {
o(&options)
}
if options.Leader == nil {
options.Leader = etcd.NewLeader()
}
if options.Task == nil {
options.Task = local.NewTask()
}
return &syncCron{
opts: options,
}
}

179
sync/etcd/etcd.go Normal file
View File

@ -0,0 +1,179 @@
// Package etcd is an etcd implementation of lock
package etcd
import (
"context"
"errors"
"log"
"path"
"strings"
gosync "sync"
client "github.com/coreos/etcd/clientv3"
cc "github.com/coreos/etcd/clientv3/concurrency"
"github.com/micro/go-micro/v2/sync"
)
type etcdSync struct {
options sync.Options
path string
client *client.Client
mtx gosync.Mutex
locks map[string]*etcdLock
}
type etcdLock struct {
s *cc.Session
m *cc.Mutex
}
type etcdLeader struct {
opts sync.LeaderOptions
s *cc.Session
e *cc.Election
id string
}
func (e *etcdSync) Leader(id string, opts ...sync.LeaderOption) (sync.Leader, error) {
var options sync.LeaderOptions
for _, o := range opts {
o(&options)
}
// make path
path := path.Join(e.path, strings.Replace(e.options.Prefix+id, "/", "-", -1))
s, err := cc.NewSession(e.client)
if err != nil {
return nil, err
}
l := cc.NewElection(s, path)
if err := l.Campaign(context.TODO(), id); err != nil {
return nil, err
}
return &etcdLeader{
opts: options,
e: l,
id: id,
}, nil
}
func (e *etcdLeader) Status() chan bool {
ch := make(chan bool, 1)
ech := e.e.Observe(context.Background())
go func() {
for r := range ech {
if string(r.Kvs[0].Value) != e.id {
ch <- true
close(ch)
return
}
}
}()
return ch
}
func (e *etcdLeader) Resign() error {
return e.e.Resign(context.Background())
}
func (e *etcdSync) Init(opts ...sync.Option) error {
for _, o := range opts {
o(&e.options)
}
return nil
}
func (e *etcdSync) Options() sync.Options {
return e.options
}
func (e *etcdSync) Lock(id string, opts ...sync.LockOption) error {
var options sync.LockOptions
for _, o := range opts {
o(&options)
}
// make path
path := path.Join(e.path, strings.Replace(e.options.Prefix+id, "/", "-", -1))
var sopts []cc.SessionOption
if options.TTL > 0 {
sopts = append(sopts, cc.WithTTL(int(options.TTL.Seconds())))
}
s, err := cc.NewSession(e.client, sopts...)
if err != nil {
return err
}
m := cc.NewMutex(s, path)
if err := m.Lock(context.TODO()); err != nil {
return err
}
e.mtx.Lock()
e.locks[id] = &etcdLock{
s: s,
m: m,
}
e.mtx.Unlock()
return nil
}
func (e *etcdSync) Unlock(id string) error {
e.mtx.Lock()
defer e.mtx.Unlock()
v, ok := e.locks[id]
if !ok {
return errors.New("lock not found")
}
err := v.m.Unlock(context.Background())
delete(e.locks, id)
return err
}
func (e *etcdSync) String() string {
return "etcd"
}
func NewSync(opts ...sync.Option) sync.Sync {
var options sync.Options
for _, o := range opts {
o(&options)
}
var endpoints []string
for _, addr := range options.Nodes {
if len(addr) > 0 {
endpoints = append(endpoints, addr)
}
}
if len(endpoints) == 0 {
endpoints = []string{"http://127.0.0.1:2379"}
}
// TODO: parse addresses
c, err := client.New(client.Config{
Endpoints: endpoints,
})
if err != nil {
log.Fatal(err)
}
return &etcdSync{
path: "/micro/sync",
client: c,
options: options,
locks: make(map[string]*etcdLock),
}
}

27
sync/event/event.go
View File

@ -1,27 +0,0 @@
// Package event provides a distributed log interface
package event
// Event provides a distributed log interface
type Event interface {
// Log retrieves the log with an id/name
Log(id string) (Log, error)
}
// Log is an individual event log
type Log interface {
// Close the log handle
Close() error
// Log ID
Id() string
// Read will read the next record
Read() (*Record, error)
// Go to an offset
Seek(offset int64) error
// Write an event to the log
Write(*Record) error
}
type Record struct {
Metadata map[string]interface{}
Data []byte
}

136
sync/leader/etcd/etcd.go
View File

@ -1,136 +0,0 @@
package etcd
import (
"context"
"log"
"path"
"strings"
client "github.com/coreos/etcd/clientv3"
cc "github.com/coreos/etcd/clientv3/concurrency"
"github.com/micro/go-micro/v2/sync/leader"
)
type etcdLeader struct {
opts leader.Options
path string
client *client.Client
}
type etcdElected struct {
s *cc.Session
e *cc.Election
id string
}
func (e *etcdLeader) Elect(id string, opts ...leader.ElectOption) (leader.Elected, error) {
var options leader.ElectOptions
for _, o := range opts {
o(&options)
}
// make path
path := path.Join(e.path, strings.Replace(id, "/", "-", -1))
s, err := cc.NewSession(e.client)
if err != nil {
return nil, err
}
l := cc.NewElection(s, path)
if err := l.Campaign(context.TODO(), id); err != nil {
return nil, err
}
return &etcdElected{
e: l,
id: id,
}, nil
}
func (e *etcdLeader) Follow() chan string {
ch := make(chan string)
s, err := cc.NewSession(e.client)
if err != nil {
return ch
}
l := cc.NewElection(s, e.path)
ech := l.Observe(context.Background())
go func() {
for r := range ech {
ch <- string(r.Kvs[0].Value)
}
}()
return ch
}
func (e *etcdLeader) String() string {
return "etcd"
}
func (e *etcdElected) Reelect() error {
return e.e.Campaign(context.TODO(), e.id)
}
func (e *etcdElected) Revoked() chan bool {
ch := make(chan bool, 1)
ech := e.e.Observe(context.Background())
go func() {
for r := range ech {
if string(r.Kvs[0].Value) != e.id {
ch <- true
close(ch)
return
}
}
}()
return ch
}
func (e *etcdElected) Resign() error {
return e.e.Resign(context.Background())
}
func (e *etcdElected) Id() string {
return e.id
}
func NewLeader(opts ...leader.Option) leader.Leader {
var options leader.Options
for _, o := range opts {
o(&options)
}
var endpoints []string
for _, addr := range options.Nodes {
if len(addr) > 0 {
endpoints = append(endpoints, addr)
}
}
if len(endpoints) == 0 {
endpoints = []string{"http://127.0.0.1:2379"}
}
// TODO: parse addresses
c, err := client.New(client.Config{
Endpoints: endpoints,
})
if err != nil {
log.Fatal(err)
}
return &etcdLeader{
path: "/micro/leader",
client: c,
opts: options,
}
}

25
sync/leader/leader.go
View File

@ -1,25 +0,0 @@
// Package leader provides leader election
package leader
// Leader provides leadership election
type Leader interface {
// elect leader
Elect(id string, opts ...ElectOption) (Elected, error)
// follow the leader
Follow() chan string
}
type Elected interface {
// id of leader
Id() string
// seek re-election
Reelect() error
// resign leadership
Resign() error
// observe leadership revocation
Revoked() chan bool
}
type Option func(o *Options)
type ElectOption func(o *ElectOptions)

22
sync/leader/options.go
View File

@ -1,22 +0,0 @@
package leader
type Options struct {
Nodes []string
Group string
}
type ElectOptions struct{}
// Nodes sets the addresses of the underlying systems
func Nodes(a ...string) Option {
return func(o *Options) {
o.Nodes = a
}
}
// Group sets the group name for coordinating leadership
func Group(g string) Option {
return func(o *Options) {
o.Group = g
}
}

113
sync/lock/etcd/etcd.go
View File

@ -1,113 +0,0 @@
// Package etcd is an etcd implementation of lock
package etcd
import (
"context"
"errors"
"log"
"path"
"strings"
"sync"
client "github.com/coreos/etcd/clientv3"
cc "github.com/coreos/etcd/clientv3/concurrency"
"github.com/micro/go-micro/v2/sync/lock"
)
type etcdLock struct {
opts lock.Options
path string
client *client.Client
sync.Mutex
locks map[string]*elock
}
type elock struct {
s *cc.Session
m *cc.Mutex
}
func (e *etcdLock) Acquire(id string, opts ...lock.AcquireOption) error {
var options lock.AcquireOptions
for _, o := range opts {
o(&options)
}
// make path
path := path.Join(e.path, strings.Replace(e.opts.Prefix+id, "/", "-", -1))
var sopts []cc.SessionOption
if options.TTL > 0 {
sopts = append(sopts, cc.WithTTL(int(options.TTL.Seconds())))
}
s, err := cc.NewSession(e.client, sopts...)
if err != nil {
return err
}
m := cc.NewMutex(s, path)
if err := m.Lock(context.TODO()); err != nil {
return err
}
e.Lock()
e.locks[id] = &elock{
s: s,
m: m,
}
e.Unlock()
return nil
}
func (e *etcdLock) Release(id string) error {
e.Lock()
defer e.Unlock()
v, ok := e.locks[id]
if !ok {
return errors.New("lock not found")
}
err := v.m.Unlock(context.Background())
delete(e.locks, id)
return err
}
func (e *etcdLock) String() string {
return "etcd"
}
func NewLock(opts ...lock.Option) lock.Lock {
var options lock.Options
for _, o := range opts {
o(&options)
}
var endpoints []string
for _, addr := range options.Nodes {
if len(addr) > 0 {
endpoints = append(endpoints, addr)
}
}
if len(endpoints) == 0 {
endpoints = []string{"http://127.0.0.1:2379"}
}
// TODO: parse addresses
c, err := client.New(client.Config{
Endpoints: endpoints,
})
if err != nil {
log.Fatal(err)
}
return &etcdLock{
path: "/micro/lock",
client: c,
opts: options,
locks: make(map[string]*elock),
}
}

135
sync/lock/http/http.go
View File

@ -1,135 +0,0 @@
// Package http adds a http lock implementation
package http
import (
"errors"
"fmt"
"hash/crc32"
"io/ioutil"
"net/http"
"net/url"
"path/filepath"
"strings"
"github.com/micro/go-micro/v2/sync/lock"
)
var (
DefaultPath = "/sync/lock"
DefaultAddress = "localhost:8080"
)
type httpLock struct {
opts lock.Options
}
func (h *httpLock) url(do, id string) (string, error) {
sum := crc32.ChecksumIEEE([]byte(id))
node := h.opts.Nodes[sum%uint32(len(h.opts.Nodes))]
// parse the host:port or whatever
uri, err := url.Parse(node)
if err != nil {
return "", err
}
if len(uri.Scheme) == 0 {
uri.Scheme = "http"
}
// set path
// build path
path := filepath.Join(DefaultPath, do, h.opts.Prefix, id)
uri.Path = path
// return url
return uri.String(), nil
}
func (h *httpLock) Acquire(id string, opts ...lock.AcquireOption) error {
var options lock.AcquireOptions
for _, o := range opts {
o(&options)
}
uri, err := h.url("acquire", id)
if err != nil {
return err
}
ttl := fmt.Sprintf("%d", int64(options.TTL.Seconds()))
wait := fmt.Sprintf("%d", int64(options.Wait.Seconds()))
rsp, err := http.PostForm(uri, url.Values{
"id": {id},
"ttl": {ttl},
"wait": {wait},
})
if err != nil {
return err
}
defer rsp.Body.Close()
b, err := ioutil.ReadAll(rsp.Body)
if err != nil {
return err
}
// success
if rsp.StatusCode == 200 {
return nil
}
// return error
return errors.New(string(b))
}
func (h *httpLock) Release(id string) error {
uri, err := h.url("release", id)
if err != nil {
return err
}
vals := url.Values{
"id": {id},
}
req, err := http.NewRequest("DELETE", uri, strings.NewReader(vals.Encode()))
if err != nil {
return err
}
rsp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer rsp.Body.Close()
b, err := ioutil.ReadAll(rsp.Body)
if err != nil {
return err
}
// success
if rsp.StatusCode == 200 {
return nil
}
// return error
return errors.New(string(b))
}
func NewLock(opts ...lock.Option) lock.Lock {
var options lock.Options
for _, o := range opts {
o(&options)
}
if len(options.Nodes) == 0 {
options.Nodes = []string{DefaultAddress}
}
return &httpLock{
opts: options,
}
}

45
sync/lock/http/server/server.go
View File

@ -1,45 +0,0 @@
// Package server implements the sync http server
package server
import (
"net/http"
"github.com/micro/go-micro/v2/sync/lock"
lkhttp "github.com/micro/go-micro/v2/sync/lock/http"
)
func Handler(lk lock.Lock) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc(lkhttp.DefaultPath, func(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
id := r.Form.Get("id")
if len(id) == 0 {
return
}
switch r.Method {
case "POST":
err := lk.Acquire(id)
if err != nil {
http.Error(w, err.Error(), 500)
}
case "DELETE":
err := lk.Release(id)
if err != nil {
http.Error(w, err.Error(), 500)
}
}
})
return mux
}
func Server(lk lock.Lock) *http.Server {
server := &http.Server{
Addr: lkhttp.DefaultAddress,
Handler: Handler(lk),
}
return server
}

32
sync/lock/lock.go
View File

@ -1,32 +0,0 @@
// Package lock provides distributed locking
package lock
import (
"errors"
"time"
)
var (
ErrLockTimeout = errors.New("lock timeout")
)
// Lock is a distributed locking interface
type Lock interface {
// Acquire a lock with given id
Acquire(id string, opts ...AcquireOption) error
// Release the lock with given id
Release(id string) error
}
type Options struct {
Nodes []string
Prefix string
}
type AcquireOptions struct {
TTL time.Duration
Wait time.Duration
}
type Option func(o *Options)
type AcquireOption func(o *AcquireOptions)

142
sync/lock/memory/memory.go
View File

@ -1,142 +0,0 @@
// Package memory provides a sync.Mutex implementation of the lock for local use
package memory
import (
"sync"
"time"
lock "github.com/micro/go-micro/v2/sync/lock"
)
type memoryLock struct {
sync.RWMutex
locks map[string]*mlock
}
type mlock struct {
id string
time time.Time
ttl time.Duration
release chan bool
}
func (m *memoryLock) Acquire(id string, opts ...lock.AcquireOption) error {
// lock our access
m.Lock()
var options lock.AcquireOptions
for _, o := range opts {
o(&options)
}
lk, ok := m.locks[id]
if !ok {
m.locks[id] = &mlock{
id: id,
time: time.Now(),
ttl: options.TTL,
release: make(chan bool),
}
// unlock
m.Unlock()
return nil
}
m.Unlock()
// set wait time
var wait <-chan time.Time
var ttl <-chan time.Time
// decide if we should wait
if options.Wait > time.Duration(0) {
wait = time.After(options.Wait)
}
// check the ttl of the lock
if lk.ttl > time.Duration(0) {
// time lived for the lock
live := time.Since(lk.time)
// set a timer for the leftover ttl
if live > lk.ttl {
// release the lock if it expired
_ = m.Release(id)
} else {
ttl = time.After(live)
}
}
lockLoop:
for {
// wait for the lock to be released
select {
case <-lk.release:
m.Lock()
// someone locked before us
lk, ok = m.locks[id]
if ok {
m.Unlock()
continue
}
// got chance to lock
m.locks[id] = &mlock{
id: id,
time: time.Now(),
ttl: options.TTL,
release: make(chan bool),
}
m.Unlock()
break lockLoop
case <-ttl:
// ttl exceeded
_ = m.Release(id)
// TODO: check the ttl again above
ttl = nil
// try acquire
continue
case <-wait:
return lock.ErrLockTimeout
}
}
return nil
}
func (m *memoryLock) Release(id string) error {
m.Lock()
defer m.Unlock()
lk, ok := m.locks[id]
// no lock exists
if !ok {
return nil
}
// delete the lock
delete(m.locks, id)
select {
case <-lk.release:
return nil
default:
close(lk.release)
}
return nil
}
func NewLock(opts ...lock.Option) lock.Lock {
var options lock.Options
for _, o := range opts {
o(&options)
}
return &memoryLock{
locks: make(map[string]*mlock),
}
}

33
sync/lock/options.go
View File

@ -1,33 +0,0 @@
package lock
import (
"time"
)
// Nodes sets the addresses the underlying lock implementation
func Nodes(a ...string) Option {
return func(o *Options) {
o.Nodes = a
}
}
// Prefix sets a prefix to any lock ids used
func Prefix(p string) Option {
return func(o *Options) {
o.Prefix = p
}
}
// TTL sets the lock ttl
func TTL(t time.Duration) AcquireOption {
return func(o *AcquireOptions) {
o.TTL = t
}
}
// Wait sets the wait time
func Wait(t time.Duration) AcquireOption {
return func(o *AcquireOptions) {
o.Wait = t
}
}

166
sync/map.go
View File

@ -1,166 +0,0 @@
package sync
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"sort"
"github.com/micro/go-micro/v2/store"
ckv "github.com/micro/go-micro/v2/store/etcd"
lock "github.com/micro/go-micro/v2/sync/lock/etcd"
)
type syncMap struct {
opts Options
}
func ekey(k interface{}) string {
b, _ := json.Marshal(k)
return base64.StdEncoding.EncodeToString(b)
}
func (m *syncMap) Read(key, val interface{}) error {
if key == nil {
return fmt.Errorf("key is nil")
}
kstr := ekey(key)
// lock
if err := m.opts.Lock.Acquire(kstr); err != nil {
return err
}
defer m.opts.Lock.Release(kstr)
// get key
kval, err := m.opts.Store.Read(kstr)
if err != nil {
return err
}
if len(kval) == 0 {
return store.ErrNotFound
}
// decode value
return json.Unmarshal(kval[0].Value, val)
}
func (m *syncMap) Write(key, val interface{}) error {
if key == nil {
return fmt.Errorf("key is nil")
}
kstr := ekey(key)
// lock
if err := m.opts.Lock.Acquire(kstr); err != nil {
return err
}
defer m.opts.Lock.Release(kstr)
// encode value
b, err := json.Marshal(val)
if err != nil {
return err
}
// set key
return m.opts.Store.Write(&store.Record{
Key: kstr,
Value: b,
})
}
func (m *syncMap) Delete(key interface{}) error {
if key == nil {
return fmt.Errorf("key is nil")
}
kstr := ekey(key)
// lock
if err := m.opts.Lock.Acquire(kstr); err != nil {
return err
}
defer m.opts.Lock.Release(kstr)
return m.opts.Store.Delete(kstr)
}
func (m *syncMap) Iterate(fn func(key, val interface{}) error) error {
keyvals, err := m.opts.Store.Read("", store.ReadPrefix())
if err != nil {
return err
}
sort.Slice(keyvals, func(i, j int) bool {
return keyvals[i].Key < keyvals[j].Key
})
for _, keyval := range keyvals {
// lock
if err := m.opts.Lock.Acquire(keyval.Key); err != nil {
return err
}
// unlock
defer m.opts.Lock.Release(keyval.Key)
// unmarshal value
var val interface{}
if len(keyval.Value) > 0 && keyval.Value[0] == '{' {
if err := json.Unmarshal(keyval.Value, &val); err != nil {
return err
}
} else {
val = keyval.Value
}
// exec func
if err := fn(keyval.Key, val); err != nil {
return err
}
// save val
b, err := json.Marshal(val)
if err != nil {
return err
}
// no save
if i := bytes.Compare(keyval.Value, b); i == 0 {
return nil
}
// set key
if err := m.opts.Store.Write(&store.Record{
Key: keyval.Key,
Value: b,
}); err != nil {
return err
}
}
return nil
}
func NewMap(opts ...Option) Map {
var options Options
for _, o := range opts {
o(&options)
}
if options.Lock == nil {
options.Lock = lock.NewLock()
}
if options.Store == nil {
options.Store = ckv.NewStore()
}
return &syncMap{
opts: options,
}
}

202
sync/memory/memory.go Normal file
View File

@ -0,0 +1,202 @@
// Package memory provides a sync.Mutex implementation of the lock for local use
package memory
import (
gosync "sync"
"time"
"github.com/micro/go-micro/v2/sync"
)
type memorySync struct {
options sync.Options
mtx gosync.RWMutex
locks map[string]*memoryLock
}
type memoryLock struct {
id string
time time.Time
ttl time.Duration
release chan bool
}
type memoryLeader struct {
opts sync.LeaderOptions
id string
resign func(id string) error
status chan bool
}
func (m *memoryLeader) Resign() error {
return m.resign(m.id)
}
func (m *memoryLeader) Status() chan bool {
return m.status
}
func (m *memorySync) Leader(id string, opts ...sync.LeaderOption) (sync.Leader, error) {
var once gosync.Once
var options sync.LeaderOptions
for _, o := range opts {
o(&options)
}
// acquire a lock for the id
if err := m.Lock(id); err != nil {
return nil, err
}
// return the leader
return &memoryLeader{
opts: options,
id: id,
resign: func(id string) error {
once.Do(func() {
m.Unlock(id)
})
return nil
},
// TODO: signal when Unlock is called
status: make(chan bool, 1),
}, nil
}
func (m *memorySync) Init(opts ...sync.Option) error {
for _, o := range opts {
o(&m.options)
}
return nil
}
func (m *memorySync) Options() sync.Options {
return m.options
}
func (m *memorySync) Lock(id string, opts ...sync.LockOption) error {
// lock our access
m.mtx.Lock()
var options sync.LockOptions
for _, o := range opts {
o(&options)
}
lk, ok := m.locks[id]
if !ok {
m.locks[id] = &memoryLock{
id: id,
time: time.Now(),
ttl: options.TTL,
release: make(chan bool),
}
// unlock
m.mtx.Unlock()
return nil
}
m.mtx.Unlock()
// set wait time
var wait <-chan time.Time
var ttl <-chan time.Time
// decide if we should wait
if options.Wait > time.Duration(0) {
wait = time.After(options.Wait)
}
// check the ttl of the lock
if lk.ttl > time.Duration(0) {
// time lived for the lock
live := time.Since(lk.time)
// set a timer for the leftover ttl
if live > lk.ttl {
// release the lock if it expired
_ = m.Unlock(id)
} else {
ttl = time.After(live)
}
}
lockLoop:
for {
// wait for the lock to be released
select {
case <-lk.release:
m.mtx.Lock()
// someone locked before us
lk, ok = m.locks[id]
if ok {
m.mtx.Unlock()
continue
}
// got chance to lock
m.locks[id] = &memoryLock{
id: id,
time: time.Now(),
ttl: options.TTL,
release: make(chan bool),
}
m.mtx.Unlock()
break lockLoop
case <-ttl:
// ttl exceeded
_ = m.Unlock(id)
// TODO: check the ttl again above
ttl = nil
// try acquire
continue
case <-wait:
return sync.ErrLockTimeout
}
}
return nil
}
func (m *memorySync) Unlock(id string) error {
m.mtx.Lock()
defer m.mtx.Unlock()
lk, ok := m.locks[id]
// no lock exists
if !ok {
return nil
}
// delete the lock
delete(m.locks, id)
select {
case <-lk.release:
return nil
default:
close(lk.release)
}
return nil
}
func (m *memorySync) String() string {
return "memory"
}
func NewSync(opts ...sync.Option) sync.Sync {
var options sync.Options
for _, o := range opts {
o(&options)
}
return &memorySync{
options: options,
locks: make(map[string]*memoryLock),
}
}

33
sync/options.go
View File

@ -1,36 +1,33 @@
package sync
import (
"github.com/micro/go-micro/v2/store"
"github.com/micro/go-micro/v2/sync/leader"
"github.com/micro/go-micro/v2/sync/lock"
"github.com/micro/go-micro/v2/sync/time"
"time"
)
// WithLeader sets the leader election implementation opton
func WithLeader(l leader.Leader) Option {
// Nodes sets the addresses to use
func Nodes(a ...string) Option {
return func(o *Options) {
o.Leader = l
o.Nodes = a
}
}
// WithLock sets the locking implementation option
func WithLock(l lock.Lock) Option {
// Prefix sets a prefix to any lock ids used
func Prefix(p string) Option {
return func(o *Options) {
o.Lock = l
o.Prefix = p
}
}
// WithStore sets the store implementation option
func WithStore(s store.Store) Option {
return func(o *Options) {
o.Store = s
// LockTTL sets the lock ttl
func LockTTL(t time.Duration) LockOption {
return func(o *LockOptions) {
o.TTL = t
}
}
// WithTime sets the time implementation option
func WithTime(t time.Time) Option {
return func(o *Options) {
o.Time = t
// LockWait sets the wait time
func LockWait(t time.Duration) LockOption {
return func(o *LockOptions) {
o.Wait = t
}
}

114
sync/store/cache.go
View File

@ -1,114 +0,0 @@
// Package store syncs multiple go-micro stores
package store
import (
"fmt"
"sync"
"time"
"github.com/ef-ds/deque"
"github.com/micro/go-micro/v2/store"
"github.com/pkg/errors"
)
// Cache implements a cache in front of go-micro Stores
type Cache interface {
store.Store
// Force a full sync
Sync() error
}
type cache struct {
sOptions store.Options
cOptions Options
pendingWrites []*deque.Deque
pendingWriteTickers []*time.Ticker
sync.RWMutex
}
// NewCache returns a new Cache
func NewCache(opts ...Option) Cache {
c := &cache{}
for _, o := range opts {
o(&c.cOptions)
}
if c.cOptions.SyncInterval == 0 {
c.cOptions.SyncInterval = 1 * time.Minute
}
if c.cOptions.SyncMultiplier == 0 {
c.cOptions.SyncMultiplier = 5
}
return c
}
func (c *cache) Close() error {
return nil
}
// Init initialises the storeOptions
func (c *cache) Init(opts ...store.Option) error {
for _, o := range opts {
o(&c.sOptions)
}
if len(c.cOptions.Stores) == 0 {
return errors.New("the cache has no stores")
}
if c.sOptions.Context == nil {
return errors.New("please provide a context to the cache. Cancelling the context signals that the cache is being disposed and syncs the cache")
}
for _, s := range c.cOptions.Stores {
if err := s.Init(); err != nil {
return errors.Wrapf(err, "Store %s failed to Init()", s.String())
}
}
c.pendingWrites = make([]*deque.Deque, len(c.cOptions.Stores)-1)
c.pendingWriteTickers = make([]*time.Ticker, len(c.cOptions.Stores)-1)
for i := 0; i < len(c.pendingWrites); i++ {
c.pendingWrites[i] = deque.New()
c.pendingWrites[i].Init()
c.pendingWriteTickers[i] = time.NewTicker(c.cOptions.SyncInterval * time.Duration(intpow(c.cOptions.SyncMultiplier, int64(i))))
}
go c.cacheManager()
return nil
}
// Options returns the cache's store options
func (c *cache) Options() store.Options {
return c.sOptions
}
// String returns a printable string describing the cache
func (c *cache) String() string {
backends := make([]string, len(c.cOptions.Stores))
for i, s := range c.cOptions.Stores {
backends[i] = s.String()
}
return fmt.Sprintf("cache %v", backends)
}
func (c *cache) List(opts ...store.ListOption) ([]string, error) {
return c.cOptions.Stores[0].List(opts...)
}
func (c *cache) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) {
return c.cOptions.Stores[0].Read(key, opts...)
}
func (c *cache) Write(r *store.Record, opts ...store.WriteOption) error {
return c.cOptions.Stores[0].Write(r, opts...)
}
// Delete removes a key from the cache
func (c *cache) Delete(key string, opts ...store.DeleteOption) error {
return c.cOptions.Stores[0].Delete(key, opts...)
}
func (c *cache) Sync() error {
return nil
}
type internalRecord struct {
key string
value []byte
expiresAt time.Time
}

68
sync/sync.go
View File

@ -1,41 +1,53 @@
// Package sync is a distributed synchronization framework
// Package sync is an interface for distributed synchronization
package sync
import (
"github.com/micro/go-micro/v2/store"
"github.com/micro/go-micro/v2/sync/leader"
"github.com/micro/go-micro/v2/sync/lock"
"github.com/micro/go-micro/v2/sync/task"
"github.com/micro/go-micro/v2/sync/time"
"errors"
"time"
)
// Map provides synchronized access to key-value storage.
// It uses the store interface and lock interface to
// provide a consistent storage mechanism.
type Map interface {
// Read value with given key
Read(key, val interface{}) error
// Write value with given key
Write(key, val interface{}) error
// Delete value with given key
Delete(key interface{}) error
// Iterate over all key/vals. Value changes are saved
Iterate(func(key, val interface{}) error) error
var (
ErrLockTimeout = errors.New("lock timeout")
)
// Sync is an interface for distributed synchronization
type Sync interface {
// Initialise options
Init(...Option) error
// Return the options
Options() Options
// Elect a leader
Leader(id string, opts ...LeaderOption) (Leader, error)
// Lock acquires a lock
Lock(id string, opts ...LockOption) error
// Unlock releases a lock
Unlock(id string) error
// Sync implementation
String() string
}
// Cron is a distributed scheduler using leader election
// and distributed task runners. It uses the leader and
// task interfaces.
type Cron interface {
Schedule(task.Schedule, task.Command) error
// Leader provides leadership election
type Leader interface {
// resign leadership
Resign() error
// status returns when leadership is lost
Status() chan bool
}
type Options struct {
Leader leader.Leader
Lock lock.Lock
Store store.Store
Task task.Task
Time time.Time
Nodes []string
Prefix string
}
type Option func(o *Options)
type LeaderOptions struct{}
type LeaderOption func(o *LeaderOptions)
type LockOptions struct {
TTL time.Duration
Wait time.Duration
}
type LockOption func(o *LockOptions)

227
sync/task/broker/broker.go
View File

@ -1,227 +0,0 @@
// Package broker provides a distributed task manager built on the micro broker
package broker
import (
"context"
"errors"
"fmt"
"math/rand"
"strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/micro/go-micro/v2/broker"
"github.com/micro/go-micro/v2/sync/task"
)
type brokerKey struct{}
// Task is a broker task
type Task struct {
// a micro broker
Broker broker.Broker
// Options
Options task.Options
mtx sync.RWMutex
status string
}
func returnError(err error, ch chan error) {
select {
case ch <- err:
default:
}
}
func (t *Task) Run(c task.Command) error {
// connect
t.Broker.Connect()
// unique id for this runner
id := uuid.New().String()
// topic of the command
topic := fmt.Sprintf("task.%s", c.Name)
// global error
errCh := make(chan error, t.Options.Pool)
// subscribe for distributed work
workFn := func(p broker.Event) error {
msg := p.Message()
// get command name
command := msg.Header["Command"]
// check the command is what we expect
if command != c.Name {
returnError(errors.New("received unknown command: "+command), errCh)
return nil
}
// new task created
switch msg.Header["Status"] {
case "start":
// artificially delay start of processing
time.Sleep(time.Millisecond * time.Duration(10+rand.Intn(100)))
// execute the function
err := c.Func()
status := "done"
errors := ""
if err != nil {
status = "error"
errors = err.Error()
}
// create response
msg := &broker.Message{
Header: map[string]string{
"Command": c.Name,
"Error": errors,
"Id": id,
"Status": status,
"Timestamp": fmt.Sprintf("%d", time.Now().Unix()),
},
// Body is nil, may be used in future
}
// publish end of task
if err := t.Broker.Publish(topic, msg); err != nil {
returnError(err, errCh)
}
}
return nil
}
// subscribe for the pool size
for i := 0; i < t.Options.Pool; i++ {
err := func() error {
// subscribe to work
subWork, err := t.Broker.Subscribe(topic, workFn, broker.Queue(fmt.Sprintf("work.%d", i)))
if err != nil {
return err
}
// unsubscribe on completion
defer subWork.Unsubscribe()
return nil
}()
if err != nil {
return err
}
}
// subscribe to all status messages
subStatus, err := t.Broker.Subscribe(topic, func(p broker.Event) error {
msg := p.Message()
// get command name
command := msg.Header["Command"]
// check the command is what we expect
if command != c.Name {
return nil
}
// check task status
switch msg.Header["Status"] {
// task is complete
case "done":
errCh <- nil
// someone failed
case "error":
returnError(errors.New(msg.Header["Error"]), errCh)
}
return nil
})
if err != nil {
return err
}
defer subStatus.Unsubscribe()
// a new task
msg := &broker.Message{
Header: map[string]string{
"Command": c.Name,
"Id": id,
"Status": "start",
"Timestamp": fmt.Sprintf("%d", time.Now().Unix()),
},
}
// artificially delay the start of the task
time.Sleep(time.Millisecond * time.Duration(10+rand.Intn(100)))
// publish the task
if err := t.Broker.Publish(topic, msg); err != nil {
return err
}
var gerrors []string
// wait for all responses
for i := 0; i < t.Options.Pool; i++ {
// check errors
err := <-errCh
// append to errors
if err != nil {
gerrors = append(gerrors, err.Error())
}
}
// return the errors
if len(gerrors) > 0 {
return errors.New("errors: " + strings.Join(gerrors, "\n"))
}
return nil
}
func (t *Task) Status() string {
t.mtx.RLock()
defer t.mtx.RUnlock()
return t.status
}
// Broker sets the micro broker
func WithBroker(b broker.Broker) task.Option {
return func(o *task.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, brokerKey{}, b)
}
}
// NewTask returns a new broker task
func NewTask(opts ...task.Option) task.Task {
options := task.Options{
Context: context.Background(),
}
for _, o := range opts {
o(&options)
}
if options.Pool == 0 {
options.Pool = 1
}
b, ok := options.Context.Value(brokerKey{}).(broker.Broker)
if !ok {
b = broker.DefaultBroker
}
return &Task{
Broker: b,
Options: options,
}
}

59
sync/task/local/local.go
View File

@ -1,59 +0,0 @@
// Package local provides a local task runner
package local
import (
"fmt"
"sync"
"github.com/micro/go-micro/v2/sync/task"
)
type localTask struct {
opts task.Options
mtx sync.RWMutex
status string
}
func (l *localTask) Run(t task.Command) error {
ch := make(chan error, l.opts.Pool)
for i := 0; i < l.opts.Pool; i++ {
go func() {
ch <- t.Execute()
}()
}
var err error
for i := 0; i < l.opts.Pool; i++ {
er := <-ch
if err != nil {
err = er
l.mtx.Lock()
l.status = fmt.Sprintf("command [%s] status: %s", t.Name, err.Error())
l.mtx.Unlock()
}
}
close(ch)
return err
}
func (l *localTask) Status() string {
l.mtx.RLock()
defer l.mtx.RUnlock()
return l.status
}
func NewTask(opts ...task.Option) task.Task {
var options task.Options
for _, o := range opts {
o(&options)
}
if options.Pool == 0 {
options.Pool = 1
}
return &localTask{
opts: options,
}
}

85
sync/task/task.go
View File

@ -1,85 +0,0 @@
// Package task provides an interface for distributed jobs
package task
import (
"context"
"fmt"
"time"
)
// Task represents a distributed task
type Task interface {
// Run runs a command immediately until completion
Run(Command) error
// Status provides status of last execution
Status() string
}
// Command to be executed
type Command struct {
Name string
Func func() error
}
// Schedule represents a time or interval at which a task should run
type Schedule struct {
// When to start the schedule. Zero time means immediately
Time time.Time
// Non zero interval dictates an ongoing schedule
Interval time.Duration
}
type Options struct {
// Pool size for workers
Pool int
// Alternative options
Context context.Context
}
type Option func(o *Options)
func (c Command) Execute() error {
return c.Func()
}
func (c Command) String() string {
return c.Name
}
func (s Schedule) Run() <-chan time.Time {
d := s.Time.Sub(time.Now())
ch := make(chan time.Time, 1)
go func() {
// wait for start time
<-time.After(d)
// zero interval
if s.Interval == time.Duration(0) {
ch <- time.Now()
close(ch)
return
}
// start ticker
ticker := time.NewTicker(s.Interval)
defer ticker.Stop()
for t := range ticker.C {
ch <- t
}
}()
return ch
}
func (s Schedule) String() string {
return fmt.Sprintf("%d-%d", s.Time.Unix(), s.Interval)
}
// WithPool sets the pool size for concurrent work
func WithPool(i int) Option {
return func(o *Options) {
o.Pool = i
}
}

18
sync/time/local/local.go
View File

@ -1,18 +0,0 @@
// Package local provides a local clock
package local
import (
gotime "time"
"github.com/micro/go-micro/v2/sync/time"
)
type Time struct{}
func (t *Time) Now() (gotime.Time, error) {
return gotime.Now(), nil
}
func NewTime(opts ...time.Option) time.Time {
return new(Time)
}

51
sync/time/ntp/ntp.go
View File

@ -1,51 +0,0 @@
// Package ntp provides ntp synchronized time
package ntp
import (
"context"
gotime "time"
"github.com/beevik/ntp"
"github.com/micro/go-micro/v2/sync/time"
)
type ntpTime struct {
server string
}
type ntpServerKey struct{}
func (n *ntpTime) Now() (gotime.Time, error) {
return ntp.Time(n.server)
}
// NewTime returns ntp time
func NewTime(opts ...time.Option) time.Time {
options := time.Options{
Context: context.Background(),
}
for _, o := range opts {
o(&options)
}
server := "time.google.com"
if k, ok := options.Context.Value(ntpServerKey{}).(string); ok {
server = k
}
return &ntpTime{
server: server,
}
}
// WithServer sets the ntp server
func WithServer(s string) time.Option {
return func(o *time.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, ntpServerKey{}, s)
}
}

18
sync/time/time.go
View File

@ -1,18 +0,0 @@
// Package time provides clock synchronization
package time
import (
"context"
"time"
)
// Time returns synchronized time
type Time interface {
Now() (time.Time, error)
}
type Options struct {
Context context.Context
}
type Option func(o *Options)

23
util/mdns/.gitignore vendored Normal file
View File

@ -0,0 +1,23 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test

501
util/mdns/client.go Normal file
View File

@ -0,0 +1,501 @@
package mdns
import (
"context"
"fmt"
"log"
"net"
"strings"
"sync"
"time"
"github.com/miekg/dns"
"golang.org/x/net/ipv4"
"golang.org/x/net/ipv6"
)
// ServiceEntry is returned after we query for a service
type ServiceEntry struct {
Name string
Host string
AddrV4 net.IP
AddrV6 net.IP
Port int
Info string
InfoFields []string
TTL int
Type uint16
Addr net.IP // @Deprecated
hasTXT bool
sent bool
}
// complete is used to check if we have all the info we need
func (s *ServiceEntry) complete() bool {
return (s.AddrV4 != nil || s.AddrV6 != nil || s.Addr != nil) && s.Port != 0 && s.hasTXT
}
// QueryParam is used to customize how a Lookup is performed
type QueryParam struct {
Service string // Service to lookup
Domain string // Lookup domain, default "local"
Type uint16 // Lookup type, defaults to dns.TypePTR
Context context.Context // Context
Timeout time.Duration // Lookup timeout, default 1 second. Ignored if Context is provided
Interface *net.Interface // Multicast interface to use
Entries chan<- *ServiceEntry // Entries Channel
WantUnicastResponse bool // Unicast response desired, as per 5.4 in RFC
}
// DefaultParams is used to return a default set of QueryParam's
func DefaultParams(service string) *QueryParam {
return &QueryParam{
Service: service,
Domain: "local",
Timeout: time.Second,
Entries: make(chan *ServiceEntry),
WantUnicastResponse: false, // TODO(reddaly): Change this default.
}
}
// Query looks up a given service, in a domain, waiting at most
// for a timeout before finishing the query. The results are streamed
// to a channel. Sends will not block, so clients should make sure to
// either read or buffer.
func Query(params *QueryParam) error {
// Create a new client
client, err := newClient()
if err != nil {
return err
}
defer client.Close()
// Set the multicast interface
if params.Interface != nil {
if err := client.setInterface(params.Interface, false); err != nil {
return err
}
}
// Ensure defaults are set
if params.Domain == "" {
params.Domain = "local"
}
if params.Context == nil {
if params.Timeout == 0 {
params.Timeout = time.Second
}
params.Context, _ = context.WithTimeout(context.Background(), params.Timeout)
if err != nil {
return err
}
}
// Run the query
return client.query(params)
}
// Listen listens indefinitely for multicast updates
func Listen(entries chan<- *ServiceEntry, exit chan struct{}) error {
// Create a new client
client, err := newClient()
if err != nil {
return err
}
defer client.Close()
client.setInterface(nil, true)
// Start listening for response packets
msgCh := make(chan *dns.Msg, 32)
go client.recv(client.ipv4UnicastConn, msgCh)
go client.recv(client.ipv6UnicastConn, msgCh)
go client.recv(client.ipv4MulticastConn, msgCh)
go client.recv(client.ipv6MulticastConn, msgCh)
ip := make(map[string]*ServiceEntry)
for {
select {
case <-exit:
return nil
case <-client.closedCh:
return nil
case m := <-msgCh:
e := messageToEntry(m, ip)
if e == nil {
continue
}
// Check if this entry is complete
if e.complete() {
if e.sent {
continue
}
e.sent = true
entries <- e
ip = make(map[string]*ServiceEntry)
} else {
// Fire off a node specific query
m := new(dns.Msg)
m.SetQuestion(e.Name, dns.TypePTR)
m.RecursionDesired = false
if err := client.sendQuery(m); err != nil {
log.Printf("[ERR] mdns: Failed to query instance %s: %v", e.Name, err)
}
}
}
}
return nil
}
// Lookup is the same as Query, however it uses all the default parameters
func Lookup(service string, entries chan<- *ServiceEntry) error {
params := DefaultParams(service)
params.Entries = entries
return Query(params)
}
// Client provides a query interface that can be used to
// search for service providers using mDNS
type client struct {
ipv4UnicastConn *net.UDPConn
ipv6UnicastConn *net.UDPConn
ipv4MulticastConn *net.UDPConn
ipv6MulticastConn *net.UDPConn
closed bool
closedCh chan struct{} // TODO(reddaly): This doesn't appear to be used.
closeLock sync.Mutex
}
// NewClient creates a new mdns Client that can be used to query
// for records
func newClient() (*client, error) {
// TODO(reddaly): At least attempt to bind to the port required in the spec.
// Create a IPv4 listener
uconn4, err4 := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
uconn6, err6 := net.ListenUDP("udp6", &net.UDPAddr{IP: net.IPv6zero, Port: 0})
if err4 != nil && err6 != nil {
log.Printf("[ERR] mdns: Failed to bind to udp port: %v %v", err4, err6)
}
if uconn4 == nil && uconn6 == nil {
return nil, fmt.Errorf("failed to bind to any unicast udp port")
}
if uconn4 == nil {
uconn4 = &net.UDPConn{}
}
if uconn6 == nil {
uconn6 = &net.UDPConn{}
}
mconn4, err4 := net.ListenUDP("udp4", mdnsWildcardAddrIPv4)
mconn6, err6 := net.ListenUDP("udp6", mdnsWildcardAddrIPv6)
if err4 != nil && err6 != nil {
log.Printf("[ERR] mdns: Failed to bind to udp port: %v %v", err4, err6)
}
if mconn4 == nil && mconn6 == nil {
return nil, fmt.Errorf("failed to bind to any multicast udp port")
}
if mconn4 == nil {
mconn4 = &net.UDPConn{}
}
if mconn6 == nil {
mconn6 = &net.UDPConn{}
}
p1 := ipv4.NewPacketConn(mconn4)
p2 := ipv6.NewPacketConn(mconn6)
p1.SetMulticastLoopback(true)
p2.SetMulticastLoopback(true)
ifaces, err := net.Interfaces()
if err != nil {
return nil, err
}
var errCount1, errCount2 int
for _, iface := range ifaces {
if err := p1.JoinGroup(&iface, &net.UDPAddr{IP: mdnsGroupIPv4}); err != nil {
errCount1++
}
if err := p2.JoinGroup(&iface, &net.UDPAddr{IP: mdnsGroupIPv6}); err != nil {
errCount2++
}
}
if len(ifaces) == errCount1 && len(ifaces) == errCount2 {
return nil, fmt.Errorf("Failed to join multicast group on all interfaces!")
}
c := &client{
ipv4MulticastConn: mconn4,
ipv6MulticastConn: mconn6,
ipv4UnicastConn: uconn4,
ipv6UnicastConn: uconn6,
closedCh: make(chan struct{}),
}
return c, nil
}
// Close is used to cleanup the client
func (c *client) Close() error {
c.closeLock.Lock()
defer c.closeLock.Unlock()
if c.closed {
return nil
}
c.closed = true
close(c.closedCh)
if c.ipv4UnicastConn != nil {
c.ipv4UnicastConn.Close()
}
if c.ipv6UnicastConn != nil {
c.ipv6UnicastConn.Close()
}
if c.ipv4MulticastConn != nil {
c.ipv4MulticastConn.Close()
}
if c.ipv6MulticastConn != nil {
c.ipv6MulticastConn.Close()
}
return nil
}
// setInterface is used to set the query interface, uses sytem
// default if not provided
func (c *client) setInterface(iface *net.Interface, loopback bool) error {
p := ipv4.NewPacketConn(c.ipv4UnicastConn)
if err := p.JoinGroup(iface, &net.UDPAddr{IP: mdnsGroupIPv4}); err != nil {
return err
}
p2 := ipv6.NewPacketConn(c.ipv6UnicastConn)
if err := p2.JoinGroup(iface, &net.UDPAddr{IP: mdnsGroupIPv6}); err != nil {
return err
}
p = ipv4.NewPacketConn(c.ipv4MulticastConn)
if err := p.JoinGroup(iface, &net.UDPAddr{IP: mdnsGroupIPv4}); err != nil {
return err
}
p2 = ipv6.NewPacketConn(c.ipv6MulticastConn)
if err := p2.JoinGroup(iface, &net.UDPAddr{IP: mdnsGroupIPv6}); err != nil {
return err
}
if loopback {
p.SetMulticastLoopback(true)
p2.SetMulticastLoopback(true)
}
return nil
}
// query is used to perform a lookup and stream results
func (c *client) query(params *QueryParam) error {
// Create the service name
serviceAddr := fmt.Sprintf("%s.%s.", trimDot(params.Service), trimDot(params.Domain))
// Start listening for response packets
msgCh := make(chan *dns.Msg, 32)
go c.recv(c.ipv4UnicastConn, msgCh)
go c.recv(c.ipv6UnicastConn, msgCh)
go c.recv(c.ipv4MulticastConn, msgCh)
go c.recv(c.ipv6MulticastConn, msgCh)
// Send the query
m := new(dns.Msg)
if params.Type == dns.TypeNone {
m.SetQuestion(serviceAddr, dns.TypePTR)
} else {
m.SetQuestion(serviceAddr, params.Type)
}
// RFC 6762, section 18.12. Repurposing of Top Bit of qclass in Question
// Section
//
// In the Question Section of a Multicast DNS query, the top bit of the qclass
// field is used to indicate that unicast responses are preferred for this
// particular question. (See Section 5.4.)
if params.WantUnicastResponse {
m.Question[0].Qclass |= 1 << 15
}
m.RecursionDesired = false
if err := c.sendQuery(m); err != nil {
return err
}
// Map the in-progress responses
inprogress := make(map[string]*ServiceEntry)
for {
select {
case resp := <-msgCh:
inp := messageToEntry(resp, inprogress)
if inp == nil {
continue
}
// Check if this entry is complete
if inp.complete() {
if inp.sent {
continue
}
inp.sent = true
select {
case params.Entries <- inp:
case <-params.Context.Done():
return nil
}
} else {
// Fire off a node specific query
m := new(dns.Msg)
m.SetQuestion(inp.Name, inp.Type)
m.RecursionDesired = false
if err := c.sendQuery(m); err != nil {
log.Printf("[ERR] mdns: Failed to query instance %s: %v", inp.Name, err)
}
}
case <-params.Context.Done():
return nil
}
}
}
// sendQuery is used to multicast a query out
func (c *client) sendQuery(q *dns.Msg) error {
buf, err := q.Pack()
if err != nil {
return err
}
if c.ipv4UnicastConn != nil {
c.ipv4UnicastConn.WriteToUDP(buf, ipv4Addr)
}
if c.ipv6UnicastConn != nil {
c.ipv6UnicastConn.WriteToUDP(buf, ipv6Addr)
}
return nil
}
// recv is used to receive until we get a shutdown
func (c *client) recv(l *net.UDPConn, msgCh chan *dns.Msg) {
if l == nil {
return
}
buf := make([]byte, 65536)
for {
c.closeLock.Lock()
if c.closed {
c.closeLock.Unlock()
return
}
c.closeLock.Unlock()
n, err := l.Read(buf)
if err != nil {
continue
}
msg := new(dns.Msg)
if err := msg.Unpack(buf[:n]); err != nil {
continue
}
select {
case msgCh <- msg:
case <-c.closedCh:
return
}
}
}
// ensureName is used to ensure the named node is in progress
func ensureName(inprogress map[string]*ServiceEntry, name string, typ uint16) *ServiceEntry {
if inp, ok := inprogress[name]; ok {
return inp
}
inp := &ServiceEntry{
Name: name,
Type: typ,
}
inprogress[name] = inp
return inp
}
// alias is used to setup an alias between two entries
func alias(inprogress map[string]*ServiceEntry, src, dst string, typ uint16) {
srcEntry := ensureName(inprogress, src, typ)
inprogress[dst] = srcEntry
}
func messageToEntry(m *dns.Msg, inprogress map[string]*ServiceEntry) *ServiceEntry {
var inp *ServiceEntry
for _, answer := range append(m.Answer, m.Extra...) {
// TODO(reddaly): Check that response corresponds to serviceAddr?
switch rr := answer.(type) {
case *dns.PTR:
// Create new entry for this
inp = ensureName(inprogress, rr.Ptr, rr.Hdr.Rrtype)
if inp.complete() {
continue
}
case *dns.SRV:
// Check for a target mismatch
if rr.Target != rr.Hdr.Name {
alias(inprogress, rr.Hdr.Name, rr.Target, rr.Hdr.Rrtype)
}
// Get the port
inp = ensureName(inprogress, rr.Hdr.Name, rr.Hdr.Rrtype)
if inp.complete() {
continue
}
inp.Host = rr.Target
inp.Port = int(rr.Port)
case *dns.TXT:
// Pull out the txt
inp = ensureName(inprogress, rr.Hdr.Name, rr.Hdr.Rrtype)
if inp.complete() {
continue
}
inp.Info = strings.Join(rr.Txt, "|")
inp.InfoFields = rr.Txt
inp.hasTXT = true
case *dns.A:
// Pull out the IP
inp = ensureName(inprogress, rr.Hdr.Name, rr.Hdr.Rrtype)
if inp.complete() {
continue
}
inp.Addr = rr.A // @Deprecated
inp.AddrV4 = rr.A
case *dns.AAAA:
// Pull out the IP
inp = ensureName(inprogress, rr.Hdr.Name, rr.Hdr.Rrtype)
if inp.complete() {
continue
}
inp.Addr = rr.AAAA // @Deprecated
inp.AddrV6 = rr.AAAA
}
if inp != nil {
inp.TTL = int(answer.Header().Ttl)
}
}
return inp
}

84
util/mdns/dns_sd.go Normal file
View File

@ -0,0 +1,84 @@
package mdns
import "github.com/miekg/dns"
// DNSSDService is a service that complies with the DNS-SD (RFC 6762) and MDNS
// (RFC 6762) specs for local, multicast-DNS-based discovery.
//
// DNSSDService implements the Zone interface and wraps an MDNSService instance.
// To deploy an mDNS service that is compliant with DNS-SD, it's recommended to
// register only the wrapped instance with the server.
//
// Example usage:
// service := &mdns.DNSSDService{
// MDNSService: &mdns.MDNSService{
// Instance: "My Foobar Service",
// Service: "_foobar._tcp",
// Port: 8000,
// }
// }
// server, err := mdns.NewServer(&mdns.Config{Zone: service})
// if err != nil {
// log.Fatalf("Error creating server: %v", err)
// }
// defer server.Shutdown()
type DNSSDService struct {
MDNSService *MDNSService
}
// Records returns DNS records in response to a DNS question.
//
// This function returns the DNS response of the underlying MDNSService
// instance. It also returns a PTR record for a request for "
// _services._dns-sd._udp.<Domain>", as described in section 9 of RFC 6763
// ("Service Type Enumeration"), to allow browsing of the underlying MDNSService
// instance.
func (s *DNSSDService) Records(q dns.Question) []dns.RR {
var recs []dns.RR
if q.Name == "_services._dns-sd._udp."+s.MDNSService.Domain+"." {
recs = s.dnssdMetaQueryRecords(q)
}
return append(recs, s.MDNSService.Records(q)...)
}
// dnssdMetaQueryRecords returns the DNS records in response to a "meta-query"
// issued to browse for DNS-SD services, as per section 9. of RFC6763.
//
// A meta-query has a name of the form "_services._dns-sd._udp.<Domain>" where
// Domain is a fully-qualified domain, such as "local."
func (s *DNSSDService) dnssdMetaQueryRecords(q dns.Question) []dns.RR {
// Intended behavior, as described in the RFC:
// ...it may be useful for network administrators to find the list of
// advertised service types on the network, even if those Service Names
// are just opaque identifiers and not particularly informative in
// isolation.
//
// For this purpose, a special meta-query is defined. A DNS query for PTR
// records with the name "_services._dns-sd._udp.<Domain>" yields a set of
// PTR records, where the rdata of each PTR record is the two-abel
// <Service> name, plus the same domain, e.g., "_http._tcp.<Domain>".
// Including the domain in the PTR rdata allows for slightly better name
// compression in Unicast DNS responses, but only the first two labels are
// relevant for the purposes of service type enumeration. These two-label
// service types can then be used to construct subsequent Service Instance
// Enumeration PTR queries, in this <Domain> or others, to discover
// instances of that service type.
return []dns.RR{
&dns.PTR{
Hdr: dns.RR_Header{
Name: q.Name,
Rrtype: dns.TypePTR,
Class: dns.ClassINET,
Ttl: defaultTTL,
},
Ptr: s.MDNSService.serviceAddr,
},
}
}
// Announcement returns DNS records that should be broadcast during the initial
// availability of the service, as described in section 8.3 of RFC 6762.
// TODO(reddaly): Add this when Announcement is added to the mdns.Zone interface.
//func (s *DNSSDService) Announcement() []dns.RR {
// return s.MDNSService.Announcement()
//}

68
util/mdns/dns_sd_test.go Normal file
View File

@ -0,0 +1,68 @@
package mdns
import (
"reflect"
"testing"
)
import "github.com/miekg/dns"
type mockMDNSService struct{}
func (s *mockMDNSService) Records(q dns.Question) []dns.RR {
return []dns.RR{
&dns.PTR{
Hdr: dns.RR_Header{
Name: "fakerecord",
Rrtype: dns.TypePTR,
Class: dns.ClassINET,
Ttl: 42,
},
Ptr: "fake.local.",
},
}
}
func (s *mockMDNSService) Announcement() []dns.RR {
return []dns.RR{
&dns.PTR{
Hdr: dns.RR_Header{
Name: "fakeannounce",
Rrtype: dns.TypePTR,
Class: dns.ClassINET,
Ttl: 42,
},
Ptr: "fake.local.",
},
}
}
func TestDNSSDServiceRecords(t *testing.T) {
s := &DNSSDService{
MDNSService: &MDNSService{
serviceAddr: "_foobar._tcp.local.",
Domain: "local",
},
}
q := dns.Question{
Name: "_services._dns-sd._udp.local.",
Qtype: dns.TypePTR,
Qclass: dns.ClassINET,
}
recs := s.Records(q)
if got, want := len(recs), 1; got != want {
t.Fatalf("s.Records(%v) returned %v records, want %v", q, got, want)
}
want := dns.RR(&dns.PTR{
Hdr: dns.RR_Header{
Name: "_services._dns-sd._udp.local.",
Rrtype: dns.TypePTR,
Class: dns.ClassINET,
Ttl: defaultTTL,
},
Ptr: "_foobar._tcp.local.",
})
if got := recs[0]; !reflect.DeepEqual(got, want) {
t.Errorf("s.Records()[0] = %v, want %v", got, want)
}
}

476
util/mdns/server.go Normal file
View File

@ -0,0 +1,476 @@
package mdns
import (
"fmt"
"log"
"math/rand"
"net"
"sync"
"sync/atomic"
"time"
"github.com/miekg/dns"
"golang.org/x/net/ipv4"
"golang.org/x/net/ipv6"
)
var (
mdnsGroupIPv4 = net.ParseIP("224.0.0.251")
mdnsGroupIPv6 = net.ParseIP("ff02::fb")
// mDNS wildcard addresses
mdnsWildcardAddrIPv4 = &net.UDPAddr{
IP: net.ParseIP("224.0.0.0"),
Port: 5353,
}
mdnsWildcardAddrIPv6 = &net.UDPAddr{
IP: net.ParseIP("ff02::"),
Port: 5353,
}
// mDNS endpoint addresses
ipv4Addr = &net.UDPAddr{
IP: mdnsGroupIPv4,
Port: 5353,
}
ipv6Addr = &net.UDPAddr{
IP: mdnsGroupIPv6,
Port: 5353,
}
)
// Config is used to configure the mDNS server
type Config struct {
// Zone must be provided to support responding to queries
Zone Zone
// Iface if provided binds the multicast listener to the given
// interface. If not provided, the system default multicase interface
// is used.
Iface *net.Interface
// Port If it is not 0, replace the port 5353 with this port number.
Port int
}
// mDNS server is used to listen for mDNS queries and respond if we
// have a matching local record
type Server struct {
config *Config
ipv4List *net.UDPConn
ipv6List *net.UDPConn
shutdown bool
shutdownCh chan struct{}
shutdownLock sync.Mutex
wg sync.WaitGroup
}
// NewServer is used to create a new mDNS server from a config
func NewServer(config *Config) (*Server, error) {
setCustomPort(config.Port)
// Create the listeners
// Create wildcard connections (because :5353 can be already taken by other apps)
ipv4List, _ := net.ListenUDP("udp4", mdnsWildcardAddrIPv4)
ipv6List, _ := net.ListenUDP("udp6", mdnsWildcardAddrIPv6)
if ipv4List == nil && ipv6List == nil {
return nil, fmt.Errorf("[ERR] mdns: Failed to bind to any udp port!")
}
if ipv4List == nil {
ipv4List = &net.UDPConn{}
}
if ipv6List == nil {
ipv6List = &net.UDPConn{}
}
// Join multicast groups to receive announcements
p1 := ipv4.NewPacketConn(ipv4List)
p2 := ipv6.NewPacketConn(ipv6List)
p1.SetMulticastLoopback(true)
p2.SetMulticastLoopback(true)
if config.Iface != nil {
if err := p1.JoinGroup(config.Iface, &net.UDPAddr{IP: mdnsGroupIPv4}); err != nil {
return nil, err
}
if err := p2.JoinGroup(config.Iface, &net.UDPAddr{IP: mdnsGroupIPv6}); err != nil {
return nil, err
}
} else {
ifaces, err := net.Interfaces()
if err != nil {
return nil, err
}
errCount1, errCount2 := 0, 0
for _, iface := range ifaces {
if err := p1.JoinGroup(&iface, &net.UDPAddr{IP: mdnsGroupIPv4}); err != nil {
errCount1++
}
if err := p2.JoinGroup(&iface, &net.UDPAddr{IP: mdnsGroupIPv6}); err != nil {
errCount2++
}
}
if len(ifaces) == errCount1 && len(ifaces) == errCount2 {
return nil, fmt.Errorf("Failed to join multicast group on all interfaces!")
}
}
s := &Server{
config: config,
ipv4List: ipv4List,
ipv6List: ipv6List,
shutdownCh: make(chan struct{}),
}
go s.recv(s.ipv4List)
go s.recv(s.ipv6List)
s.wg.Add(1)
go s.probe()
return s, nil
}
// Shutdown is used to shutdown the listener
func (s *Server) Shutdown() error {
s.shutdownLock.Lock()
defer s.shutdownLock.Unlock()
if s.shutdown {
return nil
}
s.shutdown = true
close(s.shutdownCh)
s.unregister()
if s.ipv4List != nil {
s.ipv4List.Close()
}
if s.ipv6List != nil {
s.ipv6List.Close()
}
s.wg.Wait()
return nil
}
// recv is a long running routine to receive packets from an interface
func (s *Server) recv(c *net.UDPConn) {
if c == nil {
return
}
buf := make([]byte, 65536)
for {
s.shutdownLock.Lock()
if s.shutdown {
s.shutdownLock.Unlock()
return
}
s.shutdownLock.Unlock()
n, from, err := c.ReadFrom(buf)
if err != nil {
continue
}
if err := s.parsePacket(buf[:n], from); err != nil {
log.Printf("[ERR] mdns: Failed to handle query: %v", err)
}
}
}
// parsePacket is used to parse an incoming packet
func (s *Server) parsePacket(packet []byte, from net.Addr) error {
var msg dns.Msg
if err := msg.Unpack(packet); err != nil {
log.Printf("[ERR] mdns: Failed to unpack packet: %v", err)
return err
}
// TODO: This is a bit of a hack
// We decided to ignore some mDNS answers for the time being
// See: https://tools.ietf.org/html/rfc6762#section-7.2
msg.Truncated = false
return s.handleQuery(&msg, from)
}
// handleQuery is used to handle an incoming query
func (s *Server) handleQuery(query *dns.Msg, from net.Addr) error {
if query.Opcode != dns.OpcodeQuery {
// "In both multicast query and multicast response messages, the OPCODE MUST
// be zero on transmission (only standard queries are currently supported
// over multicast). Multicast DNS messages received with an OPCODE other
// than zero MUST be silently ignored." Note: OpcodeQuery == 0
return fmt.Errorf("mdns: received query with non-zero Opcode %v: %v", query.Opcode, *query)
}
if query.Rcode != 0 {
// "In both multicast query and multicast response messages, the Response
// Code MUST be zero on transmission. Multicast DNS messages received with
// non-zero Response Codes MUST be silently ignored."
return fmt.Errorf("mdns: received query with non-zero Rcode %v: %v", query.Rcode, *query)
}
// TODO(reddaly): Handle "TC (Truncated) Bit":
// In query messages, if the TC bit is set, it means that additional
// Known-Answer records may be following shortly. A responder SHOULD
// record this fact, and wait for those additional Known-Answer records,
// before deciding whether to respond. If the TC bit is clear, it means
// that the querying host has no additional Known Answers.
if query.Truncated {
return fmt.Errorf("[ERR] mdns: support for DNS requests with high truncated bit not implemented: %v", *query)
}
var unicastAnswer, multicastAnswer []dns.RR
// Handle each question
for _, q := range query.Question {
mrecs, urecs := s.handleQuestion(q)
multicastAnswer = append(multicastAnswer, mrecs...)
unicastAnswer = append(unicastAnswer, urecs...)
}
// See section 18 of RFC 6762 for rules about DNS headers.
resp := func(unicast bool) *dns.Msg {
// 18.1: ID (Query Identifier)
// 0 for multicast response, query.Id for unicast response
id := uint16(0)
if unicast {
id = query.Id
}
var answer []dns.RR
if unicast {
answer = unicastAnswer
} else {
answer = multicastAnswer
}
if len(answer) == 0 {
return nil
}
return &dns.Msg{
MsgHdr: dns.MsgHdr{
Id: id,
// 18.2: QR (Query/Response) Bit - must be set to 1 in response.
Response: true,
// 18.3: OPCODE - must be zero in response (OpcodeQuery == 0)
Opcode: dns.OpcodeQuery,
// 18.4: AA (Authoritative Answer) Bit - must be set to 1
Authoritative: true,
// The following fields must all be set to 0:
// 18.5: TC (TRUNCATED) Bit
// 18.6: RD (Recursion Desired) Bit
// 18.7: RA (Recursion Available) Bit
// 18.8: Z (Zero) Bit
// 18.9: AD (Authentic Data) Bit
// 18.10: CD (Checking Disabled) Bit
// 18.11: RCODE (Response Code)
},
// 18.12 pertains to questions (handled by handleQuestion)
// 18.13 pertains to resource records (handled by handleQuestion)
// 18.14: Name Compression - responses should be compressed (though see
// caveats in the RFC), so set the Compress bit (part of the dns library
// API, not part of the DNS packet) to true.
Compress: true,
Answer: answer,
}
}
if mresp := resp(false); mresp != nil {
if err := s.sendResponse(mresp, from); err != nil {
return fmt.Errorf("mdns: error sending multicast response: %v", err)
}
}
if uresp := resp(true); uresp != nil {
if err := s.sendResponse(uresp, from); err != nil {
return fmt.Errorf("mdns: error sending unicast response: %v", err)
}
}
return nil
}
// handleQuestion is used to handle an incoming question
//
// The response to a question may be transmitted over multicast, unicast, or
// both. The return values are DNS records for each transmission type.
func (s *Server) handleQuestion(q dns.Question) (multicastRecs, unicastRecs []dns.RR) {
records := s.config.Zone.Records(q)
if len(records) == 0 {
return nil, nil
}
// Handle unicast and multicast responses.
// TODO(reddaly): The decision about sending over unicast vs. multicast is not
// yet fully compliant with RFC 6762. For example, the unicast bit should be
// ignored if the records in question are close to TTL expiration. For now,
// we just use the unicast bit to make the decision, as per the spec:
// RFC 6762, section 18.12. Repurposing of Top Bit of qclass in Question
// Section
//
// In the Question Section of a Multicast DNS query, the top bit of the
// qclass field is used to indicate that unicast responses are preferred
// for this particular question. (See Section 5.4.)
if q.Qclass&(1<<15) != 0 {
return nil, records
}
return records, nil
}
func (s *Server) probe() {
defer s.wg.Done()
sd, ok := s.config.Zone.(*MDNSService)
if !ok {
return
}
name := fmt.Sprintf("%s.%s.%s.", sd.Instance, trimDot(sd.Service), trimDot(sd.Domain))
q := new(dns.Msg)
q.SetQuestion(name, dns.TypePTR)
q.RecursionDesired = false
srv := &dns.SRV{
Hdr: dns.RR_Header{
Name: name,
Rrtype: dns.TypeSRV,
Class: dns.ClassINET,
Ttl: defaultTTL,
},
Priority: 0,
Weight: 0,
Port: uint16(sd.Port),
Target: sd.HostName,
}
txt := &dns.TXT{
Hdr: dns.RR_Header{
Name: name,
Rrtype: dns.TypeTXT,
Class: dns.ClassINET,
Ttl: defaultTTL,
},
Txt: sd.TXT,
}
q.Ns = []dns.RR{srv, txt}
randomizer := rand.New(rand.NewSource(time.Now().UnixNano()))
for i := 0; i < 3; i++ {
if err := s.SendMulticast(q); err != nil {
log.Println("[ERR] mdns: failed to send probe:", err.Error())
}
time.Sleep(time.Duration(randomizer.Intn(250)) * time.Millisecond)
}
resp := new(dns.Msg)
resp.MsgHdr.Response = true
// set for query
q.SetQuestion(name, dns.TypeANY)
resp.Answer = append(resp.Answer, s.config.Zone.Records(q.Question[0])...)
// reset
q.SetQuestion(name, dns.TypePTR)
// From RFC6762
// The Multicast DNS responder MUST send at least two unsolicited
// responses, one second apart. To provide increased robustness against
// packet loss, a responder MAY send up to eight unsolicited responses,
// provided that the interval between unsolicited responses increases by
// at least a factor of two with every response sent.
timeout := 1 * time.Second
timer := time.NewTimer(timeout)
for i := 0; i < 3; i++ {
if err := s.SendMulticast(resp); err != nil {
log.Println("[ERR] mdns: failed to send announcement:", err.Error())
}
select {
case <-timer.C:
timeout *= 2
timer.Reset(timeout)
case <-s.shutdownCh:
timer.Stop()
return
}
}
}
// multicastResponse us used to send a multicast response packet
func (s *Server) SendMulticast(msg *dns.Msg) error {
buf, err := msg.Pack()
if err != nil {
return err
}
if s.ipv4List != nil {
s.ipv4List.WriteToUDP(buf, ipv4Addr)
}
if s.ipv6List != nil {
s.ipv6List.WriteToUDP(buf, ipv6Addr)
}
return nil
}
// sendResponse is used to send a response packet
func (s *Server) sendResponse(resp *dns.Msg, from net.Addr) error {
// TODO(reddaly): Respect the unicast argument, and allow sending responses
// over multicast.
buf, err := resp.Pack()
if err != nil {
return err
}
// Determine the socket to send from
addr := from.(*net.UDPAddr)
if addr.IP.To4() != nil {
_, err = s.ipv4List.WriteToUDP(buf, addr)
return err
} else {
_, err = s.ipv6List.WriteToUDP(buf, addr)
return err
}
}
func (s *Server) unregister() error {
sd, ok := s.config.Zone.(*MDNSService)
if !ok {
return nil
}
atomic.StoreUint32(&sd.TTL, 0)
name := fmt.Sprintf("%s.%s.%s.", sd.Instance, trimDot(sd.Service), trimDot(sd.Domain))
q := new(dns.Msg)
q.SetQuestion(name, dns.TypeANY)
resp := new(dns.Msg)
resp.MsgHdr.Response = true
resp.Answer = append(resp.Answer, s.config.Zone.Records(q.Question[0])...)
return s.SendMulticast(resp)
}
func setCustomPort(port int) {
if port != 0 {
if mdnsWildcardAddrIPv4.Port != port {
mdnsWildcardAddrIPv4.Port = port
}
if mdnsWildcardAddrIPv6.Port != port {
mdnsWildcardAddrIPv6.Port = port
}
if ipv4Addr.Port != port {
ipv4Addr.Port = port
}
if ipv6Addr.Port != port {
ipv6Addr.Port = port
}
}
}

61
util/mdns/server_test.go Normal file
View File

@ -0,0 +1,61 @@
package mdns
import (
"testing"
"time"
)
func TestServer_StartStop(t *testing.T) {
s := makeService(t)
serv, err := NewServer(&Config{Zone: s})
if err != nil {
t.Fatalf("err: %v", err)
}
defer serv.Shutdown()
}
func TestServer_Lookup(t *testing.T) {
serv, err := NewServer(&Config{Zone: makeServiceWithServiceName(t, "_foobar._tcp")})
if err != nil {
t.Fatalf("err: %v", err)
}
defer serv.Shutdown()
entries := make(chan *ServiceEntry, 1)
found := false
doneCh := make(chan struct{})
go func() {
select {
case e := <-entries:
if e.Name != "hostname._foobar._tcp.local." {
t.Fatalf("bad: %v", e)
}
if e.Port != 80 {
t.Fatalf("bad: %v", e)
}
if e.Info != "Local web server" {
t.Fatalf("bad: %v", e)
}
found = true
case <-time.After(80 * time.Millisecond):
t.Fatalf("timeout")
}
close(doneCh)
}()
params := &QueryParam{
Service: "_foobar._tcp",
Domain: "local",
Timeout: 50 * time.Millisecond,
Entries: entries,
}
err = Query(params)
if err != nil {
t.Fatalf("err: %v", err)
}
<-doneCh
if !found {
t.Fatalf("record not found")
}
}

309
util/mdns/zone.go Normal file
View File

@ -0,0 +1,309 @@
package mdns
import (
"fmt"
"net"
"os"
"strings"
"sync/atomic"
"github.com/miekg/dns"
)
const (
// defaultTTL is the default TTL value in returned DNS records in seconds.
defaultTTL = 120
)
// Zone is the interface used to integrate with the server and
// to serve records dynamically
type Zone interface {
// Records returns DNS records in response to a DNS question.
Records(q dns.Question) []dns.RR
}
// MDNSService is used to export a named service by implementing a Zone
type MDNSService struct {
Instance string // Instance name (e.g. "hostService name")
Service string // Service name (e.g. "_http._tcp.")
Domain string // If blank, assumes "local"
HostName string // Host machine DNS name (e.g. "mymachine.net.")
Port int // Service Port
IPs []net.IP // IP addresses for the service's host
TXT []string // Service TXT records
TTL uint32
serviceAddr string // Fully qualified service address
instanceAddr string // Fully qualified instance address
enumAddr string // _services._dns-sd._udp.<domain>
}
// validateFQDN returns an error if the passed string is not a fully qualified
// hdomain name (more specifically, a hostname).
func validateFQDN(s string) error {
if len(s) == 0 {
return fmt.Errorf("FQDN must not be blank")
}
if s[len(s)-1] != '.' {
return fmt.Errorf("FQDN must end in period: %s", s)
}
// TODO(reddaly): Perform full validation.
return nil
}
// NewMDNSService returns a new instance of MDNSService.
//
// If domain, hostName, or ips is set to the zero value, then a default value
// will be inferred from the operating system.
//
// TODO(reddaly): This interface may need to change to account for "unique
// record" conflict rules of the mDNS protocol. Upon startup, the server should
// check to ensure that the instance name does not conflict with other instance
// names, and, if required, select a new name. There may also be conflicting
// hostName A/AAAA records.
func NewMDNSService(instance, service, domain, hostName string, port int, ips []net.IP, txt []string) (*MDNSService, error) {
// Sanity check inputs
if instance == "" {
return nil, fmt.Errorf("missing service instance name")
}
if service == "" {
return nil, fmt.Errorf("missing service name")
}
if port == 0 {
return nil, fmt.Errorf("missing service port")
}
// Set default domain
if domain == "" {
domain = "local."
}
if err := validateFQDN(domain); err != nil {
return nil, fmt.Errorf("domain %q is not a fully-qualified domain name: %v", domain, err)
}
// Get host information if no host is specified.
if hostName == "" {
var err error
hostName, err = os.Hostname()
if err != nil {
return nil, fmt.Errorf("could not determine host: %v", err)
}
hostName = fmt.Sprintf("%s.", hostName)
}
if err := validateFQDN(hostName); err != nil {
return nil, fmt.Errorf("hostName %q is not a fully-qualified domain name: %v", hostName, err)
}
if len(ips) == 0 {
var err error
ips, err = net.LookupIP(trimDot(hostName))
if err != nil {
// Try appending the host domain suffix and lookup again
// (required for Linux-based hosts)
tmpHostName := fmt.Sprintf("%s%s", hostName, domain)
ips, err = net.LookupIP(trimDot(tmpHostName))
if err != nil {
return nil, fmt.Errorf("could not determine host IP addresses for %s", hostName)
}
}
}
for _, ip := range ips {
if ip.To4() == nil && ip.To16() == nil {
return nil, fmt.Errorf("invalid IP address in IPs list: %v", ip)
}
}
return &MDNSService{
Instance: instance,
Service: service,
Domain: domain,
HostName: hostName,
Port: port,
IPs: ips,
TXT: txt,
TTL: defaultTTL,
serviceAddr: fmt.Sprintf("%s.%s.", trimDot(service), trimDot(domain)),
instanceAddr: fmt.Sprintf("%s.%s.%s.", instance, trimDot(service), trimDot(domain)),
enumAddr: fmt.Sprintf("_services._dns-sd._udp.%s.", trimDot(domain)),
}, nil
}
// trimDot is used to trim the dots from the start or end of a string
func trimDot(s string) string {
return strings.Trim(s, ".")
}
// Records returns DNS records in response to a DNS question.
func (m *MDNSService) Records(q dns.Question) []dns.RR {
switch q.Name {
case m.enumAddr:
return m.serviceEnum(q)
case m.serviceAddr:
return m.serviceRecords(q)
case m.instanceAddr:
return m.instanceRecords(q)
case m.HostName:
if q.Qtype == dns.TypeA || q.Qtype == dns.TypeAAAA {
return m.instanceRecords(q)
}
fallthrough
default:
return nil
}
}
func (m *MDNSService) serviceEnum(q dns.Question) []dns.RR {
switch q.Qtype {
case dns.TypeANY:
fallthrough
case dns.TypePTR:
rr := &dns.PTR{
Hdr: dns.RR_Header{
Name: q.Name,
Rrtype: dns.TypePTR,
Class: dns.ClassINET,
Ttl: atomic.LoadUint32(&m.TTL),
},
Ptr: m.serviceAddr,
}
return []dns.RR{rr}
default:
return nil
}
}
// serviceRecords is called when the query matches the service name
func (m *MDNSService) serviceRecords(q dns.Question) []dns.RR {
switch q.Qtype {
case dns.TypeANY:
fallthrough
case dns.TypePTR:
// Build a PTR response for the service
rr := &dns.PTR{
Hdr: dns.RR_Header{
Name: q.Name,
Rrtype: dns.TypePTR,
Class: dns.ClassINET,
Ttl: atomic.LoadUint32(&m.TTL),
},
Ptr: m.instanceAddr,
}
servRec := []dns.RR{rr}
// Get the instance records
instRecs := m.instanceRecords(dns.Question{
Name: m.instanceAddr,
Qtype: dns.TypeANY,
})
// Return the service record with the instance records
return append(servRec, instRecs...)
default:
return nil
}
}
// serviceRecords is called when the query matches the instance name
func (m *MDNSService) instanceRecords(q dns.Question) []dns.RR {
switch q.Qtype {
case dns.TypeANY:
// Get the SRV, which includes A and AAAA
recs := m.instanceRecords(dns.Question{
Name: m.instanceAddr,
Qtype: dns.TypeSRV,
})
// Add the TXT record
recs = append(recs, m.instanceRecords(dns.Question{
Name: m.instanceAddr,
Qtype: dns.TypeTXT,
})...)
return recs
case dns.TypeA:
var rr []dns.RR
for _, ip := range m.IPs {
if ip4 := ip.To4(); ip4 != nil {
rr = append(rr, &dns.A{
Hdr: dns.RR_Header{
Name: m.HostName,
Rrtype: dns.TypeA,
Class: dns.ClassINET,
Ttl: atomic.LoadUint32(&m.TTL),
},
A: ip4,
})
}
}
return rr
case dns.TypeAAAA:
var rr []dns.RR
for _, ip := range m.IPs {
if ip.To4() != nil {
// TODO(reddaly): IPv4 addresses could be encoded in IPv6 format and
// putinto AAAA records, but the current logic puts ipv4-encodable
// addresses into the A records exclusively. Perhaps this should be
// configurable?
continue
}
if ip16 := ip.To16(); ip16 != nil {
rr = append(rr, &dns.AAAA{
Hdr: dns.RR_Header{
Name: m.HostName,
Rrtype: dns.TypeAAAA,
Class: dns.ClassINET,
Ttl: atomic.LoadUint32(&m.TTL),
},
AAAA: ip16,
})
}
}
return rr
case dns.TypeSRV:
// Create the SRV Record
srv := &dns.SRV{
Hdr: dns.RR_Header{
Name: q.Name,
Rrtype: dns.TypeSRV,
Class: dns.ClassINET,
Ttl: atomic.LoadUint32(&m.TTL),
},
Priority: 10,
Weight: 1,
Port: uint16(m.Port),
Target: m.HostName,
}
recs := []dns.RR{srv}
// Add the A record
recs = append(recs, m.instanceRecords(dns.Question{
Name: m.instanceAddr,
Qtype: dns.TypeA,
})...)
// Add the AAAA record
recs = append(recs, m.instanceRecords(dns.Question{
Name: m.instanceAddr,
Qtype: dns.TypeAAAA,
})...)
return recs
case dns.TypeTXT:
txt := &dns.TXT{
Hdr: dns.RR_Header{
Name: q.Name,
Rrtype: dns.TypeTXT,
Class: dns.ClassINET,
Ttl: atomic.LoadUint32(&m.TTL),
},
Txt: m.TXT,
}
return []dns.RR{txt}
}
return nil
}

275
util/mdns/zone_test.go Normal file
View File

@ -0,0 +1,275 @@
package mdns
import (
"bytes"
"net"
"reflect"
"testing"
"github.com/miekg/dns"
)
func makeService(t *testing.T) *MDNSService {
return makeServiceWithServiceName(t, "_http._tcp")
}
func makeServiceWithServiceName(t *testing.T, service string) *MDNSService {
m, err := NewMDNSService(
"hostname",
service,
"local.",
"testhost.",
80, // port
[]net.IP{net.IP([]byte{192, 168, 0, 42}), net.ParseIP("2620:0:1000:1900:b0c2:d0b2:c411:18bc")},
[]string{"Local web server"}) // TXT
if err != nil {
t.Fatalf("err: %v", err)
}
return m
}
func TestNewMDNSService_BadParams(t *testing.T) {
for _, test := range []struct {
testName string
hostName string
domain string
}{
{
"NewMDNSService should fail when passed hostName that is not a legal fully-qualified domain name",
"hostname", // not legal FQDN - should be "hostname." or "hostname.local.", etc.
"local.", // legal
},
{
"NewMDNSService should fail when passed domain that is not a legal fully-qualified domain name",
"hostname.", // legal
"local", // should be "local."
},
} {
_, err := NewMDNSService(
"instance name",
"_http._tcp",
test.domain,
test.hostName,
80, // port
[]net.IP{net.IP([]byte{192, 168, 0, 42})},
[]string{"Local web server"}) // TXT
if err == nil {
t.Fatalf("%s: error expected, but got none", test.testName)
}
}
}
func TestMDNSService_BadAddr(t *testing.T) {
s := makeService(t)
q := dns.Question{
Name: "random",
Qtype: dns.TypeANY,
}
recs := s.Records(q)
if len(recs) != 0 {
t.Fatalf("bad: %v", recs)
}
}
func TestMDNSService_ServiceAddr(t *testing.T) {
s := makeService(t)
q := dns.Question{
Name: "_http._tcp.local.",
Qtype: dns.TypeANY,
}
recs := s.Records(q)
if got, want := len(recs), 5; got != want {
t.Fatalf("got %d records, want %d: %v", got, want, recs)
}
if ptr, ok := recs[0].(*dns.PTR); !ok {
t.Errorf("recs[0] should be PTR record, got: %v, all records: %v", recs[0], recs)
} else if got, want := ptr.Ptr, "hostname._http._tcp.local."; got != want {
t.Fatalf("bad PTR record %v: got %v, want %v", ptr, got, want)
}
if _, ok := recs[1].(*dns.SRV); !ok {
t.Errorf("recs[1] should be SRV record, got: %v, all reccords: %v", recs[1], recs)
}
if _, ok := recs[2].(*dns.A); !ok {
t.Errorf("recs[2] should be A record, got: %v, all records: %v", recs[2], recs)
}
if _, ok := recs[3].(*dns.AAAA); !ok {
t.Errorf("recs[3] should be AAAA record, got: %v, all records: %v", recs[3], recs)
}
if _, ok := recs[4].(*dns.TXT); !ok {
t.Errorf("recs[4] should be TXT record, got: %v, all records: %v", recs[4], recs)
}
q.Qtype = dns.TypePTR
if recs2 := s.Records(q); !reflect.DeepEqual(recs, recs2) {
t.Fatalf("PTR question should return same result as ANY question: ANY => %v, PTR => %v", recs, recs2)
}
}
func TestMDNSService_InstanceAddr_ANY(t *testing.T) {
s := makeService(t)
q := dns.Question{
Name: "hostname._http._tcp.local.",
Qtype: dns.TypeANY,
}
recs := s.Records(q)
if len(recs) != 4 {
t.Fatalf("bad: %v", recs)
}
if _, ok := recs[0].(*dns.SRV); !ok {
t.Fatalf("bad: %v", recs[0])
}
if _, ok := recs[1].(*dns.A); !ok {
t.Fatalf("bad: %v", recs[1])
}
if _, ok := recs[2].(*dns.AAAA); !ok {
t.Fatalf("bad: %v", recs[2])
}
if _, ok := recs[3].(*dns.TXT); !ok {
t.Fatalf("bad: %v", recs[3])
}
}
func TestMDNSService_InstanceAddr_SRV(t *testing.T) {
s := makeService(t)
q := dns.Question{
Name: "hostname._http._tcp.local.",
Qtype: dns.TypeSRV,
}
recs := s.Records(q)
if len(recs) != 3 {
t.Fatalf("bad: %v", recs)
}
srv, ok := recs[0].(*dns.SRV)
if !ok {
t.Fatalf("bad: %v", recs[0])
}
if _, ok := recs[1].(*dns.A); !ok {
t.Fatalf("bad: %v", recs[1])
}
if _, ok := recs[2].(*dns.AAAA); !ok {
t.Fatalf("bad: %v", recs[2])
}
if srv.Port != uint16(s.Port) {
t.Fatalf("bad: %v", recs[0])
}
}
func TestMDNSService_InstanceAddr_A(t *testing.T) {
s := makeService(t)
q := dns.Question{
Name: "hostname._http._tcp.local.",
Qtype: dns.TypeA,
}
recs := s.Records(q)
if len(recs) != 1 {
t.Fatalf("bad: %v", recs)
}
a, ok := recs[0].(*dns.A)
if !ok {
t.Fatalf("bad: %v", recs[0])
}
if !bytes.Equal(a.A, []byte{192, 168, 0, 42}) {
t.Fatalf("bad: %v", recs[0])
}
}
func TestMDNSService_InstanceAddr_AAAA(t *testing.T) {
s := makeService(t)
q := dns.Question{
Name: "hostname._http._tcp.local.",
Qtype: dns.TypeAAAA,
}
recs := s.Records(q)
if len(recs) != 1 {
t.Fatalf("bad: %v", recs)
}
a4, ok := recs[0].(*dns.AAAA)
if !ok {
t.Fatalf("bad: %v", recs[0])
}
ip6 := net.ParseIP("2620:0:1000:1900:b0c2:d0b2:c411:18bc")
if got := len(ip6); got != net.IPv6len {
t.Fatalf("test IP failed to parse (len = %d, want %d)", got, net.IPv6len)
}
if !bytes.Equal(a4.AAAA, ip6) {
t.Fatalf("bad: %v", recs[0])
}
}
func TestMDNSService_InstanceAddr_TXT(t *testing.T) {
s := makeService(t)
q := dns.Question{
Name: "hostname._http._tcp.local.",
Qtype: dns.TypeTXT,
}
recs := s.Records(q)
if len(recs) != 1 {
t.Fatalf("bad: %v", recs)
}
txt, ok := recs[0].(*dns.TXT)
if !ok {
t.Fatalf("bad: %v", recs[0])
}
if got, want := txt.Txt, s.TXT; !reflect.DeepEqual(got, want) {
t.Fatalf("TXT record mismatch for %v: got %v, want %v", recs[0], got, want)
}
}
func TestMDNSService_HostNameQuery(t *testing.T) {
s := makeService(t)
for _, test := range []struct {
q dns.Question
want []dns.RR
}{
{
dns.Question{Name: "testhost.", Qtype: dns.TypeA},
[]dns.RR{&dns.A{
Hdr: dns.RR_Header{
Name: "testhost.",
Rrtype: dns.TypeA,
Class: dns.ClassINET,
Ttl: 120,
},
A: net.IP([]byte{192, 168, 0, 42}),
}},
},
{
dns.Question{Name: "testhost.", Qtype: dns.TypeAAAA},
[]dns.RR{&dns.AAAA{
Hdr: dns.RR_Header{
Name: "testhost.",
Rrtype: dns.TypeAAAA,
Class: dns.ClassINET,
Ttl: 120,
},
AAAA: net.ParseIP("2620:0:1000:1900:b0c2:d0b2:c411:18bc"),
}},
},
} {
if got := s.Records(test.q); !reflect.DeepEqual(got, test.want) {
t.Errorf("hostname query failed: s.Records(%v) = %v, want %v", test.q, got, test.want)
}
}
}
func TestMDNSService_serviceEnum_PTR(t *testing.T) {
s := makeService(t)
q := dns.Question{
Name: "_services._dns-sd._udp.local.",
Qtype: dns.TypePTR,
}
recs := s.Records(q)
if len(recs) != 1 {
t.Fatalf("bad: %v", recs)
}
if ptr, ok := recs[0].(*dns.PTR); !ok {
t.Errorf("recs[0] should be PTR record, got: %v, all records: %v", recs[0], recs)
} else if got, want := ptr.Ptr, "_http._tcp.local."; got != want {
t.Fatalf("bad PTR record %v: got %v, want %v", ptr, got, want)
}
}

44
registry/util.go → util/registry/util.go
View File

@ -1,7 +1,11 @@
package registry
func addNodes(old, neu []*Node) []*Node {
nodes := make([]*Node, len(neu))
import (
"github.com/micro/go-micro/v2/registry"
)
func addNodes(old, neu []*registry.Node) []*registry.Node {
nodes := make([]*registry.Node, len(neu))
// add all new nodes
for i, n := range neu {
node := *n
@ -31,8 +35,8 @@ func addNodes(old, neu []*Node) []*Node {
return nodes
}
func delNodes(old, del []*Node) []*Node {
var nodes []*Node
func delNodes(old, del []*registry.Node) []*registry.Node {
var nodes []*registry.Node
for _, o := range old {
var rem bool
for _, n := range del {
@ -49,24 +53,24 @@ func delNodes(old, del []*Node) []*Node {
}
// CopyService make a copy of service
func CopyService(service *Service) *Service {
func CopyService(service *registry.Service) *registry.Service {
// copy service
s := new(Service)
s := new(registry.Service)
*s = *service
// copy nodes
nodes := make([]*Node, len(service.Nodes))
nodes := make([]*registry.Node, len(service.Nodes))
for j, node := range service.Nodes {
n := new(Node)
n := new(registry.Node)
*n = *node
nodes[j] = n
}
s.Nodes = nodes
// copy endpoints
eps := make([]*Endpoint, len(service.Endpoints))
eps := make([]*registry.Endpoint, len(service.Endpoints))
for j, ep := range service.Endpoints {
e := new(Endpoint)
e := new(registry.Endpoint)
*e = *ep
eps[j] = e
}
@ -75,8 +79,8 @@ func CopyService(service *Service) *Service {
}
// Copy makes a copy of services
func Copy(current []*Service) []*Service {
services := make([]*Service, len(current))
func Copy(current []*registry.Service) []*registry.Service {
services := make([]*registry.Service, len(current))
for i, service := range current {
services[i] = CopyService(service)
}
@ -84,14 +88,14 @@ func Copy(current []*Service) []*Service {
}
// Merge merges two lists of services and returns a new copy
func Merge(olist []*Service, nlist []*Service) []*Service {
var srv []*Service
func Merge(olist []*registry.Service, nlist []*registry.Service) []*registry.Service {
var srv []*registry.Service
for _, n := range nlist {
var seen bool
for _, o := range olist {
if o.Version == n.Version {
sp := new(Service)
sp := new(registry.Service)
// make copy
*sp = *o
// set nodes
@ -102,25 +106,25 @@ func Merge(olist []*Service, nlist []*Service) []*Service {
srv = append(srv, sp)
break
} else {
sp := new(Service)
sp := new(registry.Service)
// make copy
*sp = *o
srv = append(srv, sp)
}
}
if !seen {
srv = append(srv, Copy([]*Service{n})...)
srv = append(srv, Copy([]*registry.Service{n})...)
}
}
return srv
}
// Remove removes services and returns a new copy
func Remove(old, del []*Service) []*Service {
var services []*Service
func Remove(old, del []*registry.Service) []*registry.Service {
var services []*registry.Service
for _, o := range old {
srv := new(Service)
srv := new(registry.Service)
*srv = *o
var rem bool

16
registry/util_test.go → util/registry/util_test.go
View File

@ -3,14 +3,16 @@ package registry
import (
"os"
"testing"
"github.com/micro/go-micro/v2/registry"
)
func TestRemove(t *testing.T) {
services := []*Service{
services := []*registry.Service{
{
Name: "foo",
Version: "1.0.0",
Nodes: []*Node{
Nodes: []*registry.Node{
{
Id: "foo-123",
Address: "localhost:9999",
@ -20,7 +22,7 @@ func TestRemove(t *testing.T) {
{
Name: "foo",
Version: "1.0.0",
Nodes: []*Node{
Nodes: []*registry.Node{
{
Id: "foo-123",
Address: "localhost:6666",
@ -29,7 +31,7 @@ func TestRemove(t *testing.T) {
},
}
servs := Remove([]*Service{services[0]}, []*Service{services[1]})
servs := Remove([]*registry.Service{services[0]}, []*registry.Service{services[1]})
if i := len(servs); i > 0 {
t.Errorf("Expected 0 nodes, got %d: %+v", i, servs)
}
@ -39,11 +41,11 @@ func TestRemove(t *testing.T) {
}
func TestRemoveNodes(t *testing.T) {
services := []*Service{
services := []*registry.Service{
{
Name: "foo",
Version: "1.0.0",
Nodes: []*Node{
Nodes: []*registry.Node{
{
Id: "foo-123",
Address: "localhost:9999",
@ -57,7 +59,7 @@ func TestRemoveNodes(t *testing.T) {
{
Name: "foo",
Version: "1.0.0",
Nodes: []*Node{
Nodes: []*registry.Node{
{
Id: "foo-123",
Address: "localhost:6666",

12
sync/store/manager.go → util/sync/manager.go
View File

@ -1,4 +1,4 @@
package store
package sync
import (
"time"
@ -25,7 +25,7 @@ const (
listOp
)
func (c *cache) cacheManager() {
func (c *syncStore) syncManager() {
tickerAggregator := make(chan struct{ index int })
for i, ticker := range c.pendingWriteTickers {
go func(index int, c chan struct{ index int }, t *time.Ticker) {
@ -43,18 +43,18 @@ func (c *cache) cacheManager() {
}
}
func (c *cache) processQueue(index int) {
func (c *syncStore) processQueue(index int) {
c.Lock()
defer c.Unlock()
q := c.pendingWrites[index]
for i := 0; i < q.Len(); i++ {
r, ok := q.PopFront()
if !ok {
panic(errors.Errorf("retrieved an invalid value from the L%d cache queue", index+1))
panic(errors.Errorf("retrieved an invalid value from the L%d sync queue", index+1))
}
ir, ok := r.(*internalRecord)
if !ok {
panic(errors.Errorf("retrieved a non-internal record from the L%d cache queue", index+1))
panic(errors.Errorf("retrieved a non-internal record from the L%d sync queue", index+1))
}
if !ir.expiresAt.IsZero() && time.Now().After(ir.expiresAt) {
continue
@ -68,7 +68,7 @@ func (c *cache) processQueue(index int) {
nr.Expiry = time.Until(ir.expiresAt)
}
// Todo = internal queue also has to hold the corresponding store.WriteOptions
if err := c.cOptions.Stores[index+1].Write(nr); err != nil {
if err := c.syncOpts.Stores[index+1].Write(nr); err != nil {
// some error, so queue for retry and bail
q.PushBack(ir)
return

12
sync/store/options.go → util/sync/options.go
View File

@ -1,4 +1,4 @@
package store
package sync
import (
"time"
@ -6,9 +6,9 @@ import (
"github.com/micro/go-micro/v2/store"
)
// Options represents Cache options
// Options represents Sync options
type Options struct {
// Stores represents layers in the cache in ascending order. L0, L1, L2, etc
// Stores represents layers in the sync in ascending order. L0, L1, L2, etc
Stores []store.Store
// SyncInterval is the duration between syncs from L0 to L1
SyncInterval time.Duration
@ -16,10 +16,10 @@ type Options struct {
SyncMultiplier int64
}
// Option sets Cache Options
// Option sets Sync Options
type Option func(o *Options)
// Stores sets the layers that make up the cache
// Stores sets the layers that make up the sync
func Stores(stores ...store.Store) Option {
return func(o *Options) {
o.Stores = make([]store.Store, len(stores))
@ -36,7 +36,7 @@ func SyncInterval(d time.Duration) Option {
}
}
// SyncMultiplier sets the multiplication factor for time to wait each cache layer
// SyncMultiplier sets the multiplication factor for time to wait each sync layer
func SyncMultiplier(i int64) Option {
return func(o *Options) {
o.SyncMultiplier = i

115
util/sync/sync.go Normal file
View File

@ -0,0 +1,115 @@
// Package syncs will sync multiple stores
package sync
import (
"fmt"
"sync"
"time"
"github.com/ef-ds/deque"
"github.com/micro/go-micro/v2/store"
"github.com/pkg/errors"
)
// Sync implements a sync in for stores
type Sync interface {
// Implements the store interface
store.Store
// Force a full sync
Sync() error
}
type syncStore struct {
storeOpts store.Options
syncOpts Options
pendingWrites []*deque.Deque
pendingWriteTickers []*time.Ticker
sync.RWMutex
}
// NewSync returns a new Sync
func NewSync(opts ...Option) Sync {
c := &syncStore{}
for _, o := range opts {
o(&c.syncOpts)
}
if c.syncOpts.SyncInterval == 0 {
c.syncOpts.SyncInterval = 1 * time.Minute
}
if c.syncOpts.SyncMultiplier == 0 {
c.syncOpts.SyncMultiplier = 5
}
return c
}
func (c *syncStore) Close() error {
return nil
}
// Init initialises the storeOptions
func (c *syncStore) Init(opts ...store.Option) error {
for _, o := range opts {
o(&c.storeOpts)
}
if len(c.syncOpts.Stores) == 0 {
return errors.New("the sync has no stores")
}
if c.storeOpts.Context == nil {
return errors.New("please provide a context to the sync. Cancelling the context signals that the sync is being disposed and syncs the sync")
}
for _, s := range c.syncOpts.Stores {
if err := s.Init(); err != nil {
return errors.Wrapf(err, "Store %s failed to Init()", s.String())
}
}
c.pendingWrites = make([]*deque.Deque, len(c.syncOpts.Stores)-1)
c.pendingWriteTickers = make([]*time.Ticker, len(c.syncOpts.Stores)-1)
for i := 0; i < len(c.pendingWrites); i++ {
c.pendingWrites[i] = deque.New()
c.pendingWrites[i].Init()
c.pendingWriteTickers[i] = time.NewTicker(c.syncOpts.SyncInterval * time.Duration(intpow(c.syncOpts.SyncMultiplier, int64(i))))
}
go c.syncManager()
return nil
}
// Options returns the sync's store options
func (c *syncStore) Options() store.Options {
return c.storeOpts
}
// String returns a printable string describing the sync
func (c *syncStore) String() string {
backends := make([]string, len(c.syncOpts.Stores))
for i, s := range c.syncOpts.Stores {
backends[i] = s.String()
}
return fmt.Sprintf("sync %v", backends)
}
func (c *syncStore) List(opts ...store.ListOption) ([]string, error) {
return c.syncOpts.Stores[0].List(opts...)
}
func (c *syncStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) {
return c.syncOpts.Stores[0].Read(key, opts...)
}
func (c *syncStore) Write(r *store.Record, opts ...store.WriteOption) error {
return c.syncOpts.Stores[0].Write(r, opts...)
}
// Delete removes a key from the sync
func (c *syncStore) Delete(key string, opts ...store.DeleteOption) error {
return c.syncOpts.Stores[0].Delete(key, opts...)
}
func (c *syncStore) Sync() error {
return nil
}
type internalRecord struct {
key string
value []byte
expiresAt time.Time
}