[WIP] K8s update and runtime package changes (#895)

* First commit: outline of K8s runtime package

* Added poller. Added auto-updater into default runtime

* Added build and updated Poller interface

* Added comments and NewRuntime that accepts Options

* DefaultPoller; Runtime options

* First commit to add Kubernetes cruft

* Add comments

* Add micro- prefix to K8s runtime service names

* Get rid of import cycles. Move K8s runtime into main runtime package

* Major refactoring: Poller replaced by Notifier

POller has been replaced by Notifier which returns a channel of events
that can be consumed and acted upon.

* Added runtime configuration options

* K8s runtime is now Kubernetes runtime in dedicated pkg. Naming kung-fu.

* Fix typo in command.

* Fixed typo

* Dont Delete service when runtime stops.

runtime.Stop stops services; no need to double-stop

* Track runtime services

* Parse Unix timestamps properly

* Added deployments into K8s client. Debug logging

runtime/kubernetes/client/api/request.go

@@ -0,0 +1,223 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/micro/go-micro/runtime/kubernetes/client/watch"
+	"github.com/micro/go-micro/util/log"
+)
+
+// Request is used to construct a http request for the k8s API.
+type Request struct {
+	client    *http.Client
+	header    http.Header
+	params    url.Values
+	method    string
+	host      string
+	namespace string
+
+	resource     string
+	resourceName *string
+	body         io.Reader
+
+	err error
+}
+
+// Params is the object to pass in to set paramaters
+// on a request.
+type Params struct {
+	LabelSelector map[string]string
+	Annotations   map[string]string
+	Watch         bool
+}
+
+// verb sets method
+func (r *Request) verb(method string) *Request {
+	r.method = method
+	return r
+}
+
+// Get request
+func (r *Request) Get() *Request {
+	return r.verb("GET")
+}
+
+// Post request
+func (r *Request) Post() *Request {
+	return r.verb("POST")
+}
+
+// Put request
+func (r *Request) Put() *Request {
+	return r.verb("PUT")
+}
+
+// Patch request
+// https://github.com/kubernetes/kubernetes/blob/master/docs/devel/api-conventions.md#patch-operations
+func (r *Request) Patch() *Request {
+	return r.verb("PATCH").SetHeader("Content-Type", "application/strategic-merge-patch+json")
+}
+
+// Delete request
+func (r *Request) Delete() *Request {
+	return r.verb("DELETE")
+}
+
+// Namespace is to set the namespace to operate on
+func (r *Request) Namespace(s string) *Request {
+	r.namespace = s
+	return r
+}
+
+// Resource is the type of resource the operation is
+// for, such as "services", "endpoints" or "pods"
+func (r *Request) Resource(s string) *Request {
+	r.resource = s
+	return r
+}
+
+// Name is for targeting a specific resource by id
+func (r *Request) Name(s string) *Request {
+	r.resourceName = &s
+	return r
+}
+
+// Body pass in a body to set, this is for POST, PUT
+// and PATCH requests
+func (r *Request) Body(in interface{}) *Request {
+	b := new(bytes.Buffer)
+	if err := json.NewEncoder(b).Encode(&in); err != nil {
+		r.err = err
+		return r
+	}
+	log.Debugf("Patch body: %v", b)
+	r.body = b
+	return r
+}
+
+// Params isused to set paramters on a request
+func (r *Request) Params(p *Params) *Request {
+	for k, v := range p.LabelSelector {
+		r.params.Add("labelSelectors", k+"="+v)
+	}
+
+	return r
+}
+
+// SetHeader sets a header on a request with
+// a `key` and `value`
+func (r *Request) SetHeader(key, value string) *Request {
+	r.header.Add(key, value)
+	return r
+}
+
+// request builds the http.Request from the options
+func (r *Request) request() (*http.Request, error) {
+	var url string
+	switch r.resource {
+	case "pods":
+		// /api/v1/namespaces/{namespace}/pods
+		url = fmt.Sprintf("%s/api/v1/namespaces/%s/%s/", r.host, r.namespace, r.resource)
+	case "deployments":
+		// /apis/apps/v1/namespaces/{namespace}/deployments/{name}
+		url = fmt.Sprintf("%s/apis/apps/v1/namespaces/%s/%s/", r.host, r.namespace, r.resource)
+	}
+
+	// append resourceName if it is present
+	if r.resourceName != nil {
+		url += *r.resourceName
+	}
+
+	// append any query params
+	if len(r.params) > 0 {
+		url += "?" + r.params.Encode()
+	}
+
+	// build request
+	req, err := http.NewRequest(r.method, url, r.body)
+	if err != nil {
+		return nil, err
+	}
+
+	// set headers on request
+	req.Header = r.header
+	return req, nil
+}
+
+// Do builds and triggers the request
+func (r *Request) Do() *Response {
+	if r.err != nil {
+		return &Response{
+			err: r.err,
+		}
+	}
+
+	req, err := r.request()
+	if err != nil {
+		return &Response{
+			err: err,
+		}
+	}
+
+	log.Debugf("kubernetes api request: %v", req)
+
+	res, err := r.client.Do(req)
+	if err != nil {
+		return &Response{
+			err: err,
+		}
+	}
+
+	log.Debugf("kubernetes api response: %v", res)
+
+	// return res, err
+	return newResponse(res, err)
+}
+
+// Watch builds and triggers the request, but
+// will watch instead of return an object
+func (r *Request) Watch() (watch.Watch, error) {
+	if r.err != nil {
+		return nil, r.err
+	}
+
+	r.params.Set("watch", "true")
+
+	req, err := r.request()
+	if err != nil {
+		return nil, err
+	}
+
+	w, err := watch.NewBodyWatcher(req, r.client)
+	return w, err
+}
+
+// Options ...
+type Options struct {
+	Host        string
+	Namespace   string
+	BearerToken *string
+	Client      *http.Client
+}
+
+// NewRequest creates a k8s api request
+func NewRequest(opts *Options) *Request {
+	req := &Request{
+		header:    make(http.Header),
+		params:    make(url.Values),
+		client:    opts.Client,
+		namespace: opts.Namespace,
+		host:      opts.Host,
+	}
+
+	if opts.BearerToken != nil {
+		req.SetHeader("Authorization", "Bearer "+*opts.BearerToken)
+	}
+
+	return req
+}

runtime/kubernetes/client/api/response.go

@@ -0,0 +1,94 @@
+package api
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/micro/go-micro/util/log"
+)
+
+// Errors ...
+var (
+	ErrNotFound = errors.New("kubernetes: not found")
+	ErrDecode   = errors.New("kubernetes: error decoding")
+	ErrOther    = errors.New("kubernetes: unknown error")
+)
+
+// Status is an object that is returned when a request
+// failed or delete succeeded.
+// type Status struct {
+// 	Kind    string `json:"kind"`
+// 	Status  string `json:"status"`
+// 	Message string `json:"message"`
+// 	Reason  string `json:"reason"`
+// 	Code    int    `json:"code"`
+// }
+
+// Response ...
+type Response struct {
+	res *http.Response
+	err error
+
+	body []byte
+}
+
+// Error returns an error
+func (r *Response) Error() error {
+	return r.err
+}
+
+// StatusCode returns status code for response
+func (r *Response) StatusCode() int {
+	return r.res.StatusCode
+}
+
+// Into decode body into `data`
+func (r *Response) Into(data interface{}) error {
+	if r.err != nil {
+		return r.err
+	}
+
+	defer r.res.Body.Close()
+	decoder := json.NewDecoder(r.res.Body)
+	err := decoder.Decode(&data)
+	if err != nil {
+		return ErrDecode
+	}
+
+	return r.err
+}
+
+func newResponse(res *http.Response, err error) *Response {
+	r := &Response{
+		res: res,
+		err: err,
+	}
+
+	if err != nil {
+		return r
+	}
+
+	if r.res.StatusCode == http.StatusOK ||
+		r.res.StatusCode == http.StatusCreated ||
+		r.res.StatusCode == http.StatusNoContent {
+		// Non error status code
+		return r
+	}
+
+	if r.res.StatusCode == http.StatusNotFound {
+		r.err = ErrNotFound
+		return r
+	}
+
+	log.Logf("kubernetes: request failed with code %v", r.res.StatusCode)
+
+	b, err := ioutil.ReadAll(r.res.Body)
+	if err == nil {
+		log.Log("kubernetes: request failed with body:")
+		log.Log(string(b))
+	}
+	r.err = ErrOther
+	return r
+}

runtime/kubernetes/client/client.go

@@ -0,0 +1,102 @@
+package client
+
+import (
+	"crypto/tls"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+
+	"github.com/micro/go-micro/runtime/kubernetes/client/api"
+	"github.com/micro/go-micro/util/log"
+)
+
+var (
+	serviceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount"
+	// ErrReadNamespace is returned when the names could not be read from service account
+	ErrReadNamespace = errors.New("Could not read namespace from service account secret")
+)
+
+// Client ...
+type client struct {
+	opts *api.Options
+}
+
+// NewClientInCluster should work similarily to the official api
+// NewInClient by setting up a client configuration for use within
+// a k8s pod.
+func NewClientInCluster() *client {
+	host := "https://" + os.Getenv("KUBERNETES_SERVICE_HOST") + ":" + os.Getenv("KUBERNETES_SERVICE_PORT")
+
+	s, err := os.Stat(serviceAccountPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if s == nil || !s.IsDir() {
+		log.Fatal(errors.New("no k8s service account found"))
+	}
+
+	token, err := ioutil.ReadFile(path.Join(serviceAccountPath, "token"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	t := string(token)
+
+	ns, err := detectNamespace()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	crt, err := CertPoolFromFile(path.Join(serviceAccountPath, "ca.crt"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	c := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: crt,
+			},
+			DisableCompression: true,
+		},
+	}
+
+	return &client{
+		opts: &api.Options{
+			Client:      c,
+			Host:        host,
+			Namespace:   ns,
+			BearerToken: &t,
+		},
+	}
+}
+
+func detectNamespace() (string, error) {
+	nsPath := path.Join(serviceAccountPath, "namespace")
+
+	// Make sure it's a file and we can read it
+	if s, e := os.Stat(nsPath); e != nil {
+		return "", e
+	} else if s.IsDir() {
+		return "", ErrReadNamespace
+	}
+
+	// Read the file, and cast to a string
+	if ns, e := ioutil.ReadFile(nsPath); e != nil {
+		return string(ns), e
+	} else {
+		return string(ns), nil
+	}
+}
+
+// UpdateDeployment
+func (c *client) UpdateDeployment(name string, body interface{}) error {
+	return api.NewRequest(c.opts).
+		Patch().
+		Resource("deployments").
+		Name(name).
+		Body(body).
+		Do().
+		Error()
+}

runtime/kubernetes/client/kubernetes.go

@@ -0,0 +1,12 @@
+package client
+
+// Kubernetes client
+type Kubernetes interface {
+	// UpdateDeployment patches deployment annotations with new metadata
+	UpdateDeployment(string, interface{}) error
+}
+
+// Metadata defines api request metadata
+type Metadata struct {
+	Annotations map[string]string `json:"annotations,omitempty"`
+}

runtime/kubernetes/client/utils.go

@@ -0,0 +1,74 @@
+package client
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io/ioutil"
+)
+
+// COPIED FROM
+// https://github.com/kubernetes/kubernetes/blob/7a725418af4661067b56506faabc2d44c6d7703a/pkg/util/crypto/crypto.go
+
+// CertPoolFromFile returns an x509.CertPool containing the certificates in the given PEM-encoded file.
+// Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates
+func CertPoolFromFile(filename string) (*x509.CertPool, error) {
+	certs, err := certificatesFromFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	pool := x509.NewCertPool()
+	for _, cert := range certs {
+		pool.AddCert(cert)
+	}
+	return pool, nil
+}
+
+// certificatesFromFile returns the x509.Certificates contained in the given PEM-encoded file.
+// Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates
+func certificatesFromFile(file string) ([]*x509.Certificate, error) {
+	if len(file) == 0 {
+		return nil, errors.New("error reading certificates from an empty filename")
+	}
+	pemBlock, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	certs, err := CertsFromPEM(pemBlock)
+	if err != nil {
+		return nil, fmt.Errorf("error reading %s: %s", file, err)
+	}
+	return certs, nil
+}
+
+// CertsFromPEM returns the x509.Certificates contained in the given PEM-encoded byte array
+// Returns an error if a certificate could not be parsed, or if the data does not contain any certificates
+func CertsFromPEM(pemCerts []byte) ([]*x509.Certificate, error) {
+	ok := false
+	certs := []*x509.Certificate{}
+	for len(pemCerts) > 0 {
+		var block *pem.Block
+		block, pemCerts = pem.Decode(pemCerts)
+		if block == nil {
+			break
+		}
+		// Only use PEM "CERTIFICATE" blocks without extra headers
+		if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
+			continue
+		}
+
+		cert, err := x509.ParseCertificate(block.Bytes)
+		if err != nil {
+			return certs, err
+		}
+
+		certs = append(certs, cert)
+		ok = true
+	}
+
+	if !ok {
+		return certs, errors.New("could not read any certificates")
+	}
+	return certs, nil
+}

runtime/kubernetes/client/watch/body.go

@@ -0,0 +1,92 @@
+package watch
+
+import (
+	"bufio"
+	"encoding/json"
+	"net/http"
+	"time"
+)
+
+// bodyWatcher scans the body of a request for chunks
+type bodyWatcher struct {
+	results chan Event
+	stop    chan struct{}
+	res     *http.Response
+	req     *http.Request
+}
+
+// Changes returns the results channel
+func (wr *bodyWatcher) ResultChan() <-chan Event {
+	return wr.results
+}
+
+// Stop cancels the request
+func (wr *bodyWatcher) Stop() {
+	select {
+	case <-wr.stop:
+		return
+	default:
+		close(wr.stop)
+		close(wr.results)
+	}
+}
+
+func (wr *bodyWatcher) stream() {
+	reader := bufio.NewReader(wr.res.Body)
+
+	// ignore first few messages from stream,
+	// as they are usually old.
+	ignore := true
+
+	go func() {
+		<-time.After(time.Second)
+		ignore = false
+	}()
+
+	go func() {
+		// stop the watcher
+		defer wr.Stop()
+
+		for {
+			// read a line
+			b, err := reader.ReadBytes('\n')
+			if err != nil {
+				return
+			}
+
+			// ignore for the first second
+			if ignore {
+				continue
+			}
+
+			// send the event
+			var event Event
+			if err := json.Unmarshal(b, &event); err != nil {
+				continue
+			}
+			wr.results <- event
+		}
+	}()
+}
+
+// NewBodyWatcher creates a k8s body watcher for
+// a given http request
+func NewBodyWatcher(req *http.Request, client *http.Client) (Watch, error) {
+	stop := make(chan struct{})
+	req.Cancel = stop
+
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	wr := &bodyWatcher{
+		results: make(chan Event),
+		stop:    stop,
+		req:     req,
+		res:     res,
+	}
+
+	go wr.stream()
+	return wr, nil
+}

runtime/kubernetes/client/watch/watch.go

@@ -0,0 +1,26 @@
+package watch
+
+import "encoding/json"
+
+// Watch ...
+type Watch interface {
+	Stop()
+	ResultChan() <-chan Event
+}
+
+// EventType defines the possible types of events.
+type EventType string
+
+// EventTypes used
+const (
+	Added    EventType = "ADDED"
+	Modified EventType = "MODIFIED"
+	Deleted  EventType = "DELETED"
+	Error    EventType = "ERROR"
+)
+
+// Event represents a single event to a watched resource.
+type Event struct {
+	Type   EventType       `json:"type"`
+	Object json.RawMessage `json:"object"`
+}

runtime/kubernetes/client/watch/watch_test.go

@@ -0,0 +1,71 @@
+package watch
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+var actions = []string{
+	`{"type": "create", "object":{"foo": "bar"}}`,
+	`{"type": "delete", INVALID}`,
+	`{"type": "update", "object":{"foo": {"foo": "bar"}}}`,
+	`{"type": "delete", "object":null}`,
+}
+
+func TestBodyWatcher(t *testing.T) {
+	// set up server with handler to flush strings from ch.
+	ch := make(chan string)
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		flusher, ok := w.(http.Flusher)
+		if !ok {
+			t.Fatal("expected ResponseWriter to be a flusher")
+		}
+
+		fmt.Fprintf(w, "\n")
+		flusher.Flush()
+
+		for v := range ch {
+			fmt.Fprintf(w, "%s\n", v)
+			flusher.Flush()
+			time.Sleep(10 * time.Millisecond)
+		}
+	}))
+	defer ts.Close()
+
+	req, err := http.NewRequest("GET", ts.URL, nil)
+	if err != nil {
+		t.Fatalf("did not expect NewRequest to return err: %v", err)
+	}
+
+	// setup body watcher
+	w, err := NewBodyWatcher(req, http.DefaultClient)
+	if err != nil {
+		t.Fatalf("did not expect NewBodyWatcher to return %v", err)
+	}
+
+	<-time.After(time.Second)
+
+	// send action strings in, and expect result back
+	ch <- actions[0]
+	if r := <-w.ResultChan(); r.Type != "create" {
+		t.Fatalf("expected result to be create")
+	}
+
+	ch <- actions[1] // should be ignored as its invalid json
+	ch <- actions[2]
+	if r := <-w.ResultChan(); r.Type != "update" {
+		t.Fatalf("expected result to be update")
+	}
+
+	ch <- actions[3]
+	if r := <-w.ResultChan(); r.Type != "delete" {
+		t.Fatalf("expected result to be delete")
+	}
+
+	// stop should clean up all channels.
+	w.Stop()
+	close(ch)
+}