diff --git a/.golangci.yml b/.golangci.yml index cef170f7c..2ef168198 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -26,7 +26,7 @@ issues: linters-settings: misspell: locale: US - #ignore-words: - # - someword + ignore-words: + - cancelled goimports: local-prefixes: go.opentelemetry.io diff --git a/example/otel-collector/go.sum b/example/otel-collector/go.sum index 7f08f906d..9df9798c7 100644 --- a/example/otel-collector/go.sum +++ b/example/otel-collector/go.sum @@ -414,7 +414,6 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= diff --git a/exporters/otlp/README.md b/exporters/otlp/README.md index 653465565..24da77069 100644 --- a/exporters/otlp/README.md +++ b/exporters/otlp/README.md @@ -3,13 +3,152 @@ [![GoDoc](https://godoc.org/go.opentelemetry.io/otel?status.svg)](https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp) -This exporter converts OpenTelemetry [SpanData](https://github.com/open-telemetry/opentelemetry-go/blob/6769330394f78192df01cb59299e9e0f2e5e977b/sdk/export/trace/trace.go#L49) -to OpenTelemetry Protocol [Span](https://github.com/open-telemetry/opentelemetry-proto/blob/c20698d5bb483cf05de1a7c0e134b7c57e359674/opentelemetry/proto/trace/v1/trace.proto#L46) -and exports them to OpenTelemetry Collector. +This exporter exports OpenTelemetry spans and metrics to the OpenTelemetry Collector. -## Installation +## Installation and Setup + +The exporter can be installed using standard `go` functionality. ```bash $ go get -u go.opentelemetry.io/otel/exporters/otlp ``` + +A new exporter can be created using the `NewExporter` function. + +```golang +package main + +import ( + "log" + + "go.opentelemetry.io/otel/exporters/otlp" + "go.opentelemetry.io/otel/sdk/metric/controller/push" + "go.opentelemetry.io/otel/sdk/metric/selector/simple" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +func main() { + exporter, err := otlp.NewExporter() // Configure as needed. + if err != nil { + log.Fatalf("failed to create exporter: %v", err) + } + defer func() { + err := exporter.Stop() + if err != nil { + log.Fatalf("failed to stop exporter: %v", err) + } + }() + + // Note: The exporter can also be used as a Batcher. E.g. + // traceProvider, err := sdktrace.NewProvider( + // sdktrace.WithBatcher(exporter, + // sdktrace.WithBatchTimeout(time.Second*15), + // sdktrace.WithMaxExportBatchSize(100), + // ), + // ) + traceProvider, err := sdktrace.NewProvider(sdktrace.WithSyncer(exporter)) + if err != nil { + log.Fatal("failed to create trace provider: %v", err) + } + + pusher := push.New(simple.NewWithExactDistribution(), exporter) + pusher.Start() + metricProvider := pusher.Provider() + + // Your code here ... +} +``` + +## Configuration + +Configurations options can be specified when creating a new exporter (`NewExporter`). + +### `WorkerCount(n uint)` + +Sets the number of Goroutines to use when processing telemetry. + + +### `WithInsecure()` + +Disables client transport security for the exporter's gRPC connection just like [`grpc.WithInsecure()`](https://pkg.go.dev/google.golang.org/grpc#WithInsecure) does. +By default, client security is required unless `WithInsecure` is used. + +### `WithAddress(addr string)` + +Sets the address that the exporter will connect to the collector on. +The default address the exporter connects to is `localhost:55680`. + +### `WithReconnectionPeriod(rp time.Duration)` + +Set the delay between connection attempts after failing to connect with the collector. + +### `WithCompressor(compressor string)` + +Set the compressor for the gRPC client to use when sending requests. +The compressor used needs to have been registered with `google.golang.org/grpc/encoding` prior to using here. +This can be done by `encoding.RegisterCompressor`. +Some compressors auto-register on import, such as gzip, which can be registered by calling `import _ "google.golang.org/grpc/encoding/gzip"`. + +### `WithHeaders(headers map[string]string)` + +Headers to send when the gRPC stream connection is instantiated. + +### `WithTLSCredentials(creds "google.golang.org/grpc/credentials".TransportCredentials)` + +TLS credentials to use when talking to the server. + +### `WithGRPCServiceConfig(serviceConfig string)` + +The default gRPC service config used when . + +By default, the exporter is configured to support [retries](#retries). + +```json +{ + "methodConfig":[{ + "name":[ + { "service":"opentelemetry.proto.collector.metrics.v1.MetricsService" }, + { "service":"opentelemetry.proto.collector.trace.v1.TraceService" } + ], + "waitForReady": true, + "retryPolicy":{ + "MaxAttempts":5, + "InitialBackoff":"0.3s", + "MaxBackoff":"5s", + "BackoffMultiplier":2, + "RetryableStatusCodes":[ + "UNAVAILABLE", + "CANCELLED", + "DEADLINE_EXCEEDED", + "RESOURCE_EXHAUSTED", + "ABORTED", + "OUT_OF_RANGE", + "UNAVAILABLE", + "DATA_LOSS" + ] + } + }] +} +``` + +### `WithGRPCDialOption(opts ..."google.golang.org/grpc".DialOption)` + +Additional `grpc.DialOption` to be used. + +These options take precedence over any other set by other parts of the configuration. + +## Retries + +The exporter will not, by default, retry failed requests to the collector. +However, it is configured in a way that it can easily be enable. + +To enable retries, the `GRPC_GO_RETRY` environment variable needs to be set to `on`. For example, + +``` +GRPC_GO_RETRY=on go run . +``` + +The [default service config](https://github.com/grpc/proposal/blob/master/A6-client-retries.md) used by default is defined to retry failed requests with exponential backoff (`0.3seconds * (2)^retry`) with [a max of `5` retries](https://github.com/open-telemetry/oteps/blob/be2a3fcbaa417ebbf5845cd485d34fdf0ab4a2a4/text/0035-opentelemetry-protocol.md#export-response)). + +These retries are only attempted for reponses that are [deemed "retry-able" by the collector](https://github.com/grpc/proposal/blob/master/A6-client-retries.md#validation-of-retrypolicy). diff --git a/exporters/otlp/go.mod b/exporters/otlp/go.mod index a15cd849d..1757c65e8 100644 --- a/exporters/otlp/go.mod +++ b/exporters/otlp/go.mod @@ -4,14 +4,16 @@ replace go.opentelemetry.io/otel => ../.. require ( github.com/gogo/protobuf v1.3.1 - github.com/golang/protobuf v1.3.4 // indirect + github.com/golang/protobuf v1.3.5 // indirect github.com/google/go-cmp v0.4.0 github.com/grpc-ecosystem/grpc-gateway v1.14.3 // indirect + github.com/kr/pretty v0.2.0 // indirect github.com/open-telemetry/opentelemetry-proto v0.3.0 github.com/stretchr/testify v1.4.0 go.opentelemetry.io/otel v0.6.0 golang.org/x/text v0.3.2 // indirect - google.golang.org/grpc v1.27.1 + google.golang.org/grpc v1.29.1 + gopkg.in/yaml.v2 v2.2.8 // indirect ) go 1.13 diff --git a/exporters/otlp/go.sum b/exporters/otlp/go.sum index 9efdf481d..905ac70bc 100644 --- a/exporters/otlp/go.sum +++ b/exporters/otlp/go.sum @@ -8,10 +8,13 @@ github.com/benbjohnson/clock v1.0.0/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiU github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= @@ -23,8 +26,9 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= @@ -37,6 +41,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -94,8 +100,11 @@ google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvx google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -103,5 +112,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/exporters/otlp/options.go b/exporters/otlp/options.go index d809ec0a2..cf86a081c 100644 --- a/exporters/otlp/options.go +++ b/exporters/otlp/options.go @@ -25,6 +25,40 @@ const ( DefaultCollectorPort uint16 = 55680 DefaultCollectorHost string = "localhost" DefaultNumWorkers uint = 1 + + // For more info on gRPC service configs: + // https://github.com/grpc/proposal/blob/master/A6-client-retries.md + // + // For more info on the RetryableStatusCodes we allow here: + // https://github.com/open-telemetry/oteps/blob/be2a3fcbaa417ebbf5845cd485d34fdf0ab4a2a4/text/0035-opentelemetry-protocol.md#export-response + // + // Note: MaxAttempts > 5 are treated as 5. See + // https://github.com/grpc/proposal/blob/master/A6-client-retries.md#validation-of-retrypolicy + // for more details. + DefaultGRPCServiceConfig = `{ + "methodConfig":[{ + "name":[ + { "service":"opentelemetry.proto.collector.metrics.v1.MetricsService" }, + { "service":"opentelemetry.proto.collector.trace.v1.TraceService" } + ], + "retryPolicy":{ + "MaxAttempts":5, + "InitialBackoff":"0.3s", + "MaxBackoff":"5s", + "BackoffMultiplier":2, + "RetryableStatusCodes":[ + "UNAVAILABLE", + "CANCELLED", + "DEADLINE_EXCEEDED", + "RESOURCE_EXHAUSTED", + "ABORTED", + "OUT_OF_RANGE", + "UNAVAILABLE", + "DATA_LOSS" + ] + } + }] +}` ) type ExporterOption func(*Config) @@ -34,6 +68,7 @@ type Config struct { collectorAddr string compressor string reconnectionPeriod time.Duration + grpcServiceConfig string grpcDialOptions []grpc.DialOption headers map[string]string clientCredentials credentials.TransportCredentials @@ -106,6 +141,13 @@ func WithTLSCredentials(creds credentials.TransportCredentials) ExporterOption { } } +// WithGRPCServiceConfig defines the default gRPC service config used. +func WithGRPCServiceConfig(serviceConfig string) ExporterOption { + return func(cfg *Config) { + cfg.grpcServiceConfig = serviceConfig + } +} + // WithGRPCDialOption opens support to any grpc.DialOption to be used. If it conflicts // with some other configuration the GRPC specified via the collector the ones here will // take preference since they are set last. diff --git a/exporters/otlp/otlp.go b/exporters/otlp/otlp.go index d0e83f944..0c4d9a30a 100644 --- a/exporters/otlp/otlp.go +++ b/exporters/otlp/otlp.go @@ -72,7 +72,10 @@ func NewExporter(opts ...ExporterOption) (*Exporter, error) { func NewUnstartedExporter(opts ...ExporterOption) *Exporter { e := new(Exporter) - e.c = Config{numWorkers: DefaultNumWorkers} + e.c = Config{ + numWorkers: DefaultNumWorkers, + grpcServiceConfig: DefaultGRPCServiceConfig, + } configureOptions(&e.c, opts...) // TODO (rghetia): add resources @@ -157,7 +160,11 @@ func (e *Exporter) enableConnections(cc *grpc.ClientConn) error { func (e *Exporter) dialToCollector() (*grpc.ClientConn, error) { addr := e.prepareCollectorAddress() - var dialOpts []grpc.DialOption + + dialOpts := []grpc.DialOption{} + if e.c.grpcServiceConfig != "" { + dialOpts = append(dialOpts, grpc.WithDefaultServiceConfig(e.c.grpcServiceConfig)) + } if e.c.clientCredentials != nil { dialOpts = append(dialOpts, grpc.WithTransportCredentials(e.c.clientCredentials)) } else if e.c.canDialInsecure {