2019-08-02 22:52:55 +02:00
|
|
|
// Copyright 2019, OpenTelemetry Authors
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package trace
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2020-02-28 23:44:53 +02:00
|
|
|
"fmt"
|
|
|
|
"reflect"
|
2019-08-02 22:52:55 +02:00
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
2019-08-26 18:41:15 +02:00
|
|
|
"google.golang.org/grpc/codes"
|
|
|
|
|
2019-11-01 20:40:29 +02:00
|
|
|
"go.opentelemetry.io/otel/api/core"
|
|
|
|
apitrace "go.opentelemetry.io/otel/api/trace"
|
2019-11-05 23:08:55 +02:00
|
|
|
export "go.opentelemetry.io/otel/sdk/export/trace"
|
2019-11-01 20:40:29 +02:00
|
|
|
"go.opentelemetry.io/otel/sdk/internal"
|
2019-08-02 22:52:55 +02:00
|
|
|
)
|
|
|
|
|
2020-02-28 23:44:53 +02:00
|
|
|
const (
|
|
|
|
errorTypeKey = core.Key("error.type")
|
|
|
|
errorMessageKey = core.Key("error.message")
|
|
|
|
errorEventName = "error"
|
|
|
|
)
|
|
|
|
|
2019-08-02 22:52:55 +02:00
|
|
|
// span implements apitrace.Span interface.
|
|
|
|
type span struct {
|
|
|
|
// data contains information recorded about the span.
|
|
|
|
//
|
|
|
|
// It will be non-nil if we are exporting the span or recording events for it.
|
|
|
|
// Otherwise, data is nil, and the span is simply a carrier for the
|
|
|
|
// SpanContext, so that the trace ID is propagated.
|
2019-10-08 20:56:58 +02:00
|
|
|
data *export.SpanData
|
2019-08-02 22:52:55 +02:00
|
|
|
mu sync.Mutex // protects the contents of *data (but not the pointer value.)
|
|
|
|
spanContext core.SpanContext
|
|
|
|
|
2019-11-18 20:51:57 +02:00
|
|
|
// attributes are capped at configured limit. When the capacity is reached an oldest entry
|
2019-08-02 22:52:55 +02:00
|
|
|
// is removed to create room for a new entry.
|
2019-11-18 20:51:57 +02:00
|
|
|
attributes *attributesMap
|
2019-08-02 22:52:55 +02:00
|
|
|
|
|
|
|
// messageEvents are stored in FIFO queue capped by configured limit.
|
|
|
|
messageEvents *evictedQueue
|
|
|
|
|
|
|
|
// links are stored in FIFO queue capped by configured limit.
|
|
|
|
links *evictedQueue
|
|
|
|
|
|
|
|
// spanStore is the spanStore this span belongs to, if any, otherwise it is nil.
|
|
|
|
//*spanStore
|
|
|
|
endOnce sync.Once
|
|
|
|
|
2019-10-22 22:19:11 +02:00
|
|
|
executionTracerTaskEnd func() // ends the execution tracer span
|
|
|
|
tracer *tracer // tracer used to create span.
|
2019-08-02 22:52:55 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
var _ apitrace.Span = &span{}
|
|
|
|
|
|
|
|
func (s *span) SpanContext() core.SpanContext {
|
|
|
|
if s == nil {
|
2019-08-12 19:59:35 +02:00
|
|
|
return core.EmptySpanContext()
|
2019-08-02 22:52:55 +02:00
|
|
|
}
|
|
|
|
return s.spanContext
|
|
|
|
}
|
|
|
|
|
2019-10-11 03:07:35 +02:00
|
|
|
func (s *span) IsRecording() bool {
|
2019-08-02 22:52:55 +02:00
|
|
|
if s == nil {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return s.data != nil
|
|
|
|
}
|
|
|
|
|
2020-03-07 20:38:59 +02:00
|
|
|
func (s *span) SetStatus(code codes.Code, msg string) {
|
2019-08-02 22:52:55 +02:00
|
|
|
if s == nil {
|
|
|
|
return
|
|
|
|
}
|
2019-10-11 03:07:35 +02:00
|
|
|
if !s.IsRecording() {
|
2019-08-02 22:52:55 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
s.mu.Lock()
|
2020-03-07 20:38:59 +02:00
|
|
|
s.data.StatusCode = code
|
|
|
|
s.data.StatusMessage = msg
|
2019-08-02 22:52:55 +02:00
|
|
|
s.mu.Unlock()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *span) SetAttributes(attributes ...core.KeyValue) {
|
2019-10-11 03:07:35 +02:00
|
|
|
if !s.IsRecording() {
|
2019-08-02 22:52:55 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
s.copyToCappedAttributes(attributes...)
|
|
|
|
}
|
|
|
|
|
2019-09-27 19:48:10 +02:00
|
|
|
func (s *span) End(options ...apitrace.EndOption) {
|
2019-08-02 22:52:55 +02:00
|
|
|
if s == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if s.executionTracerTaskEnd != nil {
|
|
|
|
s.executionTracerTaskEnd()
|
|
|
|
}
|
2019-10-11 03:07:35 +02:00
|
|
|
if !s.IsRecording() {
|
2019-08-02 22:52:55 +02:00
|
|
|
return
|
|
|
|
}
|
2019-12-04 23:41:13 +02:00
|
|
|
opts := apitrace.EndConfig{}
|
2019-09-03 20:03:51 +02:00
|
|
|
for _, opt := range options {
|
|
|
|
opt(&opts)
|
|
|
|
}
|
2019-08-02 22:52:55 +02:00
|
|
|
s.endOnce.Do(func() {
|
2019-10-22 22:19:11 +02:00
|
|
|
sps, _ := s.tracer.provider.spanProcessors.Load().(spanProcessorMap)
|
2019-10-08 20:56:58 +02:00
|
|
|
mustExportOrProcess := len(sps) > 0
|
2019-09-16 22:58:15 +02:00
|
|
|
if mustExportOrProcess {
|
2019-08-02 22:52:55 +02:00
|
|
|
sd := s.makeSpanData()
|
2019-09-27 19:48:10 +02:00
|
|
|
if opts.EndTime.IsZero() {
|
2019-09-03 20:03:51 +02:00
|
|
|
sd.EndTime = internal.MonotonicEndTime(sd.StartTime)
|
|
|
|
} else {
|
2019-09-27 19:48:10 +02:00
|
|
|
sd.EndTime = opts.EndTime
|
2019-09-03 20:03:51 +02:00
|
|
|
}
|
2019-09-16 22:58:15 +02:00
|
|
|
for sp := range sps {
|
|
|
|
sp.OnEnd(sd)
|
|
|
|
}
|
2019-08-02 22:52:55 +02:00
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2020-02-28 23:44:53 +02:00
|
|
|
func (s *span) RecordError(ctx context.Context, err error, opts ...apitrace.ErrorOption) {
|
|
|
|
if s == nil || err == nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if !s.IsRecording() {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
cfg := apitrace.ErrorConfig{}
|
|
|
|
|
|
|
|
for _, o := range opts {
|
|
|
|
o(&cfg)
|
|
|
|
}
|
|
|
|
|
|
|
|
if cfg.Timestamp.IsZero() {
|
|
|
|
cfg.Timestamp = time.Now()
|
|
|
|
}
|
|
|
|
|
2020-03-07 20:38:59 +02:00
|
|
|
if cfg.StatusCode != codes.OK {
|
|
|
|
s.SetStatus(cfg.StatusCode, "")
|
2020-02-28 23:44:53 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
errType := reflect.TypeOf(err)
|
|
|
|
errTypeString := fmt.Sprintf("%s.%s", errType.PkgPath(), errType.Name())
|
|
|
|
if errTypeString == "." {
|
|
|
|
// PkgPath() and Name() may be empty for builtin Types
|
|
|
|
errTypeString = errType.String()
|
|
|
|
}
|
|
|
|
|
|
|
|
s.AddEventWithTimestamp(ctx, cfg.Timestamp, errorEventName,
|
|
|
|
errorTypeKey.String(errTypeString),
|
|
|
|
errorMessageKey.String(err.Error()),
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2019-08-02 22:52:55 +02:00
|
|
|
func (s *span) Tracer() apitrace.Tracer {
|
|
|
|
return s.tracer
|
|
|
|
}
|
|
|
|
|
2019-12-18 20:13:05 +02:00
|
|
|
func (s *span) AddEvent(ctx context.Context, name string, attrs ...core.KeyValue) {
|
2019-10-11 03:07:35 +02:00
|
|
|
if !s.IsRecording() {
|
2019-08-02 22:52:55 +02:00
|
|
|
return
|
|
|
|
}
|
2019-12-18 20:13:05 +02:00
|
|
|
s.addEventWithTimestamp(time.Now(), name, attrs...)
|
2019-08-02 22:52:55 +02:00
|
|
|
}
|
|
|
|
|
2019-12-18 20:13:05 +02:00
|
|
|
func (s *span) AddEventWithTimestamp(ctx context.Context, timestamp time.Time, name string, attrs ...core.KeyValue) {
|
2019-10-11 03:07:35 +02:00
|
|
|
if !s.IsRecording() {
|
2019-09-25 08:12:22 +02:00
|
|
|
return
|
|
|
|
}
|
2019-12-18 20:13:05 +02:00
|
|
|
s.addEventWithTimestamp(timestamp, name, attrs...)
|
2019-09-25 08:12:22 +02:00
|
|
|
}
|
|
|
|
|
2019-12-18 20:13:05 +02:00
|
|
|
func (s *span) addEventWithTimestamp(timestamp time.Time, name string, attrs ...core.KeyValue) {
|
2019-08-02 22:52:55 +02:00
|
|
|
s.mu.Lock()
|
2019-09-03 20:03:51 +02:00
|
|
|
defer s.mu.Unlock()
|
2019-10-08 20:56:58 +02:00
|
|
|
s.messageEvents.add(export.Event{
|
2019-12-18 20:13:05 +02:00
|
|
|
Name: name,
|
2019-09-09 23:59:39 +02:00
|
|
|
Attributes: attrs,
|
|
|
|
Time: timestamp,
|
2019-08-02 22:52:55 +02:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2019-08-26 20:53:12 +02:00
|
|
|
func (s *span) SetName(name string) {
|
2019-10-23 00:25:33 +02:00
|
|
|
s.mu.Lock()
|
|
|
|
defer s.mu.Unlock()
|
|
|
|
|
2019-08-26 20:53:12 +02:00
|
|
|
if s.data == nil {
|
|
|
|
// TODO: now what?
|
|
|
|
return
|
|
|
|
}
|
2020-01-14 14:54:48 +02:00
|
|
|
s.data.Name = name
|
2019-08-26 20:53:12 +02:00
|
|
|
// SAMPLING
|
2019-10-28 19:05:06 +02:00
|
|
|
noParent := !s.data.ParentSpanID.IsValid()
|
2019-08-26 20:53:12 +02:00
|
|
|
var ctx core.SpanContext
|
|
|
|
if noParent {
|
|
|
|
ctx = core.EmptySpanContext()
|
|
|
|
} else {
|
|
|
|
// FIXME: Where do we get the parent context from?
|
|
|
|
// From SpanStore?
|
|
|
|
ctx = s.data.SpanContext
|
|
|
|
}
|
|
|
|
data := samplingData{
|
|
|
|
noParent: noParent,
|
|
|
|
remoteParent: s.data.HasRemoteParent,
|
|
|
|
parent: ctx,
|
2020-01-14 14:54:48 +02:00
|
|
|
name: name,
|
2019-10-22 22:19:11 +02:00
|
|
|
cfg: s.tracer.provider.config.Load().(*Config),
|
2019-08-26 20:53:12 +02:00
|
|
|
span: s,
|
2020-03-10 17:25:11 +02:00
|
|
|
attributes: s.data.Attributes,
|
|
|
|
links: s.data.Links,
|
|
|
|
kind: s.data.SpanKind,
|
|
|
|
}
|
|
|
|
sampled := makeSamplingDecision(data)
|
|
|
|
|
|
|
|
// Adding attributes directly rather than using s.SetAttributes()
|
|
|
|
// as s.mu is already locked and attempting to do so would deadlock.
|
|
|
|
for _, a := range sampled.Attributes {
|
|
|
|
s.attributes.add(a)
|
2019-08-26 20:53:12 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-11-26 23:03:07 +02:00
|
|
|
func (s *span) addLink(link apitrace.Link) {
|
2019-10-11 03:07:35 +02:00
|
|
|
if !s.IsRecording() {
|
2019-09-21 09:26:20 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
s.mu.Lock()
|
|
|
|
defer s.mu.Unlock()
|
|
|
|
s.links.add(link)
|
|
|
|
}
|
|
|
|
|
2019-08-02 22:52:55 +02:00
|
|
|
// makeSpanData produces a SpanData representing the current state of the span.
|
|
|
|
// It requires that s.data is non-nil.
|
2019-10-08 20:56:58 +02:00
|
|
|
func (s *span) makeSpanData() *export.SpanData {
|
|
|
|
var sd export.SpanData
|
2019-08-02 22:52:55 +02:00
|
|
|
s.mu.Lock()
|
|
|
|
defer s.mu.Unlock()
|
|
|
|
sd = *s.data
|
2019-11-18 20:51:57 +02:00
|
|
|
|
|
|
|
s.attributes.toSpanData(&sd)
|
|
|
|
|
2019-08-02 22:52:55 +02:00
|
|
|
if len(s.messageEvents.queue) > 0 {
|
|
|
|
sd.MessageEvents = s.interfaceArrayToMessageEventArray()
|
|
|
|
sd.DroppedMessageEventCount = s.messageEvents.droppedCount
|
|
|
|
}
|
2019-09-21 09:26:20 +02:00
|
|
|
if len(s.links.queue) > 0 {
|
|
|
|
sd.Links = s.interfaceArrayToLinksArray()
|
|
|
|
sd.DroppedLinkCount = s.links.droppedCount
|
|
|
|
}
|
2019-08-02 22:52:55 +02:00
|
|
|
return &sd
|
|
|
|
}
|
|
|
|
|
2019-09-21 09:26:20 +02:00
|
|
|
func (s *span) interfaceArrayToLinksArray() []apitrace.Link {
|
|
|
|
linkArr := make([]apitrace.Link, 0)
|
|
|
|
for _, value := range s.links.queue {
|
|
|
|
linkArr = append(linkArr, value.(apitrace.Link))
|
|
|
|
}
|
|
|
|
return linkArr
|
|
|
|
}
|
|
|
|
|
2019-10-08 20:56:58 +02:00
|
|
|
func (s *span) interfaceArrayToMessageEventArray() []export.Event {
|
|
|
|
messageEventArr := make([]export.Event, 0)
|
2019-08-02 22:52:55 +02:00
|
|
|
for _, value := range s.messageEvents.queue {
|
2019-10-08 20:56:58 +02:00
|
|
|
messageEventArr = append(messageEventArr, value.(export.Event))
|
2019-08-02 22:52:55 +02:00
|
|
|
}
|
|
|
|
return messageEventArr
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *span) copyToCappedAttributes(attributes ...core.KeyValue) {
|
|
|
|
s.mu.Lock()
|
|
|
|
defer s.mu.Unlock()
|
|
|
|
for _, a := range attributes {
|
2019-11-18 20:51:57 +02:00
|
|
|
s.attributes.add(a)
|
2019-08-02 22:52:55 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *span) addChild() {
|
2019-10-11 03:07:35 +02:00
|
|
|
if !s.IsRecording() {
|
2019-08-02 22:52:55 +02:00
|
|
|
return
|
|
|
|
}
|
|
|
|
s.mu.Lock()
|
|
|
|
s.data.ChildSpanCount++
|
|
|
|
s.mu.Unlock()
|
|
|
|
}
|
|
|
|
|
2019-12-04 23:41:13 +02:00
|
|
|
func startSpanInternal(tr *tracer, name string, parent core.SpanContext, remoteParent bool, o apitrace.StartConfig) *span {
|
2019-08-02 22:52:55 +02:00
|
|
|
var noParent bool
|
|
|
|
span := &span{}
|
|
|
|
span.spanContext = parent
|
|
|
|
|
2019-10-22 22:19:11 +02:00
|
|
|
cfg := tr.provider.config.Load().(*Config)
|
2019-08-02 22:52:55 +02:00
|
|
|
|
2019-08-12 19:59:35 +02:00
|
|
|
if parent == core.EmptySpanContext() {
|
2019-08-02 22:52:55 +02:00
|
|
|
span.spanContext.TraceID = cfg.IDGenerator.NewTraceID()
|
|
|
|
noParent = true
|
|
|
|
}
|
|
|
|
span.spanContext.SpanID = cfg.IDGenerator.NewSpanID()
|
2019-08-26 20:53:12 +02:00
|
|
|
data := samplingData{
|
|
|
|
noParent: noParent,
|
|
|
|
remoteParent: remoteParent,
|
|
|
|
parent: parent,
|
|
|
|
name: name,
|
|
|
|
cfg: cfg,
|
|
|
|
span: span,
|
2020-03-10 17:25:11 +02:00
|
|
|
attributes: o.Attributes,
|
|
|
|
links: o.Links,
|
|
|
|
kind: o.SpanKind,
|
2019-08-02 22:52:55 +02:00
|
|
|
}
|
2020-03-10 17:25:11 +02:00
|
|
|
sampled := makeSamplingDecision(data)
|
2019-08-02 22:52:55 +02:00
|
|
|
|
|
|
|
// TODO: [rghetia] restore when spanstore is added.
|
2019-10-11 03:07:35 +02:00
|
|
|
// if !internal.LocalSpanStoreEnabled && !span.spanContext.IsSampled() && !o.Record {
|
|
|
|
if !span.spanContext.IsSampled() && !o.Record {
|
2019-08-02 22:52:55 +02:00
|
|
|
return span
|
|
|
|
}
|
|
|
|
|
2019-09-03 20:03:51 +02:00
|
|
|
startTime := o.StartTime
|
|
|
|
if startTime.IsZero() {
|
|
|
|
startTime = time.Now()
|
|
|
|
}
|
2019-10-08 20:56:58 +02:00
|
|
|
span.data = &export.SpanData{
|
2019-10-24 01:25:14 +02:00
|
|
|
SpanContext: span.spanContext,
|
|
|
|
StartTime: startTime,
|
2019-11-05 02:05:43 +02:00
|
|
|
SpanKind: apitrace.ValidateSpanKind(o.SpanKind),
|
2019-08-02 22:52:55 +02:00
|
|
|
Name: name,
|
|
|
|
HasRemoteParent: remoteParent,
|
2020-03-13 22:07:36 +02:00
|
|
|
Resource: cfg.Resource,
|
2019-08-02 22:52:55 +02:00
|
|
|
}
|
2019-11-18 20:51:57 +02:00
|
|
|
span.attributes = newAttributesMap(cfg.MaxAttributesPerSpan)
|
2019-08-02 22:52:55 +02:00
|
|
|
span.messageEvents = newEvictedQueue(cfg.MaxEventsPerSpan)
|
|
|
|
span.links = newEvictedQueue(cfg.MaxLinksPerSpan)
|
|
|
|
|
2020-03-10 17:25:11 +02:00
|
|
|
span.SetAttributes(sampled.Attributes...)
|
|
|
|
|
2019-08-02 22:52:55 +02:00
|
|
|
if !noParent {
|
|
|
|
span.data.ParentSpanID = parent.SpanID
|
|
|
|
}
|
|
|
|
// TODO: [rghetia] restore when spanstore is added.
|
|
|
|
//if internal.LocalSpanStoreEnabled {
|
|
|
|
// ss := spanStoreForNameCreateIfNew(name)
|
|
|
|
// if ss != nil {
|
|
|
|
// span.spanStore = ss
|
|
|
|
// ss.add(span)
|
|
|
|
// }
|
|
|
|
//}
|
|
|
|
|
|
|
|
return span
|
|
|
|
}
|
2019-08-26 20:53:12 +02:00
|
|
|
|
|
|
|
type samplingData struct {
|
|
|
|
noParent bool
|
|
|
|
remoteParent bool
|
|
|
|
parent core.SpanContext
|
|
|
|
name string
|
|
|
|
cfg *Config
|
|
|
|
span *span
|
2020-03-10 17:25:11 +02:00
|
|
|
attributes []core.KeyValue
|
|
|
|
links []apitrace.Link
|
|
|
|
kind apitrace.SpanKind
|
2019-08-26 20:53:12 +02:00
|
|
|
}
|
|
|
|
|
2020-03-10 17:25:11 +02:00
|
|
|
func makeSamplingDecision(data samplingData) SamplingResult {
|
2019-08-26 20:53:12 +02:00
|
|
|
if data.noParent || data.remoteParent {
|
|
|
|
// If this span is the child of a local span and no
|
|
|
|
// Sampler is set in the options, keep the parent's
|
2019-09-25 23:37:36 +02:00
|
|
|
// TraceFlags.
|
2019-08-26 20:53:12 +02:00
|
|
|
//
|
|
|
|
// Otherwise, consult the Sampler in the options if it
|
|
|
|
// is non-nil, otherwise the default sampler.
|
|
|
|
sampler := data.cfg.DefaultSampler
|
|
|
|
//if o.Sampler != nil {
|
|
|
|
// sampler = o.Sampler
|
|
|
|
//}
|
|
|
|
spanContext := &data.span.spanContext
|
2020-03-10 17:25:11 +02:00
|
|
|
sampled := sampler.ShouldSample(SamplingParameters{
|
2019-08-26 20:53:12 +02:00
|
|
|
ParentContext: data.parent,
|
|
|
|
TraceID: spanContext.TraceID,
|
|
|
|
SpanID: spanContext.SpanID,
|
|
|
|
Name: data.name,
|
2020-03-10 17:25:11 +02:00
|
|
|
HasRemoteParent: data.remoteParent,
|
|
|
|
Kind: data.kind,
|
|
|
|
Attributes: data.attributes,
|
|
|
|
Links: data.links,
|
|
|
|
})
|
|
|
|
if sampled.Decision == RecordAndSampled {
|
2019-09-25 23:37:36 +02:00
|
|
|
spanContext.TraceFlags |= core.TraceFlagsSampled
|
2019-08-26 20:53:12 +02:00
|
|
|
} else {
|
2019-09-25 23:37:36 +02:00
|
|
|
spanContext.TraceFlags &^= core.TraceFlagsSampled
|
2019-08-26 20:53:12 +02:00
|
|
|
}
|
2020-03-10 17:25:11 +02:00
|
|
|
return sampled
|
|
|
|
}
|
|
|
|
if data.parent.TraceFlags&core.TraceFlagsSampled != 0 {
|
|
|
|
return SamplingResult{Decision: RecordAndSampled}
|
2019-08-26 20:53:12 +02:00
|
|
|
}
|
2020-03-10 17:25:11 +02:00
|
|
|
return SamplingResult{Decision: NotRecord}
|
2019-08-26 20:53:12 +02:00
|
|
|
}
|