From 94bfa840e85b3b480beeebef6b36d6e0c3cf9e42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Mon, 19 Oct 2020 14:55:20 +0200 Subject: [PATCH] Add initial telemetry code, system_settings table and the task scheduler --- config.json | 3 +- server/go.mod | 7 + server/go.sum | 15 ++ server/server/server.go | 64 +++++++- server/services/config/config.go | 10 +- server/services/scheduler/scheduler.go | 77 +++++++++ .../sqlstore/migrations/postgres/bindata.go | 24 +++ .../000002_system_settings_table.down.sql | 1 + .../000002_system_settings_table.up.sql | 5 + .../sqlstore/migrations/sqlite/bindata.go | 24 +++ .../000002_system_settings_table.down.sql | 1 + .../000002_system_settings_table.up.sql | 5 + server/services/store/sqlstore/system.go | 35 +++++ server/services/store/store.go | 2 + .../services/telemetry/mocks/ServerIface.go | 147 ++++++++++++++++++ server/services/telemetry/telemetry.go | 142 +++++++++++++++++ 16 files changed, 551 insertions(+), 11 deletions(-) create mode 100644 server/services/scheduler/scheduler.go create mode 100644 server/services/store/sqlstore/migrations/postgres_files/000002_system_settings_table.down.sql create mode 100644 server/services/store/sqlstore/migrations/postgres_files/000002_system_settings_table.up.sql create mode 100644 server/services/store/sqlstore/migrations/sqlite_files/000002_system_settings_table.down.sql create mode 100644 server/services/store/sqlstore/migrations/sqlite_files/000002_system_settings_table.up.sql create mode 100644 server/services/store/sqlstore/system.go create mode 100644 server/services/telemetry/mocks/ServerIface.go create mode 100644 server/services/telemetry/telemetry.go diff --git a/config.json b/config.json index 30ff1f6c3..e3441fb84 100644 --- a/config.json +++ b/config.json @@ -6,5 +6,6 @@ "postgres_dbconfig": "dbname=octo sslmode=disable", "useSSL": false, "webpath": "./pack", - "filespath": "./files" + "filespath": "./files", + "telemetry": true } diff --git a/server/go.mod b/server/go.mod index 02fd6413a..3f7439b43 100644 --- a/server/go.mod +++ b/server/go.mod @@ -3,18 +3,25 @@ module github.com/mattermost/mattermost-octo-tasks/server go 1.15 require ( + github.com/go-ldap/ldap v3.0.3+incompatible // indirect github.com/golang-migrate/migrate v3.5.4+incompatible github.com/golang-migrate/migrate/v4 v4.13.0 + github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.4.2 github.com/jteeuwen/go-bindata v3.0.7+incompatible // indirect github.com/kr/text v0.2.0 // indirect github.com/lib/pq v1.8.0 + github.com/mattermost/mattermost-server v5.11.1+incompatible github.com/mattermost/mattermost-server/v5 v5.28.0 github.com/mattn/go-sqlite3 v2.0.3+incompatible + github.com/nicksnyder/go-i18n v1.10.1 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect + github.com/rudderlabs/analytics-go v3.2.1+incompatible github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.6.1 + go.uber.org/zap v1.15.0 golang.org/x/tools v0.0.0-20201017001424-6003fad69a88 // indirect + gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect ) diff --git a/server/go.sum b/server/go.sum index 1aa995a8f..b4f0b7f8f 100644 --- a/server/go.sum +++ b/server/go.sum @@ -264,6 +264,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-ldap/ldap v3.0.3+incompatible h1:HTeSZO8hWMS1Rgb2Ziku6b8a7qRIZZMHjsvuZyatzwk= +github.com/go-ldap/ldap v3.0.3+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= @@ -430,6 +432,7 @@ github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hashicorp/yamux v0.0.0-20200609203250-aecfd211c9ce h1:7UnVY3T/ZnHUrfviiAgIUjg2PXxsQfs5bphsG8F7Keo= github.com/hashicorp/yamux v0.0.0-20200609203250-aecfd211c9ce/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= @@ -550,12 +553,16 @@ github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQ github.com/marstr/guid v0.0.0-20170427235115-8bdf7d1a087c/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattermost/go-i18n v1.11.0 h1:1hLKqn/ZvhZ80OekjVPGYcCrBfMz+YxNNgqS+beL7zE= github.com/mattermost/go-i18n v1.11.0/go.mod h1:RyS7FDNQlzF1PsjbJWHRI35exqaKGSO9qD4iv8QjE34= +github.com/mattermost/gorp v1.6.2-0.20200624165429-2595d5e54111 h1:32+lDqjkY8hvcSnBAWEeHgigfd9FF3zWhfiDdH9j1Ko= github.com/mattermost/gorp v1.6.2-0.20200624165429-2595d5e54111/go.mod h1:QCQ3U0M9T/BlAdjKFJo0I1oe/YAgbyjNdhU8bpOLafk= github.com/mattermost/gosaml2 v0.3.2/go.mod h1:Z429EIOiEi9kbq6yHoApfzlcXpa6dzRDc6pO+Vy2Ksk= github.com/mattermost/ldap v0.0.0-20191128190019-9f62ba4b8d4d h1:2DV7VIlEv6J5R5o6tUcb3ZMKJYeeZuWZL7Rv1m23TgQ= github.com/mattermost/ldap v0.0.0-20191128190019-9f62ba4b8d4d/go.mod h1:HLbgMEI5K131jpxGazJ97AxfPDt31osq36YS1oxFQPQ= github.com/mattermost/logr v1.0.13 h1:6F/fM3csvH6Oy5sUpJuW7YyZSzZZAhJm5VcgKMxA2P8= github.com/mattermost/logr v1.0.13/go.mod h1:Mt4DPu1NXMe6JxPdwCC0XBoxXmN9eXOIRPoZarU2PXs= +github.com/mattermost/mattermost-server v1.4.0 h1:bAN0zYgjyhXPy67VTiHLg+bu8mDJ2bhx109BKk2Ddos= +github.com/mattermost/mattermost-server v5.11.1+incompatible h1:LPzKY0+2Tic/ik67qIg6VrydRCgxNXZQXOeaiJ2rMBY= +github.com/mattermost/mattermost-server v5.11.1+incompatible/go.mod h1:5L6MjAec+XXQwMIt791Ganu45GKsSiM+I0tLR9wUj8Y= github.com/mattermost/mattermost-server/v5 v5.28.0 h1:BU0cfZ3UdSh7DhbdF1XtS0Cj0HMGSJT8B6MsUKgPjks= github.com/mattermost/mattermost-server/v5 v5.28.0/go.mod h1:9FfgZY9Ywx64bzPBYo4mmR05ApyOxO+tr43eDhpWups= github.com/mattermost/rsc v0.0.0-20160330161541-bbaefb05eaa0 h1:G9tL6JXRBMzjuD1kkBtcnd42kUiT6QDwxfFYu7adM6o= @@ -612,6 +619,7 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -654,12 +662,15 @@ github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba/go.mod h1: github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= +github.com/nicksnyder/go-i18n v1.10.1 h1:isfg77E/aCD7+0lD/D00ebR2MV5vgeQ276WYyDaCRQc= +github.com/nicksnyder/go-i18n v1.10.1/go.mod h1:e4Di5xjP9oTVrC6y3C7C0HoSYXjSbhh/dU0eUV32nB4= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -769,6 +780,7 @@ github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/rudderlabs/analytics-go v3.2.1+incompatible h1:XDocL6elYIi8WhLXLklDahq+Ws3FAYVOvJSsMuYWaKk= github.com/rudderlabs/analytics-go v3.2.1+incompatible/go.mod h1:LF8/ty9kUX4PTY3l5c97K3nZZaX5Hwsvt+NBaRL/f30= github.com/russellhaering/goxmldsig v0.0.0-20180430223755-7acd5e4a6ef7/go.mod h1:Oz4y6ImuOQZxynhbSXk7btjEfNBtGlj2dcaOvXl2FSM= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -860,6 +872,7 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -1347,6 +1360,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/server/server/server.go b/server/server/server.go index 26fa296d9..e21c47d49 100644 --- a/server/server/server.go +++ b/server/server/server.go @@ -5,28 +5,42 @@ import ( "log" "os" "os/signal" + "runtime" + + "github.com/google/uuid" + "go.uber.org/zap" "github.com/mattermost/mattermost-octo-tasks/server/api" "github.com/mattermost/mattermost-octo-tasks/server/app" "github.com/mattermost/mattermost-octo-tasks/server/services/config" "github.com/mattermost/mattermost-octo-tasks/server/services/store" "github.com/mattermost/mattermost-octo-tasks/server/services/store/sqlstore" + "github.com/mattermost/mattermost-octo-tasks/server/services/telemetry" "github.com/mattermost/mattermost-octo-tasks/server/web" "github.com/mattermost/mattermost-octo-tasks/server/ws" "github.com/mattermost/mattermost-server/v5/model" "github.com/mattermost/mattermost-server/v5/services/filesstore" ) +const CurrentVersion = "0.0.1" + type Server struct { config *config.Configuration wsServer *ws.WSServer webServer *web.WebServer store store.Store filesBackend filesstore.FileBackend + telemetry *telemetry.TelemetryService + logger *zap.Logger } -func New(config *config.Configuration) (*Server, error) { - store, err := sqlstore.New(config.DBType, config.DBConfigString) +func New(cfg *config.Configuration) (*Server, error) { + logger, err := zap.NewProduction() + if err != nil { + return nil, err + } + + store, err := sqlstore.New(cfg.DBType, cfg.DBConfigString) if err != nil { log.Fatal("Unable to start the database", err) return nil, err @@ -36,16 +50,16 @@ func New(config *config.Configuration) (*Server, error) { filesBackendSettings := model.FileSettings{} filesBackendSettings.SetDefaults(false) - filesBackendSettings.Directory = &config.FilesPath + filesBackendSettings.Directory = &cfg.FilesPath filesBackend, appErr := filesstore.NewFileBackend(&filesBackendSettings, false) if appErr != nil { log.Fatal("Unable to initialize the files storage") return nil, errors.New("unable to initialize the files storage") } - appBuilder := func() *app.App { return app.New(config, store, wsServer, filesBackend) } + appBuilder := func() *app.App { return app.New(cfg, store, wsServer, filesBackend) } - webServer := web.NewWebServer(config.WebPath, config.Port, config.UseSSL) + webServer := web.NewWebServer(cfg.WebPath, cfg.Port, cfg.UseSSL) api := api.NewAPI(appBuilder) webServer.AddRoutes(wsServer) webServer.AddRoutes(api) @@ -63,13 +77,47 @@ func New(config *config.Configuration) (*Server, error) { } }() - return &Server{ - config: config, + // Init telemetry + settings, err := store.GetSystemSettings() + if err != nil { + return nil, err + } + + telemetryID := settings["TelemetryID"] + if len(telemetryID) == 0 { + telemetryID = uuid.New().String() + err := store.SetSystemSetting("TelemetryID", uuid.New().String()) + if err != nil { + return nil, err + } + } + + telemetryService := telemetry.New(telemetryID, zap.NewStdLog(logger)) + srv := &Server{ + config: cfg, wsServer: wsServer, webServer: webServer, store: store, filesBackend: filesBackend, - }, nil + telemetry: telemetryService, + logger: logger, + } + telemetryService.RegisterTracker("server", func() map[string]interface{} { + return map[string]interface{}{ + "version": CurrentVersion, + "operating_system": runtime.GOOS, + } + }) + telemetryService.RegisterTracker("config", func() map[string]interface{} { + return map[string]interface{}{ + "serverRoot": srv.config.ServerRoot == config.DefaultServerRoot, + "port": srv.config.Port == config.DefaultPort, + "useSSL": srv.config.UseSSL, + "dbType": srv.config.DBType, + } + }) + + return srv, nil } func (s *Server) Start() error { diff --git a/server/services/config/config.go b/server/services/config/config.go index 4279f24b9..bc9fffc00 100644 --- a/server/services/config/config.go +++ b/server/services/config/config.go @@ -6,6 +6,11 @@ import ( "github.com/spf13/viper" ) +const ( + DefaultServerRoot = "http://localhost:8000" + DefaultPort = 8000 +) + // Configuration is the app configuration stored in a json file type Configuration struct { ServerRoot string `json:"serverRoot" mapstructure:"serverRoot"` @@ -15,14 +20,15 @@ type Configuration struct { UseSSL bool `json:"useSSL" mapstructure:"useSSL"` WebPath string `json:"webpath" mapstructure:"webpath"` FilesPath string `json:"filespath" mapstructure:"filespath"` + Telemetry bool `json:"telemetry" mapstructure:"telemetry"` } func ReadConfigFile() (*Configuration, error) { viper.SetConfigName("config") // name of config file (without extension) viper.SetConfigType("json") // REQUIRED if the config file does not have the extension in the name viper.AddConfigPath(".") // optionally look for config in the working directory - viper.SetDefault("ServerRoot", "http://localhost:8000") - viper.SetDefault("Port", 8000) + viper.SetDefault("ServerRoot", DefaultServerRoot) + viper.SetDefault("Port", DefaultPort) viper.SetDefault("DBType", "sqlite3") viper.SetDefault("DBConfigString", "./octo.db") viper.SetDefault("WebPath", "./pack") diff --git a/server/services/scheduler/scheduler.go b/server/services/scheduler/scheduler.go new file mode 100644 index 000000000..8d9439926 --- /dev/null +++ b/server/services/scheduler/scheduler.go @@ -0,0 +1,77 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package scheduler + +import ( + "fmt" + "time" +) + +type TaskFunc func() + +type ScheduledTask struct { + Name string `json:"name"` + Interval time.Duration `json:"interval"` + Recurring bool `json:"recurring"` + function func() + cancel chan struct{} + cancelled chan struct{} +} + +func CreateTask(name string, function TaskFunc, timeToExecution time.Duration) *ScheduledTask { + return createTask(name, function, timeToExecution, false) +} + +func CreateRecurringTask(name string, function TaskFunc, interval time.Duration) *ScheduledTask { + return createTask(name, function, interval, true) +} + +func createTask(name string, function TaskFunc, interval time.Duration, recurring bool) *ScheduledTask { + task := &ScheduledTask{ + Name: name, + Interval: interval, + Recurring: recurring, + function: function, + cancel: make(chan struct{}), + cancelled: make(chan struct{}), + } + + go func() { + defer close(task.cancelled) + + ticker := time.NewTicker(interval) + defer func() { + ticker.Stop() + }() + + for { + select { + case <-ticker.C: + function() + case <-task.cancel: + return + } + + if !task.Recurring { + break + } + } + }() + + return task +} + +func (task *ScheduledTask) Cancel() { + close(task.cancel) + <-task.cancelled +} + +func (task *ScheduledTask) String() string { + return fmt.Sprintf( + "%s\nInterval: %s\nRecurring: %t\n", + task.Name, + task.Interval.String(), + task.Recurring, + ) +} diff --git a/server/services/store/sqlstore/migrations/postgres/bindata.go b/server/services/store/sqlstore/migrations/postgres/bindata.go index 075e107ce..4cba723da 100644 --- a/server/services/store/sqlstore/migrations/postgres/bindata.go +++ b/server/services/store/sqlstore/migrations/postgres/bindata.go @@ -43,6 +43,24 @@ func _000001_init_up_sql() ([]byte, error) { ) } +var __000002_system_settings_table_down_sql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x72\x09\xf2\x0f\x50\x08\x71\x74\xf2\x71\x55\x28\xae\x2c\x2e\x49\xcd\x8d\x2f\x4e\x2d\x29\xc9\xcc\x4b\x2f\xb6\xe6\x02\x04\x00\x00\xff\xff\x8b\x60\xbf\x1e\x1c\x00\x00\x00") + +func _000002_system_settings_table_down_sql() ([]byte, error) { + return bindata_read( + __000002_system_settings_table_down_sql, + "000002_system_settings_table.down.sql", + ) +} + +var __000002_system_settings_table_up_sql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x72\x0e\x72\x75\x0c\x71\x55\x08\x71\x74\xf2\x71\x55\xf0\x74\x53\xf0\xf3\x0f\x51\x70\x8d\xf0\x0c\x0e\x09\x56\x28\xae\x2c\x2e\x49\xcd\x8d\x2f\x4e\x2d\x29\xc9\xcc\x4b\x2f\x56\xd0\xe0\xe2\xcc\x4c\x51\x08\x73\x0c\x72\xf6\x70\x0c\xd2\x30\x34\x30\xd0\xd4\xe1\xe2\x2c\x4b\xcc\x29\x4d\x55\x08\x71\x8d\x08\xd1\xe1\xe2\x0c\x08\xf2\xf4\x75\x0c\x8a\x54\xf0\x76\x8d\x54\xd0\xc8\x4c\xd1\xe4\xd2\xb4\xe6\x02\x04\x00\x00\xff\xff\x17\x95\xca\x5b\x61\x00\x00\x00") + +func _000002_system_settings_table_up_sql() ([]byte, error) { + return bindata_read( + __000002_system_settings_table_up_sql, + "000002_system_settings_table.up.sql", + ) +} + // Asset loads and returns the asset for the given name. // It returns an error if the asset could not be found or // could not be loaded. @@ -67,6 +85,8 @@ func AssetNames() []string { var _bindata = map[string]func() ([]byte, error){ "000001_init.down.sql": _000001_init_down_sql, "000001_init.up.sql": _000001_init_up_sql, + "000002_system_settings_table.down.sql": _000002_system_settings_table_down_sql, + "000002_system_settings_table.up.sql": _000002_system_settings_table_up_sql, } // AssetDir returns the file names below a certain // directory embedded in the file by go-bindata. @@ -112,4 +132,8 @@ var _bintree = &_bintree_t{nil, map[string]*_bintree_t{ }}, "000001_init.up.sql": &_bintree_t{_000001_init_up_sql, map[string]*_bintree_t{ }}, + "000002_system_settings_table.down.sql": &_bintree_t{_000002_system_settings_table_down_sql, map[string]*_bintree_t{ + }}, + "000002_system_settings_table.up.sql": &_bintree_t{_000002_system_settings_table_up_sql, map[string]*_bintree_t{ + }}, }} diff --git a/server/services/store/sqlstore/migrations/postgres_files/000002_system_settings_table.down.sql b/server/services/store/sqlstore/migrations/postgres_files/000002_system_settings_table.down.sql new file mode 100644 index 000000000..9da1a9b84 --- /dev/null +++ b/server/services/store/sqlstore/migrations/postgres_files/000002_system_settings_table.down.sql @@ -0,0 +1 @@ +DROP TABLE system_settings; diff --git a/server/services/store/sqlstore/migrations/postgres_files/000002_system_settings_table.up.sql b/server/services/store/sqlstore/migrations/postgres_files/000002_system_settings_table.up.sql new file mode 100644 index 000000000..4a47b6d8e --- /dev/null +++ b/server/services/store/sqlstore/migrations/postgres_files/000002_system_settings_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS system_settings ( + id VARCHAR(100), + value TEXT, + PRIMARY KEY (id) +); diff --git a/server/services/store/sqlstore/migrations/sqlite/bindata.go b/server/services/store/sqlstore/migrations/sqlite/bindata.go index 6c9f047ae..6895c5201 100644 --- a/server/services/store/sqlstore/migrations/sqlite/bindata.go +++ b/server/services/store/sqlstore/migrations/sqlite/bindata.go @@ -43,6 +43,24 @@ func _000001_init_up_sql() ([]byte, error) { ) } +var __000002_system_settings_table_down_sql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x72\x09\xf2\x0f\x50\x08\x71\x74\xf2\x71\x55\x28\xae\x2c\x2e\x49\xcd\x8d\x2f\x4e\x2d\x29\xc9\xcc\x4b\x2f\xb6\xe6\x02\x04\x00\x00\xff\xff\x8b\x60\xbf\x1e\x1c\x00\x00\x00") + +func _000002_system_settings_table_down_sql() ([]byte, error) { + return bindata_read( + __000002_system_settings_table_down_sql, + "000002_system_settings_table.down.sql", + ) +} + +var __000002_system_settings_table_up_sql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x72\x0e\x72\x75\x0c\x71\x55\x08\x71\x74\xf2\x71\x55\xf0\x74\x53\xf0\xf3\x0f\x51\x70\x8d\xf0\x0c\x0e\x09\x56\x28\xae\x2c\x2e\x49\xcd\x8d\x2f\x4e\x2d\x29\xc9\xcc\x4b\x2f\xd6\xe0\xe2\xcc\x4c\x51\x08\x73\x0c\x72\xf6\x70\x0c\xd2\x30\x34\x30\xd0\xd4\xe1\xe2\x2c\x4b\xcc\x29\x4d\x55\x08\x71\x8d\x08\xd1\xe1\xe2\x0c\x08\xf2\xf4\x75\x0c\x8a\x54\xf0\x76\x8d\x54\xd0\xc8\x4c\xd1\xe4\xd2\xb4\xe6\x02\x04\x00\x00\xff\xff\x1e\xfb\x02\xf2\x60\x00\x00\x00") + +func _000002_system_settings_table_up_sql() ([]byte, error) { + return bindata_read( + __000002_system_settings_table_up_sql, + "000002_system_settings_table.up.sql", + ) +} + // Asset loads and returns the asset for the given name. // It returns an error if the asset could not be found or // could not be loaded. @@ -67,6 +85,8 @@ func AssetNames() []string { var _bindata = map[string]func() ([]byte, error){ "000001_init.down.sql": _000001_init_down_sql, "000001_init.up.sql": _000001_init_up_sql, + "000002_system_settings_table.down.sql": _000002_system_settings_table_down_sql, + "000002_system_settings_table.up.sql": _000002_system_settings_table_up_sql, } // AssetDir returns the file names below a certain // directory embedded in the file by go-bindata. @@ -112,4 +132,8 @@ var _bintree = &_bintree_t{nil, map[string]*_bintree_t{ }}, "000001_init.up.sql": &_bintree_t{_000001_init_up_sql, map[string]*_bintree_t{ }}, + "000002_system_settings_table.down.sql": &_bintree_t{_000002_system_settings_table_down_sql, map[string]*_bintree_t{ + }}, + "000002_system_settings_table.up.sql": &_bintree_t{_000002_system_settings_table_up_sql, map[string]*_bintree_t{ + }}, }} diff --git a/server/services/store/sqlstore/migrations/sqlite_files/000002_system_settings_table.down.sql b/server/services/store/sqlstore/migrations/sqlite_files/000002_system_settings_table.down.sql new file mode 100644 index 000000000..9da1a9b84 --- /dev/null +++ b/server/services/store/sqlstore/migrations/sqlite_files/000002_system_settings_table.down.sql @@ -0,0 +1 @@ +DROP TABLE system_settings; diff --git a/server/services/store/sqlstore/migrations/sqlite_files/000002_system_settings_table.up.sql b/server/services/store/sqlstore/migrations/sqlite_files/000002_system_settings_table.up.sql new file mode 100644 index 000000000..34af055d5 --- /dev/null +++ b/server/services/store/sqlstore/migrations/sqlite_files/000002_system_settings_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS system_settings( + id VARCHAR(100), + value TEXT, + PRIMARY KEY (id) +); diff --git a/server/services/store/sqlstore/system.go b/server/services/store/sqlstore/system.go new file mode 100644 index 000000000..41c3cbfe5 --- /dev/null +++ b/server/services/store/sqlstore/system.go @@ -0,0 +1,35 @@ +package sqlstore + +func (s *SQLStore) GetSystemSettings() (map[string]string, error) { + query := `SELECT * FROM system_settings` + + rows, err := s.db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + results := map[string]string{} + + for rows.Next() { + var id string + var value string + err := rows.Scan(&id, &value) + if err != nil { + return nil, err + } + results[id] = value + } + + return results, nil +} + +func (s *SQLStore) SetSystemSetting(id string, value string) error { + query := `INSERT INTO system_settings(id, value) VALUES ($1,$2) ON CONFLICT (id) DO UPDATE SET value=$2` + + _, err := s.db.Exec(query, id, value) + if err != nil { + return err + } + return nil +} diff --git a/server/services/store/store.go b/server/services/store/store.go index f18f92f50..a21808a34 100644 --- a/server/services/store/store.go +++ b/server/services/store/store.go @@ -13,4 +13,6 @@ type Store interface { InsertBlock(block model.Block) error DeleteBlock(blockID string) error Shutdown() error + GetSystemSettings() (map[string]string, error) + SetSystemSetting(key string, value string) error } diff --git a/server/services/telemetry/mocks/ServerIface.go b/server/services/telemetry/mocks/ServerIface.go new file mode 100644 index 000000000..91663cf36 --- /dev/null +++ b/server/services/telemetry/mocks/ServerIface.go @@ -0,0 +1,147 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +// Regenerate this file using `make telemetry-mocks`. + +package mocks + +import ( + httpservice "github.com/mattermost/mattermost-server/v5/services/httpservice" + mock "github.com/stretchr/testify/mock" + + model "github.com/mattermost/mattermost-server/v5/model" + + plugin "github.com/mattermost/mattermost-server/v5/plugin" +) + +// ServerIface is an autogenerated mock type for the ServerIface type +type ServerIface struct { + mock.Mock +} + +// Config provides a mock function with given fields: +func (_m *ServerIface) Config() *model.Config { + ret := _m.Called() + + var r0 *model.Config + if rf, ok := ret.Get(0).(func() *model.Config); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.Config) + } + } + + return r0 +} + +// GetPluginsEnvironment provides a mock function with given fields: +func (_m *ServerIface) GetPluginsEnvironment() *plugin.Environment { + ret := _m.Called() + + var r0 *plugin.Environment + if rf, ok := ret.Get(0).(func() *plugin.Environment); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*plugin.Environment) + } + } + + return r0 +} + +// GetRoleByName provides a mock function with given fields: _a0 +func (_m *ServerIface) GetRoleByName(_a0 string) (*model.Role, *model.AppError) { + ret := _m.Called(_a0) + + var r0 *model.Role + if rf, ok := ret.Get(0).(func(string) *model.Role); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.Role) + } + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(string) *model.AppError); ok { + r1 = rf(_a0) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + +// GetSchemes provides a mock function with given fields: _a0, _a1, _a2 +func (_m *ServerIface) GetSchemes(_a0 string, _a1 int, _a2 int) ([]*model.Scheme, *model.AppError) { + ret := _m.Called(_a0, _a1, _a2) + + var r0 []*model.Scheme + if rf, ok := ret.Get(0).(func(string, int, int) []*model.Scheme); ok { + r0 = rf(_a0, _a1, _a2) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*model.Scheme) + } + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(string, int, int) *model.AppError); ok { + r1 = rf(_a0, _a1, _a2) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + +// HttpService provides a mock function with given fields: +func (_m *ServerIface) HttpService() httpservice.HTTPService { + ret := _m.Called() + + var r0 httpservice.HTTPService + if rf, ok := ret.Get(0).(func() httpservice.HTTPService); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(httpservice.HTTPService) + } + } + + return r0 +} + +// IsLeader provides a mock function with given fields: +func (_m *ServerIface) IsLeader() bool { + ret := _m.Called() + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// License provides a mock function with given fields: +func (_m *ServerIface) License() *model.License { + ret := _m.Called() + + var r0 *model.License + if rf, ok := ret.Get(0).(func() *model.License); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.License) + } + } + + return r0 +} diff --git a/server/services/telemetry/telemetry.go b/server/services/telemetry/telemetry.go new file mode 100644 index 000000000..3b3c9b87e --- /dev/null +++ b/server/services/telemetry/telemetry.go @@ -0,0 +1,142 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package telemetry + +import ( + "log" + "os" + "strings" + "time" + + "github.com/mattermost/mattermost-octo-tasks/server/services/scheduler" + rudder "github.com/rudderlabs/analytics-go" +) + +const ( + DAY_MILLISECONDS = 24 * 60 * 60 * 1000 + MONTH_MILLISECONDS = 31 * DAY_MILLISECONDS + + RUDDER_KEY = "placeholder_rudder_key" + RUDDER_DATAPLANE_URL = "placeholder_rudder_dataplane_url" + + TRACK_CONFIG = "config" +) + +type telemetryTracker func() map[string]interface{} + +type TelemetryService struct { + trackers map[string]telemetryTracker + log *log.Logger + rudderClient rudder.Client + telemetryID string + timestampLastTelemetrySent time.Time +} + +type RudderConfig struct { + RudderKey string + DataplaneUrl string +} + +func New(telemetryID string, log *log.Logger) *TelemetryService { + service := &TelemetryService{ + log: log, + telemetryID: telemetryID, + trackers: map[string]telemetryTracker{}, + } + return service +} + +func (ts *TelemetryService) RegisterTracker(name string, tracker telemetryTracker) { + ts.trackers[name] = tracker +} + +func (ts *TelemetryService) getRudderConfig() RudderConfig { + if !strings.Contains(RUDDER_KEY, "placeholder") && !strings.Contains(RUDDER_DATAPLANE_URL, "placeholder") { + return RudderConfig{RUDDER_KEY, RUDDER_DATAPLANE_URL} + } else if os.Getenv("RUDDER_KEY") != "" && os.Getenv("RUDDER_DATAPLANE_URL") != "" { + return RudderConfig{os.Getenv("RUDDER_KEY"), os.Getenv("RUDDER_DATAPLANE_URL")} + } else { + return RudderConfig{} + } +} + +func (ts *TelemetryService) sendDailyTelemetry(override bool) { + config := ts.getRudderConfig() + if (config.DataplaneUrl != "" && config.RudderKey != "") || override { + ts.initRudder(config.DataplaneUrl, config.RudderKey) + for name, tracker := range ts.trackers { + ts.sendTelemetry(name, tracker()) + } + } +} + +func (ts *TelemetryService) sendTelemetry(event string, properties map[string]interface{}) { + if ts.rudderClient != nil { + var context *rudder.Context + ts.rudderClient.Enqueue(rudder.Track{ + Event: event, + UserId: ts.telemetryID, + Properties: properties, + Context: context, + }) + } +} + +func (ts *TelemetryService) initRudder(endpoint string, rudderKey string) { + if ts.rudderClient == nil { + config := rudder.Config{} + config.Logger = rudder.StdLogger(ts.log) + config.Endpoint = endpoint + // For testing + if endpoint != RUDDER_DATAPLANE_URL { + config.Verbose = true + config.BatchSize = 1 + } + client, err := rudder.NewWithConfig(rudderKey, endpoint, config) + if err != nil { + ts.log.Fatal("Failed to create Rudder instance") + return + } + client.Enqueue(rudder.Identify{ + UserId: ts.telemetryID, + }) + + ts.rudderClient = client + } +} + +func (ts *TelemetryService) doTelemetryIfNeeded(firstRun time.Time) { + hoursSinceFirstServerRun := time.Since(firstRun).Hours() + // Send once every 10 minutes for the first hour + // Send once every hour thereafter for the first 12 hours + // Send at the 24 hour mark and every 24 hours after + if hoursSinceFirstServerRun < 1 { + ts.doTelemetry() + } else if hoursSinceFirstServerRun <= 12 && time.Since(ts.timestampLastTelemetrySent) >= time.Hour { + ts.doTelemetry() + } else if hoursSinceFirstServerRun > 12 && time.Since(ts.timestampLastTelemetrySent) >= 24*time.Hour { + ts.doTelemetry() + } +} + +func (ts *TelemetryService) RunTelemetryJob(firstRun int64) { + // Send on boot + ts.doTelemetry() + scheduler.CreateRecurringTask("Telemetry", func() { + ts.doTelemetryIfNeeded(time.Unix(0, firstRun*int64(time.Millisecond))) + }, time.Minute*10) +} + +func (ts *TelemetryService) doTelemetry() { + ts.timestampLastTelemetrySent = time.Now() + ts.sendDailyTelemetry(false) +} + +// Shutdown closes the telemetry client. +func (ts *TelemetryService) Shutdown() error { + if ts.rudderClient != nil { + return ts.rudderClient.Close() + } + return nil +}