Telemetry and metrics (#496)

- total blocks by block type
- total workspaces
- blocks activity (insert/delete)
- login success / fail

logging.json

@@ -17,7 +17,8 @@
             {"id": 2, "name": "error", "color": 31},
             {"id": 1, "name": "fatal", "stacktrace": true},
             {"id": 0, "name": "panic", "stacktrace": true},
-            {"id": 500, "name": "telemetry", "color":34}
+            {"id": 500, "name": "telemetry", "color":34},
+            {"id": 501, "name": "metrics", "color":34}
         ],
         "maxqueuesize": 1000
     },

server/app/app.go

@@ -3,6 +3,7 @@ package app
 import (
 	"github.com/mattermost/focalboard/server/auth"
 	"github.com/mattermost/focalboard/server/services/config"
+	"github.com/mattermost/focalboard/server/services/metrics"
 	"github.com/mattermost/focalboard/server/services/mlog"
 	"github.com/mattermost/focalboard/server/services/store"
 	"github.com/mattermost/focalboard/server/services/webhook"
@@ -11,6 +12,15 @@ import (
 	"github.com/mattermost/mattermost-server/v5/shared/filestore"
 )
 
+type AppServices struct {
+	Auth         *auth.Auth
+	Store        store.Store
+	FilesBackend filestore.FileBackend
+	Webhook      *webhook.Client
+	Metrics      *metrics.Metrics
+	Logger       *mlog.Logger
+}
+
 type App struct {
 	config       *config.Configuration
 	store        store.Store
@@ -18,25 +28,19 @@ type App struct {
 	wsServer     *ws.Server
 	filesBackend filestore.FileBackend
 	webhook      *webhook.Client
+	metrics      *metrics.Metrics
 	logger       *mlog.Logger
 }
 
-func New(
-	config *config.Configuration,
-	store store.Store,
-	auth *auth.Auth,
-	wsServer *ws.Server,
-	filesBackend filestore.FileBackend,
-	webhook *webhook.Client,
-	logger *mlog.Logger,
-) *App {
+func New(config *config.Configuration, wsServer *ws.Server, services AppServices) *App {
 	return &App{
 		config:       config,
-		store:        store,
-		auth:         auth,
+		store:        services.Store,
+		auth:         services.Auth,
 		wsServer:     wsServer,
-		filesBackend: filesBackend,
-		webhook:      webhook,
-		logger:       logger,
+		filesBackend: services.FilesBackend,
+		webhook:      services.Webhook,
+		metrics:      services.Metrics,
+		logger:       services.Logger,
 	}
 }

server/app/auth.go

@@ -63,6 +63,7 @@ func (a *App) Login(username, email, password, mfaToken string) (string, error)
 		var err error
 		user, err = a.store.GetUserByUsername(username)
 		if err != nil {
+			a.metrics.IncrementLoginFailCount(1)
 			return "", errors.Wrap(err, "invalid username or password")
 		}
 	}
@@ -71,14 +72,17 @@ func (a *App) Login(username, email, password, mfaToken string) (string, error)
 		var err error
 		user, err = a.store.GetUserByEmail(email)
 		if err != nil {
+			a.metrics.IncrementLoginFailCount(1)
 			return "", errors.Wrap(err, "invalid username or password")
 		}
 	}
 	if user == nil {
+		a.metrics.IncrementLoginFailCount(1)
 		return "", errors.New("invalid username or password")
 	}
 
 	if !auth.ComparePassword(user.Password, password) {
+		a.metrics.IncrementLoginFailCount(1)
 		a.logger.Debug("Invalid password for user", mlog.String("userID", user.ID))
 		return "", errors.New("invalid username or password")
 	}
@@ -100,6 +104,8 @@ func (a *App) Login(username, email, password, mfaToken string) (string, error)
 		return "", errors.Wrap(err, "unable to create session")
 	}
 
+	a.metrics.IncrementLoginCount(1)
+
 	// TODO: MFA verification
 	return session.Token, nil
 }

server/app/blocks.go

@@ -30,7 +30,11 @@ func (a *App) GetParentID(c store.Container, blockID string) (string, error) {
 }
 
 func (a *App) InsertBlock(c store.Container, block model.Block) error {
-	return a.store.InsertBlock(c, block)
+	err := a.store.InsertBlock(c, block)
+	if err == nil {
+		a.metrics.IncrementBlocksInserted(1)
+	}
+	return err
 }
 
 func (a *App) InsertBlocks(c store.Container, blocks []model.Block) error {
@@ -54,6 +58,7 @@ func (a *App) InsertBlocks(c store.Container, blocks []model.Block) error {
 		}
 
 		a.wsServer.BroadcastBlockChange(c.WorkspaceID, block)
+		a.metrics.IncrementBlocksInserted(len(blocks))
 		go a.webhook.NotifyUpdate(block)
 	}
 
@@ -89,6 +94,11 @@ func (a *App) DeleteBlock(c store.Container, blockID string, modifiedBy string)
 	}
 
 	a.wsServer.BroadcastBlockDelete(c.WorkspaceID, blockID, parentID)
+	a.metrics.IncrementBlocksDeleted(1)
 
 	return nil
 }
+
+func (a *App) GetBlockCountsByType() (map[string]int64, error) {
+	return a.store.GetBlockCountsByType()
+}

server/app/helper_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/mattermost/focalboard/server/auth"
 	"github.com/mattermost/focalboard/server/services/config"
+	"github.com/mattermost/focalboard/server/services/metrics"
 	"github.com/mattermost/focalboard/server/services/mlog"
 	"github.com/mattermost/focalboard/server/services/store/mockstore"
 	"github.com/mattermost/focalboard/server/services/webhook"
@@ -29,12 +30,21 @@ func SetupTestHelper(t *testing.T) *TestHelper {
 	cfg := config.Configuration{}
 	store := mockstore.NewMockStore(ctrl)
 	auth := auth.New(&cfg, store)
-	logger := mlog.NewLogger()
-	logger.Configure("", cfg.LoggingEscapedJson)
+	logger := mlog.CreateTestLogger(t)
 	sessionToken := "TESTTOKEN"
 	wsserver := ws.NewServer(auth, sessionToken, false, logger)
 	webhook := webhook.NewClient(&cfg, logger)
-	app2 := New(&cfg, store, auth, wsserver, &mocks.FileBackend{}, webhook, logger)
+	metricsService := metrics.NewMetrics(metrics.InstanceInfo{})
+
+	appServices := AppServices{
+		Auth:         auth,
+		Store:        store,
+		FilesBackend: &mocks.FileBackend{},
+		Webhook:      webhook,
+		Metrics:      metricsService,
+		Logger:       logger,
+	}
+	app2 := New(&cfg, wsserver, appServices)
 
 	return &TestHelper{
 		App:   app2,

server/app/workspaces.go

@@ -55,3 +55,7 @@ func (a *App) UpsertWorkspaceSettings(workspace model.Workspace) error {
 func (a *App) UpsertWorkspaceSignupToken(workspace model.Workspace) error {
 	return a.store.UpsertWorkspaceSignupToken(workspace)
 }
+
+func (a *App) GetWorkspaceCount() (int64, error) {
+	return a.store.GetWorkspaceCount()
+}

server/integrationtests/clienttestlib.go

@@ -27,7 +27,7 @@ func getTestConfig() *config.Configuration {
 		connectionString = ":memory:"
 	}
 
-	logging := []byte(`
+	logging := `
 	{
 		"testing": {
 			"type": "console",
@@ -47,7 +47,7 @@ func getTestConfig() *config.Configuration {
 				{"id": 0, "name": "panic", "stacktrace": true}
 			]
 		}
-	}`)
+	}`
 
 	return &config.Configuration{
 		ServerRoot:         "http://localhost:8888",
@@ -58,7 +58,7 @@ func getTestConfig() *config.Configuration {
 		WebPath:            "./pack",
 		FilesDriver:        "local",
 		FilesPath:          "./files",
-		LoggingEscapedJson: string(logging),
+		LoggingEscapedJson: logging,
 	}
 }
 

server/server/server.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"os"
 	"runtime"
+	"sync"
 	"syscall"
 	"time"
 
@@ -19,8 +20,8 @@ import (
 	"github.com/mattermost/focalboard/server/context"
 	appModel "github.com/mattermost/focalboard/server/model"
 	"github.com/mattermost/focalboard/server/services/config"
+	"github.com/mattermost/focalboard/server/services/metrics"
 	"github.com/mattermost/focalboard/server/services/mlog"
-	"github.com/mattermost/focalboard/server/services/prometheus"
 	"github.com/mattermost/focalboard/server/services/scheduler"
 	"github.com/mattermost/focalboard/server/services/store"
 	"github.com/mattermost/focalboard/server/services/store/mattermostauthlayer"
@@ -37,22 +38,25 @@ import (
 
 const (
 	cleanupSessionTaskFrequency = 10 * time.Minute
+	updateMetricsTaskFrequency  = 15 * time.Minute
 
 	//nolint:gomnd
 	minSessionExpiryTime = int64(60 * 60 * 24 * 31) // 31 days
 )
 
 type Server struct {
-	config              *config.Configuration
-	wsServer            *ws.Server
-	webServer           *web.Server
-	store               store.Store
-	filesBackend        filestore.FileBackend
-	telemetry           *telemetry.Service
-	logger              *mlog.Logger
-	cleanUpSessionsTask *scheduler.ScheduledTask
-	promServer          *prometheus.Service
-	promInstrumentor    *prometheus.Instrumentor
+	config                 *config.Configuration
+	wsServer               *ws.Server
+	webServer              *web.Server
+	store                  store.Store
+	filesBackend           filestore.FileBackend
+	telemetry              *telemetry.Service
+	logger                 *mlog.Logger
+	cleanUpSessionsTask    *scheduler.ScheduledTask
+	metricsServer          *metrics.Service
+	metricsService         *metrics.Metrics
+	metricsUpdaterTask     *scheduler.ScheduledTask
+	servicesStartStopMutex sync.Mutex
 
 	localRouter     *mux.Router
 	localModeServer *http.Server
@@ -103,7 +107,25 @@ func New(cfg *config.Configuration, singleUserToken string, logger *mlog.Logger)
 
 	webhookClient := webhook.NewClient(cfg, logger)
 
-	appBuilder := func() *app.App { return app.New(cfg, db, authenticator, wsServer, filesBackend, webhookClient, logger) }
+	// Init metrics
+	instanceInfo := metrics.InstanceInfo{
+		Version:        appModel.CurrentVersion,
+		BuildNum:       appModel.BuildNumber,
+		Edition:        appModel.Edition,
+		InstallationID: os.Getenv("MM_CLOUD_INSTALLATION_ID"),
+	}
+	metricsService := metrics.NewMetrics(instanceInfo)
+
+	appServices := app.AppServices{
+		Auth:         authenticator,
+		Store:        db,
+		FilesBackend: filesBackend,
+		Webhook:      webhookClient,
+		Metrics:      metricsService,
+		Logger:       logger,
+	}
+	appBuilder := func() *app.App { return app.New(cfg, wsServer, appServices) }
+
 	focalboardAPI := api.NewAPI(appBuilder, singleUserToken, cfg.AuthMode, logger)
 
 	// Local router for admin APIs
@@ -120,82 +142,41 @@ func New(cfg *config.Configuration, singleUserToken string, logger *mlog.Logger)
 	webServer.AddRoutes(wsServer)
 	webServer.AddRoutes(focalboardAPI)
 
-	// Init telemetry
 	settings, err := db.GetSystemSettings()
 	if err != nil {
 		return nil, err
 	}
 
+	// Init telemetry
 	telemetryID := settings["TelemetryID"]
-
 	if len(telemetryID) == 0 {
 		telemetryID = uuid.New().String()
 		if err = db.SetSystemSetting("TelemetryID", uuid.New().String()); err != nil {
 			return nil, err
 		}
 	}
-
-	registeredUserCount, err := appBuilder().GetRegisteredUserCount()
-	if err != nil {
-		return nil, err
+	telemetryOpts := telemetryOptions{
+		appBuilder:  appBuilder,
+		cfg:         cfg,
+		telemetryID: telemetryID,
+		logger:      logger,
+		singleUser:  len(singleUserToken) > 0,
 	}
-
-	dailyActiveUsers, err := appBuilder().GetDailyActiveUsers()
-	if err != nil {
-		return nil, err
-	}
-
-	weeklyActiveUsers, err := appBuilder().GetWeeklyActiveUsers()
-	if err != nil {
-		return nil, err
-	}
-
-	monthlyActiveUsers, err := appBuilder().GetMonthlyActiveUsers()
-	if err != nil {
-		return nil, err
-	}
-
-	telemetryService := telemetry.New(telemetryID, logger.StdLogger(mlog.Telemetry))
-	telemetryService.RegisterTracker("server", func() map[string]interface{} {
-		return map[string]interface{}{
-			"version":          appModel.CurrentVersion,
-			"build_number":     appModel.BuildNumber,
-			"build_hash":       appModel.BuildHash,
-			"edition":          appModel.Edition,
-			"operating_system": runtime.GOOS,
-		}
-	})
-	telemetryService.RegisterTracker("config", func() map[string]interface{} {
-		return map[string]interface{}{
-			"serverRoot":  cfg.ServerRoot == config.DefaultServerRoot,
-			"port":        cfg.Port == config.DefaultPort,
-			"useSSL":      cfg.UseSSL,
-			"dbType":      cfg.DBType,
-			"single_user": len(singleUserToken) > 0,
-		}
-	})
-	telemetryService.RegisterTracker("activity", func() map[string]interface{} {
-		return map[string]interface{}{
-			"registered_users":     registeredUserCount,
-			"daily_active_users":   dailyActiveUsers,
-			"weekly_active_users":  weeklyActiveUsers,
-			"monthly_active_users": monthlyActiveUsers,
-		}
-	})
+	telemetryService := initTelemetry(telemetryOpts)
 
 	server := Server{
-		config:           cfg,
-		wsServer:         wsServer,
-		webServer:        webServer,
-		store:            db,
-		filesBackend:     filesBackend,
-		telemetry:        telemetryService,
-		promServer:       prometheus.New(cfg.PrometheusAddress),
-		promInstrumentor: prometheus.NewInstrumentor(appModel.CurrentVersion),
-		logger:           logger,
-		localRouter:      localRouter,
-		api:              focalboardAPI,
-		appBuilder:       appBuilder,
+		config:         cfg,
+		wsServer:       wsServer,
+		webServer:      webServer,
+		store:          db,
+		filesBackend:   filesBackend,
+		telemetry:      telemetryService,
+		metricsServer:  metrics.NewMetricsServer(cfg.PrometheusAddress, metricsService, logger),
+		metricsService: metricsService,
+		logger:         logger,
+		localRouter:    localRouter,
+		api:            focalboardAPI,
+		appBuilder:     appBuilder,
 	}
 
 	server.initHandlers()
@@ -208,6 +189,9 @@ func (s *Server) Start() error {
 
 	s.webServer.Start()
 
+	s.servicesStartStopMutex.Lock()
+	defer s.servicesStartStopMutex.Unlock()
+
 	if s.config.EnableLocalMode {
 		if err := s.startLocalModeServer(); err != nil {
 			return err
@@ -225,6 +209,28 @@ func (s *Server) Start() error {
 		}
 	}, cleanupSessionTaskFrequency)
 
+	metricsUpdater := func() {
+		app := s.appBuilder()
+		blockCounts, err := app.GetBlockCountsByType()
+		if err != nil {
+			s.logger.Error("Error updating metrics", mlog.String("group", "blocks"), mlog.Err(err))
+			return
+		}
+		s.logger.Log(mlog.Metrics, "Block metrics collected", mlog.Map("block_counts", blockCounts))
+		for blockType, count := range blockCounts {
+			s.metricsService.ObserveBlockCount(blockType, count)
+		}
+		workspaceCount, err := app.GetWorkspaceCount()
+		if err != nil {
+			s.logger.Error("Error updating metrics", mlog.String("group", "workspaces"), mlog.Err(err))
+			return
+		}
+		s.logger.Log(mlog.Metrics, "Workspace metrics collected", mlog.Int64("workspace_count", workspaceCount))
+		s.metricsService.ObserveWorkspaceCount(workspaceCount)
+	}
+	//metricsUpdater()   Calling this immediately causes integration unit tests to fail.
+	s.metricsUpdaterTask = scheduler.CreateRecurringTask("updateMetrics", metricsUpdater, updateMetricsTaskFrequency)
+
 	if s.config.Telemetry {
 		firstRun := utils.MillisFromTime(time.Now())
 		s.telemetry.RunTelemetryJob(firstRun)
@@ -233,15 +239,13 @@ func (s *Server) Start() error {
 	var group run.Group
 	if s.config.PrometheusAddress != "" {
 		group.Add(func() error {
-			if err := s.promServer.Run(); err != nil {
+			if err := s.metricsServer.Run(); err != nil {
 				return errors.Wrap(err, "PromServer Run")
 			}
 			return nil
 		}, func(error) {
-			s.promServer.Shutdown()
+			s.metricsServer.Shutdown()
 		})
-		// expose the build info as metric
-		s.promInstrumentor.ExposeBuildInfo()
 
 		if err := group.Run(); err != nil {
 			return err
@@ -257,10 +261,17 @@ func (s *Server) Shutdown() error {
 
 	s.stopLocalModeServer()
 
+	s.servicesStartStopMutex.Lock()
+	defer s.servicesStartStopMutex.Unlock()
+
 	if s.cleanUpSessionsTask != nil {
 		s.cleanUpSessionsTask.Cancel()
 	}
 
+	if s.metricsUpdaterTask != nil {
+		s.metricsUpdaterTask.Cancel()
+	}
+
 	if err := s.telemetry.Shutdown(); err != nil {
 		s.logger.Warn("Error occurred when shutting down telemetry", mlog.Err(err))
 	}
@@ -325,3 +336,81 @@ func (s *Server) GetRootRouter() *mux.Router {
 func (s *Server) SetWSHub(hub ws.Hub) {
 	s.wsServer.SetHub(hub)
 }
+
+type telemetryOptions struct {
+	appBuilder  func() *app.App
+	cfg         *config.Configuration
+	telemetryID string
+	logger      *mlog.Logger
+	singleUser  bool
+}
+
+func initTelemetry(opts telemetryOptions) *telemetry.Service {
+	telemetryService := telemetry.New(opts.telemetryID, opts.logger)
+
+	telemetryService.RegisterTracker("server", func() (telemetry.Tracker, error) {
+		return map[string]interface{}{
+			"version":          appModel.CurrentVersion,
+			"build_number":     appModel.BuildNumber,
+			"build_hash":       appModel.BuildHash,
+			"edition":          appModel.Edition,
+			"operating_system": runtime.GOOS,
+		}, nil
+	})
+	telemetryService.RegisterTracker("config", func() (telemetry.Tracker, error) {
+		return map[string]interface{}{
+			"serverRoot":  opts.cfg.ServerRoot == config.DefaultServerRoot,
+			"port":        opts.cfg.Port == config.DefaultPort,
+			"useSSL":      opts.cfg.UseSSL,
+			"dbType":      opts.cfg.DBType,
+			"single_user": opts.singleUser,
+		}, nil
+	})
+	telemetryService.RegisterTracker("activity", func() (telemetry.Tracker, error) {
+		m := make(map[string]interface{})
+		var count int
+		var err error
+		if count, err = opts.appBuilder().GetRegisteredUserCount(); err != nil {
+			return nil, err
+		}
+		m["registered_users"] = count
+
+		if count, err = opts.appBuilder().GetDailyActiveUsers(); err != nil {
+			return nil, err
+		}
+		m["daily_active_users"] = count
+
+		if count, err = opts.appBuilder().GetWeeklyActiveUsers(); err != nil {
+			return nil, err
+		}
+		m["weekly_active_users"] = count
+
+		if count, err = opts.appBuilder().GetMonthlyActiveUsers(); err != nil {
+			return nil, err
+		}
+		m["monthly_active_users"] = count
+		return m, nil
+	})
+	telemetryService.RegisterTracker("blocks", func() (telemetry.Tracker, error) {
+		blockCounts, err := opts.appBuilder().GetBlockCountsByType()
+		if err != nil {
+			return nil, err
+		}
+		m := make(map[string]interface{})
+		for k, v := range blockCounts {
+			m[k] = v
+		}
+		return m, nil
+	})
+	telemetryService.RegisterTracker("workspaces", func() (telemetry.Tracker, error) {
+		count, err := opts.appBuilder().GetWorkspaceCount()
+		if err != nil {
+			return nil, err
+		}
+		m := map[string]interface{}{
+			"workspaces": count,
+		}
+		return m, nil
+	})
+	return telemetryService
+}

server/services/metrics/metrics.go

@@ -0,0 +1,182 @@
+package metrics
+
+import (
+	"os"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	MetricsNamespace           = "focalboard"
+	MetricsSubsystemBlocks     = "blocks"
+	MetricsSubsystemWorkspaces = "workspaces"
+	MetricsSubsystemSystem     = "system"
+
+	MetricsCloudInstallationLabel = "installationId"
+)
+
+type InstanceInfo struct {
+	Version        string
+	BuildNum       string
+	Edition        string
+	InstallationID string
+}
+
+// Metrics used to instrumentate metrics in prometheus
+type Metrics struct {
+	registry *prometheus.Registry
+
+	instance  *prometheus.GaugeVec
+	startTime prometheus.Gauge
+
+	loginCount     prometheus.Counter
+	loginFailCount prometheus.Counter
+
+	blocksInsertedCount prometheus.Counter
+	blocksDeletedCount  prometheus.Counter
+
+	blockCount     *prometheus.GaugeVec
+	workspaceCount prometheus.Gauge
+
+	blockLastActivity prometheus.Gauge
+}
+
+// NewMetrics Factory method to create a new metrics collector
+func NewMetrics(info InstanceInfo) *Metrics {
+	m := &Metrics{}
+
+	m.registry = prometheus.NewRegistry()
+	options := prometheus.ProcessCollectorOpts{
+		Namespace: MetricsNamespace,
+	}
+	m.registry.MustRegister(prometheus.NewProcessCollector(options))
+	m.registry.MustRegister(prometheus.NewGoCollector())
+
+	additionalLabels := map[string]string{}
+	if info.InstallationID != "" {
+		additionalLabels[MetricsCloudInstallationLabel] = os.Getenv("MM_CLOUD_INSTALLATION_ID")
+	}
+
+	m.loginCount = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace:   MetricsNamespace,
+		Subsystem:   MetricsSubsystemSystem,
+		Name:        "login_total",
+		Help:        "Total number of logins.",
+		ConstLabels: additionalLabels,
+	})
+	m.registry.MustRegister(m.loginCount)
+
+	m.loginFailCount = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace:   MetricsNamespace,
+		Subsystem:   MetricsSubsystemSystem,
+		Name:        "login_fail_total",
+		Help:        "Total number of failed logins.",
+		ConstLabels: additionalLabels,
+	})
+	m.registry.MustRegister(m.loginFailCount)
+
+	m.instance = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace:   MetricsNamespace,
+		Subsystem:   MetricsSubsystemSystem,
+		Name:        "focalboard_instance_info",
+		Help:        "Instance information for Focalboard.",
+		ConstLabels: additionalLabels,
+	}, []string{"Version", "BuildNum", "Edition"})
+	m.registry.MustRegister(m.instance)
+	m.instance.WithLabelValues(info.Version, info.BuildNum, info.Edition).Set(1)
+
+	m.startTime = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace:   MetricsNamespace,
+		Subsystem:   MetricsSubsystemSystem,
+		Name:        "server_start_time",
+		Help:        "The time the server started.",
+		ConstLabels: additionalLabels,
+	})
+	m.startTime.SetToCurrentTime()
+	m.registry.MustRegister(m.startTime)
+
+	m.blocksInsertedCount = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace:   MetricsNamespace,
+		Subsystem:   MetricsSubsystemBlocks,
+		Name:        "blocks_inserted_total",
+		Help:        "Total number of blocks inserted.",
+		ConstLabels: additionalLabels,
+	})
+	m.registry.MustRegister(m.blocksInsertedCount)
+
+	m.blocksDeletedCount = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace:   MetricsNamespace,
+		Subsystem:   MetricsSubsystemBlocks,
+		Name:        "blocks_deleted_total",
+		Help:        "Total number of blocks deleted.",
+		ConstLabels: additionalLabels,
+	})
+	m.registry.MustRegister(m.blocksDeletedCount)
+
+	m.blockCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace:   MetricsNamespace,
+		Subsystem:   MetricsSubsystemBlocks,
+		Name:        "blocks_total",
+		Help:        "Total number of blocks.",
+		ConstLabels: additionalLabels,
+	}, []string{"BlockType"})
+	m.registry.MustRegister(m.blockCount)
+
+	m.workspaceCount = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace:   MetricsNamespace,
+		Subsystem:   MetricsSubsystemWorkspaces,
+		Name:        "workspaces_total",
+		Help:        "Total number of workspaces.",
+		ConstLabels: additionalLabels,
+	})
+	m.registry.MustRegister(m.workspaceCount)
+
+	m.blockLastActivity = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace:   MetricsNamespace,
+		Subsystem:   MetricsSubsystemBlocks,
+		Name:        "blocks_last_activity",
+		Help:        "Time of last block insert, update, delete.",
+		ConstLabels: additionalLabels,
+	})
+	m.registry.MustRegister(m.blockLastActivity)
+
+	return m
+}
+
+func (m *Metrics) IncrementLoginCount(num int) {
+	if m != nil {
+		m.loginCount.Add(float64(num))
+	}
+}
+
+func (m *Metrics) IncrementLoginFailCount(num int) {
+	if m != nil {
+		m.loginFailCount.Add(float64(num))
+	}
+}
+
+func (m *Metrics) IncrementBlocksInserted(num int) {
+	if m != nil {
+		m.blocksInsertedCount.Add(float64(num))
+		m.blockLastActivity.SetToCurrentTime()
+	}
+}
+
+func (m *Metrics) IncrementBlocksDeleted(num int) {
+	if m != nil {
+		m.blocksDeletedCount.Add(float64(num))
+		m.blockLastActivity.SetToCurrentTime()
+	}
+}
+
+func (m *Metrics) ObserveBlockCount(blockType string, count int64) {
+	if m != nil {
+		m.blockCount.WithLabelValues(blockType).Set(float64(count))
+	}
+}
+
+func (m *Metrics) ObserveWorkspaceCount(count int64) {
+	if m != nil {
+		m.workspaceCount.Set(float64(count))
+	}
+}

server/services/metrics/service.go

@@ -1,8 +1,9 @@
-package prometheus
+package metrics
 
 import (
 	"net/http"
 
+	"github.com/mattermost/focalboard/server/services/mlog"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
@@ -12,12 +13,14 @@ type Service struct {
 	*http.Server
 }
 
-// New Factory method to create a new prometheus server
-func New(address string) *Service {
+// NewMetricsServer factory method to create a new prometheus server
+func NewMetricsServer(address string, metricsService *Metrics, logger *mlog.Logger) *Service {
 	return &Service{
 		&http.Server{
-			Addr:    address,
-			Handler: promhttp.Handler(),
+			Addr: address,
+			Handler: promhttp.HandlerFor(metricsService.registry, promhttp.HandlerOpts{
+				ErrorLog: logger.StdLogger(mlog.Error),
+			}),
 		},
 	}
 }

server/services/mlog/levels.go

@@ -30,6 +30,7 @@ var (
 
 	// add more here ...
 	Telemetry = Level{ID: 500, Name: "telemetry"}
+	Metrics   = Level{ID: 501, Name: "metrics"}
 )
 
 // Combinations for LogM (log multi)

server/services/mlog/mlog.go

@@ -133,15 +133,11 @@ func (l *Logger) Configure(cfgFile string, cfgEscaped string) error {
 
 	// Add config from escaped json string
 	if cfgEscaped != "" {
-		if b, err := decodeEscapedJSONString(string(cfgEscaped)); err != nil {
-			return fmt.Errorf("error unescaping logger config as escaped json: %w", err)
-		} else {
-			var mapCfgEscaped LoggerConfig
-			if err := json.Unmarshal(b, &mapCfgEscaped); err != nil {
-				return fmt.Errorf("error decoding logger config as escaped json: %w", err)
-			}
-			cfgMap.append(mapCfgEscaped)
+		var mapCfgEscaped LoggerConfig
+		if err := json.Unmarshal([]byte(cfgEscaped), &mapCfgEscaped); err != nil {
+			return fmt.Errorf("error decoding logger config as escaped json: %w", err)
 		}
+		cfgMap.append(mapCfgEscaped)
 	}
 
 	if len(cfgMap) == 0 {
@@ -151,18 +147,6 @@ func (l *Logger) Configure(cfgFile string, cfgEscaped string) error {
 	return logrcfg.ConfigureTargets(l.log.Logr(), cfgMap, nil)
 }
 
-func decodeEscapedJSONString(s string) ([]byte, error) {
-	type wrapper struct {
-		wrap string
-	}
-	var wrapped wrapper
-	ss := fmt.Sprintf("{\"wrap\":%s}", s)
-	if err := json.Unmarshal([]byte(ss), &wrapped); err != nil {
-		return nil, err
-	}
-	return []byte(wrapped.wrap), nil
-}
-
 // With creates a new Logger with the specified fields. This is a light-weight
 // operation and can be called on demand.
 func (l *Logger) With(fields ...Field) *Logger {

server/services/prometheus/intrumentor.go

@@ -1,27 +0,0 @@
-package prometheus
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-// Instrumentor used to instrumentate metrics in prometheus
-type Instrumentor struct {
-	Version string
-}
-
-// NewInstrumentor Factory method to create a new instrumentator
-func NewInstrumentor(version string) *Instrumentor {
-	return &Instrumentor{
-		Version: version,
-	}
-}
-
-// ExposeBuildInfo exposes a gauge in prometheus for build info
-func (i *Instrumentor) ExposeBuildInfo() {
-	promauto.NewGaugeVec(prometheus.GaugeOpts{
-		Name: "focalboard_build_info",
-		Help: "Build information of Focalboard",
-	}, []string{"Version"},
-	).WithLabelValues(i.Version).Set(1)
-}

server/services/store/mockstore/mockstore.go

@@ -135,6 +135,21 @@ func (mr *MockStoreMockRecorder) GetAllBlocks(c interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllBlocks", reflect.TypeOf((*MockStore)(nil).GetAllBlocks), c)
 }
 
+// GetBlockCountsByType mocks base method.
+func (m *MockStore) GetBlockCountsByType() (map[string]int64, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetBlockCountsByType")
+	ret0, _ := ret[0].(map[string]int64)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetBlockCountsByType indicates an expected call of GetBlockCountsByType.
+func (mr *MockStoreMockRecorder) GetBlockCountsByType() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBlockCountsByType", reflect.TypeOf((*MockStore)(nil).GetBlockCountsByType))
+}
+
 // GetBlocksWithParent mocks base method.
 func (m *MockStore) GetBlocksWithParent(c store.Container, parentID string) ([]model.Block, error) {
 	m.ctrl.T.Helper()
@@ -390,6 +405,21 @@ func (mr *MockStoreMockRecorder) GetWorkspace(ID interface{}) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkspace", reflect.TypeOf((*MockStore)(nil).GetWorkspace), ID)
 }
 
+// GetWorkspaceCount mocks base method.
+func (m *MockStore) GetWorkspaceCount() (int64, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetWorkspaceCount")
+	ret0, _ := ret[0].(int64)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetWorkspaceCount indicates an expected call of GetWorkspaceCount.
+func (mr *MockStoreMockRecorder) GetWorkspaceCount() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkspaceCount", reflect.TypeOf((*MockStore)(nil).GetWorkspaceCount))
+}
+
 // HasWorkspaceAccess mocks base method.
 func (m *MockStore) HasWorkspaceAccess(arg0, arg1 string) (bool, error) {
 	m.ctrl.T.Helper()

server/services/store/sqlstore/blocks.go

@@ -435,3 +435,35 @@ func (s *SQLStore) DeleteBlock(c store.Container, blockID string, modifiedBy str
 
 	return nil
 }
+
+func (s *SQLStore) GetBlockCountsByType() (map[string]int64, error) {
+	query := s.getQueryBuilder().
+		Select(
+			"type",
+			"COUNT(*) AS count",
+		).
+		From(s.tablePrefix + "blocks").
+		GroupBy("type")
+
+	rows, err := query.Query()
+	if err != nil {
+		s.logger.Error(`GetBlockCountsByType ERROR`, mlog.Err(err))
+
+		return nil, err
+	}
+
+	m := make(map[string]int64)
+
+	for rows.Next() {
+		var blockType string
+		var count int64
+
+		err := rows.Scan(&blockType, &count)
+		if err != nil {
+			s.logger.Error("Failed to fetch block count", mlog.Err(err))
+			return nil, err
+		}
+		m[blockType] = count
+	}
+	return m, nil
+}

server/services/store/sqlstore/workspaces.go

@@ -108,3 +108,27 @@ func (s *SQLStore) GetWorkspace(ID string) (*model.Workspace, error) {
 func (s *SQLStore) HasWorkspaceAccess(userID string, workspaceID string) (bool, error) {
 	return true, nil
 }
+
+func (s *SQLStore) GetWorkspaceCount() (int64, error) {
+	query := s.getQueryBuilder().
+		Select(
+			"COUNT(*) AS count",
+		).
+		From(s.tablePrefix + "workspaces")
+
+	rows, err := query.Query()
+	if err != nil {
+		s.logger.Error("ERROR GetWorkspaceCount", mlog.Err(err))
+		return 0, err
+	}
+
+	var count int64
+
+	rows.Next()
+	err = rows.Scan(&count)
+	if err != nil {
+		s.logger.Error("Failed to fetch workspace count", mlog.Err(err))
+		return 0, err
+	}
+	return count, nil
+}

server/services/store/store.go

@@ -22,6 +22,7 @@ type Store interface {
 	GetParentID(c Container, blockID string) (string, error)
 	InsertBlock(c Container, block model.Block) error
 	DeleteBlock(c Container, blockID string, modifiedBy string) error
+	GetBlockCountsByType() (map[string]int64, error)
 
 	Shutdown() error
 
@@ -53,4 +54,5 @@ type Store interface {
 	UpsertWorkspaceSettings(workspace model.Workspace) error
 	GetWorkspace(ID string) (*model.Workspace, error)
 	HasWorkspaceAccess(userID string, workspaceID string) (bool, error)
+	GetWorkspaceCount() (int64, error)
 }

server/services/telemetry/telemetry.go

@@ -4,11 +4,11 @@
 package telemetry
 
 import (
-	"log"
 	"os"
 	"strings"
 	"time"
 
+	"github.com/mattermost/focalboard/server/services/mlog"
 	"github.com/mattermost/focalboard/server/services/scheduler"
 	rudder "github.com/rudderlabs/analytics-go"
 )
@@ -19,11 +19,13 @@ const (
 	timeBetweenTelemetryChecks = 10 * time.Minute
 )
 
-type Tracker func() map[string]interface{}
+type TrackerFunc func() (Tracker, error)
+
+type Tracker map[string]interface{}
 
 type Service struct {
-	trackers                   map[string]Tracker
-	log                        *log.Logger
+	trackers                   map[string]TrackerFunc
+	logger                     *mlog.Logger
 	rudderClient               rudder.Client
 	telemetryID                string
 	timestampLastTelemetrySent time.Time
@@ -34,18 +36,18 @@ type RudderConfig struct {
 	DataplaneURL string
 }
 
-func New(telemetryID string, log *log.Logger) *Service {
+func New(telemetryID string, logger *mlog.Logger) *Service {
 	service := &Service{
-		log:         log,
+		logger:      logger,
 		telemetryID: telemetryID,
-		trackers:    map[string]Tracker{},
+		trackers:    map[string]TrackerFunc{},
 	}
 
 	return service
 }
 
-func (ts *Service) RegisterTracker(name string, tracker Tracker) {
-	ts.trackers[name] = tracker
+func (ts *Service) RegisterTracker(name string, f TrackerFunc) {
+	ts.trackers[name] = f
 }
 
 func (ts *Service) getRudderConfig() RudderConfig {
@@ -64,7 +66,12 @@ func (ts *Service) sendDailyTelemetry(override bool) {
 		ts.initRudder(config.DataplaneURL, config.RudderKey)
 
 		for name, tracker := range ts.trackers {
-			ts.sendTelemetry(name, tracker())
+			m, err := tracker()
+			if err != nil {
+				ts.logger.Error("Error fetching telemetry data", mlog.String("name", name), mlog.Err(err))
+				continue
+			}
+			ts.sendTelemetry(name, m)
 		}
 	}
 }
@@ -84,7 +91,7 @@ func (ts *Service) sendTelemetry(event string, properties map[string]interface{}
 func (ts *Service) initRudder(endpoint, rudderKey string) {
 	if ts.rudderClient == nil {
 		config := rudder.Config{}
-		config.Logger = rudder.StdLogger(ts.log)
+		config.Logger = rudder.StdLogger(ts.logger.StdLogger(mlog.Telemetry))
 		config.Endpoint = endpoint
 		// For testing
 		if endpoint != rudderDataplaneURL {
@@ -93,8 +100,7 @@ func (ts *Service) initRudder(endpoint, rudderKey string) {
 		}
 		client, err := rudder.NewWithConfig(rudderKey, endpoint, config)
 		if err != nil {
-			ts.log.Fatal("Failed to create Rudder instance")
-
+			ts.logger.Fatal("Failed to create Rudder instance")
 			return
 		}
 		client.Enqueue(rudder.Identify{