focalboard/server/services/store/sqlstore/data_retention.go

// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore

import (
	"database/sql"
	"strings"
	"time"

	"github.com/pkg/errors"

	sq "github.com/Masterminds/squirrel"
	_ "github.com/lib/pq" // postgres driver
	"github.com/mattermost/focalboard/server/model"
	_ "github.com/mattn/go-sqlite3" // sqlite driver

	"github.com/mattermost/mattermost-server/v6/shared/mlog"
)

type RetentionTableDeletionInfo struct {
	Table         string
	PrimaryKeys   []string
	BoardIDColumn string
}

func (s *SQLStore) runDataRetention(db sq.BaseRunner, globalRetentionDate int64, batchSize int64) (int64, error) {
	s.logger.Info("Start Boards Data Retention",
		mlog.String("Global Retention Date", time.Unix(globalRetentionDate/1000, 0).String()),
		mlog.Int64("Raw Date", globalRetentionDate))
	deleteTables := []RetentionTableDeletionInfo{
		{
			Table:         "blocks",
			PrimaryKeys:   []string{"id"},
			BoardIDColumn: "board_id",
		},
		{
			Table:         "blocks_history",
			PrimaryKeys:   []string{"id"},
			BoardIDColumn: "board_id",
		},
		{
			Table:         "boards",
			PrimaryKeys:   []string{"id"},
			BoardIDColumn: "id",
		},
		{
			Table:         "boards_history",
			PrimaryKeys:   []string{"id"},
			BoardIDColumn: "id",
		},
		{
			Table:         "board_members",
			PrimaryKeys:   []string{"board_id"},
			BoardIDColumn: "board_id",
		},
		{
			Table:         "board_members_history",
			PrimaryKeys:   []string{"board_id"},
			BoardIDColumn: "board_id",
		},
		{
			Table:         "sharing",
			PrimaryKeys:   []string{"id"},
			BoardIDColumn: "id",
		},
		{
			Table:         "category_boards",
			PrimaryKeys:   []string{"id"},
			BoardIDColumn: "board_id",
		},
	}

	subBuilder := s.getQueryBuilder(db).
		Select("board_id, MAX(update_at) AS maxDate").
		From(s.tablePrefix + "blocks").
		GroupBy("board_id")

	subQuery, _, _ := subBuilder.ToSql()

	builder := s.getQueryBuilder(db).
		Select("id").
		From(s.tablePrefix + "boards").
		LeftJoin("( " + subQuery + " ) As subquery ON (subquery.board_id = id)").
		Where(sq.Lt{"maxDate": globalRetentionDate}).
		Where(sq.NotEq{"team_id": "0"}).
		Where(sq.Eq{"is_template": false})

	rows, err := builder.Query()
	if err != nil {
		s.logger.Error(`dataRetention subquery ERROR`, mlog.Err(err))
		return 0, err
	}
	defer s.CloseRows(rows)
	deleteIds, err := idsFromRows(rows)
	if err != nil {
		return 0, err
	}

	totalAffected := 0
	if len(deleteIds) > 0 {
		for _, table := range deleteTables {
			affected, err := s.genericRetentionPoliciesDeletion(db, table, deleteIds, batchSize)
			if err != nil {
				return int64(totalAffected), err
			}
			totalAffected += int(affected)
		}
	}
	s.logger.Info("Complete Boards Data Retention",
		mlog.Int("Total deletion ids", len(deleteIds)),
		mlog.Int("TotalAffected", totalAffected))
	return int64(totalAffected), nil
}

func idsFromRows(rows *sql.Rows) ([]string, error) {
	deleteIds := []string{}
	for rows.Next() {
		var boardID string
		err := rows.Scan(
			&boardID,
		)
		if err != nil {
			return nil, err
		}
		deleteIds = append(deleteIds, boardID)
	}
	return deleteIds, nil
}

// genericRetentionPoliciesDeletion actually executes the DELETE query using a sq.SelectBuilder
// which selects the rows to delete.
func (s *SQLStore) genericRetentionPoliciesDeletion(
	db sq.BaseRunner,
	info RetentionTableDeletionInfo,
	deleteIds []string,
	batchSize int64,
) (int64, error) {
	whereClause := info.BoardIDColumn + " IN ('" + strings.Join(deleteIds, "','") + "')"
	deleteQuery := s.getQueryBuilder(db).
		Delete(s.tablePrefix + info.Table).
		Where(whereClause)

	if batchSize > 0 {
		deleteQuery.Limit(uint64(batchSize))
		primaryKeysStr := "(" + strings.Join(info.PrimaryKeys, ",") + ")"
		if s.dbType != model.MysqlDBType {
			selectQuery := s.getQueryBuilder(db).
				Select(primaryKeysStr).
				From(s.tablePrefix + info.Table).
				Where(whereClause).
				Limit(uint64(batchSize))

			selectString, _, _ := selectQuery.ToSql()

			deleteQuery = s.getQueryBuilder(db).
				Delete(s.tablePrefix + info.Table).
				Where(primaryKeysStr + " IN (" + selectString + ")")
		}
	}

	var totalRowsAffected int64
	var batchRowsAffected int64
	for {
		result, err := deleteQuery.Exec()
		if err != nil {
			return 0, errors.Wrap(err, "failed to delete "+info.Table)
		}

		batchRowsAffected, err = result.RowsAffected()
		if err != nil {
			return 0, errors.Wrap(err, "failed to get rows affected for "+info.Table)
		}
		totalRowsAffected += batchRowsAffected
		if batchRowsAffected != batchSize {
			break
		}
	}
	return totalRowsAffected, nil
}