1
0
mirror of https://github.com/mattermost/focalboard.git synced 2024-12-03 08:45:40 +02:00
focalboard/server/app/import.go
Doug Lauder a47baf7f23
Multi version archive import (#2220)
* Fixed panic in PropDef.GetValue for `person` property types.
- the GetUserByID API can return a nil user and nil error

* return userid

* support old archive format

* move template init to app layer

* move app init

* init app

* revert

* ignore GetDefaultTemplate blocks call in store mock

* ignore GetDefaultTemplate blocks call in store mock2

* ignore InsertBlockss call in store mock3

* ignore RemoveDefaultTemplates call in store mock4

* ignore WriteFile call in files mock5

* ignore WriteFile call in files mock6

* ignore WriteFile call in files mock7

* ignore WriteFile call in files mock8

* fix unit tests

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
2022-02-02 11:25:06 -07:00

209 lines
5.5 KiB
Go

package app
import (
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"path"
"path/filepath"
"strings"
"github.com/krolaw/zipstream"
"github.com/mattermost/focalboard/server/model"
"github.com/mattermost/focalboard/server/services/store"
"github.com/mattermost/focalboard/server/utils"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
)
const (
archiveVersion = 2
legacyFileBegin = "{\"version\":1"
)
// ImportArchive imports an archive containing zero or more boards, plus all
// associated content, including cards, content blocks, views, and images.
//
// Archives are ZIP files containing a `version.json` file and zero or more
// directories, each containing a `board.jsonl` and zero or more image files.
func (a *App) ImportArchive(r io.Reader, opt model.ImportArchiveOptions) error {
// peek at the first bytes to see if this is a legacy archive format
br := bufio.NewReader(r)
peek, err := br.Peek(len(legacyFileBegin))
if err == nil && string(peek) == legacyFileBegin {
a.logger.Debug("importing legacy archive")
_, errImport := a.ImportBoardJSONL(br, opt)
return errImport
}
a.logger.Debug("importing archive")
zr := zipstream.NewReader(br)
boardMap := make(map[string]string) // maps old board ids to new
for {
hdr, err := zr.Next()
if err != nil {
if errors.Is(err, io.EOF) {
a.logger.Debug("import archive - done", mlog.Int("boards_imported", len(boardMap)))
return nil
}
return err
}
dir, filename := path.Split(hdr.Name)
dir = path.Clean(dir)
switch filename {
case "version.json":
ver, errVer := parseVersionFile(zr)
if errVer != nil {
return errVer
}
if ver != archiveVersion {
return model.NewErrUnsupportedArchiveVersion(ver, archiveVersion)
}
case "board.jsonl":
boardID, err := a.ImportBoardJSONL(zr, opt)
if err != nil {
return fmt.Errorf("cannot import board %s: %w", dir, err)
}
boardMap[dir] = boardID
default:
// import file/image; dir is the old board id
boardID, ok := boardMap[dir]
if !ok {
a.logger.Error("skipping orphan image in archive",
mlog.String("dir", dir),
mlog.String("filename", filename),
)
continue
}
// save file with original filename so it matches name in image block.
filePath := filepath.Join(opt.WorkspaceID, boardID, filename)
_, err := a.filesBackend.WriteFile(zr, filePath)
if err != nil {
return fmt.Errorf("cannot import file %s for board %s: %w", filename, dir, err)
}
}
a.logger.Trace("import archive file",
mlog.String("dir", dir),
mlog.String("filename", filename),
)
}
}
// ImportBoardJSONL imports a JSONL file containing blocks for one board. The resulting
// board id is returned.
func (a *App) ImportBoardJSONL(r io.Reader, opt model.ImportArchiveOptions) (string, error) {
// TODO: Stream this once `model.GenerateBlockIDs` can take a stream of blocks.
// We don't want to load the whole file in memory, even though it's a single board.
blocks := make([]model.Block, 0, 10)
lineReader := bufio.NewReader(r)
userID := opt.ModifiedBy
if userID == model.SingleUser {
userID = ""
}
now := utils.GetMillis()
lineNum := 1
for {
line, errRead := readLine(lineReader)
if len(line) != 0 {
var skip bool
if lineNum == 1 {
// first line might be a header tag (old archive format)
if strings.HasPrefix(string(line), legacyFileBegin) {
skip = true
}
}
if !skip {
var archiveLine model.ArchiveLine
if err := json.Unmarshal(line, &archiveLine); err != nil {
return "", fmt.Errorf("error parsing archive line %d: %w", lineNum, err)
}
switch archiveLine.Type {
case "block":
var block model.Block
if err2 := json.Unmarshal(archiveLine.Data, &block); err2 != nil {
return "", fmt.Errorf("invalid block in archive line %d: %w", lineNum, err2)
}
block.ModifiedBy = userID
block.UpdateAt = now
blocks = append(blocks, block)
default:
return "", model.NewErrUnsupportedArchiveLineType(lineNum, archiveLine.Type)
}
}
}
if errRead != nil {
if errors.Is(errRead, io.EOF) {
break
}
return "", fmt.Errorf("error reading archive line %d: %w", lineNum, errRead)
}
lineNum++
}
modInfoCache := make(map[string]interface{})
modBlocks := make([]model.Block, 0, len(blocks))
for _, block := range blocks {
b := block
if opt.BlockModifier != nil && !opt.BlockModifier(&b, modInfoCache) {
a.logger.Debug("skipping insert block per block modifier",
mlog.String("blockID", block.ID),
mlog.String("block_type", block.Type.String()),
)
continue
}
modBlocks = append(modBlocks, b)
}
blocks = model.GenerateBlockIDs(modBlocks, a.logger)
container := store.Container{
WorkspaceID: opt.WorkspaceID,
}
var err error
blocks, err = a.InsertBlocks(container, blocks, opt.ModifiedBy, false)
if err != nil {
return "", fmt.Errorf("error inserting archive blocks: %w", err)
}
// find new board id
for _, block := range blocks {
if block.Type == model.TypeBoard {
return block.ID, nil
}
}
return "", fmt.Errorf("missing board in archive: %w", model.ErrInvalidBoardBlock)
}
func parseVersionFile(r io.Reader) (int, error) {
file, err := io.ReadAll(r)
if err != nil {
return 0, fmt.Errorf("cannot read version.json: %w", err)
}
var header model.ArchiveHeader
if err := json.Unmarshal(file, &header); err != nil {
return 0, fmt.Errorf("cannot parse version.json: %w", err)
}
return header.Version, nil
}
func readLine(r *bufio.Reader) ([]byte, error) {
line, err := r.ReadBytes('\n')
line = bytes.TrimSpace(line)
return line, err
}