Use JSONL format for archive

import/asana/importAsana.ts

@@ -3,7 +3,7 @@
 import * as fs from 'fs'
 import minimist from 'minimist'
 import {exit} from 'process'
-import {IArchive} from '../../webapp/src/blocks/archive'
+import {ArchiveUtils} from '../../webapp/src/blocks/archive'
 import {IBlock} from '../../webapp/src/blocks/block'
 import {IPropertyOption, IPropertyTemplate, MutableBoard} from '../../webapp/src/blocks/board'
 import {MutableBoardView} from '../../webapp/src/blocks/boardView'
@@ -49,10 +49,11 @@ function main() {
     const input = JSON.parse(inputData) as Asana
 
     // Convert
-    const output = convert(input)
+    const blocks = convert(input)
 
     // Save output
-    const outputData = JSON.stringify(output)
+    // TODO: Stream output
+    const outputData = ArchiveUtils.buildBlockArchive(blocks)
     fs.writeFileSync(outputFile, outputData)
 
     console.log(`Exported to ${outputFile}`)
@@ -87,17 +88,11 @@ function getSections(input: Asana, projectId: string): Workspace[] {
     return [...sectionMap.values()]
 }
 
-function convert(input: Asana): IArchive {
+function convert(input: Asana): IBlock[] {
-    const archive: IArchive = {
-        version: 1,
-        date: Date.now(),
-        blocks: []
-    }
-
     const projects = getProjects(input)
     if (projects.length < 1) {
         console.error('No projects found')
-        return archive
+        return []
     }
 
     // TODO: Handle multiple projects
@@ -181,12 +176,10 @@ function convert(input: Asana): IArchive {
         }
     })
 
-    archive.blocks = blocks
-
     console.log('')
     console.log(`Found ${input.data.length} card(s).`)
 
-    return archive
+    return blocks
 }
 
 function showHelp() {

import/asana/package.json

@@ -7,8 +7,8 @@
   "scripts": {
     "lint": "eslint --ext .tsx,.ts . --quiet --cache",
 	"fix": "eslint --ext .tsx,.ts . --quiet --fix --cache",
-    "test": "ts-node importAsana.ts -i test/asana.json -o test/archive.focalboard",
+    "test": "ts-node importAsana.ts -i test/asana.json -o test/asana-import.focalboard",
-    "debug:test": "node --inspect=5858 -r ts-node/register importAsana.ts -i test/asana.json -o test/archive.focalboard"
+    "debug:test": "node --inspect=5858 -r ts-node/register importAsana.ts -i test/asana.json -o test/asana-import.focalboard"
   },
   "keywords": [],
   "author": "",

import/notion/importNotion.ts

@@ -3,7 +3,7 @@ import * as fs from 'fs'
 import minimist from 'minimist'
 import path from 'path'
 import {exit} from 'process'
-import {IArchive} from '../../webapp/src/blocks/archive'
+import {ArchiveUtils} from '../../webapp/src/blocks/archive'
 import {IBlock} from '../../webapp/src/blocks/block'
 import {IPropertyTemplate, MutableBoard} from '../../webapp/src/blocks/board'
 import {MutableBoardView} from '../../webapp/src/blocks/boardView'
@@ -70,10 +70,11 @@ async function main() {
     markdownFolder = path.join(inputFolder, basename)
 
     // Convert
-    const output = convert(input, title)
+    const blocks = convert(input, title)
 
     // Save output
-    const outputData = JSON.stringify(output)
+    // TODO: Stream output
+    const outputData = ArchiveUtils.buildBlockArchive(blocks)
     fs.writeFileSync(outputFile, outputData)
 
     console.log(`Exported to ${outputFile}`)
@@ -115,15 +116,9 @@ function getColumns(input: any[]) {
     return keys.slice(1)
 }
 
-function convert(input: any[], title: string): IArchive {
+function convert(input: any[], title: string): IBlock[] {
     const blocks: IBlock[] = []
 
-    const archive: IArchive = {
-        version: 1,
-        date: Date.now(),
-        blocks
-    }
-
     // Board
     const board = new MutableBoard()
     console.log(`Board: ${title}`)
@@ -160,7 +155,7 @@ function convert(input: any[], title: string): IArchive {
         console.log(keys)
         if (keys.length < 1) {
             console.error(`Expected at least one column`)
-            return archive
+            return blocks
         }
 
         const titleKey = keys[0]
@@ -216,7 +211,7 @@ function convert(input: any[], title: string): IArchive {
     console.log('')
     console.log(`Found ${input.length} card(s).`)
 
-    return archive
+    return blocks
 }
 
 function showHelp() {

import/notion/package.json

@@ -7,8 +7,8 @@
   "scripts": {
 	"lint": "eslint --ext .tsx,.ts . --quiet --cache",
 	"fix": "eslint --ext .tsx,.ts . --quiet --fix --cache",
-	"test": "ts-node importNotion.ts -i test/export -o test/archive.focalboard",
+	"test": "ts-node importNotion.ts -i test/export -o test/notion-import.focalboard",
-    "debug:test": "node --inspect=5858 -r ts-node/register importNotion.ts -i test/export -o test/archive.focalboard"
+    "debug:test": "node --inspect=5858 -r ts-node/register importNotion.ts -i test/export -o test/notion-import.focalboard"
   },
   "keywords": [],
   "author": "",

import/trello/importTrello.ts

@@ -3,7 +3,7 @@
 import * as fs from 'fs'
 import minimist from 'minimist'
 import {exit} from 'process'
-import {IArchive} from '../../webapp/src/blocks/archive'
+import {ArchiveUtils} from '../../webapp/src/blocks/archive'
 import {IBlock} from '../../webapp/src/blocks/block'
 import {IPropertyOption, IPropertyTemplate, MutableBoard} from '../../webapp/src/blocks/board'
 import {MutableBoardView} from '../../webapp/src/blocks/boardView'
@@ -49,16 +49,17 @@ function main() {
     const input = JSON.parse(inputData) as Trello
 
     // Convert
-    const output = convert(input)
+    const blocks = convert(input)
 
     // Save output
-    const outputData = JSON.stringify(output)
+    // TODO: Stream output
+    const outputData = ArchiveUtils.buildBlockArchive(blocks)
     fs.writeFileSync(outputFile, outputData)
 
     console.log(`Exported to ${outputFile}`)
 }
 
-function convert(input: Trello): IArchive {
+function convert(input: Trello): IBlock[] {
     const blocks: IBlock[] = []
 
     // Board
@@ -136,16 +137,10 @@ function convert(input: Trello): IArchive {
         }
     })
 
-    const archive: IArchive = {
-        version: 1,
-        date: Date.now(),
-        blocks
-    }
-
     console.log('')
     console.log(`Found ${input.cards.length} card(s).`)
 
-    return archive
+    return blocks
 }
 
 function showHelp() {

import/trello/package.json

@@ -7,8 +7,8 @@
   "scripts": {
 	"lint": "eslint --ext .tsx,.ts . --quiet --cache",
 	"fix": "eslint --ext .tsx,.ts . --quiet --fix --cache",
-    "test": "ts-node importTrello.ts -i test/trello.json -o test/archive.focalboard",
+    "test": "ts-node importTrello.ts -i test/trello.json -o test/trello-import.focalboard",
-    "debug:test": "node --inspect=5858 -r ts-node/register importTrello.ts -i test/trello.json -o test/archive.focalboard"
+    "debug:test": "node --inspect=5858 -r ts-node/register importTrello.ts -i test/trello.json -o test/trello-import.focalboard"
   },
   "keywords": [],
   "author": "",

webapp/src/archiver.ts

@@ -1,7 +1,7 @@
 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
 // See LICENSE.txt for license information.
-import {IArchive} from './blocks/archive'
+import {ArchiveUtils, IArchiveHeader, IArchiveLine, IBlockArchiveLine} from './blocks/archive'
-import {IMutableBlock} from './blocks/block'
+import {IBlock, IMutableBlock} from './blocks/block'
 import mutator from './mutator'
 import {Utils} from './utils'
 import {BoardTree} from './viewModel/boardTree'
@@ -9,28 +9,16 @@ import {BoardTree} from './viewModel/boardTree'
 class Archiver {
     static async exportBoardTree(boardTree: BoardTree): Promise<void> {
         const blocks = boardTree.allBlocks
-        const archive: IArchive = {
+        this.exportArchive(blocks)
-            version: 1,
-            date: Date.now(),
-            blocks,
-        }
-
-        this.exportArchive(archive)
     }
 
     static async exportFullArchive(): Promise<void> {
         const blocks = await mutator.exportFullArchive()
-        const archive: IArchive = {
+        this.exportArchive(blocks)
-            version: 1,
-            date: Date.now(),
-            blocks,
-        }
-
-        this.exportArchive(archive)
     }
 
-    private static exportArchive(archive: IArchive): void {
+    private static exportArchive(blocks: readonly IBlock[]): void {
-        const content = JSON.stringify(archive)
+        const content = ArchiveUtils.buildBlockArchive(blocks)
 
         const date = new Date()
         const filename = `archive-${date.getFullYear()}-${date.getMonth() + 1}-${date.getDate()}.focalboard`
@@ -48,32 +36,73 @@ class Archiver {
         // TODO: Remove or reuse link
     }
 
+    private static async readBlocksFromFile(file: File): Promise<IBlock[]> {
+        // TODO: Read input as a stream, line by line
+        const contents = await (new Response(file)).text()
+        Utils.log(`Import ${contents.length} bytes.`)
+
+        const blocks: IBlock[] = []
+        const allLineStrings = contents.split('\n')
+        if (allLineStrings.length >= 2) {
+            const headerString = allLineStrings[0]
+            const header = JSON.parse(headerString) as IArchiveHeader
+            if (header.date && header.version >= 1) {
+                const date = new Date(header.date)
+                Utils.log(`Import archive, version: ${header.version}, date/time: ${date.toLocaleString()}, ${blocks.length} block(s).`)
+
+                const lineStrings = allLineStrings.slice(1)
+                for (const lineString of lineStrings) {
+                    if (!lineString) {
+                        // Ignore empty lines, e.g. last line
+                        continue
+                    }
+
+                    const line = JSON.parse(lineString) as IArchiveLine
+                    if (!line || !line.type || !line.data) {
+                        Utils.logError('importFullArchive ERROR parsing line')
+                        continue
+                    }
+                    switch (line.type) {
+                    case 'block': {
+                        const blockLine = line as IBlockArchiveLine
+                        const block = blockLine.data
+                        blocks.push(block)
+                        break
+                    }
+                    }
+                }
+            } else {
+                Utils.logError('importFullArchive ERROR parsing header')
+            }
+        }
+
+        return blocks
+    }
+
     static importFullArchive(onComplete?: () => void): void {
         const input = document.createElement('input')
         input.type = 'file'
         input.accept = '.focalboard'
         input.onchange = async () => {
             const file = input.files && input.files[0]
-            const contents = await (new Response(file)).text()
+            if (file) {
-            Utils.log(`Import ${contents.length} bytes.`)
+                const blocks = await Archiver.readBlocksFromFile(file)
-            const archive: IArchive = JSON.parse(contents)
-            const {blocks} = archive
-            const date = new Date(archive.date)
-            Utils.log(`Import archive, version: ${archive.version}, date/time: ${date.toLocaleString()}, ${blocks.length} block(s).`)
 
-            // Basic error checking
+                // Basic error checking
-            let filteredBlocks = blocks.filter((o) => Boolean(o.id))
+                let filteredBlocks = blocks.filter((o) => Boolean(o.id))
 
-            Utils.log(`Import ${filteredBlocks.length} filtered blocks with ids.`)
+                Utils.log(`Import ${filteredBlocks.length} filtered blocks with ids.`)
 
-            this.fixRootIds(filteredBlocks)
+                this.fixRootIds(filteredBlocks)
 
-            filteredBlocks = filteredBlocks.filter((o) => Boolean(o.rootId))
+                filteredBlocks = filteredBlocks.filter((o) => Boolean(o.rootId))
 
-            Utils.log(`Import ${filteredBlocks.length} filtered blocks with rootIds.`)
+                Utils.log(`Import ${filteredBlocks.length} filtered blocks with rootIds.`)
+
+                await mutator.importFullArchive(filteredBlocks)
+                Utils.log('Import completed')
+            }
 
-            await mutator.importFullArchive(filteredBlocks)
-            Utils.log('Import completed')
             onComplete?.()
         }
 

webapp/src/blocks/archive.ts

@@ -2,10 +2,43 @@
 // See LICENSE.txt for license information.
 import {IBlock} from './block'
 
-interface IArchive {
+interface IArchiveHeader {
     version: number
     date: number
-    blocks: readonly IBlock[]
 }
 
-export {IArchive}
+interface IArchiveLine {
+    type: string,
+    data: any,
+}
+
+// This schema allows the expansion of additional line types in the future
+interface IBlockArchiveLine extends IArchiveLine {
+    type: 'block',
+    data: IBlock
+}
+
+class ArchiveUtils {
+    static buildBlockArchive(blocks: readonly IBlock[]): string {
+        const header: IArchiveHeader = {
+            version: 1,
+            date: Date.now(),
+        }
+
+        const headerString = JSON.stringify(header)
+        let content = headerString + '\n'
+        for (const block of blocks) {
+            const line: IBlockArchiveLine = {
+                type: 'block',
+                data: block,
+            }
+            const lineString = JSON.stringify(line)
+            content += lineString
+            content += '\n'
+        }
+
+        return content
+    }
+}
+
+export {IArchiveHeader, IArchiveLine, IBlockArchiveLine, ArchiveUtils}