self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-21 09:38:01 +02:00
Code Issues Releases Activity

Chore: Converts ENEX import file to TypeScript

Browse Source
This commit is contained in:
Laurent Cozic 2021-09-23 13:15:31 +01:00
parent f33088fbe0
commit 62f81b4315
7 changed files with 102 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.eslintignore
View File

@ -1000,6 +1000,9 @@ packages/lib/import-enex-md-gen.js.map
packages/lib/import-enex-md-gen.test.d.ts packages/lib/import-enex-md-gen.test.d.ts
packages/lib/import-enex-md-gen.test.js packages/lib/import-enex-md-gen.test.js
packages/lib/import-enex-md-gen.test.js.map packages/lib/import-enex-md-gen.test.js.map
packages/lib/import-enex.d.ts
packages/lib/import-enex.js
packages/lib/import-enex.js.map
packages/lib/locale.d.ts packages/lib/locale.d.ts
packages/lib/locale.js packages/lib/locale.js
packages/lib/locale.js.map packages/lib/locale.js.map

3
.gitignore vendored
View File

@ -985,6 +985,9 @@ packages/lib/import-enex-md-gen.js.map
packages/lib/import-enex-md-gen.test.d.ts packages/lib/import-enex-md-gen.test.d.ts
packages/lib/import-enex-md-gen.test.js packages/lib/import-enex-md-gen.test.js
packages/lib/import-enex-md-gen.test.js.map packages/lib/import-enex-md-gen.test.js.map
packages/lib/import-enex.d.ts
packages/lib/import-enex.js
packages/lib/import-enex.js.map
packages/lib/locale.d.ts packages/lib/locale.d.ts
packages/lib/locale.js packages/lib/locale.js
packages/lib/locale.js.map packages/lib/locale.js.map

2
packages/app-desktop/gui/ImportScreen.jsx
View File

@ -4,7 +4,7 @@ const Folder = require('@joplin/lib/models/Folder').default;
const { themeStyle } = require('@joplin/lib/theme'); const { themeStyle } = require('@joplin/lib/theme');
const { _ } = require('@joplin/lib/locale'); const { _ } = require('@joplin/lib/locale');
const { filename, basename } = require('@joplin/lib/path-utils'); const { filename, basename } = require('@joplin/lib/path-utils');
const { importEnex } = require('@joplin/lib/import-enex'); const importEnex = require('@joplin/lib/import-enex').default;
class ImportScreenComponent extends React.Component { class ImportScreenComponent extends React.Component {
UNSAFE_componentWillMount() { UNSAFE_componentWillMount() {

2
packages/lib/import-enex-md-gen.test.ts
View File

@ -6,7 +6,7 @@ const os = require('os');
const { filename } = require('./path-utils'); const { filename } = require('./path-utils');
import { setupDatabaseAndSynchronizer, switchClient, expectNotThrow, supportDir } from './testing/test-utils'; import { setupDatabaseAndSynchronizer, switchClient, expectNotThrow, supportDir } from './testing/test-utils';
const { enexXmlToMd } = require('./import-enex-md-gen.js'); const { enexXmlToMd } = require('./import-enex-md-gen.js');
const { importEnex } = require('./import-enex'); import importEnex from './import-enex';
import Note from './models/Note'; import Note from './models/Note';
import Tag from './models/Tag'; import Tag from './models/Tag';
import Resource from './models/Resource'; import Resource from './models/Resource';

144
packages/lib/import-enex.js → packages/lib/import-enex.ts
View File

@ -1,26 +1,27 @@
const uuid = require('./uuid').default; import uuid from './uuid';
import BaseModel from './BaseModel';
import Note from './models/Note';
import Tag from './models/Tag';
import Resource from './models/Resource';
import Setting from './models/Setting';
import time from './time';
import shim from './shim';
import { NoteEntity } from './services/database/types';
import { enexXmlToMd } from './import-enex-md-gen';
import { MarkupToHtml } from '@joplin/renderer';
const moment = require('moment'); const moment = require('moment');
const BaseModel = require('./BaseModel').default;
const Note = require('./models/Note').default;
const Tag = require('./models/Tag').default;
const Resource = require('./models/Resource').default;
const Setting = require('./models/Setting').default;
const { MarkupToHtml } = require('@joplin/renderer');
const { wrapError } = require('./errorUtils'); const { wrapError } = require('./errorUtils');
const { enexXmlToMd } = require('./import-enex-md-gen.js');
const { enexXmlToHtml } = require('./import-enex-html-gen.js'); const { enexXmlToHtml } = require('./import-enex-html-gen.js');
const time = require('./time').default;
const Levenshtein = require('levenshtein'); const Levenshtein = require('levenshtein');
const md5 = require('md5'); const md5 = require('md5');
const { Base64Decode } = require('base64-stream'); const { Base64Decode } = require('base64-stream');
const md5File = require('md5-file'); const md5File = require('md5-file');
const shim = require('./shim').default;
const { mime } = require('./mime-utils'); const { mime } = require('./mime-utils');
// const Promise = require('promise'); // const Promise = require('promise');
const fs = require('fs-extra'); const fs = require('fs-extra');
function dateToTimestamp(s, defaultValue = null) { function dateToTimestamp(s: string, defaultValue: number = null): number {
// Most dates seem to be in this format // Most dates seem to be in this format
let m = moment(s, 'YYYYMMDDTHHmmssZ'); let m = moment(s, 'YYYYMMDDTHHmmssZ');
@ -36,12 +37,12 @@ function dateToTimestamp(s, defaultValue = null) {
return m.toDate().getTime(); return m.toDate().getTime();
} }
function extractRecognitionObjId(recognitionXml) { function extractRecognitionObjId(recognitionXml: string) {
const r = recognitionXml.match(/objID="(.*?)"/); const r = recognitionXml.match(/objID="(.*?)"/);
return r && r.length >= 2 ? r[1] : null; return r && r.length >= 2 ? r[1] : null;
} }
async function decodeBase64File(sourceFilePath, destFilePath) { async function decodeBase64File(sourceFilePath: string, destFilePath: string) {
// When something goes wrong with streams you can get an error "EBADF, Bad file descriptor" // When something goes wrong with streams you can get an error "EBADF, Bad file descriptor"
// with no strack trace to tell where the error happened. // with no strack trace to tell where the error happened.
@ -73,17 +74,17 @@ async function decodeBase64File(sourceFilePath, destFilePath) {
destStream.on('finish', () => { destStream.on('finish', () => {
fs.fdatasyncSync(destFile); fs.fdatasyncSync(destFile);
fs.closeSync(destFile); fs.closeSync(destFile);
resolve(); resolve(null);
}); });
sourceStream.on('error', (error) => reject(error)); sourceStream.on('error', (error: any) => reject(error));
destStream.on('error', (error) => reject(error)); destStream.on('error', (error: any) => reject(error));
}); });
} }
async function md5FileAsync(filePath) { async function md5FileAsync(filePath: string): Promise<string> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
md5File(filePath, (error, hash) => { md5File(filePath, (error: any, hash: string) => {
if (error) { if (error) {
reject(error); reject(error);
return; return;
@ -94,24 +95,24 @@ async function md5FileAsync(filePath) {
}); });
} }
function removeUndefinedProperties(note) { function removeUndefinedProperties(note: NoteEntity) {
const output = {}; const output: any = {};
for (const n in note) { for (const n in note) {
if (!note.hasOwnProperty(n)) continue; if (!note.hasOwnProperty(n)) continue;
const v = note[n]; const v = (note as any)[n];
if (v === undefined || v === null) continue; if (v === undefined || v === null) continue;
output[n] = v; output[n] = v;
} }
return output; return output;
} }
function levenshteinPercent(s1, s2) { function levenshteinPercent(s1: string, s2: string) {
const l = new Levenshtein(s1, s2); const l = new Levenshtein(s1, s2);
if (!s1.length || !s2.length) return 1; if (!s1.length || !s2.length) return 1;
return Math.abs(l.distance / s1.length); return Math.abs(l.distance / s1.length);
} }
async function fuzzyMatch(note) { async function fuzzyMatch(note: ExtractedNote) {
if (note.created_time < time.unixMs() - 1000 * 60 * 60 * 24 * 360) { if (note.created_time < time.unixMs() - 1000 * 60 * 60 * 24 * 360) {
const notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ? AND title = ?', [note.created_time, note.title]); const notes = await Note.modelSelectAll('SELECT * FROM notes WHERE is_conflict = 0 AND created_time = ? AND title = ?', [note.created_time, note.title]);
return notes.length !== 1 ? null : notes[0]; return notes.length !== 1 ? null : notes[0];
@ -137,9 +138,30 @@ async function fuzzyMatch(note) {
return null; return null;
} }
interface ExtractedResource {
hasData?: boolean;
id?: string;
size?: number;
dataFilePath?: string;
dataEncoding?: string;
data?: string;
filename?: string;
sourceUrl?: string;
mime?: string;
title?: string;
}
interface ExtractedNote extends NoteEntity {
resources?: ExtractedResource[];
tags?: string[];
title?: string;
bodyXml?: string;
// is_todo?: boolean;
}
// At this point we have the resource has it's been parsed from the XML, but additional // At this point we have the resource has it's been parsed from the XML, but additional
// processing needs to be done to get the final resource file, its size, MD5, etc. // processing needs to be done to get the final resource file, its size, MD5, etc.
async function processNoteResource(resource) { async function processNoteResource(resource: ExtractedResource) {
if (!resource.hasData) { if (!resource.hasData) {
// Some resources have no data, go figure, so we need a special case for this. // Some resources have no data, go figure, so we need a special case for this.
resource.id = md5(Date.now() + Math.random()); resource.id = md5(Date.now() + Math.random());
@ -175,7 +197,7 @@ async function processNoteResource(resource) {
return resource; return resource;
} }
async function saveNoteResources(note) { async function saveNoteResources(note: ExtractedNote) {
let resourcesCreated = 0; let resourcesCreated = 0;
for (let i = 0; i < note.resources.length; i++) { for (let i = 0; i < note.resources.length; i++) {
const resource = note.resources[i]; const resource = note.resources[i];
@ -198,7 +220,7 @@ async function saveNoteResources(note) {
return resourcesCreated; return resourcesCreated;
} }
async function saveNoteTags(note) { async function saveNoteTags(note: ExtractedNote) {
let notesTagged = 0; let notesTagged = 0;
for (let i = 0; i < note.tags.length; i++) { for (let i = 0; i < note.tags.length; i++) {
const tagTitle = note.tags[i]; const tagTitle = note.tags[i];
@ -213,12 +235,19 @@ async function saveNoteTags(note) {
return notesTagged; return notesTagged;
} }
async function saveNoteToStorage(note, importOptions) { interface ImportOptions {
fuzzyMatching?: boolean;
onProgress?: Function;
onError?: Function;
outputFormat?: string;
}
async function saveNoteToStorage(note: ExtractedNote, importOptions: ImportOptions) {
importOptions = Object.assign({}, { importOptions = Object.assign({}, {
fuzzyMatching: false, fuzzyMatching: false,
}, importOptions); }, importOptions);
note = Note.filter(note); note = Note.filter(note as any);
const existingNote = importOptions.fuzzyMatching ? await fuzzyMatch(note) : null; const existingNote = importOptions.fuzzyMatching ? await fuzzyMatch(note) : null;
@ -230,7 +259,7 @@ async function saveNoteToStorage(note, importOptions) {
notesTagged: 0, notesTagged: 0,
}; };
const resourcesCreated = await saveNoteResources(note, importOptions); const resourcesCreated = await saveNoteResources(note);
result.resourcesCreated += resourcesCreated; result.resourcesCreated += resourcesCreated;
const notesTagged = await saveNoteTags(note); const notesTagged = await saveNoteTags(note);
@ -262,16 +291,25 @@ async function saveNoteToStorage(note, importOptions) {
return result; return result;
} }
function importEnex(parentFolderId, filePath, importOptions = null) { interface Node {
name: string;
attributes: Record<string, any>;
}
interface NoteResourceRecognition {
objID?: string;
}
export default function importEnex(parentFolderId: string, filePath: string, importOptions: ImportOptions = null) {
if (!importOptions) importOptions = {}; if (!importOptions) importOptions = {};
if (!('fuzzyMatching' in importOptions)) importOptions.fuzzyMatching = false; if (!('fuzzyMatching' in importOptions)) importOptions.fuzzyMatching = false;
if (!('onProgress' in importOptions)) importOptions.onProgress = function() {}; if (!('onProgress' in importOptions)) importOptions.onProgress = function() {};
if (!('onError' in importOptions)) importOptions.onError = function() {}; if (!('onError' in importOptions)) importOptions.onError = function() {};
function handleSaxStreamEvent(fn) { function handleSaxStreamEvent(fn: Function) {
return function(...args) { return function(...args: any[]) {
// Pass the parser to the wrapped function for debugging purposes // Pass the parser to the wrapped function for debugging purposes
if (this._parser) fn._parser = this._parser; if (this._parser) (fn as any)._parser = this._parser;
try { try {
fn.call(this, ...args); fn.call(this, ...args);
@ -301,16 +339,16 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
const strict = true; const strict = true;
const saxStream = require('@joplin/fork-sax').createStream(strict, options); const saxStream = require('@joplin/fork-sax').createStream(strict, options);
const nodes = []; // LIFO list of nodes so that we know in which node we are in the onText event const nodes: Node[] = []; // LIFO list of nodes so that we know in which node we are in the onText event
let note = null; let note: ExtractedNote = null;
let noteAttributes = null; let noteAttributes: Record<string, any> = null;
let noteResource = null; let noteResource: ExtractedResource = null;
let noteResourceAttributes = null; let noteResourceAttributes: Record<string, any> = null;
let noteResourceRecognition = null; let noteResourceRecognition: NoteResourceRecognition = null;
const notes = []; const notes: ExtractedNote[] = [];
let processingNotes = false; let processingNotes = false;
const createErrorWithNoteTitle = (fnThis, error) => { const createErrorWithNoteTitle = (fnThis: any, error: any) => {
const line = []; const line = [];
const parser = fnThis ? fnThis._parser : null; const parser = fnThis ? fnThis._parser : null;
@ -329,7 +367,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
return error; return error;
}; };
stream.on('error', function(error) { stream.on('error', function(error: any) {
importOptions.onError(createErrorWithNoteTitle(this, error)); importOptions.onError(createErrorWithNoteTitle(this, error));
}); });
@ -417,11 +455,11 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
return true; return true;
} }
saxStream.on('error', function(error) { saxStream.on('error', function(error: any) {
importOptions.onError(createErrorWithNoteTitle(this, error)); importOptions.onError(createErrorWithNoteTitle(this, error));
}); });
saxStream.on('text', handleSaxStreamEvent(function(text) { saxStream.on('text', handleSaxStreamEvent(function(text: string) {
const n = currentNodeName(); const n = currentNodeName();
if (noteAttributes) { if (noteAttributes) {
@ -443,8 +481,8 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
fs.appendFileSync(noteResource.dataFilePath, text); fs.appendFileSync(noteResource.dataFilePath, text);
} else { } else {
if (!(n in noteResource)) noteResource[n] = ''; if (!(n in noteResource)) (noteResource as any)[n] = '';
noteResource[n] += text; (noteResource as any)[n] += text;
} }
} else if (note) { } else if (note) {
if (n == 'title') { if (n == 'title') {
@ -465,7 +503,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
} }
})); }));
saxStream.on('opentag', handleSaxStreamEvent(function(node) { saxStream.on('opentag', handleSaxStreamEvent(function(node: Node) {
const n = node.name.toLowerCase(); const n = node.name.toLowerCase();
nodes.push(node); nodes.push(node);
@ -488,7 +526,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
} }
})); }));
saxStream.on('cdata', handleSaxStreamEvent(function(data) { saxStream.on('cdata', handleSaxStreamEvent(function(data: any) {
const n = currentNodeName(); const n = currentNodeName();
if (noteResourceRecognition) { if (noteResourceRecognition) {
@ -500,7 +538,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
} }
})); }));
saxStream.on('closetag', handleSaxStreamEvent(function(n) { saxStream.on('closetag', handleSaxStreamEvent(function(n: string) {
nodes.pop(); nodes.pop();
if (n == 'note') { if (n == 'note') {
@ -529,7 +567,7 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
note.longitude = noteAttributes.longitude; note.longitude = noteAttributes.longitude;
note.altitude = noteAttributes.altitude; note.altitude = noteAttributes.altitude;
note.author = noteAttributes.author ? noteAttributes.author.trim() : ''; note.author = noteAttributes.author ? noteAttributes.author.trim() : '';
note.is_todo = noteAttributes['reminder-order'] !== '0' && !!noteAttributes['reminder-order']; note.is_todo = noteAttributes['reminder-order'] !== '0' && !!noteAttributes['reminder-order'] as any;
note.todo_due = dateToTimestamp(noteAttributes['reminder-time'], 0); note.todo_due = dateToTimestamp(noteAttributes['reminder-time'], 0);
note.todo_completed = dateToTimestamp(noteAttributes['reminder-done-time'], 0); note.todo_completed = dateToTimestamp(noteAttributes['reminder-done-time'], 0);
note.order = dateToTimestamp(noteAttributes['reminder-order'], 0); note.order = dateToTimestamp(noteAttributes['reminder-order'], 0);
@ -572,10 +610,10 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
saxStream.on('end', handleSaxStreamEvent(function() { saxStream.on('end', handleSaxStreamEvent(function() {
// Wait till there is no more notes to process. // Wait till there is no more notes to process.
const iid = shim.setInterval(() => { const iid = shim.setInterval(() => {
processNotes().then(allDone => { void processNotes().then(allDone => {
if (allDone) { if (allDone) {
shim.clearTimeout(iid); shim.clearTimeout(iid);
resolve(); resolve(null);
} }
}); });
}, 500); }, 500);
@ -584,5 +622,3 @@ function importEnex(parentFolderId, filePath, importOptions = null) {
stream.pipe(saxStream); stream.pipe(saxStream);
}); });
} }
module.exports = { importEnex };

5
packages/lib/services/interop/InteropService_Importer_EnexToHtml.ts
View File

@ -1,12 +1,11 @@
import { ImportExportResult } from './types'; import { ImportExportResult } from './types';
import InteropService_Importer_Base from './InteropService_Importer_Base'; import InteropService_Importer_Base from './InteropService_Importer_Base';
import Folder from '../../models/Folder'; import Folder from '../../models/Folder';
import importEnex from '../../import-enex';
const { filename } = require('../../path-utils'); const { filename } = require('../../path-utils');
export default class InteropService_Importer_EnexToHtml extends InteropService_Importer_Base { export default class InteropService_Importer_EnexToHtml extends InteropService_Importer_Base {
async exec(result: ImportExportResult): Promise<ImportExportResult> { public async exec(result: ImportExportResult): Promise<ImportExportResult> {
const { importEnex } = require('../../import-enex');
let folder = this.options_.destinationFolder; let folder = this.options_.destinationFolder;
if (!folder) { if (!folder) {

6
packages/lib/services/interop/InteropService_Importer_EnexToMd.ts
View File

@ -1,13 +1,11 @@
import { ImportExportResult } from './types'; import { ImportExportResult } from './types';
import importEnex from '../../import-enex';
import InteropService_Importer_Base from './InteropService_Importer_Base'; import InteropService_Importer_Base from './InteropService_Importer_Base';
import Folder from '../../models/Folder'; import Folder from '../../models/Folder';
const { filename } = require('../../path-utils'); const { filename } = require('../../path-utils');
export default class InteropService_Importer_EnexToMd extends InteropService_Importer_Base { export default class InteropService_Importer_EnexToMd extends InteropService_Importer_Base {
async exec(result: ImportExportResult) { public async exec(result: ImportExportResult) {
const { importEnex } = require('../../import-enex');
let folder = this.options_.destinationFolder; let folder = this.options_.destinationFolder;
if (!folder) { if (!folder) {