diff --git a/packages/app-cli/tests/support/test_notes/yaml/note_with_byte_order_mark.md b/packages/app-cli/tests/support/test_notes/yaml/note_with_byte_order_mark.md new file mode 100644 index 000000000..fbe092a98 --- /dev/null +++ b/packages/app-cli/tests/support/test_notes/yaml/note_with_byte_order_mark.md @@ -0,0 +1,8 @@ +--- +title: Frontmatter test +tags: + - tag1 + - tag2 +--- + +This note begins with an invisible byte order mark, just before its frontmatter. diff --git a/packages/lib/services/interop/InteropService_Importer_Md.ts b/packages/lib/services/interop/InteropService_Importer_Md.ts index 92fb9af26..10d6d2f32 100644 --- a/packages/lib/services/interop/InteropService_Importer_Md.ts +++ b/packages/lib/services/interop/InteropService_Importer_Md.ts @@ -13,6 +13,7 @@ import { unique } from '../../ArrayUtils'; const { pregQuote } = require('../../string-utils-common'); import { MarkupToHtml } from '@joplin/renderer'; import { isDataUrl } from '@joplin/utils/url'; +import { stripBom } from '../../string-utils'; export default class InteropService_Importer_Md extends InteropService_Importer_Base { protected importedNotes: Record = {}; @@ -167,7 +168,8 @@ export default class InteropService_Importer_Md extends InteropService_Importer_ if (!stat) throw new Error(`Cannot read ${resolvedPath}`); const ext = fileExtension(resolvedPath); const title = filename(resolvedPath); - const body = await shim.fsDriver().readFile(resolvedPath); + const body = stripBom(await shim.fsDriver().readFile(resolvedPath)); + const note = { parent_id: parentFolderId, title: title, diff --git a/packages/lib/services/interop/InteropService_Importer_Md_frontmatter.test.ts b/packages/lib/services/interop/InteropService_Importer_Md_frontmatter.test.ts index cd8dc11b3..a3fd2eb35 100644 --- a/packages/lib/services/interop/InteropService_Importer_Md_frontmatter.test.ts +++ b/packages/lib/services/interop/InteropService_Importer_Md_frontmatter.test.ts @@ -173,4 +173,13 @@ describe('InteropService_Importer_Md_frontmatter: importMetadata', () => { const note = await importTestFile('note_with_dataurl_image.md'); expect(note.body).toBe('Street View Pegman Control'); }); + + it('should recognize frontmatter in a file that starts with a UTF8 byte order mark', async () => { + const note = await importTestFile('note_with_byte_order_mark.md'); + expect(note.title).toBe('Frontmatter test'); + expect(note.body).toBe('This note begins with an invisible byte order mark, just before its frontmatter.\n'); + + const tags = (await Tag.tagsByNoteId(note.id)).map(tag => tag.title).sort(); + expect(tags).toMatchObject(['tag1', 'tag2']); + }); }); diff --git a/packages/lib/string-utils.ts b/packages/lib/string-utils.ts index e480d19fa..59b1abc9b 100644 --- a/packages/lib/string-utils.ts +++ b/packages/lib/string-utils.ts @@ -303,3 +303,11 @@ export function scriptType(s: string) { return 'en'; } +// A UTF-8/UTF-16 byte order mark can appear at the start of a file and +// can break logic that relies on a file starting with specific text. +// See https://github.com/laurent22/joplin/issues/9868 +export const stripBom = (text: string) => { + // Remove the UTF-16 BOM --- NodeJS seems to convert UTF-8 BOMs to UTF-16 BOMs + // when reading files. + return text.replace(/^\ufeff/u, ''); +};