From 01a51589fd408ed52eaf68a887d25a85911349cf Mon Sep 17 00:00:00 2001 From: Henry Heino <46334387+personalizedrefrigerator@users.noreply.github.com> Date: Fri, 20 Feb 2026 03:26:16 -0800 Subject: [PATCH] Server: Fixes #14343: Fix certain note content is corrupted when uploaded to the server (#14379) --- .eslintignore | 2 +- .gitignore | 2 +- .../formidable-npm-2.1.2-40ba18d67f.patch | 24 +++++++++ package.json | 3 +- packages/server/package.json | 2 +- packages/tools/fuzzer/ipc/Client.ts | 45 ++++++++++++++--- ...fDebug.ts => getBinaryDiffDebugMessage.ts} | 50 ++++++++++++++++--- yarn.lock | 28 +++-------- 8 files changed, 115 insertions(+), 41 deletions(-) create mode 100644 .yarn/patches/formidable-npm-2.1.2-40ba18d67f.patch rename packages/tools/fuzzer/utils/{logDiffDebug.ts => getBinaryDiffDebugMessage.ts} (56%) diff --git a/.eslintignore b/.eslintignore index 784b97e404..45e110fe5e 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1891,11 +1891,11 @@ packages/tools/fuzzer/utils/SeededRandom.js packages/tools/fuzzer/utils/diffSortedStringArrays.test.js packages/tools/fuzzer/utils/diffSortedStringArrays.js packages/tools/fuzzer/utils/extractResourceIds.js +packages/tools/fuzzer/utils/getBinaryDiffDebugMessage.js packages/tools/fuzzer/utils/getNumberProperty.js packages/tools/fuzzer/utils/getProperty.js packages/tools/fuzzer/utils/getStringProperty.js packages/tools/fuzzer/utils/hangingIndent.js -packages/tools/fuzzer/utils/logDiffDebug.js packages/tools/fuzzer/utils/openDebugSession.js packages/tools/fuzzer/utils/randomId.test.js packages/tools/fuzzer/utils/randomId.js diff --git a/.gitignore b/.gitignore index e5072d9e27..5d70b88539 100644 --- a/.gitignore +++ b/.gitignore @@ -1864,11 +1864,11 @@ packages/tools/fuzzer/utils/SeededRandom.js packages/tools/fuzzer/utils/diffSortedStringArrays.test.js packages/tools/fuzzer/utils/diffSortedStringArrays.js packages/tools/fuzzer/utils/extractResourceIds.js +packages/tools/fuzzer/utils/getBinaryDiffDebugMessage.js packages/tools/fuzzer/utils/getNumberProperty.js packages/tools/fuzzer/utils/getProperty.js packages/tools/fuzzer/utils/getStringProperty.js packages/tools/fuzzer/utils/hangingIndent.js -packages/tools/fuzzer/utils/logDiffDebug.js packages/tools/fuzzer/utils/openDebugSession.js packages/tools/fuzzer/utils/randomId.test.js packages/tools/fuzzer/utils/randomId.js diff --git a/.yarn/patches/formidable-npm-2.1.2-40ba18d67f.patch b/.yarn/patches/formidable-npm-2.1.2-40ba18d67f.patch new file mode 100644 index 0000000000..890c0d42f6 --- /dev/null +++ b/.yarn/patches/formidable-npm-2.1.2-40ba18d67f.patch @@ -0,0 +1,24 @@ +# Resolves an issue in which notes and attachments larger than 16 KB +# could become corrupted during the upload process. +# See https://github.com/laurent22/joplin/issues/14343 +diff --git a/src/parsers/JSON.js b/src/parsers/JSON.js +index 9a096c25778c7c68be1ddd9dd78faa85bd1d8ec3..6d6bfd2d3789313a7adc8966ab8e58c3d3167356 100644 +--- a/src/parsers/JSON.js ++++ b/src/parsers/JSON.js +@@ -12,13 +12,14 @@ class JSONParser extends Transform { + } + + _transform(chunk, encoding, callback) { +- this.chunks.push(String(chunk)); // todo consider using a string decoder ++ this.chunks.push(chunk); // type: Uint8Array + callback(); + } + + _flush(callback) { + try { +- const fields = JSON.parse(this.chunks.join('')); ++ const data = Buffer.concat(this.chunks); ++ const fields = JSON.parse(data.toString('utf-8')); + Object.keys(fields).forEach((key) => { + const value = fields[key]; + this.push({ key, value }); diff --git a/package.json b/package.json index 0199c4b3e0..c3db0ff3a2 100644 --- a/package.json +++ b/package.json @@ -124,6 +124,7 @@ "depd@npm:~1.1.2": "patch:depd@npm%3A2.0.0#~/.yarn/patches/depd-npm-2.0.0-b6c51a4b43.patch", "depd@npm:2.0.0": "patch:depd@npm%3A2.0.0#~/.yarn/patches/depd-npm-2.0.0-b6c51a4b43.patch", "depd@npm:^1.1.2": "patch:depd@npm%3A2.0.0#~/.yarn/patches/depd-npm-2.0.0-b6c51a4b43.patch", - "depd@npm:^1.1.0": "patch:depd@npm%3A2.0.0#~/.yarn/patches/depd-npm-2.0.0-b6c51a4b43.patch" + "depd@npm:^1.1.0": "patch:depd@npm%3A2.0.0#~/.yarn/patches/depd-npm-2.0.0-b6c51a4b43.patch", + "formidable@npm:^2.0.1": "patch:formidable@npm%3A2.1.2#~/.yarn/patches/formidable-npm-2.1.2-40ba18d67f.patch" } } diff --git a/packages/server/package.json b/packages/server/package.json index fd58d73480..5fbbe20872 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -35,7 +35,7 @@ "bulma": "1.0.4", "compare-versions": "6.1.1", "dayjs": "1.11.19", - "formidable": "2.1.2", + "formidable": "patch:formidable@npm%3A2.1.2#~/.yarn/patches/formidable-npm-2.1.2-40ba18d67f.patch", "fs-extra": "11.3.2", "html-entities": "1.4.0", "jquery": "3.7.1", diff --git a/packages/tools/fuzzer/ipc/Client.ts b/packages/tools/fuzzer/ipc/Client.ts index aece54e739..62a6adc4e5 100644 --- a/packages/tools/fuzzer/ipc/Client.ts +++ b/packages/tools/fuzzer/ipc/Client.ts @@ -20,7 +20,7 @@ import AsyncActionQueue from '@joplin/lib/AsyncActionQueue'; import { createInterface } from 'readline/promises'; import Stream = require('stream'); import ProgressBar from '../utils/ProgressBar'; -import logDiffDebug from '../utils/logDiffDebug'; +import getDiffDebugMessage from '../utils/getBinaryDiffDebugMessage'; import { NoteEntity } from '@joplin/lib/services/database/types'; import diffSortedStringArrays from '../utils/diffSortedStringArrays'; import extractResourceIds from '../utils/extractResourceIds'; @@ -725,9 +725,7 @@ class Client implements ActionableClient { }, update: async (targetNote: NoteData) => { const keep = targetNote.body.substring( - // Problems start to appear when notes get long. - // See https://github.com/laurent22/joplin/issues/13644. - 0, Math.min(this.context_.randInt(0, targetNote.body.length), 5000), + 0, this.context_.randInt(0, targetNote.body.length), ); const append = this.context_.randomString(this.context_.randInt(0, 5000)); await this.updateNote({ @@ -824,8 +822,8 @@ class Client implements ActionableClient { } catch (error) { // Log additional information to help debug binary differences if (lastActualNote) { - logDiffDebug(lastActualNote.title, expected.title); - logDiffDebug(lastActualNote.body, expected.body); + logger.warn(getDiffDebugMessage(lastActualNote.title, expected.title)); + logger.warn(getDiffDebugMessage(lastActualNote.body, expected.body)); } // Log all transactions associated with the item this.globalActionTracker_.printActionLog(expected.id); @@ -836,7 +834,7 @@ class Client implements ActionableClient { public async createRandomNote({ parentId, id, quiet = false }: CreateRandomItemOptions) { const titleLength = this.context_.randInt(0, 256); - const bodyLength = this.context_.randInt(0, 2000); + const bodyLength = this.context_.randInt(0, 10_000); await this.createNote({ published: false, parentId, @@ -1177,6 +1175,38 @@ class Client implements ActionableClient { } }; + const assertSameBodies = (actualSorted: NoteData[], expectedSorted: NoteData[], assertionLabel: string) => { + if (actualSorted.length !== expectedSorted.length) { + throw new Error(`Input arrays have different lengths (in: ${assertionLabel})`); + } + + const differentItems = []; + for (let i = 0; i < actualSorted.length; i++) { + const actualBody = actualSorted[i].body; + const expectedBody = expectedSorted[i].body; + + if (actualBody !== expectedBody) { + differentItems.push([actualSorted[i], expectedSorted[i]]); + } + } + + if (differentItems.length) { + const message = [`Some items have different bodies (in: ${assertionLabel})`]; + for (const [actual, expected] of differentItems) { + message.push(`- item: ${actual.id}${actual.id !== expected.id ? ` (compare ${expected.id}) ` : ''}:`); + message.push(` ${hangingIndent(getDiffDebugMessage(actual.body, expected.body))}`); + + // Only display full bodies for easy-to-inspect content: + const simpleBodyExp = /^[a-z0-9;.,!?[\]() \t\n"']{0,200}$/i; + if (actual.body.match(simpleBodyExp) && expected.body.match(simpleBodyExp)) { + message.push(` actual: ${JSON.stringify(actual)}`); + message.push(` expected:${JSON.stringify(expected)}`); + } + } + throw new Error(message.join('\n')); + } + }; + const checkNoteState = async () => { const notes = [...await this.listNotes()]; const expectedNotes = [...await this.tracker_.listNotes()]; @@ -1187,6 +1217,7 @@ class Client implements ActionableClient { assertNoAdjacentEqualIds(notes, 'notes'); assertNoAdjacentEqualIds(expectedNotes, 'expectedNotes'); await assertSameIds(notes, expectedNotes, 'Note IDs should match'); + assertSameBodies(notes, expectedNotes, 'should have the same note bodies'); assert.deepEqual(notes, expectedNotes, 'should have the same notes as the expected state'); }; diff --git a/packages/tools/fuzzer/utils/logDiffDebug.ts b/packages/tools/fuzzer/utils/getBinaryDiffDebugMessage.ts similarity index 56% rename from packages/tools/fuzzer/utils/logDiffDebug.ts rename to packages/tools/fuzzer/utils/getBinaryDiffDebugMessage.ts index 644dc42db6..0cd60a6dc0 100644 --- a/packages/tools/fuzzer/utils/logDiffDebug.ts +++ b/packages/tools/fuzzer/utils/getBinaryDiffDebugMessage.ts @@ -1,23 +1,54 @@ -import Logger from '@joplin/utils/Logger'; - -const logger = Logger.create('logDiffDebug'); // Provides additional debugging information for differing strings -const logDiffDebug = (actual: string, expected: string) => { +const getDiffDebugMessage = (actual: string, expected: string) => { + if (actual === expected) return ''; + + const diffMessage = []; + + // List all characters present in actual that were not present in expected. + // This helps debug issues in which the unicode replacement character has been + // added incorrectly: + { + const inExpected = new Set(); + for (let i = 0; i < expected.length; i++) { + inExpected.add(expected.charAt(i)); + } + + const unexpected = new Set(); + for (let i = 0; i < actual.length; i++) { + const char = actual.charAt(i); + if (!inExpected.has(char)) { + unexpected.add(actual.charAt(i)); + } + } + + if (unexpected.size) { + diffMessage.push('Characters found in actual that were not in the expected state:\n'); + diffMessage.push(' ', JSON.stringify([...unexpected]), '\n\n'); + } + } + const actualBinary = new Uint8Array(new TextEncoder().encode(actual)); const expectedBinary = new Uint8Array(new TextEncoder().encode(expected)); + + if (actualBinary.length > expectedBinary.length) { + diffMessage.push(`Actual is longer than expected by ${actualBinary.length - expectedBinary.length}\n`); + } else if (expectedBinary.length > actualBinary.length) { + diffMessage.push(`Expected is longer than actual by ${expectedBinary.length - actualBinary.length}\n`); + } + for (let i = 0; i < actualBinary.length; i++) { if (i >= expectedBinary.length) { - logger.warn('Actual is longer than expected'); break; } if (expectedBinary[i] !== actualBinary[i]) { - logger.warn( + diffMessage.push( 'First binary difference at position', i, `(0x${i.toString(16)})`, ': ', expectedBinary[i], '!=', actualBinary[i], '(expected != actual)', '\n\tContext: expected[i-3:i+5] = ', [...expectedBinary.slice(i - 3, i + 5)], '\n\tContext: actual[i-3 : i+5] = ', [...actualBinary.slice(i - 3, i + 5)], '\n\tactual.byteLength = ', actualBinary.length, ', expected.byteLength = ', expectedBinary.length, + '\n\n', ); break; } @@ -28,16 +59,19 @@ const logDiffDebug = (actual: string, expected: string) => { const indexExpected = expectedBinary.length - i - 1; const indexActual = actualBinary.length - i - 1; if (expectedBinary[indexExpected] !== actualBinary[indexActual]) { - logger.warn( + diffMessage.push( 'Last binary difference (working from end)', ': ', expectedBinary[indexExpected], '!=', actualBinary[indexActual], `(expected[${indexExpected}] != actual[${indexActual}])`, '\n\tContext: expected[a-6:a+3] = ', [...expectedBinary.slice(indexExpected - 6, indexExpected + 3)], '\n\tContext: actual[b-6 : b+3] = ', [...actualBinary.slice(indexActual - 6, indexActual + 3)], '\n\twhere expected.byteLength = ', expectedBinary.byteLength, 'and actual.byteLength = ', actualBinary.byteLength, + '\n\n', ); break; } } + + return diffMessage.join(' '); }; -export default logDiffDebug; +export default getDiffDebugMessage; diff --git a/yarn.lock b/yarn.lock index 9180140594..ed068f7816 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11122,7 +11122,7 @@ __metadata: bulma: "npm:1.0.4" compare-versions: "npm:6.1.1" dayjs: "npm:1.11.19" - formidable: "npm:2.1.2" + formidable: "patch:formidable@npm%3A2.1.2#~/.yarn/patches/formidable-npm-2.1.2-40ba18d67f.patch" fs-extra: "npm:11.3.2" gulp: "npm:4.0.2" html-entities: "npm:1.4.0" @@ -12656,13 +12656,6 @@ __metadata: languageName: node linkType: hard -"@noble/hashes@npm:^1.1.5": - version: 1.8.0 - resolution: "@noble/hashes@npm:1.8.0" - checksum: 10/474b7f56bc6fb2d5b3a42132561e221b0ea4f91e590f4655312ca13667840896b34195e2b53b7f097ec080a1fdd3b58d902c2a8d0fbdf51d2e238b53808a177e - languageName: node - linkType: hard - "@nodelib/fs.scandir@npm:2.1.5": version: 2.1.5 resolution: "@nodelib/fs.scandir@npm:2.1.5" @@ -13104,15 +13097,6 @@ __metadata: languageName: node linkType: hard -"@paralleldrive/cuid2@npm:^2.2.2": - version: 2.2.2 - resolution: "@paralleldrive/cuid2@npm:2.2.2" - dependencies: - "@noble/hashes": "npm:^1.1.5" - checksum: 10/40ee269d6e47b4fed7706a2e4da7c27c3c668ebc969110d6d112277b6b16a67cce0503b53b9943f2c55035a72d225f77ea5541e03396d6429eec9252137a53b7 - languageName: node - linkType: hard - "@parcel/watcher-android-arm64@npm:2.5.1": version: 2.5.1 resolution: "@parcel/watcher-android-arm64@npm:2.5.1" @@ -29890,15 +29874,15 @@ __metadata: languageName: node linkType: hard -"formidable@npm:^2.0.1": - version: 2.1.5 - resolution: "formidable@npm:2.1.5" +"formidable@patch:formidable@npm%3A2.1.2#~/.yarn/patches/formidable-npm-2.1.2-40ba18d67f.patch": + version: 2.1.2 + resolution: "formidable@patch:formidable@npm%3A2.1.2#~/.yarn/patches/formidable-npm-2.1.2-40ba18d67f.patch::version=2.1.2&hash=f39b0e" dependencies: - "@paralleldrive/cuid2": "npm:^2.2.2" dezalgo: "npm:^1.0.4" + hexoid: "npm:^1.0.0" once: "npm:^1.4.0" qs: "npm:^6.11.0" - checksum: 10/ee96de12e91d63fe86479ffe5bf59004bb3f43e00ce7ccecd1b1ff10b5d1a89a19b1ede727e1fe57ef596c377b9f9300212a5f7bab14fd28f3c4ffe12dbb4cc7 + checksum: 10/62dd252f632d47256019e714305d2a7b4fc3129baec58088ed5e25279c678cdd30da8330f82d34524ca4be7a09279ff890f30dc4458cb268e92b56965f1f22ee languageName: node linkType: hard