Desktop,CLI: Fixes #10653: Change Resource filetype detecting strategy (#10907)

Co-authored-by: Laurent Cozic <laurent22@users.noreply.github.com>

.eslintignore

@@ -1338,6 +1338,7 @@ packages/lib/services/trash/permanentlyDeleteOldItems.test.js
 packages/lib/services/trash/permanentlyDeleteOldItems.js
 packages/lib/services/trash/restoreItems.test.js
 packages/lib/services/trash/restoreItems.js
+packages/lib/shim-init-node.test.js
 packages/lib/shim-init-node.js
 packages/lib/shim.js
 packages/lib/string-utils.test.js

.gitignore

@@ -1315,6 +1315,7 @@ packages/lib/services/trash/permanentlyDeleteOldItems.test.js
 packages/lib/services/trash/permanentlyDeleteOldItems.js
 packages/lib/services/trash/restoreItems.test.js
 packages/lib/services/trash/restoreItems.js
+packages/lib/shim-init-node.test.js
 packages/lib/shim-init-node.js
 packages/lib/shim.js
 packages/lib/string-utils.test.js

packages/app-cli/app/command-e2ee.ts

@@ -8,8 +8,7 @@ import shim from '@joplin/lib/shim';
 import * as pathUtils from '@joplin/lib/path-utils';
 import { getEncryptionEnabled, localSyncInfo } from '@joplin/lib/services/synchronizer/syncInfoUtils';
 import { generateMasterKeyAndEnableEncryption, loadMasterKeysFromSettings, masterPasswordIsValid, setupAndDisableEncryption } from '@joplin/lib/services/e2ee/utils';
-const imageType = require('image-type');
+import { fromFile as fileTypeFromFile } from 'file-type';
-const readChunk = require('read-chunk');
 
 class Command extends BaseCommand {
 	public usage() {
@@ -136,8 +135,7 @@ class Command extends BaseCommand {
 					const outputDir = options.output ? options.output : require('os').tmpdir();
 					let outFile = `${outputDir}/${pathUtils.filename(args.path)}.${Date.now()}.bin`;
 					await EncryptionService.instance().decryptFile(args.path, outFile);
-					const buffer = await readChunk(outFile, 0, 64);
+					const detectedType = await fileTypeFromFile(outFile);
-					const detectedType = imageType(buffer);
 
 					if (detectedType) {
 						const newOutFile = `${outFile}.${detectedType.ext}`;

packages/app-cli/package.json

@@ -47,15 +47,14 @@
     "aws-sdk": "2.1340.0",
     "chalk": "4.1.2",
     "compare-version": "0.1.2",
+    "file-type": "16.5.4",
     "fs-extra": "11.2.0",
     "html-entities": "1.4.0",
-    "image-type": "3.1.0",
     "keytar": "7.9.0",
     "md5": "2.3.0",
     "node-rsa": "1.1.1",
     "open": "8.4.2",
     "proper-lockfile": "4.1.2",
-    "read-chunk": "2.1.0",
     "server-destroy": "1.0.1",
     "sharp": "0.33.4",
     "sprintf-js": "1.1.3",

packages/app-cli/tests/support/valid_pdf_without_ext

@@ -0,0 +1,198 @@
+%PDF-1.3
+%âãÏÓ
+
+1 0 obj
+<<
+/Type /Catalog
+/Outlines 2 0 R
+/Pages 3 0 R
+>>
+endobj
+
+2 0 obj
+<<
+/Type /Outlines
+/Count 0
+>>
+endobj
+
+3 0 obj
+<<
+/Type /Pages
+/Count 2
+/Kids [ 4 0 R 6 0 R ] 
+>>
+endobj
+
+4 0 obj
+<<
+/Type /Page
+/Parent 3 0 R
+/Resources <<
+/Font <<
+/F1 9 0 R 
+>>
+/ProcSet 8 0 R
+>>
+/MediaBox [0 0 612.0000 792.0000]
+/Contents 5 0 R
+>>
+endobj
+
+5 0 obj
+<< /Length 1074 >>
+stream
+2 J
+BT
+0 0 0 rg
+/F1 0027 Tf
+57.3750 722.2800 Td
+( A Simple PDF File ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 688.6080 Td
+( This is a small demonstration .pdf file - ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 664.7040 Td
+( just for use in the Virtual Mechanics tutorials. More text. And more ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 652.7520 Td
+( text. And more text. And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 628.8480 Td
+( And more text. And more text. And more text. And more text. And more ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 616.8960 Td
+( text. And more text. Boring, zzzzz. And more text. And more text. And ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 604.9440 Td
+( more text. And more text. And more text. And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 592.9920 Td
+( And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 569.0880 Td
+( And more text. And more text. And more text. And more text. And more ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 557.1360 Td
+( text. And more text. And more text. Even more. Continued on page 2 ...) Tj
+ET
+endstream
+endobj
+
+6 0 obj
+<<
+/Type /Page
+/Parent 3 0 R
+/Resources <<
+/Font <<
+/F1 9 0 R 
+>>
+/ProcSet 8 0 R
+>>
+/MediaBox [0 0 612.0000 792.0000]
+/Contents 7 0 R
+>>
+endobj
+
+7 0 obj
+<< /Length 676 >>
+stream
+2 J
+BT
+0 0 0 rg
+/F1 0027 Tf
+57.3750 722.2800 Td
+( Simple PDF File 2 ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 688.6080 Td
+( ...continued from page 1. Yet more text. And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 676.6560 Td
+( And more text. And more text. And more text. And more text. And more ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 664.7040 Td
+( text. Oh, how boring typing this stuff. But not as boring as watching ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 652.7520 Td
+( paint dry. And more text. And more text. And more text. And more text. ) Tj
+ET
+BT
+/F1 0010 Tf
+69.2500 640.8000 Td
+( Boring.  More, a little more text. The end, and just as well. ) Tj
+ET
+endstream
+endobj
+
+8 0 obj
+[/PDF /Text]
+endobj
+
+9 0 obj
+<<
+/Type /Font
+/Subtype /Type1
+/Name /F1
+/BaseFont /Helvetica
+/Encoding /WinAnsiEncoding
+>>
+endobj
+
+10 0 obj
+<<
+/Creator (Rave \(http://www.nevrona.com/rave\))
+/Producer (Nevrona Designs)
+/CreationDate (D:20060301072826)
+>>
+endobj
+
+xref
+0 11
+0000000000 65535 f
+0000000019 00000 n
+0000000093 00000 n
+0000000147 00000 n
+0000000222 00000 n
+0000000390 00000 n
+0000001522 00000 n
+0000001690 00000 n
+0000002423 00000 n
+0000002456 00000 n
+0000002574 00000 n
+
+trailer
+<<
+/Size 11
+/Root 1 0 R
+/Info 10 0 R
+>>
+
+startxref
+2714
+%%EOF

packages/lib/package.json

@@ -59,6 +59,7 @@
     "diff-match-patch": "1.0.5",
     "fast-deep-equal": "3.1.3",
     "fast-xml-parser": "3.21.1",
+    "file-type": "16.5.4",
     "follow-redirects": "1.15.6",
     "form-data": "4.0.0",
     "fs-extra": "11.2.0",
@@ -66,7 +67,6 @@
     "html-entities": "1.4.0",
     "html-minifier": "4.0.0",
     "image-data-uri": "2.0.1",
-    "image-type": "3.1.0",
     "immer": "7.0.15",
     "js-yaml": "4.1.0",
     "markdown-it": "13.0.2",
@@ -83,7 +83,6 @@
     "promise": "8.3.0",
     "query-string": "7.1.3",
     "re-reselect": "4.0.1",
-    "read-chunk": "2.1.0",
     "redux": "4.2.1",
     "relative": "3.0.2",
     "reselect": "4.1.8",

packages/lib/shim-init-node.test.ts

@@ -0,0 +1,19 @@
+
+const { shimInit } = require('./shim-init-node');
+import shim from './shim';
+import { setupDatabaseAndSynchronizer, supportDir } from './testing/test-utils';
+
+describe('shim-init-node', () => {
+
+	beforeEach(async () => {
+		await setupDatabaseAndSynchronizer(1);
+		shimInit();
+	});
+
+	test('should set mime the correct mime for a PDF file even if the extension is missing', async () => {
+		const filePath = `${supportDir}/valid_pdf_without_ext`;
+		const resource = await shim.createResourceFromPath(filePath);
+
+		expect(resource.mime).toBe('application/pdf');
+	});
+});

packages/lib/shim-init-node.ts

@@ -12,6 +12,7 @@ import { ResourceEntity } from './services/database/types';
 import { TextItem } from 'pdfjs-dist/types/src/display/api';
 import replaceUnsupportedCharacters from './utils/replaceUnsupportedCharacters';
 import { FetchBlobOptions } from './types';
+import { fromFile as fileTypeFromFile } from 'file-type';
 import crypto from './services/e2ee/crypto';
 
 import FileApiDriverLocal from './file-api-driver-local';
@@ -306,9 +307,6 @@ function shimInit(options: ShimInitOptions = null) {
 			...options,
 		};
 
-		const readChunk = require('read-chunk');
-		const imageType = require('image-type');
-
 		const isUpdate = !!options.destinationResourceId;
 
 		const uuid = require('./uuid').default;
@@ -332,8 +330,7 @@ function shimInit(options: ShimInitOptions = null) {
 		let fileExt = safeFileExtension(fileExtension(filePath));
 
 		if (!resource.mime) {
-			const buffer = await readChunk(filePath, 0, 64);
+			const detectedType = await fileTypeFromFile(filePath);
-			const detectedType = imageType(buffer);
 
 			if (detectedType) {
 				fileExt = detectedType.ext;

yarn.lock

@@ -8467,6 +8467,7 @@ __metadata:
     diff-match-patch: 1.0.5
     fast-deep-equal: 3.1.3
     fast-xml-parser: 3.21.1
+    file-type: 16.5.4
     follow-redirects: 1.15.6
     form-data: 4.0.0
     fs-extra: 11.2.0
@@ -8474,7 +8475,6 @@ __metadata:
     html-entities: 1.4.0
     html-minifier: 4.0.0
     image-data-uri: 2.0.1
-    image-type: 3.1.0
     immer: 7.0.15
     jest: 29.7.0
     js-yaml: 4.1.0
@@ -8495,7 +8495,6 @@ __metadata:
     re-reselect: 4.0.1
     react: 18.3.1
     react-test-renderer: 18.3.1
-    read-chunk: 2.1.0
     redux: 4.2.1
     relative: 3.0.2
     reselect: 4.1.8
@@ -12288,6 +12287,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@tokenizer/token@npm:^0.3.0":
+  version: 0.3.0
+  resolution: "@tokenizer/token@npm:0.3.0"
+  checksum: 1d575d02d2a9f0c5a4ca5180635ebd2ad59e0f18b42a65f3d04844148b49b3db35cf00b6012a1af2d59c2ab3caca59451c5689f747ba8667ee586ad717ee58e1
+  languageName: node
+  linkType: hard
+
 "@tootallnate/once@npm:1":
   version: 1.1.2
   resolution: "@tootallnate/once@npm:1.1.2"
@@ -24514,10 +24520,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"file-type@npm:^10.9.0":
+"file-type@npm:16.5.4":
-  version: 10.11.0
+  version: 16.5.4
-  resolution: "file-type@npm:10.11.0"
+  resolution: "file-type@npm:16.5.4"
-  checksum: cadd8cd187692dcde637a3ff53bb51c5d935633fc8085e7d25bfb3b4bf995e14a43f2baf71bdcb9d7235b3e725bd158b75d25911fa2f73e5812955382228c511
+  dependencies:
+    readable-web-to-node-stream: ^3.0.0
+    strtok3: ^6.2.4
+    token-types: ^4.1.1
+  checksum: d983c0f36491c57fcb6cc70fcb02c36d6b53f312a15053263e1924e28ca8314adf0db32170801ad777f09432c32155f31715ceaee66310947731588120d7ec27
   languageName: node
   linkType: hard
 
@@ -27813,15 +27823,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"image-type@npm:3.1.0":
-  version: 3.1.0
-  resolution: "image-type@npm:3.1.0"
-  dependencies:
-    file-type: ^10.9.0
-  checksum: c50ebc7fb90a512fa3b6c08a81d2d21eac8abab9194ed64e2dec65437bea2d23201f4de0a920c37dc996aba005f7bf7fadb76ff479176a61be169eaabdee5eb6
-  languageName: node
-  linkType: hard
-
 "immediate@npm:^3.2.3":
   version: 3.3.0
   resolution: "immediate@npm:3.3.0"
@@ -30352,17 +30353,16 @@ __metadata:
     aws-sdk: 2.1340.0
     chalk: 4.1.2
     compare-version: 0.1.2
+    file-type: 16.5.4
     fs-extra: 11.2.0
     gulp: 4.0.2
     html-entities: 1.4.0
-    image-type: 3.1.0
     jest: 29.7.0
     keytar: 7.9.0
     md5: 2.3.0
     node-rsa: 1.1.1
     open: 8.4.2
     proper-lockfile: 4.1.2
-    read-chunk: 2.1.0
     server-destroy: 1.0.1
     sharp: 0.33.4
     sprintf-js: 1.1.3
@@ -37115,6 +37115,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"peek-readable@npm:^4.1.0":
+  version: 4.1.0
+  resolution: "peek-readable@npm:4.1.0"
+  checksum: 02c673f9bc816f8e4e74a054c097225ad38d457d745b775e2b96faf404a54473b2f62f5bcd496f5ebc28696708bcc5e95bed409856f4bef5ed62eae9b4ac0dab
+  languageName: node
+  linkType: hard
+
 "pend@npm:~1.2.0":
   version: 1.2.0
   resolution: "pend@npm:1.2.0"
@@ -40029,16 +40036,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"read-chunk@npm:2.1.0":
-  version: 2.1.0
-  resolution: "read-chunk@npm:2.1.0"
-  dependencies:
-    pify: ^3.0.0
-    safe-buffer: ^5.1.1
-  checksum: 03a5a3d8412c484f1d45639fcd11943932d1eda3e99d775f04f6878402d03420e62723a231c379ba6e6ac437d5f769214841e3001768b8359e7776ff727f6b4c
-  languageName: node
-  linkType: hard
-
 "read-cmd-shim@npm:^1.0.1":
   version: 1.0.5
   resolution: "read-cmd-shim@npm:1.0.5"
@@ -40284,6 +40281,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"readable-web-to-node-stream@npm:^3.0.0":
+  version: 3.0.2
+  resolution: "readable-web-to-node-stream@npm:3.0.2"
+  dependencies:
+    readable-stream: ^3.6.0
+  checksum: 8c56cc62c68513425ddfa721954875b382768f83fa20e6b31e365ee00cbe7a3d6296f66f7f1107b16cd3416d33aa9f1680475376400d62a081a88f81f0ea7f9c
+  languageName: node
+  linkType: hard
+
 "readdir-scoped-modules@npm:^1.0.0":
   version: 1.1.0
   resolution: "readdir-scoped-modules@npm:1.1.0"
@@ -44134,6 +44140,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"strtok3@npm:^6.2.4":
+  version: 6.3.0
+  resolution: "strtok3@npm:6.3.0"
+  dependencies:
+    "@tokenizer/token": ^0.3.0
+    peek-readable: ^4.1.0
+  checksum: 90732cff3f325aef7c47c511f609b593e0873ec77b5081810071cde941344e6a0ee3ccb0cae1a9f5b4e12c81a2546fd6b322fabcdfbd1dd08362c2ce5291334a
+  languageName: node
+  linkType: hard
+
 "structured-headers@npm:^0.4.1":
   version: 0.4.1
   resolution: "structured-headers@npm:0.4.1"
@@ -45414,6 +45430,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"token-types@npm:^4.1.1":
+  version: 4.2.1
+  resolution: "token-types@npm:4.2.1"
+  dependencies:
+    "@tokenizer/token": ^0.3.0
+    ieee754: ^1.2.1
+  checksum: cce256766b33e0f08ceffefa2198fb4961a417866d00780e58625999ab5c0699821407053e64eadc41b00bbb6c0d0c4d02fbd2199940d8a3ccb71e1b148ab9a2
+  languageName: node
+  linkType: hard
+
 "totalist@npm:^3.0.0":
   version: 3.0.1
   resolution: "totalist@npm:3.0.1"