diff --git a/.eslintignore b/.eslintignore index d9be63d8d..ef249faad 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1338,6 +1338,7 @@ packages/lib/services/trash/permanentlyDeleteOldItems.test.js packages/lib/services/trash/permanentlyDeleteOldItems.js packages/lib/services/trash/restoreItems.test.js packages/lib/services/trash/restoreItems.js +packages/lib/shim-init-node.test.js packages/lib/shim-init-node.js packages/lib/shim.js packages/lib/string-utils.test.js diff --git a/.gitignore b/.gitignore index df74b3ad2..71d416d63 100644 --- a/.gitignore +++ b/.gitignore @@ -1315,6 +1315,7 @@ packages/lib/services/trash/permanentlyDeleteOldItems.test.js packages/lib/services/trash/permanentlyDeleteOldItems.js packages/lib/services/trash/restoreItems.test.js packages/lib/services/trash/restoreItems.js +packages/lib/shim-init-node.test.js packages/lib/shim-init-node.js packages/lib/shim.js packages/lib/string-utils.test.js diff --git a/packages/app-cli/app/command-e2ee.ts b/packages/app-cli/app/command-e2ee.ts index bfbd15f95..794316657 100644 --- a/packages/app-cli/app/command-e2ee.ts +++ b/packages/app-cli/app/command-e2ee.ts @@ -8,8 +8,7 @@ import shim from '@joplin/lib/shim'; import * as pathUtils from '@joplin/lib/path-utils'; import { getEncryptionEnabled, localSyncInfo } from '@joplin/lib/services/synchronizer/syncInfoUtils'; import { generateMasterKeyAndEnableEncryption, loadMasterKeysFromSettings, masterPasswordIsValid, setupAndDisableEncryption } from '@joplin/lib/services/e2ee/utils'; -const imageType = require('image-type'); -const readChunk = require('read-chunk'); +import { fromFile as fileTypeFromFile } from 'file-type'; class Command extends BaseCommand { public usage() { @@ -136,8 +135,7 @@ class Command extends BaseCommand { const outputDir = options.output ? options.output : require('os').tmpdir(); let outFile = `${outputDir}/${pathUtils.filename(args.path)}.${Date.now()}.bin`; await EncryptionService.instance().decryptFile(args.path, outFile); - const buffer = await readChunk(outFile, 0, 64); - const detectedType = imageType(buffer); + const detectedType = await fileTypeFromFile(outFile); if (detectedType) { const newOutFile = `${outFile}.${detectedType.ext}`; diff --git a/packages/app-cli/package.json b/packages/app-cli/package.json index b238903c5..c00d1f609 100644 --- a/packages/app-cli/package.json +++ b/packages/app-cli/package.json @@ -47,15 +47,14 @@ "aws-sdk": "2.1340.0", "chalk": "4.1.2", "compare-version": "0.1.2", + "file-type": "16.5.4", "fs-extra": "11.2.0", "html-entities": "1.4.0", - "image-type": "3.1.0", "keytar": "7.9.0", "md5": "2.3.0", "node-rsa": "1.1.1", "open": "8.4.2", "proper-lockfile": "4.1.2", - "read-chunk": "2.1.0", "server-destroy": "1.0.1", "sharp": "0.33.4", "sprintf-js": "1.1.3", diff --git a/packages/app-cli/tests/support/valid_pdf_without_ext b/packages/app-cli/tests/support/valid_pdf_without_ext new file mode 100644 index 000000000..dbf091df9 --- /dev/null +++ b/packages/app-cli/tests/support/valid_pdf_without_ext @@ -0,0 +1,198 @@ +%PDF-1.3 +%âãÏÓ + +1 0 obj +<< +/Type /Catalog +/Outlines 2 0 R +/Pages 3 0 R +>> +endobj + +2 0 obj +<< +/Type /Outlines +/Count 0 +>> +endobj + +3 0 obj +<< +/Type /Pages +/Count 2 +/Kids [ 4 0 R 6 0 R ] +>> +endobj + +4 0 obj +<< +/Type /Page +/Parent 3 0 R +/Resources << +/Font << +/F1 9 0 R +>> +/ProcSet 8 0 R +>> +/MediaBox [0 0 612.0000 792.0000] +/Contents 5 0 R +>> +endobj + +5 0 obj +<< /Length 1074 >> +stream +2 J +BT +0 0 0 rg +/F1 0027 Tf +57.3750 722.2800 Td +( A Simple PDF File ) Tj +ET +BT +/F1 0010 Tf +69.2500 688.6080 Td +( This is a small demonstration .pdf file - ) Tj +ET +BT +/F1 0010 Tf +69.2500 664.7040 Td +( just for use in the Virtual Mechanics tutorials. More text. And more ) Tj +ET +BT +/F1 0010 Tf +69.2500 652.7520 Td +( text. And more text. And more text. And more text. ) Tj +ET +BT +/F1 0010 Tf +69.2500 628.8480 Td +( And more text. And more text. And more text. And more text. And more ) Tj +ET +BT +/F1 0010 Tf +69.2500 616.8960 Td +( text. And more text. Boring, zzzzz. And more text. And more text. And ) Tj +ET +BT +/F1 0010 Tf +69.2500 604.9440 Td +( more text. And more text. And more text. And more text. And more text. ) Tj +ET +BT +/F1 0010 Tf +69.2500 592.9920 Td +( And more text. And more text. ) Tj +ET +BT +/F1 0010 Tf +69.2500 569.0880 Td +( And more text. And more text. And more text. And more text. And more ) Tj +ET +BT +/F1 0010 Tf +69.2500 557.1360 Td +( text. And more text. And more text. Even more. Continued on page 2 ...) Tj +ET +endstream +endobj + +6 0 obj +<< +/Type /Page +/Parent 3 0 R +/Resources << +/Font << +/F1 9 0 R +>> +/ProcSet 8 0 R +>> +/MediaBox [0 0 612.0000 792.0000] +/Contents 7 0 R +>> +endobj + +7 0 obj +<< /Length 676 >> +stream +2 J +BT +0 0 0 rg +/F1 0027 Tf +57.3750 722.2800 Td +( Simple PDF File 2 ) Tj +ET +BT +/F1 0010 Tf +69.2500 688.6080 Td +( ...continued from page 1. Yet more text. And more text. And more text. ) Tj +ET +BT +/F1 0010 Tf +69.2500 676.6560 Td +( And more text. And more text. And more text. And more text. And more ) Tj +ET +BT +/F1 0010 Tf +69.2500 664.7040 Td +( text. Oh, how boring typing this stuff. But not as boring as watching ) Tj +ET +BT +/F1 0010 Tf +69.2500 652.7520 Td +( paint dry. And more text. And more text. And more text. And more text. ) Tj +ET +BT +/F1 0010 Tf +69.2500 640.8000 Td +( Boring. More, a little more text. The end, and just as well. ) Tj +ET +endstream +endobj + +8 0 obj +[/PDF /Text] +endobj + +9 0 obj +<< +/Type /Font +/Subtype /Type1 +/Name /F1 +/BaseFont /Helvetica +/Encoding /WinAnsiEncoding +>> +endobj + +10 0 obj +<< +/Creator (Rave \(http://www.nevrona.com/rave\)) +/Producer (Nevrona Designs) +/CreationDate (D:20060301072826) +>> +endobj + +xref +0 11 +0000000000 65535 f +0000000019 00000 n +0000000093 00000 n +0000000147 00000 n +0000000222 00000 n +0000000390 00000 n +0000001522 00000 n +0000001690 00000 n +0000002423 00000 n +0000002456 00000 n +0000002574 00000 n + +trailer +<< +/Size 11 +/Root 1 0 R +/Info 10 0 R +>> + +startxref +2714 +%%EOF diff --git a/packages/lib/package.json b/packages/lib/package.json index de37c065e..467a0910a 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -59,6 +59,7 @@ "diff-match-patch": "1.0.5", "fast-deep-equal": "3.1.3", "fast-xml-parser": "3.21.1", + "file-type": "16.5.4", "follow-redirects": "1.15.6", "form-data": "4.0.0", "fs-extra": "11.2.0", @@ -66,7 +67,6 @@ "html-entities": "1.4.0", "html-minifier": "4.0.0", "image-data-uri": "2.0.1", - "image-type": "3.1.0", "immer": "7.0.15", "js-yaml": "4.1.0", "markdown-it": "13.0.2", @@ -83,7 +83,6 @@ "promise": "8.3.0", "query-string": "7.1.3", "re-reselect": "4.0.1", - "read-chunk": "2.1.0", "redux": "4.2.1", "relative": "3.0.2", "reselect": "4.1.8", diff --git a/packages/lib/shim-init-node.test.ts b/packages/lib/shim-init-node.test.ts new file mode 100644 index 000000000..2a08891c6 --- /dev/null +++ b/packages/lib/shim-init-node.test.ts @@ -0,0 +1,19 @@ + +const { shimInit } = require('./shim-init-node'); +import shim from './shim'; +import { setupDatabaseAndSynchronizer, supportDir } from './testing/test-utils'; + +describe('shim-init-node', () => { + + beforeEach(async () => { + await setupDatabaseAndSynchronizer(1); + shimInit(); + }); + + test('should set mime the correct mime for a PDF file even if the extension is missing', async () => { + const filePath = `${supportDir}/valid_pdf_without_ext`; + const resource = await shim.createResourceFromPath(filePath); + + expect(resource.mime).toBe('application/pdf'); + }); +}); diff --git a/packages/lib/shim-init-node.ts b/packages/lib/shim-init-node.ts index 0d6b7f6b9..10d8b99a5 100644 --- a/packages/lib/shim-init-node.ts +++ b/packages/lib/shim-init-node.ts @@ -12,6 +12,7 @@ import { ResourceEntity } from './services/database/types'; import { TextItem } from 'pdfjs-dist/types/src/display/api'; import replaceUnsupportedCharacters from './utils/replaceUnsupportedCharacters'; import { FetchBlobOptions } from './types'; +import { fromFile as fileTypeFromFile } from 'file-type'; import crypto from './services/e2ee/crypto'; import FileApiDriverLocal from './file-api-driver-local'; @@ -306,9 +307,6 @@ function shimInit(options: ShimInitOptions = null) { ...options, }; - const readChunk = require('read-chunk'); - const imageType = require('image-type'); - const isUpdate = !!options.destinationResourceId; const uuid = require('./uuid').default; @@ -332,8 +330,7 @@ function shimInit(options: ShimInitOptions = null) { let fileExt = safeFileExtension(fileExtension(filePath)); if (!resource.mime) { - const buffer = await readChunk(filePath, 0, 64); - const detectedType = imageType(buffer); + const detectedType = await fileTypeFromFile(filePath); if (detectedType) { fileExt = detectedType.ext; diff --git a/yarn.lock b/yarn.lock index 860b93292..21eea762a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8467,6 +8467,7 @@ __metadata: diff-match-patch: 1.0.5 fast-deep-equal: 3.1.3 fast-xml-parser: 3.21.1 + file-type: 16.5.4 follow-redirects: 1.15.6 form-data: 4.0.0 fs-extra: 11.2.0 @@ -8474,7 +8475,6 @@ __metadata: html-entities: 1.4.0 html-minifier: 4.0.0 image-data-uri: 2.0.1 - image-type: 3.1.0 immer: 7.0.15 jest: 29.7.0 js-yaml: 4.1.0 @@ -8495,7 +8495,6 @@ __metadata: re-reselect: 4.0.1 react: 18.3.1 react-test-renderer: 18.3.1 - read-chunk: 2.1.0 redux: 4.2.1 relative: 3.0.2 reselect: 4.1.8 @@ -12288,6 +12287,13 @@ __metadata: languageName: node linkType: hard +"@tokenizer/token@npm:^0.3.0": + version: 0.3.0 + resolution: "@tokenizer/token@npm:0.3.0" + checksum: 1d575d02d2a9f0c5a4ca5180635ebd2ad59e0f18b42a65f3d04844148b49b3db35cf00b6012a1af2d59c2ab3caca59451c5689f747ba8667ee586ad717ee58e1 + languageName: node + linkType: hard + "@tootallnate/once@npm:1": version: 1.1.2 resolution: "@tootallnate/once@npm:1.1.2" @@ -24514,10 +24520,14 @@ __metadata: languageName: node linkType: hard -"file-type@npm:^10.9.0": - version: 10.11.0 - resolution: "file-type@npm:10.11.0" - checksum: cadd8cd187692dcde637a3ff53bb51c5d935633fc8085e7d25bfb3b4bf995e14a43f2baf71bdcb9d7235b3e725bd158b75d25911fa2f73e5812955382228c511 +"file-type@npm:16.5.4": + version: 16.5.4 + resolution: "file-type@npm:16.5.4" + dependencies: + readable-web-to-node-stream: ^3.0.0 + strtok3: ^6.2.4 + token-types: ^4.1.1 + checksum: d983c0f36491c57fcb6cc70fcb02c36d6b53f312a15053263e1924e28ca8314adf0db32170801ad777f09432c32155f31715ceaee66310947731588120d7ec27 languageName: node linkType: hard @@ -27813,15 +27823,6 @@ __metadata: languageName: node linkType: hard -"image-type@npm:3.1.0": - version: 3.1.0 - resolution: "image-type@npm:3.1.0" - dependencies: - file-type: ^10.9.0 - checksum: c50ebc7fb90a512fa3b6c08a81d2d21eac8abab9194ed64e2dec65437bea2d23201f4de0a920c37dc996aba005f7bf7fadb76ff479176a61be169eaabdee5eb6 - languageName: node - linkType: hard - "immediate@npm:^3.2.3": version: 3.3.0 resolution: "immediate@npm:3.3.0" @@ -30352,17 +30353,16 @@ __metadata: aws-sdk: 2.1340.0 chalk: 4.1.2 compare-version: 0.1.2 + file-type: 16.5.4 fs-extra: 11.2.0 gulp: 4.0.2 html-entities: 1.4.0 - image-type: 3.1.0 jest: 29.7.0 keytar: 7.9.0 md5: 2.3.0 node-rsa: 1.1.1 open: 8.4.2 proper-lockfile: 4.1.2 - read-chunk: 2.1.0 server-destroy: 1.0.1 sharp: 0.33.4 sprintf-js: 1.1.3 @@ -37115,6 +37115,13 @@ __metadata: languageName: node linkType: hard +"peek-readable@npm:^4.1.0": + version: 4.1.0 + resolution: "peek-readable@npm:4.1.0" + checksum: 02c673f9bc816f8e4e74a054c097225ad38d457d745b775e2b96faf404a54473b2f62f5bcd496f5ebc28696708bcc5e95bed409856f4bef5ed62eae9b4ac0dab + languageName: node + linkType: hard + "pend@npm:~1.2.0": version: 1.2.0 resolution: "pend@npm:1.2.0" @@ -40029,16 +40036,6 @@ __metadata: languageName: node linkType: hard -"read-chunk@npm:2.1.0": - version: 2.1.0 - resolution: "read-chunk@npm:2.1.0" - dependencies: - pify: ^3.0.0 - safe-buffer: ^5.1.1 - checksum: 03a5a3d8412c484f1d45639fcd11943932d1eda3e99d775f04f6878402d03420e62723a231c379ba6e6ac437d5f769214841e3001768b8359e7776ff727f6b4c - languageName: node - linkType: hard - "read-cmd-shim@npm:^1.0.1": version: 1.0.5 resolution: "read-cmd-shim@npm:1.0.5" @@ -40284,6 +40281,15 @@ __metadata: languageName: node linkType: hard +"readable-web-to-node-stream@npm:^3.0.0": + version: 3.0.2 + resolution: "readable-web-to-node-stream@npm:3.0.2" + dependencies: + readable-stream: ^3.6.0 + checksum: 8c56cc62c68513425ddfa721954875b382768f83fa20e6b31e365ee00cbe7a3d6296f66f7f1107b16cd3416d33aa9f1680475376400d62a081a88f81f0ea7f9c + languageName: node + linkType: hard + "readdir-scoped-modules@npm:^1.0.0": version: 1.1.0 resolution: "readdir-scoped-modules@npm:1.1.0" @@ -44134,6 +44140,16 @@ __metadata: languageName: node linkType: hard +"strtok3@npm:^6.2.4": + version: 6.3.0 + resolution: "strtok3@npm:6.3.0" + dependencies: + "@tokenizer/token": ^0.3.0 + peek-readable: ^4.1.0 + checksum: 90732cff3f325aef7c47c511f609b593e0873ec77b5081810071cde941344e6a0ee3ccb0cae1a9f5b4e12c81a2546fd6b322fabcdfbd1dd08362c2ce5291334a + languageName: node + linkType: hard + "structured-headers@npm:^0.4.1": version: 0.4.1 resolution: "structured-headers@npm:0.4.1" @@ -45414,6 +45430,16 @@ __metadata: languageName: node linkType: hard +"token-types@npm:^4.1.1": + version: 4.2.1 + resolution: "token-types@npm:4.2.1" + dependencies: + "@tokenizer/token": ^0.3.0 + ieee754: ^1.2.1 + checksum: cce256766b33e0f08ceffefa2198fb4961a417866d00780e58625999ab5c0699821407053e64eadc41b00bbb6c0d0c4d02fbd2199940d8a3ccb71e1b148ab9a2 + languageName: node + linkType: hard + "totalist@npm:^3.0.0": version: 3.0.1 resolution: "totalist@npm:3.0.1"