self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-21 09:38:01 +02:00
Code Issues Releases Activity

Desktop,CLI: Fixes #10653: Change Resource filetype detecting strategy (#10907)

Browse Source 
Co-authored-by: Laurent Cozic <laurent22@users.noreply.github.com>
This commit is contained in:
pedr 2024-11-07 10:46:08 -03:00 committed by GitHub
parent 02a0d0d0cc
commit 0aba14f581
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 278 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.eslintignore
View File

@ -1338,6 +1338,7 @@ packages/lib/services/trash/permanentlyDeleteOldItems.test.js
packages/lib/services/trash/permanentlyDeleteOldItems.js
packages/lib/services/trash/restoreItems.test.js
packages/lib/services/trash/restoreItems.js
packages/lib/shim-init-node.test.js
packages/lib/shim-init-node.js
packages/lib/shim.js
packages/lib/string-utils.test.js

1
.gitignore vendored
View File

@ -1315,6 +1315,7 @@ packages/lib/services/trash/permanentlyDeleteOldItems.test.js
packages/lib/services/trash/permanentlyDeleteOldItems.js
packages/lib/services/trash/restoreItems.test.js
packages/lib/services/trash/restoreItems.js
packages/lib/shim-init-node.test.js
packages/lib/shim-init-node.js
packages/lib/shim.js
packages/lib/string-utils.test.js

6
packages/app-cli/app/command-e2ee.ts
View File

@ -8,8 +8,7 @@ import shim from '@joplin/lib/shim';
import * as pathUtils from '@joplin/lib/path-utils';
import { getEncryptionEnabled, localSyncInfo } from '@joplin/lib/services/synchronizer/syncInfoUtils';
import { generateMasterKeyAndEnableEncryption, loadMasterKeysFromSettings, masterPasswordIsValid, setupAndDisableEncryption } from '@joplin/lib/services/e2ee/utils';
const imageType = require('image-type');
const readChunk = require('read-chunk');
import { fromFile as fileTypeFromFile } from 'file-type';
class Command extends BaseCommand {
public usage() {
@ -136,8 +135,7 @@ class Command extends BaseCommand {
const outputDir = options.output ? options.output : require('os').tmpdir();
let outFile = `${outputDir}/${pathUtils.filename(args.path)}.${Date.now()}.bin`;
await EncryptionService.instance().decryptFile(args.path, outFile);
const buffer = await readChunk(outFile, 0, 64);
const detectedType = imageType(buffer);
const detectedType = await fileTypeFromFile(outFile);
if (detectedType) {
const newOutFile = `${outFile}.${detectedType.ext}`;

3
packages/app-cli/package.json
View File

@ -47,15 +47,14 @@
"aws-sdk": "2.1340.0",
"chalk": "4.1.2",
"compare-version": "0.1.2",
"file-type": "16.5.4",
"fs-extra": "11.2.0",
"html-entities": "1.4.0",
"image-type": "3.1.0",
"keytar": "7.9.0",
"md5": "2.3.0",
"node-rsa": "1.1.1",
"open": "8.4.2",
"proper-lockfile": "4.1.2",
"read-chunk": "2.1.0",
"server-destroy": "1.0.1",
"sharp": "0.33.4",
"sprintf-js": "1.1.3",

198
packages/app-cli/tests/support/valid_pdf_without_ext Normal file
View File

@ -0,0 +1,198 @@
%PDF-1.3
%âãÏÓ
1 0 obj
<<
/Type /Catalog
/Outlines 2 0 R
/Pages 3 0 R
>>
endobj
2 0 obj
<<
/Type /Outlines
/Count 0
>>
endobj
3 0 obj
<<
/Type /Pages
/Count 2
/Kids [ 4 0 R 6 0 R ]
>>
endobj
4 0 obj
<<
/Type /Page
/Parent 3 0 R
/Resources <<
/Font <<
/F1 9 0 R
>>
/ProcSet 8 0 R
>>
/MediaBox [0 0 612.0000 792.0000]
/Contents 5 0 R
>>
endobj
5 0 obj
<< /Length 1074 >>
stream
2 J
BT
0 0 0 rg
/F1 0027 Tf
57.3750 722.2800 Td
( A Simple PDF File ) Tj
ET
BT
/F1 0010 Tf
69.2500 688.6080 Td
( This is a small demonstration .pdf file - ) Tj
ET
BT
/F1 0010 Tf
69.2500 664.7040 Td
( just for use in the Virtual Mechanics tutorials. More text. And more ) Tj
ET
BT
/F1 0010 Tf
69.2500 652.7520 Td
( text. And more text. And more text. And more text. ) Tj
ET
BT
/F1 0010 Tf
69.2500 628.8480 Td
( And more text. And more text. And more text. And more text. And more ) Tj
ET
BT
/F1 0010 Tf
69.2500 616.8960 Td
( text. And more text. Boring, zzzzz. And more text. And more text. And ) Tj
ET
BT
/F1 0010 Tf
69.2500 604.9440 Td
( more text. And more text. And more text. And more text. And more text. ) Tj
ET
BT
/F1 0010 Tf
69.2500 592.9920 Td
( And more text. And more text. ) Tj
ET
BT
/F1 0010 Tf
69.2500 569.0880 Td
( And more text. And more text. And more text. And more text. And more ) Tj
ET
BT
/F1 0010 Tf
69.2500 557.1360 Td
( text. And more text. And more text. Even more. Continued on page 2 ...) Tj
ET
endstream
endobj
6 0 obj
<<
/Type /Page
/Parent 3 0 R
/Resources <<
/Font <<
/F1 9 0 R
>>
/ProcSet 8 0 R
>>
/MediaBox [0 0 612.0000 792.0000]
/Contents 7 0 R
>>
endobj
7 0 obj
<< /Length 676 >>
stream
2 J
BT
0 0 0 rg
/F1 0027 Tf
57.3750 722.2800 Td
( Simple PDF File 2 ) Tj
ET
BT
/F1 0010 Tf
69.2500 688.6080 Td
( ...continued from page 1. Yet more text. And more text. And more text. ) Tj
ET
BT
/F1 0010 Tf
69.2500 676.6560 Td
( And more text. And more text. And more text. And more text. And more ) Tj
ET
BT
/F1 0010 Tf
69.2500 664.7040 Td
( text. Oh, how boring typing this stuff. But not as boring as watching ) Tj
ET
BT
/F1 0010 Tf
69.2500 652.7520 Td
( paint dry. And more text. And more text. And more text. And more text. ) Tj
ET
BT
/F1 0010 Tf
69.2500 640.8000 Td
( Boring. More, a little more text. The end, and just as well. ) Tj
ET
endstream
endobj
8 0 obj
[/PDF /Text]
endobj
9 0 obj
<<
/Type /Font
/Subtype /Type1
/Name /F1
/BaseFont /Helvetica
/Encoding /WinAnsiEncoding
>>
endobj
10 0 obj
<<
/Creator (Rave \(http://www.nevrona.com/rave\))
/Producer (Nevrona Designs)
/CreationDate (D:20060301072826)
>>
endobj
xref
0 11
0000000000 65535 f
0000000019 00000 n
0000000093 00000 n
0000000147 00000 n
0000000222 00000 n
0000000390 00000 n
0000001522 00000 n
0000001690 00000 n
0000002423 00000 n
0000002456 00000 n
0000002574 00000 n
trailer
<<
/Size 11
/Root 1 0 R
/Info 10 0 R
>>
startxref
2714
%%EOF

3
packages/lib/package.json
View File

@ -59,6 +59,7 @@
"diff-match-patch": "1.0.5",
"fast-deep-equal": "3.1.3",
"fast-xml-parser": "3.21.1",
"file-type": "16.5.4",
"follow-redirects": "1.15.6",
"form-data": "4.0.0",
"fs-extra": "11.2.0",
@ -66,7 +67,6 @@
"html-entities": "1.4.0",
"html-minifier": "4.0.0",
"image-data-uri": "2.0.1",
"image-type": "3.1.0",
"immer": "7.0.15",
"js-yaml": "4.1.0",
"markdown-it": "13.0.2",
@ -83,7 +83,6 @@
"promise": "8.3.0",
"query-string": "7.1.3",
"re-reselect": "4.0.1",
"read-chunk": "2.1.0",
"redux": "4.2.1",
"relative": "3.0.2",
"reselect": "4.1.8",

19
packages/lib/shim-init-node.test.ts Normal file
View File

@ -0,0 +1,19 @@
const { shimInit } = require('./shim-init-node');
import shim from './shim';
import { setupDatabaseAndSynchronizer, supportDir } from './testing/test-utils';
describe('shim-init-node', () => {
beforeEach(async () => {
await setupDatabaseAndSynchronizer(1);
shimInit();
});
test('should set mime the correct mime for a PDF file even if the extension is missing', async () => {
const filePath = `${supportDir}/valid_pdf_without_ext`;
const resource = await shim.createResourceFromPath(filePath);
expect(resource.mime).toBe('application/pdf');
});
});

7
packages/lib/shim-init-node.ts
View File

@ -12,6 +12,7 @@ import { ResourceEntity } from './services/database/types';
import { TextItem } from 'pdfjs-dist/types/src/display/api';
import replaceUnsupportedCharacters from './utils/replaceUnsupportedCharacters';
import { FetchBlobOptions } from './types';
import { fromFile as fileTypeFromFile } from 'file-type';
import crypto from './services/e2ee/crypto';
import FileApiDriverLocal from './file-api-driver-local';
@ -306,9 +307,6 @@ function shimInit(options: ShimInitOptions = null) {
...options,
};
const readChunk = require('read-chunk');
const imageType = require('image-type');
const isUpdate = !!options.destinationResourceId;
const uuid = require('./uuid').default;
@ -332,8 +330,7 @@ function shimInit(options: ShimInitOptions = null) {
let fileExt = safeFileExtension(fileExtension(filePath));
if (!resource.mime) {
const buffer = await readChunk(filePath, 0, 64);
const detectedType = imageType(buffer);
const detectedType = await fileTypeFromFile(filePath);
if (detectedType) {
fileExt = detectedType.ext;

80
yarn.lock
View File

@ -8467,6 +8467,7 @@ __metadata:
diff-match-patch: 1.0.5
fast-deep-equal: 3.1.3
fast-xml-parser: 3.21.1
file-type: 16.5.4
follow-redirects: 1.15.6
form-data: 4.0.0
fs-extra: 11.2.0
@ -8474,7 +8475,6 @@ __metadata:
html-entities: 1.4.0
html-minifier: 4.0.0
image-data-uri: 2.0.1
image-type: 3.1.0
immer: 7.0.15
jest: 29.7.0
js-yaml: 4.1.0
@ -8495,7 +8495,6 @@ __metadata:
re-reselect: 4.0.1
react: 18.3.1
react-test-renderer: 18.3.1
read-chunk: 2.1.0
redux: 4.2.1
relative: 3.0.2
reselect: 4.1.8
@ -12288,6 +12287,13 @@ __metadata:
languageName: node
linkType: hard
"@tokenizer/token@npm:^0.3.0":
version: 0.3.0
resolution: "@tokenizer/token@npm:0.3.0"
checksum: 1d575d02d2a9f0c5a4ca5180635ebd2ad59e0f18b42a65f3d04844148b49b3db35cf00b6012a1af2d59c2ab3caca59451c5689f747ba8667ee586ad717ee58e1
languageName: node
linkType: hard
"@tootallnate/once@npm:1":
version: 1.1.2
resolution: "@tootallnate/once@npm:1.1.2"
@ -24514,10 +24520,14 @@ __metadata:
languageName: node
linkType: hard
"file-type@npm:^10.9.0":
version: 10.11.0
resolution: "file-type@npm:10.11.0"
checksum: cadd8cd187692dcde637a3ff53bb51c5d935633fc8085e7d25bfb3b4bf995e14a43f2baf71bdcb9d7235b3e725bd158b75d25911fa2f73e5812955382228c511
"file-type@npm:16.5.4":
version: 16.5.4
resolution: "file-type@npm:16.5.4"
dependencies:
readable-web-to-node-stream: ^3.0.0
strtok3: ^6.2.4
token-types: ^4.1.1
checksum: d983c0f36491c57fcb6cc70fcb02c36d6b53f312a15053263e1924e28ca8314adf0db32170801ad777f09432c32155f31715ceaee66310947731588120d7ec27
languageName: node
linkType: hard
@ -27813,15 +27823,6 @@ __metadata:
languageName: node
linkType: hard
"image-type@npm:3.1.0":
version: 3.1.0
resolution: "image-type@npm:3.1.0"
dependencies:
file-type: ^10.9.0
checksum: c50ebc7fb90a512fa3b6c08a81d2d21eac8abab9194ed64e2dec65437bea2d23201f4de0a920c37dc996aba005f7bf7fadb76ff479176a61be169eaabdee5eb6
languageName: node
linkType: hard
"immediate@npm:^3.2.3":
version: 3.3.0
resolution: "immediate@npm:3.3.0"
@ -30352,17 +30353,16 @@ __metadata:
aws-sdk: 2.1340.0
chalk: 4.1.2
compare-version: 0.1.2
file-type: 16.5.4
fs-extra: 11.2.0
gulp: 4.0.2
html-entities: 1.4.0
image-type: 3.1.0
jest: 29.7.0
keytar: 7.9.0
md5: 2.3.0
node-rsa: 1.1.1
open: 8.4.2
proper-lockfile: 4.1.2
read-chunk: 2.1.0
server-destroy: 1.0.1
sharp: 0.33.4
sprintf-js: 1.1.3
@ -37115,6 +37115,13 @@ __metadata:
languageName: node
linkType: hard
"peek-readable@npm:^4.1.0":
version: 4.1.0
resolution: "peek-readable@npm:4.1.0"
checksum: 02c673f9bc816f8e4e74a054c097225ad38d457d745b775e2b96faf404a54473b2f62f5bcd496f5ebc28696708bcc5e95bed409856f4bef5ed62eae9b4ac0dab
languageName: node
linkType: hard
"pend@npm:~1.2.0":
version: 1.2.0
resolution: "pend@npm:1.2.0"
@ -40029,16 +40036,6 @@ __metadata:
languageName: node
linkType: hard
"read-chunk@npm:2.1.0":
version: 2.1.0
resolution: "read-chunk@npm:2.1.0"
dependencies:
pify: ^3.0.0
safe-buffer: ^5.1.1
checksum: 03a5a3d8412c484f1d45639fcd11943932d1eda3e99d775f04f6878402d03420e62723a231c379ba6e6ac437d5f769214841e3001768b8359e7776ff727f6b4c
languageName: node
linkType: hard
"read-cmd-shim@npm:^1.0.1":
version: 1.0.5
resolution: "read-cmd-shim@npm:1.0.5"
@ -40284,6 +40281,15 @@ __metadata:
languageName: node
linkType: hard
"readable-web-to-node-stream@npm:^3.0.0":
version: 3.0.2
resolution: "readable-web-to-node-stream@npm:3.0.2"
dependencies:
readable-stream: ^3.6.0
checksum: 8c56cc62c68513425ddfa721954875b382768f83fa20e6b31e365ee00cbe7a3d6296f66f7f1107b16cd3416d33aa9f1680475376400d62a081a88f81f0ea7f9c
languageName: node
linkType: hard
"readdir-scoped-modules@npm:^1.0.0":
version: 1.1.0
resolution: "readdir-scoped-modules@npm:1.1.0"
@ -44134,6 +44140,16 @@ __metadata:
languageName: node
linkType: hard
"strtok3@npm:^6.2.4":
version: 6.3.0
resolution: "strtok3@npm:6.3.0"
dependencies:
"@tokenizer/token": ^0.3.0
peek-readable: ^4.1.0
checksum: 90732cff3f325aef7c47c511f609b593e0873ec77b5081810071cde941344e6a0ee3ccb0cae1a9f5b4e12c81a2546fd6b322fabcdfbd1dd08362c2ce5291334a
languageName: node
linkType: hard
"structured-headers@npm:^0.4.1":
version: 0.4.1
resolution: "structured-headers@npm:0.4.1"
@ -45414,6 +45430,16 @@ __metadata:
languageName: node
linkType: hard
"token-types@npm:^4.1.1":
version: 4.2.1
resolution: "token-types@npm:4.2.1"
dependencies:
"@tokenizer/token": ^0.3.0
ieee754: ^1.2.1
checksum: cce256766b33e0f08ceffefa2198fb4961a417866d00780e58625999ab5c0699821407053e64eadc41b00bbb6c0d0c4d02fbd2199940d8a3ccb71e1b148ab9a2
languageName: node
linkType: hard
"totalist@npm:^3.0.0":
version: 3.0.1
resolution: "totalist@npm:3.0.1"