import { createNoteAndResource, newOcrService, ocrSampleDir, resourceFetcher, setupDatabaseAndSynchronizer, supportDir, switchClient, synchronizerStart } from '../../testing/test-utils'; import { supportedMimeTypes } from './OcrService'; import Resource from '../../models/Resource'; import { ResourceEntity, ResourceOcrStatus } from '../database/types'; import { msleep } from '@joplin/utils/time'; import Logger from '@joplin/utils/Logger'; describe('OcrService', () => { beforeEach(async () => { await setupDatabaseAndSynchronizer(1); await setupDatabaseAndSynchronizer(2); await switchClient(1); }); it('should process resources', async () => { const { resource: resource1 } = await createNoteAndResource({ path: `${ocrSampleDir}/testocr.png` }); const { resource: resource2 } = await createNoteAndResource({ path: `${supportDir}/photo.jpg` }); const { resource: resource3 } = await createNoteAndResource({ path: `${ocrSampleDir}/with_bullets.png` }); // Wait to make sure that updated_time is updated await msleep(1); expect(await Resource.needOcrCount(supportedMimeTypes)).toBe(3); const service = newOcrService(); await service.processResources(); const expectedText = 'This is a lot of 12 point text to test the\n' + 'ocr code and see if it works on all types\n' + 'of file format.\n' + 'The quick brown dog jumped over the\n' + 'lazy fox. The quick brown dog jumped\n' + 'over the lazy fox. The quick brown dog\n' + 'jumped over the lazy fox. The quick\n' + 'brown dog jumped over the lazy fox.'; const processedResource1: ResourceEntity = await Resource.load(resource1.id); expect(processedResource1.ocr_text).toBe(expectedText); expect(processedResource1.ocr_status).toBe(ResourceOcrStatus.Done); expect(processedResource1.ocr_error).toBe(''); const details = Resource.unserializeOcrDetails(processedResource1.ocr_details); const lines = details.map(l => l.words.map(w => w.t).join(' ')).join('\n'); expect(lines).toBe(expectedText); expect(details[0].words[0].t).toBe('This'); expect(details[0].words[0]).toEqual({ 't': 'This', 'bb': [36, 96, 92, 116], 'bl': [36, 96, 116, 116] }); // Also check that the resource blob has not been updated expect(processedResource1.blob_updated_time).toBe(resource1.blob_updated_time); expect(processedResource1.updated_time).toBeGreaterThan(resource1.updated_time); const processedResource2: ResourceEntity = await Resource.load(resource2.id); expect(processedResource2.ocr_text).toBe(''); expect(processedResource2.ocr_status).toBe(ResourceOcrStatus.Done); expect(processedResource2.ocr_error).toBe(''); const processedResource3: ResourceEntity = await Resource.load(resource3.id); expect(processedResource3.ocr_text).toBe('Declaration\n' + '| declare that:\n' + '® | will arrive in the UK within the next 48 hours\n' + '® | understand | have to provide proof of a negative COVID 19 test prior to departure to the UK (unless\n' + 'exempt)\n' + '® | have provided my seat number, if relevant\n' + '® The information | have entered in this form is correct\n' + '® | understand it could be a criminal offence to provide false details and | may be prosecuted\n' + 'If any of your information changes once you have submitted your details, such as travel details, seat number, or\n' + 'contact information, you must complete a new form.\n' + '| confirm that | understand and agree with the above declarations.', ); expect(processedResource3.ocr_status).toBe(ResourceOcrStatus.Done); expect(processedResource3.ocr_error).toBe(''); // Also check that the resource blob has not been updated expect(processedResource2.blob_updated_time).toBe(resource2.blob_updated_time); expect(processedResource2.updated_time).toBeGreaterThan(resource2.updated_time); await service.dispose(); }); it('should process PDF resources', async () => { const { resource } = await createNoteAndResource({ path: `${ocrSampleDir}/dummy.pdf` }); const service = newOcrService(); await service.processResources(); const processedResource: ResourceEntity = await Resource.load(resource.id); expect(processedResource.ocr_text).toBe('Dummy PDF file'); expect(processedResource.ocr_status).toBe(ResourceOcrStatus.Done); expect(processedResource.ocr_error).toBe(''); await service.dispose(); }); it('should handle case where resource blob has not yet been downloaded', async () => { await createNoteAndResource({ path: `${ocrSampleDir}/dummy.pdf` }); await synchronizerStart(); await switchClient(2); await synchronizerStart(); await msleep(1); const service = newOcrService(); await service.processResources(); { const resource: ResourceEntity = (await Resource.all())[0]; expect(resource.ocr_text).toBe(''); expect(resource.ocr_error).toBe(''); expect(resource.ocr_status).toBe(ResourceOcrStatus.Todo); } await resourceFetcher().startAndWait(); await service.processResources(); { const resource: ResourceEntity = (await Resource.all())[0]; expect(resource.ocr_text).toBe('Dummy PDF file'); expect(resource.ocr_error).toBe(''); expect(resource.ocr_status).toBe(ResourceOcrStatus.Done); } await service.dispose(); }); it('should handle case where resource blob cannot be downloaded', async () => { await createNoteAndResource({ path: `${ocrSampleDir}/dummy.pdf` }); await synchronizerStart(); await switchClient(2); await synchronizerStart(); const resource: ResourceEntity = (await Resource.all())[0]; // ---------------------------------------------------------------- // Fetch status is an error so OCR status will be an error too // ---------------------------------------------------------------- await Resource.setLocalState(resource.id, { resource_id: resource.id, fetch_status: Resource.FETCH_STATUS_ERROR, fetch_error: 'cannot be downloaded', }); const service = newOcrService(); // The service will print a warnign so we disable it in tests Logger.globalLogger.enabled = false; await service.processResources(); Logger.globalLogger.enabled = true; { const resource: ResourceEntity = (await Resource.all())[0]; expect(resource.ocr_text).toBe(''); expect(resource.ocr_error).toContain('Cannot process resource'); expect(resource.ocr_error).toContain('cannot be downloaded'); expect(resource.ocr_status).toBe(ResourceOcrStatus.Error); } // ---------------------------------------------------------------- // After the fetch status is reset and the resource downloaded, it // should also retry OCR and succeed. // ---------------------------------------------------------------- await Resource.resetFetchErrorStatus(resource.id); await resourceFetcher().startAndWait(); await service.processResources(); { const resource: ResourceEntity = (await Resource.all())[0]; expect(resource.ocr_text).toBe('Dummy PDF file'); expect(resource.ocr_error).toBe(''); expect(resource.ocr_status).toBe(ResourceOcrStatus.Done); } await service.dispose(); }); it('should handle conflicts if two clients process the same resource then sync', async () => { await createNoteAndResource({ path: `${ocrSampleDir}/dummy.pdf` }); const service1 = newOcrService(); await synchronizerStart(); await service1.processResources(); await switchClient(2); await synchronizerStart(); await msleep(1); await resourceFetcher().startAndWait(); const service2 = newOcrService(); await service2.processResources(); await synchronizerStart(); const expectedResouceUpatedTime = (await Resource.all())[0].updated_time; await switchClient(1); await synchronizerStart(); // A conflict happened during sync, but it is resolved by keeping the // remote version. expect((await Resource.all()).length).toBe(1); { const resource: ResourceEntity = (await Resource.all())[0]; expect(resource.ocr_text).toBe('Dummy PDF file'); expect(resource.ocr_error).toBe(''); expect(resource.ocr_status).toBe(ResourceOcrStatus.Done); expect(resource.updated_time).toBe(expectedResouceUpatedTime); } await service1.dispose(); await service2.dispose(); }); // Use this to quickly test with specific images: // it('should process resources 2', async () => { // await createNoteAndResource({ path: `${require('os').homedir()}/Desktop/AllClients.png` }); // const service = newOcrService(); // await service.processResources(); // console.info(await Resource.all()); // await service.dispose(); // }); });