joplin/packages/lib/services/synchronizer/Synchronizer.resources.test.ts

import time from '../../time';
import shim from '../../shim';
import Setting from '../../models/Setting';
import { NoteEntity, ResourceEntity } from '../../services/database/types';
import { remoteNotesFoldersResources, remoteResources } from '../../testing/test-utils-synchronizer';
import { synchronizerStart, tempFilePath, resourceFetcher, supportDir, setupDatabaseAndSynchronizer, synchronizer, fileApi, switchClient, syncTargetId, encryptionService, loadEncryptionMasterKey, fileContentEqual, checkThrowAsync, msleep } from '../../testing/test-utils';
import Folder from '../../models/Folder';
import Note from '../../models/Note';
import Resource from '../../models/Resource';
import ResourceFetcher from '../../services/ResourceFetcher';
import BaseItem from '../../models/BaseItem';
import { ModelType } from '../../BaseModel';
import { setEncryptionEnabled } from '../synchronizer/syncInfoUtils';
import { loadMasterKeysFromSettings } from '../e2ee/utils';

let insideBeforeEach = false;

describe('Synchronizer.resources', () => {

	beforeEach(async () => {
		insideBeforeEach = true;

		await setupDatabaseAndSynchronizer(1);
		await setupDatabaseAndSynchronizer(2);
		await switchClient(1);

		insideBeforeEach = false;
	});

	it('should sync resources', async () => {
		while (insideBeforeEach) await time.msleep(500);

		const folder1 = await Folder.save({ title: 'folder1' });
		const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
		await shim.attachFileToNote(note1, `${supportDir}/photo.jpg`);
		const resource1 = (await Resource.all())[0];
		const resourcePath1 = Resource.fullPath(resource1);
		await synchronizerStart();
		expect((await remoteNotesFoldersResources()).length).toBe(3);

		await switchClient(2);

		await synchronizerStart();
		const allResources = await Resource.all();
		expect(allResources.length).toBe(1);
		let resource1_2 = allResources[0];
		let ls = await Resource.localState(resource1_2);
		expect(resource1_2.id).toBe(resource1.id);
		expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_IDLE);

		const fetcher = new ResourceFetcher(() => { return synchronizer().api(); });
		fetcher.queueDownload_(resource1_2.id);
		await fetcher.waitForAllFinished();

		resource1_2 = await Resource.load(resource1.id);
		ls = await Resource.localState(resource1_2);
		expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_DONE);

		const resourcePath1_2 = Resource.fullPath(resource1_2);
		expect(fileContentEqual(resourcePath1, resourcePath1_2)).toBe(true);
	});

	it('should handle resource download errors', async () => {
		while (insideBeforeEach) await time.msleep(500);

		const folder1 = await Folder.save({ title: 'folder1' });
		const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
		await shim.attachFileToNote(note1, `${supportDir}/photo.jpg`);
		let resource1 = (await Resource.all())[0];
		await synchronizerStart();

		await switchClient(2);

		await synchronizerStart();

		const fetcher = new ResourceFetcher(() => {
			return {
			// Simulate a failed download
				// eslint-disable-next-line @typescript-eslint/ban-types -- Old code before rule was applied
				get: () => { return new Promise((_resolve: Function, reject: Function) => { reject(new Error('did not work')); }); },
			};
		});
		fetcher.queueDownload_(resource1.id);
		await fetcher.waitForAllFinished();

		resource1 = await Resource.load(resource1.id);
		const ls = await Resource.localState(resource1);
		expect(ls.fetch_status).toBe(Resource.FETCH_STATUS_ERROR);
		expect(ls.fetch_error).toBe('did not work');
	});

	it('should set the resource file size if it is missing', async () => {
		while (insideBeforeEach) await time.msleep(500);

		const folder1 = await Folder.save({ title: 'folder1' });
		const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
		await shim.attachFileToNote(note1, `${supportDir}/photo.jpg`);
		await synchronizerStart();

		await switchClient(2);

		await synchronizerStart();
		let r1 = (await Resource.all())[0];
		await Resource.setFileSizeOnly(r1.id, -1);
		r1 = await Resource.load(r1.id);
		expect(r1.size).toBe(-1);

		const fetcher = new ResourceFetcher(() => { return synchronizer().api(); });
		fetcher.queueDownload_(r1.id);
		await fetcher.waitForAllFinished();
		r1 = await Resource.load(r1.id);
		expect(r1.size).toBe(2720);
	});

	it('should delete resources', async () => {
		while (insideBeforeEach) await time.msleep(500);

		const folder1 = await Folder.save({ title: 'folder1' });
		const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
		await shim.attachFileToNote(note1, `${supportDir}/photo.jpg`);
		const resource1 = (await Resource.all())[0];
		const resourcePath1 = Resource.fullPath(resource1);
		await synchronizerStart();

		await switchClient(2);

		await synchronizerStart();
		let allResources = await Resource.all();
		expect(allResources.length).toBe(1);
		expect((await remoteNotesFoldersResources()).length).toBe(3);
		await Resource.delete(resource1.id);
		await synchronizerStart();
		expect((await remoteNotesFoldersResources()).length).toBe(2);

		const remoteBlob = await fileApi().stat(`.resource/${resource1.id}`);
		expect(!remoteBlob).toBe(true);

		await switchClient(1);

		expect(await shim.fsDriver().exists(resourcePath1)).toBe(true);
		await synchronizerStart();
		allResources = await Resource.all();
		expect(allResources.length).toBe(0);
		expect(await shim.fsDriver().exists(resourcePath1)).toBe(false);
	});

	it('should encrypt resources', async () => {
		setEncryptionEnabled(true);
		const masterKey = await loadEncryptionMasterKey();

		const folder1 = await Folder.save({ title: 'folder1' });
		const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
		await shim.attachFileToNote(note1, `${supportDir}/photo.jpg`);
		const resource1 = (await Resource.all())[0];
		const resourcePath1 = Resource.fullPath(resource1);
		await synchronizerStart();

		await switchClient(2);

		await synchronizerStart();
		Setting.setObjectValue('encryption.passwordCache', masterKey.id, '123456');
		await loadMasterKeysFromSettings(encryptionService());

		const fetcher = new ResourceFetcher(() => { return synchronizer().api(); });
		fetcher.queueDownload_(resource1.id);
		await fetcher.waitForAllFinished();

		let resource1_2 = (await Resource.all())[0];
		resource1_2 = await Resource.decrypt(resource1_2);
		const resourcePath1_2 = Resource.fullPath(resource1_2);

		expect(fileContentEqual(resourcePath1, resourcePath1_2)).toBe(true);
	});

	it('should sync resource blob changes', async () => {
		const tempFile = tempFilePath('txt');
		await shim.fsDriver().writeFile(tempFile, '1234', 'utf8');
		const folder1 = await Folder.save({ title: 'folder1' });
		const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
		await shim.attachFileToNote(note1, tempFile);
		await synchronizerStart();

		await switchClient(2);

		await synchronizerStart();
		await resourceFetcher().start();
		await resourceFetcher().waitForAllFinished();
		let resource1_2 = (await Resource.all())[0];
		const modFile = tempFilePath('txt');
		await shim.fsDriver().writeFile(modFile, '1234 MOD', 'utf8');
		await Resource.updateResourceBlobContent(resource1_2.id, modFile);
		const originalSize = resource1_2.size;
		resource1_2 = (await Resource.all())[0];
		const newSize = resource1_2.size;
		expect(originalSize).toBe(4);
		expect(newSize).toBe(8);
		await synchronizerStart();

		await switchClient(1);

		await synchronizerStart();
		await resourceFetcher().start();
		await resourceFetcher().waitForAllFinished();
		const resource1_1 = (await Resource.all())[0];
		expect(resource1_1.size).toBe(newSize);
		expect(await Resource.resourceBlobContent(resource1_1.id, 'utf8')).toBe('1234 MOD');
	});

	it('should handle resource conflicts', async () => {
		{
			const tempFile = tempFilePath('txt');
			await shim.fsDriver().writeFile(tempFile, '1234', 'utf8');
			const folder1 = await Folder.save({ title: 'folder1' });
			const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
			await shim.attachFileToNote(note1, tempFile);
			await synchronizerStart();
		}

		await switchClient(2);

		{
			await synchronizerStart();
			await resourceFetcher().start();
			await resourceFetcher().waitForAllFinished();
			const resource = (await Resource.all())[0];
			const modFile2 = tempFilePath('txt');
			await shim.fsDriver().writeFile(modFile2, '1234 MOD 2', 'utf8');
			await Resource.updateResourceBlobContent(resource.id, modFile2);
			await synchronizerStart();
		}

		await switchClient(1);

		{
			// Going to modify a resource without syncing first, which will cause a conflict
			const resource = (await Resource.all())[0];
			const modFile1 = tempFilePath('txt');
			await shim.fsDriver().writeFile(modFile1, '1234 MOD 1', 'utf8');
			await Resource.updateResourceBlobContent(resource.id, modFile1);
			await synchronizerStart(); // CONFLICT

			// If we try to read the resource content now, it should throw because the local
			// content has been moved to the conflict notebook, and the new local content
			// has not been downloaded yet.
			await checkThrowAsync(async () => await Resource.resourceBlobContent(resource.id));

			// Now download resources, and our local content would have been overwritten by
			// the content from client 2
			await resourceFetcher().start();
			await resourceFetcher().waitForAllFinished();
			const localContent = await Resource.resourceBlobContent(resource.id, 'utf8');
			expect(localContent).toBe('1234 MOD 2');

			// Check that the Conflict note has been generated, with the conflict resource
			// attached to it, and check that it has the original content.
			const allNotes = await Note.all();
			expect(allNotes.length).toBe(2);
			const resourceConflictFolderId = await Resource.resourceConflictFolderId();
			const conflictNote = allNotes.find((v: NoteEntity) => {
				return v.parent_id === resourceConflictFolderId;
			});
			expect(!!conflictNote).toBe(true);
			const resourceIds = await Note.linkedResourceIds(conflictNote.body);
			expect(resourceIds.length).toBe(1);
			const conflictContent = await Resource.resourceBlobContent(resourceIds[0], 'utf8');
			expect(conflictContent).toBe('1234 MOD 1');

			// Also check that the conflict folder has been created and that it
			// is a top folder.
			const resourceConflictFolder = await Folder.load(resourceConflictFolderId);
			expect(resourceConflictFolder).toBeTruthy();
			expect(resourceConflictFolder.parent_id).toBeFalsy();
		}
	});

	it('should handle resource conflicts if a resource is changed locally but deleted remotely', async () => {
		{
			const tempFile = tempFilePath('txt');
			await shim.fsDriver().writeFile(tempFile, '1234', 'utf8');
			const folder1 = await Folder.save({ title: 'folder1' });
			const note1 = await Note.save({ title: 'ma note', parent_id: folder1.id });
			await shim.attachFileToNote(note1, tempFile);
			await synchronizerStart();
		}

		await switchClient(2);

		{
			await synchronizerStart();
			await resourceFetcher().startAndWait();
		}

		await switchClient(1);

		{
			const resource = (await Resource.all())[0];
			await Resource.delete(resource.id);
			await synchronizerStart();

		}

		await switchClient(2);

		{
			const originalResource = (await Resource.all())[0];
			await Resource.save({ id: originalResource.id, title: 'modified resource' });
			await synchronizerStart(); // CONFLICT

			const deletedResource = await Resource.load(originalResource.id);
			expect(!deletedResource).toBe(true);

			const allResources = await Resource.all();
			expect(allResources.length).toBe(1);
			const conflictResource = allResources[0];
			expect(originalResource.id).not.toBe(conflictResource.id);
			expect(conflictResource.title).toBe('modified resource');
		}
	});

	it('should not upload a resource if it has not been fetched yet', async () => {
		// In some rare cases, the synchronizer might try to upload a resource even though it
		// doesn't have the resource file. It can happen in this situation:
		// - C1 create resource
		// - C1 sync
		// - C2 sync
		// - C2 resource metadata is received but ResourceFetcher hasn't downloaded the file yet
		// - C2 enables E2EE - all the items are marked for forced sync
		// - C2 sync
		// The synchronizer will try to upload the resource, even though it doesn't have the file,
		// so we need to make sure it doesn't. But also that once it gets the file, the resource
		// does get uploaded.

		const note1 = await Note.save({ title: 'note' });
		await shim.attachFileToNote(note1, `${supportDir}/photo.jpg`);
		const resource = (await Resource.all())[0];
		await Resource.setLocalState(resource.id, { fetch_status: Resource.FETCH_STATUS_IDLE });
		await synchronizerStart();

		expect((await remoteResources()).length).toBe(0);

		await Resource.setLocalState(resource.id, { fetch_status: Resource.FETCH_STATUS_DONE });
		await synchronizerStart();

		// At first, the resource is marked as cannot sync, so even after
		// synchronisation, nothing should happen.
		expect((await remoteResources()).length).toBe(0);

		// The user can retry the item, in which case sync should happen.
		await BaseItem.saveSyncEnabled(ModelType.Resource, resource.id);
		await synchronizerStart();
		expect((await remoteResources()).length).toBe(1);
	});

	it('should not download resources over the limit', async () => {
		const note1 = await Note.save({ title: 'note' });
		await shim.attachFileToNote(note1, `${supportDir}/photo.jpg`);
		await synchronizer().start();

		await switchClient(2);

		const previousMax = synchronizer().maxResourceSize_;
		synchronizer().maxResourceSize_ = 1;
		await synchronizerStart();
		synchronizer().maxResourceSize_ = previousMax;

		const syncItems = await BaseItem.allSyncItems(syncTargetId());
		expect(syncItems.length).toBe(2);
		expect(syncItems[1].item_location).toBe(BaseItem.SYNC_ITEM_LOCATION_REMOTE);
		expect(syncItems[1].sync_disabled).toBe(1);
	});

	it('should not upload blob if it has not changed', async () => {
		const note = await Note.save({});
		await shim.attachFileToNote(note, `${supportDir}/sample.txt`);
		const resource: ResourceEntity = (await Resource.all())[0];
		const resourcePath = `.resource/${resource.id}`;

		await synchronizer().api().put(resourcePath, 'before upload');
		expect(await synchronizer().api().get(resourcePath)).toBe('before upload');
		await synchronizerStart();
		expect(await synchronizer().api().get(resourcePath)).toBe('just testing');

		// ----------------------------------------------------------------------
		// Change metadata only and check that blob is not uploaded. To do this,
		// we manually overwrite the data on the sync target, then sync. If the
		// synchronizer doesn't upload the blob, this manually changed data
		// should remain.
		// ----------------------------------------------------------------------

		await Resource.save({ id: resource.id, title: 'my new title' });
		await synchronizer().api().put(resourcePath, 'check if changed');
		await synchronizerStart();
		expect(await synchronizer().api().get(resourcePath)).toBe('check if changed');

		// ----------------------------------------------------------------------
		// Now change the blob, and check that the remote item has been
		// overwritten.
		// ----------------------------------------------------------------------

		await Resource.updateResourceBlobContent(resource.id, `${supportDir}/sample.txt`);
		await synchronizerStart();
		expect(await synchronizer().api().get(resourcePath)).toBe('just testing');

		// ----------------------------------------------------------------------
		// Change the blob, then change the metadata, and sync. Even though
		// blob_updated_time is earlier than updated_time, it should still
		// update everything on the sync target, because both times are after
		// the item sync_time.
		// ----------------------------------------------------------------------

		await Resource.updateResourceBlobContent(resource.id, `${supportDir}/sample2.txt`);
		await msleep(1);
		await Resource.save({ id: resource.id, title: 'my new title 2' });
		await synchronizerStart();
		expect(await synchronizer().api().get(resourcePath)).toBe('just testing 2');
		expect(await synchronizer().api().get(`${resource.id}.md`)).toContain('my new title 2');
	});

});