joplin/CliClient/app/fuzzing.js

require('source-map-support').install();
require('babel-plugin-transform-runtime');

import { time } from 'lib/time-utils.js';
import { Logger } from 'lib/logger.js';
import lodash from 'lodash';

const exec = require('child_process').exec
const fs = require('fs-extra');

const baseDir = '/var/www/joplin/CliClient/tests/fuzzing';
const syncDir = baseDir + '/sync';
const joplinAppPath = __dirname + '/main.js';
let syncDurations = [];

const logger = new Logger();
logger.addTarget('console');
logger.setLevel(Logger.LEVEL_DEBUG);

process.on('unhandledRejection', (reason, p) => {
	console.error('Unhandled promise rejection', p, 'reason:', reason);
});

function createClient(id) {
	return {
		'id': id,
		'profileDir': baseDir + '/client' + id,
	};
}

async function createClients() {
	let output = [];
	let promises = [];
	for (let clientId = 0; clientId < 2; clientId++) {
		let client = createClient(clientId);
		promises.push(fs.remove(client.profileDir));
		promises.push(execCommand(client, 'config sync.target local').then(() => { return execCommand(client, 'config sync.local.path ' + syncDir); }));
		output.push(client);
	}

	await Promise.all(promises);

	return output;
}

function randomElement(array) {
	if (!array.length) return null;
	return array[Math.floor(Math.random() * array.length)];
}

function randomWord() {
	const words = ['belief','scandalous','flawless','wrestle','sort','moldy','carve','incompetent','cruel','awful','fang','holistic','makeshift','synonymous','questionable','soft','drop','boot','whimsical','stir','idea','adhesive','present','hilarious','unusual','divergent','probable','depend','suck','belong','advise','straight','encouraging','wing','clam','serve','fill','nostalgic','dysfunctional','aggressive','floor','baby','grease','sisters','print','switch','control','victorious','cracker','dream','wistful','adaptable','reminiscent','inquisitive','pushy','unaccountable','receive','guttural','two','protect','skin','unbiased','plastic','loutish','zip','used','divide','communicate','dear','muddled','dinosaurs','grip','trees','well-off','calendar','chickens','irate','deranged','trip','stream','white','poison','attack','obtain','theory','laborer','omniscient','brake','maniacal','curvy','smoke','babies','punch','hammer','toothbrush','same','crown','jagged','peep','difficult','reject','merciful','useless','doctor','mix','wicked','plant','quickest','roll','suffer','curly','brother','frighten','cold','tremendous','move','knot','lame','imaginary','capricious','raspy','aunt','loving','wink','wooden','hop','free','drab','fire','instrument','border','frame','silent','glue','decorate','distance','powerful','pig','admit','fix','pour','flesh','profuse','skinny','learn','filthy','dress','bloody','produce','innocent','meaty','pray','slimy','sun','kindhearted','dime','exclusive','boast','neat','ruthless','recess','grieving','daily','hateful','ignorant','fence','spring','slim','education','overflow','plastic','gaping','chew','detect','right','lunch','gainful','argue','cloistered','horses','orange','shame','bitter','able','sail','magical','exist','force','wheel','best','suit','spurious','partner','request','dog','gusty','money','gaze','lonely','company','pale','tempt','rat','flame','wobble','superficial','stop','protective','stare','tongue','heal','railway','idiotic','roll','puffy','turn','meeting','new','frightening','sophisticated','poke','elderly','room','stimulating','increase','moor','secret','lean','occur','country','damp','evanescent','alluring','oafish','join','thundering','cars','awesome','advice','unruly','ray','wind','anxious','fly','hammer','adventurous','shop','cook','trucks','nonchalant','addition','base','abashed','excuse','giants','dramatic','piquant','coach','possess','poor','finger','wide-eyed','aquatic','welcome','instruct','expert','evasive','hug','cute','return','mice','damage','turkey','quiet','bewildered','tidy','pointless','outrageous','medical','foolish','curve','grandiose','gullible','hapless','gleaming','third','grin','pipe','egg','act','physical','eager','side','milk','tearful','fertile','average','glamorous','strange','yak','terrific','thin','near','snails','flowery','authority','fish','curious','perpetual','healthy','health','match','fade','chemical','economic','drawer','avoid','lying','minister','lick','powder','decay','desire','furry','faint','beam','sordid','fax','tail','bawdy','cherry','letter','clover','ladybug','teeth','behavior','black','amazing','pink','waste','island','forgetful','needless','lock','waves','boundary','receipt','handy','religion','hypnotic','aftermath','explain','sense','mundane','rambunctious','second','preserve','alarm','dusty','event','blow','weigh','value','glorious','jail','sigh','cemetery','serious','yummy','cattle','understood','limit','alert','fear','lucky','tested','surround','dolls','pleasant','disillusioned','discover','tray','night','seemly','liquid','worry','pen','bent','gruesome','war','teeny-tiny','common','judge','symptomatic','bed','trot','unequaled','flowers','friends','damaged','peel','skip','show','twist','worthless','brush','look','behave','imperfect','week','petite','direction','soda','lively','coal','coil','release','berserk','books','impossible','replace','cough','chunky','torpid','discreet','material','bomb','soothe','crack','hope','license','frightened','breathe','maddening','calculator','committee','paltry','green','subsequent','arrest','gigantic','tasty','met
	return randomElement(words);
}

function execCommand(client, command, options = {}) {
	let exePath = 'node ' + joplinAppPath;
	let cmd = exePath + ' --profile ' + client.profileDir + ' ' + command;
	logger.info(client.id + ': ' + command);

	if (options.killAfter) {
		logger.info('Kill after: ' + options.killAfter);
	}

	return new Promise((resolve, reject) => {
		let childProcess = exec(cmd, (error, stdout, stderr) => {
			if (error) {
				if (error.signal == 'SIGTERM') {
					resolve('Process was killed');
				} else {
					logger.error(stderr);
					reject(error);
				}
			} else {
				resolve(stdout);
			}
		});

		if (options.killAfter) {
			setTimeout(() => {
				logger.info('Sending kill signal...');
				childProcess.kill();
			}, options.killAfter);
		}
	});
}

async function execRandomCommand(client) {
	let possibleCommands = [
		['mkbook {word}', 40], // CREATE FOLDER
		['mknote {word}', 100], // CREATE NOTE
		[async () => { // DELETE RANDOM ITEM
			let items = await execCommand(client, 'dump');
			items = JSON.parse(items);
			let item = randomElement(items);
			if (!item) return;

			if (item.type_ == 1) {
				return execCommand(client, 'rm -f ' + item.title);
			} else if (item.type_ == 2) {
				return execCommand(client, 'rm -f ' + '../' + item.title);
			} else {
				throw new Error('Unknown type: ' + item.type_);
			}
		}, 30],
		[async () => { // SYNC
			let avgSyncDuration = averageSyncDuration();
			let options = {};
			if (!isNaN(avgSyncDuration)) {
				// if (Math.random() >= 0.5) {
				// 	options.killAfter = avgSyncDuration * Math.random();
				// }
			}
			return execCommand(client, 'sync --random-failures', options);
		}, 40],
		[async () => { // UPDATE RANDOM ITEM
			let items = await execCommand(client, 'dump');
			items = JSON.parse(items);
			let item = randomElement(items);
			if (!item) return;

			return execCommand(client, 'set ' + item.id + ' title "' + randomWord() + '"');
		}, 50],
	];

	let cmd = null;
	while (true) {
		cmd = randomElement(possibleCommands);
		let r = 1 + Math.floor(Math.random() * 100);
		if (r <= cmd[1]) break;
	}

	cmd = cmd[0];

	if (typeof cmd === 'function') {
		return cmd();
	} else {
		cmd = cmd.replace('{word}', randomWord());
		return execCommand(client, cmd);
	}
}

function averageSyncDuration() {
	return lodash.mean(syncDurations);
}

function randomNextCheckTime() {
	let output = time.unixMs() + 1000 + Math.random() * 1000 * 120;
	logger.info('Next sync check: ' + time.unixMsToIso(output) + ' (' + (Math.round((output - time.unixMs()) / 1000)) + ' sec.)');
	return output;
}

function findItem(items, itemId) {
	for (let i = 0; i  < items.length; i++) {
		if (items[i].id == itemId) return items[i];
	}
	return null;
}

function compareItems(item1, item2) {
	let output = [];
	for (let n in item1) {
		if (!item1.hasOwnProperty(n)) continue;
		if (n == 'sync_time') continue;
		let p1 = item1[n];
		let p2 = item2[n];
		if (p1 !== p2) output.push(n);
	}
	return output;
}

function findMissingItems_(items1, items2) {
	let output = [];

	for (let i = 0; i < items1.length; i++) {
		let item1 = items1[i];
		let found = false;
		for (let j = 0; j < items2.length; j++) {
			let item2 = items2[j];
			if (item1.id == item2.id) {
				found = true;
				break;
			}
		}

		if (!found) {
			output.push(item1);
		}
	}

	return output;
}

function findMissingItems(items1, items2) {
	return [
		findMissingItems_(items1, items2),
		findMissingItems_(items2, items1),
	];
}

async function compareClientItems(clientItems) {
	let itemCounts = [];
	for (let i = 0; i < clientItems.length; i++) {
		let items = clientItems[i];
		itemCounts.push(items.length);
	}
	logger.info('Item count: ' + itemCounts.join(', '));
	
	let missingItems = findMissingItems(clientItems[0], clientItems[1]);
	if (missingItems[0].length || missingItems[1].length) {
		logger.error('Items are different');
		logger.error(missingItems);
		process.exit(1);
	}

	let differences = [];
	let items = clientItems[0];
	for (let i = 0; i < items.length; i++) {
		let item1 = items[i];
		for (let clientId = 1; clientId < clientItems.length; clientId++) {
			let item2 = findItem(clientItems[clientId], item1.id);
			if (!item2) {
				logger.error('Item not found on client ' + clientId + ':');
				logger.error(item1);
				process.exit(1);
			}

			let diff = compareItems(item1, item2);
			if (diff.length) {
				differences.push({
					item1: item1,
					item2: item2,
				});
			}
		}
	}

	if (differences.length) {
		logger.error('Found differences between items:');
		logger.error(differences);
		process.exit(1);
	}
}

async function main(argv) {
	await fs.remove(syncDir);
	
	let clients = await createClients();
	let activeCommandCounts = [];
	let clientId = 0;

	for (let i = 0; i < clients.length; i++) {
		clients[i].activeCommandCount = 0;
	}

	function handleCommand(clientId) {
		if (clients[clientId].activeCommandCount >= 1) return;

		clients[clientId].activeCommandCount++;

		execRandomCommand(clients[clientId]).catch((error) => {
			logger.info('Client ' + clientId + ':');
			logger.error(error);
		}).then((r) => {
			if (r) {
				logger.info('Client ' + clientId + ':');
				logger.info(r);
			}
			clients[clientId].activeCommandCount--;
		});
	}

	let nextSyncCheckTime = randomNextCheckTime();
	let state = 'commands';

	setInterval(async () => {
		if (state == 'waitForSyncCheck') return;

		if (state == 'syncCheck') {
			state = 'waitForSyncCheck';
			let clientItems = [];
			// Up to 3 sync operations must be performed by each clients in order for them
			// to be perfectly in sync - in order for each items to send their changes
			// and get those from the other clients, and to also get changes that are
			// made as a result of a sync operation (eg. renaming a folder that conflicts
			// with another one).
			for (let loopCount = 0; loopCount < 3; loopCount++) {
				for (let i = 0; i < clients.length; i++) {
					let beforeTime = time.unixMs();
					await execCommand(clients[i], 'sync');
					syncDurations.push(time.unixMs() - beforeTime);
					if (syncDurations.length > 20) syncDurations.splice(0, 1);
					if (loopCount === 2) {
						let dump = await execCommand(clients[i], 'dump');
						clientItems[i] = JSON.parse(dump);
					}
				}
			}

			await compareClientItems(clientItems);

			nextSyncCheckTime = randomNextCheckTime();
			state = 'commands';
			return;
		}

		if (state == 'waitForClients') {
			for (let i = 0; i < clients.length; i++) {
				if (clients[i].activeCommandCount > 0) return;
			}

			state = 'syncCheck';
			return;
		}

		if (state == 'commands') {
			if (nextSyncCheckTime <= time.unixMs()) {
				state = 'waitForClients';
				return;
			}

			handleCommand(clientId);
			clientId++;
			if (clientId >= clients.length) clientId = 0;
		}
	}, 100);
}

main(process.argv).catch((error) => {
	logger.error(error);
});