self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-01-11 18:24:43 +02:00
Code Issues Releases Activity

All: Refactored so that memory and file sync target use same delta logic

Browse Source
This commit is contained in:
Laurent Cozic 2018-01-21 19:45:32 +00:00
parent 1bfeed377a
commit 1a1e264fa4
4 changed files with 129 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
ReactNativeClient/lib/file-api-driver-local.js
View File

@ -1,5 +1,5 @@
const BaseItem = require('lib/models/BaseItem.js');
const { time } = require('lib/time-utils.js');
const { basicDelta } = require('lib/file-api');
// NOTE: when synchronising with the file system the time resolution is the second (unlike milliseconds for OneDrive for instance).
// What it means is that if, for example, client 1 changes a note at time t, and client 2 changes the same note within the same second,
@ -67,113 +67,15 @@ class FileApiDriverLocal {
}
}
contextFromOptions_(options) {
let output = {
timestamp: 0,
filesAtTimestamp: [],
statsCache: null,
async delta(path, options) {
const getStatFn = async (path) => {
const stats = await this.fsDriver().readDirStats(path);
return this.metadataFromStats_(stats);
};
if (!options || !options.context) return output;
const d = new Date(options.context.timestamp);
output.timestamp = isNaN(d.getTime()) ? 0 : options.context.timestamp;
output.filesAtTimestamp = Array.isArray(options.context.filesAtTimestamp) ? options.context.filesAtTimestamp.slice() : [];
output.statsCache = options.context && options.context.statsCache ? options.context.statsCache : null;
return output;
}
async delta(path, options) {
const outputLimit = 1000;
const itemIds = await options.allItemIdsHandler();
try {
const context = this.contextFromOptions_(options);
let newContext = {
timestamp: context.timestamp,
filesAtTimestamp: context.filesAtTimestamp.slice(),
statsCache: context.statsCache,
};
// Stats are cached until all items have been processed (until hasMore is false)
if (newContext.statsCache === null) {
const stats = await this.fsDriver().readDirStats(path);
newContext.statsCache = this.metadataFromStats_(stats);
newContext.statsCache.sort(function(a, b) {
return a.updated_time - b.updated_time;
});
}
let output = [];
// Find out which files have been changed since the last time. Note that we keep
// both the timestamp of the most recent change, *and* the items that exactly match
// this timestamp. This to handle cases where an item is modified while this delta
// function is running. For example:
// t0: Item 1 is changed
// t0: Sync items - run delta function
// t0: While delta() is running, modify Item 2
// Since item 2 was modified within the same millisecond, it would be skipped in the
// next sync if we relied exclusively on a timestamp.
for (let i = 0; i < newContext.statsCache.length; i++) {
const stat = newContext.statsCache[i];
if (stat.isDir) continue;
if (stat.updated_time < context.timestamp) continue;
// Special case for items that exactly match the timestamp
if (stat.updated_time === context.timestamp) {
if (context.filesAtTimestamp.indexOf(stat.path) >= 0) continue;
}
if (stat.updated_time > newContext.timestamp) {
newContext.timestamp = stat.updated_time;
newContext.filesAtTimestamp = [];
}
newContext.filesAtTimestamp.push(stat.path);
output.push(stat);
if (output.length >= outputLimit) break;
}
if (!Array.isArray(itemIds)) throw new Error('Delta API not supported - local IDs must be provided');
let deletedItems = [];
for (let i = 0; i < itemIds.length; i++) {
if (output.length + deletedItems.length >= outputLimit) break;
const itemId = itemIds[i];
let found = false;
for (let j = 0; j < newContext.statsCache.length; j++) {
const item = newContext.statsCache[j];
if (BaseItem.pathToId(item.path) == itemId) {
found = true;
break;
}
}
if (!found) {
deletedItems.push({
path: BaseItem.systemPath(itemId),
isDeleted: true,
});
}
}
output = output.concat(deletedItems);
const hasMore = output.length >= outputLimit;
if (!hasMore) newContext.statsCache = null;
return {
hasMore: hasMore,
context: newContext,
items: output,
};
const output = await basicDelta(path, getStatFn, options);
return output;
} catch(error) {
throw this.fsErrorToJsError_(error, path);
}

50
ReactNativeClient/lib/file-api-driver-memory.js
View File

@ -1,5 +1,6 @@
const { time } = require('lib/time-utils.js');
const fs = require('fs-extra');
const { basicDelta } = require('lib/file-api');
class FileApiDriverMemory {
@ -144,48 +145,17 @@ class FileApiDriverMemory {
}
async delta(path, options = null) {
let limit = 3;
let output = {
hasMore: false,
context: {},
items: [],
const getStatFn = async (path) => {
let output = this.items_.slice();
for (let i = 0; i < output.length; i++) {
const item = Object.assign({}, output[i]);
item.path = item.path.substr(path.length + 1);
output[i] = item;
}
return output;
};
let context = options ? options.context : null;
let fromTime = 0;
if (context) fromTime = context.fromTime;
let sortedItems = this.items_.slice().concat(this.deletedItems_);
sortedItems.sort((a, b) => {
if (a.updated_time < b.updated_time) return -1;
if (a.updated_time > b.updated_time) return +1;
return 0;
});
let hasMore = false;
let items = [];
let maxTime = 0;
for (let i = 0; i < sortedItems.length; i++) {
let item = sortedItems[i];
if (item.updated_time >= fromTime) {
item = Object.assign({}, item);
item.path = item.path.substr(path.length + 1);
items.push(item);
if (item.updated_time > maxTime) maxTime = item.updated_time;
}
if (items.length >= limit) {
hasMore = true;
break;
}
}
output.items = items;
output.hasMore = hasMore;
output.context = { fromTime: maxTime };
const output = await basicDelta(path, getStatFn, options);
return output;
}

112
ReactNativeClient/lib/file-api.js
View File

@ -1,6 +1,7 @@
const { isHidden } = require('lib/path-utils.js');
const { Logger } = require('lib/logger.js');
const { shim } = require('lib/shim');
const BaseItem = require('lib/models/BaseItem.js');
const JoplinError = require('lib/JoplinError');
class FileApi {
@ -120,4 +121,113 @@ class FileApi {
}
module.exports = { FileApi };
function basicDeltaContextFromOptions_(options) {
let output = {
timestamp: 0,
filesAtTimestamp: [],
statsCache: null,
};
if (!options || !options.context) return output;
const d = new Date(options.context.timestamp);
output.timestamp = isNaN(d.getTime()) ? 0 : options.context.timestamp;
output.filesAtTimestamp = Array.isArray(options.context.filesAtTimestamp) ? options.context.filesAtTimestamp.slice() : [];
output.statsCache = options.context && options.context.statsCache ? options.context.statsCache : null;
return output;
}
// This is the basic delta algorithm, which can be used in case the cloud service does not have
// a built-on delta API. OneDrive and Dropbox have one for example, but Nextcloud and obviously
// the file system do not.
async function basicDelta(path, getStatFn, options) {
const outputLimit = 1000;
const itemIds = await options.allItemIdsHandler();
if (!Array.isArray(itemIds)) throw new Error('Delta API not supported - local IDs must be provided');
const context = basicDeltaContextFromOptions_(options);
let newContext = {
timestamp: context.timestamp,
filesAtTimestamp: context.filesAtTimestamp.slice(),
statsCache: context.statsCache,
};
// Stats are cached until all items have been processed (until hasMore is false)
if (newContext.statsCache === null) {
newContext.statsCache = await getStatFn(path);
newContext.statsCache.sort(function(a, b) {
return a.updated_time - b.updated_time;
});
}
let output = [];
// Find out which files have been changed since the last time. Note that we keep
// both the timestamp of the most recent change, *and* the items that exactly match
// this timestamp. This to handle cases where an item is modified while this delta
// function is running. For example:
// t0: Item 1 is changed
// t0: Sync items - run delta function
// t0: While delta() is running, modify Item 2
// Since item 2 was modified within the same millisecond, it would be skipped in the
// next sync if we relied exclusively on a timestamp.
for (let i = 0; i < newContext.statsCache.length; i++) {
const stat = newContext.statsCache[i];
if (stat.isDir) continue;
if (stat.updated_time < context.timestamp) continue;
// Special case for items that exactly match the timestamp
if (stat.updated_time === context.timestamp) {
if (context.filesAtTimestamp.indexOf(stat.path) >= 0) continue;
}
if (stat.updated_time > newContext.timestamp) {
newContext.timestamp = stat.updated_time;
newContext.filesAtTimestamp = [];
}
newContext.filesAtTimestamp.push(stat.path);
output.push(stat);
if (output.length >= outputLimit) break;
}
let deletedItems = [];
for (let i = 0; i < itemIds.length; i++) {
if (output.length + deletedItems.length >= outputLimit) break;
const itemId = itemIds[i];
let found = false;
for (let j = 0; j < newContext.statsCache.length; j++) {
const item = newContext.statsCache[j];
if (BaseItem.pathToId(item.path) == itemId) {
found = true;
break;
}
}
if (!found) {
deletedItems.push({
path: BaseItem.systemPath(itemId),
isDeleted: true,
});
}
}
output = output.concat(deletedItems);
const hasMore = output.length >= outputLimit;
if (!hasMore) newContext.statsCache = null;
return {
hasMore: hasMore,
context: newContext,
items: output,
};
}
module.exports = { FileApi, basicDelta };

2
ReactNativeClient/lib/synchronizer.js
View File

@ -501,7 +501,7 @@ class Synchronizer {
if (action == 'createLocal' || action == 'updateLocal') {
if (content === null) {
this.logger().warn('Remote has been deleted between now and the list() call? In that case it will be handled during the next sync: ' + path);
this.logger().warn('Remote has been deleted between now and the delta() call? In that case it will be handled during the next sync: ' + path);
continue;
}
content = ItemClass.filter(content);