self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2024-11-24 08:12:24 +02:00
Code Issues Releases Activity

All: Refactored filesystem sync driver to support mobile

Browse Source
This commit is contained in:
Laurent Cozic 2018-01-17 18:51:15 +00:00
parent f632580eed
commit 1a5c8d126d
7 changed files with 234 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CliClient/app/main.js
View File

@ -23,12 +23,14 @@ const { Logger } = require('lib/logger.js');
const { FsDriverNode } = require('lib/fs-driver-node.js');
const { shimInit } = require('lib/shim-init-node.js');
const { _ } = require('lib/locale.js');
const { FileApiDriverLocal } = require('lib/file-api-driver-local.js');
const EncryptionService = require('lib/services/EncryptionService');
const fsDriver = new FsDriverNode();
Logger.fsDriver_ = fsDriver;
Resource.fsDriver_ = fsDriver;
EncryptionService.fsDriver_ = fsDriver;
FileApiDriverLocal.fsDriver_ = fsDriver;
// That's not good, but it's to avoid circular dependency issues
// in the BaseItem class.

20
CliClient/tests/test-utils.js
View File

@ -38,6 +38,7 @@ const fsDriver = new FsDriverNode();
Logger.fsDriver_ = fsDriver;
Resource.fsDriver_ = fsDriver;
EncryptionService.fsDriver_ = fsDriver;
FileApiDriverLocal.fsDriver_ = fsDriver;
const logDir = __dirname + '/../tests/logs';
fs.mkdirpSync(logDir, 0o755);
@ -142,25 +143,6 @@ async function setupDatabase(id = null) {
BaseModel.db_ = databases_[id];
await Setting.load();
//return setupDatabase(id);
// return databases_[id].open({ name: filePath }).then(() => {
// BaseModel.db_ = databases_[id];
// return setupDatabase(id);
// });
// return fs.unlink(filePath).catch(() => {
// // Don't care if the file doesn't exist
// }).then(() => {
// databases_[id] = new JoplinDatabase(new DatabaseDriverNode());
// return databases_[id].open({ name: filePath }).then(() => {
// BaseModel.db_ = databases_[id];
// return setupDatabase(id);
// });
// });
}
function resourceDir(id = null) {

2
ElectronClient/app/main-html.js
View File

@ -17,11 +17,13 @@ const { FsDriverNode } = require('lib/fs-driver-node.js');
const { shimInit } = require('lib/shim-init-node.js');
const EncryptionService = require('lib/services/EncryptionService');
const { bridge } = require('electron').remote.require('./bridge');
const { FileApiDriverLocal } = require('lib/file-api-driver-local.js');
const fsDriver = new FsDriverNode();
Logger.fsDriver_ = fsDriver;
Resource.fsDriver_ = fsDriver;
EncryptionService.fsDriver_ = fsDriver;
FileApiDriverLocal.fsDriver_ = fsDriver;
// That's not good, but it's to avoid circular dependency issues
// in the BaseItem class.

242
ReactNativeClient/lib/file-api-driver-local.js
View File

@ -26,66 +26,55 @@ class FileApiDriverLocal {
return output;
}
stat(path) {
return new Promise((resolve, reject) => {
fs.stat(path, (error, s) => {
if (error) {
if (error.code == 'ENOENT') {
resolve(null);
} else {
reject(this.fsErrorToJsError_(error));
}
return;
}
resolve(this.metadataFromStats_(path, s));
});
});
fsDriver() {
if (!FileApiDriverLocal.fsDriver_) throw new Error('FileApiDriverLocal.fsDriver_ not set!');
return FileApiDriverLocal.fsDriver_;
}
statTimeToTimestampMs_(time) {
let m = moment(time, 'YYYY-MM-DDTHH:mm:ss.SSSZ');
if (!m.isValid()) {
throw new Error('Invalid date: ' + time);
async stat(path) {
try {
const s = await this.fsDriver().stat(path);
return this.metadataFromStat_(s);
} catch (error) {
if (error.code == 'ENOENT') return null;
throw this.fsErrorToJsError_(error);
}
return m.toDate().getTime();
}
metadataFromStats_(path, stats) {
metadataFromStat_(stat) {
return {
path: path,
created_time: this.statTimeToTimestampMs_(stats.birthtime),
updated_time: this.statTimeToTimestampMs_(stats.mtime),
created_time_orig: stats.birthtime,
updated_time_orig: stats.mtime,
isDir: stats.isDirectory(),
path: stat.path,
created_time: stat.birthtime.getTime(),
updated_time: stat.mtime.getTime(),
created_time_orig: stat.birthtime,
updated_time_orig: stat.mtime,
isDir: stat.isDirectory(),
};
}
setTimestamp(path, timestampMs) {
return new Promise((resolve, reject) => {
let t = Math.floor(timestampMs / 1000);
fs.utimes(path, t, t, (error) => {
if (error) {
reject(this.fsErrorToJsError_(error));
return;
}
resolve();
});
});
metadataFromStats_(stats) {
let output = [];
for (let i = 0; i < stats.length; i++) {
const mdStat = this.metadataFromStat_(stats[i]);
output.push(mdStat);
}
return output;
}
async setTimestamp(path, timestampMs) {
try {
await this.fsDriver().setTimestamp(path, new Date(timestampMs));
} catch (error) {
throw this.fsErrorToJsError_(error);
}
}
async delta(path, options) {
const itemIds = await options.allItemIdsHandler();
try {
let items = await fs.readdir(path);
let output = [];
for (let i = 0; i < items.length; i++) {
let stat = await this.stat(path + '/' + items[i]);
if (!stat) continue; // Has been deleted between the readdir() call and now
stat.path = items[i];
output.push(stat);
}
const stats = await this.fsDriver().readDirStats(path);
let output = this.metadataFromStats_(stats);
if (!Array.isArray(itemIds)) throw new Error('Delta API not supported - local IDs must be provided');
@ -123,14 +112,8 @@ class FileApiDriverLocal {
async list(path, options) {
try {
let items = await fs.readdir(path);
let output = [];
for (let i = 0; i < items.length; i++) {
let stat = await this.stat(path + '/' + items[i]);
if (!stat) continue; // Has been deleted between the readdir() call and now
stat.path = items[i];
output.push(stat);
}
const stats = await this.fsDriver().readDirStats(path);
const output = this.metadataFromStats_(stats);
return {
items: output,
@ -147,9 +130,11 @@ class FileApiDriverLocal {
try {
if (options.target === 'file') {
output = await fs.copy(path, options.path, { overwrite: true });
//output = await fs.copy(path, options.path, { overwrite: true });
output = await this.fsDriver().copy(path, options.path);
} else {
output = await fs.readFile(path, options.encoding);
//output = await fs.readFile(path, options.encoding);
output = await this.fsDriver().readFile(path, options.encoding);
}
} catch (error) {
if (error.code == 'ENOENT') return null;
@ -159,78 +144,107 @@ class FileApiDriverLocal {
return output;
}
mkdir(path) {
return new Promise((resolve, reject) => {
fs.exists(path, (exists) => {
if (exists) {
resolve();
return;
}
async mkdir(path) {
if (await this.fsDriver().exists(path)) return;
try {
await this.fsDriver().mkdir(path);
} catch (error) {
throw this.fsErrorToJsError_(error);
}
// return new Promise((resolve, reject) => {
// fs.exists(path, (exists) => {
// if (exists) {
// resolve();
// return;
// }
fs.mkdirp(path, (error) => {
if (error) {
reject(this.fsErrorToJsError_(error));
} else {
resolve();
}
});
});
});
// fs.mkdirp(path, (error) => {
// if (error) {
// reject(this.fsErrorToJsError_(error));
// } else {
// resolve();
// }
// });
// });
// });
}
async put(path, content, options = null) {
if (!options) options = {};
if (options.source === 'file') content = await fs.readFile(options.path);
try {
if (options.source === 'file') {
await this.fsDriver().copy(options.path, path);
return;
}
await this.fsDriver().writeFile(path, content, 'utf8');
} catch (error) {
throw this.fsErrorToJsError_(error);
}
return new Promise((resolve, reject) => {
fs.writeFile(path, content, function(error) {
if (error) {
reject(this.fsErrorToJsError_(error));
} else {
resolve();
}
});
});
// if (!options) options = {};
// if (options.source === 'file') content = await fs.readFile(options.path);
// return new Promise((resolve, reject) => {
// fs.writeFile(path, content, function(error) {
// if (error) {
// reject(this.fsErrorToJsError_(error));
// } else {
// resolve();
// }
// });
// });
}
delete(path) {
return new Promise((resolve, reject) => {
fs.unlink(path, function(error) {
if (error) {
if (error && error.code == 'ENOENT') {
// File doesn't exist - it's fine
resolve();
} else {
reject(this.fsErrorToJsError_(error));
}
} else {
resolve();
}
});
});
async delete(path) {
try {
await this.fsDriver().unlink(path);
} catch (error) {
throw this.fsErrorToJsError_(error);
}
// return new Promise((resolve, reject) => {
// fs.unlink(path, function(error) {
// if (error) {
// if (error && error.code == 'ENOENT') {
// // File doesn't exist - it's fine
// resolve();
// } else {
// reject(this.fsErrorToJsError_(error));
// }
// } else {
// resolve();
// }
// });
// });
}
async move(oldPath, newPath) {
let lastError = null;
for (let i = 0; i < 5; i++) {
try {
let output = await fs.move(oldPath, newPath, { overwrite: true });
return output;
} catch (error) {
lastError = error;
// Normally cannot happen with the `overwrite` flag but sometime it still does.
// In this case, retry.
if (error.code == 'EEXIST') {
await time.sleep(1);
continue;
}
throw this.fsErrorToJsError_(error);
}
}
return this.fsDriver().move(oldPath, newPath);
throw lastError;
// let lastError = null;
// for (let i = 0; i < 5; i++) {
// try {
// let output = await fs.move(oldPath, newPath, { overwrite: true });
// return output;
// } catch (error) {
// lastError = error;
// // Normally cannot happen with the `overwrite` flag but sometime it still does.
// // In this case, retry.
// if (error.code == 'EEXIST') {
// await time.sleep(1);
// continue;
// }
// throw this.fsErrorToJsError_(error);
// }
// }
// throw lastError;
}
format() {

63
ReactNativeClient/lib/fs-driver-node.js
View File

@ -1,4 +1,5 @@
const fs = require('fs-extra');
const { time } = require('lib/time-utils.js');
class FsDriverNode {
@ -15,14 +16,62 @@ class FsDriverNode {
return fs.writeFile(path, buffer);
}
move(source, dest) {
return fs.move(source, dest, { overwrite: true });
writeFile(path, string, encoding = 'base64') {
return fs.writeFile(path, string, { encoding: encoding });
}
async move(source, dest) {
let lastError = null;
for (let i = 0; i < 5; i++) {
try {
const output = await fs.move(source, dest, { overwrite: true });
return output;
} catch (error) {
lastError = error;
// Normally cannot happen with the `overwrite` flag but sometime it still does.
// In this case, retry.
if (error.code == 'EEXIST') {
await time.sleep(1);
continue;
}
throw this.fsErrorToJsError_(error);
}
}
throw lastError;
}
exists(path) {
return fs.pathExists(path);
}
async mkdir(path) {
return fs.mkdirp(path);
}
async stat(path) {
const s = await fs.stat(path);
s.path = path;
return s;
}
async setTimestamp(path, timestampDate) {
return fs.utimes(path, timestampDate, timestampDate);
}
async readDirStats(path) {
let items = await fs.readdir(path);
let output = [];
for (let i = 0; i < items.length; i++) {
let stat = await this.stat(path + '/' + items[i]);
if (!stat) continue; // Has been deleted between the readdir() call and now
stat.path = stat.path.substr(path.length + 1);
output.push(stat);
}
return output;
}
open(path, mode) {
return fs.open(path, mode);
}
@ -31,8 +80,14 @@ class FsDriverNode {
return fs.close(handle);
}
readFile(path) {
return fs.readFile(path);
readFile(path, encoding = 'utf8') {
if (encoding === 'Buffer') return fs.readFile(path); // Returns the raw buffer
return fs.readFile(path, encoding);
}
// Always overwrite destination
async copy(source, dest) {
return fs.copy(source, dest, { overwrite: true });
}
async unlink(path) {

44
ReactNativeClient/lib/fs-driver-rn.js
View File

@ -10,6 +10,10 @@ class FsDriverRN {
return RNFS.appendFile(path, string, encoding);
}
writeFile(path, string, encoding = 'base64') {
return RNFS.writeFile(path, string, encoding);
}
writeBinaryFile(path, content) {
throw new Error('Not implemented');
}
@ -22,11 +26,30 @@ class FsDriverRN {
return RNFS.exists(path);
}
async mkdir(path) {
return fs.mkdir(path);
}
async stat(path) {
const r = await RNFS.stat(path);
// Returns a format compatible with Node.js format
return {
birthtime: r.ctime, // Confusingly, "ctime" normally means "change time" but here it's used as "creation time"
mtime: r.mtime,
isDirectory: () => return r.isDirectory(),
path: path,
};
}
async setTimestamp(path, timestampDate) {
return RNFS.touch(path, timestampDate);
}
async open(path, mode) {
// Note: RNFS.read() doesn't provide any way to know if the end of file has been reached.
// So instead we stat the file here and use stat.size to manually check for end of file.
// Bug: https://github.com/itinance/react-native-fs/issues/342
const stat = await RNFS.stat(path);
const stat = await this.stat(path);
return {
path: path,
offset: 0,
@ -39,8 +62,23 @@ class FsDriverRN {
return null;
}
readFile(path) {
throw new Error('Not implemented');
readFile(path, encoding = 'utf8') {
if (encoding === 'Buffer') throw new Error('Raw buffer output not supported for FsDriverRN.readFile');
return RNFS.readFile(path, encoding);
}
// Always overwrite destination
async copy(source, dest) {
let retry = false;
try {
await RNFS.copyFile(source, dest);
} catch (error) {
// On iOS it will throw an error if the file already exist
retry = true;
await this.unlink(dest);
}
if (retry) await RNFS.copyFile(source, dest);
}
async unlink(path) {

2
ReactNativeClient/lib/models/Resource.js
View File

@ -120,7 +120,7 @@ class Resource extends BaseItem {
}
static async content(resource) {
return this.fsDriver().readFile(this.fullPath(resource));
return this.fsDriver().readFile(this.fullPath(resource), 'Buffer');
}
static setContent(resource, content) {