self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-10-31 00:07:48 +02:00
Code Issues Releases Activity

sync resources too

Browse Source
This commit is contained in:
Laurent Cozic
2017-07-02 13:02:07 +01:00
parent d56fb48299
commit 1aeedb80f7
8 changed files with 99 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CliClient/app/main.js
View File

@@ -11,6 +11,7 @@ import { Database } from 'lib/database.js';
import { DatabaseDriverNode } from 'lib/database-driver-node.js';
import { BaseModel } from 'lib/base-model.js';
import { Folder } from 'lib/models/folder.js';
import { Resource } from 'lib/models/resource.js';
import { BaseItem } from 'lib/models/base-item.js';
import { Note } from 'lib/models/note.js';
import { Setting } from 'lib/models/setting.js';

11
lib/database.js
View File

@@ -71,11 +71,12 @@ CREATE TABLE note_tags (
CREATE TABLE resources (
id TEXT PRIMARY KEY,
title TEXT,
mime TEXT,
filename TEXT,
created_time INT,
updated_time INT
title TEXT NOT NULL DEFAULT "",
mime TEXT NOT NULL,
filename TEXT NOT NULL,
created_time INT NOT NULL,
updated_time INT NOT NULL,
sync_time INT NOT NULL DEFAULT 0
);
CREATE TABLE note_resources (

31
lib/file-api-driver-local.js
View File

@@ -70,22 +70,21 @@ class FileApiDriverLocal {
};
}
get(path) {
return new Promise((resolve, reject) => {
fs.readFile(path, 'utf8', (error, content) => {
if (error) {
if (error.code == 'ENOENT') {
// Return null in this case so that it's possible to get a file
// without checking if it exists first.
resolve(null);
} else {
reject(error);
}
return;
}
return resolve(content);
});
});
async get(path, options) {
let output = null;
try {
if (options.encoding == 'binary') {
output = fs.readFile(path);
} else {
output = fs.readFile(path, options.encoding);
}
} catch (error) {
if (error.code == 'ENOENT') return null;
throw error;
}
return output;
}
mkdir(path) {

5
lib/file-api.js
View File

@@ -62,9 +62,10 @@ class FileApi {
});
}
get(path) {
get(path, options = {}) {
if (!options.encoding) options.encoding = 'utf8';
this.logger().debug('get ' + this.fullPath_(path));
return this.driver_.get(this.fullPath_(path));
return this.driver_.get(this.fullPath_(path), options);
}
put(path, content) {

42
lib/models/base-item.js
View File

@@ -1,7 +1,4 @@
import { BaseModel } from 'lib/base-model.js';
import { Note } from 'lib/models/note.js';
import { Folder } from 'lib/models/folder.js';
import { Setting } from 'lib/models/setting.js';
import { Database } from 'lib/database.js';
import { time } from 'lib/time-utils.js';
import moment from 'moment';
@@ -12,6 +9,14 @@ class BaseItem extends BaseModel {
return true;
}
// Need to dynamically load the classes like this to avoid circular dependencies
static getClass(name) {
if (!this.classes_) this.classes_ = {};
if (this.classes_[name]) return this.classes_[name];
this.classes_[name] = require('lib/models/' + name.toLowerCase() + '.js')[name];
return this.classes_[name];
}
static systemPath(itemOrId) {
if (typeof itemOrId === 'string') return itemOrId + '.md';
return itemOrId.id + '.md';
@@ -22,18 +27,19 @@ class BaseItem extends BaseModel {
if (typeof item === 'object') {
if (!('type_' in item)) throw new Error('Item does not have a type_ property');
return item.type_ == BaseModel.MODEL_TYPE_NOTE ? Note : Folder;
return this.itemClass(item.type_);
} else {
if (Number(item) === BaseModel.MODEL_TYPE_NOTE) return Note;
if (Number(item) === BaseModel.MODEL_TYPE_FOLDER) return Folder;
if (Number(item) === BaseModel.MODEL_TYPE_NOTE) return this.getClass('Note');
if (Number(item) === BaseModel.MODEL_TYPE_FOLDER) return this.getClass('Folder');
if (Number(item) === BaseModel.MODEL_TYPE_RESOURCE) return this.getClass('Resource');
throw new Error('Unknown type: ' + item);
}
}
// Returns the IDs of the items that have been synced at least once
static async syncedItems() {
let folders = await Folder.modelSelectAll('SELECT id FROM folders WHERE sync_time > 0');
let notes = await Note.modelSelectAll('SELECT id FROM notes WHERE is_conflict = 0 AND sync_time > 0');
let folders = await this.getClass('Folder').modelSelectAll('SELECT id FROM folders WHERE sync_time > 0');
let notes = await this.getClass('Note').modelSelectAll('SELECT id FROM notes WHERE is_conflict = 0 AND sync_time > 0');
return folders.concat(notes);
}
@@ -47,9 +53,9 @@ class BaseItem extends BaseModel {
}
static loadItemById(id) {
return Note.load(id).then((item) => {
return this.getClass('Note').load(id).then((item) => {
if (item) return item;
return Folder.load(id);
return this.getClass('Folder').load(id);
});
}
@@ -156,13 +162,15 @@ class BaseItem extends BaseModel {
return output;
}
static itemsThatNeedSync(limit = 100) {
return Folder.modelSelectAll('SELECT * FROM folders WHERE sync_time < updated_time LIMIT ' + limit).then((items) => {
if (items.length) return { hasMore: true, items: items };
return Note.modelSelectAll('SELECT * FROM notes WHERE sync_time < updated_time AND is_conflict = 0 LIMIT ' + limit).then((items) => {
return { hasMore: items.length >= limit, items: items };
});
});
static async itemsThatNeedSync(limit = 100) {
let items = await this.getClass('Folder').modelSelectAll('SELECT * FROM folders WHERE sync_time < updated_time LIMIT ' + limit);
if (items.length) return { hasMore: true, items: items };
items = await this.getClass('Resource').modelSelectAll('SELECT * FROM resources WHERE sync_time < updated_time LIMIT ' + limit);
if (items.length) return { hasMore: true, items: items };
items = await this.getClass('Note').modelSelectAll('SELECT * FROM notes WHERE sync_time < updated_time AND is_conflict = 0 LIMIT ' + limit);
return { hasMore: items.length >= limit, items: items };
}
}

2
lib/models/note.js
View File

@@ -16,7 +16,7 @@ class Note extends BaseItem {
static serialize(note, type = null, shownKeys = null) {
let fieldNames = this.fieldNames();
fieldNames.push('type_');
lodash.pull(fieldNames, 'is_conflict', 'sync_time');
lodash.pull(fieldNames, 'is_conflict', 'sync_time', 'body'); // Exclude 'body' since it's going to be added separately at the top of the note
return super.serialize(note, 'note', fieldNames);
}

24
lib/models/resource.js
View File

@@ -1,9 +1,11 @@
import { BaseModel } from 'lib/base-model.js';
import { BaseItem } from 'lib/models/base-item.js';
import { Setting } from 'lib/models/setting.js';
import { mime } from 'lib/mime-utils.js';
import { filename } from 'lib/path-utils.js';
import lodash from 'lodash';
class Resource extends BaseModel {
class Resource extends BaseItem {
static tableName() {
return 'resources';
@@ -13,6 +15,13 @@ class Resource extends BaseModel {
return BaseModel.MODEL_TYPE_RESOURCE;
}
static serialize(item, type = null, shownKeys = null) {
let fieldNames = this.fieldNames();
fieldNames.push('type_');
lodash.pull(fieldNames, 'sync_time');
return super.serialize(item, 'resource', fieldNames);
}
static fullPath(resource) {
let extension = mime.toFileExtension(resource.mime);
extension = extension ? '.' + extension : '';
@@ -23,6 +32,19 @@ class Resource extends BaseModel {
return filename(path);
}
static content(resource) {
// TODO: node-only, and should probably be done with streams
const fs = require('fs-extra');
return fs.readFile(this.fullPath(resource));
}
static setContent(resource, content) {
// TODO: node-only, and should probably be done with streams
const fs = require('fs-extra');
let buffer = new Buffer(content);
return fs.writeFile(this.fullPath(resource), buffer);
}
}
export { Resource };

37
lib/synchronizer.js
View File

@@ -3,6 +3,7 @@ require('babel-plugin-transform-runtime');
import { BaseItem } from 'lib/models/base-item.js';
import { Folder } from 'lib/models/folder.js';
import { Note } from 'lib/models/note.js';
import { Resource } from 'lib/models/resource.js';
import { BaseModel } from 'lib/base-model.js';
import { sprintf } from 'sprintf-js';
import { time } from 'lib/time-utils.js';
@@ -16,6 +17,7 @@ class Synchronizer {
this.db_ = db;
this.api_ = api;
this.syncDirName_ = '.sync';
this.resourceDirName_ = '.resource';
this.logger_ = new Logger();
}
@@ -68,14 +70,10 @@ class Synchronizer {
}
let folderCount = await Folder.count();
let noteCount = await Note.count();
let resourceCount = await Resource.count();
this.logger().info('Total folders: ' + folderCount);
this.logger().info('Total notes: ' + noteCount);
}
async createWorkDir() {
if (this.syncWorkDir_) return this.syncWorkDir_;
let dir = await this.api().mkdir(this.syncDirName_);
return this.syncDirName_;
this.logger().info('Total resources: ' + resourceCount);
}
randomFailure(options, name) {
@@ -122,12 +120,13 @@ class Synchronizer {
createRemote: 0,
updateRemote: 0,
deleteRemote: 0,
folderConflict: 0,
itemConflict: 0,
noteConflict: 0,
};
try {
await this.createWorkDir();
await this.api().mkdir(this.syncDirName_);
await this.api().mkdir(this.resourceDirName_);
let donePaths = [];
while (true) {
@@ -156,8 +155,8 @@ class Synchronizer {
action = 'createRemote';
reason = 'remote does not exist, and local is new and has never been synced';
} else {
// Note or folder was modified after having been deleted remotely
action = local.type_ == BaseModel.MODEL_TYPE_NOTE ? 'noteConflict' : 'folderConflict';
// Note or item was modified after having been deleted remotely
action = local.type_ == BaseModel.MODEL_TYPE_NOTE ? 'noteConflict' : 'itemConflict';
reason = 'remote has been deleted, but local has changes';
}
} else {
@@ -165,7 +164,7 @@ class Synchronizer {
// Since, in this loop, we are only dealing with notes that require sync, if the
// remote has been modified after the sync time, it means both notes have been
// modified and so there's a conflict.
action = local.type_ == BaseModel.MODEL_TYPE_NOTE ? 'noteConflict' : 'folderConflict';
action = local.type_ == BaseModel.MODEL_TYPE_NOTE ? 'noteConflict' : 'itemConflict';
reason = 'both remote and local have changes';
} else {
action = 'updateRemote';
@@ -175,6 +174,12 @@ class Synchronizer {
this.logSyncOperation(action, local, remote, reason);
if (local.type_ == BaseModel.MODEL_TYPE_RESOURCE && (action == 'createRemote' || (action == 'itemConflict' && remote))) {
let remoteContentPath = this.resourceDirName_ + '/' + local.id;
let resourceContent = await Resource.content(local);
await this.api().put(remoteContentPath, resourceContent);
}
if (action == 'createRemote' || action == 'updateRemote') {
// Make the operation atomic by doing the work on a copy of the file
@@ -189,7 +194,7 @@ class Synchronizer {
await ItemClass.save({ id: local.id, sync_time: time.unixMs(), type_: local.type_ }, { autoTimestamp: false });
} else if (action == 'folderConflict') {
} else if (action == 'itemConflict') {
if (remote) {
let remoteContent = await this.api().get(path);
@@ -303,6 +308,14 @@ class Synchronizer {
newContent.sync_time = time.unixMs();
let options = { autoTimestamp: false };
if (action == 'createLocal') options.isNew = true;
if (newContent.type_ == BaseModel.MODEL_TYPE_RESOURCE && action == 'createLocal') {
let localResourceContentPath = Resource.fullPath(newContent);
let remoteResourceContentPath = this.resourceDirName_ + '/' + newContent.id;
let remoteResourceContent = await this.api().get(remoteResourceContentPath, { encoding: 'binary' });
await Resource.setContent(newContent, remoteResourceContent);
}
try {
await ItemClass.save(newContent, options);
} catch (error) {