1
0
mirror of https://github.com/laurent22/joplin.git synced 2024-11-27 08:21:03 +02:00

Download new items during sync

This commit is contained in:
Laurent Cozic 2017-01-05 18:59:01 +01:00
parent 67d9104374
commit bb0af42fee
17 changed files with 256 additions and 99 deletions

View File

@ -15,25 +15,13 @@ Application::Application(int &argc, char **argv) :
QGuiApplication(argc, argv),
db_(jop::db()),
api_("http://joplin.local"),
synchronizer_(api_, db_),
synchronizer_(api_.baseUrl(), db_),
folderModel_(db_)
{
jop::db().initialize("D:/Web/www/joplin/QtClient/data/notes.sqlite");
// QVector<Change> changes = Change::all();
// foreach (Change change, changes) {
// qDebug() << change.value("item_id").toString() << change.value("type").toInt() << change.mergedFields();
// }
// qDebug() << "=====================================";
// changes = Change::mergedChanges(changes);
// foreach (Change change, changes) {
// qDebug() << change.value("item_id").toString() << change.value("type").toInt() << change.mergedFields();
// }
// This is linked to where the QSettings will be saved. In other words,
// if these values are changed, the settings will be reset and saved
// somewhere else.
@ -57,25 +45,25 @@ Application::Application(int &argc, char **argv) :
connect(rootObject, SIGNAL(currentNoteChanged()), this, SLOT(view_currentNoteChanged()));
connect(rootObject, SIGNAL(addFolderButtonClicked()), this, SLOT(view_addFolderButtonClicked()));
connect(&dispatcher(), SIGNAL(folderCreated(QString)), this, SLOT(dispatcher_folderCreated(QString)));
connect(&dispatcher(), SIGNAL(folderUpdated(QString)), this, SLOT(dispatcher_folderUpdated(QString)));
connect(&dispatcher(), SIGNAL(folderDeleted(QString)), this, SLOT(dispatcher_folderDeleted(QString)));
view_.show();
synchronizerTimer_.setInterval(1000 * 60);
synchronizerTimer_.start();
connect(&synchronizerTimer_, SIGNAL(timeout()), this, SLOT(synchronizerTimer_timeout()));
connect(&api_, SIGNAL(requestDone(const QJsonObject&, const QString&)), this, SLOT(api_requestDone(const QJsonObject&, const QString&)));
QString sessionId = settings.value("sessionId").toString();
//if (sessionId == "") {
QUrlQuery postData;
postData.addQueryItem("email", "laurent@cozic.net");
postData.addQueryItem("password", "12345678");
postData.addQueryItem("client_id", "B6E12222B6E12222");
api_.post("sessions", QUrlQuery(), postData, "getSession");
// } else {
// afterSessionInitialization();
// }
//emit jop::dispatcher().folderCreated("test");
//.folderCreated("tes");
QUrlQuery postData;
postData.addQueryItem("email", "laurent@cozic.net");
postData.addQueryItem("password", "12345678");
postData.addQueryItem("client_id", "B6E12222B6E12222");
api_.post("sessions", QUrlQuery(), postData, "getSession");
}
void Application::api_requestDone(const QJsonObject& response, const QString& tag) {
@ -90,6 +78,26 @@ void Application::api_requestDone(const QJsonObject& response, const QString& ta
}
}
void Application::dispatcher_folderCreated(const QString &folderId) {
qDebug() << "Folder created" << folderId;
synchronizerTimer_.start(1000 * 3);
}
void Application::dispatcher_folderUpdated(const QString &folderId) {
qDebug() << "Folder udpated" << folderId;
synchronizerTimer_.start(1000 * 3);
}
void Application::dispatcher_folderDeleted(const QString &folderId) {
qDebug() << "Folder deleted" << folderId;
synchronizerTimer_.start(1000 * 3);
}
void Application::synchronizerTimer_timeout() {
synchronizerTimer_.start(1000 * 60);
synchronizer_.start();
}
QString Application::selectedFolderId() const {
QObject* rootObject = (QObject*)view_.rootObject();
@ -114,6 +122,7 @@ void Application::afterSessionInitialization() {
QString sessionId = settings.value("sessionId").toString();
qDebug() << "Session:" << sessionId;
api_.setSessionId(sessionId);
synchronizer_.setSessionId(sessionId);
synchronizer_.start();
}

View File

@ -33,6 +33,7 @@ private:
QmlNote selectedQmlNote_;
WebApi api_;
Synchronizer synchronizer_;
QTimer synchronizerTimer_;
void afterSessionInitialization();
@ -44,6 +45,12 @@ public slots:
void view_addFolderButtonClicked();
void api_requestDone(const QJsonObject& response, const QString& tag);
void dispatcher_folderCreated(const QString& folderId);
void dispatcher_folderUpdated(const QString& folderId);
void dispatcher_folderDeleted(const QString& folderId);
void synchronizerTimer_timeout();
};
}

View File

@ -2,16 +2,22 @@
using namespace jop;
Dispatcher::Dispatcher() {
Dispatcher::Dispatcher() {}
void Dispatcher::emitFolderCreated(const QString &folderId) {
emit folderCreated(folderId);
}
Dispatcher instance_;
void Dispatcher::emitFolderUpdated(const QString &folderId) {
emit folderUpdated(folderId);
}
void Dispatcher::emitFolderDeleted(const QString &folderId) {
emit folderDeleted(folderId);
}
Dispatcher dispatcherInstance_;
Dispatcher& jop::dispatcher() {
return instance_;
return dispatcherInstance_;
}
//Dispatcher &Dispatcher::instance() {
// return instance_;
//}

View File

@ -1,6 +1,8 @@
#ifndef DISPATCHER_H
#define DISPATCHER_H
#include <stable.h>
namespace jop {
class Dispatcher : public QObject {
@ -10,15 +12,15 @@ class Dispatcher : public QObject {
public:
Dispatcher();
//static Dispatcher& instance();
void emitFolderCreated(const QString& folderId);
void emitFolderUpdated(const QString& folderId);
void emitFolderDeleted(const QString& folderId);
signals:
void folderCreated(const QString& id);
private:
//static Dispatcher& instance_;
void folderCreated(const QString& folderId);
void folderUpdated(const QString& folderId);
void folderDeleted(const QString& folderId);
};

View File

@ -1,5 +1,6 @@
#include "basemodel.h"
#include "dispatcher.h"
#include "models/change.h"
#include "database.h"
#include "uuid.h"
@ -9,10 +10,8 @@ using namespace jop;
QMap<int, QVector<BaseModel::Field>> BaseModel::tableFields_;
QHash<QString, QVariant> BaseModel::cache_;
BaseModel::BaseModel() {}
BaseModel::BaseModel(const QSqlQuery &query) {
loadSqlQuery(query);
BaseModel::BaseModel() {
isNew_ = -1;
}
QStringList BaseModel::changedFields() const {
@ -92,22 +91,22 @@ bool BaseModel::save() {
cacheDelete(QString("%1:count").arg(tableName));
}
bool output = false;
bool isSaved = false;
jop::db().transaction();
if (isNew) {
QSqlQuery q = jop::db().buildSqlQuery(Database::Insert, tableName, values);
jop::db().execQuery(q);
output = jop::db().errorCheck(q);
if (output) setValue("id", values["id"]);
isSaved = jop::db().errorCheck(q);
if (isSaved) setValue("id", values["id"]);
} else {
QSqlQuery q = jop::db().buildSqlQuery(Database::Update, tableName, values, QString("%1 = '%2'").arg(primaryKey()).arg(value("id").toString()));
jop::db().execQuery(q);
output = jop::db().errorCheck(q);
isSaved = jop::db().errorCheck(q);
}
if (output && trackChanges()) {
if (isSaved && trackChanges()) {
if (isNew) {
Change change;
change.setValue("item_id", id());
@ -128,7 +127,17 @@ bool BaseModel::save() {
jop::db().commit();
return output;
if (isSaved && table() == jop::FoldersTable) {
if (isNew) {
dispatcher().emitFolderCreated(id().toString());
} else {
dispatcher().emitFolderUpdated(id().toString());
}
}
isNew_ = -1;
return isSaved;
}
bool BaseModel::dispose() {
@ -138,11 +147,11 @@ bool BaseModel::dispose() {
q.bindValue(":id", id().toString());
jop::db().execQuery(q);
bool output = jop::db().errorCheck(q);
bool isDeleted = jop::db().errorCheck(q);
if (output) cacheDelete(QString("%1:count").arg(tableName));
if (isDeleted) cacheDelete(QString("%1:count").arg(tableName));
if (output && trackChanges()) {
if (isDeleted && trackChanges()) {
Change change;
change.setValue("item_id", id());
change.setValue("item_type", table());
@ -150,7 +159,11 @@ bool BaseModel::dispose() {
change.save();
}
return output;
if (isDeleted && table() == jop::FoldersTable) {
dispatcher().emitFolderDeleted(id().toString());
}
return isDeleted;
}
Table BaseModel::table() const {
@ -171,6 +184,8 @@ bool BaseModel::trackChanges() const {
}
bool BaseModel::isNew() const {
if (isNew_ == 0) return false;
if (isNew_ == 1) return true;
return !valueIsSet(primaryKey());
}
@ -228,6 +243,9 @@ bool BaseModel::isValidFieldName(Table table, const QString &name) {
return false;
}
// When loading a QSqlQuery, all the values are cleared and replaced by those
// from the QSqlQuery. All the fields are marked as NOT changed as it's assumed
// the object is already in the database (since loaded from there).
void BaseModel::loadSqlQuery(const QSqlQuery &query) {
values_.clear();
QSqlRecord record = query.record();
@ -241,17 +259,45 @@ void BaseModel::loadSqlQuery(const QSqlQuery &query) {
}
if (field.type == QMetaType::QString) {
values_.insert(field.name, Value(query.value(idx).toString()));
//values_.insert(field.name, Value(query.value(idx).toString()));
setValue(field.name, query.value(idx).toString());
} else if (field.type == QMetaType::Int) {
values_.insert(field.name, Value(query.value(idx).toInt()));
//values_.insert(field.name, Value(query.value(idx).toInt()));
setValue(field.name, query.value(idx).toInt());
} else {
qCritical() << "Unsupported value type" << field.name;
}
}
isNew_ = -1;
changedFields_.clear();
}
// When loading a QJsonObject, all the values are cleared and replaced by those
// from the QJsonObject. All the fields are marked as changed since it's
// assumed that the object comes from the web service.
void BaseModel::loadJsonObject(const QJsonObject &jsonObject) {
values_.clear();
changedFields_.clear();
QVector<BaseModel::Field> fields = BaseModel::tableFields(table());
foreach (BaseModel::Field field, fields) {
if (field.type == QMetaType::QString) {
//values_.insert(field.name, Value(jsonObject[field.name].toString()));
setValue(field.name, jsonObject[field.name].toString());
} else if (field.type == QMetaType::Int) {
//values_.insert(field.name, Value(jsonObject[field.name].toInt()));
setValue(field.name, jsonObject[field.name].toInt());
} else {
qCritical() << "Unsupported value type" << field.name;
}
}
isNew_ = 1;
}
QHash<QString, BaseModel::Value> BaseModel::values() const {
return values_;
}

View File

@ -40,12 +40,11 @@ public:
};
BaseModel();
BaseModel(const QSqlQuery& query);
QStringList changedFields() const;
static int count(jop::Table table);
bool load(const QString& id);
bool save();
bool dispose();
virtual bool save();
virtual bool dispose();
virtual Table table() const;
virtual QString primaryKey() const;
@ -60,6 +59,7 @@ public:
static bool isValidFieldName(Table table, const QString& name);
void loadSqlQuery(const QSqlQuery& query);
void loadJsonObject(const QJsonObject& jsonObject);
QHash<QString, Value> values() const;
Value value(const QString& name) const;
bool valueIsSet(const QString& name) const;
@ -84,6 +84,8 @@ protected:
static QMap<int, QVector<BaseModel::Field>> tableFields_;
static QHash<QString, QVariant> cache_;
int isNew_;
};
}

View File

@ -3,12 +3,6 @@
using namespace jop;
Change::Change() {}
Change::Change(const QSqlQuery &query) {
loadSqlQuery(query);
}
Table Change::table() const {
return jop::ChangesTable;
}
@ -25,7 +19,8 @@ QVector<Change> Change::all(int limit) {
QVector<Change> output;
while (q.next()) {
Change change(q);
Change change;
change.loadSqlQuery(q);
output.push_back(change);
}

View File

@ -13,8 +13,6 @@ public:
enum Type { Undefined, Create, Update, Delete };
Change();
Change(const QSqlQuery& query);
Table table() const;
static QVector<Change> all(int limit = 100);

View File

@ -1,5 +1,6 @@
#include "models/folder.h"
#include "dispatcher.h"
#include "database.h"
#include "uuid.h"

View File

@ -19,6 +19,9 @@ public:
bool primaryKeyIsUuid() const;
bool trackChanges() const;
// bool save();
// bool dispose();
private:
};

View File

@ -26,9 +26,7 @@
#include <QJsonObject>
#include <QJsonParseError>
#include <QBuffer>
#include <QJsonArray>
#include <QTimer>
#endif // __cplusplus

View File

@ -4,16 +4,29 @@
using namespace jop;
Synchronizer::Synchronizer(WebApi& api, Database &database) : api_(api), db_(database) {
Synchronizer::Synchronizer(const QString &apiUrl, Database &database) : api_(apiUrl), db_(database) {
qDebug() << api_.baseUrl();
state_ = Idle;
uploadsRemaining_ = 0;
downloadsRemaining_ = 0;
connect(&api_, SIGNAL(requestDone(QJsonObject,QString)), this, SLOT(api_requestDone(QJsonObject,QString)));
}
void Synchronizer::start() {
if (state_ != Idle) {
qWarning() << "Cannot start synchronizer because synchronization already in progress. State: " << state_;
return;
}
qDebug() << "Starting synchronizer...";
state_ = UploadingChanges;
QVector<Change> changes = Change::all();
changes = Change::mergedChanges(changes);
uploadsRemaining_ = changes.size();
foreach (Change change, changes) {
jop::Table itemType = (jop::Table)change.value("item_type").toInt();
QString itemId = change.value("item_id").toString();
@ -28,7 +41,7 @@ void Synchronizer::start() {
Folder folder;
folder.load(itemId);
QUrlQuery data = valuesToUrlQuery(folder.values());
api_.put("folders/" + folder.id().toString(), QUrlQuery(), data, "putFolder:" + folder.id().toString());
api_.put("folders/" + folder.id().toString(), QUrlQuery(), data, "upload:putFolder:" + folder.id().toString());
} else if (type == Change::Update) {
@ -39,15 +52,23 @@ void Synchronizer::start() {
foreach (QString field, mergedFields) {
data.addQueryItem(field, folder.value(field).toString());
}
api_.patch("folders/" + folder.id().toString(), QUrlQuery(), data, "patchFolder:" + folder.id().toString());
api_.patch("folders/" + folder.id().toString(), QUrlQuery(), data, "upload:patchFolder:" + folder.id().toString());
} else if (type == Change::Delete) {
api_.del("folders/" + itemId, QUrlQuery(), QUrlQuery(), "deleteFolder:" + itemId);
api_.del("folders/" + itemId, QUrlQuery(), QUrlQuery(), "upload:deleteFolder:" + itemId);
}
}
}
if (!uploadsRemaining_) {
downloadChanges();
}
}
void Synchronizer::setSessionId(const QString &v) {
api_.setSessionId(v);
}
QUrlQuery Synchronizer::valuesToUrlQuery(const QHash<QString, Change::Value>& values) const {
@ -58,31 +79,90 @@ QUrlQuery Synchronizer::valuesToUrlQuery(const QHash<QString, Change::Value>& va
return query;
}
void Synchronizer::api_requestDone(const QJsonObject& response, const QString& tag) {
qDebug() << "WebApi: done" << tag;
void Synchronizer::downloadChanges() {
state_ = DownloadingChanges;
//QUrlQuery data = valuesToUrlQuery(folder.values());
api_.get("synchronizer", QUrlQuery(), QUrlQuery(), "download:getSynchronizer");
}
void Synchronizer::api_requestDone(const QJsonObject& response, const QString& tag) {
QStringList parts = tag.split(':');
QString action = tag;
QString category = parts[0];
QString action = parts[1];
QString id = "";
if (parts.size() == 2) {
action = parts[0];
id = parts[1];
if (parts.size() == 3) {
id = parts[2];
}
qDebug() << "WebApi: done" << category << action << id;
// TODO: check for error
qDebug() << "Synced folder" << id;
if (category == "upload") {
uploadsRemaining_--;
if (action == "putFolder") {
Change::disposeByItemId(id);
}
qDebug() << "Synced folder" << id;
if (action == "patchFolder") {
Change::disposeByItemId(id);
}
if (action == "putFolder") {
Change::disposeByItemId(id);
}
if (action == "deleteFolder") {
Change::disposeByItemId(id);
if (action == "patchFolder") {
Change::disposeByItemId(id);
}
if (action == "deleteFolder") {
Change::disposeByItemId(id);
}
if (uploadsRemaining_ < 0) {
qWarning() << "Mismatch on operations done:" << uploadsRemaining_;
}
if (uploadsRemaining_ <= 0) {
uploadsRemaining_ = 0;
downloadChanges();
}
} else if (category == "download") {
if (action == "getSynchronizer") {
QJsonArray items = response["items"].toArray();
foreach (QJsonValue item, items) {
QJsonObject obj = item.toObject();
QString itemId = obj["item_id"].toString();
QString itemType = obj["item_type"].toString();
QString operationType = obj["type"].toString();
QString path = itemType + "s"; // That should remain true
if (operationType == "create") {
api_.get(path + "/" + itemId, QUrlQuery(), QUrlQuery(), "download:getFolder:" + itemId);
}
downloadsRemaining_++;
}
} else {
downloadsRemaining_--;
if (action == "getFolder") {
Folder folder;
folder.loadJsonObject(response);
folder.save();
// TODO: save last rev ID
}
if (downloadsRemaining_ < 0) {
qCritical() << "Mismatch on download operations done" << downloadsRemaining_;
}
if (downloadsRemaining_ <= 0) {
qDebug() << "All download operations complete";
downloadsRemaining_ = 0;
state_ = Idle;
}
}
} else {
qCritical() << "Invalid category" << category;
}
}

View File

@ -14,14 +14,21 @@ class Synchronizer : public QObject {
public:
Synchronizer(WebApi &api, Database& database);
enum SynchronizationState { Idle, UploadingChanges, DownloadingChanges };
Synchronizer(const QString& apiUrl, Database& database);
void start();
void setSessionId(const QString& v);
private:
QUrlQuery valuesToUrlQuery(const QHash<QString, BaseModel::Value> &values) const;
WebApi& api_;
WebApi api_;
Database& db_;
SynchronizationState state_;
int uploadsRemaining_;
int downloadsRemaining_;
void downloadChanges();
public slots:

View File

@ -99,7 +99,7 @@ void WebApi::processQueue() {
if (r.method != jop::GET && r.method != jop::DEL) {
cmd << "--data" << "'" + r.data.toString(QUrl::FullyEncoded) + "'";
}
cmd << url;
cmd << "'" + url + "'";
qDebug().noquote() << cmd.join(" ");
@ -116,7 +116,7 @@ void WebApi::request_finished(QNetworkReply *reply) {
qWarning().noquote() << QString(responseBodyBA);
} else {
response = doc.object();
if (!response["error"].isNull()) {
if (response.contains("error") && !response["error"].isNull()) {
qWarning().noquote() << "API error:" << QString(responseBodyBA);
}
}

View File

@ -21,6 +21,7 @@ class FoldersController extends ApiController {
if ($request->isMethod('POST')) {
$folder = new Folder();
$folder->fromPublicArray($request->request->all());
$folder->owner_id = $this->user()->id;
$folder->save();
return static::successResponse($folder);
}

View File

@ -6,7 +6,7 @@ use Sensio\Bundle\FrameworkExtraBundle\Configuration\Route;
use Symfony\Bundle\FrameworkBundle\Controller\Controller;
use Symfony\Component\HttpFoundation\Request;
use AppBundle\Controller\ApiController;
use AppBundle\Model\Action;
use AppBundle\Model\Change;
use AppBundle\Exception\UnauthorizedException;
/*
@ -103,11 +103,13 @@ class SynchronizerController extends ApiController {
* @Route("/synchronizer")
*/
public function allAction(Request $request) {
$id = (int)$request->query->get('last_id');
$lastChangeId = (int)$request->query->get('last_id');
if (!$this->user() || !$this->session()) throw new UnauthorizedException();
$actions = Action::actionsDoneAfterId($this->user()->id, $this->session()->client_id, $id);
$actions = Change::changesDoneAfterId($this->user()->id, $this->session()->client_id, $lastChangeId);
// $actions['user_id'] = Change::hex($this->user()->id);
// $actions['client_id'] = Change::hex($this->session()->client_id);
return static::successResponse($actions);
}

View File

@ -10,7 +10,7 @@ class Change extends BaseModel {
'type' => array('create', 'update', 'delete'),
);
static public function changesDoneAfterId($userId, $clientId, $changeId) {
static public function changesDoneAfterId($userId, $clientId, $fromChangeId) {
// Simplification:
//
// - If create, update, delete => return nothing
@ -19,7 +19,7 @@ class Change extends BaseModel {
// - If update, update, update => return last
$limit = 100;
$changes = self::where('id', '>', $changeId)
$changes = self::where('id', '>', $fromChangeId)
->where('user_id', '=', $userId)
->where('client_id', '!=', $clientId)
->orderBy('id')