self-hosted/joplin
1
0
Fork 0
mirror of https://github.com/laurent22/joplin.git synced 2025-03-26 21:12:59 +02:00
Code Issues Releases Activity

Desktop: Add synchronization tools to clear local sync state or data

Browse Source
This commit is contained in:
Laurent Cozic 2021-05-03 12:55:38 +02:00
parent da4c61653f
commit a6caa357c8
14 changed files with 229 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.eslintignore
View File

@ -106,6 +106,9 @@ packages/app-cli/tests/Synchronizer.sharing.js.map
packages/app-cli/tests/Synchronizer.tags.d.ts
packages/app-cli/tests/Synchronizer.tags.js
packages/app-cli/tests/Synchronizer.tags.js.map
packages/app-cli/tests/Synchronizer.tools.d.ts
packages/app-cli/tests/Synchronizer.tools.js
packages/app-cli/tests/Synchronizer.tools.js.map
packages/app-cli/tests/fsDriver.d.ts
packages/app-cli/tests/fsDriver.js
packages/app-cli/tests/fsDriver.js.map
@ -1348,6 +1351,9 @@ packages/lib/services/synchronizer/migrations/1.js.map
packages/lib/services/synchronizer/migrations/2.d.ts
packages/lib/services/synchronizer/migrations/2.js
packages/lib/services/synchronizer/migrations/2.js.map
packages/lib/services/synchronizer/tools.d.ts
packages/lib/services/synchronizer/tools.js
packages/lib/services/synchronizer/tools.js.map
packages/lib/services/synchronizer/utils/types.d.ts
packages/lib/services/synchronizer/utils/types.js
packages/lib/services/synchronizer/utils/types.js.map

8
.gitignore vendored
View File

@ -93,6 +93,9 @@ packages/app-cli/tests/Synchronizer.sharing.js.map
packages/app-cli/tests/Synchronizer.tags.d.ts
packages/app-cli/tests/Synchronizer.tags.js
packages/app-cli/tests/Synchronizer.tags.js.map
packages/app-cli/tests/Synchronizer.tools.d.ts
packages/app-cli/tests/Synchronizer.tools.js
packages/app-cli/tests/Synchronizer.tools.js.map
packages/app-cli/tests/fsDriver.d.ts
packages/app-cli/tests/fsDriver.js
packages/app-cli/tests/fsDriver.js.map
@ -1335,6 +1338,9 @@ packages/lib/services/synchronizer/migrations/1.js.map
packages/lib/services/synchronizer/migrations/2.d.ts
packages/lib/services/synchronizer/migrations/2.js
packages/lib/services/synchronizer/migrations/2.js.map
packages/lib/services/synchronizer/tools.d.ts
packages/lib/services/synchronizer/tools.js
packages/lib/services/synchronizer/tools.js.map
packages/lib/services/synchronizer/utils/types.d.ts
packages/lib/services/synchronizer/utils/types.js
packages/lib/services/synchronizer/utils/types.js.map
@ -1516,5 +1522,3 @@ packages/tools/tool-utils.d.ts
packages/tools/tool-utils.js
packages/tools/tool-utils.js.map
# AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD
packages/app-desktop/gui/getWindowTitle.js
packages/app-desktop/gui/getWindowTitle.test.js

53
packages/app-cli/tests/Synchronizer.tools.ts Normal file
View File

@ -0,0 +1,53 @@
import { allNotesFolders, remoteNotesAndFolders } from './test-utils-synchronizer';
import { afterAllCleanUp, synchronizerStart, setupDatabaseAndSynchronizer, switchClient, fileApi, db } from './test-utils';
import Folder from '@joplin/lib/models/Folder';
import Note from '@joplin/lib/models/Note';
import { clearLocalDataForRedownload, clearLocalSyncStateForReupload } from '@joplin/lib/services/synchronizer/tools';
describe('Synchronizer.tools', function() {
beforeEach(async (done) => {
await setupDatabaseAndSynchronizer(1);
await setupDatabaseAndSynchronizer(2);
await switchClient(1);
done();
});
afterAll(async () => {
await afterAllCleanUp();
});
it('should clear local sync data, and re-upload everything to sync target', (async () => {
await Folder.save({ title: 'test' });
await synchronizerStart();
await fileApi().clearRoot();
await clearLocalSyncStateForReupload(db());
// Now that the local sync state has been cleared, it should re-upload
// the items as if it was a new sync target. It should also not delete*
// any local data.
await synchronizerStart();
expect((await remoteNotesAndFolders()).length).toBe(1);
expect((await Folder.all()).length).toBe(1);
}));
it('should clear local data, and re-downlaod everything from sync target', (async () => {
const folder = await Folder.save({ title: 'test' });
await Note.save({ title: 'test note', parent_id: folder.id });
await synchronizerStart();
await clearLocalDataForRedownload(db());
expect((await allNotesFolders()).length).toBe(0);
await synchronizerStart();
expect((await allNotesFolders()).length).toBe(2);
expect((await remoteNotesAndFolders()).length).toBe(2);
}));
});

15
packages/app-cli/tests/test-utils.ts
View File

@ -1,7 +1,7 @@
/* eslint-disable require-atomic-updates */
import BaseApplication from '@joplin/lib/BaseApplication';
import BaseModel from '@joplin/lib/BaseModel';
import Logger, { TargetType, LoggerWrapper } from '@joplin/lib/Logger';
import Logger, { TargetType, LoggerWrapper, LogLevel } from '@joplin/lib/Logger';
import Setting from '@joplin/lib/models/Setting';
import BaseService from '@joplin/lib/services/BaseService';
import FsDriverNode from '@joplin/lib/fs-driver-node';
@ -69,7 +69,6 @@ const suiteName_ = uuid.createNano();
const databases_: any[] = [];
let synchronizers_: any[] = [];
const synchronizerContexts_: any = {};
const fileApis_: any = {};
const encryptionServices_: any[] = [];
const revisionServices_: any[] = [];
@ -168,7 +167,7 @@ dbLogger.setLevel(Logger.LEVEL_WARN);
const logger = new Logger();
logger.addTarget(TargetType.Console);
logger.setLevel(Logger.LEVEL_WARN); // Set to DEBUG to display sync process in console
logger.setLevel(LogLevel.Warn); // Set to DEBUG to display sync process in console
Logger.initializeGlobalLogger(logger);
@ -390,7 +389,6 @@ async function setupDatabaseAndSynchronizer(id: number, options: any = null) {
syncTarget.setFileApi(fileApi());
syncTarget.setLogger(logger);
synchronizers_[id] = await syncTarget.synchronizer();
synchronizerContexts_[id] = null;
}
encryptionServices_[id] = new EncryptionService();
@ -420,11 +418,16 @@ function synchronizer(id: number = null) {
// the client.
async function synchronizerStart(id: number = null, extraOptions: any = null) {
if (id === null) id = currentClient_;
const context = synchronizerContexts_[id];
const contextKey = `sync.${syncTargetId()}.context`;
const context = Setting.value(contextKey);
const options = Object.assign({}, extraOptions);
if (context) options.context = context;
const newContext = await synchronizer(id).start(options);
synchronizerContexts_[id] = newContext;
Setting.setValue(contextKey, JSON.stringify(newContext));
return newContext;
}

64
packages/app-desktop/gui/ConfigScreen/ConfigScreen.tsx
View File

@ -4,7 +4,7 @@ import ButtonBar from './ButtonBar';
import Button, { ButtonLevel } from '../Button/Button';
import { _ } from '@joplin/lib/locale';
import bridge from '../../services/bridge';
import Setting from '@joplin/lib/models/Setting';
import Setting, { SyncStartupOperation } from '@joplin/lib/models/Setting';
import control_PluginsStates from './controls/plugins/PluginsStates';
const { connect } = require('react-redux');
@ -51,6 +51,7 @@ class ConfigScreenComponent extends React.Component<any, any> {
this.renderLabel = this.renderLabel.bind(this);
this.renderDescription = this.renderDescription.bind(this);
this.renderHeader = this.renderHeader.bind(this);
this.handleSettingButton = this.handleSettingButton.bind(this);
}
async checkSyncConfig_() {
@ -82,6 +83,22 @@ class ConfigScreenComponent extends React.Component<any, any> {
}
}
private async handleSettingButton(key: string) {
if (key === 'sync.clearLocalSyncStateButton') {
if (!confirm('This cannot be undone. Do you want to continue?')) return;
Setting.setValue('sync.startupOperation', SyncStartupOperation.ClearLocalSyncState);
await Setting.saveAll();
bridge().restart();
} else if (key === 'sync.clearLocalDataButton') {
if (!confirm('This cannot be undone. Do you want to continue?')) return;
Setting.setValue('sync.startupOperation', SyncStartupOperation.ClearLocalData);
await Setting.saveAll();
bridge().restart();
} else {
throw new Error(`Unhandled key: ${key}`);
}
}
sectionByName(name: string) {
const sections = shared.settingsSections({ device: 'desktop', settings: this.state.settings });
for (const section of sections) {
@ -202,49 +219,6 @@ class ConfigScreenComponent extends React.Component<any, any> {
</div>
);
}
// if (syncTargetMd.name === 'nextcloud') {
// const syncTarget = settings['sync.5.syncTargets'][settings['sync.5.path']];
// let status = _('Unknown');
// let errorMessage = null;
// if (this.state.checkNextcloudAppResult === 'checking') {
// status = _('Checking...');
// } else if (syncTarget) {
// if (syncTarget.uuid) status = _('OK');
// if (syncTarget.error) {
// status = _('Error');
// errorMessage = syncTarget.error;
// }
// }
// const statusComp = !errorMessage || this.state.checkNextcloudAppResult === 'checking' || !this.state.showNextcloudAppLog ? null : (
// <div style={statusStyle}>
// <p style={theme.textStyle}>{_('The Joplin Nextcloud App is either not installed or misconfigured. Please see the full error message below:')}</p>
// <pre>{errorMessage}</pre>
// </div>
// );
// const showLogButton = !errorMessage || this.state.showNextcloudAppLog ? null : (
// <a style={theme.urlStyle} href="#" onClick={this.showLogButton_click}>[{_('Show Log')}]</a>
// );
// const appStatusStyle = Object.assign({}, theme.textStyle, { fontWeight: 'bold' });
// settingComps.push(
// <div key="nextcloud_app_check" style={this.rowStyle_}>
// <span style={theme.textStyle}>Beta: {_('Joplin Nextcloud App status:')} </span><span style={appStatusStyle}>{status}</span>
// &nbsp;&nbsp;
// {showLogButton}
// &nbsp;&nbsp;
// <Button level={ButtonLevel.Secondary} style={{ display: 'inline-block' }} title={_('Check Status')} disabled={this.state.checkNextcloudAppResult === 'checking'} onClick={this.checkNextcloudAppButton_click}/>
// &nbsp;&nbsp;
// <a style={theme.urlStyle} href="#" onClick={this.nextcloudAppHelpLink_click}>[{_('Help')}]</a>
// {statusComp}
// </div>
// );
// }
}
let advancedSettingsButton = null;
@ -637,7 +611,7 @@ class ConfigScreenComponent extends React.Component<any, any> {
<div style={labelStyle}>
<label>{md.label()}</label>
</div>
<Button level={ButtonLevel.Secondary} title={_('Edit')} onClick={md.onClick}/>
<Button level={ButtonLevel.Secondary} title={md.label()} onClick={md.onClick ? md.onClick : () => this.handleSettingButton(key)}/>
{descriptionComp}
</div>
);

25
packages/app-desktop/gui/StatusScreen/StatusScreen.tsx
View File

@ -5,10 +5,12 @@ import { _ } from '@joplin/lib/locale';
const { connect } = require('react-redux');
import Setting from '@joplin/lib/models/Setting';
const bridge = require('electron').remote.require('./bridge').default;
const { themeStyle } = require('@joplin/lib/theme');
import ReportService from '@joplin/lib/services/ReportService';
import Button, { ButtonLevel } from '../Button/Button';
import bridge from '../../services/bridge';
const fs = require('fs-extra');
import styled from 'styled-components';
interface Props {
themeId: string;
@ -16,6 +18,10 @@ interface Props {
dispatch: Function;
}
const StyledAdvancedToolItem = styled.div`
margin-bottom: 1em;
`;
async function exportDebugReportClick() {
const filename = `syncReport-${new Date().getTime()}.csv`;
@ -134,14 +140,25 @@ function StatusScreen(props: Props) {
return <div>{sectionsHtml}</div>;
}
function renderTools() {
return (
<div>
<h2 key="section_tools" style={theme.h2Style}>
{_('Advanced tools')}
</h2>
<StyledAdvancedToolItem>
<Button level={ButtonLevel.Primary} title={_('Export debug report')} onClick={() => exportDebugReportClick()}/>
</StyledAdvancedToolItem>
</div>
);
}
const body = renderBodyHtml(report);
return (
<div style={style}>
<div style={containerStyle}>
<a style={theme.textStyle} onClick={() => exportDebugReportClick()} href="#">
Export debug report
</a>
{renderTools()}
{body}
</div>
<ButtonBar

16
packages/lib/BaseApplication.ts
View File

@ -1,4 +1,4 @@
import Setting from './models/Setting';
import Setting, { SyncStartupOperation } from './models/Setting';
import Logger, { TargetType, LoggerWrapper } from './Logger';
import shim from './shim';
import BaseService from './services/BaseService';
@ -44,6 +44,7 @@ import ResourceService from './services/ResourceService';
import DecryptionWorker from './services/DecryptionWorker';
const { loadKeychainServiceAndSettings } = require('./services/SettingUtils');
import MigrationService from './services/MigrationService';
import { clearLocalDataForRedownload, clearLocalSyncStateForReupload } from './services/synchronizer/tools';
const { toSystemSlashes } = require('./path-utils');
const { setAutoFreeze } = require('immer');
@ -650,6 +651,18 @@ export default class BaseApplication {
return toSystemSlashes(output, 'linux');
}
private async handleSyncToolActions() {
if (Setting.value('sync.startupOperation') === SyncStartupOperation.ClearLocalSyncState) {
await clearLocalSyncStateForReupload(reg.db());
} else if (Setting.value('sync.startupOperation') === SyncStartupOperation.ClearLocalData) {
await clearLocalDataForRedownload(reg.db());
} else if (Setting.value('sync.startupOperation') === SyncStartupOperation.None) {
// Nothing
} else {
throw new Error(`Invalid sync.startupOperation value: ${Setting.value('sync.startupOperation')}`);
}
}
async start(argv: string[]): Promise<any> {
const startFlags = await this.handleStartFlags_(argv);
@ -740,6 +753,7 @@ export default class BaseApplication {
BaseModel.setDb(this.database_);
await loadKeychainServiceAndSettings(KeychainServiceDriver);
await this.handleSyncToolActions();
appLogger.info(`Client ID: ${Setting.value('clientId')}`);

2
packages/lib/Logger.ts
View File

@ -12,7 +12,7 @@ export enum TargetType {
Console = 'console',
}
enum LogLevel {
export enum LogLevel {
None = 0,
Error = 10,
Warn = 20,

4
packages/lib/SyncTargetRegistry.js
View File

@ -15,6 +15,10 @@ class SyncTargetRegistry {
};
}
static allIds() {
return Object.keys(this.reg_);
}
static nameToId(name) {
for (const n in this.reg_) {
if (!this.reg_.hasOwnProperty(n)) continue;

2
packages/lib/Synchronizer.ts
View File

@ -297,7 +297,7 @@ export default class Synchronizer {
// 1. UPLOAD: Send to the sync target the items that have changed since the last sync.
// 2. DELETE_REMOTE: Delete on the sync target, the items that have been deleted locally.
// 3. DELTA: Find on the sync target the items that have been modified or deleted and apply the changes locally.
async start(options: any = null) {
public async start(options: any = null) {
if (!options) options = {};
if (this.state() != 'idle') {

2
packages/lib/database.ts
View File

@ -68,7 +68,7 @@ export default class Database {
this.logger().info('Database was open successfully');
}
escapeField(field: string) {
public escapeField(field: string) {
if (field == '*') return '*';
const p = field.split('.');
if (p.length == 1) return `\`${field}\``;

2
packages/lib/models/BaseItem.ts
View File

@ -681,7 +681,7 @@ export default class BaseItem extends BaseModel {
return output;
}
static syncItemTypes() {
public static syncItemTypes(): ModelType[] {
return BaseItem.syncItemDefinitions_.map((def: any) => {
return def.type;
});

47
packages/lib/models/Setting.ts
View File

@ -82,6 +82,12 @@ export interface SettingSection {
source?: SettingSectionSource;
}
export enum SyncStartupOperation {
None = 0,
ClearLocalSyncState = 1,
ClearLocalData = 2,
}
interface SettingSections {
[key: string]: SettingSection;
}
@ -284,6 +290,12 @@ class Setting extends BaseModel {
public: false,
},
'sync.startupOperation': {
value: SyncStartupOperation.None,
type: SettingItemType.Int,
public: false,
},
'sync.2.path': {
value: '',
type: SettingItemType.String,
@ -925,6 +937,29 @@ class Setting extends BaseModel {
description: () => 'CSS file support is provided for your convenience, but they are advanced settings, and styles you define may break from one version to the next. If you want to use them, please know that it might require regular development work from you to keep them working. The Joplin team cannot make a commitment to keep the application HTML structure stable.',
},
'sync.clearLocalSyncStateButton': {
value: null,
type: SettingItemType.Button,
public: true,
appTypes: ['desktop'],
label: () => _('Re-upload local data to sync target'),
section: 'sync',
advanced: true,
description: () => 'If the data on the sync target is incorrect or empty, you can use this button to force a re-upload of your data to the sync target. Application will have to be restarted',
},
'sync.clearLocalDataButton': {
value: null,
type: SettingItemType.Button,
public: true,
appTypes: ['desktop'],
label: () => _('Delete local data and re-download from sync target'),
section: 'sync',
advanced: true,
description: () => 'If the data on the sync target is correct but your local data is not, you can use this button to clear your local data and force re-downloading everything from the sync target. As your local data will be deleted first, it is recommended to export your data as JEX first. Application will have to be restarted',
},
autoUpdateEnabled: { value: false, type: SettingItemType.Bool, storage: SettingStorage.File, section: 'application', public: platform !== 'linux', appTypes: ['desktop'], label: () => _('Automatically update the application') },
'autoUpdate.includePreReleases': { value: false, type: SettingItemType.Bool, section: 'application', storage: SettingStorage.File, public: true, appTypes: ['desktop'], label: () => _('Get pre-releases when checking for updates'), description: () => _('See the pre-release page for more details: %s', 'https://joplinapp.org/prereleases') },
'clipperServer.autoStart': { value: false, type: SettingItemType.Bool, storage: SettingStorage.File, public: false },
@ -1194,7 +1229,13 @@ class Setting extends BaseModel {
return output;
}
static keyExists(key: string) {
// Resets the key to its default value.
public static resetKey(key: string) {
const md = this.settingMetadata(key);
this.setValue(key, md.value);
}
public static keyExists(key: string) {
return key in this.metadata();
}
@ -1343,7 +1384,7 @@ class Setting extends BaseModel {
this.constants_[key] = value;
}
static setValue(key: string, value: any) {
public static setValue(key: string, value: any) {
if (!this.cache_) throw new Error('Settings have not been initialized!');
value = this.formatValue(key, value);
@ -1585,7 +1626,7 @@ class Setting extends BaseModel {
return output;
}
static async saveAll() {
public static async saveAll() {
if (Setting.autoSaveEnabled && !this.saveTimeoutId_) return Promise.resolve();
this.logger().debug('Saving settings...');

48
packages/lib/services/synchronizer/tools.ts Normal file
View File

@ -0,0 +1,48 @@
import { SqlQuery } from '../../database';
import JoplinDatabase from '../../JoplinDatabase';
import BaseItem from '../../models/BaseItem';
import Setting from '../../models/Setting';
const SyncTargetRegistry = require('../../SyncTargetRegistry');
async function clearSyncContext() {
const syncTargetIds = SyncTargetRegistry.allIds();
for (const syncTargetId of syncTargetIds) {
const key = `sync.${syncTargetId}.context`;
if (!Setting.keyExists(key)) break;
Setting.resetKey(key);
}
await Setting.saveAll();
}
export async function clearLocalSyncStateForReupload(db: JoplinDatabase) {
const queries: SqlQuery[] = [
{ sql: 'DELETE FROM deleted_items' },
{ sql: 'DELETE FROM sync_items' },
];
await db.transactionExecBatch(queries);
await clearSyncContext();
}
export async function clearLocalDataForRedownload(db: JoplinDatabase) {
const queries: SqlQuery[] = [
{ sql: 'DELETE FROM deleted_items' },
{ sql: 'DELETE FROM sync_items' },
{ sql: 'DELETE FROM item_changes' },
{ sql: 'DELETE FROM note_resources' },
];
const syncItemTypes = BaseItem.syncItemTypes();
for (const syncItemType of syncItemTypes) {
const SyncItemClass = BaseItem.getClassByItemType(syncItemType);
queries.push({ sql: `DELETE FROM ${db.escapeField(SyncItemClass.tableName())}` });
}
await db.transactionExecBatch(queries);
await clearSyncContext();
}