2018-03-09 22:59:12 +02:00
const { uuid } = require ( 'lib/uuid.js' ) ;
const { promiseChain } = require ( 'lib/promise-utils.js' ) ;
const { time } = require ( 'lib/time-utils.js' ) ;
const { Database } = require ( 'lib/database.js' ) ;
2018-09-29 14:29:07 +02:00
const { sprintf } = require ( 'sprintf-js' ) ;
2018-10-08 20:11:53 +02:00
const Resource = require ( 'lib/models/Resource' ) ;
2017-07-06 21:48:17 +02:00
const structureSql = `
CREATE TABLE folders (
id TEXT PRIMARY KEY ,
title TEXT NOT NULL DEFAULT "" ,
created _time INT NOT NULL ,
2017-07-16 14:53:59 +02:00
updated _time INT NOT NULL
2017-07-06 21:48:17 +02:00
) ;
CREATE INDEX folders _title ON folders ( title ) ;
CREATE INDEX folders _updated _time ON folders ( updated _time ) ;
CREATE TABLE notes (
id TEXT PRIMARY KEY ,
parent _id TEXT NOT NULL DEFAULT "" ,
title TEXT NOT NULL DEFAULT "" ,
body TEXT NOT NULL DEFAULT "" ,
created _time INT NOT NULL ,
updated _time INT NOT NULL ,
is _conflict INT NOT NULL DEFAULT 0 ,
latitude NUMERIC NOT NULL DEFAULT 0 ,
longitude NUMERIC NOT NULL DEFAULT 0 ,
altitude NUMERIC NOT NULL DEFAULT 0 ,
author TEXT NOT NULL DEFAULT "" ,
source _url TEXT NOT NULL DEFAULT "" ,
is _todo INT NOT NULL DEFAULT 0 ,
todo _due INT NOT NULL DEFAULT 0 ,
todo _completed INT NOT NULL DEFAULT 0 ,
source TEXT NOT NULL DEFAULT "" ,
source _application TEXT NOT NULL DEFAULT "" ,
application _data TEXT NOT NULL DEFAULT "" ,
\ ` order \` INT NOT NULL DEFAULT 0
) ;
CREATE INDEX notes _title ON notes ( title ) ;
CREATE INDEX notes _updated _time ON notes ( updated _time ) ;
CREATE INDEX notes _is _conflict ON notes ( is _conflict ) ;
CREATE INDEX notes _is _todo ON notes ( is _todo ) ;
CREATE INDEX notes _order ON notes ( \ ` order \` );
CREATE TABLE tags (
id TEXT PRIMARY KEY ,
title TEXT NOT NULL DEFAULT "" ,
created _time INT NOT NULL ,
2017-07-16 14:53:59 +02:00
updated _time INT NOT NULL
2017-07-06 21:48:17 +02:00
) ;
CREATE INDEX tags _title ON tags ( title ) ;
CREATE INDEX tags _updated _time ON tags ( updated _time ) ;
CREATE TABLE note _tags (
id TEXT PRIMARY KEY ,
note _id TEXT NOT NULL ,
tag _id TEXT NOT NULL ,
created _time INT NOT NULL ,
2017-07-16 14:53:59 +02:00
updated _time INT NOT NULL
2017-07-06 21:48:17 +02:00
) ;
CREATE INDEX note _tags _note _id ON note _tags ( note _id ) ;
CREATE INDEX note _tags _tag _id ON note _tags ( tag _id ) ;
CREATE INDEX note _tags _updated _time ON note _tags ( updated _time ) ;
CREATE TABLE resources (
id TEXT PRIMARY KEY ,
title TEXT NOT NULL DEFAULT "" ,
mime TEXT NOT NULL ,
filename TEXT NOT NULL DEFAULT "" ,
created _time INT NOT NULL ,
2017-07-16 14:53:59 +02:00
updated _time INT NOT NULL
2017-07-06 21:48:17 +02:00
) ;
CREATE INDEX resources _title ON resources ( title ) ;
CREATE INDEX resources _updated _time ON resources ( updated _time ) ;
CREATE TABLE settings (
\ ` key \` TEXT PRIMARY KEY,
\ ` value \` TEXT,
\ ` type \` INT NOT NULL
) ;
CREATE TABLE table _fields (
id INTEGER PRIMARY KEY ,
table _name TEXT NOT NULL ,
field _name TEXT NOT NULL ,
field _type INT NOT NULL ,
field _default TEXT
) ;
2017-07-16 14:53:59 +02:00
CREATE TABLE sync _items (
id INTEGER PRIMARY KEY ,
sync _target INT NOT NULL ,
sync _time INT NOT NULL DEFAULT 0 ,
item _type INT NOT NULL ,
item _id TEXT NOT NULL
) ;
CREATE INDEX sync _items _sync _time ON sync _items ( sync _time ) ;
CREATE INDEX sync _items _sync _target ON sync _items ( sync _target ) ;
CREATE INDEX sync _items _item _type ON sync _items ( item _type ) ;
CREATE INDEX sync _items _item _id ON sync _items ( item _id ) ;
2017-07-19 21:15:55 +02:00
CREATE TABLE deleted _items (
id INTEGER PRIMARY KEY ,
item _type INT NOT NULL ,
item _id TEXT NOT NULL ,
deleted _time INT NOT NULL
) ;
CREATE TABLE version (
version INT NOT NULL
) ;
2017-07-06 21:48:17 +02:00
INSERT INTO version ( version ) VALUES ( 1 ) ;
` ;
class JoplinDatabase extends Database {
2018-03-09 22:59:12 +02:00
2017-07-06 21:48:17 +02:00
constructor ( driver ) {
super ( driver ) ;
this . initialized _ = false ;
this . tableFields _ = null ;
}
initialized ( ) {
return this . initialized _ ;
}
async open ( options ) {
await super . open ( options ) ;
return this . initialize ( ) ;
}
tableFieldNames ( tableName ) {
let tf = this . tableFields ( tableName ) ;
let output = [ ] ;
for ( let i = 0 ; i < tf . length ; i ++ ) {
output . push ( tf [ i ] . name ) ;
}
return output ;
}
2018-09-28 22:03:28 +02:00
tableFields ( tableName , options = null ) {
if ( options === null ) options = { } ;
2018-03-09 22:59:12 +02:00
if ( ! this . tableFields _ ) throw new Error ( 'Fields have not been loaded yet' ) ;
if ( ! this . tableFields _ [ tableName ] ) throw new Error ( 'Unknown table: ' + tableName ) ;
2018-09-28 22:03:28 +02:00
const output = this . tableFields _ [ tableName ] . slice ( ) ;
if ( options . includeDescription ) {
for ( let i = 0 ; i < output . length ; i ++ ) {
output [ i ] . description = this . fieldDescription ( tableName , output [ i ] . name ) ;
}
}
return output ;
}
2018-09-29 14:29:07 +02:00
fieldDescription ( tableName , fieldName ) {
const sp = sprintf ;
2018-09-28 22:03:28 +02:00
if ( ! this . tableDescriptions _ ) {
this . tableDescriptions _ = {
notes : {
2018-09-29 14:29:07 +02:00
parent _id : sp ( 'ID of the notebook that contains this note. Change this ID to move the note to a different notebook.' ) ,
body : sp ( 'The note body, in Markdown. May also contain HTML.' ) ,
is _conflict : sp ( 'Tells whether the note is a conflict or not.' ) ,
is _todo : sp ( 'Tells whether this note is a todo or not.' ) ,
todo _due : sp ( 'When the todo is due. An alarm will be triggered on that date.' ) ,
todo _completed : sp ( 'Tells whether todo is completed or not. This is a timestamp in milliseconds.' ) ,
source _url : sp ( 'The full URL where the note comes from.' ) ,
2018-09-28 22:03:28 +02:00
} ,
folders : { } ,
resources : { } ,
tags : { } ,
} ;
const baseItems = [ 'notes' , 'folders' , 'tags' , 'resources' ] ;
for ( let i = 0 ; i < baseItems . length ; i ++ ) {
const n = baseItems [ i ] ;
const singular = n . substr ( 0 , n . length - 1 ) ;
2018-09-29 14:29:07 +02:00
this . tableDescriptions _ [ n ] . title = sp ( 'The %s title.' , singular ) ;
this . tableDescriptions _ [ n ] . created _time = sp ( 'When the %s was created.' , singular ) ;
this . tableDescriptions _ [ n ] . updated _time = sp ( 'When the %s was last updated.' , singular ) ;
this . tableDescriptions _ [ n ] . user _created _time = sp ( 'When the %s was created. It may differ from created_time as it can be manually set by the user.' , singular ) ;
this . tableDescriptions _ [ n ] . user _updated _time = sp ( 'When the %s was last updated. It may differ from updated_time as it can be manually set by the user.' , singular ) ;
2018-09-28 22:03:28 +02:00
}
}
const d = this . tableDescriptions _ [ tableName ] ;
return d && d [ fieldName ] ? d [ fieldName ] : '' ;
2017-07-06 21:48:17 +02:00
}
refreshTableFields ( ) {
2018-03-09 22:59:12 +02:00
this . logger ( ) . info ( 'Initializing tables...' ) ;
2017-07-06 21:48:17 +02:00
let queries = [ ] ;
2018-03-09 22:59:12 +02:00
queries . push ( this . wrapQuery ( 'DELETE FROM table_fields' ) ) ;
return this . selectAll ( 'SELECT name FROM sqlite_master WHERE type="table"' ) . then ( ( tableRows ) => {
let chain = [ ] ;
for ( let i = 0 ; i < tableRows . length ; i ++ ) {
let tableName = tableRows [ i ] . name ;
if ( tableName == 'android_metadata' ) continue ;
if ( tableName == 'table_fields' ) continue ;
if ( tableName == 'sqlite_sequence' ) continue ;
chain . push ( ( ) => {
return this . selectAll ( 'PRAGMA table_info("' + tableName + '")' ) . then ( ( pragmas ) => {
for ( let i = 0 ; i < pragmas . length ; i ++ ) {
let item = pragmas [ i ] ;
// In SQLite, if the default value is a string it has double quotes around it, so remove them here
let defaultValue = item . dflt _value ;
if ( typeof defaultValue == 'string' && defaultValue . length >= 2 && defaultValue [ 0 ] == '"' && defaultValue [ defaultValue . length - 1 ] == '"' ) {
defaultValue = defaultValue . substr ( 1 , defaultValue . length - 2 ) ;
2017-07-06 21:48:17 +02:00
}
2018-03-09 22:59:12 +02:00
let q = Database . insertQuery ( 'table_fields' , {
table _name : tableName ,
field _name : item . name ,
field _type : Database . enumId ( 'fieldType' , item . type ) ,
field _default : defaultValue ,
} ) ;
queries . push ( q ) ;
}
2017-07-06 21:48:17 +02:00
} ) ;
2018-03-09 22:59:12 +02:00
} ) ;
}
2017-07-06 21:48:17 +02:00
2018-03-09 22:59:12 +02:00
return promiseChain ( chain ) ;
} ) . then ( ( ) => {
return this . transactionExecBatch ( queries ) ;
} ) ;
2017-07-06 21:48:17 +02:00
}
2017-07-19 21:15:55 +02:00
async upgradeDatabase ( fromVersion ) {
// INSTRUCTIONS TO UPGRADE THE DATABASE:
//
// 1. Add the new version number to the existingDatabaseVersions array
// 2. Add the upgrade logic to the "switch (targetVersion)" statement below
2017-10-22 19:12:16 +02:00
// IMPORTANT:
//
2018-03-09 22:59:12 +02:00
// Whenever adding a new database property, some additional logic might be needed
2017-10-22 19:12:16 +02:00
// in the synchronizer to handle this property. For example, when adding a property
// that should have a default value, existing remote items will not have this
// default value and thus might cause problems. In that case, the default value
// must be set in the synchronizer too.
2018-10-07 21:11:33 +02:00
const existingDatabaseVersions = [ 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 ] ;
2017-07-19 21:15:55 +02:00
let currentVersionIndex = existingDatabaseVersions . indexOf ( fromVersion ) ;
2017-12-28 21:14:03 +02:00
2017-12-02 17:18:15 +02:00
// currentVersionIndex < 0 if for the case where an old version of Joplin used with a newer
// version of the database, so that migration is not run in this case.
2018-05-06 13:11:59 +02:00
if ( currentVersionIndex < 0 ) throw new Error ( 'Unknown profile version. Most likely this is an old version of Joplin, while the profile was created by a newer version. Please upgrade Joplin at https://joplin.cozic.net and try again.' ) ;
2017-12-28 21:14:03 +02:00
if ( currentVersionIndex == existingDatabaseVersions . length - 1 ) return false ;
2017-07-19 21:15:55 +02:00
while ( currentVersionIndex < existingDatabaseVersions . length - 1 ) {
const targetVersion = existingDatabaseVersions [ currentVersionIndex + 1 ] ;
this . logger ( ) . info ( "Converting database to version " + targetVersion ) ;
let queries = [ ] ;
2017-07-26 19:49:01 +02:00
if ( targetVersion == 1 ) {
queries = this . wrapQueries ( this . sqlStringToLines ( structureSql ) ) ;
}
2018-03-09 22:59:12 +02:00
2017-07-19 21:15:55 +02:00
if ( targetVersion == 2 ) {
const newTableSql = `
CREATE TABLE deleted _items (
id INTEGER PRIMARY KEY ,
item _type INT NOT NULL ,
item _id TEXT NOT NULL ,
deleted _time INT NOT NULL ,
sync _target INT NOT NULL
) ;
` ;
2018-03-09 22:59:12 +02:00
queries . push ( { sql : 'DROP TABLE deleted_items' } ) ;
2017-07-19 21:15:55 +02:00
queries . push ( { sql : this . sqlStringToLines ( newTableSql ) [ 0 ] } ) ;
queries . push ( { sql : "CREATE INDEX deleted_items_sync_target ON deleted_items (sync_target)" } ) ;
}
2017-07-25 23:55:26 +02:00
if ( targetVersion == 3 ) {
2018-03-09 22:59:12 +02:00
queries = this . alterColumnQueries ( 'settings' , { key : 'TEXT PRIMARY KEY' , value : 'TEXT' } ) ;
2017-07-25 23:55:26 +02:00
}
2017-08-19 22:56:28 +02:00
if ( targetVersion == 4 ) {
queries . push ( "INSERT INTO settings (`key`, `value`) VALUES ('sync.3.context', (SELECT `value` FROM settings WHERE `key` = 'sync.context'))" ) ;
queries . push ( 'DELETE FROM settings WHERE `key` = "sync.context"' ) ;
}
2017-08-20 22:11:32 +02:00
if ( targetVersion == 5 ) {
2018-03-09 22:59:12 +02:00
const tableNames = [ 'notes' , 'folders' , 'tags' , 'note_tags' , 'resources' ] ;
2017-08-20 22:11:32 +02:00
for ( let i = 0 ; i < tableNames . length ; i ++ ) {
const n = tableNames [ i ] ;
2018-03-09 22:59:12 +02:00
queries . push ( 'ALTER TABLE ' + n + ' ADD COLUMN user_created_time INT NOT NULL DEFAULT 0' ) ;
queries . push ( 'ALTER TABLE ' + n + ' ADD COLUMN user_updated_time INT NOT NULL DEFAULT 0' ) ;
queries . push ( 'UPDATE ' + n + ' SET user_created_time = created_time' ) ;
queries . push ( 'UPDATE ' + n + ' SET user_updated_time = updated_time' ) ;
queries . push ( 'CREATE INDEX ' + n + '_user_updated_time ON ' + n + ' (user_updated_time)' ) ;
2017-08-20 22:11:32 +02:00
}
}
2017-11-28 00:50:46 +02:00
if ( targetVersion == 6 ) {
2018-03-09 22:59:12 +02:00
queries . push ( 'CREATE TABLE alarms (id INTEGER PRIMARY KEY AUTOINCREMENT, note_id TEXT NOT NULL, trigger_time INT NOT NULL)' ) ;
queries . push ( 'CREATE INDEX alarm_note_id ON alarms (note_id)' ) ;
2017-11-28 00:50:46 +02:00
}
2017-12-02 01:15:49 +02:00
if ( targetVersion == 7 ) {
queries . push ( 'ALTER TABLE resources ADD COLUMN file_extension TEXT NOT NULL DEFAULT ""' ) ;
}
2017-12-05 01:38:09 +02:00
if ( targetVersion == 8 ) {
queries . push ( 'ALTER TABLE sync_items ADD COLUMN sync_disabled INT NOT NULL DEFAULT "0"' ) ;
queries . push ( 'ALTER TABLE sync_items ADD COLUMN sync_disabled_reason TEXT NOT NULL DEFAULT ""' ) ;
}
2017-12-13 20:57:40 +02:00
if ( targetVersion == 9 ) {
2017-12-14 20:53:08 +02:00
const newTableSql = `
CREATE TABLE master _keys (
id TEXT PRIMARY KEY ,
created _time INT NOT NULL ,
updated _time INT NOT NULL ,
source _application TEXT NOT NULL ,
encryption _method INT NOT NULL ,
checksum TEXT NOT NULL ,
content TEXT NOT NULL
) ;
` ;
queries . push ( this . sqlStringToLines ( newTableSql ) [ 0 ] ) ;
2018-03-09 22:59:12 +02:00
const tableNames = [ 'notes' , 'folders' , 'tags' , 'note_tags' , 'resources' ] ;
2017-12-14 19:58:10 +02:00
for ( let i = 0 ; i < tableNames . length ; i ++ ) {
const n = tableNames [ i ] ;
2018-03-09 22:59:12 +02:00
queries . push ( 'ALTER TABLE ' + n + ' ADD COLUMN encryption_cipher_text TEXT NOT NULL DEFAULT ""' ) ;
queries . push ( 'ALTER TABLE ' + n + ' ADD COLUMN encryption_applied INT NOT NULL DEFAULT 0' ) ;
queries . push ( 'CREATE INDEX ' + n + '_encryption_applied ON ' + n + ' (encryption_applied)' ) ;
2017-12-14 19:58:10 +02:00
}
2017-12-14 23:12:02 +02:00
2018-03-09 22:59:12 +02:00
queries . push ( 'ALTER TABLE sync_items ADD COLUMN force_sync INT NOT NULL DEFAULT 0' ) ;
queries . push ( 'ALTER TABLE resources ADD COLUMN encryption_blob_encrypted INT NOT NULL DEFAULT 0' ) ;
2017-12-13 20:57:40 +02:00
}
2018-03-16 16:32:47 +02:00
const upgradeVersion10 = ( ) => {
2018-03-13 01:40:43 +02:00
const itemChangesTable = `
CREATE TABLE item _changes (
2018-03-15 20:08:46 +02:00
id INTEGER PRIMARY KEY AUTOINCREMENT ,
2018-03-13 01:40:43 +02:00
item _type INT NOT NULL ,
item _id TEXT NOT NULL ,
type INT NOT NULL ,
created _time INT NOT NULL
) ;
` ;
const noteResourcesTable = `
CREATE TABLE note _resources (
id INTEGER PRIMARY KEY ,
note _id TEXT NOT NULL ,
2018-03-15 20:08:46 +02:00
resource _id TEXT NOT NULL ,
is _associated INT NOT NULL ,
last _seen _time INT NOT NULL
2018-03-13 01:40:43 +02:00
) ;
` ;
queries . push ( this . sqlStringToLines ( itemChangesTable ) [ 0 ] ) ;
queries . push ( 'CREATE INDEX item_changes_item_id ON item_changes (item_id)' ) ;
queries . push ( 'CREATE INDEX item_changes_created_time ON item_changes (created_time)' ) ;
queries . push ( 'CREATE INDEX item_changes_item_type ON item_changes (item_type)' ) ;
queries . push ( this . sqlStringToLines ( noteResourcesTable ) [ 0 ] ) ;
queries . push ( 'CREATE INDEX note_resources_note_id ON note_resources (note_id)' ) ;
queries . push ( 'CREATE INDEX note_resources_resource_id ON note_resources (resource_id)' ) ;
queries . push ( { sql : 'INSERT INTO item_changes (item_type, item_id, type, created_time) SELECT 1, id, 1, ? FROM notes' , params : [ Date . now ( ) ] } ) ;
}
2018-03-16 16:32:47 +02:00
if ( targetVersion == 10 ) {
upgradeVersion10 ( ) ;
}
if ( targetVersion == 11 ) {
2018-03-16 19:39:44 +02:00
// This trick was needed because Electron Builder incorrectly released a dev branch containing v10 as it was
// still being developed, and the db schema was not final at that time. So this v11 was created to
// make sure any invalid db schema that was accidentally created was deleted and recreated.
2018-03-16 16:32:47 +02:00
queries . push ( 'DROP TABLE item_changes' ) ;
queries . push ( 'DROP TABLE note_resources' ) ;
upgradeVersion10 ( ) ;
}
2018-05-06 13:11:59 +02:00
if ( targetVersion == 12 ) {
queries . push ( 'ALTER TABLE folders ADD COLUMN parent_id TEXT NOT NULL DEFAULT ""' ) ;
}
2018-10-07 21:11:33 +02:00
if ( targetVersion == 13 ) {
2018-10-10 19:53:09 +02:00
queries . push ( 'ALTER TABLE resources ADD COLUMN fetch_status INT NOT NULL DEFAULT "2"' ) ;
2018-10-07 21:11:33 +02:00
queries . push ( 'ALTER TABLE resources ADD COLUMN fetch_error TEXT NOT NULL DEFAULT ""' ) ;
2018-10-08 20:11:53 +02:00
queries . push ( { sql : 'UPDATE resources SET fetch_status = ?' , params : [ Resource . FETCH _STATUS _DONE ] } ) ;
2018-10-07 21:11:33 +02:00
}
2018-03-09 22:59:12 +02:00
queries . push ( { sql : 'UPDATE version SET version = ?' , params : [ targetVersion ] } ) ;
2017-07-19 21:15:55 +02:00
await this . transactionExecBatch ( queries ) ;
currentVersionIndex ++ ;
}
return true ;
}
2017-07-06 21:48:17 +02:00
async initialize ( ) {
2018-03-09 22:59:12 +02:00
this . logger ( ) . info ( 'Checking for database schema update...' ) ;
2017-07-06 21:48:17 +02:00
2017-07-26 19:49:01 +02:00
let versionRow = null ;
try {
// Will throw if the database has not been created yet, but this is handled below
2018-03-09 22:59:12 +02:00
versionRow = await this . selectOne ( 'SELECT * FROM version LIMIT 1' ) ;
2017-07-26 19:49:01 +02:00
} catch ( error ) {
2018-03-09 22:59:12 +02:00
if ( error . message && ( error . message . indexOf ( 'no such table: version' ) >= 0 ) ) {
2017-07-26 22:00:16 +02:00
// Ignore
} else {
console . info ( error ) ;
}
2017-07-26 19:49:01 +02:00
}
2017-07-06 21:48:17 +02:00
2017-07-26 19:49:01 +02:00
const version = ! versionRow ? 0 : versionRow . version ;
2018-03-09 22:59:12 +02:00
this . logger ( ) . info ( 'Current database version' , version ) ;
2017-07-06 21:48:17 +02:00
2017-07-26 19:49:01 +02:00
const upgraded = await this . upgradeDatabase ( version ) ;
if ( upgraded ) await this . refreshTableFields ( ) ;
2017-07-06 21:48:17 +02:00
2017-07-26 19:49:01 +02:00
this . tableFields _ = { } ;
2017-07-06 21:48:17 +02:00
2018-03-09 22:59:12 +02:00
let rows = await this . selectAll ( 'SELECT * FROM table_fields' ) ;
2017-07-06 21:48:17 +02:00
2017-07-26 19:49:01 +02:00
for ( let i = 0 ; i < rows . length ; i ++ ) {
let row = rows [ i ] ;
if ( ! this . tableFields _ [ row . table _name ] ) this . tableFields _ [ row . table _name ] = [ ] ;
this . tableFields _ [ row . table _name ] . push ( {
name : row . field _name ,
type : row . field _type ,
default : Database . formatValue ( row . field _type , row . field _default ) ,
} ) ;
2017-07-06 21:48:17 +02:00
}
}
2018-03-09 22:59:12 +02:00
2017-07-06 21:48:17 +02:00
}
Database . TYPE _INT = 1 ;
Database . TYPE _TEXT = 2 ;
Database . TYPE _NUMERIC = 3 ;
2018-03-09 22:59:12 +02:00
module . exports = { JoplinDatabase } ;