2018-03-09 22:59:12 +02:00
const { promiseChain } = require ( 'lib/promise-utils.js' ) ;
const { Database } = require ( 'lib/database.js' ) ;
2018-09-29 14:29:07 +02:00
const { sprintf } = require ( 'sprintf-js' ) ;
2018-10-08 20:11:53 +02:00
const Resource = require ( 'lib/models/Resource' ) ;
2017-07-06 21:48:17 +02:00
const structureSql = `
CREATE TABLE folders (
id TEXT PRIMARY KEY ,
title TEXT NOT NULL DEFAULT "" ,
created _time INT NOT NULL ,
2017-07-16 14:53:59 +02:00
updated _time INT NOT NULL
2017-07-06 21:48:17 +02:00
) ;
CREATE INDEX folders _title ON folders ( title ) ;
CREATE INDEX folders _updated _time ON folders ( updated _time ) ;
CREATE TABLE notes (
id TEXT PRIMARY KEY ,
parent _id TEXT NOT NULL DEFAULT "" ,
title TEXT NOT NULL DEFAULT "" ,
body TEXT NOT NULL DEFAULT "" ,
created _time INT NOT NULL ,
updated _time INT NOT NULL ,
is _conflict INT NOT NULL DEFAULT 0 ,
latitude NUMERIC NOT NULL DEFAULT 0 ,
longitude NUMERIC NOT NULL DEFAULT 0 ,
altitude NUMERIC NOT NULL DEFAULT 0 ,
author TEXT NOT NULL DEFAULT "" ,
source _url TEXT NOT NULL DEFAULT "" ,
is _todo INT NOT NULL DEFAULT 0 ,
todo _due INT NOT NULL DEFAULT 0 ,
todo _completed INT NOT NULL DEFAULT 0 ,
source TEXT NOT NULL DEFAULT "" ,
source _application TEXT NOT NULL DEFAULT "" ,
application _data TEXT NOT NULL DEFAULT "" ,
\ ` order \` INT NOT NULL DEFAULT 0
) ;
CREATE INDEX notes _title ON notes ( title ) ;
CREATE INDEX notes _updated _time ON notes ( updated _time ) ;
CREATE INDEX notes _is _conflict ON notes ( is _conflict ) ;
CREATE INDEX notes _is _todo ON notes ( is _todo ) ;
CREATE INDEX notes _order ON notes ( \ ` order \` );
CREATE TABLE tags (
id TEXT PRIMARY KEY ,
title TEXT NOT NULL DEFAULT "" ,
created _time INT NOT NULL ,
2017-07-16 14:53:59 +02:00
updated _time INT NOT NULL
2017-07-06 21:48:17 +02:00
) ;
CREATE INDEX tags _title ON tags ( title ) ;
CREATE INDEX tags _updated _time ON tags ( updated _time ) ;
CREATE TABLE note _tags (
id TEXT PRIMARY KEY ,
note _id TEXT NOT NULL ,
tag _id TEXT NOT NULL ,
created _time INT NOT NULL ,
2017-07-16 14:53:59 +02:00
updated _time INT NOT NULL
2017-07-06 21:48:17 +02:00
) ;
CREATE INDEX note _tags _note _id ON note _tags ( note _id ) ;
CREATE INDEX note _tags _tag _id ON note _tags ( tag _id ) ;
CREATE INDEX note _tags _updated _time ON note _tags ( updated _time ) ;
CREATE TABLE resources (
id TEXT PRIMARY KEY ,
title TEXT NOT NULL DEFAULT "" ,
mime TEXT NOT NULL ,
filename TEXT NOT NULL DEFAULT "" ,
created _time INT NOT NULL ,
2017-07-16 14:53:59 +02:00
updated _time INT NOT NULL
2017-07-06 21:48:17 +02:00
) ;
CREATE INDEX resources _title ON resources ( title ) ;
CREATE INDEX resources _updated _time ON resources ( updated _time ) ;
CREATE TABLE settings (
\ ` key \` TEXT PRIMARY KEY,
\ ` value \` TEXT,
\ ` type \` INT NOT NULL
) ;
CREATE TABLE table _fields (
id INTEGER PRIMARY KEY ,
table _name TEXT NOT NULL ,
field _name TEXT NOT NULL ,
field _type INT NOT NULL ,
field _default TEXT
) ;
2017-07-16 14:53:59 +02:00
CREATE TABLE sync _items (
id INTEGER PRIMARY KEY ,
sync _target INT NOT NULL ,
sync _time INT NOT NULL DEFAULT 0 ,
item _type INT NOT NULL ,
item _id TEXT NOT NULL
) ;
CREATE INDEX sync _items _sync _time ON sync _items ( sync _time ) ;
CREATE INDEX sync _items _sync _target ON sync _items ( sync _target ) ;
CREATE INDEX sync _items _item _type ON sync _items ( item _type ) ;
CREATE INDEX sync _items _item _id ON sync _items ( item _id ) ;
2017-07-19 21:15:55 +02:00
CREATE TABLE deleted _items (
id INTEGER PRIMARY KEY ,
item _type INT NOT NULL ,
item _id TEXT NOT NULL ,
deleted _time INT NOT NULL
) ;
CREATE TABLE version (
version INT NOT NULL
) ;
2017-07-06 21:48:17 +02:00
INSERT INTO version ( version ) VALUES ( 1 ) ;
` ;
class JoplinDatabase extends Database {
constructor ( driver ) {
super ( driver ) ;
this . initialized _ = false ;
this . tableFields _ = null ;
2018-12-28 22:40:29 +02:00
this . version _ = null ;
2017-07-06 21:48:17 +02:00
}
initialized ( ) {
return this . initialized _ ;
}
async open ( options ) {
await super . open ( options ) ;
return this . initialize ( ) ;
}
tableFieldNames ( tableName ) {
let tf = this . tableFields ( tableName ) ;
let output = [ ] ;
for ( let i = 0 ; i < tf . length ; i ++ ) {
output . push ( tf [ i ] . name ) ;
}
return output ;
}
2018-09-28 22:03:28 +02:00
tableFields ( tableName , options = null ) {
if ( options === null ) options = { } ;
2018-03-09 22:59:12 +02:00
if ( ! this . tableFields _ ) throw new Error ( 'Fields have not been loaded yet' ) ;
2019-09-19 23:51:18 +02:00
if ( ! this . tableFields _ [ tableName ] ) throw new Error ( ` Unknown table: ${ tableName } ` ) ;
2018-09-28 22:03:28 +02:00
const output = this . tableFields _ [ tableName ] . slice ( ) ;
if ( options . includeDescription ) {
for ( let i = 0 ; i < output . length ; i ++ ) {
output [ i ] . description = this . fieldDescription ( tableName , output [ i ] . name ) ;
}
}
return output ;
}
2019-05-26 20:39:07 +02:00
async clearForTesting ( ) {
const tableNames = [
'notes' ,
'folders' ,
'resources' ,
'tags' ,
'note_tags' ,
// 'master_keys',
'item_changes' ,
'note_resources' ,
// 'settings',
'deleted_items' ,
'sync_items' ,
'notes_normalized' ,
'revisions' ,
'resources_to_download' ,
2019-06-08 00:11:08 +02:00
'key_values' ,
2019-05-26 20:39:07 +02:00
] ;
const queries = [ ] ;
for ( const n of tableNames ) {
2019-09-19 23:51:18 +02:00
queries . push ( ` DELETE FROM ${ n } ` ) ;
queries . push ( ` DELETE FROM sqlite_sequence WHERE name=" ${ n } " ` ) ; // Reset autoincremented IDs
2019-05-26 20:39:07 +02:00
}
2019-06-08 00:11:08 +02:00
queries . push ( 'DELETE FROM settings WHERE key="sync.1.context"' ) ;
queries . push ( 'DELETE FROM settings WHERE key="sync.2.context"' ) ;
queries . push ( 'DELETE FROM settings WHERE key="sync.3.context"' ) ;
queries . push ( 'DELETE FROM settings WHERE key="sync.4.context"' ) ;
queries . push ( 'DELETE FROM settings WHERE key="sync.5.context"' ) ;
queries . push ( 'DELETE FROM settings WHERE key="sync.6.context"' ) ;
2019-05-26 20:39:07 +02:00
queries . push ( 'DELETE FROM settings WHERE key="sync.7.context"' ) ;
2019-07-29 15:43:53 +02:00
2019-06-28 01:51:02 +02:00
queries . push ( 'DELETE FROM settings WHERE key="revisionService.lastProcessedChangeId"' ) ;
queries . push ( 'DELETE FROM settings WHERE key="resourceService.lastProcessedChangeId"' ) ;
queries . push ( 'DELETE FROM settings WHERE key="searchEngine.lastProcessedChangeId"' ) ;
2019-05-26 20:39:07 +02:00
await this . transactionExecBatch ( queries ) ;
}
2019-09-13 00:16:42 +02:00
createDefaultRow ( ) {
2018-11-13 02:45:08 +02:00
const row = { } ;
const fields = this . tableFields ( 'resource_local_states' ) ;
for ( let i = 0 ; i < fields . length ; i ++ ) {
const f = fields [ i ] ;
row [ f . name ] = Database . formatValue ( f . type , f . default ) ;
}
return row ;
}
2019-07-29 15:43:53 +02:00
fieldDescription ( tableName , fieldName ) {
2018-09-29 14:29:07 +02:00
const sp = sprintf ;
2018-09-28 22:03:28 +02:00
if ( ! this . tableDescriptions _ ) {
this . tableDescriptions _ = {
notes : {
2018-09-29 14:29:07 +02:00
parent _id : sp ( 'ID of the notebook that contains this note. Change this ID to move the note to a different notebook.' ) ,
body : sp ( 'The note body, in Markdown. May also contain HTML.' ) ,
is _conflict : sp ( 'Tells whether the note is a conflict or not.' ) ,
is _todo : sp ( 'Tells whether this note is a todo or not.' ) ,
todo _due : sp ( 'When the todo is due. An alarm will be triggered on that date.' ) ,
todo _completed : sp ( 'Tells whether todo is completed or not. This is a timestamp in milliseconds.' ) ,
source _url : sp ( 'The full URL where the note comes from.' ) ,
2018-09-28 22:03:28 +02:00
} ,
folders : { } ,
resources : { } ,
tags : { } ,
} ;
const baseItems = [ 'notes' , 'folders' , 'tags' , 'resources' ] ;
for ( let i = 0 ; i < baseItems . length ; i ++ ) {
const n = baseItems [ i ] ;
const singular = n . substr ( 0 , n . length - 1 ) ;
2018-09-29 14:29:07 +02:00
this . tableDescriptions _ [ n ] . title = sp ( 'The %s title.' , singular ) ;
this . tableDescriptions _ [ n ] . created _time = sp ( 'When the %s was created.' , singular ) ;
this . tableDescriptions _ [ n ] . updated _time = sp ( 'When the %s was last updated.' , singular ) ;
this . tableDescriptions _ [ n ] . user _created _time = sp ( 'When the %s was created. It may differ from created_time as it can be manually set by the user.' , singular ) ;
this . tableDescriptions _ [ n ] . user _updated _time = sp ( 'When the %s was last updated. It may differ from updated_time as it can be manually set by the user.' , singular ) ;
2018-09-28 22:03:28 +02:00
}
}
const d = this . tableDescriptions _ [ tableName ] ;
return d && d [ fieldName ] ? d [ fieldName ] : '' ;
2017-07-06 21:48:17 +02:00
}
refreshTableFields ( ) {
2018-03-09 22:59:12 +02:00
this . logger ( ) . info ( 'Initializing tables...' ) ;
2017-07-06 21:48:17 +02:00
let queries = [ ] ;
2018-03-09 22:59:12 +02:00
queries . push ( this . wrapQuery ( 'DELETE FROM table_fields' ) ) ;
2019-07-29 15:43:53 +02:00
return this . selectAll ( 'SELECT name FROM sqlite_master WHERE type="table"' )
. then ( tableRows => {
let chain = [ ] ;
for ( let i = 0 ; i < tableRows . length ; i ++ ) {
let tableName = tableRows [ i ] . name ;
if ( tableName == 'android_metadata' ) continue ;
if ( tableName == 'table_fields' ) continue ;
if ( tableName == 'sqlite_sequence' ) continue ;
if ( tableName . indexOf ( 'notes_fts' ) === 0 ) continue ;
chain . push ( ( ) => {
2019-09-19 23:51:18 +02:00
return this . selectAll ( ` PRAGMA table_info(" ${ tableName } ") ` ) . then ( pragmas => {
2019-07-29 15:43:53 +02:00
for ( let i = 0 ; i < pragmas . length ; i ++ ) {
let item = pragmas [ i ] ;
// In SQLite, if the default value is a string it has double quotes around it, so remove them here
let defaultValue = item . dflt _value ;
if ( typeof defaultValue == 'string' && defaultValue . length >= 2 && defaultValue [ 0 ] == '"' && defaultValue [ defaultValue . length - 1 ] == '"' ) {
defaultValue = defaultValue . substr ( 1 , defaultValue . length - 2 ) ;
}
let q = Database . insertQuery ( 'table_fields' , {
table _name : tableName ,
field _name : item . name ,
field _type : Database . enumId ( 'fieldType' , item . type ) ,
field _default : defaultValue ,
} ) ;
queries . push ( q ) ;
2017-07-06 21:48:17 +02:00
}
2019-07-29 15:43:53 +02:00
} ) ;
2017-07-06 21:48:17 +02:00
} ) ;
2019-07-29 15:43:53 +02:00
}
2017-07-06 21:48:17 +02:00
2019-07-29 15:43:53 +02:00
return promiseChain ( chain ) ;
} )
. then ( ( ) => {
return this . transactionExecBatch ( queries ) ;
} ) ;
2017-07-06 21:48:17 +02:00
}
2019-12-29 19:58:40 +02:00
addMigrationFile ( num ) {
const timestamp = Date . now ( ) ;
return { sql : 'INSERT INTO migrations (number, created_time, updated_time) VALUES (?, ?, ?)' , params : [ num , timestamp , timestamp ] } ;
}
2017-07-19 21:15:55 +02:00
async upgradeDatabase ( fromVersion ) {
// INSTRUCTIONS TO UPGRADE THE DATABASE:
//
// 1. Add the new version number to the existingDatabaseVersions array
// 2. Add the upgrade logic to the "switch (targetVersion)" statement below
2017-10-22 19:12:16 +02:00
// IMPORTANT:
//
2019-07-29 15:43:53 +02:00
// Whenever adding a new database property, some additional logic might be needed
2017-10-22 19:12:16 +02:00
// in the synchronizer to handle this property. For example, when adding a property
// that should have a default value, existing remote items will not have this
// default value and thus might cause problems. In that case, the default value
// must be set in the synchronizer too.
2018-12-27 23:49:19 +02:00
// Note: v16 and v17 don't do anything. They were used to debug an issue.
2019-12-29 19:58:40 +02:00
const existingDatabaseVersions = [ 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10 , 11 , 12 , 13 , 14 , 15 , 16 , 17 , 18 , 19 , 20 , 21 , 22 , 23 , 24 , 25 , 26 , 27 ] ;
2017-07-19 21:15:55 +02:00
let currentVersionIndex = existingDatabaseVersions . indexOf ( fromVersion ) ;
2017-12-28 21:14:03 +02:00
2017-12-02 17:18:15 +02:00
// currentVersionIndex < 0 if for the case where an old version of Joplin used with a newer
// version of the database, so that migration is not run in this case.
2019-04-18 15:59:17 +02:00
if ( currentVersionIndex < 0 ) throw new Error ( 'Unknown profile version. Most likely this is an old version of Joplin, while the profile was created by a newer version. Please upgrade Joplin at https://joplinapp.org and try again.' ) ;
2018-05-06 13:11:59 +02:00
2018-12-28 22:40:29 +02:00
if ( currentVersionIndex == existingDatabaseVersions . length - 1 ) return fromVersion ;
let latestVersion = fromVersion ;
2017-12-28 21:14:03 +02:00
2017-07-19 21:15:55 +02:00
while ( currentVersionIndex < existingDatabaseVersions . length - 1 ) {
const targetVersion = existingDatabaseVersions [ currentVersionIndex + 1 ] ;
2019-09-19 23:51:18 +02:00
this . logger ( ) . info ( ` Converting database to version ${ targetVersion } ` ) ;
2017-07-19 21:15:55 +02:00
let queries = [ ] ;
2017-07-26 19:49:01 +02:00
if ( targetVersion == 1 ) {
queries = this . wrapQueries ( this . sqlStringToLines ( structureSql ) ) ;
}
2019-07-29 15:43:53 +02:00
2017-07-19 21:15:55 +02:00
if ( targetVersion == 2 ) {
const newTableSql = `
CREATE TABLE deleted _items (
id INTEGER PRIMARY KEY ,
item _type INT NOT NULL ,
item _id TEXT NOT NULL ,
deleted _time INT NOT NULL ,
sync _target INT NOT NULL
) ;
` ;
2018-03-09 22:59:12 +02:00
queries . push ( { sql : 'DROP TABLE deleted_items' } ) ;
2017-07-19 21:15:55 +02:00
queries . push ( { sql : this . sqlStringToLines ( newTableSql ) [ 0 ] } ) ;
2019-07-29 15:43:53 +02:00
queries . push ( { sql : 'CREATE INDEX deleted_items_sync_target ON deleted_items (sync_target)' } ) ;
2017-07-19 21:15:55 +02:00
}
2017-07-25 23:55:26 +02:00
if ( targetVersion == 3 ) {
2018-03-09 22:59:12 +02:00
queries = this . alterColumnQueries ( 'settings' , { key : 'TEXT PRIMARY KEY' , value : 'TEXT' } ) ;
2017-07-25 23:55:26 +02:00
}
2017-08-19 22:56:28 +02:00
if ( targetVersion == 4 ) {
2019-07-30 09:35:42 +02:00
queries . push ( 'INSERT INTO settings (`key`, `value`) VALUES (\'sync.3.context\', (SELECT `value` FROM settings WHERE `key` = \'sync.context\'))' ) ;
2017-08-19 22:56:28 +02:00
queries . push ( 'DELETE FROM settings WHERE `key` = "sync.context"' ) ;
}
2017-08-20 22:11:32 +02:00
if ( targetVersion == 5 ) {
2018-03-09 22:59:12 +02:00
const tableNames = [ 'notes' , 'folders' , 'tags' , 'note_tags' , 'resources' ] ;
2017-08-20 22:11:32 +02:00
for ( let i = 0 ; i < tableNames . length ; i ++ ) {
const n = tableNames [ i ] ;
2019-09-19 23:51:18 +02:00
queries . push ( ` ALTER TABLE ${ n } ADD COLUMN user_created_time INT NOT NULL DEFAULT 0 ` ) ;
queries . push ( ` ALTER TABLE ${ n } ADD COLUMN user_updated_time INT NOT NULL DEFAULT 0 ` ) ;
queries . push ( ` UPDATE ${ n } SET user_created_time = created_time ` ) ;
queries . push ( ` UPDATE ${ n } SET user_updated_time = updated_time ` ) ;
queries . push ( ` CREATE INDEX ${ n } _user_updated_time ON ${ n } (user_updated_time) ` ) ;
2017-08-20 22:11:32 +02:00
}
}
2017-11-28 00:50:46 +02:00
if ( targetVersion == 6 ) {
2018-03-09 22:59:12 +02:00
queries . push ( 'CREATE TABLE alarms (id INTEGER PRIMARY KEY AUTOINCREMENT, note_id TEXT NOT NULL, trigger_time INT NOT NULL)' ) ;
queries . push ( 'CREATE INDEX alarm_note_id ON alarms (note_id)' ) ;
2017-11-28 00:50:46 +02:00
}
2017-12-02 01:15:49 +02:00
if ( targetVersion == 7 ) {
queries . push ( 'ALTER TABLE resources ADD COLUMN file_extension TEXT NOT NULL DEFAULT ""' ) ;
}
2017-12-05 01:38:09 +02:00
if ( targetVersion == 8 ) {
queries . push ( 'ALTER TABLE sync_items ADD COLUMN sync_disabled INT NOT NULL DEFAULT "0"' ) ;
queries . push ( 'ALTER TABLE sync_items ADD COLUMN sync_disabled_reason TEXT NOT NULL DEFAULT ""' ) ;
}
2017-12-13 20:57:40 +02:00
if ( targetVersion == 9 ) {
2017-12-14 20:53:08 +02:00
const newTableSql = `
CREATE TABLE master _keys (
id TEXT PRIMARY KEY ,
created _time INT NOT NULL ,
updated _time INT NOT NULL ,
source _application TEXT NOT NULL ,
encryption _method INT NOT NULL ,
checksum TEXT NOT NULL ,
content TEXT NOT NULL
) ;
` ;
queries . push ( this . sqlStringToLines ( newTableSql ) [ 0 ] ) ;
2018-03-09 22:59:12 +02:00
const tableNames = [ 'notes' , 'folders' , 'tags' , 'note_tags' , 'resources' ] ;
2017-12-14 19:58:10 +02:00
for ( let i = 0 ; i < tableNames . length ; i ++ ) {
const n = tableNames [ i ] ;
2019-09-19 23:51:18 +02:00
queries . push ( ` ALTER TABLE ${ n } ADD COLUMN encryption_cipher_text TEXT NOT NULL DEFAULT "" ` ) ;
queries . push ( ` ALTER TABLE ${ n } ADD COLUMN encryption_applied INT NOT NULL DEFAULT 0 ` ) ;
queries . push ( ` CREATE INDEX ${ n } _encryption_applied ON ${ n } (encryption_applied) ` ) ;
2017-12-14 19:58:10 +02:00
}
2017-12-14 23:12:02 +02:00
2018-03-09 22:59:12 +02:00
queries . push ( 'ALTER TABLE sync_items ADD COLUMN force_sync INT NOT NULL DEFAULT 0' ) ;
queries . push ( 'ALTER TABLE resources ADD COLUMN encryption_blob_encrypted INT NOT NULL DEFAULT 0' ) ;
2017-12-13 20:57:40 +02:00
}
2018-03-16 16:32:47 +02:00
const upgradeVersion10 = ( ) => {
2018-03-13 01:40:43 +02:00
const itemChangesTable = `
CREATE TABLE item _changes (
2018-03-15 20:08:46 +02:00
id INTEGER PRIMARY KEY AUTOINCREMENT ,
2018-03-13 01:40:43 +02:00
item _type INT NOT NULL ,
item _id TEXT NOT NULL ,
type INT NOT NULL ,
created _time INT NOT NULL
) ;
` ;
const noteResourcesTable = `
CREATE TABLE note _resources (
id INTEGER PRIMARY KEY ,
note _id TEXT NOT NULL ,
2018-03-15 20:08:46 +02:00
resource _id TEXT NOT NULL ,
is _associated INT NOT NULL ,
last _seen _time INT NOT NULL
2018-03-13 01:40:43 +02:00
) ;
` ;
queries . push ( this . sqlStringToLines ( itemChangesTable ) [ 0 ] ) ;
queries . push ( 'CREATE INDEX item_changes_item_id ON item_changes (item_id)' ) ;
queries . push ( 'CREATE INDEX item_changes_created_time ON item_changes (created_time)' ) ;
queries . push ( 'CREATE INDEX item_changes_item_type ON item_changes (item_type)' ) ;
queries . push ( this . sqlStringToLines ( noteResourcesTable ) [ 0 ] ) ;
queries . push ( 'CREATE INDEX note_resources_note_id ON note_resources (note_id)' ) ;
queries . push ( 'CREATE INDEX note_resources_resource_id ON note_resources (resource_id)' ) ;
queries . push ( { sql : 'INSERT INTO item_changes (item_type, item_id, type, created_time) SELECT 1, id, 1, ? FROM notes' , params : [ Date . now ( ) ] } ) ;
2019-07-29 15:43:53 +02:00
} ;
2018-03-13 01:40:43 +02:00
2018-03-16 16:32:47 +02:00
if ( targetVersion == 10 ) {
upgradeVersion10 ( ) ;
}
if ( targetVersion == 11 ) {
2018-03-16 19:39:44 +02:00
// This trick was needed because Electron Builder incorrectly released a dev branch containing v10 as it was
// still being developed, and the db schema was not final at that time. So this v11 was created to
// make sure any invalid db schema that was accidentally created was deleted and recreated.
2018-03-16 16:32:47 +02:00
queries . push ( 'DROP TABLE item_changes' ) ;
queries . push ( 'DROP TABLE note_resources' ) ;
upgradeVersion10 ( ) ;
}
2018-05-06 13:11:59 +02:00
if ( targetVersion == 12 ) {
queries . push ( 'ALTER TABLE folders ADD COLUMN parent_id TEXT NOT NULL DEFAULT ""' ) ;
}
2018-10-07 21:11:33 +02:00
if ( targetVersion == 13 ) {
2018-10-10 19:53:09 +02:00
queries . push ( 'ALTER TABLE resources ADD COLUMN fetch_status INT NOT NULL DEFAULT "2"' ) ;
2018-10-07 21:11:33 +02:00
queries . push ( 'ALTER TABLE resources ADD COLUMN fetch_error TEXT NOT NULL DEFAULT ""' ) ;
2018-10-08 20:11:53 +02:00
queries . push ( { sql : 'UPDATE resources SET fetch_status = ?' , params : [ Resource . FETCH _STATUS _DONE ] } ) ;
2018-10-07 21:11:33 +02:00
}
2018-11-13 02:45:08 +02:00
if ( targetVersion == 14 ) {
const resourceLocalStates = `
CREATE TABLE resource _local _states (
id INTEGER PRIMARY KEY ,
resource _id TEXT NOT NULL ,
fetch _status INT NOT NULL DEFAULT "2" ,
fetch _error TEXT NOT NULL DEFAULT ""
) ;
` ;
queries . push ( this . sqlStringToLines ( resourceLocalStates ) [ 0 ] ) ;
queries . push ( 'INSERT INTO resource_local_states SELECT null, id, fetch_status, fetch_error FROM resources' ) ;
queries . push ( 'CREATE INDEX resource_local_states_resource_id ON resource_local_states (resource_id)' ) ;
queries . push ( 'CREATE INDEX resource_local_states_resource_fetch_status ON resource_local_states (fetch_status)' ) ;
2019-07-29 15:43:53 +02:00
queries = queries . concat (
this . alterColumnQueries ( 'resources' , {
id : 'TEXT PRIMARY KEY' ,
title : 'TEXT NOT NULL DEFAULT ""' ,
mime : 'TEXT NOT NULL' ,
filename : 'TEXT NOT NULL DEFAULT ""' ,
created _time : 'INT NOT NULL' ,
updated _time : 'INT NOT NULL' ,
user _created _time : 'INT NOT NULL DEFAULT 0' ,
user _updated _time : 'INT NOT NULL DEFAULT 0' ,
file _extension : 'TEXT NOT NULL DEFAULT ""' ,
encryption _cipher _text : 'TEXT NOT NULL DEFAULT ""' ,
encryption _applied : 'INT NOT NULL DEFAULT 0' ,
encryption _blob _encrypted : 'INT NOT NULL DEFAULT 0' ,
} )
) ;
2018-11-13 02:45:08 +02:00
}
2018-12-10 20:58:49 +02:00
if ( targetVersion == 15 ) {
queries . push ( 'CREATE VIRTUAL TABLE notes_fts USING fts4(content="notes", notindexed="id", id, title, body)' ) ;
queries . push ( 'INSERT INTO notes_fts(docid, id, title, body) SELECT rowid, id, title, body FROM notes WHERE is_conflict = 0 AND encryption_applied = 0' ) ;
2018-12-12 23:40:05 +02:00
// Keep the content tables (notes) and the FTS table (notes_fts) in sync.
// More info at https://www.sqlite.org/fts3.html#_external_content_fts4_tables_
queries . push ( `
CREATE TRIGGER notes _fts _before _update BEFORE UPDATE ON notes BEGIN
DELETE FROM notes _fts WHERE docid = old . rowid ;
END ; ` );
queries . push ( `
CREATE TRIGGER notes _fts _before _delete BEFORE DELETE ON notes BEGIN
DELETE FROM notes _fts WHERE docid = old . rowid ;
END ; ` );
queries . push ( `
CREATE TRIGGER notes _after _update AFTER UPDATE ON notes BEGIN
INSERT INTO notes _fts ( docid , id , title , body ) SELECT rowid , id , title , body FROM notes WHERE is _conflict = 0 AND encryption _applied = 0 AND new . rowid = notes . rowid ;
END ; ` );
queries . push ( `
CREATE TRIGGER notes _after _insert AFTER INSERT ON notes BEGIN
INSERT INTO notes _fts ( docid , id , title , body ) SELECT rowid , id , title , body FROM notes WHERE is _conflict = 0 AND encryption _applied = 0 AND new . rowid = notes . rowid ;
END ; ` );
2018-12-10 20:58:49 +02:00
}
2019-01-18 19:56:56 +02:00
if ( targetVersion == 18 ) {
2018-12-29 21:19:18 +02:00
const notesNormalized = `
CREATE TABLE notes _normalized (
id TEXT NOT NULL ,
title TEXT NOT NULL DEFAULT "" ,
body TEXT NOT NULL DEFAULT ""
) ;
` ;
queries . push ( this . sqlStringToLines ( notesNormalized ) [ 0 ] ) ;
queries . push ( 'CREATE INDEX notes_normalized_id ON notes_normalized (id)' ) ;
queries . push ( 'DROP TRIGGER IF EXISTS notes_fts_before_update' ) ;
queries . push ( 'DROP TRIGGER IF EXISTS notes_fts_before_delete' ) ;
queries . push ( 'DROP TRIGGER IF EXISTS notes_after_update' ) ;
queries . push ( 'DROP TRIGGER IF EXISTS notes_after_insert' ) ;
queries . push ( 'DROP TABLE IF EXISTS notes_fts' ) ;
queries . push ( 'CREATE VIRTUAL TABLE notes_fts USING fts4(content="notes_normalized", notindexed="id", id, title, body)' ) ;
// Keep the content tables (notes) and the FTS table (notes_fts) in sync.
// More info at https://www.sqlite.org/fts3.html#_external_content_fts4_tables_
queries . push ( `
CREATE TRIGGER notes _fts _before _update BEFORE UPDATE ON notes _normalized BEGIN
DELETE FROM notes _fts WHERE docid = old . rowid ;
END ; ` );
queries . push ( `
CREATE TRIGGER notes _fts _before _delete BEFORE DELETE ON notes _normalized BEGIN
DELETE FROM notes _fts WHERE docid = old . rowid ;
END ; ` );
queries . push ( `
CREATE TRIGGER notes _after _update AFTER UPDATE ON notes _normalized BEGIN
INSERT INTO notes _fts ( docid , id , title , body ) SELECT rowid , id , title , body FROM notes _normalized WHERE new . rowid = notes _normalized . rowid ;
END ; ` );
queries . push ( `
CREATE TRIGGER notes _after _insert AFTER INSERT ON notes _normalized BEGIN
INSERT INTO notes _fts ( docid , id , title , body ) SELECT rowid , id , title , body FROM notes _normalized WHERE new . rowid = notes _normalized . rowid ;
END ; ` );
}
2019-05-06 22:35:29 +02:00
if ( targetVersion == 19 ) {
const newTableSql = `
CREATE TABLE revisions (
id TEXT PRIMARY KEY ,
parent _id TEXT NOT NULL DEFAULT "" ,
item _type INT NOT NULL ,
item _id TEXT NOT NULL ,
item _updated _time INT NOT NULL ,
title _diff TEXT NOT NULL DEFAULT "" ,
body _diff TEXT NOT NULL DEFAULT "" ,
metadata _diff TEXT NOT NULL DEFAULT "" ,
encryption _cipher _text TEXT NOT NULL DEFAULT "" ,
encryption _applied INT NOT NULL DEFAULT 0 ,
updated _time INT NOT NULL ,
created _time INT NOT NULL
) ;
` ;
queries . push ( this . sqlStringToLines ( newTableSql ) [ 0 ] ) ;
queries . push ( 'CREATE INDEX revisions_parent_id ON revisions (parent_id)' ) ;
queries . push ( 'CREATE INDEX revisions_item_type ON revisions (item_type)' ) ;
queries . push ( 'CREATE INDEX revisions_item_id ON revisions (item_id)' ) ;
queries . push ( 'CREATE INDEX revisions_item_updated_time ON revisions (item_updated_time)' ) ;
queries . push ( 'CREATE INDEX revisions_updated_time ON revisions (updated_time)' ) ;
queries . push ( 'ALTER TABLE item_changes ADD COLUMN source INT NOT NULL DEFAULT 1' ) ;
queries . push ( 'ALTER TABLE item_changes ADD COLUMN before_change_item TEXT NOT NULL DEFAULT ""' ) ;
}
2019-05-11 18:55:40 +02:00
if ( targetVersion == 20 ) {
const newTableSql = `
CREATE TABLE migrations (
id INTEGER PRIMARY KEY ,
number INTEGER NOT NULL ,
updated _time INT NOT NULL ,
created _time INT NOT NULL
) ;
` ;
queries . push ( this . sqlStringToLines ( newTableSql ) [ 0 ] ) ;
queries . push ( 'ALTER TABLE resources ADD COLUMN `size` INT NOT NULL DEFAULT -1' ) ;
2019-12-29 19:58:40 +02:00
queries . push ( this . addMigrationFile ( 20 ) ) ;
2019-05-11 18:55:40 +02:00
}
2019-05-12 02:15:52 +02:00
if ( targetVersion == 21 ) {
queries . push ( 'ALTER TABLE sync_items ADD COLUMN item_location INT NOT NULL DEFAULT 1' ) ;
}
2019-05-22 16:56:07 +02:00
if ( targetVersion == 22 ) {
const newTableSql = `
CREATE TABLE resources _to _download (
id INTEGER PRIMARY KEY ,
resource _id TEXT NOT NULL ,
updated _time INT NOT NULL ,
created _time INT NOT NULL
) ;
` ;
queries . push ( this . sqlStringToLines ( newTableSql ) [ 0 ] ) ;
queries . push ( 'CREATE INDEX resources_to_download_resource_id ON resources_to_download (resource_id)' ) ;
queries . push ( 'CREATE INDEX resources_to_download_updated_time ON resources_to_download (updated_time)' ) ;
}
2019-06-07 10:05:15 +02:00
if ( targetVersion == 23 ) {
const newTableSql = `
CREATE TABLE key _values (
id INTEGER PRIMARY KEY ,
\ ` key \` TEXT NOT NULL,
\ ` value \` TEXT NOT NULL,
\ ` type \` INT NOT NULL,
updated _time INT NOT NULL
) ;
` ;
queries . push ( this . sqlStringToLines ( newTableSql ) [ 0 ] ) ;
queries . push ( 'CREATE UNIQUE INDEX key_values_key ON key_values (key)' ) ;
}
2019-07-14 17:00:02 +02:00
if ( targetVersion == 24 ) {
queries . push ( 'ALTER TABLE notes ADD COLUMN `markup_language` INT NOT NULL DEFAULT 1' ) ; // 1: Markdown, 2: HTML
}
2019-11-11 08:14:56 +02:00
if ( targetVersion == 25 ) {
queries . push ( ` CREATE VIEW tags_with_note_count AS
SELECT tags . id as id , tags . title as title , tags . created _time as created _time , tags . updated _time as updated _time , COUNT ( notes . id ) as note _count
FROM tags
LEFT JOIN note _tags nt on nt . tag _id = tags . id
LEFT JOIN notes on notes . id = nt . note _id
WHERE notes . id IS NOT NULL
GROUP BY tags . id ` );
}
2019-12-17 14:45:57 +02:00
if ( targetVersion == 26 ) {
const tableNames = [ 'notes' , 'folders' , 'tags' , 'note_tags' , 'resources' ] ;
for ( let i = 0 ; i < tableNames . length ; i ++ ) {
const n = tableNames [ i ] ;
queries . push ( ` ALTER TABLE ${ n } ADD COLUMN is_shared INT NOT NULL DEFAULT 0 ` ) ;
}
}
2019-12-29 19:58:40 +02:00
if ( targetVersion == 27 ) {
queries . push ( this . addMigrationFile ( 27 ) ) ;
}
2018-03-09 22:59:12 +02:00
queries . push ( { sql : 'UPDATE version SET version = ?' , params : [ targetVersion ] } ) ;
2017-07-19 21:15:55 +02:00
2018-12-28 22:40:29 +02:00
try {
await this . transactionExecBatch ( queries ) ;
} catch ( error ) {
2019-01-18 19:56:56 +02:00
if ( targetVersion === 15 || targetVersion === 18 ) {
this . logger ( ) . warn ( 'Could not upgrade to database v15 or v18 - FTS feature will not be used' , error ) ;
2018-12-28 22:40:29 +02:00
} else {
throw error ;
}
}
latestVersion = targetVersion ;
2019-07-29 15:43:53 +02:00
2017-07-19 21:15:55 +02:00
currentVersionIndex ++ ;
}
2018-12-28 22:40:29 +02:00
return latestVersion ;
}
async ftsEnabled ( ) {
try {
await this . selectOne ( 'SELECT count(*) FROM notes_fts' ) ;
} catch ( error ) {
this . logger ( ) . warn ( 'FTS check failed' , error ) ;
return false ;
}
this . logger ( ) . info ( 'FTS check succeeded' ) ;
2017-07-19 21:15:55 +02:00
return true ;
}
2018-12-28 22:40:29 +02:00
version ( ) {
return this . version _ ;
}
2017-07-06 21:48:17 +02:00
async initialize ( ) {
2018-03-09 22:59:12 +02:00
this . logger ( ) . info ( 'Checking for database schema update...' ) ;
2017-07-06 21:48:17 +02:00
2017-07-26 19:49:01 +02:00
let versionRow = null ;
try {
// Will throw if the database has not been created yet, but this is handled below
2018-03-09 22:59:12 +02:00
versionRow = await this . selectOne ( 'SELECT * FROM version LIMIT 1' ) ;
2017-07-26 19:49:01 +02:00
} catch ( error ) {
2019-07-29 15:43:53 +02:00
if ( error . message && error . message . indexOf ( 'no such table: version' ) >= 0 ) {
2017-07-26 22:00:16 +02:00
// Ignore
} else {
console . info ( error ) ;
}
2017-07-26 19:49:01 +02:00
}
2017-07-06 21:48:17 +02:00
2017-07-26 19:49:01 +02:00
const version = ! versionRow ? 0 : versionRow . version ;
2018-12-28 22:40:29 +02:00
this . version _ = version ;
2018-03-09 22:59:12 +02:00
this . logger ( ) . info ( 'Current database version' , version ) ;
2017-07-06 21:48:17 +02:00
2018-12-28 22:40:29 +02:00
const newVersion = await this . upgradeDatabase ( version ) ;
this . version _ = newVersion ;
if ( newVersion !== version ) await this . refreshTableFields ( ) ;
2017-07-06 21:48:17 +02:00
2017-07-26 19:49:01 +02:00
this . tableFields _ = { } ;
2017-07-06 21:48:17 +02:00
2018-03-09 22:59:12 +02:00
let rows = await this . selectAll ( 'SELECT * FROM table_fields' ) ;
2017-07-06 21:48:17 +02:00
2017-07-26 19:49:01 +02:00
for ( let i = 0 ; i < rows . length ; i ++ ) {
let row = rows [ i ] ;
if ( ! this . tableFields _ [ row . table _name ] ) this . tableFields _ [ row . table _name ] = [ ] ;
this . tableFields _ [ row . table _name ] . push ( {
name : row . field _name ,
type : row . field _type ,
default : Database . formatValue ( row . field _type , row . field _default ) ,
} ) ;
2017-07-06 21:48:17 +02:00
}
}
}
Database . TYPE _INT = 1 ;
Database . TYPE _TEXT = 2 ;
Database . TYPE _NUMERIC = 3 ;
2019-07-29 15:43:53 +02:00
module . exports = { JoplinDatabase } ;