Server v2.6.14

.github/workflows/github-actions-main.yml

@@ -5,7 +5,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-latest, ubuntu-latest, windows-2016]
+        os: [macos-latest, ubuntu-latest, windows-latest]
     steps:
 
       # Silence apt-get update errors (for example when a module doesn't

docker-compose.db-dev.yml

@@ -6,7 +6,7 @@ version: '3'
 
 services:
     db:
-        image: postgres:13.1
+        image: postgres:13
         command: postgres -c work_mem=100000
         ports:
             - "5432:5432"
@@ -14,7 +14,7 @@ services:
             - POSTGRES_PASSWORD=joplin
             - POSTGRES_USER=joplin
             - POSTGRES_DB=joplin
-        
+     
         # Use this to specify additional Postgres
         # config parameters:
         #

docker-compose.server-dev.yml

@@ -18,7 +18,7 @@ services:
             - POSTGRES_PORT=5432
             - POSTGRES_HOST=localhost
     db:
-        image: postgres:13.1
+        image: postgres:13
         ports:
             - "5432:5432"
         environment:

docker-compose.server.yml

@@ -8,7 +8,7 @@ version: '3'
 
 services:
     db:
-        image: postgres:13.1
+        image: postgres:13
         volumes:
             - ./data/postgres:/var/lib/postgresql/data
         ports:

packages/app-mobile/android/app/build.gradle

@@ -146,8 +146,8 @@ android {
         applicationId "net.cozic.joplin"
         minSdkVersion rootProject.ext.minSdkVersion
         targetSdkVersion rootProject.ext.targetSdkVersion
-		versionCode 2097659
-		versionName "2.6.3"
+		versionCode 2097660
+		versionName "2.6.4"
 		ndk {
 			abiFilters "armeabi-v7a", "x86", "arm64-v8a", "x86_64"
 		}
@@ -215,6 +215,16 @@ android {
 
         }
     }
+
+    // To fix this error:
+    //
+    // > Execution failed for task ':app:lintVitalRelease'
+    //
+    // https://stackoverflow.com/a/62603296/561309
+    lintOptions {
+        disable 'InvalidPackage'
+        checkReleaseBuilds false
+    }
 }
 
 dependencies {

packages/app-mobile/android/gradle.properties

@@ -27,3 +27,10 @@ android.enableJetifier=true
 
 # Version of flipper SDK to use with React Native
 FLIPPER_VERSION=0.99.0
+
+# To fix this error:
+#
+# > Failed to transform bcprov-jdk15on-1.68.jar
+#
+# https://github.com/robolectric/robolectric/issues/6521
+android.jetifier.ignorelist=bcprov

packages/lib/BaseApplication.ts

@@ -258,6 +258,20 @@ export default class BaseApplication {
 				continue;
 			}
 
+			if (arg.indexOf('--enable-features=') === 0) {
+				// Electron-specific flag - ignore it
+				// Allows users to run the app on native wayland
+				argv.splice(0, 1);
+				continue;
+			}
+
+			if (arg.indexOf('--ozone-platform=') === 0) {
+				// Electron-specific flag - ignore it
+				// Allows users to run the app on native wayland
+				argv.splice(0, 1);
+				continue;
+			}
+
 			if (arg.length && arg[0] == '-') {
 				throw new JoplinError(_('Unknown flag: %s', arg), 'flagError');
 			} else {

packages/server/jest.config.js

@@ -10,7 +10,7 @@ module.exports = {
 
 	testEnvironment: 'node',
 
-	slowTestThreshold: 40,
+	slowTestThreshold: 60,
 
 	setupFilesAfterEnv: [`${__dirname}/jest.setup.js`],
 };

packages/server/package-lock.json

@@ -1,12 +1,12 @@
 {
 	"name": "@joplin/server",
-	"version": "2.6.13",
+	"version": "2.6.14",
 	"lockfileVersion": 2,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "@joplin/server",
-			"version": "2.6.13",
+			"version": "2.6.14",
 			"dependencies": {
 				"@aws-sdk/client-s3": "^3.40.0",
 				"@fortawesome/fontawesome-free": "^5.15.1",

packages/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@joplin/server",
-  "version": "2.6.13",
+  "version": "2.6.14",
   "private": true,
   "scripts": {
     "start-dev": "npm run build && JOPLIN_IS_TESTING=1 nodemon --config nodemon.json --ext ts,js,mustache,css,tsx dist/app.js --env dev",

packages/server/src/commands/StorageCommand.ts

@@ -14,9 +14,10 @@ enum ArgvCommand {
 
 interface Argv {
 	command: ArgvCommand;
-	connection: string;
+	connection?: string;
 	batchSize?: number;
 	maxContentSize?: number;
+	maxProcessedItems?: number;
 }
 
 export default class StorageCommand extends BaseCommand {
@@ -52,6 +53,10 @@ export default class StorageCommand extends BaseCommand {
 				type: 'number',
 				description: 'Max content size',
 			},
+			'max-processed-items': {
+				type: 'number',
+				description: 'Max number of items to process before stopping',
+			},
 			'connection': {
 				description: 'storage connection string',
 				type: 'string',
@@ -85,11 +90,14 @@ export default class StorageCommand extends BaseCommand {
 			},
 
 			[ArgvCommand.DeleteDatabaseContentColumn]: async () => {
+				const maxProcessedItems = argv.maxProcessedItems;
+
 				logger.info(`Batch size: ${batchSize}`);
 
 				await runContext.models.item().deleteDatabaseContentColumn({
 					batchSize,
 					logger,
+					maxProcessedItems,
 				});
 			},
 		};

packages/server/src/models/ItemModel.test.ts

@@ -462,6 +462,41 @@ describe('ItemModel', function() {
 		expect(await models().item().dbContent(note1.id)).toEqual(Buffer.from(''));
 	});
 
+	test('should delete the database item content - maxProcessedItems handling', async function() {
+		if (isSqlite(db())) {
+			expect(1).toBe(1);
+			return;
+		}
+
+		const { user: user1 } = await createUserAndSession(1);
+
+		await createItemTree3(user1.id, '', '', [
+			{
+				id: '000000000000000000000000000000F1',
+				children: [
+					{ id: '00000000000000000000000000000001' },
+					{ id: '00000000000000000000000000000002' },
+					{ id: '00000000000000000000000000000003' },
+					{ id: '00000000000000000000000000000004' },
+				],
+			},
+		]);
+
+		await models().item().deleteDatabaseContentColumn({ batchSize: 2, maxProcessedItems: 4 });
+
+		const itemIds = (await models().item().all()).map(it => it.id);
+		const contents = await Promise.all([
+			models().item().dbContent(itemIds[0]),
+			models().item().dbContent(itemIds[1]),
+			models().item().dbContent(itemIds[2]),
+			models().item().dbContent(itemIds[3]),
+			models().item().dbContent(itemIds[4]),
+		]);
+
+		const emptyOnes = contents.filter(c => c.toString() === '');
+		expect(emptyOnes.length).toBe(4);
+	});
+
 	// test('should stop importing item if it has been deleted', async function() {
 	// 	const { user: user1 } = await createUserAndSession(1);
 

packages/server/src/models/ItemModel.ts

@@ -30,6 +30,7 @@ export interface ImportContentToStorageOptions {
 export interface DeleteDatabaseContentOptions {
 	batchSize?: number;
 	logger?: Logger | LoggerWrapper;
+	maxProcessedItems?: number;
 }
 
 export interface SaveFromRawContentItem {
@@ -409,20 +410,21 @@ export default class ItemModel extends BaseModel<Item> {
 		options = {
 			batchSize: 1000,
 			logger: new Logger(),
+			maxProcessedItems: 0,
 			...options,
 		};
 
-		const itemCount = (await this.db(this.tableName)
-			.count('id', { as: 'total' })
-			.where('content', '!=', Buffer.from(''))
-			.first())['total'];
+		// const itemCount = (await this.db(this.tableName)
+		// 	.count('id', { as: 'total' })
+		// 	.where('content', '!=', Buffer.from(''))
+		// 	.first())['total'];
 
 		let totalDone = 0;
 
 		// UPDATE items SET content = '\x' WHERE id IN (SELECT id FROM items WHERE content != '\x' LIMIT 5000);
 
 		while (true) {
-			options.logger.info(`Processing items ${totalDone} / ${itemCount}`);
+			options.logger.info(`Processing items ${totalDone}`);
 
 			const updatedRows = await this
 				.db(this.tableName)
@@ -440,6 +442,11 @@ export default class ItemModel extends BaseModel<Item> {
 				return;
 			}
 
+			if (options.maxProcessedItems && totalDone + options.batchSize > options.maxProcessedItems) {
+				options.logger.info(`Processed ${totalDone} items out of requested ${options.maxProcessedItems}`);
+				return;
+			}
+
 			await msleep(1000);
 		}
 	}

packages/tools/tool-utils.ts

@@ -6,6 +6,20 @@ const execa = require('execa');
 const { splitCommandString } = require('@joplin/lib/string-utils');
 const moment = require('moment');
 
+export interface GitHubReleaseAsset {
+	name: string;
+	browser_download_url: string;
+}
+
+export interface GitHubRelease {
+	assets: GitHubReleaseAsset[];
+	tag_name: string;
+	upload_url: string;
+	html_url: string;
+	prerelease: boolean;
+	draft: boolean;
+}
+
 function quotePath(path: string) {
 	if (!path) return '';
 	if (path.indexOf('"') < 0 && path.indexOf(' ') < 0) return path;
@@ -364,7 +378,39 @@ export function githubOauthToken() {
 	return readCredentialFile('github_oauth_token.txt');
 }
 
-export async function githubRelease(project: string, tagName: string, options: any = null) {
+// Note that the GitHub API releases/latest is broken on the joplin-android repo
+// as of Nov 2021 (last working on 3 November 2021, first broken on 19
+// November). It used to return the latest **published** release but now it
+// retuns... some release, always the same one, but not the latest one. GitHub
+// says that nothing has changed on the API, although it used to work. So since
+// we can't use /latest anymore, we need to fetch all the releases to find the
+// latest published one.
+export async function gitHubLatestRelease(repoName: string): Promise<GitHubRelease> {
+	let pageNum = 1;
+
+	while (true) {
+		const response: any = await fetch(`https://api.github.com/repos/laurent22/${repoName}/releases?page=${pageNum}`, {
+			headers: {
+				'Content-Type': 'application/json',
+				'User-Agent': 'Joplin Readme Updater',
+			},
+		});
+
+		if (!response.ok) throw new Error(`Cannot fetch releases: ${response.statusText}`);
+
+		const releases = await response.json();
+		if (!releases.length) throw new Error('Cannot find latest release');
+
+		for (const release of releases) {
+			if (release.prerelease || release.draft) continue;
+			return release;
+		}
+
+		pageNum++;
+	}
+}
+
+export async function githubRelease(project: string, tagName: string, options: any = null): Promise<GitHubRelease> {
 	options = Object.assign({}, {
 		isDraft: false,
 		isPreRelease: false,

packages/tools/update-readme-download.ts

@@ -1,18 +1,8 @@
 import * as fs from 'fs-extra';
 import { fileExtension } from '@joplin/lib/path-utils';
-const request = require('request');
+import { gitHubLatestRelease, GitHubRelease } from './tool-utils';
 const readmePath = `${__dirname}/../../README.md`;
 
-interface GitHubReleaseAsset {
-	name: string;
-	browser_download_url: string;
-}
-
-interface GitHubRelease {
-	assets: GitHubReleaseAsset[];
-	tag_name: string;
-}
-
 async function msleep(ms: number) {
 	return new Promise((resolve) => {
 		setTimeout(() => {
@@ -21,25 +11,6 @@ async function msleep(ms: number) {
 	});
 }
 
-async function gitHubLatestRelease(repoName: string): Promise<GitHubRelease> {
-	return new Promise((resolve, reject) => {
-		request.get({
-			url: `https://api.github.com/repos/laurent22/${repoName}/releases/latest`,
-			json: true,
-			headers: { 'User-Agent': 'Joplin Readme Updater' },
-		}, (error: any, response: any, data: any) => {
-			if (error) {
-				reject(error);
-			} else if (response.statusCode !== 200) {
-				console.warn(data);
-				reject(new Error(`Error HTTP ${response.statusCode}`));
-			} else {
-				resolve(data);
-			}
-		});
-	});
-}
-
 function downloadUrl(release: GitHubRelease, os: string, portable = false) {
 	if (!release || !release.assets || !release.assets.length) return null;
 
@@ -122,8 +93,8 @@ async function main(argv: any) {
 	// Disable for now due to broken /latest API end point, which returns a
 	// version from 6 months ago.
 
-	// if (androidUrl) content = content.replace(/(https:\/\/github.com\/laurent22\/joplin-android\/releases\/download\/android-v\d+\.\d+\.\d+\/joplin-v\d+\.\d+\.\d+\.apk)/, androidUrl);
-	// if (android32Url) content = content.replace(/(https:\/\/github.com\/laurent22\/joplin-android\/releases\/download\/android-v\d+\.\d+\.\d+\/joplin-v\d+\.\d+\.\d+-32bit\.apk)/, android32Url);
+	if (androidUrl) content = content.replace(/(https:\/\/github.com\/laurent22\/joplin-android\/releases\/download\/android-v\d+\.\d+\.\d+\/joplin-v\d+\.\d+\.\d+\.apk)/, androidUrl);
+	if (android32Url) content = content.replace(/(https:\/\/github.com\/laurent22\/joplin-android\/releases\/download\/android-v\d+\.\d+\.\d+\/joplin-v\d+\.\d+\.\d+-32bit\.apk)/, android32Url);
 
 	setReadmeContent(content);
 

readme/changelog_android.md

@@ -1,5 +1,14 @@
 # Joplin Android app changelog
 
+## [android-v2.6.4](https://github.com/laurent22/joplin/releases/tag/android-v2.6.4) (Pre-release) - 2021-12-01T11:38:49Z
+
+- Improved: Also duplicate resources when duplicating a note (c0a8c33)
+- Improved: Improved S3 sync error handling and reliability, and upgraded S3 SDK (#5312 by Lee Matos)
+- Fixed: Alarm setting buttons were no longer visible (#5777)
+- Fixed: Alarms were not being triggered in some cases (#5798) (#5216 by Roman Musin)
+- Fixed: Fixed opening attachments (6950c40)
+- Fixed: Handle duplicate attachments when the parent notebook is shared (#5796)
+
 ## [android-v2.6.3](https://github.com/laurent22/joplin/releases/tag/android-v2.6.3) (Pre-release) - 2021-11-21T16:59:46Z
 
 - New: Add date format YYYY/MM/DD (#5759 by Helmut K. C. Tessarek)

readme/changelog_server.md

@@ -1,5 +1,9 @@
 # Joplin Server Changelog
 
+## [server-v2.6.14](https://github.com/laurent22/joplin/releases/tag/server-v2.6.14) - 2021-12-02T16:29:54Z
+
+- Improved: Improved storage command (122afd6)
+
 ## [server-v2.6.13](https://github.com/laurent22/joplin/releases/tag/server-v2.6.13) - 2021-11-29T18:41:28Z
 
 - New: Added command to delete database item content (01048f5)