chore(mobile): add login integration tests and reorganize CI definitions (#1417)

* Add integration tests for the login process

* Reorganize tests

* Test wrong instance URL

* Run mobile unit tests in CI

* Fix CI

* Pin Flutter Version to 3.3.10

* Push something stupid to re-trigger CI

.github/workflows/test.yml

@@ -39,3 +39,56 @@ jobs:
 
       - name: Run tests
         run: cd web && npm ci && npm run check:all
+
+  mobile-unit-tests:
+    name: Run mobile unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Flutter SDK
+        uses: subosito/flutter-action@v2
+        with:
+          channel: 'stable'
+          flutter-version: '3.3.10'
+      - name: Run tests
+        working-directory: ./mobile
+        run: flutter test
+
+  mobile-integration-tests:
+    name: Run mobile end-to-end integration tests
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'adopt'
+          java-version: '11'
+      - name: Cache android SDK
+        uses: actions/cache@v3
+        id: android-sdk
+        with:
+          key: android-sdk
+          path: |
+            /usr/local/lib/android/
+            ~/.android
+      - name: Setup Android SDK
+        if: steps.android-sdk.outputs.cache-hit != 'true'
+        uses: android-actions/setup-android@v2
+      - name: Setup Flutter SDK
+        uses: subosito/flutter-action@v2
+        with:
+          channel: 'stable'
+          flutter-version: '3.3.10'
+      - name: Run integration tests
+        uses: reactivecircus/android-emulator-runner@v2.27.0
+        with:
+          working-directory: ./mobile
+          api-level: 29
+          arch: x86_64
+          profile: pixel
+          target: default
+          emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim
+          disable-linux-hw-accel: false
+          script: |
+            flutter pub get
+            flutter test integration_test

.github/workflows/test_mobile.yml

@@ -1,44 +0,0 @@
-name: Flutter Integration Tests
-
-on:
-  push:
-    branches: [ "main" ]
-  pull_request:
-
-jobs:
-  build:
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-java@v3
-        with:
-          distribution: 'adopt'
-          java-version: '11'
-      - name: Cache android SDK
-        uses: actions/cache@v3
-        id: android-sdk
-        with:
-          key: android-sdk
-          path: |
-            /usr/local/lib/android/
-            ~/.android
-      - name: Setup Android SDK
-        if: steps.android-sdk.outputs.cache-hit != 'true'
-        uses: android-actions/setup-android@v2
-      - name: Setup Flutter SDK
-        uses: subosito/flutter-action@v2
-        with:
-          channel: 'stable'
-      - name: Run integration tests
-        uses: reactivecircus/android-emulator-runner@v2.27.0
-        with:
-          working-directory: ./mobile
-          api-level: 29
-          arch: x86_64
-          profile: pixel
-          target: default
-          emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim
-          disable-linux-hw-accel: false
-          script: |
-            flutter pub get
-            flutter test integration_test

mobile/assets/i18n/en-US.json

@@ -110,6 +110,8 @@
   "experimental_settings_title": "Experimental",
   "home_page_add_to_album_conflicts": "Added {added} assets to album {album}. {failed} assets are already in the album.",
   "home_page_add_to_album_success": "Added {added} assets to album {album}.",
+  "home_page_building_timeline": "Building the timeline",
+  "home_page_first_time_notice": "If this is your first time using the app, please make sure to choose a backup album(s) so that the timeline can populate photos and videos in the album(s).",
   "library_page_albums": "Albums",
   "library_page_new_album": "New album",
   "login_form_button_text": "Login",

mobile/integration_test/module_login/login_input_validation_test.dart

@@ -8,12 +8,11 @@ void main() async {
   await ImmichTestHelper.initialize();
 
   group("Login input validation test", () {
-    immichWidgetTest("Test leading/trailing whitespace", (tester) async {
+    immichWidgetTest("Test leading/trailing whitespace", (tester, helper) async {
-      await ImmichTestLoginHelper.waitForLoginScreen(tester);
+      await helper.loginHelper.waitForLoginScreen();
-      await ImmichTestLoginHelper.acknowledgeNewServerVersion(tester);
+      await helper.loginHelper.acknowledgeNewServerVersion();
 
-      await ImmichTestLoginHelper.enterLoginCredentials(
+      await helper.loginHelper.enterCredentials(
-        tester,
         email: " demo@immich.app"
       );
 
@@ -21,8 +20,7 @@ void main() async {
 
       expect(find.text("login_form_err_leading_whitespace".tr()), findsOneWidget);
 
-      await ImmichTestLoginHelper.enterLoginCredentials(
+      await helper.loginHelper.enterCredentials(
-          tester,
           email: "demo@immich.app "
       );
 
@@ -31,12 +29,11 @@ void main() async {
       expect(find.text("login_form_err_trailing_whitespace".tr()), findsOneWidget);
     });
 
-    immichWidgetTest("Test invalid email", (tester) async {
+    immichWidgetTest("Test invalid email", (tester, helper) async {
-      await ImmichTestLoginHelper.waitForLoginScreen(tester);
+      await helper.loginHelper.waitForLoginScreen();
-      await ImmichTestLoginHelper.acknowledgeNewServerVersion(tester);
+      await helper.loginHelper.acknowledgeNewServerVersion();
 
-      await ImmichTestLoginHelper.enterLoginCredentials(
+      await helper.loginHelper.enterCredentials(
-          tester,
           email: "demo.immich.app"
       );
 

mobile/integration_test/module_login/login_test.dart

@@ -0,0 +1,39 @@
+import 'dart:io';
+
+import 'package:flutter_test/flutter_test.dart';
+
+import '../test_utils/general_helper.dart';
+import '../test_utils/login_helper.dart';
+
+void main() async {
+  await ImmichTestHelper.initialize();
+
+  group("Login tests", () {
+    immichWidgetTest("Test correct credentials", (tester, helper) async {
+      await helper.loginHelper.waitForLoginScreen();
+      await helper.loginHelper.acknowledgeNewServerVersion();
+      await helper.loginHelper
+          .enterCredentialsOf(LoginCredentials.testInstance);
+      await helper.loginHelper.pressLoginButton();
+      await helper.loginHelper.assertLoginSuccess();
+    });
+
+    immichWidgetTest("Test login with wrong password", (tester, helper) async {
+      await helper.loginHelper.waitForLoginScreen();
+      await helper.loginHelper.acknowledgeNewServerVersion();
+      await helper.loginHelper.enterCredentialsOf(
+          LoginCredentials.testInstanceButWithWrongPassword);
+      await helper.loginHelper.pressLoginButton();
+      await helper.loginHelper.assertLoginFailed();
+    });
+
+    immichWidgetTest("Test login with wrong server URL", (tester, helper) async {
+      await helper.loginHelper.waitForLoginScreen();
+      await helper.loginHelper.acknowledgeNewServerVersion();
+      await helper.loginHelper.enterCredentialsOf(
+          LoginCredentials.wrongInstanceUrl);
+      await helper.loginHelper.pressLoginButton();
+      await helper.loginHelper.assertLoginFailed();
+    });
+  });
+}

mobile/integration_test/test_utils/general_helper.dart

@@ -1,14 +1,28 @@
 
 import 'package:easy_localization/easy_localization.dart';
+import 'package:flutter/foundation.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:hive/hive.dart';
 import 'package:immich_mobile/main.dart';
 import 'package:integration_test/integration_test.dart';
-
+import 'package:meta/meta.dart';
 import 'package:immich_mobile/main.dart' as app;
 
+import 'login_helper.dart';
+
 class ImmichTestHelper {
 
+  final WidgetTester tester;
+
+  ImmichTestHelper(this.tester);
+
+  ImmichTestLoginHelper? _loginHelper;
+
+  ImmichTestLoginHelper get loginHelper {
+    _loginHelper ??= ImmichTestLoginHelper(tester);
+    return _loginHelper!;
+  }
+
   static Future<IntegrationTestWidgetsFlutterBinding> initialize() async {
     final binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized();
     binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;
@@ -32,9 +46,12 @@ class ImmichTestHelper {
 
 }
 
-void immichWidgetTest(String description, Future<void> Function(WidgetTester) test) {
+@isTest
-  testWidgets(description, (widgetTester) async {
+void immichWidgetTest(String description, Future<void> Function(WidgetTester, ImmichTestHelper) test) {
-    await ImmichTestHelper.loadApp(widgetTester);
+
-    await test(widgetTester);
+    testWidgets(description, (widgetTester) async {
-  });
+        await ImmichTestHelper.loadApp(widgetTester);
+        await test(widgetTester, ImmichTestHelper(widgetTester));
+    }, semanticsEnabled: false);
+
 }

mobile/integration_test/test_utils/login_helper.dart

@@ -1,10 +1,15 @@
 import 'dart:async';
+import 'package:easy_localization/easy_localization.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_test/flutter_test.dart';
+import 'package:immich_mobile/modules/home/ui/asset_grid/immich_asset_grid.dart';
 
 class ImmichTestLoginHelper {
-  static Future<void> waitForLoginScreen(WidgetTester tester,
+  final WidgetTester tester;
-      {int timeoutSeconds = 20}) async {
+
+  ImmichTestLoginHelper(this.tester);
+
+  Future<void> waitForLoginScreen({int timeoutSeconds = 20}) async {
     for (var i = 0; i < timeoutSeconds; i++) {
       // Search for "IMMICH" test in the app bar
       final result = find.text("IMMICH");
@@ -21,7 +26,7 @@ class ImmichTestLoginHelper {
     fail("Timeout while waiting for login screen");
   }
 
-  static Future<bool> acknowledgeNewServerVersion(WidgetTester tester) async {
+  Future<bool> acknowledgeNewServerVersion() async {
     final result = find.text("Acknowledge");
     if (!tester.any(result)) {
       return false;
@@ -33,8 +38,7 @@ class ImmichTestLoginHelper {
     return true;
   }
 
-  static Future<void> enterLoginCredentials(
+  Future<void> enterCredentials({
-    WidgetTester tester, {
     String server = "",
     String email = "",
     String password = "",
@@ -50,6 +54,70 @@ class ImmichTestLoginHelper {
     await tester.pump(const Duration(milliseconds: 500));
     await tester.enterText(loginForms.at(2), server);
 
-    await tester.pump(const Duration(milliseconds: 500));
+    await tester.testTextInput.receiveAction(TextInputAction.done);
+    await tester.pumpAndSettle();
+  }
+
+  Future<void> enterCredentialsOf(LoginCredentials credentials) async {
+    await enterCredentials(
+      server: credentials.server,
+      email: credentials.email,
+      password: credentials.password,
+    );
+  }
+
+  Future<void> pressLoginButton() async {
+    final button = find.textContaining("login_form_button_text".tr());
+    await tester.tap(button);
+  }
+
+  Future<void> assertLoginSuccess({int timeoutSeconds = 15}) async {
+    for (var i = 0; i < timeoutSeconds * 2; i++) {
+      if (tester.any(find.text("home_page_building_timeline".tr()))) {
+        return;
+      }
+
+      await tester.pump(const Duration(milliseconds: 500));
+    }
+
+    fail("Login failed.");
+  }
+
+  Future<void> assertLoginFailed({int timeoutSeconds = 15}) async {
+    for (var i = 0; i < timeoutSeconds * 2; i++) {
+      if (tester.any(find.text("login_form_failed_login".tr()))) {
+        return;
+      }
+
+      await tester.pump(const Duration(milliseconds: 500));
+    }
+
+    fail("Timeout.");
   }
 }
+
+enum LoginCredentials {
+  testInstance(
+    "https://flutter-int-test.preview.immich.app",
+    "demo@immich.app",
+    "demo",
+  ),
+
+  testInstanceButWithWrongPassword(
+    "https://flutter-int-test.preview.immich.app",
+    "demo@immich.app",
+    "wrong",
+  ),
+
+  wrongInstanceUrl(
+  "https://does-not-exist.preview.immich.app",
+  "demo@immich.app",
+  "demo",
+  );
+
+  const LoginCredentials(this.server, this.email, this.password);
+
+  final String server;
+  final String email;
+  final String password;
+}

mobile/lib/modules/home/views/home_page.dart

@@ -2,6 +2,7 @@ import 'dart:async';
 
 import 'package:auto_route/auto_route.dart';
 import 'package:easy_localization/easy_localization.dart';
+import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_hooks/flutter_hooks.dart';
 import 'package:fluttertoast/fluttertoast.dart';
@@ -52,7 +53,10 @@ class HomePage extends HookConsumerWidget {
         });
 
         return () {
-          selectionEnabledHook.dispose();
+          // This does not work in tests
+          if (kReleaseMode) {
+            selectionEnabledHook.dispose();
+          }
         };
       },
       [],
@@ -162,28 +166,28 @@ class HomePage extends HookConsumerWidget {
               Padding(
                 padding: const EdgeInsets.only(top: 16.0),
                 child: Text(
-                  'Building the timeline',
+                  'home_page_building_timeline',
                   style: TextStyle(
                     fontWeight: FontWeight.w600,
                     fontSize: 16,
                     color: Theme.of(context).primaryColor,
                   ),
-                ),
+                ).tr(),
               ),
               AnimatedOpacity(
                 duration: const Duration(milliseconds: 500),
                 opacity: tipOneOpacity.value,
-                child: const SizedBox(
+                child: SizedBox(
                   width: 250,
                   child: Padding(
-                    padding: EdgeInsets.only(top: 8.0),
+                    padding: const EdgeInsets.only(top: 8.0),
-                    child: Text(
+                    child: const Text(
-                      'If this is your first time using the app, please make sure to choose a backup album(s) so that the timeline can populate photos and videos in the album(s).',
+                      'home_page_first_time_notice',
                       textAlign: TextAlign.justify,
                       style: TextStyle(
                         fontSize: 12,
                       ),
-                    ),
+                    ).tr(),
                   ),
                 ),
               )