test: migrate all packages from Flutter driver to integration tests (#9649)

* Remove Flutter Driver tests

* Remove `drive` dependency

* Add `integration_test` as dependency

* Setup integration test and migrate `firebase_core`

* Migrate `firebase_analytics`

* Migrate `cloud_functions`

* Migrate `firebase_app_check`

* Migrate `firebase_app_installations`

* Migrate `firebase_auth`

* Migrate `firebase_crashlytics`

* Migrate `firebase_database`

* Add missing `IntegrationTestWidgetsFlutterBinding` to `firebase_database`

* Migrate `firebase_dynamic_links`

* Migrate `firebase_messaging`

* Migrate `firebase_ml_model_downloader`

* Migrate `firebase_remote_config`

* Migrate `firebase_storage`

* Migrate `cloud_firestore`

* Migrate `cloud_firestore_odm_e2e.dart`

* Fix port and move to group for `cloud_firestore`

* Migrate `firebase_performance`

* Fix android build

* Update test commands

* Delete unused `cloud_functions` Flutter driver tests

* Delete unused `firebase_analytics` Flutter driver tests

* Delete unused `firebase_auth` Flutter driver tests

* Delete unused `firebase_core` Flutter driver tests

* Remove unused `dev_dependencies` in `firebase_in_app_messaging_example`

* Remove unused `dev_dependencies` in `firebase_ml_model_downloader`

* Remove unused `dev_dependencies` in `firebase_performance`

* Migrate `flutterfire_ui`

* Remove unused `dev_dependencies` in `flutterfire_ui_example`

* Sort dev dependencies in `tests`

* Remove unnecessary `test` dependency in `cloud_firestore`  

* Remove unnecessary `test` dependency in `cloud_firestore_example`  

* Sort dev dependencies in `cloud_firestore_odm`

* Remove unnecessary `test` and `flutter_driver` dependency in `cloud_firestore_example`  

* Remove unnecessary `test` dependency in `cloud_functions_example`  

* Remove unnecessary `test` dependency in `firebase_analytics_example`  

* Remove unnecessary `test` dependency in `firebase_app_check`  

* Remove unnecessary `test` dependency in `firebase_app_installations`  

* Remove unnecessary `test` & `flutter_driver` dependency in `firebase_app_installations_example`  

* Remove unnecessary `test` dependency in `firebase_auth`

* Remove unnecessary `test` dependency in `firebase_crashlytics`

* Remove unnecessary `test` dependency in `firebase_database`

* Remove unnecessary `test` dependency in `firebase_in_app_messaging`

* Remove unnecessary `test` dependency in `firebase_messaging`

* Remove unnecessary `test` dependency in `firebase_remote_config`

* Remove unnecessary `test` dependency in `firebase_storage`

* Remove `test:e2e` and `test:e2e:web` melos command

* Update `CONTRIBUTING.md` for running integration tests

* Remove drive commands from `tasks.json`

* Revert "Remove drive commands from `tasks.json`"

This reverts commit 915af70.

* Update `tasks.json`

* Fix `build_example.sh`

* Update `drive_example.sh`

* Remove unused `test_driver/integration/*` in `build.yaml` of `cloud_firestore_odm_generator`

* Revert "Remove `test:e2e` and `test:e2e:web` melos command"

This reverts commit e0af8b7.

* Bring back the `melos run test:e2e` and fix macos & web integration tests

* Add missing `--driver=./test_driver/integration_test.dart` in `tasks.json`

* Changed to sync `cloud_firestore_odm_e2e_test.dart`

* Check flakiness

* Skip "subscribeToTopic" and "unsubscribeFromTopic" for Android

* comment out `concurrency` for e2e tests

* Skip "times out when the provided timeout option is exceeded" test

* Remove testing stuff

* Remove `IntegrationTestWidgetsFlutterBinding.ensureInitialized()`

* Revert "Remove `IntegrationTestWidgetsFlutterBinding.ensureInitialized()`"

This reverts commit f117e20.

* Re-trigger CI

* Re-trigger CI

Author: nilsreichardt

.github/workflows/e2e_tests.yaml

@@ -80,7 +80,7 @@ jobs:
       - name: "Build Application"
         working-directory: tests
         run: |
-          flutter build apk --debug --target=./test_driver/driver_e2e.dart --dart-define=CI=true --no-android-gradle-daemon
+          flutter build apk --debug --target=./integration_test/e2e_test.dart --dart-define=CI=true --no-android-gradle-daemon
       - name: Start Firebase Emulator
         run: cd ./.github/workflows/scripts && ./start-firebase-emulator.sh
       - name: "E2E Tests"
@@ -98,7 +98,7 @@ jobs:
           script: |
             sleep 15
             $ANDROID_HOME/platform-tools/adb logcat '*:D' > adb-log.txt &
-            flutter drive --target=./test_driver/driver_e2e.dart --dart-define=CI=true
+            flutter test integration_test/e2e_test.dart --dart-define=CI=true
 
       - name: Compress Emulator Log
         if: always()
@@ -159,7 +159,7 @@ jobs:
           export CCACHE_DEPEND=true
           export CCACHE_INODECACHE=true
           ccache -s
-          flutter build ios --no-codesign --simulator --debug --target=./test_driver/driver_e2e.dart --dart-define=CI=true
+          flutter build ios --no-codesign --simulator --debug --target=./integration_test/e2e_test.dart --dart-define=CI=true
           ccache -s
       - name: Start Firebase Emulator
         run: cd ./.github/workflows/scripts && ./start-firebase-emulator.sh
@@ -174,7 +174,7 @@ jobs:
           sleep 15
           # Uncomment following line to have simulator logs printed out for debugging purposes.
           # xcrun simctl spawn booted log stream --predicate 'eventMessage contains "flutter"' &
-          flutter drive -d "$SIMULATOR" --target=./test_driver/driver_e2e.dart --dart-define=CI=true
+          flutter test integration_test/e2e_test.dart -d "$SIMULATOR" --dart-define=CI=true
           FLUTTER_DRIVE_EXIT_CODE=$?
           xcrun simctl shutdown "$SIMULATOR"
           exit $FLUTTER_DRIVE_EXIT_CODE
@@ -225,14 +225,17 @@ jobs:
           export CCACHE_DEPEND=true
           export CCACHE_INODECACHE=true
           ccache -s
-          flutter build macos --debug --target=./test_driver/driver_e2e.dart --device-id=macos --dart-define=CI=true
+          flutter build macos --debug --target=./integration_test/e2e_test.dart --device-id=macos --dart-define=CI=true
           ccache -s
       - name: Start Firebase Emulator
         run: cd ./.github/workflows/scripts && ./start-firebase-emulator.sh
       - name: "E2E Tests"
         working-directory: tests
         run: |
-          flutter drive -d macos --target=./test_driver/driver_e2e.dart --dart-define=CI=true
+          flutter test \
+            integration_test/e2e_test.dart \
+            -d macos \
+            --dart-define=CI=true
 
   web:
     runs-on: ubuntu-latest
@@ -263,6 +266,14 @@ jobs:
         run: cd ./.github/workflows/scripts && ./start-firebase-emulator.sh
       - name: "E2E Tests"
         working-directory: tests
+        # Web devices are not supported for the `flutter test` command yet. As a
+        # workaround we can use the `flutter drive` command. Tracking issue:
+        # https://github.com/flutter/flutter/issues/66264
         run: |
           chromedriver --port=4444 &
-          flutter drive --verbose-system-logs --device-id=web-server --target=./test_driver/driver_e2e.dart --dart-define=CI=true
+          flutter drive \
+            --verbose-system-logs \
+            -d web-server \
+            --driver=test_driver/integration_test.dart \
+            --target=integration_test/e2e_test.dart \
+            --dart-define=CI=true

.github/workflows/scripts/build-example.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-DEFAULT_TARGET="./test_driver/MELOS_PARENT_PACKAGE_NAME_e2e.dart"
+DEFAULT_TARGET="./integration_test/MELOS_PARENT_PACKAGE_NAME_e2e_test.dart"
 
 ACTION=$1
 TARGET_FILE=${2:-$DEFAULT_TARGET}

.github/workflows/scripts/drive-example.sh

@@ -6,8 +6,8 @@ if [ "$ACTION" == "android" ]
 then
   # Sleep to allow emulator to settle.
   sleep 15
-  melos exec -c 1 --fail-fast --scope="$FLUTTERFIRE_PLUGIN_SCOPE_EXAMPLE" --dir-exists=test_driver -- \
-    flutter drive $FLUTTER_COMMAND_FLAGS --no-pub --target=./test_driver/MELOS_PARENT_PACKAGE_NAME_e2e.dart --dart-define=CI=true
+  melos exec -c 1 --fail-fast --scope="$FLUTTERFIRE_PLUGIN_SCOPE_EXAMPLE" --dir-exists=integration_test -- \
+    flutter test integration_test/MELOS_PARENT_PACKAGE_NAME_e2e_test.dart $FLUTTER_COMMAND_FLAGS --no-pub --dart-define=CI=true
   exit
 fi
 
@@ -21,8 +21,8 @@ then
   sleep 15
   # Uncomment following line to have simulator logs printed out for debugging purposes.
   # xcrun simctl spawn booted log stream --predicate 'eventMessage contains "flutter"' &
-  melos exec -c 1 --fail-fast --scope="$FLUTTERFIRE_PLUGIN_SCOPE_EXAMPLE" --dir-exists=test_driver -- \
-    flutter drive $FLUTTER_COMMAND_FLAGS -d \"$SIMULATOR\" --no-pub --target=./test_driver/MELOS_PARENT_PACKAGE_NAME_e2e.dart --dart-define=CI=true
+  melos exec -c 1 --fail-fast --scope="$FLUTTERFIRE_PLUGIN_SCOPE_EXAMPLE" --dir-exists=integration_test -- \
+    flutter test integration_test/MELOS_PARENT_PACKAGE_NAME_e2e_test.dart $FLUTTER_COMMAND_FLAGS -d \"$SIMULATOR\" --no-pub --dart-define=CI=true
   MELOS_EXIT_CODE=$?
   xcrun simctl shutdown "$SIMULATOR"
   exit $MELOS_EXIT_CODE
@@ -31,7 +31,7 @@ fi
 if [ "$ACTION" == "macos" ]
 then
   melos exec -c 1 --fail-fast --scope="$FLUTTERFIRE_PLUGIN_SCOPE_EXAMPLE" --dir-exists=test_driver -- \
-    flutter drive $FLUTTER_COMMAND_FLAGS -d macos --no-pub --target=./test_driver/MELOS_PARENT_PACKAGE_NAME_e2e.dart --dart-define=CI=true
+    flutter test integration_test/MELOS_PARENT_PACKAGE_NAME_e2e_test.dart $FLUTTER_COMMAND_FLAGS -d macos --no-pub --dart-define=CI=true
   exit
 fi
 
@@ -40,6 +40,6 @@ then
   melos bootstrap --scope="*firebase_core*" --scope="$FLUTTERFIRE_PLUGIN_SCOPE"
   chromedriver --port=4444 &
   melos exec -c 1 --scope="$FLUTTERFIRE_PLUGIN_SCOPE_EXAMPLE" --dir-exists=web -- \
-    flutter drive $FLUTTER_COMMAND_FLAGS --no-pub --verbose-system-logs --device-id=web-server --target=./test_driver/MELOS_PARENT_PACKAGE_NAME_e2e.dart --dart-define=CI=true
+    flutter drive $FLUTTER_COMMAND_FLAGS --no-pub --verbose-system-logs --device-id=web-server --target=./integration_test/MELOS_PARENT_PACKAGE_NAME_e2e_test.dart --driver=./test_driver/integration_test.dart --dart-define=CI=true
   exit
 fi

.vscode/tasks.json

@@ -62,14 +62,14 @@
     {
       "label": "firestore -> test:e2e",
       "type": "shell",
-      "command": "melos exec -c 1 --scope=\"*cloud_firestore_example*\" --fail-fast --dir-exists=test_driver -- flutter drive --no-pub --target=./test_driver/MELOS_PARENT_PACKAGE_NAME_e2e.dart",
+      "command": "melos exec -c 1 --scope=\"*cloud_firestore_example*\" --fail-fast --dir-exists=integration_test -- flutter drive --no-pub --target=./integration_test/MELOS_PARENT_PACKAGE_NAME_e2e_test.dart",
       "detail": "Run end-to-end testing for Android & iOS via `flutter drive` on the Firestore example application",
       "problemMatcher": []
     },
     {
       "label": "firestore -> test:e2e:web",
       "type": "shell",
-      "command": "melos exec -c 1 --scope=\"*cloud_firestore_example*\" --dir-exists=web --fail-fast --dir-exists=test_driver -- flutter drive --release -d chrome --no-pub --verbose-system-logs --browser-name=chrome --target=./test_driver/MELOS_PARENT_PACKAGE_NAME_e2e.dart",
+      "command": "melos exec -c 1 --scope=\"*cloud_firestore_example*\" --dir-exists=web --fail-fast --dir-exists=test_driver -- flutter drive --release -d chrome --no-pub --verbose-system-logs --browser-name=chrome --target=./integration_test/MELOS_PARENT_PACKAGE_NAME_e2e_test.dart --driver=./test_driver/integration_test.dart",
       "detail": "Run end-to-end testing for Web via `flutter drive` on the Firestore example application",
       "problemMatcher": []
     },
@@ -122,20 +122,6 @@
       "detail": "Run `flutter test` on all Auth packages (excluding web & example)",
       "problemMatcher": []
     },
-    {
-      "label": "auth -> test:e2e",
-      "type": "shell",
-      "command": "melos exec -c 1 --scope=\"*firebase_auth_example*\" --fail-fast --dir-exists=test_driver -- flutter drive --no-pub --target=./test_driver/MELOS_PARENT_PACKAGE_NAME_e2e.dart",
-      "detail": "Run end-to-end testing for Android & iOS via `flutter drive` on the Auth example application",
-      "problemMatcher": []
-    },
-    {
-      "label": "auth -> test:e2e:web",
-      "type": "shell",
-      "command": "melos exec -c 1 --scope=\"*firebase_auth_example*\" --dir-exists=web --fail-fast --dir-exists=test_driver -- flutter drive --release -d chrome --no-pub --verbose-system-logs --browser-name=chrome --target=./test_driver/MELOS_PARENT_PACKAGE_NAME_e2e.dart",
-      "detail": "Run end-to-end testing for Web via `flutter drive` on the Auth example application",
-      "problemMatcher": []
-    },
     {
       "label": "auth -> test:web",
       "type": "shell",

CONTRIBUTING.md

@@ -111,21 +111,30 @@ flutter test
 ### End-to-end (e2e) tests
 
 E2e tests are those which directly communicate with Firebase, whose results cannot be mocked. These tests run directly from
-an example application. To run e2e tests, run the `flutter drive` command from the plugins main `example` directory, targeting the
-entry e2e test file.
+an example application. To run e2e tests, run the `flutter test` (for Android, iOS & macOS) or the `flutter drive` (for web) 
+command from the plugins main `example` directory, targeting the entry e2e test file.
 
 > Some packages use Firebase Emulator Suite to run tests. To learn more, [visit the official documentation](https://firebase.google.com/docs/emulator-suite).
 
+To start the Firebase Emulator, run these commands:
+
+```bash
+cd .github/workflows/scripts
+firebase emulators:start --only auth,firestore,functions,storage,database --project flutterfire-e2e-tests
+```
+
+To run tests against Android, iOS or macOS environments, run these commands:
+
 ```bash
 cd packages/firebase_auth/firebase_auth/example
-flutter drive --target=./test_driver/firebase_auth_e2e.dart
+flutter test integration_test/firebase_auth_e2e_test.dart
 ```
 
 To run tests against web environments, run the command as a release build:
 
 ```bash
 cd packages/firebase_auth/firebase_auth/example
-flutter drive --target=./test_driver/firebase_auth_e2e.dart --release -d chrome
+flutter drive --target=./integration_test/firebase_auth_e2e_test.dart --driver=./test_driver/integration_test.dart --release -d chrome
 ```
 
 ### Using Melos

melos.yaml

@@ -151,18 +151,18 @@ scripts:
   test:e2e:
     run: |
       melos exec -c 1 --fail-fast -- \
-        "flutter drive --no-pub --target=./test_driver/driver_e2e.dart"
+        "flutter test integration_test/e2e_test.dart --no-pub"
     description: |
       Run all e2e tests.
     select-package:
       dir-exists:
-        - test_driver
+        - integration_test
       scope: '*tests*'
 
   test:e2e:web:
     run: |
       melos exec -c 1 --fail-fast -- \
-        "flutter drive --no-pub --target=./test_driver/driver_e2e.dart -d chrome"
+        "flutter drive --no-pub --target=./integration_test/e2e_test.dart --driver=./test_driver/integration_test.dart -d chrome"
     description: |
       Run all e2e tests on web platform. Please ensure you have "chromedriver" installed and running.
     select-package:

packages/cloud_firestore/cloud_firestore/example/test_driver/cloud_firestore_e2e.dart renamed to packages/cloud_firestore/cloud_firestore/example/integration_test/cloud_firestore_e2e_test.dart

@@ -3,9 +3,9 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'package:cloud_firestore/cloud_firestore.dart';
-import 'package:drive/drive.dart' as drive;
 import 'package:firebase_core/firebase_core.dart';
 import 'package:flutter_test/flutter_test.dart';
+import 'package:integration_test/integration_test.dart';
 
 import 'collection_reference_e2e.dart';
 import 'document_change_e2e.dart';
@@ -24,29 +24,31 @@ import 'second_app_e2e.dart';
 
 bool kUseFirestoreEmulator = true;
 
-void testsMain() {
-  setUpAll(() async {
-    await Firebase.initializeApp(options: TestFirebaseConfig.platformOptions);
-
-    if (kUseFirestoreEmulator) {
-      FirebaseFirestore.instance.useFirestoreEmulator('localhost', 8080);
-    }
+void main() {
+  IntegrationTestWidgetsFlutterBinding.ensureInitialized();
+
+  group('cloud_firestore', () {
+    setUpAll(() async {
+      await Firebase.initializeApp(options: TestFirebaseConfig.platformOptions);
+
+      if (kUseFirestoreEmulator) {
+        FirebaseFirestore.instance.useFirestoreEmulator('localhost', 8080);
+      }
+    });
+
+    runInstanceTests();
+
+    runCollectionReferenceTests();
+    runDocumentChangeTests();
+    runDocumentReferenceTests();
+    runFieldValueTests();
+    runGeoPointTests();
+    runQueryTests();
+    runSnapshotMetadataTests();
+    runTimestampTests();
+    runTransactionTests();
+    runWriteBatchTests();
+    runLoadBundleTests();
+    runSecondAppTests();
   });
-
-  runInstanceTests();
-
-  runCollectionReferenceTests();
-  runDocumentChangeTests();
-  runDocumentReferenceTests();
-  runFieldValueTests();
-  runGeoPointTests();
-  runQueryTests();
-  runSnapshotMetadataTests();
-  runTimestampTests();
-  runTransactionTests();
-  runWriteBatchTests();
-  runLoadBundleTests();
-  runSecondAppTests();
 }
-
-void main() => drive.main(testsMain);

packages/cloud_firestore/cloud_firestore/example/test_driver/collection_reference_e2e.dart renamed to packages/cloud_firestore/cloud_firestore/example/integration_test/collection_reference_e2e.dart

@@ -29,10 +29,8 @@ dependency_overrides:
     path: ../../../firebase_core/firebase_core_web
 
 dev_dependencies:
-  drive: ^1.0.0-1.0.nullsafety.1
-  flutter_driver:
+  integration_test:
     sdk: flutter
-  test: any
 
 flutter:
   uses-material-design: true

packages/cloud_firestore/cloud_firestore/example/test_driver/document_change_e2e.dart renamed to packages/cloud_firestore/cloud_firestore/example/integration_test/document_change_e2e.dart

@@ -0,0 +1,3 @@
+import 'package:integration_test/integration_test_driver.dart';
+
+Future<void> main() => integrationDriver();

packages/cloud_firestore/cloud_firestore/example/test_driver/document_reference_e2e.dart renamed to packages/cloud_firestore/cloud_firestore/example/integration_test/document_reference_e2e.dart

@@ -28,7 +28,6 @@ dev_dependencies:
   flutter_test:
     sdk: flutter
   mockito: ^5.0.0
-  test: any
 
 flutter:
   plugin:

packages/cloud_firestore/cloud_firestore/example/test_driver/field_value_e2e.dart renamed to packages/cloud_firestore/cloud_firestore/example/integration_test/field_value_e2e.dart

@@ -7,16 +7,12 @@ targets:
           include:
             - lib/*
             - lib/**
-            - test_driver/integration/*
-            - test_driver/integration/**/*
       json_serializable:
         enabled: true
         generate_for:
           include:
             - lib/*
             - lib/**
-            - test_driver/integration/*
-            - test_driver/integration/**/*
         options:
           create_field_map: true
       source_gen|combining_builder:

packages/cloud_firestore/cloud_firestore/example/test_driver/firebase_config.dart renamed to packages/cloud_firestore/cloud_firestore/example/integration_test/firebase_config.dart

@@ -0,0 +1,41 @@
+// Copyright 2021, the Chromium project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:cloud_firestore/cloud_firestore.dart';
+import 'package:firebase_core/firebase_core.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:integration_test/integration_test.dart';
+
+import 'collection_reference_test.dart' as collection_reference_test;
+import 'document_reference_test.dart' as document_reference_test;
+import 'path_test.dart' as path_test;
+import 'query_reference_test.dart' as query_reference_test;
+
+void main() {
+  IntegrationTestWidgetsFlutterBinding.ensureInitialized();
+
+  group('cloud_firestore_odm', () {
+    setUpAll(() async {
+      await Firebase.initializeApp(
+        options: const FirebaseOptions(
+          apiKey: 'AIzaSyAHAsf51D0A407EklG1bs-5wA7EbyfNFg0',
+          appId: '1:448618578101:ios:3a3c8ae9cb0b6408ac3efc',
+          messagingSenderId: '448618578101',
+          projectId: 'react-native-firebase-testing',
+          authDomain: 'react-native-firebase-testing.firebaseapp.com',
+          iosClientId:
+              '448618578101-m53gtqfnqipj12pts10590l37npccd2r.apps.googleusercontent.com',
+        ),
+      );
+      FirebaseFirestore.instance.useFirestoreEmulator('localhost', 8080);
+    });
+
+    collection_reference_test.main();
+    document_reference_test.main();
+    // TODO CI tests configuration currently not compatible with widget testing
+    // firestore_builder_test.main();
+    query_reference_test.main();
+    path_test.main();
+  });
+}

packages/cloud_firestore/cloud_firestore/example/test_driver/firebase_options_secondary.dart renamed to packages/cloud_firestore/cloud_firestore/example/integration_test/firebase_options_secondary.dart

@@ -24,10 +24,11 @@ dev_dependencies:
     path: ../../cloud_firestore_odm_generator
   cloud_firestore_odm_generator_integration_test:
     path: ../../cloud_firestore_odm_generator/cloud_firestore_odm_generator_integration_test
-  drive: ^1.0.0-1.0.nullsafety.1
   flutter_test:
     sdk: flutter
   freezed: ^2.0.0
+  integration_test:
+    sdk: flutter
   http: ^0.13.5
   json_serializable: ^6.3.0
   mockito: ^5.0.0