Merge pull request #965 from ImranR98/dev

Parallelize Update Checking (#963), Bugfixes (#954, #962)

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-name: android
+name: Release (Manual/Draft)
 
 on:
   workflow_dispatch:
@@ -25,12 +25,16 @@ jobs:
         env:
           KEYSTORE_BASE64: ${{ secrets.KEYSTORE_BASE64 }}
           KEYSTORE_PASS: ${{ secrets.KEYSTORE_PASS }}
+          GPG_KEY: ${{ secrets.GPG_KEY }}
         run: |
           echo "${KEYSTORE_BASE64}" | base64 -d > apksign.keystore
+          echo "$GPG_KEY" | gpg --import
           for apk in ./build/app/outputs/flutter-apk/*-release*.apk; do
-            out=${apk/-release/-release-signed}
+            unsignedFn=${apk/-release/-unsigned}
-            ${ANDROID_HOME}/build-tools/30.0.2/apksigner sign --ks apksign.keystore --ks-pass env:KEYSTORE_PASS --out "${out}" "${apk}"
+            mv "$apk" "$unsignedFn"
-            echo "$(sha256sum ${out})"
+            ${ANDROID_HOME}/build-tools/30.0.2/apksigner sign --ks apksign.keystore --ks-pass env:KEYSTORE_PASS --out "${apk}" "${unsignedFn}"
+            sha256sum ${apk} | cut -d " " -f 1 > "$apk".sha256
+            gpg --sign --detach-sig "$apk".sha256
           done
           rm apksign.keystore
         
@@ -39,14 +43,19 @@ jobs:
         run: |
            VERSION=$(grep -oP "currentVersion = '\K[^']+" lib/main.dart)
            echo "::set-output name=version::$VERSION"
+           TAG=$(grep -oP "'.*\\\$currentVersion.*'" lib/main.dart | head -c -2 | tail -c +2 | sed "s/\$currentVersion/$VERSION/g")
+           echo "::set-output name=tag::$TAG"
+           if [ -n "$(echo $TAG | grep -oP '\-beta$')" ]; then BETA=true; else BETA=false; fi
+           echo "::set-output name=beta::$BETA"
       
       - name: Create Release And Upload APKs
         uses: ncipollo/release-action@v1
         with:
           token: ${{ secrets.GAT }}
-          tag: "v${{ steps.extract_version.outputs.version }}-beta"
+          tag: "${{ steps.extract_version.outputs.tag }}"
-          prerelease: true
+          prerelease: "${{ steps.extract_version.outputs.beta }}"
-          artifacts: ./build/app/outputs/flutter-apk/*-signed*.apk          
+          artifacts: ./build/app/outputs/flutter-apk/*-release*.apk*
+          draft: true
                   
       - name: Archive Reports For Job
         uses: actions/upload-artifact@v3

lib/app_sources/html.dart

@@ -120,14 +120,14 @@ class HTML extends AppSource {
         GeneratedFormTextField('matchGroupToUse',
             label: tr('matchGroupToUse'),
             required: false,
-            hint: '1',
+            hint: '0',
             textInputType: const TextInputType.numberWithOptions(),
             additionalValidators: [
               (value) {
                 if (value?.isEmpty == true) {
                   value = null;
                 }
-                value ??= '1';
+                value ??= '0';
                 return intValidator(value);
               }
             ])
@@ -216,8 +216,12 @@ class HTML extends AppSource {
         if (match.isEmpty) {
           throw NoVersionError();
         }
-        version = match.last
+        String matchGroupString =
-            .group(int.parse(additionalSettings['matchGroupToUse'] as String));
+            (additionalSettings['matchGroupToUse'] as String).trim();
+        if (matchGroupString.isEmpty) {
+          matchGroupString = "0";
+        }
+        version = match.last.group(int.parse(matchGroupString));
         if (version?.isEmpty == true) {
           throw NoVersionError();
         }

lib/app_sources/uptodown.dart

@@ -80,4 +80,20 @@ class Uptodown extends AppSource {
         version, getApkUrlsFromUrls([apkUrl]), AppNames(author, appName),
         releaseDate: relDate);
   }
+
+  @override
+  Future<String> apkUrlPrefetchModifier(
+      String apkUrl, String standardUrl) async {
+    var res = await sourceRequest(apkUrl);
+    if (res.statusCode != 200) {
+      throw getObtainiumHttpError(res);
+    }
+    var html = parse(res.body);
+    var finalUrl =
+        (html.querySelector('.post-download')?.attributes['data-url']);
+    if (finalUrl == null) {
+      throw NoAPKError();
+    }
+    return finalUrl;
+  }
 }

lib/custom_errors.dart

@@ -65,25 +65,30 @@ class NotImplementedError extends ObtainiumError {
 }
 
 class MultiAppMultiError extends ObtainiumError {
-  Map<String, List<String>> content = {};
+  Map<String, dynamic> rawErrors = {};
+  Map<String, List<String>> idsByErrorString = {};
+  Map<String, String> appIdNames = {};
 
   MultiAppMultiError() : super(tr('placeholder'), unexpected: true);
 
-  add(String appId, String string) {
+  add(String appId, dynamic error, {String? appName}) {
-    var tempIds = content.remove(string);
+    rawErrors[appId] = error;
+    var string = error.toString();
+    var tempIds = idsByErrorString.remove(string);
     tempIds ??= [];
     tempIds.add(appId);
-    content.putIfAbsent(string, () => tempIds!);
+    idsByErrorString.putIfAbsent(string, () => tempIds!);
+    if (appName != null) {
+      appIdNames[appId] = appName;
+    }
   }
 
+  String errorString(String appId) =>
+      '${appIdNames.containsKey(appId) ? '${appIdNames[appId]} ($appId)' : appId}: ${rawErrors[appId].toString()}';
+
   @override
-  String toString() {
+  String toString() =>
-    String finalString = '';
+      idsByErrorString.keys.map((e) => errorString(e)).join('\n\n');
-    for (var e in content.keys) {
-      finalString += '$e: ${content[e].toString()}\n\n';
-    }
-    return finalString;
-  }
 }
 
 showError(dynamic e, BuildContext context) {

lib/main.dart

@@ -19,7 +19,7 @@ import 'package:easy_localization/src/easy_localization_controller.dart';
 // ignore: implementation_imports
 import 'package:easy_localization/src/localization.dart';
 
-const String currentVersion = '0.14.22';
+const String currentVersion = '0.14.23';
 const String currentReleaseTag =
     'v$currentVersion-beta'; // KEEP THIS IN SYNC WITH GITHUB RELEASES
 

lib/pages/apps.dart

@@ -68,7 +68,7 @@ class AppsPageState extends State<AppsPage> {
         refreshingSince = DateTime.now();
       });
       return appsProvider.checkUpdates().catchError((e) {
-        showError(e, context);
+        showError(e is Map ? e['errors'] : e, context);
         return <App>[];
       }).whenComplete(() {
         setState(() {
@@ -833,7 +833,7 @@ class AppsPageState extends State<AppsPage> {
                 items: const [],
                 initValid: true,
                 message: tr('installStatusOfXWillBeResetExplanation',
-                    args: [plural('app', selectedAppIds.length)]),
+                    args: [plural('apps', selectedAppIds.length)]),
               );
             });
         if (values != null) {

lib/pages/import_export.dart

@@ -217,7 +217,8 @@ class _ImportExportPageState extends State<ImportExportPage> {
               if (errors.isEmpty) {
                 // ignore: use_build_context_synchronously
                 showError(
-                    tr('importedX', args: [plural('app', selectedUrls.length)]),
+                    tr('importedX',
+                        args: [plural('apps', selectedUrls.length)]),
                     context);
               } else {
                 // ignore: use_build_context_synchronously
@@ -274,7 +275,7 @@ class _ImportExportPageState extends State<ImportExportPage> {
             if (errors.isEmpty) {
               // ignore: use_build_context_synchronously
               showError(
-                  tr('importedX', args: [plural('app', selectedUrls.length)]),
+                  tr('importedX', args: [plural('apps', selectedUrls.length)]),
                   context);
             } else {
               // ignore: use_build_context_synchronously

lib/providers/apps_provider.dart

@@ -449,7 +449,7 @@ class AppsProvider with ChangeNotifier {
           } catch (e) {
             logs.add(
                 'Could not install APK from XAPK \'${file.path}\': ${e.toString()}');
-            errors.add(dir.appId, e.toString());
+            errors.add(dir.appId, e, appName: apps[dir.appId]?.name);
           }
         } else if (file.path.toLowerCase().endsWith('.obb')) {
           await moveObbFile(file, dir.appId);
@@ -457,7 +457,7 @@ class AppsProvider with ChangeNotifier {
       }
       if (somethingInstalled) {
         dir.file.delete(recursive: true);
-      } else if (errors.content.isNotEmpty) {
+      } else if (errors.idsByErrorString.isNotEmpty) {
         throw errors;
       }
     } finally {
@@ -677,11 +677,11 @@ class AppsProvider with ChangeNotifier {
         }
         installedIds.add(id);
       } catch (e) {
-        errors.add(id, e.toString());
+        errors.add(id, e, appName: apps[id]?.name);
       }
     }
 
-    if (errors.content.isNotEmpty) {
+    if (errors.idsByErrorString.isNotEmpty) {
       throw errors;
     }
 
@@ -738,7 +738,6 @@ class AppsProvider with ChangeNotifier {
     var naiveStandardVersionDetection = SourceProvider()
         .getSource(app.url, overrideSource: app.overrideSource)
         .naiveStandardVersionDetection;
-    ;
     // FIRST, COMPARE THE APP'S REPORTED AND REAL INSTALLED VERSIONS, WHERE ONE IS NULL
     if (installedInfo == null && app.installedVersion != null && !trackOnly) {
       // App says it's installed but isn't really (and isn't track only) - set to not installed
@@ -1069,7 +1068,8 @@ class AppsProvider with ChangeNotifier {
 
   Future<List<App>> checkUpdates(
       {DateTime? ignoreAppsCheckedAfter,
-      bool throwErrorsForRetry = false}) async {
+      bool throwErrorsForRetry = false,
+      List<String>? specificIds}) async {
     List<App> updates = [];
     MultiAppMultiError errors = MultiAppMultiError();
     if (!gettingUpdates) {
@@ -1077,27 +1077,33 @@ class AppsProvider with ChangeNotifier {
       try {
         List<String> appIds = getAppsSortedByUpdateCheckTime(
             ignoreAppsCheckedAfter: ignoreAppsCheckedAfter);
-        for (int i = 0; i < appIds.length; i++) {
+        if (specificIds != null) {
+          appIds = appIds.where((aId) => specificIds.contains(aId)).toList();
+        }
+        await Future.wait(appIds.map((appId) async {
           App? newApp;
           try {
-            newApp = await checkUpdate(appIds[i]);
+            newApp = await checkUpdate(appId);
           } catch (e) {
             if ((e is RateLimitError || e is SocketException) &&
                 throwErrorsForRetry) {
               rethrow;
             }
-            errors.add(appIds[i], e.toString());
+            errors.add(appId, e, appName: apps[appId]?.name);
           }
           if (newApp != null) {
             updates.add(newApp);
           }
-        }
+        }), eagerError: true);
       } finally {
         gettingUpdates = false;
       }
     }
-    if (errors.content.isNotEmpty) {
+    if (errors.idsByErrorString.isNotEmpty) {
-      throw errors;
+      var res = <String, dynamic>{};
+      res['errors'] = errors;
+      res['updates'] = updates;
+      throw res;
     }
     return updates;
   }
@@ -1314,18 +1320,16 @@ class _APKOriginWarningDialogState extends State<APKOriginWarningDialog> {
 
 /// Background updater function
 ///
-/// @param List<String>? toCheck: The appIds to check for updates (default to all apps sorted by last update check time)
+/// @param List<MapEntry<String, int>>? toCheck: The appIds to check for updates (with the number of previous attempts made per appid) (defaults to all apps)
 ///
 /// @param List<String>? toInstall: The appIds to attempt to update (defaults to an empty array)
 ///
-/// @param int? attemptCount: The number of times the function has failed up to this point (defaults to 0)
-///
 /// When toCheck is empty, the function is in "install mode" (else it is in "update mode").
 /// In update mode, all apps in toCheck are checked for updates.
 /// If an update is available, the appId is either added to toInstall (if a background update is possible) or the user is notified.
-/// If there is an error, the function tries to continue after a few minutes (duration depends on the error), up to a maximum of 5 tries.
+/// If there are errors, the task is run again for the remaining apps after a few minutes (duration depends on the errors), up to a maximum of 5 tries for any app.
 ///
-/// Once all update checks are complete, the function is called again in install mode.
+/// Once all update checks are complete, the task is run again in install mode.
 /// In this mode, all apps in toInstall are downloaded and installed in the background (install result is unknown).
 /// If there is an error, the function tries to continue after a few minutes (duration depends on the error), up to a maximum of 5 tries.
 ///
@@ -1372,87 +1376,109 @@ Future<void> bgUpdateCheck(int taskId, Map<String, dynamic>? params) async {
       'BG ${installMode ? 'install' : 'update'} task $taskId: Started (${installMode ? toInstall.length : toCheck.length}).');
 
   if (!installMode) {
-    // If in update mode...
+    // If in update mode, we check for updates.
-    var didCompleteChecking = false;
+    // We divide the results into 4 groups:
-    CheckingUpdatesNotification? notif;
+    // - toNotify - Apps with updates that the user will be notified about (can't be silently installed)
+    // - toRetry - Apps with update check errors that will be retried in a while
+    // - toThrow - Apps with update check errors that the user will be notified about (no retry)
+    // - toInstall - Apps with updates that will be installed silently
+    // After grouping the updates, we take care of toNotify and toThrow first
+    // Then if toRetry is not empty, we schedule another update task to run in a while (toInstall is retained)
+    // If toRetry is empty, we take care of toInstall
+
+    // Init. vars.
+    List<App> updates = [];
+    List<App> toNotify = [];
+    List<MapEntry<String, int>> toRetry = [];
+    var retryAfterXSeconds = 0;
+    List<String> toThrow = [];
     var networkRestricted = false;
     if (appsProvider.settingsProvider.bgUpdatesOnWiFiOnly) {
       var netResult = await (Connectivity().checkConnectivity());
       networkRestricted = (netResult != ConnectivityResult.wifi) &&
           (netResult != ConnectivityResult.ethernet);
     }
-    // Loop through all updates and check each
+    MultiAppMultiError? errors;
-    List<App> toNotify = [];
+    CheckingUpdatesNotification notif =
+        CheckingUpdatesNotification(plural('apps', toCheck.length));
+
     try {
-      for (int i = 0; i < toCheck.length; i++) {
+      // Check for updates
-        var appId = toCheck[i].key;
+      notificationsProvider.notify(notif, cancelExisting: true);
-        var attemptCount = toCheck[i].value + 1;
+      updates = await appsProvider.checkUpdates(
-        AppInMemory? app = appsProvider.apps[appId];
+          specificIds: toCheck.map((e) => e.key).toList());
-        if (app?.app.installedVersion != null) {
-          try {
-            notificationsProvider.notify(
-                notif = CheckingUpdatesNotification(app?.name ?? appId),
-                cancelExisting: true);
-            App? newApp = await appsProvider.checkUpdate(appId);
-            if (newApp != null) {
-              if (networkRestricted ||
-                  !(await appsProvider.canInstallSilently(app!.app))) {
-                toNotify.add(newApp);
-              } else {
-                toInstall.add(MapEntry(appId, 0));
-              }
-            }
-            if (i == (toCheck.length - 1)) {
-              didCompleteChecking = true;
-            }
     } catch (e) {
-            // If you got an error, move the offender to the back of the line (increment their fail count) and schedule another task to continue checking shortly
+      // If there were errors, group them into toRetry and toThrow
+      if (e is Map) {
+        updates = e['updates'];
+        errors = e['errors'];
+        errors!.rawErrors.forEach((key, err) {
           logs.add(
-                'BG update task $taskId: Got error on checking for $appId \'${e.toString()}\'.');
+              'BG update task $taskId: Got error on checking for $key \'${err.toString()}\'.');
-            if (attemptCount < maxAttempts) {
+          var toCheckApp = toCheck.where((element) => element.key == key).first;
-              var remainingSeconds = e is RateLimitError
+          if (toCheckApp.value < maxAttempts) {
-                  ? (i == 0 ? (e.remainingMinutes * 60) : (5 * 60))
+            toRetry.add(MapEntry(toCheckApp.key, toCheckApp.value + 1));
+            var minRetryIntervalForThisApp = err is RateLimitError
+                ? (err.remainingMinutes * 60)
                 : e is ClientException
                     ? (15 * 60)
-                      : pow(attemptCount, 2).toInt();
+                    : pow(toCheckApp.value + 1, 2).toInt();
-              logs.add(
+            if (minRetryIntervalForThisApp > retryAfterXSeconds) {
-                  'BG update task $taskId: Will continue in $remainingSeconds seconds (with $appId moved to the end of the line).');
+              retryAfterXSeconds = minRetryIntervalForThisApp;
-              var remainingToCheck = moveStrToEndMapEntryWithCount(
+            }
-                  toCheck.sublist(i), MapEntry(appId, attemptCount));
-              AndroidAlarmManager.oneShot(Duration(seconds: remainingSeconds),
-                  taskId + 1, bgUpdateCheck,
-                  params: {
-                    'toCheck': remainingToCheck
-                        .map(
-                            (entry) => {'key': entry.key, 'value': entry.value})
-                        .toList(),
-                    'toInstall': toInstall
-                        .map(
-                            (entry) => {'key': entry.key, 'value': entry.value})
-                        .toList(),
-                  });
-              break;
           } else {
-              // If the offender has reached its fail limit, notify the user and remove it from the list (task can continue)
+            toThrow.add(key);
-              toCheck.removeAt(i);
+          }
-              i--;
+        });
-              notificationsProvider
+      } else {
-                  .notify(ErrorCheckingUpdatesNotification(e.toString()));
+        // We don't expect to ever get here in any situation so no need to catch
+        logs.add('Fatal error in BG update task: ${e.toString()}');
+        rethrow;
       }
     } finally {
-            if (notif != null) {
       notificationsProvider.cancel(notif.id);
     }
+
+    // Group the updates into toNotify and toInstall
+    for (var i = 0; i < updates.length; i++) {
+      if (networkRestricted ||
+          !(await appsProvider.canInstallSilently(updates[i]))) {
+        toNotify.add(updates[i]);
+      } else {
+        toInstall.add(MapEntry(updates[i].id, 0));
       }
     }
-      }
+
-    } finally {
+    // Send the update notification
     if (toNotify.isNotEmpty) {
       notificationsProvider.notify(UpdateNotification(toNotify));
     }
+
+    // Send the error notifications
+    if (toThrow.isNotEmpty) {
+      for (var appId in toThrow) {
+        notificationsProvider.notify(ErrorCheckingUpdatesNotification(
+            errors!.errorString(appId),
+            id: Random().nextInt(10000)));
       }
-    // If you're done checking and found some silently installable updates, schedule another task which will run in install mode
+    }
-    if (didCompleteChecking && toInstall.isNotEmpty) {
+
+    // if there are update checks to retry, schedule a retry task
+    if (toRetry.isNotEmpty) {
+      logs.add(
+          'BG update task $taskId: Will retry in $retryAfterXSeconds seconds.');
+      AndroidAlarmManager.oneShot(
+          Duration(seconds: retryAfterXSeconds), taskId + 1, bgUpdateCheck,
+          params: {
+            'toCheck': toRetry
+                .map((entry) => {'key': entry.key, 'value': entry.value})
+                .toList(),
+            'toInstall': toInstall
+                .map((entry) => {'key': entry.key, 'value': entry.value})
+                .toList(),
+          });
+    } else if (toInstall.isNotEmpty) {
+      // If there are no more update checks, schedule an install task
       logs.add(
           'BG update task $taskId: Done. Scheduling install task to run immediately.');
       AndroidAlarmManager.oneShot(
@@ -1463,11 +1489,14 @@ Future<void> bgUpdateCheck(int taskId, Map<String, dynamic>? params) async {
                 .map((entry) => {'key': entry.key, 'value': entry.value})
                 .toList()
           });
-    } else if (didCompleteChecking) {
+    } else {
       logs.add('BG install task $taskId: Done.');
     }
-  } else {
+  }
-    // If in install mode...
+
+  if (installMode) {
+    // If in install mode, we install silent updates.
+
     var didCompleteInstalling = false;
     var tempObtArr = toInstall.where((element) => element.key == obtainiumId);
     if (tempObtArr.isNotEmpty) {

pubspec.lock

@@ -46,10 +46,10 @@ packages:
     dependency: transitive
     description:
       name: archive
-      sha256: d4dc11707abb32ef756ab95678c0d6df54003d98277f7c9aeda14c48e7a38c2f
+      sha256: "06a96f1249f38a00435b3b0c9a3246d934d7dbc8183fc7c9e56989860edb99d4"
       url: "https://pub.dev"
     source: hosted
-    version: "3.4.3"
+    version: "3.4.4"
   args:
     dependency: transitive
     description:
@@ -291,10 +291,10 @@ packages:
     dependency: "direct main"
     description:
       name: flutter_local_notifications
-      sha256: "3002092e5b8ce2f86c3361422e52e6db6776c23ee21e0b2f71b892bf4259ef04"
+      sha256: "6d11ea777496061e583623aaf31923f93a9409ef8fcaeeefdd6cd78bf4fe5bb3"
       url: "https://pub.dev"
     source: hosted
-    version: "15.1.1"
+    version: "16.1.0"
   flutter_local_notifications_linux:
     dependency: transitive
     description:
@@ -320,10 +320,10 @@ packages:
     dependency: "direct main"
     description:
       name: flutter_markdown
-      sha256: a10979814c5f4ddbe2b6143fba25d927599e21e3ba65b3862995960606fae78f
+      sha256: "8afc9a6aa6d8e8063523192ba837149dbf3d377a37c0b0fc579149a1fbd4a619"
       url: "https://pub.dev"
     source: hosted
-    version: "0.6.17+3"
+    version: "0.6.18"
   flutter_plugin_android_lifecycle:
     dependency: transitive
     description:
@@ -538,10 +538,10 @@ packages:
     dependency: "direct main"
     description:
       name: permission_handler
-      sha256: ad65ba9af42a3d067203641de3fd9f547ded1410bad3b84400c2b4899faede70
+      sha256: "284a66179cabdf942f838543e10413246f06424d960c92ba95c84439154fcac8"
       url: "https://pub.dev"
     source: hosted
-    version: "11.0.0"
+    version: "11.0.1"
   permission_handler_android:
     dependency: transitive
     description:
@@ -879,10 +879,10 @@ packages:
     dependency: "direct main"
     description:
       name: webview_flutter
-      sha256: "053d454c9475546b4382e9498601fb46293cdac9b3ca93f1a738375bc9a1eee4"
+      sha256: c1ab9b81090705c6069197d9fdc1625e587b52b8d70cdde2339d177ad0dbb98e
       url: "https://pub.dev"
     source: hosted
-    version: "4.3.0"
+    version: "4.4.1"
   webview_flutter_android:
     dependency: transitive
     description:
@@ -903,10 +903,10 @@ packages:
     dependency: transitive
     description:
       name: webview_flutter_wkwebview
-      sha256: "3c7d56ca4b82654ad1f58aeefb8d593a59224f26d6b2bf8feed074361eb34c86"
+      sha256: "30b9af6bdd457b44c08748b9190d23208b5165357cc2eb57914fee1366c42974"
       url: "https://pub.dev"
     source: hosted
-    version: "3.9.0"
+    version: "3.9.1"
   win32:
     dependency: transitive
     description:

pubspec.yaml

@@ -17,7 +17,7 @@ publish_to: 'none' # Remove this line if you wish to publish to pub.dev
 # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html
 # In Windows, build-name is used as the major, minor, and patch parts
 # of the product and file versions while build-number is used as the build suffix.
-version: 0.14.22+214 # When changing this, update the tag in main() accordingly
+version: 0.14.23+215 # When changing this, update the tag in main() accordingly
 
 environment:
   sdk: '>=3.0.0 <4.0.0'
@@ -38,7 +38,7 @@ dependencies:
   cupertino_icons: ^1.0.5
   path_provider: ^2.0.11
   flutter_fgbg: ^0.3.0 # Try removing reliance on this
-  flutter_local_notifications: ^15.1.0+1
+  flutter_local_notifications: ^16.1.0
   provider: ^6.0.3
   http: ^1.0.0
   webview_flutter: ^4.0.0