From 21fdfc1eef72462e4a9fe4ada9869de0f87dcca7 Mon Sep 17 00:00:00 2001 From: Imran Remtulla Date: Fri, 6 Oct 2023 19:23:18 -0400 Subject: [PATCH 1/3] Attempting to parallelize update checks --- lib/custom_errors.dart | 5 +- lib/pages/apps.dart | 2 +- lib/providers/apps_provider.dart | 194 ++++++++++++++++++------------- 3 files changed, 115 insertions(+), 86 deletions(-) diff --git a/lib/custom_errors.dart b/lib/custom_errors.dart index 6343c87..4d807c5 100644 --- a/lib/custom_errors.dart +++ b/lib/custom_errors.dart @@ -65,11 +65,14 @@ class NotImplementedError extends ObtainiumError { } class MultiAppMultiError extends ObtainiumError { + Map rawErrors = {}; Map> content = {}; MultiAppMultiError() : super(tr('placeholder'), unexpected: true); - add(String appId, String string) { + add(String appId, dynamic error) { + rawErrors[appId] = error; + var string = error.toString(); var tempIds = content.remove(string); tempIds ??= []; tempIds.add(appId); diff --git a/lib/pages/apps.dart b/lib/pages/apps.dart index 3f4cc30..cb7ce46 100644 --- a/lib/pages/apps.dart +++ b/lib/pages/apps.dart @@ -68,7 +68,7 @@ class AppsPageState extends State { refreshingSince = DateTime.now(); }); return appsProvider.checkUpdates().catchError((e) { - showError(e, context); + showError(e is Map ? e['errors'] : e, context); return []; }).whenComplete(() { setState(() { diff --git a/lib/providers/apps_provider.dart b/lib/providers/apps_provider.dart index 2dc7498..1a9e052 100644 --- a/lib/providers/apps_provider.dart +++ b/lib/providers/apps_provider.dart @@ -449,7 +449,7 @@ class AppsProvider with ChangeNotifier { } catch (e) { logs.add( 'Could not install APK from XAPK \'${file.path}\': ${e.toString()}'); - errors.add(dir.appId, e.toString()); + errors.add(dir.appId, e); } } else if (file.path.toLowerCase().endsWith('.obb')) { await moveObbFile(file, dir.appId); @@ -677,7 +677,7 @@ class AppsProvider with ChangeNotifier { } installedIds.add(id); } catch (e) { - errors.add(id, e.toString()); + errors.add(id, e); } } @@ -1069,7 +1069,8 @@ class AppsProvider with ChangeNotifier { Future> checkUpdates( {DateTime? ignoreAppsCheckedAfter, - bool throwErrorsForRetry = false}) async { + bool throwErrorsForRetry = false, + List? specificIds}) async { List updates = []; MultiAppMultiError errors = MultiAppMultiError(); if (!gettingUpdates) { @@ -1077,27 +1078,33 @@ class AppsProvider with ChangeNotifier { try { List appIds = getAppsSortedByUpdateCheckTime( ignoreAppsCheckedAfter: ignoreAppsCheckedAfter); - for (int i = 0; i < appIds.length; i++) { + if (specificIds != null) { + appIds = appIds.where((aId) => specificIds.contains(aId)).toList(); + } + await Future.wait(appIds.map((appId) async { App? newApp; try { - newApp = await checkUpdate(appIds[i]); + newApp = await checkUpdate(appId); } catch (e) { if ((e is RateLimitError || e is SocketException) && throwErrorsForRetry) { rethrow; } - errors.add(appIds[i], e.toString()); + errors.add(appId, e); } if (newApp != null) { updates.add(newApp); } - } + }), eagerError: true); } finally { gettingUpdates = false; } } if (errors.content.isNotEmpty) { - throw errors; + var res = Map(); + res['errors'] = errors; + res['updates'] = updates; + throw res; } return updates; } @@ -1314,18 +1321,16 @@ class _APKOriginWarningDialogState extends State { /// Background updater function /// -/// @param List? toCheck: The appIds to check for updates (default to all apps sorted by last update check time) +/// @param List>? toCheck: The appIds to check for updates (with the number of previous attempts made per appid) (defaults to all apps) /// /// @param List? toInstall: The appIds to attempt to update (defaults to an empty array) /// -/// @param int? attemptCount: The number of times the function has failed up to this point (defaults to 0) -/// /// When toCheck is empty, the function is in "install mode" (else it is in "update mode"). /// In update mode, all apps in toCheck are checked for updates. /// If an update is available, the appId is either added to toInstall (if a background update is possible) or the user is notified. -/// If there is an error, the function tries to continue after a few minutes (duration depends on the error), up to a maximum of 5 tries. +/// If there are errors, the task is run again for the remaining apps after a few minutes (duration depends on the errors), up to a maximum of 5 tries for any app. /// -/// Once all update checks are complete, the function is called again in install mode. +/// Once all update checks are complete, the task is run again in install mode. /// In this mode, all apps in toInstall are downloaded and installed in the background (install result is unknown). /// If there is an error, the function tries to continue after a few minutes (duration depends on the error), up to a maximum of 5 tries. /// @@ -1372,87 +1377,105 @@ Future bgUpdateCheck(int taskId, Map? params) async { 'BG ${installMode ? 'install' : 'update'} task $taskId: Started (${installMode ? toInstall.length : toCheck.length}).'); if (!installMode) { - // If in update mode... - var didCompleteChecking = false; - CheckingUpdatesNotification? notif; + // If in update mode, we check for updates. + // We divide the results into 4 groups: + // - toNotify - Apps with updates that the user will be notified about (can't be silently installed) + // - toRetry - Apps with update check errors that will be retried in a while + // - toThrow - Apps with update check errors that the user will be notified about (no retry) + // - toInstall - Apps with updates that will be installed silently + // After grouping the updates, we take care of toNotify and toThrow first + // Then if toRetry is not empty, we schedule another update task to run in a while (toInstall is retained) + // If toRetry is empty, we take care of toInstall + + // Init. vars. + List updates = []; + List toNotify = []; + List> toRetry = []; + var retryAfterXSeconds = 0; + List> toThrow = []; var networkRestricted = false; if (appsProvider.settingsProvider.bgUpdatesOnWiFiOnly) { var netResult = await (Connectivity().checkConnectivity()); networkRestricted = (netResult != ConnectivityResult.wifi) && (netResult != ConnectivityResult.ethernet); } - // Loop through all updates and check each - List toNotify = []; + CheckingUpdatesNotification notif = + CheckingUpdatesNotification(plural('app', toCheck.length)); + try { - for (int i = 0; i < toCheck.length; i++) { - var appId = toCheck[i].key; - var attemptCount = toCheck[i].value + 1; - AppInMemory? app = appsProvider.apps[appId]; - if (app?.app.installedVersion != null) { - try { - notificationsProvider.notify( - notif = CheckingUpdatesNotification(app?.name ?? appId), - cancelExisting: true); - App? newApp = await appsProvider.checkUpdate(appId); - if (newApp != null) { - if (networkRestricted || - !(await appsProvider.canInstallSilently(app!.app))) { - toNotify.add(newApp); - } else { - toInstall.add(MapEntry(appId, 0)); - } - } - if (i == (toCheck.length - 1)) { - didCompleteChecking = true; - } - } catch (e) { - // If you got an error, move the offender to the back of the line (increment their fail count) and schedule another task to continue checking shortly - logs.add( - 'BG update task $taskId: Got error on checking for $appId \'${e.toString()}\'.'); - if (attemptCount < maxAttempts) { - var remainingSeconds = e is RateLimitError - ? (i == 0 ? (e.remainingMinutes * 60) : (5 * 60)) - : e is ClientException - ? (15 * 60) - : pow(attemptCount, 2).toInt(); - logs.add( - 'BG update task $taskId: Will continue in $remainingSeconds seconds (with $appId moved to the end of the line).'); - var remainingToCheck = moveStrToEndMapEntryWithCount( - toCheck.sublist(i), MapEntry(appId, attemptCount)); - AndroidAlarmManager.oneShot(Duration(seconds: remainingSeconds), - taskId + 1, bgUpdateCheck, - params: { - 'toCheck': remainingToCheck - .map( - (entry) => {'key': entry.key, 'value': entry.value}) - .toList(), - 'toInstall': toInstall - .map( - (entry) => {'key': entry.key, 'value': entry.value}) - .toList(), - }); - break; - } else { - // If the offender has reached its fail limit, notify the user and remove it from the list (task can continue) - toCheck.removeAt(i); - i--; - notificationsProvider - .notify(ErrorCheckingUpdatesNotification(e.toString())); - } - } finally { - if (notif != null) { - notificationsProvider.cancel(notif.id); + // Check for updates + notificationsProvider.notify(notif, cancelExisting: true); + updates = await appsProvider.checkUpdates( + specificIds: toCheck.map((e) => e.key).toList()); + } catch (e) { + // If there were errors, group them into toRetry and toThrow + if (e is Map) { + updates = e['updates']; + MultiAppMultiError errors = e['errors']; + errors.rawErrors.forEach((key, err) { + logs.add( + 'BG update task $taskId: Got error on checking for $key \'${err.toString()}\'.'); + var toCheckApp = toCheck.where((element) => element.key == key).first; + if (toCheckApp.value < maxAttempts) { + toRetry.add(MapEntry(toCheckApp.key, toCheckApp.value + 1)); + var minRetryIntervalForThisApp = err is RateLimitError + ? (err.remainingMinutes * 60) + : e is ClientException + ? (15 * 60) + : pow(toCheckApp.value, 2).toInt(); + if (minRetryIntervalForThisApp > retryAfterXSeconds) { + retryAfterXSeconds = minRetryIntervalForThisApp; } + } else { + toThrow.add(MapEntry(key, err)); } - } + }); + } else { + // We don't expect to ever get here in any situation so no need to catch + logs.add('Fatal error in BG update task: ${e.toString()}'); + rethrow; } } finally { - if (toNotify.isNotEmpty) { - notificationsProvider.notify(UpdateNotification(toNotify)); + notificationsProvider.cancel(notif.id); + } + + // Group the updates into toNotify and toInstall + for (var i = 0; i < updates.length; i++) { + if (networkRestricted || + !(await appsProvider.canInstallSilently(updates[i]))) { + toNotify.add(updates[i]); + } else { + toInstall.add(MapEntry(updates[i].id, 0)); } } - // If you're done checking and found some silently installable updates, schedule another task which will run in install mode - if (didCompleteChecking && toInstall.isNotEmpty) { + + // Send the update notification + if (toNotify.isNotEmpty) { + notificationsProvider.notify(UpdateNotification(toNotify)); + } + + // Send the error notifications + if (toThrow.isNotEmpty) { + for (var element in toThrow) { + notificationsProvider.notify(ErrorCheckingUpdatesNotification( + '${element.key}: ${element.value.toString()}')); + } + } + + // if there are update checks to retry, schedule a retry task + if (toRetry.isNotEmpty) { + logs.add( + 'BG update task $taskId: Will retry in $retryAfterXSeconds seconds.'); + AndroidAlarmManager.oneShot( + Duration(seconds: retryAfterXSeconds), taskId + 1, bgUpdateCheck, + params: { + 'toCheck': toRetry, + 'toInstall': toInstall + .map((entry) => {'key': entry.key, 'value': entry.value}) + .toList(), + }); + } else if (toInstall.isNotEmpty) { + // If there are no more update checks, schedule an install task logs.add( 'BG update task $taskId: Done. Scheduling install task to run immediately.'); AndroidAlarmManager.oneShot( @@ -1463,11 +1486,14 @@ Future bgUpdateCheck(int taskId, Map? params) async { .map((entry) => {'key': entry.key, 'value': entry.value}) .toList() }); - } else if (didCompleteChecking) { + } else { logs.add('BG install task $taskId: Done.'); } - } else { - // If in install mode... + } + + if (installMode) { + // If in install mode, we install silent updates. + var didCompleteInstalling = false; var tempObtArr = toInstall.where((element) => element.key == obtainiumId); if (tempObtArr.isNotEmpty) { From e9e9adb1744eeaff9d5cf3a1ccf6c668378287e2 Mon Sep 17 00:00:00 2001 From: Imran Remtulla Date: Fri, 6 Oct 2023 19:34:08 -0400 Subject: [PATCH 2/3] Tweaks --- lib/providers/apps_provider.dart | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/providers/apps_provider.dart b/lib/providers/apps_provider.dart index 1a9e052..31824c2 100644 --- a/lib/providers/apps_provider.dart +++ b/lib/providers/apps_provider.dart @@ -1458,7 +1458,8 @@ Future bgUpdateCheck(int taskId, Map? params) async { if (toThrow.isNotEmpty) { for (var element in toThrow) { notificationsProvider.notify(ErrorCheckingUpdatesNotification( - '${element.key}: ${element.value.toString()}')); + '${element.key}: ${element.value.toString()}', + id: Random().nextInt(10000))); } } @@ -1469,7 +1470,9 @@ Future bgUpdateCheck(int taskId, Map? params) async { AndroidAlarmManager.oneShot( Duration(seconds: retryAfterXSeconds), taskId + 1, bgUpdateCheck, params: { - 'toCheck': toRetry, + 'toCheck': toRetry + .map((entry) => {'key': entry.key, 'value': entry.value}) + .toList(), 'toInstall': toInstall .map((entry) => {'key': entry.key, 'value': entry.value}) .toList(), From 8163cd5c8f7ab9706d4711d64e553007b82e73a2 Mon Sep 17 00:00:00 2001 From: Imran Remtulla Date: Fri, 6 Oct 2023 19:58:46 -0400 Subject: [PATCH 3/3] Improvements, bugfixes --- lib/custom_errors.dart | 24 +++++++++++++----------- lib/pages/apps.dart | 2 +- lib/pages/import_export.dart | 5 +++-- lib/providers/apps_provider.dart | 29 +++++++++++++++-------------- 4 files changed, 32 insertions(+), 28 deletions(-) diff --git a/lib/custom_errors.dart b/lib/custom_errors.dart index 4d807c5..04e6104 100644 --- a/lib/custom_errors.dart +++ b/lib/custom_errors.dart @@ -66,27 +66,29 @@ class NotImplementedError extends ObtainiumError { class MultiAppMultiError extends ObtainiumError { Map rawErrors = {}; - Map> content = {}; + Map> idsByErrorString = {}; + Map appIdNames = {}; MultiAppMultiError() : super(tr('placeholder'), unexpected: true); - add(String appId, dynamic error) { + add(String appId, dynamic error, {String? appName}) { rawErrors[appId] = error; var string = error.toString(); - var tempIds = content.remove(string); + var tempIds = idsByErrorString.remove(string); tempIds ??= []; tempIds.add(appId); - content.putIfAbsent(string, () => tempIds!); + idsByErrorString.putIfAbsent(string, () => tempIds!); + if (appName != null) { + appIdNames[appId] = appName; + } } + String errorString(String appId) => + '${appIdNames.containsKey(appId) ? '${appIdNames[appId]} ($appId)' : appId}: ${rawErrors[appId].toString()}'; + @override - String toString() { - String finalString = ''; - for (var e in content.keys) { - finalString += '$e: ${content[e].toString()}\n\n'; - } - return finalString; - } + String toString() => + idsByErrorString.keys.map((e) => errorString(e)).join('\n\n'); } showError(dynamic e, BuildContext context) { diff --git a/lib/pages/apps.dart b/lib/pages/apps.dart index cb7ce46..fe98aa9 100644 --- a/lib/pages/apps.dart +++ b/lib/pages/apps.dart @@ -833,7 +833,7 @@ class AppsPageState extends State { items: const [], initValid: true, message: tr('installStatusOfXWillBeResetExplanation', - args: [plural('app', selectedAppIds.length)]), + args: [plural('apps', selectedAppIds.length)]), ); }); if (values != null) { diff --git a/lib/pages/import_export.dart b/lib/pages/import_export.dart index 264f3a9..68de7f5 100644 --- a/lib/pages/import_export.dart +++ b/lib/pages/import_export.dart @@ -217,7 +217,8 @@ class _ImportExportPageState extends State { if (errors.isEmpty) { // ignore: use_build_context_synchronously showError( - tr('importedX', args: [plural('app', selectedUrls.length)]), + tr('importedX', + args: [plural('apps', selectedUrls.length)]), context); } else { // ignore: use_build_context_synchronously @@ -274,7 +275,7 @@ class _ImportExportPageState extends State { if (errors.isEmpty) { // ignore: use_build_context_synchronously showError( - tr('importedX', args: [plural('app', selectedUrls.length)]), + tr('importedX', args: [plural('apps', selectedUrls.length)]), context); } else { // ignore: use_build_context_synchronously diff --git a/lib/providers/apps_provider.dart b/lib/providers/apps_provider.dart index 31824c2..7c150c6 100644 --- a/lib/providers/apps_provider.dart +++ b/lib/providers/apps_provider.dart @@ -449,7 +449,7 @@ class AppsProvider with ChangeNotifier { } catch (e) { logs.add( 'Could not install APK from XAPK \'${file.path}\': ${e.toString()}'); - errors.add(dir.appId, e); + errors.add(dir.appId, e, appName: apps[dir.appId]?.name); } } else if (file.path.toLowerCase().endsWith('.obb')) { await moveObbFile(file, dir.appId); @@ -457,7 +457,7 @@ class AppsProvider with ChangeNotifier { } if (somethingInstalled) { dir.file.delete(recursive: true); - } else if (errors.content.isNotEmpty) { + } else if (errors.idsByErrorString.isNotEmpty) { throw errors; } } finally { @@ -677,11 +677,11 @@ class AppsProvider with ChangeNotifier { } installedIds.add(id); } catch (e) { - errors.add(id, e); + errors.add(id, e, appName: apps[id]?.name); } } - if (errors.content.isNotEmpty) { + if (errors.idsByErrorString.isNotEmpty) { throw errors; } @@ -1090,7 +1090,7 @@ class AppsProvider with ChangeNotifier { throwErrorsForRetry) { rethrow; } - errors.add(appId, e); + errors.add(appId, e, appName: apps[appId]?.name); } if (newApp != null) { updates.add(newApp); @@ -1100,7 +1100,7 @@ class AppsProvider with ChangeNotifier { gettingUpdates = false; } } - if (errors.content.isNotEmpty) { + if (errors.idsByErrorString.isNotEmpty) { var res = Map(); res['errors'] = errors; res['updates'] = updates; @@ -1392,15 +1392,16 @@ Future bgUpdateCheck(int taskId, Map? params) async { List toNotify = []; List> toRetry = []; var retryAfterXSeconds = 0; - List> toThrow = []; + List toThrow = []; var networkRestricted = false; if (appsProvider.settingsProvider.bgUpdatesOnWiFiOnly) { var netResult = await (Connectivity().checkConnectivity()); networkRestricted = (netResult != ConnectivityResult.wifi) && (netResult != ConnectivityResult.ethernet); } + MultiAppMultiError? errors; CheckingUpdatesNotification notif = - CheckingUpdatesNotification(plural('app', toCheck.length)); + CheckingUpdatesNotification(plural('apps', toCheck.length)); try { // Check for updates @@ -1411,8 +1412,8 @@ Future bgUpdateCheck(int taskId, Map? params) async { // If there were errors, group them into toRetry and toThrow if (e is Map) { updates = e['updates']; - MultiAppMultiError errors = e['errors']; - errors.rawErrors.forEach((key, err) { + errors = e['errors']; + errors!.rawErrors.forEach((key, err) { logs.add( 'BG update task $taskId: Got error on checking for $key \'${err.toString()}\'.'); var toCheckApp = toCheck.where((element) => element.key == key).first; @@ -1422,12 +1423,12 @@ Future bgUpdateCheck(int taskId, Map? params) async { ? (err.remainingMinutes * 60) : e is ClientException ? (15 * 60) - : pow(toCheckApp.value, 2).toInt(); + : pow(toCheckApp.value + 1, 2).toInt(); if (minRetryIntervalForThisApp > retryAfterXSeconds) { retryAfterXSeconds = minRetryIntervalForThisApp; } } else { - toThrow.add(MapEntry(key, err)); + toThrow.add(key); } }); } else { @@ -1456,9 +1457,9 @@ Future bgUpdateCheck(int taskId, Map? params) async { // Send the error notifications if (toThrow.isNotEmpty) { - for (var element in toThrow) { + for (var appId in toThrow) { notificationsProvider.notify(ErrorCheckingUpdatesNotification( - '${element.key}: ${element.value.toString()}', + errors!.errorString(appId), id: Random().nextInt(10000))); } }