From fb9e66332df81a8f27a7bdcb69e7c2da2bb26dd0 Mon Sep 17 00:00:00 2001 From: Imran Remtulla Date: Fri, 5 May 2023 22:35:32 -0400 Subject: [PATCH] APKPure, SourceHut, Bugfixes --- lib/app_sources/apkcombo.dart | 116 ++++++++++++++++++++ lib/app_sources/apkmirror.dart | 2 +- lib/app_sources/apkpure.dart | 69 ++++++++++++ lib/app_sources/codeberg.dart | 1 + lib/app_sources/fdroid.dart | 4 +- lib/app_sources/fdroidrepo.dart | 3 +- lib/app_sources/github.dart | 5 +- lib/app_sources/gitlab.dart | 3 +- lib/app_sources/html.dart | 170 +++++++++++++++-------------- lib/app_sources/izzyondroid.dart | 4 +- lib/app_sources/jenkins.dart | 6 +- lib/app_sources/mullvad.dart | 2 +- lib/app_sources/neutroncode.dart | 2 +- lib/app_sources/signal.dart | 2 +- lib/app_sources/sourceforge.dart | 3 +- lib/app_sources/sourcehut.dart | 101 +++++++++++++++++ lib/app_sources/steammobile.dart | 2 +- lib/app_sources/telegramapp.dart | 2 +- lib/app_sources/vlc.dart | 6 +- lib/app_sources/whatsapp.dart | 16 +-- lib/pages/add_app.dart | 6 +- lib/providers/apps_provider.dart | 20 ++-- lib/providers/source_provider.dart | 22 +++- 23 files changed, 445 insertions(+), 122 deletions(-) create mode 100644 lib/app_sources/apkcombo.dart create mode 100644 lib/app_sources/apkpure.dart create mode 100644 lib/app_sources/sourcehut.dart diff --git a/lib/app_sources/apkcombo.dart b/lib/app_sources/apkcombo.dart new file mode 100644 index 0000000..6c5dfa6 --- /dev/null +++ b/lib/app_sources/apkcombo.dart @@ -0,0 +1,116 @@ +import 'package:easy_localization/easy_localization.dart'; +import 'package:html/parser.dart'; +import 'package:http/http.dart'; +import 'package:obtainium/custom_errors.dart'; +import 'package:obtainium/providers/source_provider.dart'; + +class APKCombo extends AppSource { + APKCombo() { + host = 'apkcombo.com'; + } + + @override + String sourceSpecificStandardizeURL(String url) { + RegExp standardUrlRegEx = RegExp('^https?://$host/+[^/]+/+[^/]+'); + var match = standardUrlRegEx.firstMatch(url.toLowerCase()); + if (match == null) { + throw InvalidURLError(name); + } + return url.substring(0, match.end); + } + + @override + String? tryInferringAppId(String standardUrl, + {Map additionalSettings = const {}}) { + return Uri.parse(standardUrl).pathSegments.last; + } + + @override + Map get requestHeaders => { + "User-Agent": "curl/8.0.1", + "Accept": "*/*", + "Connection": "keep-alive", + "Host": "$host" + }; + + Future>> getApkUrls(String standardUrl) async { + var res = await sourceRequest('$standardUrl/download/apk'); + if (res.statusCode != 200) { + throw getObtainiumHttpError(res); + } + var html = parse(res.body); + return html + .querySelectorAll('#variants-tab > div > ul > li') + .map((e) { + String? arch = e + .querySelector('code') + ?.text + .trim() + .replaceAll(',', '') + .replaceAll(':', '-') + .replaceAll(' ', '-'); + return e.querySelectorAll('a').map((e) { + String? url = e.attributes['href']; + if (url != null && + !Uri.parse(url).path.toLowerCase().endsWith('.apk')) { + url = null; + } + String verCode = + e.querySelector('.info .header .vercode')?.text.trim() ?? ''; + return MapEntry( + arch != null ? '$arch-$verCode.apk' : '', url ?? ''); + }).toList(); + }) + .reduce((value, element) => [...value, ...element]) + .where((element) => element.value.isNotEmpty) + .toList(); + } + + @override + Future apkUrlPrefetchModifier( + String apkUrl, String standardUrl) async { + var freshURLs = await getApkUrls(standardUrl); + var path2Match = Uri.parse(apkUrl).path; + for (var url in freshURLs) { + if (Uri.parse(url.value).path == path2Match) { + return url.value; + } + } + throw NoAPKError(); + } + + @override + Future getLatestAPKDetails( + String standardUrl, + Map additionalSettings, + ) async { + String appId = tryInferringAppId(standardUrl)!; + String host = Uri.parse(standardUrl).host; + var preres = await sourceRequest(standardUrl); + if (preres.statusCode != 200) { + throw getObtainiumHttpError(preres); + } + var res = parse(preres.body); + String? version = res.querySelector('div.version')?.text.trim(); + if (version == null) { + throw NoVersionError(); + } + String appName = res.querySelector('div.app_name')?.text.trim() ?? appId; + String author = res.querySelector('div.author')?.text.trim() ?? appName; + List infoArray = res + .querySelectorAll('div.information-table > .item > div.value') + .map((e) => e.text.trim()) + .toList(); + DateTime? releaseDate; + if (infoArray.length >= 2) { + try { + releaseDate = DateFormat('MMMM d, yyyy').parse(infoArray[1]); + } catch (e) { + // ignore + } + } + return APKDetails( + version, await getApkUrls(standardUrl), AppNames(author, appName), + releaseDate: releaseDate); + } +} diff --git a/lib/app_sources/apkmirror.dart b/lib/app_sources/apkmirror.dart index 396e599..2265cc7 100644 --- a/lib/app_sources/apkmirror.dart +++ b/lib/app_sources/apkmirror.dart @@ -57,7 +57,7 @@ class APKMirror extends AppSource { true ? additionalSettings['filterReleaseTitlesByRegEx'] : null; - Response res = await get(Uri.parse('$standardUrl/feed')); + Response res = await sourceRequest('$standardUrl/feed'); if (res.statusCode == 200) { var items = parse(res.body).querySelectorAll('item'); dynamic targetRelease; diff --git a/lib/app_sources/apkpure.dart b/lib/app_sources/apkpure.dart new file mode 100644 index 0000000..437ae4c --- /dev/null +++ b/lib/app_sources/apkpure.dart @@ -0,0 +1,69 @@ +import 'package:easy_localization/easy_localization.dart'; +import 'package:html/parser.dart'; +import 'package:http/http.dart'; +import 'package:obtainium/custom_errors.dart'; +import 'package:obtainium/providers/source_provider.dart'; + +class APKPure extends AppSource { + APKPure() { + host = 'apkpure.com'; + } + + @override + String sourceSpecificStandardizeURL(String url) { + RegExp standardUrlRegExB = RegExp('^https?://m.$host/+[^/]+/+[^/]+'); + RegExpMatch? match = standardUrlRegExB.firstMatch(url.toLowerCase()); + if (match != null) { + url = 'https://$host/${Uri.parse(url).path}'; + } + RegExp standardUrlRegExA = RegExp('^https?://$host/+[^/]+/+[^/]+'); + match = standardUrlRegExA.firstMatch(url.toLowerCase()); + if (match == null) { + throw InvalidURLError(name); + } + return url.substring(0, match.end); + } + + @override + String? tryInferringAppId(String standardUrl, + {Map additionalSettings = const {}}) { + return Uri.parse(standardUrl).pathSegments.last; + } + + @override + Future getLatestAPKDetails( + String standardUrl, + Map additionalSettings, + ) async { + String appId = tryInferringAppId(standardUrl)!; + String host = Uri.parse(standardUrl).host; + var res = await sourceRequest('$standardUrl/download'); + if (res.statusCode == 200) { + var html = parse(res.body); + String? version = html.querySelector('span.info-sdk span')?.text.trim(); + if (version == null) { + throw NoVersionError(); + } + String? dateString = + html.querySelector('span.info-other span.date')?.text.trim(); + DateTime? releaseDate = dateString != null + ? DateFormat('MMMM d, yyyy').parse(dateString) + : null; + List> apkUrls = [ + MapEntry('$appId.apk', 'https://d.$host/b/APK/$appId?version=latest') + ]; + String author = html + .querySelector('span.info-sdk') + ?.text + .trim() + .substring(version.length + 4) ?? + Uri.parse(standardUrl).pathSegments.reversed.last; + String appName = + html.querySelector('h1.info-title')?.text.trim() ?? appId; + return APKDetails(version, apkUrls, AppNames(author, appName), + releaseDate: releaseDate); + } else { + throw getObtainiumHttpError(res); + } + } +} diff --git a/lib/app_sources/codeberg.dart b/lib/app_sources/codeberg.dart index a45d620..d50332d 100644 --- a/lib/app_sources/codeberg.dart +++ b/lib/app_sources/codeberg.dart @@ -9,6 +9,7 @@ import 'package:obtainium/providers/source_provider.dart'; class Codeberg extends AppSource { Codeberg() { host = 'codeberg.org'; + overrideEligible = true; additionalSourceSpecificSettingFormItems = []; diff --git a/lib/app_sources/fdroid.dart b/lib/app_sources/fdroid.dart index 318a983..88b4f5e 100644 --- a/lib/app_sources/fdroid.dart +++ b/lib/app_sources/fdroid.dart @@ -66,14 +66,14 @@ class FDroid extends AppSource { String? appId = tryInferringAppId(standardUrl); String host = Uri.parse(standardUrl).host; return getAPKUrlsFromFDroidPackagesAPIResponse( - await get(Uri.parse('https://$host/api/v1/packages/$appId')), + await sourceRequest('https://$host/api/v1/packages/$appId'), 'https://$host/repo/$appId', standardUrl); } @override Future>> search(String query) async { - Response res = await get(Uri.parse('https://search.$host/?q=$query')); + Response res = await sourceRequest('https://search.$host/?q=$query'); if (res.statusCode == 200) { Map> urlsWithDescriptions = {}; parse(res.body).querySelectorAll('.package-header').forEach((e) { diff --git a/lib/app_sources/fdroidrepo.dart b/lib/app_sources/fdroidrepo.dart index 33c8edf..184e139 100644 --- a/lib/app_sources/fdroidrepo.dart +++ b/lib/app_sources/fdroidrepo.dart @@ -8,6 +8,7 @@ import 'package:obtainium/providers/source_provider.dart'; class FDroidRepo extends AppSource { FDroidRepo() { name = tr('fdroidThirdPartyRepo'); + overrideEligible = true; additionalSourceAppSpecificSettingFormItems = [ [ @@ -28,7 +29,7 @@ class FDroidRepo extends AppSource { if (appIdOrName == null) { throw NoReleasesError(); } - var res = await get(Uri.parse('$standardUrl/index.xml')); + var res = await sourceRequest('$standardUrl/index.xml'); if (res.statusCode == 200) { var body = parse(res.body); var foundApps = body.querySelectorAll('application').where((element) { diff --git a/lib/app_sources/github.dart b/lib/app_sources/github.dart index e6b67ae..a9a76c3 100644 --- a/lib/app_sources/github.dart +++ b/lib/app_sources/github.dart @@ -11,6 +11,7 @@ import 'package:url_launcher/url_launcher_string.dart'; class GitHub extends AppSource { GitHub() { host = 'github.com'; + overrideEligible = true; additionalSourceSpecificSettingFormItems = [ GeneratedFormTextField('github-creds', @@ -108,7 +109,7 @@ class GitHub extends AppSource { true ? additionalSettings['filterReleaseTitlesByRegEx'] : null; - Response res = await get(Uri.parse(requestUrl)); + Response res = await sourceRequest(requestUrl); if (res.statusCode == 200) { var releases = jsonDecode(res.body) as List; @@ -216,7 +217,7 @@ class GitHub extends AppSource { Future>> searchCommon( String query, String requestUrl, String rootProp, {Function(Response)? onHttpErrorCode}) async { - Response res = await get(Uri.parse(requestUrl)); + Response res = await sourceRequest(requestUrl); if (res.statusCode == 200) { Map> urlsWithDescriptions = {}; for (var e in (jsonDecode(res.body)[rootProp] as List)) { diff --git a/lib/app_sources/gitlab.dart b/lib/app_sources/gitlab.dart index 150cb04..07f8369 100644 --- a/lib/app_sources/gitlab.dart +++ b/lib/app_sources/gitlab.dart @@ -9,6 +9,7 @@ import 'package:easy_localization/easy_localization.dart'; class GitLab extends AppSource { GitLab() { host = 'gitlab.com'; + overrideEligible = true; additionalSourceAppSpecificSettingFormItems = [ [ @@ -39,7 +40,7 @@ class GitLab extends AppSource { ) async { bool fallbackToOlderReleases = additionalSettings['fallbackToOlderReleases'] == true; - Response res = await get(Uri.parse('$standardUrl/-/tags?format=atom')); + Response res = await sourceRequest('$standardUrl/-/tags?format=atom'); if (res.statusCode == 200) { var standardUri = Uri.parse(standardUrl); var parsedHtml = parse(res.body); diff --git a/lib/app_sources/html.dart b/lib/app_sources/html.dart index c4c039f..d9ec173 100644 --- a/lib/app_sources/html.dart +++ b/lib/app_sources/html.dart @@ -4,84 +4,109 @@ import 'package:http/http.dart'; import 'package:obtainium/custom_errors.dart'; import 'package:obtainium/providers/source_provider.dart'; +String ensureAbsoluteUrl(String ambiguousUrl, Uri referenceAbsoluteUrl) { + try { + Uri.parse(ambiguousUrl).origin; + return ambiguousUrl; + } catch (err) { + // is relative + } + var currPathSegments = referenceAbsoluteUrl.path + .split('/') + .where((element) => element.trim().isNotEmpty) + .toList(); + if (ambiguousUrl.startsWith('/') || currPathSegments.isEmpty) { + return '${referenceAbsoluteUrl.origin}/$ambiguousUrl'; + } else if (ambiguousUrl.split('/').length == 1) { + return '${referenceAbsoluteUrl.origin}/${currPathSegments.join('/')}/$ambiguousUrl'; + } else { + return '${referenceAbsoluteUrl.origin}/${currPathSegments.sublist(0, currPathSegments.length - 1).join('/')}/$ambiguousUrl'; + } +} + +int compareAlphaNumeric(String a, String b) { + List aParts = _splitAlphaNumeric(a); + List bParts = _splitAlphaNumeric(b); + + for (int i = 0; i < aParts.length && i < bParts.length; i++) { + String aPart = aParts[i]; + String bPart = bParts[i]; + + bool aIsNumber = _isNumeric(aPart); + bool bIsNumber = _isNumeric(bPart); + + if (aIsNumber && bIsNumber) { + int aNumber = int.parse(aPart); + int bNumber = int.parse(bPart); + int cmp = aNumber.compareTo(bNumber); + if (cmp != 0) { + return cmp; + } + } else if (!aIsNumber && !bIsNumber) { + int cmp = aPart.compareTo(bPart); + if (cmp != 0) { + return cmp; + } + } else { + // Alphanumeric strings come before numeric strings + return aIsNumber ? 1 : -1; + } + } + + return aParts.length.compareTo(bParts.length); +} + +List _splitAlphaNumeric(String s) { + List parts = []; + StringBuffer sb = StringBuffer(); + + bool isNumeric = _isNumeric(s[0]); + sb.write(s[0]); + + for (int i = 1; i < s.length; i++) { + bool currentIsNumeric = _isNumeric(s[i]); + if (currentIsNumeric == isNumeric) { + sb.write(s[i]); + } else { + parts.add(sb.toString()); + sb.clear(); + sb.write(s[i]); + isNumeric = currentIsNumeric; + } + } + + parts.add(sb.toString()); + + return parts; +} + +bool _isNumeric(String s) { + return s.codeUnitAt(0) >= 48 && s.codeUnitAt(0) <= 57; +} + class HTML extends AppSource { + HTML() { + overrideEligible = true; + } + @override String sourceSpecificStandardizeURL(String url) { return url; } - int compareAlphaNumeric(String a, String b) { - List aParts = _splitAlphaNumeric(a); - List bParts = _splitAlphaNumeric(b); - - for (int i = 0; i < aParts.length && i < bParts.length; i++) { - String aPart = aParts[i]; - String bPart = bParts[i]; - - bool aIsNumber = _isNumeric(aPart); - bool bIsNumber = _isNumeric(bPart); - - if (aIsNumber && bIsNumber) { - int aNumber = int.parse(aPart); - int bNumber = int.parse(bPart); - int cmp = aNumber.compareTo(bNumber); - if (cmp != 0) { - return cmp; - } - } else if (!aIsNumber && !bIsNumber) { - int cmp = aPart.compareTo(bPart); - if (cmp != 0) { - return cmp; - } - } else { - // Alphanumeric strings come before numeric strings - return aIsNumber ? 1 : -1; - } - } - - return aParts.length.compareTo(bParts.length); - } - - List _splitAlphaNumeric(String s) { - List parts = []; - StringBuffer sb = StringBuffer(); - - bool isNumeric = _isNumeric(s[0]); - sb.write(s[0]); - - for (int i = 1; i < s.length; i++) { - bool currentIsNumeric = _isNumeric(s[i]); - if (currentIsNumeric == isNumeric) { - sb.write(s[i]); - } else { - parts.add(sb.toString()); - sb.clear(); - sb.write(s[i]); - isNumeric = currentIsNumeric; - } - } - - parts.add(sb.toString()); - - return parts; - } - - bool _isNumeric(String s) { - return s.codeUnitAt(0) >= 48 && s.codeUnitAt(0) <= 57; - } - @override Future getLatestAPKDetails( String standardUrl, Map additionalSettings, ) async { var uri = Uri.parse(standardUrl); - Response res = await get(uri); + Response res = await sourceRequest(standardUrl); if (res.statusCode == 200) { List links = parse(res.body) .querySelectorAll('a') .map((element) => element.attributes['href'] ?? '') - .where((element) => element.toLowerCase().endsWith('.apk')) + .where((element) => + Uri.parse(element).path.toLowerCase().endsWith('.apk')) .toList(); links.sort( (a, b) => compareAlphaNumeric(a.split('/').last, b.split('/').last)); @@ -95,25 +120,8 @@ class HTML extends AppSource { var rel = links.last; var apkName = rel.split('/').last; var version = apkName.substring(0, apkName.length - 4); - List apkUrls = [rel].map((e) { - try { - Uri.parse(e).origin; - return e; - } catch (err) { - // is relative - } - var currPathSegments = uri.path - .split('/') - .where((element) => element.trim().isNotEmpty) - .toList(); - if (e.startsWith('/') || currPathSegments.isEmpty) { - return '${uri.origin}/$e'; - } else if (e.split('/').length == 1) { - return '${uri.origin}/${currPathSegments.join('/')}/$e'; - } else { - return '${uri.origin}/${currPathSegments.sublist(0, currPathSegments.length - 1).join('/')}/$e'; - } - }).toList(); + List apkUrls = + [rel].map((e) => ensureAbsoluteUrl(e, uri)).toList(); return APKDetails( version, getApkUrlsFromUrls(apkUrls), AppNames(uri.host, tr('app'))); } else { diff --git a/lib/app_sources/izzyondroid.dart b/lib/app_sources/izzyondroid.dart index 0fbceb9..cdab17a 100644 --- a/lib/app_sources/izzyondroid.dart +++ b/lib/app_sources/izzyondroid.dart @@ -31,8 +31,8 @@ class IzzyOnDroid extends AppSource { ) async { String? appId = tryInferringAppId(standardUrl); return FDroid().getAPKUrlsFromFDroidPackagesAPIResponse( - await get( - Uri.parse('https://apt.izzysoft.de/fdroid/api/v1/packages/$appId')), + await sourceRequest( + 'https://apt.izzysoft.de/fdroid/api/v1/packages/$appId'), 'https://android.izzysoft.de/frepo/$appId', standardUrl); } diff --git a/lib/app_sources/jenkins.dart b/lib/app_sources/jenkins.dart index 366b76d..804b3fb 100644 --- a/lib/app_sources/jenkins.dart +++ b/lib/app_sources/jenkins.dart @@ -6,6 +6,7 @@ import 'package:obtainium/providers/source_provider.dart'; class Jenkins extends AppSource { Jenkins() { + overrideEligible = true; overrideVersionDetectionFormDefault('releaseDateAsVersion', true); } @@ -30,7 +31,7 @@ class Jenkins extends AppSource { ) async { standardUrl = trimJobUrl(standardUrl); Response res = - await get(Uri.parse('$standardUrl/lastSuccessfulBuild/api/json')); + await sourceRequest('$standardUrl/lastSuccessfulBuild/api/json'); if (res.statusCode == 200) { var json = jsonDecode(res.body); var releaseDate = json['timestamp'] == null @@ -55,9 +56,6 @@ class Jenkins extends AppSource { .where((url) => url.value.isNotEmpty && url.key.toLowerCase().endsWith('.apk')) .toList(); - if (apkUrls.isEmpty) { - throw NoAPKError(); - } return APKDetails( version, apkUrls, diff --git a/lib/app_sources/mullvad.dart b/lib/app_sources/mullvad.dart index 64ef092..4e15d2f 100644 --- a/lib/app_sources/mullvad.dart +++ b/lib/app_sources/mullvad.dart @@ -28,7 +28,7 @@ class Mullvad extends AppSource { String standardUrl, Map additionalSettings, ) async { - Response res = await get(Uri.parse('$standardUrl/en/download/android')); + Response res = await sourceRequest('$standardUrl/en/download/android'); if (res.statusCode == 200) { var versions = parse(res.body) .querySelectorAll('p') diff --git a/lib/app_sources/neutroncode.dart b/lib/app_sources/neutroncode.dart index 45374a9..4fbec3c 100644 --- a/lib/app_sources/neutroncode.dart +++ b/lib/app_sources/neutroncode.dart @@ -78,7 +78,7 @@ class NeutronCode extends AppSource { String standardUrl, Map additionalSettings, ) async { - Response res = await get(Uri.parse(standardUrl)); + Response res = await sourceRequest(standardUrl); if (res.statusCode == 200) { var http = parse(res.body); var name = http.querySelector('.pd-title')?.innerHtml; diff --git a/lib/app_sources/signal.dart b/lib/app_sources/signal.dart index 40d6953..f7bda25 100644 --- a/lib/app_sources/signal.dart +++ b/lib/app_sources/signal.dart @@ -19,7 +19,7 @@ class Signal extends AppSource { Map additionalSettings, ) async { Response res = - await get(Uri.parse('https://updates.$host/android/latest.json')); + await sourceRequest('https://updates.$host/android/latest.json'); if (res.statusCode == 200) { var json = jsonDecode(res.body); String? apkUrl = json['url']; diff --git a/lib/app_sources/sourceforge.dart b/lib/app_sources/sourceforge.dart index ee2f22b..794eef1 100644 --- a/lib/app_sources/sourceforge.dart +++ b/lib/app_sources/sourceforge.dart @@ -6,6 +6,7 @@ import 'package:obtainium/providers/source_provider.dart'; class SourceForge extends AppSource { SourceForge() { host = 'sourceforge.net'; + overrideEligible = true; } @override @@ -29,7 +30,7 @@ class SourceForge extends AppSource { String standardUrl, Map additionalSettings, ) async { - Response res = await get(Uri.parse('$standardUrl/rss?path=/')); + Response res = await sourceRequest('$standardUrl/rss?path=/'); if (res.statusCode == 200) { var parsedHtml = parse(res.body); var allDownloadLinks = diff --git a/lib/app_sources/sourcehut.dart b/lib/app_sources/sourcehut.dart new file mode 100644 index 0000000..b85758f --- /dev/null +++ b/lib/app_sources/sourcehut.dart @@ -0,0 +1,101 @@ +import 'package:html/parser.dart'; +import 'package:http/http.dart'; +import 'package:obtainium/app_sources/github.dart'; +import 'package:obtainium/app_sources/html.dart'; +import 'package:obtainium/custom_errors.dart'; +import 'package:obtainium/providers/source_provider.dart'; +import 'package:obtainium/components/generated_form.dart'; +import 'package:easy_localization/easy_localization.dart'; + +class SourceHut extends AppSource { + SourceHut() { + host = 'git.sr.ht'; + overrideEligible = true; + + additionalSourceAppSpecificSettingFormItems = [ + [ + GeneratedFormSwitch('fallbackToOlderReleases', + label: tr('fallbackToOlderReleases'), defaultValue: true) + ] + ]; + } + + @override + String sourceSpecificStandardizeURL(String url) { + RegExp standardUrlRegEx = RegExp('^https?://$host/[^/]+/[^/]+'); + RegExpMatch? match = standardUrlRegEx.firstMatch(url.toLowerCase()); + if (match == null) { + throw InvalidURLError(name); + } + return url.substring(0, match.end); + } + + @override + String? changeLogPageFromStandardUrl(String standardUrl) => standardUrl; + + @override + Future getLatestAPKDetails( + String standardUrl, + Map additionalSettings, + ) async { + Uri standardUri = Uri.parse(standardUrl); + String appName = standardUri.pathSegments.last; + bool fallbackToOlderReleases = + additionalSettings['fallbackToOlderReleases'] == true; + Response res = await sourceRequest('$standardUrl/refs/rss.xml'); + if (res.statusCode == 200) { + var parsedHtml = parse(res.body); + List apkDetailsList = []; + int ind = 0; + + for (var entry in parsedHtml.querySelectorAll('item').sublist(0, 6)) { + // Limit 5 for speed + if (!fallbackToOlderReleases && ind > 0) { + break; + } + String? version = entry.querySelector('title')?.text.trim(); + if (version == null) { + throw NoVersionError(); + } + String? releaseDateString = entry.querySelector('pubDate')?.innerHtml; + var link = entry.querySelector('link'); + String releasePage = '$standardUrl/refs/$version'; + DateTime? releaseDate = releaseDateString != null + ? DateFormat('EEE, dd MMM yyyy HH:mm:ss Z').parse(releaseDateString) + : null; + var res2 = await sourceRequest(releasePage); + List> apkUrls = []; + if (res2.statusCode == 200) { + apkUrls = getApkUrlsFromUrls(parse(res2.body) + .querySelectorAll('a') + .map((e) => e.attributes['href'] ?? '') + .where((e) => e.toLowerCase().endsWith('.apk')) + .map((e) => ensureAbsoluteUrl(e, standardUri)) + .toList()); + } + apkDetailsList.add(APKDetails( + version, + apkUrls, + AppNames(entry.querySelector('author')?.innerHtml.trim() ?? appName, + appName), + releaseDate: releaseDate)); + ind++; + } + if (apkDetailsList.isEmpty) { + throw NoReleasesError(); + } + if (fallbackToOlderReleases) { + if (additionalSettings['trackOnly'] != true) { + apkDetailsList = + apkDetailsList.where((e) => e.apkUrls.isNotEmpty).toList(); + } + if (apkDetailsList.isEmpty) { + throw NoReleasesError(); + } + } + return apkDetailsList.first; + } else { + throw getObtainiumHttpError(res); + } + } +} diff --git a/lib/app_sources/steammobile.dart b/lib/app_sources/steammobile.dart index 305c89a..65c518b 100644 --- a/lib/app_sources/steammobile.dart +++ b/lib/app_sources/steammobile.dart @@ -29,7 +29,7 @@ class SteamMobile extends AppSource { String standardUrl, Map additionalSettings, ) async { - Response res = await get(Uri.parse('https://$host/mobile')); + Response res = await sourceRequest('https://$host/mobile'); if (res.statusCode == 200) { var apkNamePrefix = additionalSettings['app'] as String?; if (apkNamePrefix == null) { diff --git a/lib/app_sources/telegramapp.dart b/lib/app_sources/telegramapp.dart index 3c02156..44042e0 100644 --- a/lib/app_sources/telegramapp.dart +++ b/lib/app_sources/telegramapp.dart @@ -20,7 +20,7 @@ class TelegramApp extends AppSource { String standardUrl, Map additionalSettings, ) async { - Response res = await get(Uri.parse('https://t.me/s/TAndroidAPK')); + Response res = await sourceRequest('https://t.me/s/TAndroidAPK'); if (res.statusCode == 200) { var http = parse(res.body); var messages = diff --git a/lib/app_sources/vlc.dart b/lib/app_sources/vlc.dart index b105068..06ad878 100644 --- a/lib/app_sources/vlc.dart +++ b/lib/app_sources/vlc.dart @@ -19,8 +19,8 @@ class VLC extends AppSource { String standardUrl, Map additionalSettings, ) async { - Response res = await get( - Uri.parse('https://www.videolan.org/vlc/download-android.html')); + Response res = await sourceRequest( + 'https://www.videolan.org/vlc/download-android.html'); if (res.statusCode == 200) { var dwUrlBase = 'get.videolan.org/vlc-android'; var dwLinks = parse(res.body) @@ -38,7 +38,7 @@ class VLC extends AppSource { throw NoVersionError(); } String? targetUrl = 'https://$dwUrlBase/$version/'; - Response res2 = await get(Uri.parse(targetUrl)); + Response res2 = await sourceRequest(targetUrl); String mirrorDwBase = 'https://plug-mirror.rcac.purdue.edu/vlc/vlc-android/$version/'; List apkUrls = []; diff --git a/lib/app_sources/whatsapp.dart b/lib/app_sources/whatsapp.dart index 85deb84..f17d8a0 100644 --- a/lib/app_sources/whatsapp.dart +++ b/lib/app_sources/whatsapp.dart @@ -14,21 +14,21 @@ class WhatsApp extends AppSource { } @override - Future apkUrlPrefetchModifier(String apkUrl) async { - Response res = await get(Uri.parse('https://www.whatsapp.com/android')); + Future apkUrlPrefetchModifier( + String apkUrl, String standardUrl) async { + Response res = await sourceRequest('https://www.whatsapp.com/android'); if (res.statusCode == 200) { var targetLinks = parse(res.body) .querySelectorAll('a') - .map((e) => e.attributes['href']) - .where((e) => e != null) + .map((e) => e.attributes['href'] ?? '') + .where((e) => e.isNotEmpty) .where((e) => - e!.contains('scontent.whatsapp.net') && - e.contains('WhatsApp.apk')) + e.contains('content.whatsapp.net') && e.contains('WhatsApp.apk')) .toList(); if (targetLinks.isEmpty) { throw NoAPKError(); } - return targetLinks[0]!; + return targetLinks[0]; } else { throw getObtainiumHttpError(res); } @@ -39,7 +39,7 @@ class WhatsApp extends AppSource { String standardUrl, Map additionalSettings, ) async { - Response res = await get(Uri.parse('https://www.whatsapp.com/android')); + Response res = await sourceRequest('https://www.whatsapp.com/android'); if (res.statusCode == 200) { var targetElements = parse(res.body) .querySelectorAll('p') diff --git a/lib/pages/add_app.dart b/lib/pages/add_app.dart index b0654c3..3cef280 100644 --- a/lib/pages/add_app.dart +++ b/lib/pages/add_app.dart @@ -302,8 +302,10 @@ class _AddAppPageState extends State { 'overrideSource', defaultValue: HTML().runtimeType.toString(), [ - ...sourceProvider.sources.map( - (s) => MapEntry(s.runtimeType.toString(), s.name)) + ...sourceProvider.sources + .where((s) => s.overrideEligible) + .map((s) => + MapEntry(s.runtimeType.toString(), s.name)) ], label: tr('overrideSource')) ] diff --git a/lib/providers/apps_provider.dart b/lib/providers/apps_provider.dart index 9cc445c..e4f611c 100644 --- a/lib/providers/apps_provider.dart +++ b/lib/providers/apps_provider.dart @@ -125,10 +125,13 @@ class AppsProvider with ChangeNotifier { } downloadFile(String url, String fileName, Function? onProgress, - {bool useExisting = true}) async { + {bool useExisting = true, Map? headers}) async { var destDir = (await getExternalCacheDirectories())!.first.path; - StreamedResponse response = - await Client().send(Request('GET', Uri.parse(url))); + var req = Request('GET', Uri.parse(url)); + if (headers != null) { + req.headers.addAll(headers); + } + StreamedResponse response = await Client().send(req); File downloadedFile = File('$destDir/$fileName'); if (!(downloadedFile.existsSync() && useExisting)) { File tempDownloadedFile = File('${downloadedFile.path}.part'); @@ -170,15 +173,16 @@ class AppsProvider with ChangeNotifier { notifyListeners(); } try { - String downloadUrl = await SourceProvider() - .getSource(app.url, overrideSource: app.overrideSource) - .apkUrlPrefetchModifier(app.apkUrls[app.preferredApkIndex].value); + AppSource source = SourceProvider() + .getSource(app.url, overrideSource: app.overrideSource); + String downloadUrl = await source.apkUrlPrefetchModifier( + app.apkUrls[app.preferredApkIndex].value, app.url); var fileName = '${app.id}-${downloadUrl.hashCode}.apk'; var notif = DownloadNotification(app.finalName, 100); notificationsProvider?.cancel(notif.id); int? prevProg; - File downloadedFile = - await downloadFile(downloadUrl, fileName, (double? progress) { + File downloadedFile = await downloadFile(downloadUrl, fileName, + headers: source.requestHeaders, (double? progress) { int? prog = progress?.ceil(); if (apps[app.id] != null) { apps[app.id]!.downloadProgress = progress; diff --git a/lib/providers/source_provider.dart b/lib/providers/source_provider.dart index 556c294..f498496 100644 --- a/lib/providers/source_provider.dart +++ b/lib/providers/source_provider.dart @@ -7,7 +7,9 @@ import 'package:device_info_plus/device_info_plus.dart'; import 'package:easy_localization/easy_localization.dart'; import 'package:html/dom.dart'; import 'package:http/http.dart'; +import 'package:obtainium/app_sources/apkcombo.dart'; import 'package:obtainium/app_sources/apkmirror.dart'; +import 'package:obtainium/app_sources/apkpure.dart'; import 'package:obtainium/app_sources/codeberg.dart'; import 'package:obtainium/app_sources/fdroid.dart'; import 'package:obtainium/app_sources/fdroidrepo.dart'; @@ -20,6 +22,7 @@ import 'package:obtainium/app_sources/mullvad.dart'; import 'package:obtainium/app_sources/neutroncode.dart'; import 'package:obtainium/app_sources/signal.dart'; import 'package:obtainium/app_sources/sourceforge.dart'; +import 'package:obtainium/app_sources/sourcehut.dart'; import 'package:obtainium/app_sources/steammobile.dart'; import 'package:obtainium/app_sources/telegramapp.dart'; import 'package:obtainium/app_sources/vlc.dart'; @@ -315,6 +318,7 @@ abstract class AppSource { late String name; bool enforceTrackOnly = false; bool changeLogIfAnyIsMarkDown = true; + bool overrideEligible = false; AppSource() { name = runtimeType.toString(); @@ -344,6 +348,18 @@ abstract class AppSource { return url; } + Map? get requestHeaders => null; + + Future sourceRequest(String url) async { + if (requestHeaders != null) { + var req = Request('GET', Uri.parse(url)); + req.headers.addAll(requestHeaders!); + return Response.fromStream(await Client().send(req)); + } else { + return get(Uri.parse(url)); + } + } + String sourceSpecificStandardizeURL(String url) { throw NotImplementedError(); } @@ -410,7 +426,8 @@ abstract class AppSource { return null; } - Future apkUrlPrefetchModifier(String apkUrl) async { + Future apkUrlPrefetchModifier( + String apkUrl, String standardUrl) async { return apkUrl; } @@ -459,7 +476,10 @@ class SourceProvider { FDroidRepo(), Jenkins(), SourceForge(), + SourceHut(), APKMirror(), + APKPure(), + // APKCombo(), // Can't get past their scraping blocking yet (get 403 Forbidden) Mullvad(), Signal(), VLC(),