From 81c0c9d4aa4f6ae61e0dbcde5ac66cf1a6fa9435 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Mon, 28 Oct 2024 22:47:39 +0100 Subject: [PATCH] Drop stats/known_apks.txt Use repo/index-v2.json instead. Also fix deprecated datetime.utcnow(). --- MANIFEST.in | 1 - examples/config.yml | 5 --- fdroidserver/common.py | 72 ++++++-------------------------------- fdroidserver/index.py | 2 +- fdroidserver/nightly.py | 7 ++-- fdroidserver/scanner.py | 6 ++-- fdroidserver/update.py | 12 +++---- tests/common.TestCase | 2 -- tests/config.py | 2 -- tests/nightly.TestCase | 2 -- tests/run-tests | 12 +++---- tests/scanner.TestCase | 8 +++-- tests/stats/known_apks.txt | 18 ---------- tests/update.TestCase | 4 --- 14 files changed, 33 insertions(+), 120 deletions(-) delete mode 100644 tests/stats/known_apks.txt diff --git a/MANIFEST.in b/MANIFEST.in index 295d9832..0e24a132 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -860,7 +860,6 @@ include tests/source-files/ut.ewh.audiometrytest/settings.gradle include tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties include tests/source-files/Zillode/syncthing-silk/build.gradle include tests/SpeedoMeterApp.main_1.apk -include tests/stats/known_apks.txt include tests/testcommon.py include tests/test-gradlew-fdroid include tests/triple-t-2/build/org.piwigo.android/app/build.gradle diff --git a/examples/config.yml b/examples/config.yml index 1f722244..cdcbb731 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -401,11 +401,6 @@ # configured to allow push access (e.g. ssh key, username/password, etc) # binary_transparency_remote: git@gitlab.com:fdroid/binary-transparency-log.git -# If you want to keep the "added" and "last updated" dates for each -# app and APK in your repo, enable this. The name comes from an old -# system for tracking statistics that is no longer included. -# update_stats: true - # Set this to true to always use a build server. This saves specifying the # --server option on dedicated secure build server hosts. # build_server_always: true diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 6d459c7c..bc9413c1 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -162,7 +162,6 @@ default_config = { 'make_current_version_link': False, 'current_version_name_source': 'Name', 'deploy_process_logs': False, - 'update_stats': False, 'repo_maxage': 0, 'build_server_always': False, 'keystore': 'keystore.p12', @@ -2555,40 +2554,17 @@ class KnownApks: this is parsed as a list from the end to allow the filename to have any combo of spaces. """ - self.path = os.path.join('stats', 'known_apks.txt') self.apks = {} - if os.path.isfile(self.path): - with open(self.path, 'r', encoding='utf-8') as f: - for line in f: - t = line.rstrip().split(' ') - if len(t) == 2: - self.apks[t[0]] = (t[1], None) - else: - appid = t[-2] - date = datetime.strptime(t[-1], '%Y-%m-%d') - filename = line[0:line.rfind(appid) - 1] - self.apks[filename] = (appid, date) - check_system_clock(date, self.path) - self.changed = False - - def writeifchanged(self): - if not self.changed: - return - - if not os.path.exists('stats'): - os.mkdir('stats') - - lst = [] - for apk, app in self.apks.items(): - appid, added = app - line = apk + ' ' + appid - if added: - line += ' ' + added.strftime('%Y-%m-%d') - lst.append(line) - - with open(self.path, 'w') as f: - for line in sorted(lst, key=natural_key): - f.write(line + '\n') + for part in ('repo', 'archive'): + path = os.path.join(part, 'index-v2.json') + if os.path.isfile(path): + with open(path, 'r', encoding='utf-8') as f: + index = json.load(f) + for appid, data in index["packages"].items(): + for version in data["versions"].values(): + filename = version["file"]["name"][1:] + date = datetime.fromtimestamp(version["added"] // 1000, tz=timezone.utc) + self.apks[filename] = (appid, date) def recordapk(self, apkName, app, default_date=None): """ @@ -2601,38 +2577,12 @@ class KnownApks: """ if apkName not in self.apks: if default_date is None: - default_date = datetime.utcnow() + default_date = datetime.now(timezone.utc) self.apks[apkName] = (app, default_date) self.changed = True _ignored, added = self.apks[apkName] return added - def getapp(self, apkname): - """Look up information - given the 'apkname'. - - Returns (app id, date added/None). - Or returns None for an unknown apk. - """ - if apkname in self.apks: - return self.apks[apkname] - return None - - def getlatest(self, num): - """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first.""" - apps = {} - for apk, app in self.apks.items(): - appid, added = app - if added: - if appid in apps: - if apps[appid] > added: - apps[appid] = added - else: - apps[appid] = added - sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:] - lst = [app for app, _ignored in sortedapps] - lst.reverse() - return lst - def get_file_extension(filename): """Get the normalized file extension, can be blank string but never None.""" diff --git a/fdroidserver/index.py b/fdroidserver/index.py index a3bcf9a3..5ca59662 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -77,7 +77,7 @@ def make(apps, apks, repodir, archive): sortedapps[appid] = apps[appid] repodict = collections.OrderedDict() - repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc) + repodict['timestamp'] = datetime.now(timezone.utc) repodict['version'] = METADATA_VERSION if common.config['repo_maxage'] != 0: diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 1b7251ee..84c2da2e 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -367,7 +367,7 @@ You can use it with the [F-Droid](https://f-droid.org/) Android app. Last updated: {date}'''.format(repo_git_base=repo_git_base, repo_url=repo_url, - date=datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')) + date=datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')) with open(readme_path, 'w') as fp: fp.write(readme) mirror_git_repo.git.add(all=True) @@ -422,7 +422,6 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, 'keypass': PASSWORD, 'keydname': DISTINGUISHED_NAME, 'make_current_version_link': False, - 'update_stats': True, } with open('config.yml', 'w') as fp: yaml.dump(config, fp, default_flow_style=False) @@ -496,7 +495,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, common.local_rsync( options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/' ) - common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/') + stats = repo_basedir + '/stats/' + if os.path.exists(stats): + common.local_rsync(options, [stats], git_mirror_statsdir + '/') mirror_git_repo.git.add(all=True) mirror_git_repo.index.commit("update app metadata") diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index d8251e23..6a392b2b 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -29,7 +29,7 @@ import urllib.request import zipfile from argparse import ArgumentParser from dataclasses import dataclass, field, fields -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from enum import IntEnum from pathlib import Path from tempfile import TemporaryDirectory @@ -330,7 +330,7 @@ def get_embedded_classes(apkfile, depth=0): def _datetime_now(): """Get datetime.now(), using this funciton allows mocking it for testing.""" - return datetime.utcnow() + return datetime.now(timezone.utc) def _scanner_cachedir(): @@ -389,7 +389,7 @@ class SignatureDataController: last_updated = self.data.get("last_updated", None) if last_updated: try: - last_updated = datetime.fromtimestamp(last_updated) + last_updated = datetime.fromtimestamp(last_updated, timezone.utc) except ValueError as e: raise SignatureDataMalformedException() from e except TypeError as e: diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 81bbe8bb..d5c3d922 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -362,7 +362,7 @@ def get_cache(): if not isinstance(v['antiFeatures'], dict): v['antiFeatures'] = {k: {} for k in sorted(v['antiFeatures'])} if 'added' in v: - v['added'] = datetime.fromtimestamp(v['added']) + v['added'] = datetime.fromtimestamp(v['added'], tz=timezone.utc) return apkcache @@ -2001,7 +2001,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir) if use_date_from_apk: - default_date_param = datetime.fromtimestamp(os.stat(apkfile).st_mtime) + default_date_param = datetime.fromtimestamp(os.stat(apkfile).st_mtime, tz=timezone.utc) else: default_date_param = None @@ -2420,12 +2420,12 @@ def create_metadata_from_template(apk): def read_added_date_from_all_apks(apps, apks): """No summary. - Added dates come from the stats/known_apks.txt file but are + Added dates come from the repo/index-v2.json file but are read when scanning apks and thus need to be applied form apk level to app level for _all_ apps and not only from non-archived ones - TODO: read the added dates directly from known_apks.txt instead of + TODO: read the added dates directly from index-v2.json instead of going through apks that way it also works for for repos that don't keep an archive of apks. """ @@ -2772,10 +2772,6 @@ def main(): from . import btlog btlog.make_binary_transparency_log(repodirs) - if config['update_stats']: - # Update known apks info... - knownapks.writeifchanged() - status_update_json(apps, apks + archapks) logging.info(_("Finished")) diff --git a/tests/common.TestCase b/tests/common.TestCase index a3a54bb7..385033cf 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1898,7 +1898,6 @@ class CommonTest(unittest.TestCase): self.assertFalse(os.path.exists('config.yml')) self.assertFalse(os.path.exists('config.py')) config = fdroidserver.common.read_config() - self.assertFalse(config.get('update_stats')) self.assertIsNotNone(config.get('char_limits')) def test_with_zero_size_config(self): @@ -1908,7 +1907,6 @@ class CommonTest(unittest.TestCase): self.assertTrue(os.path.exists('config.yml')) self.assertFalse(os.path.exists('config.py')) config = fdroidserver.common.read_config() - self.assertFalse(config.get('update_stats')) self.assertIsNotNone(config.get('char_limits')) def test_with_config_yml(self): diff --git a/tests/config.py b/tests/config.py index 899ee68c..fa118db2 100644 --- a/tests/config.py +++ b/tests/config.py @@ -25,8 +25,6 @@ mirrors = ( 'https://foo.bar/fdroid', ) -update_stats = True - install_list = 'org.adaway' uninstall_list = ('com.android.vending', 'com.facebook.orca', ) diff --git a/tests/nightly.TestCase b/tests/nightly.TestCase index 44e85f04..b1f9a8eb 100755 --- a/tests/nightly.TestCase +++ b/tests/nightly.TestCase @@ -274,7 +274,6 @@ class NightlyTest(unittest.TestCase): 'repo_name': 'f-droid/test-nightly', 'repo_url': mirror_url + '/repo', 'servergitmirrors': [{"url": git_url}], - 'update_stats': True, } with open('config.yml') as fp: config = yaml.safe_load(fp) @@ -347,7 +346,6 @@ class NightlyTest(unittest.TestCase): 'repo_name': 'fdroid/test-nightly', 'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo', 'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly'}], - 'update_stats': True, } with open('config.yml') as fp: config = yaml.safe_load(fp) diff --git a/tests/run-tests b/tests/run-tests index 31cb4939..6ba54e03 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -273,7 +273,6 @@ cp -a \ $WORKSPACE/tests/config \ $WORKSPACE/tests/metadata \ $WORKSPACE/tests/repo \ - $WORKSPACE/tests/stats \ $REPOROOT/ cp -a $WORKSPACE/tests/gnupghome $GNUPGHOME chmod 0700 $GNUPGHOME @@ -645,14 +644,13 @@ printf '\narchive_older: 3\n' >> config.yml mkdir -p {repo,archive,metadata,stats} cp $WORKSPACE/tests/repo/com.politedroid_5.apk archive cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo +cp $WORKSPACE/tests/repo/index-v2.json repo cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata #TODO: the timestamp of the oldest apk in the file should be used, even if that # doesn't exist anymore -echo "com.politedroid_4.apk com.politedroid 2016-01-01" > stats/known_apks.txt -echo "com.politedroid_5.apk com.politedroid 2017-01-01" >> stats/known_apks.txt -echo "com.politedroid_6.apk com.politedroid 2018-01-01" >> stats/known_apks.txt $sed -i -e 's/ArchivePolicy:.*/ArchivePolicy: 1 versions/' metadata/com.politedroid.yml -timestamp=1483228800 # $(date -u --date=2017-01-01 +%s)000 +timestamp=1498176000000 # $(date -u --date=2017-01-01 +%s)000 +pwd $fdroid update --pretty --nosign grep -F "\"added\": $timestamp" repo/index-v1.json @@ -1078,7 +1076,7 @@ GIT_REMOTE=`create_test_dir` GNUPGHOME=$REPOROOT/gnupghome cd $REPOROOT fdroid_init_with_prebuilt_keystore -cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/ +cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $REPOROOT/ echo "binary_transparency_remote: $GIT_REMOTE" >> config.yml $fdroid update --verbose $fdroid deploy --verbose @@ -1224,7 +1222,7 @@ $git config receive.denyCurrentBranch updateInstead cd $OFFLINE_ROOT fdroid_init_with_prebuilt_keystore printf '\narchive_older: 3\n' >> config.yml -cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $OFFLINE_ROOT/ +cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $OFFLINE_ROOT/ mkdir $OFFLINE_ROOT/unsigned cp $WORKSPACE/tests/urzip-release-unsigned.apk $OFFLINE_ROOT/unsigned diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index 83b6e8ff..6c965723 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -14,7 +14,7 @@ import unittest import uuid import zipfile from dataclasses import asdict -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from unittest import mock if sys.version_info >= (3, 11): @@ -592,7 +592,7 @@ class Test_SignatureDataController(unittest.TestCase): sdc = fdroidserver.scanner.SignatureDataController( 'nnn', 'fff.yml', 'https://example.com/test.json' ) - sdc.data['last_updated'] = datetime.utcnow().timestamp() + sdc.data['last_updated'] = datetime.now(timezone.utc).timestamp() sdc.check_last_updated() def test_check_last_updated_exception_cache_outdated(self): @@ -600,7 +600,9 @@ class Test_SignatureDataController(unittest.TestCase): 'nnn', 'fff.yml', 'https://example.com/test.json' ) sdc.cache_duration = timedelta(days=7) - sdc.data['last_updated'] = (datetime.utcnow() - timedelta(days=30)).timestamp() + sdc.data['last_updated'] = ( + datetime.now(timezone.utc) - timedelta(days=30) + ).timestamp() with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException): sdc.check_last_updated() diff --git a/tests/stats/known_apks.txt b/tests/stats/known_apks.txt deleted file mode 100644 index ee162b10..00000000 --- a/tests/stats/known_apks.txt +++ /dev/null @@ -1,18 +0,0 @@ -com.example.test.helloworld_1.apk com.example.test.helloworld 2023-02-17 -com.politedroid_3.apk com.politedroid 2017-06-23 -com.politedroid_4.apk com.politedroid 2017-06-23 -com.politedroid_5.apk com.politedroid 2017-06-23 -com.politedroid_6.apk com.politedroid 2017-06-23 -duplicate.permisssions_9999999.apk duplicate.permisssions 2017-12-22 -fake.ota.update_1234.zip fake.ota.update 2016-03-10 -info.zwanenburg.caffeinetile_4.apk info.zwanenburg.caffeinetile 2018-10-10 -no.min.target.sdk_987.apk no.min.target.sdk 2018-10-10 -obb.main.oldversion_1444412523.apk obb.main.oldversion 2013-12-31 -obb.main.twoversions_1101613.apk obb.main.twoversions 2015-10-12 -obb.main.twoversions_1101615.apk obb.main.twoversions 2016-01-01 -obb.main.twoversions_1101617.apk obb.main.twoversions 2016-06-20 -obb.mainpatch.current_1619.apk obb.mainpatch.current 2016-04-23 -obb.mainpatch.current_1619_another-release-key.apk obb.mainpatch.current 2017-06-01 -souch.smsbypass_9.apk souch.smsbypass 2018-04-26 -urzip-; Рахма́, [rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢·.apk info.guardianproject.urzip 2016-06-23 -v1.v2.sig_1020.apk v1.v2.sig 2023-02-17 diff --git a/tests/update.TestCase b/tests/update.TestCase index bf5dc7e7..ce91339d 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -697,14 +697,10 @@ class UpdateTest(unittest.TestCase): os.chdir(self.testdir) os.mkdir('repo') - os.mkdir('stats') - with open(os.path.join('stats', 'known_apks.txt'), 'w') as fp: - fp.write('se.manyver_30.apk se.manyver 2018-10-10\n') filename = 'Norway_bouvet_europe_2.obf.zip' shutil.copy(os.path.join(self.basedir, filename), 'repo') knownapks = fdroidserver.common.KnownApks() files, fcachechanged = fdroidserver.update.scan_repo_files(dict(), 'repo', knownapks, False) - knownapks.writeifchanged() self.assertTrue(fcachechanged) info = files[0]