Drop stats/known_apks.txt
Use repo/index-v2.json instead. Also fix deprecated datetime.utcnow().
This commit is contained in:
Jochen Sprickerhof
Hans-Christoph Steiner
parent
83a3227235
commit
81c0c9d4aa
@ -860,7 +860,6 @@ include tests/source-files/ut.ewh.audiometrytest/settings.gradle
|
||||
include tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties
|
||||
include tests/source-files/Zillode/syncthing-silk/build.gradle
|
||||
include tests/SpeedoMeterApp.main_1.apk
|
||||
include tests/stats/known_apks.txt
|
||||
include tests/testcommon.py
|
||||
include tests/test-gradlew-fdroid
|
||||
include tests/triple-t-2/build/org.piwigo.android/app/build.gradle
|
||||
|
||||
@ -401,11 +401,6 @@
|
||||
# configured to allow push access (e.g. ssh key, username/password, etc)
|
||||
# binary_transparency_remote: git@gitlab.com:fdroid/binary-transparency-log.git
|
||||
|
||||
# If you want to keep the "added" and "last updated" dates for each
|
||||
# app and APK in your repo, enable this. The name comes from an old
|
||||
# system for tracking statistics that is no longer included.
|
||||
# update_stats: true
|
||||
|
||||
# Set this to true to always use a build server. This saves specifying the
|
||||
# --server option on dedicated secure build server hosts.
|
||||
# build_server_always: true
|
||||
|
||||
@ -162,7 +162,6 @@ default_config = {
|
||||
'make_current_version_link': False,
|
||||
'current_version_name_source': 'Name',
|
||||
'deploy_process_logs': False,
|
||||
'update_stats': False,
|
||||
'repo_maxage': 0,
|
||||
'build_server_always': False,
|
||||
'keystore': 'keystore.p12',
|
||||
@ -2555,40 +2554,17 @@ class KnownApks:
|
||||
this is parsed as a list from the end to allow the filename to
|
||||
have any combo of spaces.
|
||||
"""
|
||||
self.path = os.path.join('stats', 'known_apks.txt')
|
||||
self.apks = {}
|
||||
if os.path.isfile(self.path):
|
||||
with open(self.path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
t = line.rstrip().split(' ')
|
||||
if len(t) == 2:
|
||||
self.apks[t[0]] = (t[1], None)
|
||||
else:
|
||||
appid = t[-2]
|
||||
date = datetime.strptime(t[-1], '%Y-%m-%d')
|
||||
filename = line[0:line.rfind(appid) - 1]
|
||||
self.apks[filename] = (appid, date)
|
||||
check_system_clock(date, self.path)
|
||||
self.changed = False
|
||||
|
||||
def writeifchanged(self):
|
||||
if not self.changed:
|
||||
return
|
||||
|
||||
if not os.path.exists('stats'):
|
||||
os.mkdir('stats')
|
||||
|
||||
lst = []
|
||||
for apk, app in self.apks.items():
|
||||
appid, added = app
|
||||
line = apk + ' ' + appid
|
||||
if added:
|
||||
line += ' ' + added.strftime('%Y-%m-%d')
|
||||
lst.append(line)
|
||||
|
||||
with open(self.path, 'w') as f:
|
||||
for line in sorted(lst, key=natural_key):
|
||||
f.write(line + '\n')
|
||||
for part in ('repo', 'archive'):
|
||||
path = os.path.join(part, 'index-v2.json')
|
||||
if os.path.isfile(path):
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
index = json.load(f)
|
||||
for appid, data in index["packages"].items():
|
||||
for version in data["versions"].values():
|
||||
filename = version["file"]["name"][1:]
|
||||
date = datetime.fromtimestamp(version["added"] // 1000, tz=timezone.utc)
|
||||
self.apks[filename] = (appid, date)
|
||||
|
||||
def recordapk(self, apkName, app, default_date=None):
|
||||
"""
|
||||
@ -2601,38 +2577,12 @@ class KnownApks:
|
||||
"""
|
||||
if apkName not in self.apks:
|
||||
if default_date is None:
|
||||
default_date = datetime.utcnow()
|
||||
default_date = datetime.now(timezone.utc)
|
||||
self.apks[apkName] = (app, default_date)
|
||||
self.changed = True
|
||||
_ignored, added = self.apks[apkName]
|
||||
return added
|
||||
|
||||
def getapp(self, apkname):
|
||||
"""Look up information - given the 'apkname'.
|
||||
|
||||
Returns (app id, date added/None).
|
||||
Or returns None for an unknown apk.
|
||||
"""
|
||||
if apkname in self.apks:
|
||||
return self.apks[apkname]
|
||||
return None
|
||||
|
||||
def getlatest(self, num):
|
||||
"""Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first."""
|
||||
apps = {}
|
||||
for apk, app in self.apks.items():
|
||||
appid, added = app
|
||||
if added:
|
||||
if appid in apps:
|
||||
if apps[appid] > added:
|
||||
apps[appid] = added
|
||||
else:
|
||||
apps[appid] = added
|
||||
sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
|
||||
lst = [app for app, _ignored in sortedapps]
|
||||
lst.reverse()
|
||||
return lst
|
||||
|
||||
|
||||
def get_file_extension(filename):
|
||||
"""Get the normalized file extension, can be blank string but never None."""
|
||||
|
||||
@ -77,7 +77,7 @@ def make(apps, apks, repodir, archive):
|
||||
sortedapps[appid] = apps[appid]
|
||||
|
||||
repodict = collections.OrderedDict()
|
||||
repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc)
|
||||
repodict['timestamp'] = datetime.now(timezone.utc)
|
||||
repodict['version'] = METADATA_VERSION
|
||||
|
||||
if common.config['repo_maxage'] != 0:
|
||||
|
||||
@ -367,7 +367,7 @@ You can use it with the [F-Droid](https://f-droid.org/) Android app.
|
||||
|
||||
Last updated: {date}'''.format(repo_git_base=repo_git_base,
|
||||
repo_url=repo_url,
|
||||
date=datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC'))
|
||||
date=datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC'))
|
||||
with open(readme_path, 'w') as fp:
|
||||
fp.write(readme)
|
||||
mirror_git_repo.git.add(all=True)
|
||||
@ -422,7 +422,6 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
|
||||
'keypass': PASSWORD,
|
||||
'keydname': DISTINGUISHED_NAME,
|
||||
'make_current_version_link': False,
|
||||
'update_stats': True,
|
||||
}
|
||||
with open('config.yml', 'w') as fp:
|
||||
yaml.dump(config, fp, default_flow_style=False)
|
||||
@ -496,7 +495,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
|
||||
common.local_rsync(
|
||||
options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
|
||||
)
|
||||
common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/')
|
||||
stats = repo_basedir + '/stats/'
|
||||
if os.path.exists(stats):
|
||||
common.local_rsync(options, [stats], git_mirror_statsdir + '/')
|
||||
mirror_git_repo.git.add(all=True)
|
||||
mirror_git_repo.index.commit("update app metadata")
|
||||
|
||||
|
||||
@ -29,7 +29,7 @@ import urllib.request
|
||||
import zipfile
|
||||
from argparse import ArgumentParser
|
||||
from dataclasses import dataclass, field, fields
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import IntEnum
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
@ -330,7 +330,7 @@ def get_embedded_classes(apkfile, depth=0):
|
||||
|
||||
def _datetime_now():
|
||||
"""Get datetime.now(), using this funciton allows mocking it for testing."""
|
||||
return datetime.utcnow()
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _scanner_cachedir():
|
||||
@ -389,7 +389,7 @@ class SignatureDataController:
|
||||
last_updated = self.data.get("last_updated", None)
|
||||
if last_updated:
|
||||
try:
|
||||
last_updated = datetime.fromtimestamp(last_updated)
|
||||
last_updated = datetime.fromtimestamp(last_updated, timezone.utc)
|
||||
except ValueError as e:
|
||||
raise SignatureDataMalformedException() from e
|
||||
except TypeError as e:
|
||||
|
||||
@ -362,7 +362,7 @@ def get_cache():
|
||||
if not isinstance(v['antiFeatures'], dict):
|
||||
v['antiFeatures'] = {k: {} for k in sorted(v['antiFeatures'])}
|
||||
if 'added' in v:
|
||||
v['added'] = datetime.fromtimestamp(v['added'])
|
||||
v['added'] = datetime.fromtimestamp(v['added'], tz=timezone.utc)
|
||||
|
||||
return apkcache
|
||||
|
||||
@ -2001,7 +2001,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
|
||||
fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
|
||||
|
||||
if use_date_from_apk:
|
||||
default_date_param = datetime.fromtimestamp(os.stat(apkfile).st_mtime)
|
||||
default_date_param = datetime.fromtimestamp(os.stat(apkfile).st_mtime, tz=timezone.utc)
|
||||
else:
|
||||
default_date_param = None
|
||||
|
||||
@ -2420,12 +2420,12 @@ def create_metadata_from_template(apk):
|
||||
def read_added_date_from_all_apks(apps, apks):
|
||||
"""No summary.
|
||||
|
||||
Added dates come from the stats/known_apks.txt file but are
|
||||
Added dates come from the repo/index-v2.json file but are
|
||||
read when scanning apks and thus need to be applied form apk
|
||||
level to app level for _all_ apps and not only from non-archived
|
||||
ones
|
||||
|
||||
TODO: read the added dates directly from known_apks.txt instead of
|
||||
TODO: read the added dates directly from index-v2.json instead of
|
||||
going through apks that way it also works for for repos that
|
||||
don't keep an archive of apks.
|
||||
"""
|
||||
@ -2772,10 +2772,6 @@ def main():
|
||||
from . import btlog
|
||||
btlog.make_binary_transparency_log(repodirs)
|
||||
|
||||
if config['update_stats']:
|
||||
# Update known apks info...
|
||||
knownapks.writeifchanged()
|
||||
|
||||
status_update_json(apps, apks + archapks)
|
||||
|
||||
logging.info(_("Finished"))
|
||||
|
||||
@ -1898,7 +1898,6 @@ class CommonTest(unittest.TestCase):
|
||||
self.assertFalse(os.path.exists('config.yml'))
|
||||
self.assertFalse(os.path.exists('config.py'))
|
||||
config = fdroidserver.common.read_config()
|
||||
self.assertFalse(config.get('update_stats'))
|
||||
self.assertIsNotNone(config.get('char_limits'))
|
||||
|
||||
def test_with_zero_size_config(self):
|
||||
@ -1908,7 +1907,6 @@ class CommonTest(unittest.TestCase):
|
||||
self.assertTrue(os.path.exists('config.yml'))
|
||||
self.assertFalse(os.path.exists('config.py'))
|
||||
config = fdroidserver.common.read_config()
|
||||
self.assertFalse(config.get('update_stats'))
|
||||
self.assertIsNotNone(config.get('char_limits'))
|
||||
|
||||
def test_with_config_yml(self):
|
||||
|
||||
@ -25,8 +25,6 @@ mirrors = (
|
||||
'https://foo.bar/fdroid',
|
||||
)
|
||||
|
||||
update_stats = True
|
||||
|
||||
install_list = 'org.adaway'
|
||||
uninstall_list = ('com.android.vending', 'com.facebook.orca', )
|
||||
|
||||
|
||||
@ -274,7 +274,6 @@ class NightlyTest(unittest.TestCase):
|
||||
'repo_name': 'f-droid/test-nightly',
|
||||
'repo_url': mirror_url + '/repo',
|
||||
'servergitmirrors': [{"url": git_url}],
|
||||
'update_stats': True,
|
||||
}
|
||||
with open('config.yml') as fp:
|
||||
config = yaml.safe_load(fp)
|
||||
@ -347,7 +346,6 @@ class NightlyTest(unittest.TestCase):
|
||||
'repo_name': 'fdroid/test-nightly',
|
||||
'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo',
|
||||
'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly'}],
|
||||
'update_stats': True,
|
||||
}
|
||||
with open('config.yml') as fp:
|
||||
config = yaml.safe_load(fp)
|
||||
|
||||
@ -273,7 +273,6 @@ cp -a \
|
||||
$WORKSPACE/tests/config \
|
||||
$WORKSPACE/tests/metadata \
|
||||
$WORKSPACE/tests/repo \
|
||||
$WORKSPACE/tests/stats \
|
||||
$REPOROOT/
|
||||
cp -a $WORKSPACE/tests/gnupghome $GNUPGHOME
|
||||
chmod 0700 $GNUPGHOME
|
||||
@ -645,14 +644,13 @@ printf '\narchive_older: 3\n' >> config.yml
|
||||
mkdir -p {repo,archive,metadata,stats}
|
||||
cp $WORKSPACE/tests/repo/com.politedroid_5.apk archive
|
||||
cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo
|
||||
cp $WORKSPACE/tests/repo/index-v2.json repo
|
||||
cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata
|
||||
#TODO: the timestamp of the oldest apk in the file should be used, even if that
|
||||
# doesn't exist anymore
|
||||
echo "com.politedroid_4.apk com.politedroid 2016-01-01" > stats/known_apks.txt
|
||||
echo "com.politedroid_5.apk com.politedroid 2017-01-01" >> stats/known_apks.txt
|
||||
echo "com.politedroid_6.apk com.politedroid 2018-01-01" >> stats/known_apks.txt
|
||||
$sed -i -e 's/ArchivePolicy:.*/ArchivePolicy: 1 versions/' metadata/com.politedroid.yml
|
||||
timestamp=1483228800 # $(date -u --date=2017-01-01 +%s)000
|
||||
timestamp=1498176000000 # $(date -u --date=2017-01-01 +%s)000
|
||||
pwd
|
||||
|
||||
$fdroid update --pretty --nosign
|
||||
grep -F "\"added\": $timestamp" repo/index-v1.json
|
||||
@ -1078,7 +1076,7 @@ GIT_REMOTE=`create_test_dir`
|
||||
GNUPGHOME=$REPOROOT/gnupghome
|
||||
cd $REPOROOT
|
||||
fdroid_init_with_prebuilt_keystore
|
||||
cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/
|
||||
cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $REPOROOT/
|
||||
echo "binary_transparency_remote: $GIT_REMOTE" >> config.yml
|
||||
$fdroid update --verbose
|
||||
$fdroid deploy --verbose
|
||||
@ -1224,7 +1222,7 @@ $git config receive.denyCurrentBranch updateInstead
|
||||
cd $OFFLINE_ROOT
|
||||
fdroid_init_with_prebuilt_keystore
|
||||
printf '\narchive_older: 3\n' >> config.yml
|
||||
cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $OFFLINE_ROOT/
|
||||
cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $OFFLINE_ROOT/
|
||||
mkdir $OFFLINE_ROOT/unsigned
|
||||
cp $WORKSPACE/tests/urzip-release-unsigned.apk $OFFLINE_ROOT/unsigned
|
||||
|
||||
|
||||
@ -14,7 +14,7 @@ import unittest
|
||||
import uuid
|
||||
import zipfile
|
||||
from dataclasses import asdict
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest import mock
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
@ -592,7 +592,7 @@ class Test_SignatureDataController(unittest.TestCase):
|
||||
sdc = fdroidserver.scanner.SignatureDataController(
|
||||
'nnn', 'fff.yml', 'https://example.com/test.json'
|
||||
)
|
||||
sdc.data['last_updated'] = datetime.utcnow().timestamp()
|
||||
sdc.data['last_updated'] = datetime.now(timezone.utc).timestamp()
|
||||
sdc.check_last_updated()
|
||||
|
||||
def test_check_last_updated_exception_cache_outdated(self):
|
||||
@ -600,7 +600,9 @@ class Test_SignatureDataController(unittest.TestCase):
|
||||
'nnn', 'fff.yml', 'https://example.com/test.json'
|
||||
)
|
||||
sdc.cache_duration = timedelta(days=7)
|
||||
sdc.data['last_updated'] = (datetime.utcnow() - timedelta(days=30)).timestamp()
|
||||
sdc.data['last_updated'] = (
|
||||
datetime.now(timezone.utc) - timedelta(days=30)
|
||||
).timestamp()
|
||||
with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException):
|
||||
sdc.check_last_updated()
|
||||
|
||||
|
||||
@ -1,18 +0,0 @@
|
||||
com.example.test.helloworld_1.apk com.example.test.helloworld 2023-02-17
|
||||
com.politedroid_3.apk com.politedroid 2017-06-23
|
||||
com.politedroid_4.apk com.politedroid 2017-06-23
|
||||
com.politedroid_5.apk com.politedroid 2017-06-23
|
||||
com.politedroid_6.apk com.politedroid 2017-06-23
|
||||
duplicate.permisssions_9999999.apk duplicate.permisssions 2017-12-22
|
||||
fake.ota.update_1234.zip fake.ota.update 2016-03-10
|
||||
info.zwanenburg.caffeinetile_4.apk info.zwanenburg.caffeinetile 2018-10-10
|
||||
no.min.target.sdk_987.apk no.min.target.sdk 2018-10-10
|
||||
obb.main.oldversion_1444412523.apk obb.main.oldversion 2013-12-31
|
||||
obb.main.twoversions_1101613.apk obb.main.twoversions 2015-10-12
|
||||
obb.main.twoversions_1101615.apk obb.main.twoversions 2016-01-01
|
||||
obb.main.twoversions_1101617.apk obb.main.twoversions 2016-06-20
|
||||
obb.mainpatch.current_1619.apk obb.mainpatch.current 2016-04-23
|
||||
obb.mainpatch.current_1619_another-release-key.apk obb.mainpatch.current 2017-06-01
|
||||
souch.smsbypass_9.apk souch.smsbypass 2018-04-26
|
||||
urzip-; Рахма́, [rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢·.apk info.guardianproject.urzip 2016-06-23
|
||||
v1.v2.sig_1020.apk v1.v2.sig 2023-02-17
|
||||
@ -697,14 +697,10 @@ class UpdateTest(unittest.TestCase):
|
||||
|
||||
os.chdir(self.testdir)
|
||||
os.mkdir('repo')
|
||||
os.mkdir('stats')
|
||||
with open(os.path.join('stats', 'known_apks.txt'), 'w') as fp:
|
||||
fp.write('se.manyver_30.apk se.manyver 2018-10-10\n')
|
||||
filename = 'Norway_bouvet_europe_2.obf.zip'
|
||||
shutil.copy(os.path.join(self.basedir, filename), 'repo')
|
||||
knownapks = fdroidserver.common.KnownApks()
|
||||
files, fcachechanged = fdroidserver.update.scan_repo_files(dict(), 'repo', knownapks, False)
|
||||
knownapks.writeifchanged()
|
||||
self.assertTrue(fcachechanged)
|
||||
|
||||
info = files[0]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user