1berry/fdroidserver
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Drop stats/known_apks.txt

Browse Source 
Use repo/index-v2.json instead.
Also fix deprecated datetime.utcnow().
This commit is contained in:
Jochen Sprickerhof 2024-10-28 22:47:39 +01:00 committed by Hans-Christoph Steiner
parent 83a3227235
commit 81c0c9d4aa
14 changed files with 33 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
MANIFEST.in
View File

@ -860,7 +860,6 @@ include tests/source-files/ut.ewh.audiometrytest/settings.gradle
include tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties
include tests/source-files/Zillode/syncthing-silk/build.gradle
include tests/SpeedoMeterApp.main_1.apk
include tests/stats/known_apks.txt
include tests/testcommon.py
include tests/test-gradlew-fdroid
include tests/triple-t-2/build/org.piwigo.android/app/build.gradle

5
examples/config.yml
View File

@ -401,11 +401,6 @@
# configured to allow push access (e.g. ssh key, username/password, etc)
# binary_transparency_remote: git@gitlab.com:fdroid/binary-transparency-log.git
# If you want to keep the "added" and "last updated" dates for each
# app and APK in your repo, enable this. The name comes from an old
# system for tracking statistics that is no longer included.
# update_stats: true
# Set this to true to always use a build server. This saves specifying the
# --server option on dedicated secure build server hosts.
# build_server_always: true

72
fdroidserver/common.py
View File

@ -162,7 +162,6 @@ default_config = {
'make_current_version_link': False,
'current_version_name_source': 'Name',
'deploy_process_logs': False,
'update_stats': False,
'repo_maxage': 0,
'build_server_always': False,
'keystore': 'keystore.p12',
@ -2555,40 +2554,17 @@ class KnownApks:
this is parsed as a list from the end to allow the filename to
have any combo of spaces.
"""
self.path = os.path.join('stats', 'known_apks.txt')
self.apks = {}
if os.path.isfile(self.path):
with open(self.path, 'r', encoding='utf-8') as f:
for line in f:
t = line.rstrip().split(' ')
if len(t) == 2:
self.apks[t[0]] = (t[1], None)
else:
appid = t[-2]
date = datetime.strptime(t[-1], '%Y-%m-%d')
filename = line[0:line.rfind(appid) - 1]
self.apks[filename] = (appid, date)
check_system_clock(date, self.path)
self.changed = False
def writeifchanged(self):
if not self.changed:
return
if not os.path.exists('stats'):
os.mkdir('stats')
lst = []
for apk, app in self.apks.items():
appid, added = app
line = apk + ' ' + appid
if added:
line += ' ' + added.strftime('%Y-%m-%d')
lst.append(line)
with open(self.path, 'w') as f:
for line in sorted(lst, key=natural_key):
f.write(line + '\n')
for part in ('repo', 'archive'):
path = os.path.join(part, 'index-v2.json')
if os.path.isfile(path):
with open(path, 'r', encoding='utf-8') as f:
index = json.load(f)
for appid, data in index["packages"].items():
for version in data["versions"].values():
filename = version["file"]["name"][1:]
date = datetime.fromtimestamp(version["added"] // 1000, tz=timezone.utc)
self.apks[filename] = (appid, date)
def recordapk(self, apkName, app, default_date=None):
"""
@ -2601,38 +2577,12 @@ class KnownApks:
"""
if apkName not in self.apks:
if default_date is None:
default_date = datetime.utcnow()
default_date = datetime.now(timezone.utc)
self.apks[apkName] = (app, default_date)
self.changed = True
_ignored, added = self.apks[apkName]
return added
def getapp(self, apkname):
"""Look up information - given the 'apkname'.
Returns (app id, date added/None).
Or returns None for an unknown apk.
"""
if apkname in self.apks:
return self.apks[apkname]
return None
def getlatest(self, num):
"""Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first."""
apps = {}
for apk, app in self.apks.items():
appid, added = app
if added:
if appid in apps:
if apps[appid] > added:
apps[appid] = added
else:
apps[appid] = added
sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
lst = [app for app, _ignored in sortedapps]
lst.reverse()
return lst
def get_file_extension(filename):
"""Get the normalized file extension, can be blank string but never None."""

2
fdroidserver/index.py
View File

@ -77,7 +77,7 @@ def make(apps, apks, repodir, archive):
sortedapps[appid] = apps[appid]
repodict = collections.OrderedDict()
repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc)
repodict['timestamp'] = datetime.now(timezone.utc)
repodict['version'] = METADATA_VERSION
if common.config['repo_maxage'] != 0:

7
fdroidserver/nightly.py
View File

@ -367,7 +367,7 @@ You can use it with the [F-Droid](https://f-droid.org/) Android app.
Last updated: {date}'''.format(repo_git_base=repo_git_base,
repo_url=repo_url,
date=datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC'))
date=datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC'))
with open(readme_path, 'w') as fp:
fp.write(readme)
mirror_git_repo.git.add(all=True)
@ -422,7 +422,6 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
'keypass': PASSWORD,
'keydname': DISTINGUISHED_NAME,
'make_current_version_link': False,
'update_stats': True,
}
with open('config.yml', 'w') as fp:
yaml.dump(config, fp, default_flow_style=False)
@ -496,7 +495,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
common.local_rsync(
options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
)
common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/')
stats = repo_basedir + '/stats/'
if os.path.exists(stats):
common.local_rsync(options, [stats], git_mirror_statsdir + '/')
mirror_git_repo.git.add(all=True)
mirror_git_repo.index.commit("update app metadata")

6
fdroidserver/scanner.py
View File

@ -29,7 +29,7 @@ import urllib.request
import zipfile
from argparse import ArgumentParser
from dataclasses import dataclass, field, fields
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
from enum import IntEnum
from pathlib import Path
from tempfile import TemporaryDirectory
@ -330,7 +330,7 @@ def get_embedded_classes(apkfile, depth=0):
def _datetime_now():
"""Get datetime.now(), using this funciton allows mocking it for testing."""
return datetime.utcnow()
return datetime.now(timezone.utc)
def _scanner_cachedir():
@ -389,7 +389,7 @@ class SignatureDataController:
last_updated = self.data.get("last_updated", None)
if last_updated:
try:
last_updated = datetime.fromtimestamp(last_updated)
last_updated = datetime.fromtimestamp(last_updated, timezone.utc)
except ValueError as e:
raise SignatureDataMalformedException() from e
except TypeError as e:

12
fdroidserver/update.py
View File

@ -362,7 +362,7 @@ def get_cache():
if not isinstance(v['antiFeatures'], dict):
v['antiFeatures'] = {k: {} for k in sorted(v['antiFeatures'])}
if 'added' in v:
v['added'] = datetime.fromtimestamp(v['added'])
v['added'] = datetime.fromtimestamp(v['added'], tz=timezone.utc)
return apkcache
@ -2001,7 +2001,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
if use_date_from_apk:
default_date_param = datetime.fromtimestamp(os.stat(apkfile).st_mtime)
default_date_param = datetime.fromtimestamp(os.stat(apkfile).st_mtime, tz=timezone.utc)
else:
default_date_param = None
@ -2420,12 +2420,12 @@ def create_metadata_from_template(apk):
def read_added_date_from_all_apks(apps, apks):
"""No summary.
Added dates come from the stats/known_apks.txt file but are
Added dates come from the repo/index-v2.json file but are
read when scanning apks and thus need to be applied form apk
level to app level for _all_ apps and not only from non-archived
ones
TODO: read the added dates directly from known_apks.txt instead of
TODO: read the added dates directly from index-v2.json instead of
going through apks that way it also works for for repos that
don't keep an archive of apks.
"""
@ -2772,10 +2772,6 @@ def main():
from . import btlog
btlog.make_binary_transparency_log(repodirs)
if config['update_stats']:
# Update known apks info...
knownapks.writeifchanged()
status_update_json(apps, apks + archapks)
logging.info(_("Finished"))

2
tests/common.TestCase
View File

@ -1898,7 +1898,6 @@ class CommonTest(unittest.TestCase):
self.assertFalse(os.path.exists('config.yml'))
self.assertFalse(os.path.exists('config.py'))
config = fdroidserver.common.read_config()
self.assertFalse(config.get('update_stats'))
self.assertIsNotNone(config.get('char_limits'))
def test_with_zero_size_config(self):
@ -1908,7 +1907,6 @@ class CommonTest(unittest.TestCase):
self.assertTrue(os.path.exists('config.yml'))
self.assertFalse(os.path.exists('config.py'))
config = fdroidserver.common.read_config()
self.assertFalse(config.get('update_stats'))
self.assertIsNotNone(config.get('char_limits'))
def test_with_config_yml(self):

2
tests/config.py
View File

@ -25,8 +25,6 @@ mirrors = (
'https://foo.bar/fdroid',
)
update_stats = True
install_list = 'org.adaway'
uninstall_list = ('com.android.vending', 'com.facebook.orca', )

2
tests/nightly.TestCase
View File

@ -274,7 +274,6 @@ class NightlyTest(unittest.TestCase):
'repo_name': 'f-droid/test-nightly',
'repo_url': mirror_url + '/repo',
'servergitmirrors': [{"url": git_url}],
'update_stats': True,
}
with open('config.yml') as fp:
config = yaml.safe_load(fp)
@ -347,7 +346,6 @@ class NightlyTest(unittest.TestCase):
'repo_name': 'fdroid/test-nightly',
'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo',
'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly'}],
'update_stats': True,
}
with open('config.yml') as fp:
config = yaml.safe_load(fp)

12
tests/run-tests
View File

@ -273,7 +273,6 @@ cp -a \
$WORKSPACE/tests/config \
$WORKSPACE/tests/metadata \
$WORKSPACE/tests/repo \
$WORKSPACE/tests/stats \
$REPOROOT/
cp -a $WORKSPACE/tests/gnupghome $GNUPGHOME
chmod 0700 $GNUPGHOME
@ -645,14 +644,13 @@ printf '\narchive_older: 3\n' >> config.yml
mkdir -p {repo,archive,metadata,stats}
cp $WORKSPACE/tests/repo/com.politedroid_5.apk archive
cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo
cp $WORKSPACE/tests/repo/index-v2.json repo
cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata
#TODO: the timestamp of the oldest apk in the file should be used, even if that
# doesn't exist anymore
echo "com.politedroid_4.apk com.politedroid 2016-01-01" > stats/known_apks.txt
echo "com.politedroid_5.apk com.politedroid 2017-01-01" >> stats/known_apks.txt
echo "com.politedroid_6.apk com.politedroid 2018-01-01" >> stats/known_apks.txt
$sed -i -e 's/ArchivePolicy:.*/ArchivePolicy: 1 versions/' metadata/com.politedroid.yml
timestamp=1483228800 # $(date -u --date=2017-01-01 +%s)000
timestamp=1498176000000 # $(date -u --date=2017-01-01 +%s)000
pwd
$fdroid update --pretty --nosign
grep -F "\"added\": $timestamp" repo/index-v1.json
@ -1078,7 +1076,7 @@ GIT_REMOTE=`create_test_dir`
GNUPGHOME=$REPOROOT/gnupghome
cd $REPOROOT
fdroid_init_with_prebuilt_keystore
cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/
cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $REPOROOT/
echo "binary_transparency_remote: $GIT_REMOTE" >> config.yml
$fdroid update --verbose
$fdroid deploy --verbose
@ -1224,7 +1222,7 @@ $git config receive.denyCurrentBranch updateInstead
cd $OFFLINE_ROOT
fdroid_init_with_prebuilt_keystore
printf '\narchive_older: 3\n' >> config.yml
cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $OFFLINE_ROOT/
cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $OFFLINE_ROOT/
mkdir $OFFLINE_ROOT/unsigned
cp $WORKSPACE/tests/urzip-release-unsigned.apk $OFFLINE_ROOT/unsigned

8
tests/scanner.TestCase
View File

@ -14,7 +14,7 @@ import unittest
import uuid
import zipfile
from dataclasses import asdict
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
from unittest import mock
if sys.version_info >= (3, 11):
@ -592,7 +592,7 @@ class Test_SignatureDataController(unittest.TestCase):
sdc = fdroidserver.scanner.SignatureDataController(
'nnn', 'fff.yml', 'https://example.com/test.json'
)
sdc.data['last_updated'] = datetime.utcnow().timestamp()
sdc.data['last_updated'] = datetime.now(timezone.utc).timestamp()
sdc.check_last_updated()
def test_check_last_updated_exception_cache_outdated(self):
@ -600,7 +600,9 @@ class Test_SignatureDataController(unittest.TestCase):
'nnn', 'fff.yml', 'https://example.com/test.json'
)
sdc.cache_duration = timedelta(days=7)
sdc.data['last_updated'] = (datetime.utcnow() - timedelta(days=30)).timestamp()
sdc.data['last_updated'] = (
datetime.now(timezone.utc) - timedelta(days=30)
).timestamp()
with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException):
sdc.check_last_updated()

18
tests/stats/known_apks.txt
View File

@ -1,18 +0,0 @@
com.example.test.helloworld_1.apk com.example.test.helloworld 2023-02-17
com.politedroid_3.apk com.politedroid 2017-06-23
com.politedroid_4.apk com.politedroid 2017-06-23
com.politedroid_5.apk com.politedroid 2017-06-23
com.politedroid_6.apk com.politedroid 2017-06-23
duplicate.permisssions_9999999.apk duplicate.permisssions 2017-12-22
fake.ota.update_1234.zip fake.ota.update 2016-03-10
info.zwanenburg.caffeinetile_4.apk info.zwanenburg.caffeinetile 2018-10-10
no.min.target.sdk_987.apk no.min.target.sdk 2018-10-10
obb.main.oldversion_1444412523.apk obb.main.oldversion 2013-12-31
obb.main.twoversions_1101613.apk obb.main.twoversions 2015-10-12
obb.main.twoversions_1101615.apk obb.main.twoversions 2016-01-01
obb.main.twoversions_1101617.apk obb.main.twoversions 2016-06-20
obb.mainpatch.current_1619.apk obb.mainpatch.current 2016-04-23
obb.mainpatch.current_1619_another-release-key.apk obb.mainpatch.current 2017-06-01
souch.smsbypass_9.apk souch.smsbypass 2018-04-26
urzip-; Рахма́, [rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢·.apk info.guardianproject.urzip 2016-06-23
v1.v2.sig_1020.apk v1.v2.sig 2023-02-17

4
tests/update.TestCase
View File

@ -697,14 +697,10 @@ class UpdateTest(unittest.TestCase):
os.chdir(self.testdir)
os.mkdir('repo')
os.mkdir('stats')
with open(os.path.join('stats', 'known_apks.txt'), 'w') as fp:
fp.write('se.manyver_30.apk se.manyver 2018-10-10\n')
filename = 'Norway_bouvet_europe_2.obf.zip'
shutil.copy(os.path.join(self.basedir, filename), 'repo')
knownapks = fdroidserver.common.KnownApks()
files, fcachechanged = fdroidserver.update.scan_repo_files(dict(), 'repo', knownapks, False)
knownapks.writeifchanged()
self.assertTrue(fcachechanged)
info = files[0]