From 642499ec944acb11d62f053e5747e6a151eb7872 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 24 Feb 2025 12:48:51 +0100 Subject: [PATCH] purge config.py handling, it is no longer supported --- .gitignore | 1 - .gitlab-ci.yml | 4 +- MANIFEST.in | 2 +- fdroidserver/common.py | 49 +++------- fdroidserver/init.py | 2 +- tests/config.py | 31 ------- tests/config.yml | 27 ++++++ tests/test_common.py | 198 +++++++++++++---------------------------- tests/test_index.py | 9 +- tests/test_nightly.py | 2 - tests/test_publish.py | 17 ++-- 11 files changed, 117 insertions(+), 225 deletions(-) delete mode 100644 tests/config.py create mode 100644 tests/config.yml diff --git a/.gitignore b/.gitignore index 04e92ad6..ce3a0e9a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,6 @@ tmp/ /tests/repo/status # files used in manual testing -/config.py /config.yml /tmp/ /logs/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 394c41a4..69b10ae6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -555,11 +555,11 @@ servergitmirrors: - ./tests/key-tricks.py - ssh-keyscan gitlab.com >> /root/.ssh/known_hosts - test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo - - cp tests/config.py tests/keystore.jks /tmp/fdroid/ + - cp tests/config.yml tests/keystore.jks /tmp/fdroid/ - cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/ - cd /tmp/fdroid - touch fdroid-icon.png - - printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py + - printf "\nservergitmirrors\x3a 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.yml - $PYTHONPATH/fdroid update --verbose --create-metadata - $PYTHONPATH/fdroid deploy --verbose - export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json` diff --git a/MANIFEST.in b/MANIFEST.in index 4f14a48d..0dac052c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -548,7 +548,7 @@ include tests/build-tools/28.0.3/aapt-output-souch.smsbypass_9.txt include tests/build-tools/generate.sh include tests/check-fdroid-apk include tests/com.fake.IpaApp_1000000000001.ipa -include tests/config.py +include tests/config.yml include tests/config/antiFeatures.yml include tests/config/categories.yml include tests/config/de/antiFeatures.yml diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 21944364..893b14da 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -531,8 +531,7 @@ def read_config(): not required, just use defaults. config.yml is the preferred form because no code is executed when - reading it. config.py is deprecated and supported for backwards - compatibility. + reading it. config.py is deprecated and no longer supported. config.yml requires ASCII or UTF-8 encoding because this code does not auto-detect the file's encoding. That is left up to the YAML @@ -550,10 +549,6 @@ def read_config(): config_file = 'config.yml' old_config_file = 'config.py' - if os.path.exists(config_file) and os.path.exists(old_config_file): - logging.error(_("""Conflicting config files! Using {newfile}, ignoring {oldfile}!""") - .format(oldfile=old_config_file, newfile=config_file)) - if os.path.exists(config_file): logging.debug(_("Reading '{config_file}'").format(config_file=config_file)) with open(config_file, encoding='utf-8') as fp: @@ -561,19 +556,13 @@ def read_config(): if not config: config = {} config_type_check(config_file, config) - elif os.path.exists(old_config_file): - logging.warning(_("""{oldfile} is deprecated, use {newfile}""") - .format(oldfile=old_config_file, newfile=config_file)) - with io.open(old_config_file, "rb") as fp: - code = compile(fp.read(), old_config_file, 'exec') - exec(code, None, config) # nosec TODO automatically migrate - for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'): - if k in config: - if not type(config[k]) in (str, list, tuple): - logging.warning( - _("'{field}' will be in random order! Use () or [] brackets if order is important!") - .format(field=k)) + if os.path.exists(old_config_file): + logging.warning( + _("""Ignoring deprecated {oldfile}, use {newfile}!""").format( + oldfile=old_config_file, newfile=config_file + ) + ) # smartcardoptions must be a list since its command line args for Popen smartcardoptions = config.get('smartcardoptions') @@ -4203,7 +4192,7 @@ def load_stats_fdroid_signing_key_fingerprints(): def write_to_config(thisconfig, key, value=None, config_file=None): - """Write a key/value to the local config.yml or config.py. + """Write a key/value to the local config.yml. NOTE: only supports writing string variables. @@ -4222,8 +4211,6 @@ def write_to_config(thisconfig, key, value=None, config_file=None): value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key] if config_file: cfg = config_file - elif os.path.exists('config.py') and not os.path.exists('config.yml'): - cfg = 'config.py' else: cfg = 'config.yml' @@ -4239,19 +4226,8 @@ def write_to_config(thisconfig, key, value=None, config_file=None): if not lines[-1].endswith('\n'): lines[-1] += '\n' - # regex for finding and replacing python string variable - # definitions/initializations - if cfg.endswith('.py'): - pattern = re.compile(r'^[\s#]*' + key + r'\s*=\s*"[^"]*"') - repl = key + ' = "' + value + '"' - pattern2 = re.compile(r'^[\s#]*' + key + r"\s*=\s*'[^']*'") - repl2 = key + " = '" + value + "'" - else: - # assume .yml as default - pattern = re.compile(r'^[\s#]*' + key + r':.*') - repl = yaml.dump({key: value}, default_flow_style=False) - pattern2 = pattern - repl2 = repl + pattern = re.compile(r'^[\s#]*' + key + r':.*\n') + repl = yaml.dump({key: value}) # If we replaced this line once, we make sure won't be a # second instance of this line for this key in the document. @@ -4259,18 +4235,15 @@ def write_to_config(thisconfig, key, value=None, config_file=None): # edit config file with open(cfg, 'w') as f: for line in lines: - if pattern.match(line) or pattern2.match(line): + if pattern.match(line): if not didRepl: line = pattern.sub(repl, line) - line = pattern2.sub(repl2, line) f.write(line) didRepl = True else: f.write(line) if not didRepl: - f.write('\n') f.write(repl) - f.write('\n') def parse_xml(path): diff --git a/fdroidserver/init.py b/fdroidserver/init.py index b77fea72..326ecf4c 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -138,7 +138,7 @@ def main(): _("Android SDK not found at {path}!").format(path=test_config['sdk_path']) ) - if not os.path.exists('config.yml') and not os.path.exists('config.py'): + if not os.path.exists('config.yml'): # 'metadata' and 'tmp' are created in fdroid if not os.path.exists('repo'): os.mkdir('repo') diff --git a/tests/config.py b/tests/config.py deleted file mode 100644 index fa118db2..00000000 --- a/tests/config.py +++ /dev/null @@ -1,31 +0,0 @@ - -# TODO convert to config.yml! - -repo_url = "https://MyFirstFDroidRepo.org/fdroid/repo" -repo_name = "My First F-Droid Repo Demo" -repo_description = """This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/fdroid.""" - -archive_older = 3 -archive_url = "https://f-droid.org/archive" -archive_name = "My First F-Droid Archive Demo" -archive_description = """ -The repository of older versions of applications from the main demo repository. -""" - -make_current_version_link = False - -repo_keyalias = "sova" -keystore = "keystore.jks" -keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=" -keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=" -keydname = "CN=sova, OU=F-Droid" - -mirrors = ( - 'http://foobarfoobarfoobar.onion/fdroid', - 'https://foo.bar/fdroid', -) - -install_list = 'org.adaway' -uninstall_list = ('com.android.vending', 'com.facebook.orca', ) - -repo_key_sha256 = "f49af3f11efddf20dffd70f5e3117b9976674167adca280e6b1932a0601b26f6" diff --git a/tests/config.yml b/tests/config.yml new file mode 100644 index 00000000..b6f62a44 --- /dev/null +++ b/tests/config.yml @@ -0,0 +1,27 @@ +--- + +repo_url: https://MyFirstFDroidRepo.org/fdroid/repo +repo_name: My First F-Droid Repo Demo +repo_description: This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/fdroid. + +archive_older: 3 +archive_url: https://f-droid.org/archive +archive_name: My First F-Droid Archive Demo +archive_description: The repository of older versions of applications from the main demo repository. + +make_current_version_link: false + +repo_keyalias: sova +keystore: keystore.jks +keystorepass: "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=" +keypass: "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=" +keydname: "CN=sova, OU=F-Droid" + +mirrors: + - http://foobarfoobarfoobar.onion/fdroid + - https://foo.bar/fdroid + +install_list: org.adaway +uninstall_list: ['com.android.vending', 'com.facebook.orca'] + +repo_key_sha256: f49af3f11efddf20dffd70f5e3117b9976674167adca280e6b1932a0601b26f6 diff --git a/tests/test_common.py b/tests/test_common.py index 79b676aa..34f83773 100755 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -58,6 +58,9 @@ class CommonTest(unittest.TestCase): os.makedirs(self.tmpdir) os.chdir(basedir) + self.verbose = '-v' in sys.argv or '--verbose' in sys.argv + fdroidserver.common.set_console_logging(self.verbose) + # these are declared as None at the top of the module file fdroidserver.common.config = None fdroidserver.common.options = None @@ -642,65 +645,65 @@ class CommonTest(unittest.TestCase): ) def test_write_to_config(self): - with tempfile.TemporaryDirectory() as tmpPath: - cfgPath = os.path.join(tmpPath, 'config.py') - with open(cfgPath, 'w') as f: - f.write( - textwrap.dedent( - """\ + """Test that config items can be added without messing up config.yml. + + The '_orig' key are where the original string values of paths + are stored. Paths have tilde expansion and env vars replaced + in fill_config_defaults(). + + """ + os.chdir(self.testdir) + config_yml = 'config.yml' + Path(config_yml).write_text( + textwrap.dedent( + """\ + # abc + # test: 'example value' + a_path: ~/android-sdk + + # comment + do_not_touch: good value + a_path: "!!!" + + key: "123" # inline""" + ) + ) + + config = {'key': 111, 'a_path_orig': '~/android-sdk'} + fdroidserver.common.write_to_config(config, 'key', config_file=config_yml) + fdroidserver.common.write_to_config( + config, 'a_path', config_file=config_yml + ) + fdroidserver.common.write_to_config( + config, 'test', value='test value', config_file=config_yml + ) + fdroidserver.common.write_to_config( + config, 'new_key', value='new', config_file=config_yml + ) + + with open(config_yml) as fp: + self.assertEqual( + fp.read(), + textwrap.dedent( + """\ # abc - # test = 'example value' - default_me= '%%%' + test: test value + a_path: ~/android-sdk # comment - do_not_touch = "good value" - default_me="!!!" + do_not_touch: good value - key="123" # inline""" - ) - ) - - cfg = {'key': '111', 'default_me_orig': 'orig'} - fdroidserver.common.write_to_config(cfg, 'key', config_file=cfgPath) - fdroidserver.common.write_to_config(cfg, 'default_me', config_file=cfgPath) - fdroidserver.common.write_to_config(cfg, 'test', value='test value', config_file=cfgPath) - fdroidserver.common.write_to_config(cfg, 'new_key', value='new', config_file=cfgPath) - - with open(cfgPath, 'r') as f: - self.assertEqual( - f.read(), - textwrap.dedent( - """\ - # abc - test = 'test value' - default_me = 'orig' - - # comment - do_not_touch = "good value" - - key = "111" # inline - - new_key = "new" + key: 111 + new_key: new """ - ), - ) + ), + ) def test_write_to_config_when_empty(self): - with tempfile.TemporaryDirectory() as tmpPath: - cfgPath = os.path.join(tmpPath, 'config.py') - with open(cfgPath, 'w') as f: - pass - fdroidserver.common.write_to_config({}, 'key', 'val', cfgPath) - with open(cfgPath, 'r') as f: - self.assertEqual( - f.read(), - textwrap.dedent( - """\ - - key = "val" - """ - ), - ) + config_yml = Path(self.testdir) / 'config.yml' + config_yml.write_text('') + fdroidserver.common.write_to_config({}, 'key', 'val', config_yml) + self.assertEqual(config_yml.read_text(), 'key: val\n') def test_apk_name_regex(self): good = [ @@ -1883,7 +1886,6 @@ class CommonTest(unittest.TestCase): """It should set defaults if no config file is found""" os.chdir(self.tmpdir) self.assertFalse(os.path.exists('config.yml')) - self.assertFalse(os.path.exists('config.py')) config = fdroidserver.common.read_config() self.assertIsNotNone(config.get('char_limits')) @@ -1892,7 +1894,6 @@ class CommonTest(unittest.TestCase): os.chdir(self.tmpdir) open('config.yml', 'w').close() self.assertTrue(os.path.exists('config.yml')) - self.assertFalse(os.path.exists('config.py')) config = fdroidserver.common.read_config() self.assertIsNotNone(config.get('char_limits')) @@ -1902,7 +1903,6 @@ class CommonTest(unittest.TestCase): with open('config.yml', 'w') as fp: fp.write('apksigner: yml') self.assertTrue(os.path.exists('config.yml')) - self.assertFalse(os.path.exists('config.py')) config = fdroidserver.common.read_config() self.assertEqual('yml', config.get('apksigner')) @@ -1913,7 +1913,6 @@ class CommonTest(unittest.TestCase): with open('config.yml', 'w', encoding='utf-8') as fp: fp.write('apksigner: ' + teststr) self.assertTrue(os.path.exists('config.yml')) - self.assertFalse(os.path.exists('config.py')) config = fdroidserver.common.read_config() self.assertEqual(teststr, config.get('apksigner')) @@ -1924,7 +1923,6 @@ class CommonTest(unittest.TestCase): with open('config.yml', 'w') as fp: yaml.dump({'apksigner': teststr}, fp) self.assertTrue(os.path.exists('config.yml')) - self.assertFalse(os.path.exists('config.py')) config = fdroidserver.common.read_config() self.assertEqual(teststr, config.get('apksigner')) @@ -1936,7 +1934,6 @@ class CommonTest(unittest.TestCase): with open('config.yml', 'w') as fp: fp.write("""keypass: {'env': 'SECRET'}""") self.assertTrue(os.path.exists('config.yml')) - self.assertFalse(os.path.exists('config.py')) config = fdroidserver.common.read_config() self.assertEqual(os.getenv('SECRET', 'fail'), config.get('keypass')) @@ -1952,16 +1949,6 @@ class CommonTest(unittest.TestCase): with self.assertRaises(yaml.scanner.ScannerError): fdroidserver.common.read_config() - def test_with_config_py(self): - """Make sure it is still possible to use config.py alone.""" - os.chdir(self.tmpdir) - with open('config.py', 'w') as fp: - fp.write('apksigner = "py"') - self.assertFalse(os.path.exists('config.yml')) - self.assertTrue(os.path.exists('config.py')) - config = fdroidserver.common.read_config() - self.assertEqual("py", config.get('apksigner')) - def test_config_perm_warning(self): """Exercise the code path that issues a warning about unsafe permissions.""" os.chdir(self.tmpdir) @@ -1973,24 +1960,6 @@ class CommonTest(unittest.TestCase): os.remove(fp.name) fdroidserver.common.config = None - with open('config.py', 'w') as fp: - fp.write('keystore = "foo.jks"') - self.assertTrue(os.path.exists(fp.name)) - os.chmod(fp.name, 0o666) # nosec B103 - fdroidserver.common.read_config() - - def test_with_both_config_yml_py(self): - """If config.yml and config.py are present, config.py should be ignored.""" - os.chdir(self.tmpdir) - with open('config.yml', 'w') as fp: - fp.write('apksigner: yml') - with open('config.py', 'w') as fp: - fp.write('apksigner = "py"') - self.assertTrue(os.path.exists('config.yml')) - self.assertTrue(os.path.exists('config.py')) - config = fdroidserver.common.read_config() - self.assertEqual('yml', config.get('apksigner')) - def test_config_repo_url(self): """repo_url ends in /repo, archive_url ends in /archive.""" os.chdir(self.tmpdir) @@ -2037,28 +2006,12 @@ class CommonTest(unittest.TestCase): os.chdir(self.tmpdir) with open('config.yml', 'w') as fp: fp.write('apksigner: yml') + os.chmod('config.yml', 0o0600) self.assertTrue(os.path.exists(fp.name)) - self.assertFalse(os.path.exists('config.py')) config = fdroidserver.common.read_config() self.assertFalse('keypass' in config) self.assertEqual('yml', config.get('apksigner')) fdroidserver.common.write_to_config(config, 'keypass', 'mysecretpassword') - with open(fp.name) as fp: - print(fp.read()) - fdroidserver.common.config = None - config = fdroidserver.common.read_config() - self.assertEqual('mysecretpassword', config['keypass']) - - def test_write_to_config_py(self): - os.chdir(self.tmpdir) - with open('config.py', 'w') as fp: - fp.write('apksigner = "py"') - self.assertTrue(os.path.exists(fp.name)) - self.assertFalse(os.path.exists('config.yml')) - config = fdroidserver.common.read_config() - self.assertFalse('keypass' in config) - self.assertEqual('py', config.get('apksigner')) - fdroidserver.common.write_to_config(config, 'keypass', 'mysecretpassword') fdroidserver.common.config = None config = fdroidserver.common.read_config() self.assertEqual('mysecretpassword', config['keypass']) @@ -2068,7 +2021,6 @@ class CommonTest(unittest.TestCase): with open('config.yml', 'w') as fp: fp.write('java_paths:\n 8: /usr/lib/jvm/java-8-openjdk\n') self.assertTrue(os.path.exists(fp.name)) - self.assertFalse(os.path.exists('config.py')) config = fdroidserver.common.read_config() self.assertEqual('/usr/lib/jvm/java-8-openjdk', config.get('java_paths', {}).get('8')) @@ -2143,10 +2095,11 @@ class CommonTest(unittest.TestCase): def test_loading_config_buildserver_yml(self): """Smoke check to make sure this file is properly parsed""" - os.chdir(self.tmpdir) - shutil.copy(os.path.join(basedir, '..', 'buildserver', 'config.buildserver.yml'), - 'config.yml') - self.assertFalse(os.path.exists('config.py')) + os.chdir(self.testdir) + shutil.copy( + os.path.join(basedir, '..', 'buildserver', 'config.buildserver.yml'), + 'config.yml', + ) fdroidserver.common.read_config() def test_setup_status_output(self): @@ -2742,35 +2695,6 @@ class CommonTest(unittest.TestCase): config['smartcardoptions'], ) - def test_get_smartcardoptions_config_py(self): - os.chdir(self.tmpdir) - with open('config.py', 'w') as fp: - fp.write( - textwrap.dedent( - """ - smartcardoptions = ''' - \t-storetype\tPKCS11 - \t-providerClass\tsun.security.pkcs11.SunPKCS11 - \t-providerArg\t/etc/pkcs11_java.cfg - - ''' - """ - ) - ) - config = fdroidserver.common.read_config() - fdroidserver.common.config = config - self.assertEqual( - [ - '-storetype', - 'PKCS11', - '-providerClass', - 'sun.security.pkcs11.SunPKCS11', - '-providerArg', - '/etc/pkcs11_java.cfg', - ], - config['smartcardoptions'], - ) - def test_load_localized_config(self): """It should load""" antiFeatures = fdroidserver.common.load_localized_config( diff --git a/tests/test_index.py b/tests/test_index.py index ea08653b..8f1b1e86 100755 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -38,8 +38,8 @@ class IndexTest(unittest.TestCase): cls.index_v1_jar = basedir / 'repo' / 'index-v1.jar' def setUp(self): - (basedir / 'config.py').chmod(0o600) - os.chdir(basedir) # so read_config() can find config.py + (basedir / 'config.yml').chmod(0o600) + os.chdir(basedir) # so read_config() can find config.yml common.config = None common.options = Options @@ -380,8 +380,7 @@ class IndexTest(unittest.TestCase): with zipfile.ZipFile(jarfile, 'w', zipfile.ZIP_DEFLATED) as jar: jar.writestr('publishsigkeys.json', json.dumps(sigkeyfps)) publish.sign_sig_key_fingerprint_list(jarfile) - with open('config.py', 'w'): - pass + Path('config.yml').write_text('') index.v1_sort_packages( i, common.load_stats_fdroid_signing_key_fingerprints() @@ -701,7 +700,7 @@ class IndexTest(unittest.TestCase): ) def test_add_mirrors_to_repodict(self): - """Test based on the contents of tests/config.py""" + """Test based on the contents of tests/config.yml""" repodict = {'address': common.config['repo_url']} index.add_mirrors_to_repodict('repo', repodict) self.assertEqual( diff --git a/tests/test_nightly.py b/tests/test_nightly.py index 09d8fbcf..bc2567e2 100755 --- a/tests/test_nightly.py +++ b/tests/test_nightly.py @@ -250,7 +250,6 @@ class NightlyTest(unittest.TestCase): raise self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']]) - self.assertFalse(os.path.exists('config.py')) git_url = 'git@github.com:f-droid/test-nightly' mirror_url = index.get_mirror_service_urls({"url": git_url})[0] expected = { @@ -324,7 +323,6 @@ class NightlyTest(unittest.TestCase): raise self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']]) - self.assertFalse(os.path.exists('config.py')) expected = { 'archive_description': 'Old nightly builds that have been archived.', 'archive_name': 'fdroid/test-nightly archive', diff --git a/tests/test_publish.py b/tests/test_publish.py index 2f8be36a..eaf2f542 100755 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -13,10 +13,12 @@ import json import os import pathlib +import ruamel.yaml import shutil import sys import unittest import tempfile +from pathlib import Path from unittest import mock from fdroidserver import publish @@ -94,8 +96,7 @@ class PublishTest(unittest.TestCase): ] os.chdir(self.testdir) - with open('config.py', 'w') as f: - pass + Path('config.yml').write_text('') publish.store_stats_fdroid_signing_key_fingerprints(appids, indent=2) @@ -116,11 +117,13 @@ class PublishTest(unittest.TestCase): } self.assertEqual(expected, common.load_stats_fdroid_signing_key_fingerprints()) - with open('config.py', 'r') as f: - self.assertEqual( - '\nrepo_key_sha256 = "c58460800c7b250a619c30c13b07b7359a43e5af71a4352d86c58ae18c9f6d41"\n', - f.read(), - ) + yaml = ruamel.yaml.YAML(typ='safe') + with open('config.yml') as fp: + config = yaml.load(fp) + self.assertEqual( + 'c58460800c7b250a619c30c13b07b7359a43e5af71a4352d86c58ae18c9f6d41', + config['repo_key_sha256'], + ) def test_store_and_load_fdroid_signing_key_fingerprints_with_missmatch(self): common.config = {}