verify: handle corrupt verified.json

verified.json can get quite large on verification.f-droid.org, and for some
unknown reason, it sometimes corrupts it when writing it out.  All the data
is already available in all the other JSON files, so this just automatically
reconstructs it.  Its a hack, but it took me much less time than I've
already spent trying to troubleshoot why it writes out corrupt verified.json.

fdroidserver/verify.py

@@ -80,6 +80,30 @@ def _add_diffoscope_info(d):
         pass
 
 
+def get_verified_json(path):
+    """Get the full collection of reports that is written out to verified.json."""
+    if os.path.exists(path):
+        try:
+            with open(path) as fp:
+                return json.load(fp)
+        except Exception as e:
+            logging.info(f'{path}: {e}')
+
+    data = OrderedDict()
+    data['packages'] = OrderedDict()
+
+    for f in glob.glob(os.path.join(os.path.dirname(path), '*.apk.json')):
+        with open(f) as fp:
+            reports = json.load(fp)
+        for report in reports.values():
+            packageName = report['local']['packageName']
+            if packageName not in data['packages']:
+                data['packages'][packageName] = []
+            data['packages'][packageName].append(report)
+
+    return data
+
+
 def write_json_report(url, remote_apk, unsigned_apk, compare_result):
     """Write out the results of the verify run to JSON.
 
@@ -122,12 +146,7 @@ def write_json_report(url, remote_apk, unsigned_apk, compare_result):
 
     if output['verified']:
         jsonfile = 'unsigned/verified.json'
-        if os.path.exists(jsonfile):
+        data = get_verified_json(jsonfile)
-            with open(jsonfile) as fp:
-                data = json.load(fp)
-        else:
-            data = OrderedDict()
-            data['packages'] = OrderedDict()
         packageName = output['local']['packageName']
 
         if packageName not in data['packages']:

tests/org.fdroid.fdroid_1019051.apk.json

@@ -0,0 +1,43 @@
+{
+  "1708238023.6572325": {
+    "diffoscope": {
+      "Available-in-Debian-packages": [
+        "coreboot-utils",
+        "ghc",
+        "libxmlb-dev",
+        "radare2"
+      ],
+      "External-Tools-Required": [
+        "cbfstool",
+        "ghc",
+        "lipo",
+        "otool",
+        "radare2",
+        "xb-tool"
+      ],
+      "Missing-Python-Modules": [
+        "pypdf",
+        "r2pipe"
+      ],
+      "VERSION": "240"
+    },
+    "local": {
+      "file": "unsigned/org.fdroid.fdroid_1019051.apk",
+      "packageName": "org.fdroid.fdroid",
+      "sha256": "0eec78236ec5ebb8f416c611717bd659b75d2b6600ef71a50c922efc99dbdca2",
+      "timestamp": 1708238023.6572325,
+      "versionCode": 1019051,
+      "versionName": "1.19.1"
+    },
+    "remote": {
+      "file": "tmp/org.fdroid.fdroid_1019051.apk",
+      "packageName": "org.fdroid.fdroid",
+      "sha256": "162cb14b93bd9b665fff4256b4fc91cfe75fb72335a02b1d0febe606220b50f4",
+      "timestamp": 1715356428.522411,
+      "versionCode": 1019051,
+      "versionName": "1.19.1"
+    },
+    "url": "https://f-droid.org/repo/org.fdroid.fdroid_1019051.apk",
+    "verified": true
+  }
+}

tests/test_verify.py

@@ -44,10 +44,53 @@ class VerifyTest(unittest.TestCase):
         os.chdir(self.tempdir.name)
         self.repodir = Path('repo')
         self.repodir.mkdir()
+        self.apk_reports_json = basedir / 'org.fdroid.fdroid_1019051.apk.json'
 
     def tearDown(self):
         self.tempdir.cleanup()
 
+    def test_get_verified_json_creation(self):
+        self.assertEqual({'packages': {}}, verify.get_verified_json('does-not-exist'))
+
+    def test_get_verified_json_existing(self):
+        f = 'verified.json'
+        reports = {'packages': {'placeholder': {}}}
+        with open(f, 'w') as fp:
+            json.dump(reports, fp)
+        self.assertEqual(reports, verify.get_verified_json(f))
+
+    def test_get_verified_json_pull_in_one_report(self):
+        shutil.copy(self.apk_reports_json, self.tempdir.name)
+        with open(self.apk_reports_json) as fp:
+            reports = json.load(fp)
+        self.assertEqual(
+            {'packages': {'org.fdroid.fdroid': [reports['1708238023.6572325']]}},
+            verify.get_verified_json('does-not-exist'),
+        )
+
+    def test_get_verified_json_ignore_corrupt(self):
+        f = 'verified.json'
+        with open(f, 'w') as fp:
+            fp.write("""{"packages": {"placeholder": {""")
+        shutil.copy(self.apk_reports_json, self.tempdir.name)
+        with open(self.apk_reports_json) as fp:
+            reports = json.load(fp)
+        self.assertEqual(
+            {'packages': {'org.fdroid.fdroid': [reports['1708238023.6572325']]}},
+            verify.get_verified_json(f),
+        )
+
+    def test_get_verified_json_ignore_apk_reports(self):
+        """When an intact verified.json exists, it should ignore the .apk.json reports."""
+        f = 'verified.json'
+        placeholder = {'packages': {'placeholder': {}}}
+        with open(f, 'w') as fp:
+            json.dump(placeholder, fp)
+        shutil.copy(self.apk_reports_json, self.tempdir.name)
+        with open(self.apk_reports_json) as fp:
+            json.load(fp)
+        self.assertEqual(placeholder, verify.get_verified_json(f))
+
     @patch('fdroidserver.common.sha256sum')
     def test_write_json_report(self, sha256sum):
         sha256sum.return_value = (