[Git][reproducible-builds/diffoscope][master] 3 commits: Make the code clearer around generating the Debian substvars.
Chris Lamb (@lamby)
gitlab at salsa.debian.org
Tue Nov 29 10:14:41 UTC 2022
Chris Lamb pushed to branch master at Reproducible Builds / diffoscope
Commits:
58fd63c8 by Chris Lamb at 2022-11-29T10:14:02+00:00
Make the code clearer around generating the Debian substvars.
- - - - -
1852890a by FC Stegerman at 2022-11-29T10:14:03+00:00
As an optimisation, don't run apktool if no differences are detected before the signing block. (Closes: reproducible-builds/diffoscope!105)
Signed-off-by: Chris Lamb <lamby at debian.org>
- - - - -
792115b9 by Chris Lamb at 2022-11-29T10:14:03+00:00
Make sure we recommend apksigcopier. (Re: reproducible-builds/diffoscope!105)
- - - - -
5 changed files:
- debian/control
- debian/tests/control
- diffoscope/comparators/apk.py
- diffoscope/external_tools.py
- diffoscope/main.py
Changes:
=====================================
debian/control
=====================================
@@ -9,6 +9,7 @@ Uploaders:
Build-Depends:
abootimg <!nocheck>,
androguard <!nocheck>,
+ apksigcopier <!nocheck>,
apksigner <!nocheck>,
apktool [!ppc64el !s390x] <!nocheck>,
bash-completion,
=====================================
debian/tests/control
=====================================
@@ -7,7 +7,7 @@
# $ mv debian/tests/control.tmp debian/tests/control
Tests: pytest-with-recommends
-Depends: python3-all, diffoscope, black, python3-pytest, python3-h5py, file, linux-image-amd64 [amd64] | linux-image-generic [amd64], abootimg, acl, apksigner, apktool [!ppc64el !s390x], binutils-multiarch, bzip2, caca-utils, colord, coreboot-utils, db-util, default-jdk-headless | default-jdk | java-sdk, device-tree-compiler, docx2txt, e2fsprogs, enjarify, ffmpeg, fontforge-extras, fonttools, fp-utils [!ppc64el !s390x], genisoimage, gettext, ghc, ghostscript, giflib-tools, gnumeric, gnupg, gnupg-utils, hdf5-tools, html2text, imagemagick, jsbeautifier, libarchive-tools, libxmlb-dev, llvm, lz4 | liblz4-tool, lzip, mono-utils, ocaml-nox, odt2txt, oggvideotools [!s390x], openssh-client, openssl, pgpdump, poppler-utils, procyon-decompiler, python3-pdfminer, r-base-core, rpm2cpio, sng, sqlite3, squashfs-tools, tcpdump, u-boot-tools, unzip, wabt, xmlbeans, xxd, xz-utils, zip, zstd, androguard, python3-argcomplete, python3-binwalk, python3-defusedxml, python3-distro, python3-guestfs, python3-jsondiff, python3-progressbar, python3-pypdf2, python3-debian, python3-pyxattr, python3-rpm, python3-tlsh
+Depends: python3-all, diffoscope, black, python3-pytest, python3-h5py, file, linux-image-amd64 [amd64] | linux-image-generic [amd64], abootimg, acl, apksigcopier, apksigner, apktool [!ppc64el !s390x], binutils-multiarch, bzip2, caca-utils, colord, coreboot-utils, db-util, default-jdk-headless | default-jdk | java-sdk, device-tree-compiler, docx2txt, e2fsprogs, enjarify, ffmpeg, fontforge-extras, fonttools, fp-utils [!ppc64el !s390x], genisoimage, gettext, ghc, ghostscript, giflib-tools, gnumeric, gnupg, gnupg-utils, hdf5-tools, html2text, imagemagick, jsbeautifier, libarchive-tools, libxmlb-dev, llvm, lz4 | liblz4-tool, lzip, mono-utils, ocaml-nox, odt2txt, oggvideotools [!s390x], openssh-client, openssl, pgpdump, poppler-utils, procyon-decompiler, python3-pdfminer, r-base-core, rpm2cpio, sng, sqlite3, squashfs-tools, tcpdump, u-boot-tools, unzip, wabt, xmlbeans, xxd, xz-utils, zip, zstd,
Tests: pytest
Depends: python3-all, diffoscope, python3-pytest, python3-h5py, file, python3-tlsh
=====================================
diffoscope/comparators/apk.py
=====================================
@@ -3,6 +3,7 @@
#
# Copyright © 2016 Reiner Herrmann <reiner at reiner-h.de>
# Copyright © 2016-2021 Chris Lamb <lamby at debian.org>
+# Copyright © 2022 FC Stegerman <flx at obfusk.net>
#
# diffoscope is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -230,7 +231,20 @@ class ApkFile(ZipFileBase):
FILE_EXTENSION_SUFFIX = {".apk"}
CONTAINER_CLASSES = [ApkContainer, ZipContainer]
+ @property
+ def as_container(self):
+ # If we found no differences before the APK Signing Block we return None
+ # here to prevent apktool from being run needlessly (which can take up a
+ # significant amount of extra time) via ApkContainer (since there's no
+ # API that allows us to selectively disable use of container classes in
+ # cases like these).
+ if getattr(self, "_disable_container_compare", False):
+ return None # don't run apktool
+ return super().as_container
+
def compare_details(self, other, source=None):
+ self.check_differences_before_signing_block(other)
+
differences = zipinfo_differences(self, other)
try:
@@ -258,6 +272,42 @@ class ApkFile(ZipFileBase):
return differences
+ def check_differences_before_signing_block(self, other):
+ try:
+ self._check_differences_before_signing_block(other)
+ except (RequiredToolNotFound, ImportError):
+ self.add_comment(
+ "'apksigcopier' Python package not installed; unconditionally running 'apktool'."
+ )
+ return
+
+ @tool_required("apksigcopier")
+ def _check_differences_before_signing_block(self, other):
+ import apksigcopier
+
+ try:
+ offset_self, _ = apksigcopier.extract_v2_sig(self.path)
+ offset_other, _ = apksigcopier.extract_v2_sig(other.path)
+ except Exception:
+ return
+
+ if offset_self != offset_other:
+ return
+
+ with open(self.path, "rb") as fh_self:
+ with open(other.path, "rb") as fh_other:
+ while fh_self.tell() < offset_self:
+ size = min(offset_self - fh_self.tell(), 4096)
+ if fh_self.read(size) != fh_other.read(size):
+ return
+
+ self.add_comment(
+ "No differences before APK Signing Block; not running 'apktool'."
+ )
+
+ self._disable_container_compare = True
+ other._disable_container_compare = True
+
def get_v2_signing_keys(path):
from androguard.core.bytecodes import apk
=====================================
diffoscope/external_tools.py
=====================================
@@ -25,6 +25,7 @@ that might resolve to, for example, `/usr/bin/abootimg`..
EXTERNAL_TOOLS = {
"abootimg": {"debian": "abootimg", "guix": "abootimg"},
"androguard": {"debian": "androguard"},
+ "apksigcopier": {"debian": "apksigcopier"},
"apktool": {"debian": "apktool"},
"apksigner": {"debian": "apksigner"},
"db_dump": {"debian": "db-util", "guix": "bdb"},
=====================================
diffoscope/main.py
=====================================
@@ -566,18 +566,19 @@ class ListMissingToolsAction(ListToolsAction):
class ListDebianSubstvarsAction(argparse._StoreTrueAction):
def __call__(self, *args, **kwargs):
+ tools = set()
+
# Attempt to import all comparators so tool_required.all is as
# populated as possible...
ComparatorManager().reload()
+ tools.update(tool_required.all)
# ... however for the generated substvar to be effective/deterministic
# regardless of the currently installed packages, we special-case some
# tools (NB. not package names) as their modules may not have been
# imported by the `ComparatorManager().reload()` call above. (#908072)
- tools = set(
- ("gpg", "rpm2cpio") # comparators/debian.py # comparators/rpm.py
- )
- tools.update(tool_required.all)
+ tools.add("gpg") # comparators/debian.py
+ tools.add("rpm2cpio") # comparators/rpm.py
packages = set()
packages_minimal = set()
View it on GitLab: https://salsa.debian.org/reproducible-builds/diffoscope/-/compare/ecada35b88d0bd78e0542cda94eba377740c283d...792115b9f6c3c61e8a957ce8c73fe6c78713aede
--
View it on GitLab: https://salsa.debian.org/reproducible-builds/diffoscope/-/compare/ecada35b88d0bd78e0542cda94eba377740c283d...792115b9f6c3c61e8a957ce8c73fe6c78713aede
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.reproducible-builds.org/pipermail/rb-commits/attachments/20221129/20a8e34e/attachment.htm>
More information about the rb-commits
mailing list