[Git][reproducible-builds/diffoscope][master] 2 commits: Add support for UKI files. (Closes: reproducible-builds/diffoscope!146)
Chris Lamb (@lamby)
gitlab at salsa.debian.org
Tue Oct 15 13:53:01 UTC 2024
Chris Lamb pushed to branch master at Reproducible Builds / diffoscope
Commits:
9d5b5d32 by Jelle van der Waa at 2024-10-14T10:23:26-07:00
Add support for UKI files. (Closes: reproducible-builds/diffoscope!146)
Diffoscope supported diffing UKI's with objdump in Pe32PlusFile, but
this doesn't work too well with 200M+ UKI's. It takes a long time to
generate the diff, ukify from systemd-ukify gives a user-friendly dump
of the sections with hashes so one can easily see what part of the UKI
is not reproducible.
Signed-off-by: Chris Lamb <lamby at debian.org>
- - - - -
0736b361 by Chris Lamb at 2024-10-14T10:23:34-07:00
Don't try and test with systemd-ukify in Debian stable.
- - - - -
9 changed files:
- .gitlab-ci.yml
- debian/control
- debian/tests/control
- diffoscope/comparators/__init__.py
- + diffoscope/comparators/uki.py
- diffoscope/external_tools.py
- + tests/comparators/test_uki.py
- + tests/data/dummyx64.efi.stub
- + tests/data/uki_expected_diff
Changes:
=====================================
.gitlab-ci.yml
=====================================
@@ -39,7 +39,7 @@ stable-bpo:
<<: *test
image: debian:stable-backports
before_script:
- - set -e; for pkg in coreboot-utils python3-pypdf; do
+ - set -e; for pkg in coreboot-utils python3-pypdf systemd-ukify; do
sed -i "/$pkg/d" debian/control ;
done
=====================================
debian/control
=====================================
@@ -89,6 +89,7 @@ Build-Depends:
sng <!nocheck>,
sqlite3 <!nocheck>,
squashfs-tools <!nocheck>,
+ systemd-ukify <!nocheck>,
tcpdump <!nocheck>,
u-boot-tools <!nocheck>,
unzip <!nocheck>,
=====================================
debian/tests/control
=====================================
@@ -7,7 +7,7 @@
# $ mv debian/tests/control.tmp debian/tests/control
Tests: pytest-with-recommends
-Depends: python3-all, diffoscope, black, python3-pytest, python3-h5py, file, linux-image-amd64 [amd64] | linux-image-generic [amd64], 7zip, abootimg, acl, apksigcopier, apksigner, apktool, binutils-multiarch, bzip2, caca-utils, colord, coreboot-utils [!risv64], db-util, default-jdk-headless | default-jdk | java-sdk, device-tree-compiler, docx2txt, e2fsprogs, enjarify, ffmpeg, fontforge-extras, fonttools, fp-utils [!riscv64 !s390x], genisoimage, gettext, ghc, ghostscript, giflib-tools, gnumeric, gnupg-utils, gpg, hdf5-tools, html2text, imagemagick, jsbeautifier, libarchive-tools, libxmlb-utils, llvm, lz4, lzip, mono-utils [!riscv64], ocaml-nox, odt2txt, oggvideotools [!s390x], openssh-client, openssl, perl, pgpdump, poppler-utils, procyon-decompiler, python3-pdfminer, r-base-core, rpm2cpio, sng, sqlite3, squashfs-tools, tcpdump, u-boot-tools, unzip, wabt, xmlbeans, xxd, xz-utils, zip, zstd, python3-argcomplete, python3-binwalk, python3-defusedxml, python3-distro, python3-guestfs, python3-jsondiff, python3-progressbar, python3-pypdf, python3-debian, python3-pyxattr, python3-rpm, python3-tlsh
+Depends: python3-all, diffoscope, black, python3-pytest, python3-h5py, file, linux-image-amd64 [amd64] | linux-image-generic [amd64], 7zip, abootimg, acl, apksigcopier, apksigner, apktool, binutils-multiarch, bzip2, caca-utils, colord, coreboot-utils [!risv64], db-util, default-jdk-headless | default-jdk | java-sdk, device-tree-compiler, docx2txt, e2fsprogs, enjarify, ffmpeg, fontforge-extras, fonttools, fp-utils [!riscv64 !s390x], genisoimage, gettext, ghc, ghostscript, giflib-tools, gnumeric, gnupg-utils, gpg, hdf5-tools, html2text, imagemagick, jsbeautifier, libarchive-tools, libxmlb-utils, llvm, lz4, lzip, mono-utils [!riscv64], ocaml-nox, odt2txt, oggvideotools [!s390x], openssh-client, openssl, perl, pgpdump, poppler-utils, procyon-decompiler, python3-pdfminer, r-base-core, rpm2cpio, sng, sqlite3, squashfs-tools, systemd-ukify, tcpdump, u-boot-tools, unzip, wabt, xmlbeans, xxd, xz-utils, zip, zstd, python3-argcomplete, python3-binwalk, python3-defusedxml, python3-distro, python3-guestfs, python3-jsondiff, python3-progressbar, python3-pypdf, python3-debian, python3-pyxattr, python3-rpm, python3-tlsh
Tests: pytest
Depends: python3-all, diffoscope, python3-pytest, python3-h5py, file, python3-tlsh
=====================================
diffoscope/comparators/__init__.py
=====================================
@@ -108,6 +108,7 @@ class ComparatorManager:
("openssh.PublicKeyFile",),
("gif.GifFile",),
("pcap.PcapFile",),
+ ("uki.UKIFile",),
("pe32.Pe32PlusFile",),
("pgp.PgpFile",),
("pgp.PgpSignature",),
=====================================
diffoscope/comparators/uki.py
=====================================
@@ -0,0 +1,57 @@
+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2024 Jelle van der Waa <jelle at archlinux.org>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope. If not, see <https://www.gnu.org/licenses/>.
+
+
+import re
+import logging
+
+from diffoscope.tools import tool_required
+from diffoscope.difference import Difference
+
+from .utils.file import File
+from .utils.command import Command, our_check_output
+
+logger = logging.getLogger(__name__)
+
+
+class UKIDump(Command):
+ @tool_required("ukify")
+ def cmdline(self):
+ return ["ukify", "inspect", self.path]
+
+
+class UKIFile(File):
+ DESCRIPTION = "UKI image (.efi)"
+ FILE_EXTENSION_SUFFIX = {".efi"}
+ FILE_TYPE_RE = re.compile(r"^PE32\+")
+
+ @classmethod
+ def recognizes(cls, file):
+ if not super().recognizes(file):
+ return False
+
+ try:
+ our_check_output(
+ ["objdump", "-h", "--section", ".linux", file.path],
+ )
+ return True
+ except subprocess.SubprocessError:
+ return False
+
+ def compare_details(self, other, source=None):
+ return [Difference.from_operation(UKIDump, self.path, other.path)]
=====================================
diffoscope/external_tools.py
=====================================
@@ -224,6 +224,10 @@ EXTERNAL_TOOLS = {
"tar": {"debian": "tar", "arch": "tar", "guix": "tar"},
"tcpdump": {"debian": "tcpdump", "arch": "tcpdump", "guix": "tcpdump"},
"ttx": {"debian": "fonttools", "guix": "python-fonttools"},
+ "ukify": {
+ "debian": "systemd-ukify",
+ "arch": "systemd-ukify",
+ },
"unsquashfs": {
"debian": "squashfs-tools",
"arch": "squashfs-tools",
=====================================
tests/comparators/test_uki.py
=====================================
@@ -0,0 +1,81 @@
+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2024 Jelle van der Waa <jelle at archlinux.org>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope. If not, see <https://www.gnu.org/licenses/>.
+
+import pytest
+
+from diffoscope.comparators.binary import FilesystemFile
+from diffoscope.comparators.uki import UKIFile
+from diffoscope.comparators.utils.command import our_check_output
+from diffoscope.comparators.utils.specialize import specialize
+
+from ..utils.data import assert_diff, load_fixture
+from ..utils.tools import skip_unless_tools_exist
+
+efi_stub = load_fixture("dummyx64.efi.stub")
+
+
+def uki_fixture(prefix, os_release, uname):
+ @pytest.fixture
+ def uki(tmpdir, efi_stub):
+ input_ = str(tmpdir.join(prefix))
+ output = str(tmpdir.join("{}.unsigned.efi".format(prefix)))
+
+ with open(input_, "w") as fp:
+ fp.write("kernel")
+
+ our_check_output(
+ [
+ "ukify",
+ "build",
+ "--stub",
+ efi_stub.path,
+ "--linux",
+ input_,
+ "--os-release",
+ os_release,
+ "--uname",
+ uname,
+ ],
+ cwd=tmpdir,
+ )
+
+ return specialize(FilesystemFile(output))
+
+ return uki
+
+
+uki1 = uki_fixture("linux1", "arch", "6.11")
+uki2 = uki_fixture("linux2", "debian", "6.12")
+
+
+ at pytest.fixture
+def differences(uki1, uki2):
+ return uki1.compare(uki2).details
+
+
+ at skip_unless_tools_exist("objdump")
+ at skip_unless_tools_exist("ukify")
+def test_no_differences(uki1):
+ difference = uki1.compare(uki1)
+ assert difference is None
+
+
+ at skip_unless_tools_exist("objdump")
+ at skip_unless_tools_exist("ukify")
+def test_diff(differences):
+ assert_diff(differences[0], "uki_expected_diff")
=====================================
tests/data/dummyx64.efi.stub
=====================================
Binary files /dev/null and b/tests/data/dummyx64.efi.stub differ
=====================================
tests/data/uki_expected_diff
=====================================
Binary files /dev/null and b/tests/data/uki_expected_diff differ
View it on GitLab: https://salsa.debian.org/reproducible-builds/diffoscope/-/compare/1724e4c0703b0b43fa0e49a171535620835e3022...0736b36104dbe68ee34b0aeaaf31cc0d73d4a967
--
View it on GitLab: https://salsa.debian.org/reproducible-builds/diffoscope/-/compare/1724e4c0703b0b43fa0e49a171535620835e3022...0736b36104dbe68ee34b0aeaaf31cc0d73d4a967
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.reproducible-builds.org/pipermail/rb-commits/attachments/20241015/902562ef/attachment.htm>
More information about the rb-commits
mailing list