[Git][reproducible-builds/diffoscope][master] comparators: Add FIT comparator
Chris Lamb
gitlab at salsa.debian.org
Tue Jan 5 15:00:39 UTC 2021
Chris Lamb pushed to branch master at Reproducible Builds / diffoscope
Commits:
ebd3bdcb by Conrad Ratschan at 2021-01-05T14:50:12+00:00
comparators: Add FIT comparator
Add comparator for U-Boot Flattened Image Tree files. Add required tools
and tests.
- - - - -
5 changed files:
- diffoscope/comparators/__init__.py
- + diffoscope/comparators/fit.py
- diffoscope/external_tools.py
- + tests/comparators/test_fit.py
- + tests/data/fit_expected_diff
Changes:
=====================================
diffoscope/comparators/__init__.py
=====================================
@@ -105,6 +105,7 @@ class ComparatorManager:
("pgp.PgpFile",),
("pgp.PgpSignature",),
("kbx.KbxFile",),
+ ("fit.FlattenedImageTreeFile",),
("dtb.DeviceTreeFile",),
("ogg.OggFile",),
("uimage.UimageFile",),
=====================================
diffoscope/comparators/fit.py
=====================================
@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2020-2021 Conrad Ratschan <ratschance at gmail.com>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope. If not, see <https://www.gnu.org/licenses/>.
+import logging
+import os
+import re
+
+from diffoscope.tools import tool_required, tool_check_installed
+from diffoscope.difference import Difference
+from .utils import command
+from .utils.archive import Archive
+
+from .utils.file import File
+from .utils.command import Command
+
+logger = logging.getLogger(__name__)
+
+
+class FitContainer(Archive):
+ # Match the image string in dumpimage list output. Example: " Image 0 (ramdisk at 0)"
+ IMAGE_RE = re.compile(r"^\sImage ([0-9]+) \((.+)\)", re.MULTILINE)
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._members = {}
+
+ def open_archive(self):
+ return self
+
+ def close_archive(self):
+ pass
+
+ @tool_required("dumpimage")
+ def get_member_names(self):
+ image_info = command.our_check_output(
+ ["dumpimage", "-l", self.source.path],
+ )
+ members = []
+ for match in re.finditer(
+ self.IMAGE_RE, image_info.decode(encoding="utf-8")
+ ):
+ pos, member_name = match.group(1, 2)
+ # Save mapping of name -> position as dumpimage takes position as an argument
+ self._members[member_name] = pos
+ members.append(member_name)
+ return members
+
+ @tool_required("dumpimage")
+ def extract(self, member_name, dest_dir):
+ pos = self._members[member_name]
+ dest_path = os.path.join(dest_dir, os.path.basename(member_name))
+ logger.debug("fit image extracting %s to %s", member_name, dest_path)
+
+ command.our_check_output(
+ [
+ "dumpimage",
+ "-T",
+ "flat_dt",
+ "-p",
+ pos,
+ self.source.path,
+ "-o",
+ dest_path,
+ ],
+ )
+ return dest_path
+
+
+class FlattenedImageTreeContents(Command):
+ @tool_required("dumpimage")
+ def cmdline(self):
+ return ["dumpimage", "-l", self.path]
+
+
+class FlattenedImageTreeFile(File):
+ """
+ Flattened Image Trees (FIT) are a newer boot image format used by U-Boot. This
+ format allows for multiple kernels, root filesystems, device trees, boot
+ configurations, and checksums to be packaged into one file. It leverages the
+ Flattened Device Tree Blob file format.
+ """
+
+ DESCRIPTION = "Flattened Image Tree blob files"
+ FILE_TYPE_RE = re.compile(r"^Device Tree Blob")
+ CONTAINER_CLASSES = [FitContainer]
+
+ @classmethod
+ def recognizes(cls, file):
+ if not cls.FILE_TYPE_RE.search(file.magic_file_type):
+ return False
+
+ if not tool_check_installed("fdtdump"):
+ return False
+
+ # Since the file type is the same as a Device Tree Blob, use fdtget (same
+ # package as fdtdump) to differentiate between FIT/DTB
+ root_nodes = (
+ command.our_check_output(["fdtget", file.path, "-l", "/"])
+ .decode(encoding="utf-8")
+ .strip()
+ .split("\n")
+ )
+ root_props = (
+ command.our_check_output(["fdtget", file.path, "-p", "/"])
+ .decode(encoding="utf-8")
+ .strip()
+ .split("\n")
+ )
+
+ # Check for mandatory FIT items used in U-Boot's FIT image verification routine
+ return "description" in root_props and "images" in root_nodes
+
+ def compare_details(self, other, source=None):
+ return [
+ Difference.from_operation(
+ FlattenedImageTreeContents, self.path, other.path
+ )
+ ]
=====================================
diffoscope/external_tools.py
=====================================
@@ -51,6 +51,11 @@ EXTERNAL_TOOLS = {
"cpio": {"debian": "cpio", "arch": "cpio", "guix": "cpio"},
"diff": {"debian": "diffutils", "arch": "diffutils", "guix": "diffutils"},
"docx2txt": {"debian": "docx2txt", "arch": "docx2txt", "guix": "docx2txt"},
+ "dumpimage": {
+ "debian": "u-boot-tools",
+ "arch": "uboot-tools",
+ "guix": "u-boot-tools",
+ },
"enjarify": {"debian": "enjarify", "arch": "enjarify", "guix": "enjarify"},
"fdtdump": {
"debian": "device-tree-compiler",
=====================================
tests/comparators/test_fit.py
=====================================
@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2020 Conrad Ratschan <ratschance at gmail.com>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope. If not, see <https://www.gnu.org/licenses/>.
+import os
+import subprocess
+
+import pytest
+
+from diffoscope.comparators.binary import FilesystemFile
+from diffoscope.comparators.fit import FlattenedImageTreeFile
+from diffoscope.comparators.utils.specialize import specialize
+
+from ..utils.data import data, get_data, load_fixture
+from ..utils.tools import skip_unless_tools_exist
+from ..utils.nonexisting import assert_non_existing
+
+cpio1 = load_fixture("test1.cpio")
+cpio2 = load_fixture("test2.cpio")
+
+
+def fit_fixture(prefix, entrypoint):
+ @pytest.fixture
+ def uboot_fit(tmp_path):
+ cpio = data("{}.cpio".format(prefix))
+ fit_out = tmp_path / "{}.itb".format(prefix)
+
+ # Time must be controlled for reproducible FIT images
+ time_env = os.environ.copy()
+ time_env["SOURCE_DATE_EPOCH"] = "1609459200"
+ time_env["TZ"] = "UTC"
+ _ = subprocess.run(
+ [
+ "mkimage",
+ "-f",
+ "auto",
+ "-A",
+ "arm",
+ "-O",
+ "linux",
+ "-T",
+ "ramdisk",
+ "-C",
+ "none",
+ "-e",
+ entrypoint,
+ "-d",
+ cpio,
+ fit_out,
+ ],
+ capture_output=True,
+ env=time_env,
+ )
+ return specialize(FilesystemFile(str(fit_out)))
+
+ return uboot_fit
+
+
+uboot_fit1 = fit_fixture("test1", "0x1000")
+uboot_fit2 = fit_fixture("test2", "0x2000")
+
+
+ at skip_unless_tools_exist("dumpimage")
+ at skip_unless_tools_exist("fdtdump")
+def test_identification(uboot_fit1, uboot_fit2):
+ assert isinstance(uboot_fit1, FlattenedImageTreeFile)
+ assert isinstance(uboot_fit2, FlattenedImageTreeFile)
+
+
+ at skip_unless_tools_exist("dumpimage")
+ at skip_unless_tools_exist("fdtdump")
+def test_no_differences(uboot_fit1):
+ difference = uboot_fit1.compare(uboot_fit1)
+ assert difference is None
+
+
+ at pytest.fixture
+def differences(uboot_fit1, uboot_fit2):
+ return uboot_fit1.compare(uboot_fit2).details
+
+
+ at pytest.fixture
+def nested_differences(uboot_fit1, uboot_fit2):
+ return uboot_fit1.compare(uboot_fit2).details[1].details
+
+
+ at skip_unless_tools_exist("dumpimage")
+ at skip_unless_tools_exist("fdtdump")
+def test_file_differences(differences):
+ expected_diff = get_data("fit_expected_diff")
+ assert differences[0].unified_diff == expected_diff
+
+
+ at skip_unless_tools_exist("cpio")
+ at skip_unless_tools_exist("dumpimage")
+ at skip_unless_tools_exist("fdtdump")
+def test_nested_listing(nested_differences):
+ expected_diff = get_data("cpio_listing_expected_diff")
+ assert nested_differences[0].unified_diff == expected_diff
+
+
+ at skip_unless_tools_exist("cpio")
+ at skip_unless_tools_exist("dumpimage")
+ at skip_unless_tools_exist("fdtdump")
+def test_nested_symlink(nested_differences):
+ assert nested_differences[1].source1 == "dir/link"
+ assert nested_differences[1].comment == "symlink"
+ expected_diff = get_data("symlink_expected_diff")
+ assert nested_differences[1].unified_diff == expected_diff
+
+
+ at skip_unless_tools_exist("cpio")
+ at skip_unless_tools_exist("dumpimage")
+ at skip_unless_tools_exist("fdtdump")
+def test_nested_compressed_files(nested_differences):
+ assert nested_differences[2].source1 == "dir/text"
+ assert nested_differences[2].source2 == "dir/text"
+ expected_diff = get_data("text_ascii_expected_diff")
+ assert nested_differences[2].unified_diff == expected_diff
+
+
+ at skip_unless_tools_exist("cpio")
+ at skip_unless_tools_exist("dumpimage")
+ at skip_unless_tools_exist("fdtdump")
+def test_compare_non_existing(monkeypatch, uboot_fit1):
+ assert_non_existing(monkeypatch, uboot_fit1)
=====================================
tests/data/fit_expected_diff
=====================================
@@ -0,0 +1,15 @@
+@@ -5,13 +5,13 @@
+ Created: Fri Jan 1 00:00:00 2021
+ Type: RAMDisk Image
+ Compression: uncompressed
+ Data Size: 1024 Bytes = 1.00 KiB = 0.00 MiB
+ Architecture: ARM
+ OS: Linux
+ Load Address: 0x00000000
+- Entry Point: 0x00001000
++ Entry Point: 0x00002000
+ Default Configuration: 'conf-1'
+ Configuration 0 (conf-1)
+ Description: unavailable
+ Kernel: unavailable
+ Init Ramdisk: ramdisk-1
View it on GitLab: https://salsa.debian.org/reproducible-builds/diffoscope/-/commit/ebd3bdcb6127350be8a4e3716f81dc640690e4c8
--
View it on GitLab: https://salsa.debian.org/reproducible-builds/diffoscope/-/commit/ebd3bdcb6127350be8a4e3716f81dc640690e4c8
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.reproducible-builds.org/pipermail/rb-commits/attachments/20210105/4d4e41d7/attachment.htm>
More information about the rb-commits
mailing list