[Git][reproducible-builds/diffoscope][pax] diffoscope/comparators/tar.py: support for POSIX.1-2001 PAX headers
Johannes Schauer Marin Rodrigues (@josch)
gitlab at salsa.debian.org
Tue Jun 30 20:42:19 UTC 2026
Johannes Schauer Marin Rodrigues pushed to branch pax at Reproducible Builds / diffoscope
Commits:
7c7a4cae by Johannes Schauer Marin Rodrigues at 2026-06-30T22:41:53+02:00
diffoscope/comparators/tar.py: support for POSIX.1-2001 PAX headers
Uses python tarfile to find differences in PAX headers.
The libarchive based reader is not able to spot differences in
POSIX.1-2001 PAX headers. libarchive itself is capable of reading this
information (the archive_entry_xattr_* functions) but it doesn't seem to
expose the relevant functions via libarchive.ffi.
The PAX header stores information like
* very long filenames and linknames
* extended attributes
* POSIX ACLs
* atime and ctime with sub-second resolution
* GNU/Hurd translators
* sparse files
* files larger than 8 GiB
* device major and minor numbers with more than 21 bits
* uid/gid with more than 21 bits
If libarchive is unable to see a difference between the tarballs we fall
back to the Python tarfile module and try to print more information.
- - - - -
2 changed files:
- diffoscope/comparators/tar.py
- tests/comparators/test_tar.py
Changes:
=====================================
diffoscope/comparators/tar.py
=====================================
@@ -24,6 +24,149 @@ from diffoscope.difference import Difference
from .utils.file import File
from .utils.libarchive import LibarchiveContainer, list_libarchive
+import tarfile
+import datetime
+import struct
+
+def timestamp2str(ts):
+ try:
+ return str(datetime.datetime.fromtimestamp(float(ts), datetime.UTC))
+ except ValueError:
+ return f"Unable to parse: {ts}"
+
+def read_number(v):
+ try:
+ return int(v)
+ except ValueError:
+ return f"Unable to parse: {ts}"
+
+# from https://salsa.debian.org/josch/seedfiles-integration-tests/-/blob/main/test.py
+def parse_xattr(acl):
+ # see uapi/linux/posix_acl.h
+ POSIX_ACL_XATTR_VERSION = 0x0002
+ assert (POSIX_ACL_XATTR_VERSION,) == struct.unpack("<I", acl[:4])
+ acl = acl[4:]
+ assert (1,) == struct.unpack("<H", acl[:2]) # a_refcount
+ (count,) = struct.unpack("<H", acl[2:4])
+ assert b"\xff\xff\xff\xff" == acl[4:8] # filler?
+ # make sure that the acl minus the header is a multiple of 8
+ assert len(acl) % 8 == 0
+ assert count >= len(acl) / 8
+ count = len(acl) / 8
+ result = ""
+ # see linux/posix_acl.h
+ while acl:
+ e_tag, e_perm, e_id = struct.unpack("<hHI", acl[:8])
+ match e_tag:
+ case 0x01: e_tag = "ACL_USER_OBJ"
+ case 0x02: e_tag = "ACL_USER"
+ case 0x04: e_tag = "ACL_GROUP_OBJ"
+ case 0x08: e_tag = "ACL_GROUP"
+ case 0x10: e_tag = "ACL_MASK"
+ case 0x20: e_tag = "ACL_OTHER"
+ case _: e_tag = f"Unknown e_tag: {e_tag}"
+ e_perm_str = ""
+ if e_perm & 0x04:
+ e_perm_str += "r"
+ if e_perm & 0x02:
+ e_perm_str += "w"
+ if e_perm & 0x01:
+ e_perm_str += "x"
+ if e_id + 1 == 2**32:
+ e_id = "2^32-1"
+ result += f"\n {e_tag:>13} {e_perm_str:<3} {e_id}"
+ acl = acl[8:]
+ return result
+
+def list_pytarfile(path):
+ with tarfile.open(path) as tar:
+ for member in tar:
+ if not member.pax_headers:
+ continue
+ for k, v in sorted(member.pax_headers.items()):
+ p = f"{member.name} PAX header {k}"
+ match k:
+ case "GNU.sparse.numblocks":
+ yield f"{p} raw value: {v}\n"
+ case "GNU.sparse.offset":
+ yield f"{p}: {read_number(v)}\n"
+ case "GNU.sparse.numbytes":
+ yield f"{p}: {read_number(v)}\n"
+ case "GNU.sparse.size":
+ yield f"{p}: {read_number(v)}\n"
+ case "GNU.sparse.map":
+ yield f"{p} raw value: {v}\n"
+ case "GNU.sparse.major":
+ yield f"{p}: {read_number(v)}\n"
+ case "GNU.sparse.minor":
+ yield f"{p}: {read_number(v)}\n"
+ case "GNU.sparse.name":
+ yield f"{p}: {v}\n"
+ case "GNU.sparse.realsize":
+ yield f"{p}: {read_number(v)}\n"
+ case "LIBARCHIVE.creationtime":
+ yield f"{p}: {timestamp2str(v)}\n"
+ case "LIBARCHIVE.symlinktype":
+ match v:
+ case "file"|"dir": yield f"{p}: {v}\n"
+ case _: yield f"{p}: Unknown value: {v}"
+ case "LIBARCHIVE.xattr":
+ yield f"{p} raw value: {v}\n"
+ case "RHT.security.selinux":
+ yield f"{p} raw value: {v}\n"
+ case "SCHILY.acl.access":
+ yield f"{p} raw value: {v}\n"
+ case "SCHILY.acl.default":
+ yield f"{p} raw value: {v}\n"
+ case "SCHILY.acl.ace":
+ yield f"{p} raw value: {v}\n"
+ case "SCHILY.devmajor":
+ yield f"{p}: {read_number(v)}\n"
+ case "SCHILY.devminor":
+ yield f"{p}: {read_number(v)}\n"
+ case "SCHILY.fflags":
+ yield f"{p} raw value: {v}\n"
+ case "SCHILY.dev":
+ yield f"{p}: {read_number(v)}\n"
+ case "SCHILY.ino":
+ yield f"{p}: {read_number(v)}\n"
+ case "SCHILY.nlink":
+ yield f"{p}: {read_number(v)}\n"
+ case "SCHILY.realsize":
+ yield f"{p}: {read_number(v)}\n"
+ case "SCHILY.xattr.gnu.translator":
+ yield f"{p} raw value: {v}\n"
+ case "SCHILY.xattr.system.posix_acl_default":
+ yield f"{p}: {parse_xattr(v.encode(errors='surrogateescape'))}\n"
+ case "SCHILY.xattr.user":
+ yield f"{p} raw value: {v}\n"
+ case "SCHILY.xattr.trusted":
+ yield f"{p} raw value: {v}\n"
+ case "SCHILY.xattr.security":
+ yield f"{p} raw value: {v}\n"
+ case "SUN.holesdata":
+ yield f"{p} raw value: {v}\n"
+ case "atime"|"ctime"|"mtime":
+ yield f"{p}: {timestamp2str(v)}\n"
+ case "gid"|"uid":
+ yield f"{p}: {read_number(v)}\n"
+ case "gname"|"uname":
+ yield f"{p}: {v}\n"
+ case "hdrcharset":
+ match v:
+ case "BINARY":
+ yield "{p}: BINARY\n"
+ case "ISO-IR 10646 2000 UTF-8":
+ yield "{p}: UTF-8\n"
+ case _:
+ yield f"{p}: Unknown value: {v}"
+ case "linkpath":
+ yield f"{p}: {v}\n"
+ case "path":
+ yield f"{p}: {v}\n"
+ case "size":
+ yield f"{p}: {read_number(v)}\n"
+
class TarContainer(LibarchiveContainer):
pass
@@ -35,12 +178,40 @@ class TarFile(File):
FILE_TYPE_RE = re.compile(r"\btar archive\b")
def compare_details(self, other, source=None):
+ libarchive_result = Difference.from_text_readers(
+ list_libarchive(self.path),
+ list_libarchive(other.path),
+ self.path,
+ other.path,
+ source="file list",
+ )
+ if libarchive_result is not None:
+ return [libarchive_result]
+ # The libarchive based reader is not able to spot differences in
+ # POSIX.1-2001 PAX headers. libarchive itself is capable of reading
+ # this information (the archive_entry_xattr_* functions) but it
+ # doesn't seem to expose the relevant functions via libarchive.ffi.
+ #
+ # The PAX header stores information like
+ # * very long filenames and linknames
+ # * extended attributes
+ # * POSIX ACLs
+ # * atime and ctime with sub-second resolution
+ # * GNU/Hurd translators
+ # * sparse files
+ # * files larger than 8 GiB
+ # * device major and minor numbers with more than 21 bits
+ # * uid/gid with more than 21 bits
+ #
+ # If libarchive is unable to see a difference between the tarballs
+ # we fall back to the Python tarfile module and try to print more
+ # information.
return [
Difference.from_text_readers(
- list_libarchive(self.path),
- list_libarchive(other.path),
+ list_pytarfile(self.path),
+ list_pytarfile(other.path),
self.path,
other.path,
source="file list",
- )
+ ),
]
=====================================
tests/comparators/test_tar.py
=====================================
@@ -18,10 +18,16 @@
# along with diffoscope. If not, see <https://www.gnu.org/licenses/>.
import pytest
+import gzip
+import os.path
+from datetime import datetime
+import struct
from diffoscope.config import Config
from diffoscope.comparators.tar import TarFile
from diffoscope.comparators.missing_file import MissingFile
+from diffoscope.comparators.utils.specialize import specialize
+from diffoscope.comparators.binary import FilesystemFile
from ..utils.data import load_fixture, get_data
from ..utils.nonexisting import assert_non_existing
@@ -79,3 +85,196 @@ def test_no_permissions_dir_in_tarball(monkeypatch, no_permissions_tar):
# Comparing with non-existing file makes it easy to make sure all files are unpacked
monkeypatch.setattr(Config(), "new_file", True)
no_permissions_tar.compare(MissingFile("/nonexistent", no_permissions_tar))
+
+
+# The result of
+# $ tar --format=pax --no-recursion -C /var/log -cf journal.tar ./journal
+# once with and once without the --xattrs option
+tar_acl_bytes = bytes.fromhex(
+ "1f8b080085ad436a02ffedd44f4bc3301806f02813a5273f42bf8069fef6ad87"
+ "9dbc4cf02078f224a1ad30e9ac3419d44faf2fdb4419a2286e179f1f94276d42"
+ "d2439fcae23a8cb33634ed108b877e393c864efc31c54ae756c9b6535947423b"
+ "6b4a22e734afd39ab411f928f660195318f855c4ff64551ed27cd14e35558648"
+ "3932d2581e91b72ee3d97a3d5b92f59a942269cfb5afac336556f9fce6627679"
+ "752bc790d220e3734ced423ef5713ede85bae3ab6e639c1ef23907e258bcb089"
+ "385ae509e7849f9f6eeef34d6655f9cda64d7b1f965dfad9ae023e27df4a5fec"
+ "ee0caeb821ef3ff65e13775ebddbeabf2d9d16b9df67ff87be4f5fad5b7f87cd"
+ "d98efe910000000000000000000000000000000000bff10adb0252ab00280000"
+)
+
+
+tar_noacl_bytes = bytes.fromhex(
+ "1f8b0800b4ad436a02ffedd43d6a0331108661d53ec55ec0bbfa19699cc2bdcb"
+ "5c41c45bc4c459d85d83737b0f0e263f458a101b82dfa719a11990407c6abbc7"
+ "7adcf475db8f53b71b0ee36b7d717fcc9b2272aee67bf55ec50549b1a88a049b"
+ "0b41bdbae6e86ee030cd75b4abb8fb947c53e7e77dbf0eba8aaa5e34b631d94a"
+ "739285759fdebb4553b657f1daa687905749625938fc7bed25f4ddf5ceb08447"
+ "cdf973ee838a7ee4dff6bee63f1509aec9b7ccff380cf34f73d3db34f7fbedf2"
+ "4a7f24000000000000000000000000f01b278df8083500280000"
+)
+
+# We run tar with --clamp-mtime --mtime=0 but the ctime and atime
+# will still be stored in pax headers. Re-create like this:
+# $ tar --clamp-mtime --mtime=0 --format=pax --no-recursion -cf bin1.tar ./bin
+# $ touch ./bin
+# $ tar --clamp-mtime --mtime=0 --format=pax --no-recursion -cf bin2.tar ./bin
+
+tar_before_bytes = bytes.fromhex(
+ "1f8b0800000000000003edd3cd0a8240148661d75d8557a067fe6dd1be65b760"
+ "256450411a74f94d861006b508b3c5fb6c0ecc37301f0c27cb57e5755995dbea"
+ "dce4ebfa988c40226f6d37a3e1140936514e6b71de191bcf95846092f43a4699"
+ "a14bd396e758e5176ffd212369d9d6876aa142a10be38cf299b7f107bcf6c52c"
+ "a69b97d4e898cadccea6ee8eef65f7a5cfc77da35bf1e0ee530527cfb337d87f"
+ "25da25a91bb7d643bffffb53b3d9bdb9f7290700000000000000000000000026"
+ "7003acf6139900280000"
+)
+
+tar_after_bytes = bytes.fromhex(
+ "1f8b0800000000000003edd3cd0a8240148661d75d8557a0f3e39971d3be65b7"
+ "60266450811678f98d861042b508b3c5fb6c3e983370cee64bd26dd16daa625f"
+ "356dbaabcfd10c54e0b26cc8609a4af92cd2628c1227360bef5a796fa3b89be3"
+ "98a95b7b2d9a70ca2f76fd21abe2e25a9faab5f6b9c9ad5897275a7b63c439b7"
+ "0ad3f2f574e9dbf1bda42f7d3aef8ea1e25efad45ed4738e26fdd7ca4814cbbc"
+ "673d8cfd3f5edaf2f0e6dfa739000000000000000000000000b0803be9796d3f"
+ "00280000"
+)
+
+# Generates a PAX format tarball with a single entry which has the GNU/Hurd
+# translator given in the content argument attached to it in a PAX header
+def generate_pax(name, mode, type, linktarget=None, major=None, minor=None, content=None):
+ entries = []
+ if content is not None:
+ content = "SCHILY.xattr.gnu.translator=" + content
+ # In the beginning the decimal length of the field is recorded.
+ # But the length includes the length of the number itself.
+ # Luckily we only need to support decimal numbers with two digits.
+ assert 9 < len(content) + 4 < 100
+ dirname, basename = os.path.split(name)
+ paxname = dirname + "/PaxHeaders/" + basename
+ content = f"{len(content) +4} {content}\n"
+ entries.append((paxname, 0, "x", None, None, None, content))
+ entries.append((name, mode, type, linktarget, major, minor, None))
+
+ mtime = int(1612543740) # an arbitrary reproducible date
+ devtar = b""
+ for file in entries:
+ fname, mode, type, linktarget, major, minor, content = file
+ assert len(fname) <= 100
+ buf = struct.pack(
+ "@100s8s8s8s12s12s8s1s100s6s2s32s32s8s8s155s12s",
+ fname.encode(),
+ f"{mode:07o}".encode(),
+ b"0" * 7,
+ b"0" * 7,
+ f"{len(content) if content is not None else 0:011o}".encode(),
+ f"{0 if type == 'x' else mtime:011o}".encode(),
+ b" " * 8,
+ type.encode(),
+ linktarget.encode() if linktarget is not None else b"",
+ b"ustar",
+ b"00",
+ b"",
+ b"",
+ (f"{major:07o}" if major is not None else "").encode(),
+ (f"{minor:07o}" if minor is not None else "").encode(),
+ b"",
+ b"",
+ )
+ chksum = 256 + sum(struct.unpack_from("148B8x356B", buf))
+ buf = buf[:-364] + f"{chksum:06o}\0".encode() + buf[-357:]
+ devtar += buf
+ if content is not None:
+ devtar += struct.pack("@512s", content.encode())
+ return struct.pack("@10240s", devtar)
+
+
+def tar_fixture(content, prefix):
+ @pytest.fixture
+ def pax_tar(tmp_path, tar1, tar2):
+ match prefix:
+ case "test1":
+ tar = tar1
+ case "test2":
+ tar = tar2
+ case _:
+ raise Exception(f"Invalid prefix: {prefix}")
+ tar_path = tmp_path / f"{prefix}.tar"
+ with open(tar_path, "wb") as t_fp:
+ t_fp.write(content)
+ with open(tar.path, "rb") as r_fp:
+ t_fp.write(bytes(r_fp.read()))
+ return specialize(FilesystemFile(str(tar_path)))
+
+ return pax_tar
+
+
+tar_acl = tar_fixture(gzip.decompress(tar_acl_bytes), "test1")
+tar_noacl = tar_fixture(gzip.decompress(tar_noacl_bytes), "test2")
+
+tar_before = tar_fixture(gzip.decompress(tar_before_bytes), "test1")
+tar_after = tar_fixture(gzip.decompress(tar_after_bytes), "test2")
+
+
+# The translator of tar_fd1_translator is wrong -- don't do this at home, kids!
+tar_fd0_translator = tar_fixture(generate_pax(name="./dev/fd0", mode=0o640, type="0", content="/hurd/storeio\0fd0\0"), "test1")
+tar_fd1_translator = tar_fixture(generate_pax(name="./dev/fd0", mode=0o640, type="0", content="/hurd/storeio\0fd1\0"), "test2")
+
+
+def test_identification(tar_acl, tar_noacl, tar_before, tar_after, tar_fd0_translator, tar_fd1_translator):
+ assert isinstance(tar_acl, TarFile)
+ assert isinstance(tar_noacl, TarFile)
+ assert isinstance(tar_before, TarFile)
+ assert isinstance(tar_after, TarFile)
+ assert isinstance(tar_fd0_translator, TarFile)
+ assert isinstance(tar_fd1_translator, TarFile)
+
+
+ at pytest.fixture
+def differences_acl(tar_acl, tar_noacl):
+ return tar_acl.compare(tar_noacl).details
+
+
+def test_listing_acl(differences_acl):
+ assert (
+ differences_acl[0].unified_diff
+ == """@@ -1,8 +1,2 @@
+-./journal PAX header SCHILY.xattr.system.posix_acl_default:
+- ACL_USER_OBJ rwx 2^32-1
+- ACL_GROUP_OBJ rx 2^32-1
+- ACL_GROUP rx 4
+- ACL_MASK rx 2^32-1
+- ACL_OTHER rx 2^32-1
+ ./journal PAX header atime: 2026-06-29 22:01:12.232778+00:00
+ ./journal PAX header ctime: 2023-01-12 09:50:07.391583+00:00
+"""
+ )
+
+
+ at pytest.fixture
+def differences_atime(tar_before, tar_after):
+ return tar_before.compare(tar_after).details
+
+def test_listing_atime(differences_atime):
+ assert (
+ differences_atime[0].unified_diff
+ == """@@ -1,2 +1,2 @@
+-./bin PAX header atime: 2026-06-30 16:01:56.640106+00:00
+-./bin PAX header ctime: 2026-06-30 16:01:56.632106+00:00
++./bin PAX header atime: 2026-06-30 16:02:48.117226+00:00
++./bin PAX header ctime: 2026-06-30 16:02:48.117226+00:00
+"""
+ )
+
+
+ at pytest.fixture
+def differences_translator(tar_fd0_translator, tar_fd1_translator):
+ return tar_fd0_translator.compare(tar_fd1_translator).details
+
+def test_listing_translator(differences_translator):
+ assert (
+ differences_translator[0].unified_diff
+ == """@@ -1 +1 @@
+-./dev/fd0 PAX header SCHILY.xattr.gnu.translator raw value: /hurd/storeio\0fd0\0
++./dev/fd0 PAX header SCHILY.xattr.gnu.translator raw value: /hurd/storeio\0fd1\0
+"""
+ )
View it on GitLab: https://salsa.debian.org/reproducible-builds/diffoscope/-/commit/7c7a4cae90b7187b77d0bbe7d8f2234d8218e6d7
--
View it on GitLab: https://salsa.debian.org/reproducible-builds/diffoscope/-/commit/7c7a4cae90b7187b77d0bbe7d8f2234d8218e6d7
You're receiving this email because of your account on salsa.debian.org. Manage all notifications: https://salsa.debian.org/-/profile/notifications | Help: https://salsa.debian.org/help
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.reproducible-builds.org/pipermail/rb-commits/attachments/20260630/b2b11174/attachment.htm>
More information about the rb-commits
mailing list