[Git][reproducible-builds/diffoscope][master] 5 commits: Drop unused BASE_DIR global in the tests.
Chris Lamb
gitlab at salsa.debian.org
Fri Nov 8 18:51:00 UTC 2019
Chris Lamb pushed to branch master at Reproducible Builds / diffoscope
Commits:
02497c9c by Chris Lamb at 2019-11-08T18:48:08Z
Drop unused BASE_DIR global in the tests.
- - - - -
1c4b1278 by Chris Lamb at 2019-11-08T18:48:10Z
Try and ensure that new test data files are generated dynamically, ie. at least no new ones are added without "good" reasons.
- - - - -
9c66d5f6 by Chris Lamb at 2019-11-08T18:48:12Z
Truncate the tcpdump expected diff to 8KB (from ~600KB).
- - - - -
1b36f8eb by Jelle van der Waa at 2019-11-08T18:48:14Z
Add support for comparing .zst files are created by zstd. (Closes: reproducible-builds/diffoscope!34)
Signed-off-by: Chris Lamb <lamby at debian.org>
- - - - -
47542775 by Chris Lamb at 2019-11-08T18:50:45Z
Refresh OCaml test fixtures to support OCaml >= 4.08.1.
- - - - -
10 changed files:
- debian/control
- diffoscope/comparators/__init__.py
- + diffoscope/comparators/zst.py
- diffoscope/external_tools.py
- tests/comparators/test_ocaml.py
- tests/comparators/test_pcap.py
- + tests/comparators/test_zst.py
- tests/data/ocaml_expected_diff
- tests/data/pcap_expected_diff
- tests/test_source.py
Changes:
=====================================
debian/control
=====================================
@@ -83,6 +83,7 @@ Build-Depends:
wabt <!nocheck>,
xmlbeans <!nocheck>,
xxd <!nocheck> | vim-common <!nocheck>,
+ zstd <!nocheck>,
Build-Conflicts:
graphicsmagick-imagemagick-compat,
Standards-Version: 4.4.1
=====================================
diffoscope/comparators/__init__.py
=====================================
@@ -105,6 +105,7 @@ class ComparatorManager:
('ogg.OggFile',),
('xsb.XsbFile',),
('berkeley_db.BerkeleyDBFile',),
+ ('zst.ZstFile',),
)
_singleton = {}
=====================================
diffoscope/comparators/zst.py
=====================================
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2019 Jelle van der Waa <jelle at archlinux.org>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope. If not, see <https://www.gnu.org/licenses/>.
+
+import re
+import os.path
+import logging
+import subprocess
+
+from diffoscope.tools import tool_required
+
+from .utils.file import File
+from .utils.archive import Archive
+
+logger = logging.getLogger(__name__)
+
+
+class ZstContainer(Archive):
+ def open_archive(self):
+ return self
+
+ def close_archive(self):
+ pass
+
+ def get_member_names(self):
+ return [self.get_compressed_content_name('.zst')]
+
+ @tool_required('zstd')
+ def extract(self, member_name, dest_dir):
+ dest_path = os.path.join(dest_dir, member_name)
+ logger.debug('zstd extracting to %s', dest_path)
+ with open(dest_path, 'wb') as fp:
+ subprocess.check_call(
+ ["zstd", "-d", "-c", self.source.path],
+ shell=False,
+ stdout=fp,
+ stderr=None,
+ )
+ return dest_path
+
+
+class ZstFile(File):
+ DESCRIPTION = "Zstandard compressed files"
+ CONTAINER_CLASS = ZstContainer
+ FILE_TYPE_RE = re.compile(r'^Zstandard compressed data')
=====================================
diffoscope/external_tools.py
=====================================
@@ -212,6 +212,7 @@ EXTERNAL_TOOLS = {
'zipnote': {'debian': 'zip', 'guix': 'zip'},
'procyon': {'debian': 'procyon-decompiler'},
'dumpxsb': {'debian': 'xmlbeans'},
+ 'zstd': {'debian': 'zstd'},
}
# May be populated at runtime by remapped names like
=====================================
tests/comparators/test_ocaml.py
=====================================
@@ -57,7 +57,7 @@ def ocaml_version():
return out.decode('utf-8').split()[-1]
- at skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.05.0')
+ at skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.08.1')
def test_identification(cmi1):
assert isinstance(cmi1, OcamlInterfaceFile)
@@ -67,18 +67,18 @@ def differences(cmi1, cmi2):
return cmi1.compare(cmi2).details
- at skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.05.0')
+ at skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.08.1')
def test_no_differences(cmi1):
difference = cmi1.compare(cmi1)
assert difference is None
- at skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.05.0')
+ at skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.08.1')
def test_diff(differences):
expected_diff = get_data('ocaml_expected_diff')
assert differences[0].unified_diff == expected_diff
- at skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.05.0')
+ at skip_unless_tool_is_at_least('ocamlobjinfo', ocaml_version, '4.08.1')
def test_compare_non_existing(monkeypatch, cmi1):
assert_non_existing(monkeypatch, cmi1, has_null_source=False)
=====================================
tests/comparators/test_pcap.py
=====================================
@@ -46,7 +46,7 @@ def differences(pcap1, pcap2):
@skip_unless_tools_exist('tcpdump')
def test_diff(differences):
expected_diff = get_data('pcap_expected_diff')
- assert differences[0].unified_diff == expected_diff
+ assert differences[0].unified_diff[: 2 ** 13] == expected_diff[: 2 ** 13]
@skip_unless_tools_exist('tcpdump')
=====================================
tests/comparators/test_zst.py
=====================================
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2019 Jelle van der Waa <jelle at archlinux.org>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope. If not, see <https://www.gnu.org/licenses/>.
+
+import subprocess
+
+import pytest
+
+from diffoscope.comparators.zst import ZstFile
+from diffoscope.comparators.binary import FilesystemFile
+from diffoscope.comparators.utils.specialize import specialize
+
+from ..utils.tools import skip_unless_tools_exist
+
+
+def zst_fixture(prefix):
+ @pytest.fixture
+ def zstd(tmpdir):
+ input_ = str(tmpdir.join('{}'.format(prefix)))
+ output = str(tmpdir.join('{}.zst'.format(prefix)))
+
+ with open(input_, 'w') as f:
+ f.write(prefix)
+
+ subprocess.check_call(('zstd', input_))
+
+ return specialize(FilesystemFile(output))
+
+ return zstd
+
+
+zst1 = zst_fixture('test1')
+zst2 = zst_fixture('test2')
+
+
+ at skip_unless_tools_exist('zstd')
+def test_identification(zst1):
+ assert isinstance(zst1, ZstFile)
+
+
+ at skip_unless_tools_exist('zstd')
+def test_no_differences(zst1):
+ difference = zst1.compare(zst1)
+ assert difference is None
+
+
+ at pytest.fixture
+def differences(zst1, zst2):
+ return zst1.compare(zst2).details
+
+
+ at skip_unless_tools_exist('zstd')
+def test_content_source(differences):
+ assert differences[0].source1 == 'test1'
+ assert differences[0].source2 == 'test2'
=====================================
tests/data/ocaml_expected_diff
=====================================
@@ -2,7 +2,7 @@
-Unit name: Test1
+Unit name: Test2
Interfaces imported:
-- b5f47409a5d1c32a864ea3d2fbed44db Test1
-+ 9ee19396ca59dbe57206a9a88d98a7d5 Test2
- 07ea9e20ae94d62c35cfecbe7d66d3ea Pervasives
- cbd5f2d6b649925222e1e9fb63b89db6 CamlinternalFormatBasics
+- 4bf3070814d7fb8e8d365d95481f8cad Test1
++ 333f54d1aae1264e7ad64cbb437cbc4f Test2
+ ad45f251bbf98d3a0bf3b883546ecfc8 Stdlib
+ a2b1a9d869fd05813beb35645bd9cd94 CamlinternalFormatBasics
=====================================
tests/data/pcap_expected_diff
=====================================
The diff for this file was not included because it is too large.
=====================================
tests/test_source.py
=====================================
@@ -18,12 +18,224 @@
# along with diffoscope. If not, see <https://www.gnu.org/licenses/>.
import os
+import glob
import diffoscope
import subprocess
from .utils.tools import skip_unless_tools_exist
-BASE_DIR = os.path.dirname(os.path.abspath(diffoscope.__file__))
+ALLOWED_TEST_FILES = {
+ # Data files we would prefer to generate dynamically
+ 'android1.img',
+ 'android2.img',
+ 'archive1.tar',
+ 'archive2.tar',
+ 'base-files_157-r45695_ar71xx.ipk',
+ 'base-files_157-r45918_ar71xx.ipk',
+ 'binary1',
+ 'binary2',
+ 'bug881937_1.deb',
+ 'bug881937_2.deb',
+ 'bug903391_1.deb',
+ 'bug903391_2.deb',
+ 'bug903401_1.deb',
+ 'bug903401_2.deb',
+ 'bug903565_1.deb',
+ 'bug903565_2.deb',
+ 'containers',
+ 'containers/a.tar.bz2',
+ 'containers/a.tar.gz',
+ 'containers/a.tar.xz',
+ 'containers/b.tar.bz2',
+ 'containers/b.tar.gz',
+ 'containers/b.tar.xz',
+ 'containers/magic_bzip2',
+ 'containers/magic_gzip',
+ 'containers/magic_xz',
+ 'dbgsym/add/test-dbgsym-dbgsym_1_amd64.deb',
+ 'dbgsym/add/test-dbgsym_1_amd64.deb',
+ 'dbgsym/mult/test-dbgsym-dbgsym_1_amd64.deb',
+ 'dbgsym/mult/test-dbgsym_1_amd64.deb',
+ 'dbgsym/test-dbgsym_1.dsc',
+ 'dbgsym/test-dbgsym_1.tar.gz',
+ 'debian-bug-876316-control.tar.gz',
+ # Outputs
+ 'devicetree1.dtb',
+ 'devicetree2.dtb',
+ 'elfmix1.not_a',
+ 'elfmix2.a',
+ 'encrypted1.zip',
+ 'encrypted2.zip',
+ 'fuzzy-tar-in-tar1.tar',
+ 'fuzzy-tar-in-tar2.tar',
+ 'fuzzy1.tar',
+ 'fuzzy2.tar',
+ 'fuzzy3.tar',
+ 'hello1.wasm',
+ 'hello2.wasm',
+ 'no-perms.tar',
+ 'quine.gz',
+ 'quine.zip',
+ 'quine_a.deb',
+ 'quine_b.deb',
+ 'Samyak-Malayalam1.ttf',
+ 'Samyak-Malayalam2.ttf',
+ 'test1-le64.cache-4',
+ 'test1.a',
+ 'test1.apk',
+ 'test1.asc',
+ 'test1.binwalk',
+ 'test1.buildinfo',
+ 'test1.bz2',
+ 'test1.changes',
+ 'Test1.class',
+ 'Test2.class',
+ 'test1.cpio',
+ 'test1.db',
+ 'test1.deb',
+ 'test1.debsrc.tar.gz',
+ 'test1.debug',
+ 'test1.dex',
+ 'test1.docx',
+ 'test1.dsc',
+ 'test1.epub',
+ 'test1.exe',
+ 'test1.ext4',
+ 'test1.fat12',
+ 'test1.fat16',
+ 'test1.fat32',
+ 'test1.gif',
+ 'test1.git-index',
+ 'test1.gnumeric',
+ 'test1.gz',
+ 'test1.hi',
+ 'test1.icc',
+ 'test1.ico',
+ 'test1.iso',
+ 'test1.jmod',
+ 'test1.jpg',
+ 'test1.js',
+ 'test1.json',
+ 'test1.kbx',
+ 'test1.lz4',
+ 'test1.macho',
+ 'test1.mo',
+ 'test1.mozzip',
+ 'test1.mp3',
+ 'test1.o',
+ 'test1.odt',
+ 'test1.ogg',
+ 'test1.pcap',
+ 'test1.pdf',
+ 'test1.pgp',
+ 'test1.png',
+ 'test1.ppu',
+ 'test1.ps',
+ 'test1.rdx',
+ 'test1.rlib',
+ 'test1.rpm',
+ 'test1.sqlite3',
+ 'test1.squashfs',
+ 'test1.tar',
+ 'test1.xml',
+ 'test1.xsb',
+ 'test1.xz',
+ 'test1.zip',
+ 'test1_meta.ico',
+ 'test1_meta.jpg',
+ 'test2-le64.cache-4',
+ 'test2.a',
+ 'test2.apk',
+ 'test2.asc',
+ 'test2.binwalk',
+ 'test2.buildinfo',
+ 'test2.bz2',
+ 'test2.changes',
+ 'test2.cpio',
+ 'test2.db',
+ 'test2.deb',
+ 'test2.debsrc.tar.gz',
+ 'test2.debug',
+ 'test2.dex',
+ 'test2.docx',
+ 'test2.dsc',
+ 'test2.epub',
+ 'test2.exe',
+ 'test2.ext4',
+ 'test2.fat12',
+ 'test2.gif',
+ 'test2.git-index',
+ 'test2.gnumeric',
+ 'test2.gz',
+ 'test2.hi',
+ 'test2.icc',
+ 'test2.ico',
+ 'test2.iso',
+ 'test2.jmod',
+ 'test2.jpg',
+ 'test2.js',
+ 'test2.json',
+ 'test2.kbx',
+ 'test2.lz4',
+ 'test2.macho',
+ 'test2.mo',
+ 'test2.mozzip',
+ 'test2.mp3',
+ 'test2.o',
+ 'test2.odt',
+ 'test2.ogg',
+ 'test2.pcap',
+ 'test2.pdf',
+ 'test2.pgp',
+ 'test2.png',
+ 'test2.ppu',
+ 'test2.ps',
+ 'test2.rdx',
+ 'test2.rlib',
+ 'test2.rpm',
+ 'test2.sqlite3',
+ 'test2.squashfs',
+ 'test2.tar',
+ 'test2.xml',
+ 'test2.xsb',
+ 'test2.xz',
+ 'test2.zip',
+ 'test2_meta.ico',
+ 'test2_meta.jpg',
+ 'test3.apk',
+ 'test3.changes',
+ 'test3.gif',
+ 'test3.zip',
+ 'test4.changes',
+ 'test4.gif',
+ 'test_comment1.zip',
+ 'test_comment2.zip',
+ 'test_invalid.json',
+ 'test_invalid.xml',
+ 'test_iso8859-1.mo',
+ 'test_no_charset.mo',
+ 'test_openssh_pub_key1.pub',
+ 'test_openssh_pub_key2.pub',
+ 'test_weird_non_unicode_chars1.pdf',
+ 'test_weird_non_unicode_chars2.pdf',
+ 'text_ascii1',
+ 'text_ascii2',
+ 'text_iso8859',
+ 'text_order1',
+ 'text_order2',
+ 'text_unicode1',
+ 'text_unicode2',
+ 'text_unicode_binary_fallback',
+ # Outputs
+ 'debian-bug-875281.collapsed-diff.json',
+ 'order1a.json',
+ 'order1b.json',
+ 'output.colored.txt',
+ 'output.json',
+ 'output.md',
+ 'output.rst',
+ 'output.txt',
+}
@skip_unless_tools_exist('black')
@@ -36,3 +248,40 @@ def test_code_is_black_clean():
print(output)
assert len(output) == 0
+
+
+def test_does_not_add_new_test_files():
+ """
+ For a variety of reasons we are now prefering to generate any test data
+ dynamically (via pytest fixtures, etc.) rather than committing and shipping
+ such files.
+
+ Exceptions to this may be appropriate (or even required) but this test
+ ensures that test files that could be dynamically generated are not added
+ "automatically", for example by following previous/older commits.
+ """
+
+ test_dir = os.path.join(os.path.dirname(__file__), 'data')
+
+ seen = set()
+
+ for x in glob.iglob(os.path.join(test_dir, '**'), recursive=True):
+ if os.path.isdir(x):
+ continue
+
+ # Strip off common prefix
+ x = x[len(test_dir) + 1 :]
+
+ # Skip some known expected diff filename patterns
+ if (
+ x.endswith('_diff')
+ or x.endswith('_diffs')
+ or x.endswith('.diff')
+ or '_diff_' in x
+ or 'diff.' in x
+ ):
+ continue
+
+ seen.add(x)
+
+ assert seen - ALLOWED_TEST_FILES - {''} == set()
View it on GitLab: https://salsa.debian.org/reproducible-builds/diffoscope/compare/2c9fbc138a11dc6d5bb88f642c0ef245a1949d5a...475427752d104fd9f498ca5578f834528119acda
--
View it on GitLab: https://salsa.debian.org/reproducible-builds/diffoscope/compare/2c9fbc138a11dc6d5bb88f642c0ef245a1949d5a...475427752d104fd9f498ca5578f834528119acda
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.reproducible-builds.org/pipermail/rb-commits/attachments/20191108/58c8c56f/attachment.htm>
More information about the rb-commits
mailing list