[Git][reproducible-builds/diffoscope][master] 3 commits: Correctly use fallback_recognises to prevent matching .XSB binary XML files....

Chris Lamb gitlab at salsa.debian.org
Fri Aug 21 11:08:22 UTC 2020



Chris Lamb pushed to branch master at Reproducible Builds / diffoscope


Commits:
3325f4df by Chris Lamb at 2020-08-21T12:07:36+01:00
Correctly use fallback_recognises to prevent matching .XSB binary XML files. (Re: reproducible-builds/diffoscope#211)

Didn't notice that the pipelines were failing - I was not getting emails.

- - - - -
aa2e2201 by Chris Lamb at 2020-08-21T12:07:52+01:00
Support multiple options for file extension matching.

- - - - -
4b3c8c50 by Chris Lamb at 2020-08-21T12:07:52+01:00
Support extra file extensions when trying to find PGP files. (Re: reproducible-builds/diffoscope#211)

- - - - -


20 changed files:

- diffoscope/comparators/apk.py
- diffoscope/comparators/debian.py
- diffoscope/comparators/debian_fallback.py
- diffoscope/comparators/dex.py
- diffoscope/comparators/fontconfig.py
- diffoscope/comparators/gnumeric.py
- diffoscope/comparators/gzip.py
- diffoscope/comparators/ipk.py
- diffoscope/comparators/javascript.py
- diffoscope/comparators/lz4.py
- diffoscope/comparators/openssl.py
- diffoscope/comparators/pgp.py
- diffoscope/comparators/ppu.py
- diffoscope/comparators/rdata.py
- diffoscope/comparators/rust.py
- diffoscope/comparators/utils/file.py
- diffoscope/comparators/wasm.py
- diffoscope/comparators/xml.py
- diffoscope/comparators/xsb.py
- diffoscope/comparators/xz.py


Changes:

=====================================
diffoscope/comparators/apk.py
=====================================
@@ -212,7 +212,7 @@ class ApkFile(File):
     DESCRIPTION = "Android APK files"
     FILE_TYPE_HEADER_PREFIX = b"PK\x03\x04"
     FILE_TYPE_RE = re.compile(r"^((Java|Zip) archive data|Dalvik dex file)\b")
-    FILE_EXTENSION_SUFFIX = ".apk"
+    FILE_EXTENSION_SUFFIX = {".apk"}
     CONTAINER_CLASSES = [ApkContainer, ZipContainer]
 
     def compare_details(self, other, source=None):


=====================================
diffoscope/comparators/debian.py
=====================================
@@ -216,7 +216,7 @@ class DebControlFile(File):
 
 class DotChangesFile(DebControlFile):
     DESCRIPTION = "Debian .changes files"
-    FILE_EXTENSION_SUFFIX = ".changes"
+    FILE_EXTENSION_SUFFIX = {".changes"}
     FILE_TYPE_RE = re.compile(r"^(ASCII text|UTF-8 Unicode text)")
 
     @classmethod
@@ -260,7 +260,7 @@ class DotChangesFile(DebControlFile):
 
 class DotDscFile(DebControlFile):
     DESCRIPTION = "Debian source packages (.dsc)"
-    FILE_EXTENSION_SUFFIX = ".dsc"
+    FILE_EXTENSION_SUFFIX = {".dsc"}
 
     @classmethod
     def recognizes(cls, file):
@@ -295,7 +295,7 @@ class DotBuildinfoContainer(DebControlContainer):
 class DotBuildinfoFile(DebControlFile):
     DESCRIPTION = "Debian .buildinfo files"
     CONTAINER_CLASSES = [DotBuildinfoContainer]
-    FILE_EXTENSION_SUFFIX = ".buildinfo"
+    FILE_EXTENSION_SUFFIX = {".buildinfo"}
     FILE_TYPE_RE = re.compile(r"^(ASCII text|UTF-8 Unicode text)")
 
     @classmethod


=====================================
diffoscope/comparators/debian_fallback.py
=====================================
@@ -34,14 +34,14 @@ class AbstractDebianFallbackFile(TextFile):
 
 class DotChangesFile(AbstractDebianFallbackFile):
     DESCRIPTION = "Debian .changes files"
-    FILE_EXTENSION_SUFFIX = ".changes"
+    FILE_EXTENSION_SUFFIX = {".changes"}
 
 
 class DotDscFile(AbstractDebianFallbackFile):
     DESCRIPTION = "Debian source packages (.dsc)"
-    FILE_EXTENSION_SUFFIX = ".dsc"
+    FILE_EXTENSION_SUFFIX = {".dsc"}
 
 
 class DotBuildinfoFile(AbstractDebianFallbackFile):
     DESCRIPTION = "Debian .buildinfo files"
-    FILE_EXTENSION_SUFFIX = ".buildinfo"
+    FILE_EXTENSION_SUFFIX = {".buildinfo"}


=====================================
diffoscope/comparators/dex.py
=====================================
@@ -60,5 +60,5 @@ class DexContainer(Archive):
 class DexFile(File):
     DESCRIPTION = "Dalvik .dex files"
     FILE_TYPE_RE = re.compile(r"^Dalvik dex file .*\b")
-    FILE_EXTENSION_SUFFIX = ".dex"
+    FILE_EXTENSION_SUFFIX = {".dex"}
     CONTAINER_CLASSES = [DexContainer]


=====================================
diffoscope/comparators/fontconfig.py
=====================================
@@ -27,7 +27,7 @@ from .utils.file import File
 class FontconfigCacheFile(File):
     DESCRIPTION = "FreeDesktop Fontconfig cache files"
     FILE_TYPE_HEADER_PREFIX = struct.pack("<H", 0xFC04)
-    FILE_EXTENSION_SUFFIX = "-le64.cache-4"
+    FILE_EXTENSION_SUFFIX = {"-le64.cache-4"}
 
     def compare_details(self, other, source=None):
         return [


=====================================
diffoscope/comparators/gnumeric.py
=====================================
@@ -30,7 +30,7 @@ from .missing_file import MissingFile
 
 class GnumericFile(File):
     DESCRIPTION = "Gnumeric spreadsheets"
-    FILE_EXTENSION_SUFFIX = ".gnumeric"
+    FILE_EXTENSION_SUFFIX = {".gnumeric"}
 
     @tool_required("ssconvert")
     def compare_details(self, other, source=None):


=====================================
diffoscope/comparators/gzip.py
=====================================
@@ -60,5 +60,5 @@ class GzipFile(File):
     FILE_TYPE_RE = re.compile(r"^gzip compressed data\b")
 
     # Work around file(1) Debian bug #876316
-    FALLBACK_FILE_EXTENSION_SUFFIX = ".gz"
+    FALLBACK_FILE_EXTENSION_SUFFIX = {".gz"}
     FALLBACK_FILE_TYPE_HEADER_PREFIX = b"\x1f\x8b"


=====================================
diffoscope/comparators/ipk.py
=====================================
@@ -24,4 +24,4 @@ from .gzip import GzipFile
 
 class IpkFile(GzipFile):
     DESCRIPTION = "OpenWRT package archives (.ipk)"
-    FILE_EXTENSION_SUFFIX = ".ipk"
+    FILE_EXTENSION_SUFFIX = {".ipk"}


=====================================
diffoscope/comparators/javascript.py
=====================================
@@ -33,7 +33,7 @@ class JavaScriptBeautify(Command):
 
 class JavaScriptFile(File):
     DESCRIPTION = "JavaScript files"
-    FILE_EXTENSION_SUFFIX = ".js"
+    FILE_EXTENSION_SUFFIX = {".js"}
 
     def compare_details(self, other, source=None):
         return [


=====================================
diffoscope/comparators/lz4.py
=====================================
@@ -58,5 +58,5 @@ class Lz4File(File):
     FILE_TYPE_RE = re.compile(r"^LZ4 compressed data \([^\)]+\)$")
 
     # Work around file(1) Debian bug #876316
-    FALLBACK_FILE_EXTENSION_SUFFIX = ".lz4"
+    FALLBACK_FILE_EXTENSION_SUFFIX = {".lz4"}
     FALLBACK_FILE_TYPE_HEADER_PREFIX = b"\x04\x22M\x18"


=====================================
diffoscope/comparators/openssl.py
=====================================
@@ -64,7 +64,7 @@ class Pkcs7File(File):
 
 class MobileProvisionFile(File):
     DESCRIPTION = "Apple Xcode mobile provisioning files"
-    FILE_EXTENSION_SUFFIX = ".mobileprovision"
+    FILE_EXTENSION_SUFFIX = {".mobileprovision"}
 
     def compare_details(self, other, source=None):
         return [


=====================================
diffoscope/comparators/pgp.py
=====================================
@@ -48,10 +48,10 @@ class Pgpdump(Command):
 class PgpFile(File):
     DESCRIPTION = "PGP signed/encrypted messages"
     FILE_TYPE_RE = re.compile(r"^PGP message\b")
-    FALLBACK_FILE_EXTENSION_SUFFIX = ".pgp"
+    FALLBACK_FILE_EXTENSION_SUFFIX = {".pgp", ".asc", ".pub", ".sec", ".gpg"}
 
     @classmethod
-    def recognizes(cls, file):
+    def fallback_recognizes(cls, file):
         if file.magic_file_type == "data":
             try:
                 our_check_output(
@@ -65,7 +65,7 @@ class PgpFile(File):
 
             logger.debug("%s is not a PGP file", file.path)
 
-        return super().recognizes(file)
+        return False
 
     def compare_details(self, other, source=None):
         return [


=====================================
diffoscope/comparators/ppu.py
=====================================
@@ -63,7 +63,7 @@ class Ppudump(Command):
 
 class PpuFile(File):
     DESCRIPTION = "FreePascal files (.ppu)"
-    FILE_EXTENSION_SUFFIX = ".ppu"
+    FILE_EXTENSION_SUFFIX = {".ppu"}
 
     @classmethod
     def recognizes(cls, file):


=====================================
diffoscope/comparators/rdata.py
=====================================
@@ -160,7 +160,7 @@ class RdbReader(Command):
 
 class RdbFile(File):
     DESCRIPTION = "GNU R database files (.rdb)"
-    FILE_EXTENSION_SUFFIX = ".rdb"
+    FILE_EXTENSION_SUFFIX = {".rdb"}
 
     def compare_details(self, other, source=None):
         a = get_module_path_for_rdb(self)


=====================================
diffoscope/comparators/rust.py
=====================================
@@ -63,7 +63,7 @@ class RustObjectFile(File):
     DESCRIPTION = "Rust object files (.deflate)"
     CONTAINER_CLASSES = [RustObjectContainer]
     FILE_TYPE_HEADER_PREFIX = b"RUST_OBJECT\x01\x00\x00\x00"
-    FILE_EXTENSION_SUFFIX = ".deflate"
+    FILE_EXTENSION_SUFFIX = {".deflate"}
 
     def compare_details(self, other, source=None):
         return [


=====================================
diffoscope/comparators/utils/file.py
=====================================
@@ -166,7 +166,7 @@ class File(metaclass=abc.ABCMeta):
         all_tests = [
             test
             for test in (
-                (cls.FILE_EXTENSION_SUFFIX, str.endswith, file.name),
+                (cls.FILE_EXTENSION_SUFFIX, cls.any_endswith, file.name),
                 (file_type_tests, _run_tests, any),
             )
             if test[0]
@@ -199,8 +199,12 @@ class File(metaclass=abc.ABCMeta):
         all_tests = [
             test
             for test in (
-                (cls.FALLBACK_FILE_EXTENSION_SUFFIX, str.endswith, file.name),
-                (cls.FILE_EXTENSION_SUFFIX, str.endswith, file.name),
+                (
+                    cls.FALLBACK_FILE_EXTENSION_SUFFIX,
+                    cls.any_endswith,
+                    file.name,
+                ),
+                (cls.FILE_EXTENSION_SUFFIX, cls.any_endswith, file.name),
                 (
                     cls.FALLBACK_FILE_TYPE_HEADER_PREFIX,
                     bytes.startswith,
@@ -217,6 +221,19 @@ class File(metaclass=abc.ABCMeta):
 
         return _run_tests(all, all_tests) if all_tests else False
 
+    @staticmethod
+    def any_endswith(val, candidates):
+        """
+        Return true iff `val` ends with any string present in `candidates`
+        iterable.
+        """
+
+        for x in candidates:
+            if val.endswith(x):
+                return True
+
+        return False
+
     # This might be different from path and is used to do file extension matching
     @property
     def name(self):


=====================================
diffoscope/comparators/wasm.py
=====================================
@@ -35,7 +35,7 @@ class Wasm2Wat(Command):
 
 class WasmFile(File):
     DESCRIPTION = "WebAssembly binary module"
-    FILE_EXTENSION_SUFFIX = ".wasm"
+    FILE_EXTENSION_SUFFIX = {".wasm"}
 
     @classmethod
     def recognizes(cls, file):


=====================================
diffoscope/comparators/xml.py
=====================================
@@ -84,7 +84,7 @@ class XMLFile(File):
     """
 
     DESCRIPTION = "XML files"
-    FILE_EXTENSION_SUFFIX = ".xml"
+    FILE_EXTENSION_SUFFIX = {".xml"}
 
     @classmethod
     def recognizes(cls, file):


=====================================
diffoscope/comparators/xsb.py
=====================================
@@ -37,7 +37,7 @@ class Dumpxsb(Command):
 
 class XsbFile(File):
     DESCRIPTION = "XML binary schemas (.xsb)"
-    FILE_EXTENSION_SUFFIX = ".xsb"
+    FILE_EXTENSION_SUFFIX = {".xsb"}
 
     def compare_details(self, other, source=None):
         return [


=====================================
diffoscope/comparators/xz.py
=====================================
@@ -60,5 +60,5 @@ class XzFile(File):
     FILE_TYPE_RE = re.compile(r"^XZ compressed data$")
 
     # Work around file(1) Debian bug #876316
-    FALLBACK_FILE_EXTENSION_SUFFIX = ".xz"
+    FALLBACK_FILE_EXTENSION_SUFFIX = {".xz"}
     FALLBACK_FILE_TYPE_HEADER_PREFIX = b"\xfd7zXZ\x00"



View it on GitLab: https://salsa.debian.org/reproducible-builds/diffoscope/-/compare/36d740f61f8ddbce2755ff92071cce1a4f397a15...4b3c8c50f137b9108681fed847394eac740d8f32

-- 
View it on GitLab: https://salsa.debian.org/reproducible-builds/diffoscope/-/compare/36d740f61f8ddbce2755ff92071cce1a4f397a15...4b3c8c50f137b9108681fed847394eac740d8f32
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.reproducible-builds.org/pipermail/rb-commits/attachments/20200821/d39a3513/attachment.htm>


More information about the rb-commits mailing list