[Git][reproducible-builds/diffoscope-website][master] 2 commits: Move to a single data file and a Python script to update it.

Chris Lamb gitlab at salsa.debian.org
Wed May 27 10:35:14 UTC 2020



Chris Lamb pushed to branch master at Reproducible Builds / diffoscope-website


Commits:
92d13a72 by Chris Lamb at 2020-05-27T11:12:12+01:00
Move to a single data file and a Python script to update it.

- - - - -
f0663bbf by Chris Lamb at 2020-05-27T11:12:12+01:00
Generate the file formats dynamically.

- - - - -


6 changed files:

- .gitignore
- Makefile
- _data/contributors.yml → _data/diffoscope.yml
- − _data/releases.yml
- index.md
- + update


Changes:

=====================================
.gitignore
=====================================
@@ -1,2 +1 @@
 /_site/
-/diffoscope.git/


=====================================
Makefile
=====================================
@@ -1,32 +1,8 @@
-GIT = git --git-dir=$(GIT_DIR)/.git
-GIT_DIR = diffoscope.git
-
-DATA = _data/contributors.yml _data/releases.yml
-
 all:
 	jekyll build --verbose --trace
 
-data: $(DATA)
-
-$(GIT_DIR):
-	git clone https://salsa.debian.org/reproducible-builds/diffoscope.git $@
-
-_data/contributors.yml: $(GIT_DIR)
-	mkdir -p "$(dir $@)"
-	echo "# This file is automatically generated" > $@
-	$(GIT) log --format='%aN' | sort -u | \
-		grep -v root | sed -e 's@^@- @' >> $@
-
-_data/releases.yml: $(GIT_DIR)
-	mkdir -p "$(dir $@)"
-	echo "# This file is automatically generated" > $@
-	dpkg-parsechangelog -l $(GIT_DIR)/debian/changelog --format=rfc822 --from=0 | \
-		awk -F': ' ' \
-			$$1 == "Version" { print "\n- version: " $$2 } \
-			$$1 == "Timestamp" { print "  timestamp: " $$2 } \
-			' >> $@
-
 clean:
 	rm -rf _site
 
-.PHONY: $(DATA)
+data:
+	./update


=====================================
_data/contributors.yml → _data/diffoscope.yml
=====================================
@@ -1,4 +1,6 @@
 # This file is automatically generated
+
+contributors:
 - Alexis Murzeau
 - anthraxx
 - Baptiste Daroussin
@@ -22,14 +24,14 @@
 - HW42
 - James Clarke
 - Jelle van der Waa
-- Jérémy Bobbio
 - Joachim Breitner
 - Juliana Oliveira
 - Juliana Oliveira Rodrigues
 - Juliana Rodrigues
+- "J\xE9r\xE9my Bobbio"
 - klemens
 - Marc Herbert
-- Marek Marczykowski-Górecki
+- "Marek Marczykowski-G\xF3recki"
 - Maria Glukhova
 - Marius Gedminas
 - Mattia Rizzolo
@@ -41,7 +43,7 @@
 - Paul Gevers
 - Paul Wise
 - Peter De Wachter
-- Rainer Müller
+- "Rainer M\xFCller"
 - Reiner Herrmann
 - Ricardo Gaviria
 - Santiago Torres-Arias
@@ -53,3 +55,26 @@
 - Xavier Briand
 - Ximin Luo
 - Yasushi SHOJI
+description: 'File formats supported include: Android APK files, Android boot images,
+  AppleXcode mobile provisioning files, Berkeley DB database files, ColorSync colourprofiles
+  (.icc), Coreboot CBFS filesystem images, Dalvik .dex files, Debianbuildinfo files,
+  Debian .changes files, Debian source packages (.dsc), DeviceTree Compiler blob files,
+  ELF binaries, FreeDesktop Fontconfig cache files,FreePascal files (.ppu), GHC Haskell
+  .hi files, GIF image files, GNU R Rscriptfiles (.rds), GNU R database files (.rdb),
+  GPG keybox databases, Gettextmessage catalogues, Git repositories, Gnumeric spreadsheets,
+  Gzipped files,Hierarchical Data Format database, ISO 9660 CD images, JPEG images,
+  JSON files,Java .class files, Java .jmod modules, JavaScript files, LLVM IR bitcode
+  files,LZ4 compressed files, MacOS binaries, Microsoft Windows icon files, MicrosoftWord
+  .docx files, Mono ''Portable Executable'' files, Mozilla-optimized .ZIParchives,
+  Multimedia metadata, OCaml interface files, Ogg Vorbis audio files,OpenOffice .odt
+  files, OpenSSH public keys, OpenWRT package archives (.ipk),PDF documents, PGP signatures,
+  PGP signed/encrypted messages, PNG images,PostScript documents, Public Key Cryptography
+  Standards (PKCS) files (version#7), Rust object files (.deflate), SQLite databases,
+  SquashFS filesystems,TrueType font files, WebAssembly binary module, XML binary
+  schemas (.xsb), XMLfiles, XZ compressed files, ZIP archives, Zstandard compressed
+  files, ar(1)archives, bzip2 archives, character/block devices, cpio archives, directories,ext2/ext3/ext4/btrfs/fat
+  filesystems, symlinks, tape archives (.tar), tcpdumpcapture files (.pcap) and text
+  files.'
+latest_release:
+  date: 1590224080
+  version: '146'


=====================================
_data/releases.yml deleted
=====================================
@@ -1,430 +0,0 @@
-# This file is automatically generated
-
-- version: 146
-  timestamp: 1590224080
-
-- version: 145
-  timestamp: 1590222686
-
-- version: 144
-  timestamp: 1589469366
-
-- version: 143
-  timestamp: 1587989647
-
-- version: 142
-  timestamp: 1587569714
-
-- version: 141
-  timestamp: 1586974542
-
-- version: 140
-  timestamp: 1586628657
-
-- version: 139
-  timestamp: 1585945762
-
-- version: 138
-  timestamp: 1585581388
-
-- version: 137
-  timestamp: 1582046159
-
-- version: 136
-  timestamp: 1579880337
-
-- version: 135
-  timestamp: 1579011967
-
-- version: 134
-  timestamp: 1577474236
-
-- version: 133
-  timestamp: 1574935413
-
-- version: 132
-  timestamp: 1574764304
-
-- version: 131
-  timestamp: 1574540571
-
-- version: 130
-  timestamp: 1573726940
-
-- version: 129
-  timestamp: 1572261318
-
-- version: 128
-  timestamp: 1571991700
-
-- version: 127
-  timestamp: 1571929174
-
-- version: 126
-  timestamp: 1571081387
-
-- version: 125
-  timestamp: 1568799897
-
-- version: 124
-  timestamp: 1568715547
-
-- version: 123
-  timestamp: 1567862191
-
-- version: 122
-  timestamp: 1567151545
-
-- version: 121
-  timestamp: 1565974147
-
-- version: 120
-  timestamp: 1564431466
-
-- version: 119
-  timestamp: 1564424544
-
-- version: 118
-  timestamp: 1563024209
-
-- version: 117
-  timestamp: 1562678694
-
-- version: 116
-  timestamp: 1562511269
-
-- version: 115
-  timestamp: 1558447322
-
-- version: 114
-  timestamp: 1556468506
-
-- version: 113
-  timestamp: 1551736838
-
-- version: 112
-  timestamp: 1550566589
-
-- version: 111
-  timestamp: 1549800566
-
-- version: 110
-  timestamp: 1549470789
-
-- version: 108
-  timestamp: 1547407265
-
-- version: 107
-  timestamp: 1544132574
-
-- version: 106
-  timestamp: 1542967768
-
-- version: 105
-  timestamp: 1541712443
-
-- version: 104
-  timestamp: 1540472141
-
-- version: 103
-  timestamp: 1538577359
-
-- version: 102
-  timestamp: 1537692220
-
-- version: 101
-  timestamp: 1537025865
-
-- version: 100
-  timestamp: 1535789388
-
-- version: 99
-  timestamp: 1531073226
-
-- version: 98
-  timestamp: 1530256765
-
-- version: 97
-  timestamp: 1530218330
-
-- version: 96
-  timestamp: 1529180368
-
-- version: 95
-  timestamp: 1526831217
-
-- version: 94
-  timestamp: 1525585484
-
-- version: 93
-  timestamp: 1522314183
-
-- version: 92
-  timestamp: 1521728924
-
-- version: 91
-  timestamp: 1519908048
-
-- version: 90
-  timestamp: 1514314170
-
-- version: 89
-  timestamp: 1514209762
-
-- version: 88
-  timestamp: 1509809169
-
-- version: 87
-  timestamp: 1506099185
-
-- version: 86
-  timestamp: 1504470186
-
-- version: 85
-  timestamp: 1501532865
-
-- version: 84
-  timestamp: 1499855291
-
-- version: 83
-  timestamp: 1497771299
-
-- version: 82
-  timestamp: 1493216277
-
-- version: 81
-  timestamp: 1490628077
-
-- version: 80
-  timestamp: 1489680405
-
-- version: 79
-  timestamp: 1489653496
-
-- version: 78
-  timestamp: 1488126408
-
-- version: 77
-  timestamp: 1486999502
-
-- version: 76
-  timestamp: 1486679872
-
-- version: 75
-  timestamp: 1486672127
-
-- version: 74
-  timestamp: 1486612708
-
-- version: 73
-  timestamp: 1486599899
-
-- version: 72
-  timestamp: 1486597780
-
-- version: 71
-  timestamp: 1486512121
-
-- version: 70
-  timestamp: 1485445150
-
-- version: 69
-  timestamp: 1484613234
-
-- version: 68
-  timestamp: 1484526262
-
-- version: 67
-  timestamp: 1483548996
-
-- version: 66
-  timestamp: 1482607110
-
-- version: 65
-  timestamp: 1482494295
-
-- version: 64
-  timestamp: 1482403056
-
-- version: 63
-  timestamp: 1480677004
-
-- version: 62
-  timestamp: 1478014425
-
-- version: 61
-  timestamp: 1475059465
-
-- version: 60
-  timestamp: 1473610473
-
-- version: 59
-  timestamp: 1471251288
-
-- version: 58
-  timestamp: 1471166539
-
-- version: 57
-  timestamp: 1471128402
-
-- version: 56
-  timestamp: 1469656976
-
-- version: 55
-  timestamp: 1468612229
-
-- version: 54
-  timestamp: 1464970562
-
-- version: 53
-  timestamp: 1464963796
-
-- version: 52
-  timestamp: 1461176883
-
-- version: 51
-  timestamp: 1456660595
-
-- version: 50
-  timestamp: 1456598240
-
-- version: 49
-  timestamp: 1455729165
-
-- version: 48
-  timestamp: 1453806837
-
-- version: 47
-  timestamp: 1453572513
-
-- version: 46
-  timestamp: 1453218407
-
-- version: 45
-  timestamp: 1451921090
-
-- version: 44
-  timestamp: 1450453340
-
-- version: 43
-  timestamp: 1450194895
-
-- version: 42
-  timestamp: 1448045874
-
-- version: 41
-  timestamp: 1447507389
-
-- version: 40
-  timestamp: 1447412596
-
-- version: 39
-  timestamp: 1447327685
-
-- version: 38
-  timestamp: 1445072712
-
-- version: 37
-  timestamp: 1444920355
-
-- version: 36
-  timestamp: 1442913877
-
-- version: 35
-  timestamp: 1442870636
-
-- version: 34
-  timestamp: 1442487446
-
-- version: 33
-  timestamp: 1441354845
-
-- version: 32
-  timestamp: 1441303245
-
-- version: 31
-  timestamp: 1440108191
-
-- version: 30
-  timestamp: 1439115510
-
-- version: 29
-  timestamp: 1438610107
-
-- version: 28
-  timestamp: 1438528912
-
-- version: 27
-  timestamp: 1438331890
-
-- version: 26
-  timestamp: 1435483341
-
-- version: 25
-  timestamp: 1435415578
-
-- version: 24
-  timestamp: 1435011328
-
-- version: 23
-  timestamp: 1435007957
-
-- version: 22
-  timestamp: 1434315949
-
-- version: 21
-  timestamp: 1434224300
-
-- version: 20
-  timestamp: 1431871514
-
-- version: 19
-  timestamp: 1431419748
-
-- version: 18
-  timestamp: 1431163786
-
-- version: 17
-  timestamp: 1430414966
-
-- version: 16
-  timestamp: 1429175278
-
-- version: 15
-  timestamp: 1428165048
-
-- version: 14
-  timestamp: 1427984219
-
-- version: 13
-  timestamp: 1427565209
-
-- version: 12
-  timestamp: 1427478146
-
-- version: 11
-  timestamp: 1427144415
-
-- version: 10
-  timestamp: 1426855964
-
-- version: 9
-  timestamp: 1423998826
-
-- version: 8
-  timestamp: 1423533775
-
-- version: 7
-  timestamp: 1420895117
-
-- version: 6
-  timestamp: 1420628471
-
-- version: 5
-  timestamp: 1419939253
-
-- version: 4
-  timestamp: 1415532907
-
-- version: 3
-  timestamp: 1412250775


=====================================
index.md
=====================================
@@ -7,7 +7,7 @@ title: "diffoscope: in-depth comparison of files, archives, and directories"
 
 <div class="lead">In-depth comparison of files, archives, and directories.</div>
 
-**Latest release**: {{ site.data.releases[0].version }} ({{ site.data.releases[0].timestamp|date: "%d %b %Y"  }})
+**Latest release**: {{ site.data.diffoscope.latest_release.version }} ({{ site.data.diffoscope.latest_release.date|date: "%d %b %Y"  }})
 
 *diffoscope* will try to get to the bottom of what makes files or directories different. It will recursively unpack archives of many kinds and transform various binary formats into more human readable form to compare them. It can compare two tarballs, ISO images, or PDF just as easily.
 
@@ -44,7 +44,7 @@ Try it now using Docker:
 
 * Text and HTML ouput
 
-* Supported file formats: Android APK files, Android boot images, Berkeley DB database files, ColorSync colour profiles (.icc), Coreboot CBFS filesystem images, Dalvik .dex files, Debian .buildinfo files, Debian .changes files, Debian source packages (.dsc), Device Tree Compiler blob files, ELF binaries, FreeDesktop Fontconfig cache files, FreePascal files (.ppu), GHC Haskell .hi files, GIF image files, GNU R Rscript files (.rds), GNU R database files (.rdb), Gettext message catalogues, Git repositories, Gnumeric spreadsheets, Gzipped files, ISO 9660 CD images, JPEG images, JSON files, Java .class files, JavaScript files, LLVM IR bitcode files, MacOS binaries, Microsoft Windows icon files, Microsoft Word .docx files, Mono 'Portable Executable' files, Ogg Vorbis audio files, OpenOffice .odt files, OpenSSH public keys, OpenWRT package archives (.ipk), PDF documents, PGP signed/encrypted messages, PNG images, PostScript documents, RPM archives, Rust object files (.deflate), SQLite databases, SquashFS filesystems, TrueType font files, XML binary schemas (.xsb), XML files, XZ compressed files, ar(1) archives, bzip2 archives, character/block devices, cpio archives, directories, ext2/ext3/ext4/btrfs filesystems, statically-linked binaries, symlinks, tape archives (.tar), tcpdump capture files (.pcap) and text files.
+* {{ site.data.diffoscope.description }}
 
 * Fallback on hexdump comparison
 
@@ -107,4 +107,4 @@ Extending *diffoscope* to support new formats is quite straightforward in most c
 
 ## Contributors
 
-{{ site.data.contributors|array_to_sentence_string:"and" }}.
+{{ site.data.diffoscope.contributors|array_to_sentence_string:"and" }}.


=====================================
update
=====================================
@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+import os
+import yaml
+import tempfile
+import subprocess
+
+from email.utils import parsedate_to_datetime
+from debian.changelog import Changelog
+
+
+DATA_DIR = os.path.join(os.path.dirname(__file__), "_data")
+
+
+def main():
+    src = "https://salsa.debian.org/reproducible-builds/diffoscope.git"
+
+    # Prefer a local checkout if it exists
+    candidate = os.path.join(
+        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+        "diffoscope",
+        ".git",
+    )
+
+    if os.path.exists(candidate):
+        src = candidate
+
+    with tempfile.TemporaryDirectory() as t:
+        git_dir = os.path.join(t, "diffoscope")
+        subprocess.check_call(("git", "clone", src, git_dir))
+
+        releases = get_releases(git_dir)
+        debian_substvars = get_debian_substvars(git_dir)
+
+        data = {
+            "latest_release": {
+                "date": int(releases[0]["date"].timestamp()),
+                "version": releases[0]["version"],
+            },
+            "contributors": get_contributors(git_dir),
+            "description": debian_substvars["diffoscope:Description"],
+        }
+
+        with open(os.path.join(DATA_DIR, "diffoscope.yml"), "w") as f:
+            print("# This file is automatically generated\n", file=f)
+            yaml.dump(data, stream=f)
+
+
+def get_contributors(git_dir):
+    result = set()
+
+    for x in git(git_dir, "log", "--format=%aN").splitlines():
+        if x != "root":
+            result.add(x)
+
+    return list(sorted(result, key=lambda x: x.lower()))
+
+
+def get_releases(git_dir):
+    result = []
+    changelog = Changelog()
+
+    with open(os.path.join(git_dir, "debian", "changelog")) as f:
+        changelog.parse_changelog(f)
+
+    for x in changelog:
+        result.append(
+            {"version": str(x.version), "date": parsedate_to_datetime(x.date),}
+        )
+
+    return result
+
+
+def get_debian_substvars(git_dir):
+    val = subprocess.check_output(
+        ("bin/diffoscope", "--list-debian-substvars"), cwd=git_dir
+    )
+
+    data = {}
+
+    for x in val.decode("utf-8").splitlines():
+        key, _, val = x.partition("=")
+
+        data[key] = val.replace("${Newline}", "")
+
+    return data
+
+
+def git(git_dir, *args):
+    cmd = ("git",) + args
+
+    val = subprocess.check_output(cmd, env={"GIT_DIR": os.path.join(git_dir, ".git")})
+
+    return val.decode("utf-8")
+
+
+if __name__ == "__main__":
+    main()



View it on GitLab: https://salsa.debian.org/reproducible-builds/diffoscope-website/-/compare/2da6b560c390aa1840b43836ea856028afdfa1db...f0663bbf1278e28455cb3941810000db15edf131

-- 
View it on GitLab: https://salsa.debian.org/reproducible-builds/diffoscope-website/-/compare/2da6b560c390aa1840b43836ea856028afdfa1db...f0663bbf1278e28455cb3941810000db15edf131
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.reproducible-builds.org/pipermail/rb-commits/attachments/20200527/38347946/attachment.htm>


More information about the rb-commits mailing list