[reproducible-website] 01/01: Add generate-draft{,.template}
Chris Lamb
chris at chris-lamb.co.uk
Sun Apr 22 13:15:10 CEST 2018
This is an automated email from the git hooks/post-receive script.
lamby pushed a commit to branch master
in repository reproducible-website.
commit c45f3cda9ece60f9fc95e7a7c87720ca915aaaf3
Author: Chris Lamb <lamby at debian.org>
Date: Sun Apr 22 13:15:02 2018 +0200
Add generate-draft{,.template}
---
bin/generate-draft | 321 ++++++++++++++++++++++++++++++++++++++++++++
bin/generate-draft.template | 48 +++++++
2 files changed, 369 insertions(+)
diff --git a/bin/generate-draft b/bin/generate-draft
new file mode 100755
index 0000000..dfd9b7f
--- /dev/null
+++ b/bin/generate-draft
@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+
+import collections
+import datetime
+import jinja2
+import os
+import pickle
+import re
+import subprocess
+import sys
+import time
+import yaml
+
+WEEK_1_END = 1430611200 # May 3 2015, 00:00 UTC, Sunday
+
+PROJECTS = (
+ 'diffoscope',
+ 'strip-nondeterminism',
+ 'disorderfs',
+ 'reprotest',
+ 'buildinfo.debian.net',
+ 'trydiffoscope',
+ 'reproducible-website',
+ 'jenkins.debian.net',
+)
+
+
+def main(*args):
+ for x in PROJECTS + ('notes',):
+ ensure_dir(sibling_repo_gitdir(x))
+
+ week = int(args[0]) if len(args) > 0 else prev_week()
+
+ data = get_data(week)
+
+ env = jinja2.Environment(
+ loader=jinja2.FileSystemLoader(os.path.dirname(__file__))
+ )
+ print(env.get_template('generate-draft.template').render(**data))
+
+ return 0
+
+
+
+def log(msg, *args, **kwargs):
+ print("I: " + msg.format(*args, **kwargs), file=sys.stderr)
+
+
+def prev_week():
+ now = int(time.time())
+ return ((now - WEEK_1_END) // (7*24*3600) + 1)
+
+
+def sibling_repo_gitdir(path):
+ toplevel = os.path.dirname(subprocess.check_output((
+ 'git',
+ 'rev-parse',
+ '--show-toplevel',
+ )).decode('utf-8'))
+
+ return os.path.join(toplevel, path, '.git')
+
+
+def ensure_dir(path):
+ if not os.path.isdir(path):
+ raise ValueError("not a directory: {}".format(path))
+
+
+def get_data(week, max_age=3600):
+ filename = '/tmp/generate-draft-{}.pickle'.format(week)
+
+ try:
+ mtime = os.path.getmtime(filename)
+ mtime_self = os.path.getmtime(__file__)
+
+ if mtime > mtime_self and mtime >= time.time() - max_age:
+ log("Using cache from {}", filename)
+
+ with open(filename, 'rb') as f:
+ return pickle.load(f)
+ except (EOFError, OSError):
+ pass
+
+ log("Getting new data")
+
+ week_end = WEEK_1_END + (week - 1) * 7 * 24 * 3600 # exclusive
+ week_start = week_end - 7 * 24 * 3600 # inclusive
+
+ data = {x: y(week_start, week_end) for x, y in (
+ ('author', get_author),
+ ('commits', get_commits),
+ ('uploads', get_uploads),
+ ('patches', get_patches),
+ ('ftbfs_bugs', get_ftbfs_bugs),
+ ('issues_yml', get_issues_yml),
+ ('packages_yml', get_packages_yml),
+ ('packages_stats', get_packages_stats),
+ )}
+
+ data.update({
+ 'week': week,
+ 'week_start': datetime.datetime.utcfromtimestamp(week_start),
+ 'week_end': datetime.datetime.utcfromtimestamp(week_end - 1),
+ 'projects': PROJECTS,
+ })
+
+ log("Saving cache to {}", filename)
+
+ with open(filename, 'wb') as f:
+ pickle.dump(data, f)
+
+ return data
+
+
+
+def get_author(week_start, week_end):
+ return os.environ.get('DEBFULLNAME', 'FIXME')
+
+
+def get_ftbfs_bugs(week_start, week_end):
+ return bugs(
+ week_start,
+ week_end,
+ "bugs_usertags.tag = '{}'".format('ftbfs'),
+ )
+
+
+def get_patches(week_start, week_end):
+ return bugs(
+ week_start,
+ week_end,
+ "id IN (SELECT id FROM bugs_tags WHERE tag = 'patch')",
+ )
+
+
+def bugs(week_start, week_end, extra="true"):
+ log("Querying UDD for usertagged bugs with filter: {}", extra)
+
+ fields = (
+ 'id',
+ 'source',
+ 'submitter',
+ 'submitter_name',
+ 'title',
+ 'arrival',
+ )
+
+ sql = """
+ SELECT
+ {fields}
+ FROM
+ bugs
+ INNER JOIN
+ bugs_usertags USING (id)
+ WHERE
+ bugs_usertags.email = 'reproducible-builds at lists.alioth.debian.org'
+ AND
+ {extra}
+ AND
+ CAST(arrival AS DATE) BETWEEN to_timestamp(@{week_start}) AND to_timestamp(@{week_end})
+ """.format(**{
+ 'fields': ', '.join(fields),
+ 'extra': extra,
+ 'week_start': week_start,
+ 'week_end': week_end,
+ })
+
+ seen = set()
+ result = {}
+ for x in udd(sql, fields):
+ if x['id'] in seen:
+ continue
+ seen.add(x['id'])
+
+ result.setdefault(x['submitter_name'], []).append(x)
+
+ return {
+ x: list(sorted(y, key=lambda x: x['id'])) for x, y in result.items()
+ }
+
+
+def get_uploads(week_start, week_end):
+ log("Querying UDD for uploads")
+
+ fields = (
+ 'source',
+ 'version',
+ 'distribution',
+ 'signed_by_name',
+ )
+
+ data = udd("""
+ SELECT
+ {fields}
+ FROM
+ upload_history
+ WHERE
+ source IN ({sources})
+ AND
+ CAST(date AS date) BETWEEN to_timestamp(@{week_start}) AND to_timestamp(@{week_end})
+ ORDER BY
+ date
+ """.format(**{
+ 'fields': ', '.join(fields),
+ 'sources': ', '.join("'{}'".format(x) for x in PROJECTS),
+ 'week_start': week_start,
+ 'week_end': week_end,
+ }), fields)
+
+ result = {}
+ for x in data:
+ result.setdefault(x['source'], []).append(x)
+
+ return result
+
+
+def udd(query, fields):
+ lines = subprocess.check_output("""
+ echo "{}" | ssh alioth.debian.org psql --no-psqlrc service=udd
+ """.format(query), shell=True)
+
+ data = []
+
+ for line in lines.splitlines()[2:]:
+ split = line.decode('utf-8').split('|')
+
+ if len(split) != len(fields):
+ continue
+
+ row = dict(zip(fields, [x.strip() for x in split]))
+
+ data.append(row)
+
+ return data
+
+
+def get_commits(week_start, week_end):
+ return {x: commits(week_start, week_end, x) for x in PROJECTS}
+
+
+def get_issues_yml(week_start, week_end):
+ return commits(week_start, week_end, 'notes', 'issues.yml')
+
+
+def get_packages_yml(week_start, week_end):
+ return commits(week_start, week_end, 'notes', 'packages.yml')
+
+
+def open_packages_yml(date):
+ return subprocess.Popen(
+ "git show $(git rev-list -n1 --until @{0} origin/master):packages.yml".format(date),
+ shell=True,
+ cwd=sibling_repo_gitdir("notes"),
+ stdout=subprocess.PIPE).stdout
+
+
+def get_packages_stats(week_start, week_end):
+ old = yaml.safe_load(open_packages_yml(week_start))
+ new = yaml.safe_load(open_packages_yml(week_end))
+
+ removed = set(old.keys()) - set(new.keys())
+ added = set(new.keys()) - set(old.keys())
+ updated = 0
+ for name in set(old.keys()).intersection(new.keys()):
+ if old[name] != new[name]:
+ updated += 1
+ return {
+ "removed": len(removed),
+ "added": len(added),
+ "updated": updated,
+ }
+
+
+def commits(week_start, week_end, project, path='.'):
+ # Assume its in the parent dir
+ git_dir = sibling_repo_gitdir(project)
+
+ subprocess.check_call(('git', 'fetch', 'origin'), cwd=git_dir)
+
+ output = subprocess.check_output((
+ 'git',
+ 'log',
+ 'origin/master',
+ '--since', '@{}'.format(week_start),
+ '--until', '@{}'.format(week_end),
+ '--pretty=format:%an\t%h\t%s',
+ '--no-merges',
+ '--all',
+ '--',
+ path,
+ ), cwd=git_dir).decode('utf-8')
+
+ result = collections.defaultdict(list)
+ for x in output.splitlines():
+ author, sha, title = x.split('\t', 2)
+
+ for pattern in (
+ r'^dependabot$',
+ ):
+ if re.search(pattern, author) is not None:
+ continue
+
+ for pattern in (
+ r'^--fix-deterministic$',
+ r'^Add missing usertagged bugs$',
+ r'^Remove archived bugs$',
+ r'^Release .* to Debian .*$',
+ ):
+ if re.search(pattern, title) is not None:
+ continue
+
+ result[author].append({
+ 'sha': sha,
+ 'title': title.replace('_', '\_'),
+ })
+
+ return result
+
+
+if __name__ == '__main__':
+ sys.exit(main(*sys.argv[1:]))
diff --git a/bin/generate-draft.template b/bin/generate-draft.template
new file mode 100644
index 0000000..21d4603
--- /dev/null
+++ b/bin/generate-draft.template
@@ -0,0 +1,48 @@
+---
+layout: blog
+week: {{ week }}
+---
+
+Here's what happened in the [Reproducible Builds](https://reproducible-builds.org) effort between {{ week_start.strftime('%A %B') }} {{ week_start.day }} and {{ week_end.strftime('%A %B') }} {{ week_end.day }} {{ week_end.year }}:
+
+* FIXME
+
+Packages reviewed and fixed, and bugs filed
+-------------------------------------------
+
+FIXME: prune the below list so it doesn't duplicate the information above, and perhaps also run bin/review-bugs to see if anything was missed.
+
+{% for x, ys in patches.items()|sort %}* {{ x }}:
+{% for y in ys %} * [#{{ y['id'] }}](https://bugs.debian.org/{{ y['id'] }}) filed against [{{ y['source'] }}](https://tracker.debian.org/pkg/{{ y['source'] }}).
+{% endfor %}{% endfor %}
+
+In addition, build failure bugs were reported by:
+{% for k, v in ftbfs_bugs.items()|sort %}
+* {{ k }} ({{ v|length }}){% endfor %}
+
+{% for project in projects %}
+{{ project }} development
+------------{{ "-" * project|length }}
+{% for x in uploads[project] %}
+Version [{{ x['version'] }}](https://tracker.debian.org/news/FIXME) was uploaded to {{ x['distribution'] }} by {{ x['signed_by_name'] }}. It [includes contributions lready convered by posts in previous weeks](https://anonscm.debian.org/git/reproducible/{{ project }}.git/log/?h={% if project != 'diffoscope' %}debian/{% endif %}{{ x['version'] }}) as well as new ones from:
+
+{% endfor %}
+{% for x, ys in commits[project].items() %}* {{ x }}:{% for y in ys %}
+ * [{{ y['title'] }}]({% if project == "jenkins.debian.net" %}https://salsa.debian.org/qa/jenkins.debian.net/commit/{{ y['sha'] }}{% else %}https://anonscm.debian.org/git/reproducible/{{ project }}.git/commit/?id={{ y['sha'] }}{% endif %}){% endfor %}
+{% endfor %}
+{% endfor %}
+
+Reviews of unreproducible packages
+----------------------------------
+
+{{ packages_stats['added'] }} package reviews have been added, {{ packages_stats['updated'] }} have been updated and {{ packages_stats['removed'] }} have been removed in this week, adding to our [knowledge about identified issues](https://tests.reproducible-builds.org/debian/index_issues.html).
+
+FIXME issue types have been updated:
+{% for _, xs in issues_yml.items()|sort %}{% for x in xs %}
+* [{{ x['title'] }}](https://anonscm.debian.org/git/reproducible/notes.git/commit/?id={{ x['sha'] }}){% endfor %}{% endfor %}
+
+
+Misc.
+-----
+
+This week's edition was written by {{ author }} & reviewed by a bunch of Reproducible Builds folks on IRC & the mailing lists.
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/reproducible-website.git
More information about the rb-commits
mailing list