[reproducible-website] 01/01: Add generate-draft{,.template}

Chris Lamb chris at chris-lamb.co.uk
Sun Apr 22 13:15:10 CEST 2018


This is an automated email from the git hooks/post-receive script.

lamby pushed a commit to branch master
in repository reproducible-website.

commit c45f3cda9ece60f9fc95e7a7c87720ca915aaaf3
Author: Chris Lamb <lamby at debian.org>
Date:   Sun Apr 22 13:15:02 2018 +0200

    Add generate-draft{,.template}
---
 bin/generate-draft          | 321 ++++++++++++++++++++++++++++++++++++++++++++
 bin/generate-draft.template |  48 +++++++
 2 files changed, 369 insertions(+)

diff --git a/bin/generate-draft b/bin/generate-draft
new file mode 100755
index 0000000..dfd9b7f
--- /dev/null
+++ b/bin/generate-draft
@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+
+import collections
+import datetime
+import jinja2
+import os
+import pickle
+import re
+import subprocess
+import sys
+import time
+import yaml
+
+WEEK_1_END = 1430611200  # May 3 2015, 00:00 UTC, Sunday
+
+PROJECTS = (
+    'diffoscope',
+    'strip-nondeterminism',
+    'disorderfs',
+    'reprotest',
+    'buildinfo.debian.net',
+    'trydiffoscope',
+    'reproducible-website',
+    'jenkins.debian.net',
+)
+
+
+def main(*args):
+    for x in PROJECTS + ('notes',):
+        ensure_dir(sibling_repo_gitdir(x))
+
+    week = int(args[0]) if len(args) > 0 else prev_week()
+
+    data = get_data(week)
+
+    env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(os.path.dirname(__file__))
+    )
+    print(env.get_template('generate-draft.template').render(**data))
+
+    return 0
+
+
+
+def log(msg, *args, **kwargs):
+    print("I: " + msg.format(*args, **kwargs), file=sys.stderr)
+
+
+def prev_week():
+    now = int(time.time())
+    return ((now - WEEK_1_END) // (7*24*3600) + 1)
+
+
+def sibling_repo_gitdir(path):
+    toplevel = os.path.dirname(subprocess.check_output((
+        'git',
+        'rev-parse',
+        '--show-toplevel',
+    )).decode('utf-8'))
+
+    return os.path.join(toplevel, path, '.git')
+
+
+def ensure_dir(path):
+    if not os.path.isdir(path):
+        raise ValueError("not a directory: {}".format(path))
+
+
+def get_data(week, max_age=3600):
+    filename = '/tmp/generate-draft-{}.pickle'.format(week)
+
+    try:
+        mtime = os.path.getmtime(filename)
+        mtime_self = os.path.getmtime(__file__)
+
+        if mtime > mtime_self and mtime >= time.time() - max_age:
+            log("Using cache from {}", filename)
+
+            with open(filename, 'rb') as f:
+                return pickle.load(f)
+    except (EOFError, OSError):
+        pass
+
+    log("Getting new data")
+
+    week_end = WEEK_1_END + (week - 1) * 7 * 24 * 3600  # exclusive
+    week_start = week_end - 7 * 24 * 3600  # inclusive
+
+    data = {x: y(week_start, week_end) for x, y in (
+        ('author', get_author),
+        ('commits', get_commits),
+        ('uploads', get_uploads),
+        ('patches', get_patches),
+        ('ftbfs_bugs', get_ftbfs_bugs),
+        ('issues_yml', get_issues_yml),
+        ('packages_yml', get_packages_yml),
+        ('packages_stats', get_packages_stats),
+    )}
+
+    data.update({
+        'week': week,
+        'week_start': datetime.datetime.utcfromtimestamp(week_start),
+        'week_end': datetime.datetime.utcfromtimestamp(week_end - 1),
+        'projects': PROJECTS,
+    })
+
+    log("Saving cache to {}", filename)
+
+    with open(filename, 'wb') as f:
+        pickle.dump(data, f)
+
+    return data
+
+
+
+def get_author(week_start, week_end):
+    return os.environ.get('DEBFULLNAME', 'FIXME')
+
+
+def get_ftbfs_bugs(week_start, week_end):
+    return bugs(
+        week_start,
+        week_end,
+        "bugs_usertags.tag = '{}'".format('ftbfs'),
+    )
+
+
+def get_patches(week_start, week_end):
+    return bugs(
+        week_start,
+        week_end,
+        "id IN (SELECT id FROM bugs_tags WHERE tag = 'patch')",
+    )
+
+
+def bugs(week_start, week_end, extra="true"):
+    log("Querying UDD for usertagged bugs with filter: {}", extra)
+
+    fields = (
+        'id',
+        'source',
+        'submitter',
+        'submitter_name',
+        'title',
+        'arrival',
+    )
+
+    sql = """
+        SELECT
+            {fields}
+        FROM
+            bugs
+        INNER JOIN
+            bugs_usertags USING (id)
+        WHERE
+            bugs_usertags.email = 'reproducible-builds at lists.alioth.debian.org'
+        AND
+            {extra}
+        AND
+            CAST(arrival AS DATE) BETWEEN to_timestamp(@{week_start}) AND to_timestamp(@{week_end})
+    """.format(**{
+        'fields': ', '.join(fields),
+        'extra': extra,
+        'week_start': week_start,
+        'week_end': week_end,
+    })
+
+    seen = set()
+    result = {}
+    for x in udd(sql, fields):
+        if x['id'] in seen:
+            continue
+        seen.add(x['id'])
+
+        result.setdefault(x['submitter_name'], []).append(x)
+
+    return {
+        x: list(sorted(y, key=lambda x: x['id'])) for x, y in result.items()
+    }
+
+
+def get_uploads(week_start, week_end):
+    log("Querying UDD for uploads")
+
+    fields = (
+        'source',
+        'version',
+        'distribution',
+        'signed_by_name',
+    )
+
+    data = udd("""
+        SELECT
+            {fields}
+        FROM
+            upload_history
+        WHERE
+            source IN ({sources})
+        AND
+            CAST(date AS date) BETWEEN to_timestamp(@{week_start}) AND to_timestamp(@{week_end})
+        ORDER BY
+            date
+    """.format(**{
+        'fields': ', '.join(fields),
+        'sources': ', '.join("'{}'".format(x) for x in PROJECTS),
+        'week_start': week_start,
+        'week_end': week_end,
+    }), fields)
+
+    result = {}
+    for x in data:
+        result.setdefault(x['source'], []).append(x)
+
+    return result
+
+
+def udd(query, fields):
+    lines = subprocess.check_output("""
+        echo "{}" | ssh alioth.debian.org psql --no-psqlrc service=udd
+    """.format(query), shell=True)
+
+    data = []
+
+    for line in lines.splitlines()[2:]:
+        split = line.decode('utf-8').split('|')
+
+        if len(split) != len(fields):
+            continue
+
+        row = dict(zip(fields, [x.strip() for x in split]))
+
+        data.append(row)
+
+    return data
+
+
+def get_commits(week_start, week_end):
+    return {x: commits(week_start, week_end, x) for x in PROJECTS}
+
+
+def get_issues_yml(week_start, week_end):
+    return commits(week_start, week_end, 'notes', 'issues.yml')
+
+
+def get_packages_yml(week_start, week_end):
+    return commits(week_start, week_end, 'notes', 'packages.yml')
+
+
+def open_packages_yml(date):
+    return subprocess.Popen(
+        "git show $(git rev-list -n1 --until @{0} origin/master):packages.yml".format(date),
+        shell=True,
+        cwd=sibling_repo_gitdir("notes"),
+        stdout=subprocess.PIPE).stdout
+
+
+def get_packages_stats(week_start, week_end):
+    old = yaml.safe_load(open_packages_yml(week_start))
+    new = yaml.safe_load(open_packages_yml(week_end))
+
+    removed = set(old.keys()) - set(new.keys())
+    added = set(new.keys()) - set(old.keys())
+    updated = 0
+    for name in set(old.keys()).intersection(new.keys()):
+        if old[name] != new[name]:
+            updated += 1
+    return {
+        "removed": len(removed),
+        "added": len(added),
+        "updated": updated,
+    }
+
+
+def commits(week_start, week_end, project, path='.'):
+    # Assume its in the parent dir
+    git_dir = sibling_repo_gitdir(project)
+
+    subprocess.check_call(('git', 'fetch', 'origin'), cwd=git_dir)
+
+    output = subprocess.check_output((
+        'git',
+        'log',
+        'origin/master',
+        '--since', '@{}'.format(week_start),
+        '--until', '@{}'.format(week_end),
+        '--pretty=format:%an\t%h\t%s',
+        '--no-merges',
+        '--all',
+        '--',
+        path,
+    ), cwd=git_dir).decode('utf-8')
+
+    result = collections.defaultdict(list)
+    for x in output.splitlines():
+        author, sha, title = x.split('\t', 2)
+
+        for pattern in (
+            r'^dependabot$',
+        ):
+            if re.search(pattern, author) is not None:
+                continue
+
+        for pattern in (
+            r'^--fix-deterministic$',
+            r'^Add missing usertagged bugs$',
+            r'^Remove archived bugs$',
+            r'^Release .* to Debian .*$',
+        ):
+            if re.search(pattern, title) is not None:
+                continue
+
+        result[author].append({
+            'sha': sha,
+            'title': title.replace('_', '\_'),
+        })
+
+    return result
+
+
+if __name__ == '__main__':
+    sys.exit(main(*sys.argv[1:]))
diff --git a/bin/generate-draft.template b/bin/generate-draft.template
new file mode 100644
index 0000000..21d4603
--- /dev/null
+++ b/bin/generate-draft.template
@@ -0,0 +1,48 @@
+---
+layout: blog
+week: {{ week }}
+---
+
+Here's what happened in the [Reproducible Builds](https://reproducible-builds.org) effort between {{ week_start.strftime('%A %B') }} {{ week_start.day }} and {{ week_end.strftime('%A %B') }} {{ week_end.day }} {{ week_end.year }}:
+
+* FIXME
+
+Packages reviewed and fixed, and bugs filed
+-------------------------------------------
+
+FIXME: prune the below list so it doesn't duplicate the information above, and perhaps also run bin/review-bugs to see if anything was missed.
+
+{% for x, ys in patches.items()|sort %}* {{ x }}:
+{% for y in ys %}    * [#{{ y['id'] }}](https://bugs.debian.org/{{ y['id'] }}) filed against [{{ y['source'] }}](https://tracker.debian.org/pkg/{{ y['source'] }}).
+{% endfor %}{% endfor %}
+
+In addition, build failure bugs were reported by:
+{% for k, v in ftbfs_bugs.items()|sort %}
+* {{ k }} ({{ v|length }}){% endfor %}
+
+{% for project in projects %}
+{{ project }} development
+------------{{ "-" * project|length }}
+{% for x in uploads[project] %}
+Version [{{ x['version'] }}](https://tracker.debian.org/news/FIXME) was uploaded to {{ x['distribution'] }} by {{ x['signed_by_name'] }}. It [includes contributions lready convered by posts in previous weeks](https://anonscm.debian.org/git/reproducible/{{ project }}.git/log/?h={% if project != 'diffoscope' %}debian/{% endif %}{{ x['version'] }}) as well as new ones from:
+
+{% endfor %}
+{% for x, ys in commits[project].items() %}* {{ x }}:{% for y in ys %}
+    * [{{ y['title'] }}]({% if project == "jenkins.debian.net" %}https://salsa.debian.org/qa/jenkins.debian.net/commit/{{ y['sha'] }}{% else %}https://anonscm.debian.org/git/reproducible/{{ project }}.git/commit/?id={{ y['sha'] }}{% endif %}){% endfor %}
+{% endfor %}
+{% endfor %}
+
+Reviews of unreproducible packages
+----------------------------------
+
+{{ packages_stats['added'] }} package reviews have been added, {{ packages_stats['updated'] }} have been updated and {{ packages_stats['removed'] }} have been removed in this week, adding to our [knowledge about identified issues](https://tests.reproducible-builds.org/debian/index_issues.html).
+
+FIXME issue types have been updated:
+{% for _, xs in issues_yml.items()|sort %}{% for x in xs %}
+* [{{ x['title'] }}](https://anonscm.debian.org/git/reproducible/notes.git/commit/?id={{ x['sha'] }}){% endfor %}{% endfor %}
+
+
+Misc.
+-----
+
+This week's edition was written by {{ author }} & reviewed by a bunch of Reproducible Builds folks on IRC & the mailing lists.

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/reproducible-website.git


More information about the rb-commits mailing list