[Git][reproducible-builds/reproducible-website][master] 3 commits: Calculate authors dynamically.
Chris Lamb
gitlab at salsa.debian.org
Thu Jan 2 16:47:08 UTC 2020
Chris Lamb pushed to branch master at Reproducible Builds / reproducible-website
Commits:
21da60b8 by Chris Lamb at 2020-01-02T12:53:07+00:00
Calculate authors dynamically.
- - - - -
e1fc0e8a by Chris Lamb at 2020-01-02T13:02:14+00:00
Move continuous tests heading level to <h1> (vs. <h2>) to match the other pages.
- - - - -
9fdd1cbd by Chris Lamb at 2020-01-02T16:46:42+00:00
Apply "back" to bin/generate-draft.
- - - - -
3 changed files:
- bin/generate-draft
- bin/generate-draft.template
- citests.md
Changes:
=====================================
bin/generate-draft
=====================================
@@ -13,32 +13,29 @@ import time
import yaml
PROJECTS = (
- 'diffoscope',
- 'diffoscope-website',
- 'strip-nondeterminism',
- 'disorderfs',
- 'reprotest',
- 'trydiffoscope',
- 'try.diffoscope.org',
- 'reproducible-website',
- 'rb-mailx-ansible',
- 'jenkins.debian.net',
+ "diffoscope",
+ "diffoscope-website",
+ "strip-nondeterminism",
+ "disorderfs",
+ "reprotest",
+ "trydiffoscope",
+ "try.diffoscope.org",
+ "reproducible-website",
+ "rb-mailx-ansible",
+ "jenkins.debian.net",
)
def main(*args):
- for x in PROJECTS + ('reproducible-notes',):
+ for x in PROJECTS + ("reproducible-notes",):
ensure_dir(sibling_repo_gitdir(x))
- previous_month = datetime.date.today().replace(day=1) - \
- datetime.timedelta(days=1)
+ previous_month = datetime.date.today().replace(day=1) - datetime.timedelta(days=1)
data = get_data(previous_month.year, previous_month.month)
- env = jinja2.Environment(
- loader=jinja2.FileSystemLoader(os.path.dirname(__file__))
- )
- print(env.get_template('generate-draft.template').render(**data))
+ env = jinja2.Environment(loader=jinja2.FileSystemLoader(os.path.dirname(__file__)))
+ print(env.get_template("generate-draft.template").render(**data))
return 0
@@ -48,13 +45,13 @@ def log(msg, *args, **kwargs):
def sibling_repo_gitdir(path):
- toplevel = os.path.dirname(subprocess.check_output((
- 'git',
- 'rev-parse',
- '--show-toplevel',
- )).decode('utf-8'))
+ toplevel = os.path.dirname(
+ subprocess.check_output(("git", "rev-parse", "--show-toplevel",)).decode(
+ "utf-8"
+ )
+ )
- return os.path.join(toplevel, path, '.git')
+ return os.path.join(toplevel, path, ".git")
def ensure_dir(path):
@@ -63,7 +60,7 @@ def ensure_dir(path):
def get_data(year, month, max_age=3600):
- filename = '/tmp/generate-draft-{:04d}{:02d}.pickle'.format(year, month)
+ filename = "/tmp/generate-draft-{:04d}{:02d}.pickle".format(year, month)
try:
mtime = os.path.getmtime(filename)
@@ -72,7 +69,7 @@ def get_data(year, month, max_age=3600):
if mtime > mtime_self and mtime >= time.time() - max_age:
log("Using cache from {}", filename)
- with open(filename, 'rb') as f:
+ with open(filename, "rb") as f:
return pickle.load(f)
except (EOFError, OSError):
pass
@@ -81,57 +78,68 @@ def get_data(year, month, max_age=3600):
month_start = datetime.datetime(year=year, month=month, day=1)
month_end = month_start.replace(
- day=calendar.monthlen(year, month),
- hour=23,
- minute=59
+ day=calendar.monthlen(year, month), hour=23, minute=59
)
month_start_ts = int(month_start.timestamp())
month_end_ts = int(month_end.timestamp())
- data = {x: y(month_start_ts, month_end_ts) for x, y in (
- ('author', get_author),
- ('commits', get_commits),
- ('uploads', get_uploads),
- ('patches', get_patches),
- ('ftbfs_bugs', get_ftbfs_bugs),
- ('issues_yml', get_issues_yml),
- ('packages_yml', get_packages_yml),
- ('packages_stats', get_packages_stats),
- )}
-
- data.update({
- 'projects': PROJECTS,
- 'month_year': month_start.strftime('%B %Y'),
- 'title_year': '{:04d}'.format(year),
- 'title_month': '{:02d}'.format(month),
- })
+ data = {
+ x: y(month_start_ts, month_end_ts)
+ for x, y in (
+ ("authors", get_authors),
+ ("commits", get_commits),
+ ("uploads", get_uploads),
+ ("patches", get_patches),
+ ("ftbfs_bugs", get_ftbfs_bugs),
+ ("issues_yml", get_issues_yml),
+ ("packages_yml", get_packages_yml),
+ ("packages_stats", get_packages_stats),
+ )
+ }
+
+ data.update(
+ {
+ "projects": PROJECTS,
+ "month_year": month_start.strftime("%B %Y"),
+ "title_year": "{:04d}".format(year),
+ "title_month": "{:02d}".format(month),
+ }
+ )
log("Saving cache to {}", filename)
- with open(filename, 'wb') as f:
+ with open(filename, "wb") as f:
pickle.dump(data, f)
return data
-def get_author(month_start, month_end):
- return os.environ.get('DEBFULLNAME', 'FIXME')
+def get_authors(month_start, month_end):
+ lines = subprocess.check_output(
+ (
+ "git",
+ "log",
+ '--pretty=format:"%an%x09"',
+ "_reports/{}-{:02d}.md".format(month_start.year, month_start.month),
+ )
+ ).decode("utf-8")
+
+ authors = list(sorted(set(x.replace('"', "").strip() for x in lines.splitlines())))
+
+ if len(authors) < 2:
+ return " and ".join(authors)
+
+ return "{} and {}".format(", ".join(authors[:-1]), authors[-1],)
def get_ftbfs_bugs(month_start, month_end):
- return bugs(
- month_start,
- month_end,
- "bugs_usertags.tag = '{}'".format('ftbfs'),
- )
+ return bugs(month_start, month_end, "bugs_usertags.tag = '{}'".format("ftbfs"),)
def get_patches(month_start, month_end):
return bugs(
- month_start,
- month_end,
- "id IN (SELECT id FROM bugs_tags WHERE tag = 'patch')",
+ month_start, month_end, "id IN (SELECT id FROM bugs_tags WHERE tag = 'patch')",
)
@@ -139,12 +147,12 @@ def bugs(month_start, month_end, extra="true"):
log("Querying UDD for usertagged bugs with filter: {}", extra)
fields = (
- 'id',
- 'source',
- 'submitter',
- 'submitter_name',
- 'title',
- 'arrival',
+ "id",
+ "source",
+ "submitter",
+ "submitter_name",
+ "title",
+ "arrival",
)
sql = """
@@ -161,38 +169,39 @@ def bugs(month_start, month_end, extra="true"):
AND
CAST(arrival AS DATE) BETWEEN
to_timestamp(@{month_start}) AND to_timestamp(@{month_end})
- """.format(**{
- 'fields': ', '.join(fields),
- 'extra': extra,
- 'month_start': month_start,
- 'month_end': month_end,
- })
+ """.format(
+ **{
+ "fields": ", ".join(fields),
+ "extra": extra,
+ "month_start": month_start,
+ "month_end": month_end,
+ }
+ )
seen = set()
result = {}
for x in udd(sql, fields):
- if x['id'] in seen:
+ if x["id"] in seen:
continue
- seen.add(x['id'])
+ seen.add(x["id"])
- result.setdefault(x['submitter_name'], []).append(x)
+ result.setdefault(x["submitter_name"], []).append(x)
- return {
- x: list(sorted(y, key=lambda x: x['id'])) for x, y in result.items()
- }
+ return {x: list(sorted(y, key=lambda x: x["id"])) for x, y in result.items()}
def get_uploads(month_start, month_end):
log("Querying UDD for uploads")
fields = (
- 'source',
- 'version',
- 'distribution',
- 'signed_by_name',
+ "source",
+ "version",
+ "distribution",
+ "signed_by_name",
)
- data = udd("""
+ data = udd(
+ """
SELECT
{fields}
FROM
@@ -204,29 +213,38 @@ def get_uploads(month_start, month_end):
to_timestamp(@{month_start}) AND to_timestamp(@{month_end})
ORDER BY
date
- """.format(**{
- 'fields': ', '.join(fields),
- 'sources': ', '.join("'{}'".format(x) for x in PROJECTS),
- 'month_start': month_start,
- 'month_end': month_end,
- }), fields)
+ """.format(
+ **{
+ "fields": ", ".join(fields),
+ "sources": ", ".join("'{}'".format(x) for x in PROJECTS),
+ "month_start": month_start,
+ "month_end": month_end,
+ }
+ ),
+ fields,
+ )
result = {}
for x in data:
- result.setdefault(x['source'], []).append(x)
+ result.setdefault(x["source"], []).append(x)
return result
def udd(query, fields):
- lines = subprocess.check_output("""
+ lines = subprocess.check_output(
+ """
echo "{}" | ssh quantz.debian.org psql --no-psqlrc service=udd
- """.format(query), shell=True)
+ """.format(
+ query
+ ),
+ shell=True,
+ )
data = []
for line in lines.splitlines()[2:]:
- split = line.decode('utf-8').split('|')
+ split = line.decode("utf-8").split("|")
if len(split) != len(fields):
continue
@@ -243,11 +261,11 @@ def get_commits(month_start, month_end):
def get_issues_yml(month_start, month_end):
- return commits(month_start, month_end, 'reproducible-notes', 'issues.yml')
+ return commits(month_start, month_end, "reproducible-notes", "issues.yml")
def get_packages_yml(month_start, month_end):
- return commits(month_start, month_end, 'reproducible-notes', 'packages.yml')
+ return commits(month_start, month_end, "reproducible-notes", "packages.yml")
def open_packages_yml(date):
@@ -255,8 +273,9 @@ def open_packages_yml(date):
"git show $(git rev-list -n1 --until @{0} "
"origin/master):packages.yml".format(date),
shell=True,
- cwd=sibling_repo_gitdir('reproducible-notes'),
- stdout=subprocess.PIPE).stdout
+ cwd=sibling_repo_gitdir("reproducible-notes"),
+ stdout=subprocess.PIPE,
+ ).stdout
def get_packages_stats(month_start, month_end):
@@ -276,52 +295,54 @@ def get_packages_stats(month_start, month_end):
}
-def commits(month_start, month_end, project, path='.'):
+def commits(month_start, month_end, project, path="."):
# Assume its in the parent dir
git_dir = sibling_repo_gitdir(project)
- subprocess.check_call(('git', 'fetch', 'origin'), cwd=git_dir)
-
- output = subprocess.check_output((
- 'git',
- 'log',
- 'origin/master',
- '--since', '@{}'.format(month_start),
- '--until', '@{}'.format(month_end),
- '--pretty=format:%an\t%h\t%s',
- '--no-merges',
- '--all',
- '--',
- path,
- ':(exclude,glob)_blog/posts/*.md',
- ), cwd=git_dir).decode('utf-8')
+ subprocess.check_call(("git", "fetch", "origin"), cwd=git_dir)
+
+ output = subprocess.check_output(
+ (
+ "git",
+ "log",
+ "origin/master",
+ "--since",
+ "@{}".format(month_start),
+ "--until",
+ "@{}".format(month_end),
+ "--pretty=format:%an\t%h\t%s",
+ "--no-merges",
+ "--all",
+ "--",
+ path,
+ ":(exclude,glob)_blog/posts/*.md",
+ ),
+ cwd=git_dir,
+ ).decode("utf-8")
result = collections.defaultdict(list)
for x in output.splitlines():
- author, sha, title = x.split('\t', 2)
+ author, sha, title = x.split("\t", 2)
- for pattern in (
- r'^dependabot$',
- ):
+ for pattern in (r"^dependabot$",):
if re.search(pattern, author) is not None:
continue
for pattern in (
- r'^--fix-deterministic$',
- r'^Add missing usertagged bugs$',
- r'^Remove archived bugs$',
- r'^Release .* to Debian .*$',
+ r"^--fix-deterministic$",
+ r"^Add missing usertagged bugs$",
+ r"^Remove archived bugs$",
+ r"^Release .* to Debian .*$",
):
if re.search(pattern, title) is not None:
continue
- result[author].append({
- 'sha': sha,
- 'title': title.replace('_', '\_'),
- })
+ result[author].append(
+ {"sha": sha, "title": title.replace("_", "\_"),}
+ )
return result
-if __name__ == '__main__':
+if __name__ == "__main__":
sys.exit(main(*sys.argv[1:]))
=====================================
bin/generate-draft.template
=====================================
@@ -105,4 +105,4 @@ If you are interested in contributing the Reproducible Builds project, please vi
---
-This month's report was written by {{ author }}. It was subsequently reviewed by a bunch of Reproducible Builds folks on IRC and the mailing list.
+This month's report was written by {{ authors }}. It was subsequently reviewed by a bunch of Reproducible Builds folks on IRC and the mailing list.
=====================================
citests.md
=====================================
@@ -4,7 +4,8 @@ title: Continuous tests
permalink: /citests/
order: 10
---
-## Continuous tests
+
+# Continuous tests
<div markdown="1">
The following projects are tested within the reproducible test infrastructure.
View it on GitLab: https://salsa.debian.org/reproducible-builds/reproducible-website/compare/aa7cfc47bae74cfc7fe64672c69052b41d570117...9fdd1cbdc52b6eb084b038a3a655f715b182964f
--
View it on GitLab: https://salsa.debian.org/reproducible-builds/reproducible-website/compare/aa7cfc47bae74cfc7fe64672c69052b41d570117...9fdd1cbdc52b6eb084b038a3a655f715b182964f
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.reproducible-builds.org/pipermail/rb-commits/attachments/20200102/2bb5dd78/attachment.htm>
More information about the rb-commits
mailing list