[diffoscope] 01/01: wip
Chris Lamb
chris at chris-lamb.co.uk
Mon Jan 9 17:31:24 CET 2017
This is an automated email from the git hooks/post-receive script.
lamby pushed a commit to branch lamby/wip/progress-bar
in repository diffoscope.
commit 5c7b4651d6d7f805f6fb5de933c2bc1fac1e90ab
Author: Chris Lamb <lamby at debian.org>
Date: Mon Sep 26 14:54:33 2016 +0200
wip
Signed-off-by: Chris Lamb <lamby at debian.org>
---
debian/rules | 3 +-
diffoscope/comparators/directory.py | 26 ++++---
diffoscope/comparators/utils.py | 40 ++++++-----
diffoscope/main.py | 15 +++-
diffoscope/progress.py | 132 ++++++++++++++++++++++++++++++++++++
5 files changed, 185 insertions(+), 31 deletions(-)
diff --git a/debian/rules b/debian/rules
index 9eaf836..90e5761 100755
--- a/debian/rules
+++ b/debian/rules
@@ -35,7 +35,8 @@ override_dh_python3:
--recommends=rpm-python \
--recommends=tlsh \
--recommends=guestfs \
- --recommends=argcomplete
+ --recommends=argcomplete \
+ --recommends=progressbar
override_dh_gencontrol:
TOOLS="$$(bin/diffoscope --list-tools=debian | awk -F': ' '/Available-in-Debian-packages/ { print $$2 }' | \
diff --git a/diffoscope/comparators/directory.py b/diffoscope/comparators/directory.py
index cec01cb..05578bc 100644
--- a/diffoscope/comparators/directory.py
+++ b/diffoscope/comparators/directory.py
@@ -25,6 +25,7 @@ import diffoscope.comparators
from diffoscope import logger, tool_required
from diffoscope.exc import RequiredToolNotFound
+from diffoscope.progress import ProgressContext
from diffoscope.difference import Difference
from diffoscope.comparators.utils import Container, Command
from diffoscope.comparators.binary import FilesystemFile
@@ -151,17 +152,20 @@ class FilesystemDirectory(object):
other_container = DirectoryContainer(other)
my_names = my_container.get_member_names()
other_names = other_container.get_member_names()
- for name in sorted(set(my_names).intersection(other_names)):
- my_file = my_container.get_member(name)
- other_file = other_container.get_member(name)
- inner_difference = diffoscope.comparators.compare_files(
- my_file, other_file, source=name)
- meta_differences = compare_meta(my_file.name, other_file.name)
- if meta_differences and not inner_difference:
- inner_difference = Difference(None, my_file.path, other_file.path)
- if inner_difference:
- inner_difference.add_details(meta_differences)
- differences.append(inner_difference)
+ to_compare = set(my_names).intersection(other_names)
+ with ProgressContext(len(to_compare)) as p:
+ for name in sorted(to_compare):
+ my_file = my_container.get_member(name)
+ other_file = other_container.get_member(name)
+ inner_difference = diffoscope.comparators.compare_files(
+ my_file, other_file, source=name)
+ meta_differences = compare_meta(my_file.name, other_file.name)
+ if meta_differences and not inner_difference:
+ inner_difference = Difference(None, my_file.path, other_file.path)
+ if inner_difference:
+ inner_difference.add_details(meta_differences)
+ differences.append(inner_difference)
+ p.step()
if not differences:
return None
difference = Difference(None, self.path, other.path, source)
diff --git a/diffoscope/comparators/utils.py b/diffoscope/comparators/utils.py
index 95bff29..06850b2 100644
--- a/diffoscope/comparators/utils.py
+++ b/diffoscope/comparators/utils.py
@@ -31,6 +31,7 @@ import diffoscope.comparators
from diffoscope import logger, tool_required, get_temporary_directory
from diffoscope.config import Config
+from diffoscope.progress import ProgressContext
from diffoscope.comparators.binary import File, NonExistingFile
@@ -201,22 +202,29 @@ class Container(object, metaclass=abc.ABCMeta):
my_members = self.get_members()
my_reminders = collections.OrderedDict()
other_members = other.get_members()
- # keep it sorted like my members
- while my_members:
- my_member_name, my_member = my_members.popitem(last=False)
- if my_member_name in other_members:
- yield my_member, other_members.pop(my_member_name), NO_COMMENT
- else:
- my_reminders[my_member_name] = my_member
- my_members = my_reminders
- for my_name, other_name, score in diffoscope.comparators.perform_fuzzy_matching(my_members, other_members):
- comment = 'Files similar despite different names (difference score: %d)' % score
- yield my_members.pop(my_name), other_members.pop(other_name), comment
- if Config.general.new_file:
- for my_member in my_members.values():
- yield my_member, NonExistingFile('/dev/null', my_member), NO_COMMENT
- for other_member in other_members.values():
- yield NonExistingFile('/dev/null', other_member), other_member, NO_COMMENT
+
+ with ProgressContext(max(len(my_members), len(other_members))) as p:
+ # keep it sorted like my member/
+ while my_members:
+ my_member_name, my_member = my_members.popitem(last=False)
+ if my_member_name in other_members:
+ yield my_member, other_members.pop(my_member_name), NO_COMMENT
+ p.step()
+ else:
+ my_reminders[my_member_name] = my_member
+ my_members = my_reminders
+ for my_name, other_name, score in diffoscope.comparators.perform_fuzzy_matching(my_members, other_members):
+ comment = 'Files similar despite different names (difference score: %d)' % score
+ yield my_members.pop(my_name), other_members.pop(other_name), comment
+ p.step()
+ p.step()
+ if Config.general.new_file:
+ for my_member in my_members.values():
+ yield my_member, NonExistingFile('/dev/null', my_member), NO_COMMENT
+ p.step()
+ for other_member in other_members.values():
+ yield NonExistingFile('/dev/null', other_member), other_member, NO_COMMENT
+ p.step()
def compare(self, other, source=None):
return itertools.starmap(diffoscope.comparators.compare_commented_files, self.comparisons(other))
diff --git a/diffoscope/main.py b/diffoscope/main.py
index ff8382f..9431429 100644
--- a/diffoscope/main.py
+++ b/diffoscope/main.py
@@ -33,6 +33,7 @@ import diffoscope.comparators
from diffoscope import logger, VERSION, set_locale, clean_all_temp_files
from diffoscope.exc import RequiredToolNotFound
from diffoscope.config import Config
+from diffoscope.progress import Progress, ProgressContext
from diffoscope.presenters.html import output_html, output_html_directory, \
JQUERY_SYSTEM_LOCATIONS
from diffoscope.presenters.text import output_text
@@ -124,6 +125,12 @@ def create_parser():
400, 20)
parser.add_argument('--new-file', dest='new_file', action='store_true',
help='treat absent files as empty')
+ parser.add_argument('--progress', dest='progress', action='store_const',
+ const=True, help='show an (approximate) progress bar')
+ parser.add_argument('--no-progress', dest='progress', action='store_const',
+ const=False, help='do not show an (approximate) progress bar')
+ parser.add_argument('--status-fd', dest='status_fd', metavar='n', type=int,
+ help='Send machine-readable status to file descriptor n')
parser.add_argument('--css', metavar='url', dest='css_url',
help='link to an extra CSS for the HTML report')
parser.add_argument('--jquery', metavar='url', dest='jquery_url',
@@ -196,7 +203,6 @@ def maybe_set_limit(config, parsed_args, key):
elif parsed_args.no_default_limits:
setattr(config, key, float("inf"))
-
def run_diffoscope(parsed_args):
if not tlsh and Config.general.fuzzy_threshold != parsed_args.fuzzy_threshold:
logger.warning('Fuzzy-matching is currently disabled as the "tlsh" module is unavailable.')
@@ -213,8 +219,11 @@ def run_diffoscope(parsed_args):
logger.setLevel(logging.DEBUG)
set_locale()
logger.debug('Starting comparison')
- difference = diffoscope.comparators.compare_root_paths(
- parsed_args.path1, parsed_args.path2)
+ Progress().setup(parsed_args)
+ with ProgressContext(2):
+ difference = diffoscope.comparators.compare_root_paths(
+ parsed_args.path1, parsed_args.path2)
+ Progress().finish()
if difference:
# no output desired? print text
if not any((parsed_args.text_output, parsed_args.html_output, parsed_args.html_output_directory)):
diff --git a/diffoscope/progress.py b/diffoscope/progress.py
new file mode 100644
index 0000000..db2d64e
--- /dev/null
+++ b/diffoscope/progress.py
@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2016 Chris Lamb <lamby at debian.org>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+
+class Progress(object):
+ _singleton = {}
+
+ def __init__(self):
+ self.__dict__ = self._singleton
+
+ if not self._singleton:
+ self.total = 0
+ self.current = 0
+ self.observers = []
+
+ def setup(self, parsed_args):
+ # Show progress bar if user explicitly asked for it, otherwise show if
+ # STDOUT is a tty.
+ if parsed_args.progress or \
+ (parsed_args.progress is None and sys.stdout.isatty()):
+ try:
+ self.register(ProgressBar())
+ except ImportError:
+ # User asked for bar, so show them the error
+ if parsed_args.progress:
+ raise
+
+ if parsed_args.status_fd:
+ self.register(StatusFD(parsed_args.status_fd))
+
+ ##
+
+ def register(self, observer):
+ self.observers.append(observer)
+
+ def updateTotal(self, delta):
+ self.total += delta
+
+ for x in self.observers:
+ x.total(self.total)
+
+ def finish(self):
+ for x in self.observers:
+ x.finish()
+
+ def processed(self, delta=1):
+ self.current += delta
+
+ for x in self.observers:
+ x.processed(self.current)
+
+class ProgressContext(object):
+ def __init__(self, total):
+ self.total = total
+ self.current = 0
+
+ Progress().updateTotal(total)
+
+ def step(self):
+ if self.current == self.total:
+ return
+ Progress().processed(1)
+ self.current += 1
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_traceback):
+ for _ in range(self.total - self.current):
+ self.step()
+
+##
+
+class ProgressBar(object):
+ def __init__(self):
+ import progressbar
+
+ self.bar = progressbar.ProgressBar(widgets=(
+ progressbar.ETA(),
+ ' ',
+ progressbar.Bar(),
+ ' ',
+ progressbar.Percentage(),
+ ' ',
+ progressbar.SimpleProgress(),
+ ))
+ self.bar.start()
+
+ def total(self, total):
+ self.bar.maxval = total
+ self.bar.update()
+
+ def processed(self, val):
+ self.bar.currval = val
+ self.bar.update()
+
+ def finish(self):
+ self.bar.finish()
+
+class StatusFD(object):
+ def __init__(self, fileno):
+ self.fileobj = os.fdopen(fileno, 'w')
+
+ def total(self, total):
+ self._update()
+
+ def processed(self, val):
+ self._update()
+
+ def finish(self):
+ self._update()
+
+ def _update(self):
+ print('{0.current}\t{0.total}'.format(Progress(), file=self.fileobj))
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/diffoscope.git
More information about the diffoscope
mailing list