[diffoscope] 02/04: progress: weigh future elements by the already-extracted sizes of previous elements
Ximin Luo
infinity0 at debian.org
Fri May 26 16:55:54 CEST 2017
This is an automated email from the git hooks/post-receive script.
infinity0 pushed a commit to branch experimental
in repository diffoscope.
commit c8c24a99dc2706424dfaf94988919b3d6f25eb9a
Author: Ximin Luo <infinity0 at debian.org>
Date: Thu May 25 18:03:11 2017 +0200
progress: weigh future elements by the already-extracted sizes of previous elements
---
diffoscope/comparators/directory.py | 2 +-
diffoscope/comparators/utils/container.py | 8 +--
diffoscope/main.py | 2 +-
diffoscope/progress.py | 100 ++++++++++++++++++++++--------
4 files changed, 80 insertions(+), 32 deletions(-)
diff --git a/diffoscope/comparators/directory.py b/diffoscope/comparators/directory.py
index 08fec63..5365aa9 100644
--- a/diffoscope/comparators/directory.py
+++ b/diffoscope/comparators/directory.py
@@ -200,8 +200,8 @@ class DirectoryContainer(Container):
for name in sorted(to_compare):
my_file, my_size = my_members[name]
other_file, other_size = other_members[name]
+ p.begin_step(my_size + other_size, msg=name)
yield my_file, other_file, name
- p.step(my_size + other_size, msg=name)
def compare(self, other, source=None):
from .utils.compare import compare_files
diff --git a/diffoscope/comparators/utils/container.py b/diffoscope/comparators/utils/container.py
index 7114dfb..b8107d9 100644
--- a/diffoscope/comparators/utils/container.py
+++ b/diffoscope/comparators/utils/container.py
@@ -128,7 +128,7 @@ class Container(object, metaclass=abc.ABCMeta):
my_member_name, (my_member, my_size) = my_members.popitem(last=False)
if my_member_name in other_members:
other_member, other_size = other_members.pop(my_member_name)
- p.step(my_size + other_size, msg=my_member.progress_name)
+ p.begin_step(my_size + other_size, msg=my_member.progress_name)
yield my_member, other_member, NO_COMMENT
else:
my_remainders[my_member_name] = (my_member, my_size)
@@ -141,16 +141,16 @@ class Container(object, metaclass=abc.ABCMeta):
other_member, other_size = other_members.pop(other_name)
comment = "Files similar despite different names" \
" (difference score: {})".format(score)
- p.step(my_size + other_size, msg=my_name)
+ p.begin_step(my_size + other_size, msg=my_name)
yield my_member, other_member, comment
if Config().new_file:
for my_member, my_size in my_members.values():
- p.step(my_size, msg=my_member.progress_name)
+ p.begin_step(my_size, msg=my_member.progress_name)
yield my_member, MissingFile('/dev/null', my_member), NO_COMMENT
for other_member, other_size in other_members.values():
- p.step(other_size, msg=other_member.progress_name)
+ p.begin_step(other_size, msg=other_member.progress_name)
yield MissingFile('/dev/null', other_member), other_member, NO_COMMENT
def compare(self, other, source=None, no_recurse=False):
diff --git a/diffoscope/main.py b/diffoscope/main.py
index baa9448..f97607d 100644
--- a/diffoscope/main.py
+++ b/diffoscope/main.py
@@ -290,7 +290,7 @@ def run_diffoscope(parsed_args):
set_path()
set_locale()
logger.debug('Starting comparison')
- with Progress(1, parsed_args.path1):
+ with Progress():
with profile('main', 'outputs'):
difference = compare_root_paths(
parsed_args.path1, parsed_args.path2)
diff --git a/diffoscope/progress.py b/diffoscope/progress.py
index 9d4dbdf..803bdd5 100644
--- a/diffoscope/progress.py
+++ b/diffoscope/progress.py
@@ -55,8 +55,7 @@ class ProgressManager(object):
self.reset()
def reset(self):
- self.total = 0
- self.current = 0
+ self.stack = []
self.observers = []
def setup(self, parsed_args):
@@ -87,47 +86,96 @@ class ProgressManager(object):
return log_handler
+ def push(self, progress):
+ assert not self.stack or self.stack[-1].is_active()
+ self.stack.append(progress)
+
+ def pop(self, progress):
+ x = self.stack.pop()
+ assert x is progress
+ if self.stack:
+ self.stack[-1].child_done(x.total)
+
def register(self, observer):
logger.debug("Registering %s as a progress observer", observer)
-
self.observers.append(observer)
- def step(self, delta=1, msg=""):
- delta = min(self.total - self.current, delta) # clamp
-
- self.current += delta
- for x in self.observers:
- x.notify(self.current, self.total, msg)
+ def update(self, msg):
+ if self.stack:
+ cur_estimates = None
+ for progress in reversed(self.stack):
+ cur_estimates = progress.estimates(cur_estimates)
+ current, total = cur_estimates
+ else:
+ current, total = 0, 1
- def new_total(self, delta, msg):
- self.total += delta
for x in self.observers:
- x.notify(self.current, self.total, msg)
+ x.notify(current, total, msg)
def finish(self):
for x in self.observers:
x.finish()
class Progress(object):
- def __init__(self, total, msg=""):
- self.current = 0
- self.total = total
-
- ProgressManager().new_total(total, msg)
+ def __init__(self, total=None):
+ self.done = []
+ self.current_steps = 0
+ self.current_child_steps_done = 0
+ if total:
+ self.total = total
+ else:
+ self.total = 1
+ self.begin_step(1)
def __enter__(self):
+ ProgressManager().push(self)
return self
def __exit__(self, exc_type, exc_value, exc_traceback):
- self.step(self.total - self.current)
-
- def step(self, delta=1, msg=""):
- delta = min(self.total - self.current, delta) # clamp
- if not delta:
- return
-
- self.current += delta
- ProgressManager().step(delta, msg)
+ self.maybe_end()
+ ProgressManager().pop(self)
+
+ def estimates(self, cur_child_estimate=None):
+ own_done = sum(pair[0] for pair in self.done)
+ children_done = sum(pair[1] for pair in self.done)
+ all_done = own_done + children_done
+
+ if self.current_steps:
+ if self.current_child_steps_done or cur_child_estimate:
+ # something is in-progress, the calculation is slightly more complex
+ cur_child_done, cur_child_total = cur_child_estimate or (0, 0)
+ own_done += self.current_steps
+ all_done += self.current_steps + self.current_child_steps_done + cur_child_done
+ # cost of what we expect will have been done, once the current in-progress
+ # step plus all of its children, have completed
+ expected_all_done = all_done + (cur_child_total - cur_child_done)
+ return all_done, int(float(self.total) / own_done * expected_all_done)
+ else:
+ pass # nothing in progress
+ else:
+ # nothing in progress
+ assert not cur_child_estimate
+
+ # weigh self.total by (all_done/own_done)
+ return all_done, int(float(self.total) / own_done * all_done)
+
+ def is_active(self):
+ return self.current_steps
+
+ def maybe_end(self, msg=""):
+ if self.is_active():
+ self.done += [(self.current_steps, self.current_child_steps_done)]
+ self.current_steps = 0
+ self.current_child_steps_done = 0
+ ProgressManager().update(msg)
+
+ def begin_step(self, step, msg=""):
+ assert step > 0
+ self.maybe_end(msg)
+ self.current_steps = step
+
+ def child_done(self, total):
+ self.current_child_steps_done += total
class ProgressBar(object):
def __init__(self):
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/diffoscope.git
More information about the diffoscope
mailing list