From 3e822ab6ca365e8691c1a2ab3fe8d5f8d392ba29 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Sun, 9 Apr 2023 23:21:23 +0200 Subject: [PATCH] measure validation progress more precisely Signed-off-by: Csaba Kiraly --- DAS/observer.py | 13 ++++++++----- DAS/simulator.py | 8 ++++---- DAS/validator.py | 42 +++++++++++++++++++++++++++++------------- 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/DAS/observer.py b/DAS/observer.py index 235ed60..beba4ad 100644 --- a/DAS/observer.py +++ b/DAS/observer.py @@ -46,16 +46,18 @@ class Observer: arrived = 0 expected = 0 ready = 0 + validatedall = 0 validated = 0 for val in validators: if val.amIproposer == 0: - (a, e) = val.checkStatus() + (a, e, v) = val.checkStatus() arrived += a expected += e if a == e: ready += 1 - validated += val.vpn - return (arrived, expected, ready, validated) + validatedall += val.vpn + validated += v + return (arrived, expected, ready, validatedall, validated) def getProgress(self, validators): """Calculate current simulation progress with different metrics. @@ -69,14 +71,15 @@ class Observer: but counts a validator only if its support node's all validators see all interesting segments TODO: add real per validator progress counter """ - arrived, expected, ready, validated = self.checkStatus(validators) + arrived, expected, ready, validatedall, validated = self.checkStatus(validators) missingSamples = expected - arrived sampleProgress = arrived / expected nodeProgress = ready / (len(validators)-1) validatorCnt = sum([v.vpn for v in validators[1:]]) + validatorAllProgress = validatedall / validatorCnt validatorProgress = validated / validatorCnt - return missingSamples, sampleProgress, nodeProgress, validatorProgress + return missingSamples, sampleProgress, nodeProgress, validatorAllProgress, validatorProgress def getTrafficStats(self, validators): """Summary statistics of traffic measurements in a timestep.""" diff --git a/DAS/simulator.py b/DAS/simulator.py index e3d7be4..c2a099f 100644 --- a/DAS/simulator.py +++ b/DAS/simulator.py @@ -167,7 +167,7 @@ class Simulator: def run(self): """It runs the main simulation until the block is available or it gets stucked.""" self.glob.checkRowsColumns(self.validators) - arrived, expected, ready, validated = self.glob.checkStatus(self.validators) + arrived, expected, ready, validatedall, validated = self.glob.checkStatus(self.validators) missingSamples = expected - arrived missingVector = [] progressVector = [] @@ -199,9 +199,9 @@ class Simulator: self.validators[i].updateStats() trafficStatsVector.append(trafficStats) - missingSamples, sampleProgress, nodeProgress, validatorProgress = self.glob.getProgress(self.validators) - self.logger.debug("step %d, arrived %0.02f %%, ready %0.02f %%, validated %0.02f %%" - % (steps, sampleProgress*100, nodeProgress*100, validatorProgress*100), extra=self.format) + missingSamples, sampleProgress, nodeProgress, validatorAllProgress, validatorProgress = self.glob.getProgress(self.validators) + self.logger.info("step %d, arrived %0.02f %%, ready %0.02f %%, validatedall %0.02f %%, , validated %0.02f %%" + % (steps, sampleProgress*100, nodeProgress*100, validatorAllProgress*100, validatorProgress*100), extra=self.format) cnS = "samples received" cnN = "nodes ready" diff --git a/DAS/validator.py b/DAS/validator.py index e8abba0..bbc7cba 100644 --- a/DAS/validator.py +++ b/DAS/validator.py @@ -69,8 +69,13 @@ class Validator: # random.seed(self.ID) self.nodeClass = 1 if (self.ID <= shape.numberNodes * shape.class1ratio) else 2 self.vpn = self.shape.vpn1 if (self.nodeClass == 1) else self.shape.vpn2 - self.rowIDs = rows if rows else unionOfSamples(range(self.shape.blockSize), self.shape.chi, self.vpn) - self.columnIDs = columns if columns else unionOfSamples(range(self.shape.blockSize), self.shape.chi, self.vpn) + self.vRowIDs = [] + self.vColumnIDs = [] + for i in range(self.vpn): + self.vRowIDs.append(set(rows[i*self.shape.chi:(i+1)*self.shape.chi]) if rows else set(random.sample(range(self.shape.blockSize), self.shape.chi))) + self.vColumnIDs.append(set(columns[i*self.shape.chi:(i+1)*self.shape.chi]) if columns else set(random.sample(range(self.shape.blockSize), self.shape.chi))) + self.rowIDs = set.union(*self.vRowIDs) + self.columnIDs = set.union(*self.vColumnIDs) self.rowNeighbors = collections.defaultdict(dict) self.columnNeighbors = collections.defaultdict(dict) @@ -469,16 +474,27 @@ class Validator: def checkStatus(self): """It checks how many expected/arrived samples are for each assigned row/column.""" - arrived = 0 - expected = 0 - for id in self.columnIDs: - line = self.getColumn(id) - arrived += line.count(1) - expected += len(line) - for id in self.rowIDs: - line = self.getRow(id) - arrived += line.count(1) - expected += len(line) + + def checkStatus(columnIDs, rowIDs): + arrived = 0 + expected = 0 + for id in columnIDs: + line = self.getColumn(id) + arrived += line.count(1) + expected += len(line) + for id in rowIDs: + line = self.getRow(id) + arrived += line.count(1) + expected += len(line) + return arrived, expected + + arrived, expected = checkStatus(self.columnIDs, self.rowIDs) self.logger.debug("status: %d / %d", arrived, expected, extra=self.format) - return (arrived, expected) + validated = 0 + for i in range(self.vpn): + a, e = checkStatus(self.vColumnIDs[i], self.vRowIDs[i]) + if a == e: + validated+=1 + + return arrived, expected, validated