import copy, random, hashlib from distributions import normal_distribution import networksim from voting_strategy import vote import math NUM_VALIDATORS = 20 BLKTIME = 40 BY_CHAIN = False NETSPLITS = False GENESIS_STATE = 0 logging_level = 0 def log(s, lvl): if logging_level >= lvl: print(s) class Signature(): def __init__(self, signer, probs, finalized_state, sign_from): self.signer = signer # List of maps from block hash to probability self.probs = probs # Hash of the signature (for db storage purposes) self.hash = random.randrange(10**14) # Finalized state self.finalized_state = finalized_state # Finalized height self.sign_from = sign_from class Block(): def __init__(self, maker, height): # The producer of the block self.maker = maker # The height of the block self.height = height # Hash of the signature (for db storage purposes) self.hash = random.randrange(10**20) + 10**21 + 10**23 * self.height def state_transition(state, block): return state if block is None else (state ** 3 + block.hash ** 5) % 10**40 # A validator class Validator(): def __init__(self, pos, network): # Map from height to {node_id: latest_opinion} self.received_signatures = [] # List of received blocks self.received_blocks = [] # Own probability estimates self.probs = [] # All objects that this validator has received; basically a database self.received_objects = {} self.time_received = {} # The validator's ID, and its position in the queue self.pos = self.id = pos # This validator's offset from the clock self.time_offset = max(normal_distribution(200, 100)(), 0) # The highest height that this validator has seen self.max_height = 0 self.head = None # The last time the validator made a block self.last_time_made_block = -999999999999 # The validator's hash chain self.finalized_hashes = [] # Finalized states self.finalized_states = [] # The highest height that the validator has finalized self.max_finalized_height = -1 # The network object self.network = network # Last time signed self.last_time_signed = 0 # Next neight to mine self.next_height = self.pos def get_time(self): return self.network.time + self.time_offset def sign(self, block): # Initialize the probability array, the core of the signature best_guesses = [None] * len(self.received_blocks) sign_from = max(0, self.max_finalized_height - 30) for i, b in list(enumerate(self.received_blocks))[sign_from:]: if self.received_blocks[i] is None: time_delta = self.get_time() - BLKTIME * i my_opinion = 0.35 / (1 + max(0, time_delta) * 0.3 / BLKTIME) + 0.14 else: time_delta = self.time_received[b.hash] - BLKTIME * i my_opinion = 0.7 / (1 + abs(time_delta) * 0.3 / BLKTIME) + 0.15 if my_opinion == 0.5: my_opinion = 0.5001 votes = self.received_signatures[i].values() if i < len(self.received_signatures) else [] votes += [my_opinion] * (NUM_VALIDATORS - len(votes)) best_guesses[i] = min(vote(votes), 1 if self.received_blocks[i] is not None else my_opinion) if best_guesses[i] > 0.9999: while len(self.finalized_hashes) <= i: self.finalized_hashes.append(None) self.finalized_hashes[i] = self.received_blocks[i].hash elif best_guesses[i] < 0.0001: while len(self.finalized_hashes) <= i: self.finalized_hashes.append(None) self.finalized_hashes[i] = False while self.max_finalized_height < len(self.finalized_hashes) - 1 \ and self.finalized_hashes[self.max_finalized_height + 1] is not None: self.max_finalized_height += 1 last_state = self.finalized_states[-1] if len(self.finalized_states) else GENESIS_STATE self.finalized_states.append(state_transition(last_state, self.received_blocks[self.max_finalized_height])) self.probs = self.probs[:sign_from] + best_guesses[sign_from:] log('Making signature: %r' % self.probs[-10:], lvl=1) sign_from_state = self.finalized_states[sign_from - 1] if sign_from > 0 else GENESIS_STATE s = Signature(self.pos, self.probs[sign_from:], sign_from_state, sign_from) all_signatures.append(s) return s def on_receive(self, obj): # Ignore objects that we already know about if obj.hash in self.received_objects: return # When receiving a block if isinstance(obj, Block): log('received block: %d %d' % (obj.height, obj.hash), lvl=2) while len(self.received_blocks) <= obj.height: self.received_blocks.append(None) self.received_blocks[obj.height] = obj self.time_received[obj.hash] = self.get_time() # If we have not yet produced a signature at this height, do so now s = self.sign(obj) self.network.broadcast(self, s) self.on_receive(s) self.network.broadcast(self, obj) # When receiving a signature elif isinstance(obj, Signature): while len(self.received_signatures) <= len(obj.probs) + obj.sign_from: self.received_signatures.append({}) for i, p in enumerate(obj.probs): self.received_signatures[i + obj.sign_from][obj.signer] = p self.network.broadcast(self, obj) # Received an object request, respond if we have it elif isinstance(obj, ObjRequest): if obj.ask_hash in self.received_objects: self.network.direct_send(obj.sender_id, ObjResponse( self.received_objects[obj.ask_hash])) # Received an object response, add to database elif isinstance(obj, ObjResponse): self.received_objects[obj.obj.hash] = obj.obj self.received_objects[obj.hash] = obj self.time_received[obj.hash] = self.get_time() # Run every tick def tick(self): mytime = self.get_time() target_time = BLKTIME * self.next_height if mytime >= target_time: o = Block(self.pos, self.next_height) self.next_height += NUM_VALIDATORS log('making block: %d %d' % (o.height, o.hash), lvl=1) if random.random() < 0.9: self.network.broadcast(self, o) while len(self.received_blocks) <= o.height: self.received_blocks.append(None) self.received_blocks[o.height] = o self.received_objects[o.hash] = o self.time_received[o.hash] = mytime return o validator_list = [] future = {} discarded = {} finalized_blocks = {} all_signatures = [] now = [0] def who_heard_of(h, n): o = '' for x in n.agents: o += '1' if h in x.received_objects else '0' return o def get_opinions(n): o = [] maxheight = 0 for x in n.agents: maxheight = max(maxheight, len(x.probs)) for h in range(maxheight): p = '' q = '' for x in n.agents: if len(x.probs) <= h: p += '_' elif x.probs[h] < 0.5: p += str(int(5 - math.log(x.probs[h]) / math.log(0.0001) * 4) if x.probs[h] > 0.0001 else 0) elif x.probs[h] >= 0.5: p += str(int(5 + math.log(1 - x.probs[h]) / math.log(0.0001) * 4) if x.probs[h] < 0.9999 else 9) q += 'n' if len(x.received_blocks) <= h or x.received_blocks[h] is None else 'y' o.append((h, p, q)) return o def get_finalization_heights(n): o = [] for x in n.agents: o.append(x.max_finalized_height) return o # Check how often blocks that are assigned particular probabilities of # finalization by our algorithm are actually finalized def calibrate(finalized_hashes): thresholds = [0, 0.25, 0.5, 0.75] + [1 - 0.5**k for k in range(10)] + [1] signed = [0] * (len(thresholds) - 1) _finalized = [0] * (len(thresholds) - 1) _discarded = [0] * (len(thresholds) - 1) for s in all_signatures: for i, prob in enumerate(s.probs): if i + s.sign_from >= len(finalized_hashes): continue actual_result = 1 if finalized_hashes[i + s.sign_from] else 0 index = 0 while prob > thresholds[index + 1]: index += 1 signed[index] += 1 if actual_result == 1: _finalized[index] += 1 elif actual_result == 0: _discarded[index] += 1 for i in range(len(thresholds) - 1): if _finalized[i] + _discarded[i]: print 'Probability from %f to %f: %f' % (thresholds[i], thresholds[i+1], _finalized[i] * 1.0 / (_finalized[i] + _discarded[i])) print 'Percentage of blocks nonempty: %f%%' % (len([x for x in finalized_hashes if x]) * 100.0 / len(finalized_hashes)) def run(steps=4000): n = networksim.NetworkSimulator() for i in range(NUM_VALIDATORS): n.agents.append(Validator(i, n)) n.generate_peers() while len(all_signatures): all_signatures.pop() for x in future.keys(): del future[x] for x in finalized_blocks.keys(): del finalized_blocks[x] for x in discarded.keys(): del discarded[x] for i in range(steps): n.tick() if i % 250 == 0: print get_opinions(n)[-60:] finalized0 = [(v.max_finalized_height, v.finalized_hashes) for v in n.agents] finalized = sorted(finalized0, key=lambda x: len(x[1])) for j in range(len(n.agents) - 1): for k in range(len(finalized[j][1])): if finalized[j][1][k] is not None and finalized[j+1][1][k] is not None: if finalized[j][1][k] != finalized[j+1][1][k]: print finalized[j] print finalized[j+1] raise Exception("Finalization mismatch: %r %r" % (finalized[j][1][k], finalized[j+1][1][k])) print 'Finalized status: %r' % [x[0] for x in finalized0] if i == 10000 and NETSPLITS: print "###########################################################" print "Knocking off 20% of the network!!!!!" print "###########################################################" n.knock_offline_random(NUM_VALIDATORS // 5) if i == 20000 and NETSPLITS: print "###########################################################" print "Simluating a netsplit!!!!!" print "###########################################################" n.generate_peers() n.partition() if i == 30000 and NETSPLITS: print "###########################################################" print "Network health back to normal!" print "###########################################################" n.generate_peers() calibrate(n.agents[0].finalized_hashes)