Random beacon v1 (#24)

* Implement beacon verification and handling module

* Create beacon tests and fix encountered problems

* Refactor tests

* Add mixed happy/unhappy test

* Clean unused import

* Add requirements.txt

* Add beacon to package

* Resolve relative import

* Fmt

* Refactor BeaconHandler -> RandomBeaconHandler
Remove unused verification calls

* Change view bytes encoding
Extract generating private key

* Bring back old trusty carnot

* Added beaconized carnot module

* Implement flat overlay

* Refactor overlay next leader

* Implement beaconized carnot

* Fill proposed block on beaconized carnot

* Sketch and update for testing purposes

* Step up beaconized test

* Fix missing leader selection

* Fix random beacon test

* Use recovery mode for random beacon initialization

* Expose entropy as constructor parameter
This commit is contained in:
Daniel Sanchez 2023-05-18 18:29:28 +02:00 committed by GitHub
parent c2e05a48c5
commit 5a169039b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 511 additions and 15 deletions

View File

@ -13,5 +13,7 @@ jobs:
with: with:
# Semantic version range syntax or exact version of a Python version # Semantic version range syntax or exact version of a Python version
python-version: '3.x' python-version: '3.x'
- name: Install dependencies
run: pip install -r requirements.txt
- name: Run tests - name: Run tests
run: cd carnot && python -m unittest run: cd carnot && python -m unittest

View File

@ -1 +1,2 @@
from .carnot import * from .carnot import *
from .beacon import RandomBeaconHandler

90
carnot/beacon.py Normal file
View File

@ -0,0 +1,90 @@
# typing imports
from dataclasses import dataclass
from random import randint
from typing import TypeAlias
# carnot imports
# lib imports
from blspy import PrivateKey, Util, PopSchemeMPL, G2Element, G1Element
# stdlib imports
from hashlib import sha256
View: TypeAlias = int
Beacon: TypeAlias = bytes
Proof: TypeAlias = bytes # For now this is gonna be a public key, in future research we may pivot to zk proofs.
def generate_random_sk() -> PrivateKey:
seed = bytes([randint(0, 255) for _ in range(32)])
return PopSchemeMPL.key_gen(seed)
@dataclass
class RandomBeacon:
version: int
context: View
entropy: Beacon
# TODO: Just the happy path beacons owns a proof, we can set the proof to empty bytes for now.
# Probably we should separate this into two kinds of beacons and group them under a single type later on.
proof: Proof
class NormalMode:
@staticmethod
def verify(beacon: RandomBeacon) -> bool:
"""
:param proof: BLS signature
:param beacon: Beacon is signature for current view
:param view: View to verify beacon upon
:return:
"""
# TODO: Actually verify that the message is propoerly signed
sig = G2Element.from_bytes(beacon.entropy)
proof = G1Element.from_bytes(beacon.proof)
return PopSchemeMPL.verify(proof, Util.hash256(str(beacon.context).encode()), sig)
@staticmethod
def generate_beacon(private_key: PrivateKey, view: View) -> Beacon:
return bytes(PopSchemeMPL.sign(private_key, Util.hash256(str(view).encode())))
class RecoveryMode:
@staticmethod
def verify(last_beacon: RandomBeacon, beacon: RandomBeacon) -> bool:
"""
:param last_beacon: Unhappy -> last working beacon (signature), Happy -> Hash of previous beacon and next view number
:param beacon:
:param view:
:return:
"""
b = sha256(last_beacon.entropy + str(beacon.context).encode()).digest()
return b == beacon.entropy
@staticmethod
def generate_beacon(last_beacon: Beacon, view: View) -> Beacon:
return sha256(last_beacon + str(view).encode()).digest()
class RandomBeaconHandler:
def __init__(self, beacon: RandomBeacon):
"""
:param beacon: Beacon should be initialized with either the last known working beacon from recovery.
Or the hash of the genesis block in case of first consensus round.
:return: Self
"""
self.last_beacon: RandomBeacon = beacon
def verify_happy(self, new_beacon: RandomBeacon) -> bool:
if NormalMode.verify(new_beacon):
self.last_beacon = new_beacon
return True
return False
def verify_unhappy(self, new_beacon: RandomBeacon) -> bool:
if RecoveryMode.verify(self.last_beacon, new_beacon):
self.last_beacon = new_beacon
return True
return False

View File

@ -0,0 +1,85 @@
from typing import Set
from carnot import Carnot, Block, TimeoutQc, Vote, Event, Send, Quorum
from beacon import *
from overlay import EntropyOverlay
@dataclass
class BeaconizedBlock(Block):
beacon: RandomBeacon
class BeaconizedCarnot(Carnot):
def __init__(self, sk: PrivateKey, overlay: EntropyOverlay, entropy: bytes = b""):
self.sk = sk
self.pk = bytes(self.sk.get_g1())
self.random_beacon = RandomBeaconHandler(
RandomBeacon(
version=0,
context=-1,
entropy=RecoveryMode.generate_beacon(entropy, -1),
proof=self.pk
)
)
overlay.set_entropy(self.random_beacon.last_beacon.entropy)
super().__init__(self.pk, overlay=overlay)
def approve_block(self, block: BeaconizedBlock, votes: Set[Vote]) -> Event:
assert block.id() in self.safe_blocks
assert len(votes) == self.overlay.super_majority_threshold(self.id)
assert all(self.overlay.is_member_of_child_committee(self.id, vote.voter) for vote in votes)
assert all(vote.block == block.id() for vote in votes)
assert self.highest_voted_view < block.view
if self.overlay.is_member_of_root_committee(self.id):
qc = self.build_qc(block.view, block, None)
else:
qc = None
vote: Vote = Vote(
block=block.id(),
voter=self.id,
view=block.view,
qc=qc
)
self.highest_voted_view = max(self.highest_voted_view, block.view)
# root members send votes to next leader, we update our beacon first
if self.overlay.is_member_of_root_committee(self.id):
self.random_beacon.verify_happy(block.beacon)
self.overlay.set_entropy(self.random_beacon.last_beacon.entropy)
return Send(to=self.overlay.leader(), payload=vote)
# otherwise we send to the parent committee and update the beacon second
return_event = Send(to=self.overlay.parent_committee(self.id), payload=vote)
self.random_beacon.verify_happy(block.beacon)
self.overlay.set_entropy(self.random_beacon.last_beacon.entropy)
return return_event
def receive_timeout_qc(self, timeout_qc: TimeoutQc):
super().receive_timeout_qc(timeout_qc)
if timeout_qc.view < self.current_view:
return
entropy = RecoveryMode.generate_beacon(self.random_beacon.last_beacon.entropy, timeout_qc.view)
new_beacon = RandomBeacon(
version=0,
context=self.current_view,
entropy=entropy,
proof=b""
)
self.random_beacon.verify_unhappy(new_beacon)
self.overlay.set_entropy(self.random_beacon.last_beacon.entropy)
def propose_block(self, view: View, quorum: Quorum) -> Event:
beacon = RandomBeacon(
version=0,
context=self.current_view,
entropy=NormalMode.generate_beacon(self.sk, self.current_view),
proof=self.pk
)
event: Event = super().propose_block(view, quorum)
block = event.payload
block = BeaconizedBlock(view=block.view, qc=block.qc, _id=block._id, beacon=beacon)
event.payload = block
return event

View File

@ -36,9 +36,8 @@
# Please note this is still a work in progress # Please note this is still a work in progress
from dataclasses import dataclass from dataclasses import dataclass
from typing import TypeAlias, List, Set, Self, Optional, Dict, FrozenSet from typing import TypeAlias, List, Set, Self, Optional, Dict
from abc import abstractmethod from abc import abstractmethod, ABC
Id: TypeAlias = bytes Id: TypeAlias = bytes
View: TypeAlias = int View: TypeAlias = int
@ -156,6 +155,7 @@ class Send:
Event: TypeAlias = BroadCast | Send Event: TypeAlias = BroadCast | Send
class Overlay: class Overlay:
""" """
Overlay structure for a View Overlay structure for a View
@ -167,16 +167,20 @@ class Overlay:
:param _id: Node id to be checked :param _id: Node id to be checked
:return: true if node is the leader of the current view :return: true if node is the leader of the current view
""" """
pass return _id == self.leader()
@abstractmethod @abstractmethod
def leader(self, view: View) -> Id: def leader(self) -> Id:
""" """
:param view: :param view:
:return: the leader Id of the specified view :return: the leader Id of the specified view
""" """
pass pass
@abstractmethod
def next_leader(self) -> Id:
pass
@abstractmethod @abstractmethod
def is_member_of_leaf_committee(self, _id: Id) -> bool: def is_member_of_leaf_committee(self, _id: Id) -> bool:
""" """
@ -248,7 +252,7 @@ def download(view) -> Block:
class Carnot: class Carnot:
def __init__(self, _id: Id): def __init__(self, _id: Id, overlay=Overlay()):
self.id: Id = _id self.id: Id = _id
# Current View counter # Current View counter
# It is the view currently being processed by the node. Once a Qc is received, the view is considered completed # It is the view currently being processed by the node. Once a Qc is received, the view is considered completed
@ -261,9 +265,9 @@ class Carnot:
# Validated blocks with their validated QCs are included here. If commit conditions are satisfied for # Validated blocks with their validated QCs are included here. If commit conditions are satisfied for
# each one of these blocks it will be committed. # each one of these blocks it will be committed.
self.safe_blocks: Dict[Id, Block] = dict() self.safe_blocks: Dict[Id, Block] = dict()
# Whether the node timeed out in the last view and corresponding qc # Whether the node time out in the last view and corresponding qc
self.last_view_timeout_qc: Optional[TimeoutQc] = None self.last_view_timeout_qc: Optional[TimeoutQc] = None
self.overlay: Overlay = Overlay() # TODO: integrate overlay self.overlay: Overlay = overlay
# Committing conditions for a block # Committing conditions for a block
@ -394,7 +398,7 @@ class Carnot:
assert self.highest_voted_view == vote.view assert self.highest_voted_view == vote.view
if self.overlay.is_member_of_root_committee(self.id): if self.overlay.is_member_of_root_committee(self.id):
return Send(to=self.overlay.leader(self.current_view + 1), payload=vote) return Send(to=self.overlay.next_leader(), payload=vote)
def forward_new_view(self, msg: NewView) -> Optional[Event]: def forward_new_view(self, msg: NewView) -> Optional[Event]:
assert msg.view == self.current_view assert msg.view == self.current_view
@ -403,7 +407,7 @@ class Carnot:
assert self.highest_voted_view == msg.view assert self.highest_voted_view == msg.view
if self.overlay.is_member_of_root_committee(self.id): if self.overlay.is_member_of_root_committee(self.id):
return Send(to=self.overlay.leader(self.current_view + 1), payload=msg) return Send(to=self.overlay.next_leader(), payload=msg)
def build_qc(self, view: View, block: Optional[Block], new_views: Optional[Set[NewView]]) -> Qc: def build_qc(self, view: View, block: Optional[Block], new_views: Optional[Set[NewView]]) -> Qc:
# unhappy path # unhappy path
@ -475,7 +479,7 @@ class Carnot:
# A node must change its view after making sure it has the high_Qc or last_timeout_view_qc # A node must change its view after making sure it has the high_Qc or last_timeout_view_qc
# from previous view. # from previous view.
return ( return (
self.current_view == self.local_high_qc.view + 1 or self.current_view == self.local_high_qc.view + 1 or
self.current_view == self.last_view_timeout_qc.view + 1 or self.current_view == self.last_view_timeout_qc.view + 1 or
(self.current_view == self.last_view_timeout_qc.view) (self.current_view == self.last_view_timeout_qc.view)
) )
@ -553,7 +557,7 @@ class Carnot:
self.highest_voted_view = max(self.highest_voted_view, view) self.highest_voted_view = max(self.highest_voted_view, view)
if self.overlay.is_member_of_root_committee(self.id): if self.overlay.is_member_of_root_committee(self.id):
return Send(payload=timeout_msg, to=[self.overlay.leader(self.current_view + 1)]) return Send(payload=timeout_msg, to=[self.overlay.next_leader()])
return Send(payload=timeout_msg, to=self.overlay.parent_committee(self.id)) return Send(payload=timeout_msg, to=self.overlay.parent_committee(self.id))
@ -569,7 +573,7 @@ class Carnot:
self.update_timeout_qc(timeout_qc) self.update_timeout_qc(timeout_qc)
# Update our current view and go ahead with the next step # Update our current view and go ahead with the next step
self.update_current_view_from_timeout_qc(timeout_qc) self.update_current_view_from_timeout_qc(timeout_qc)
self.rebuild_overlay_from_timeout_qc(timeout_qc) # self.rebuild_overlay_from_timeout_qc(timeout_qc)
def rebuild_overlay_from_timeout_qc(self, timeout_qc: TimeoutQc): def rebuild_overlay_from_timeout_qc(self, timeout_qc: TimeoutQc):
assert timeout_qc.view >= self.current_view assert timeout_qc.view >= self.current_view

56
carnot/overlay.py Normal file
View File

@ -0,0 +1,56 @@
import random
from abc import abstractmethod
from typing import Set, Optional, List
from carnot import Overlay, Id, Committee, View
class EntropyOverlay(Overlay):
@abstractmethod
def set_entropy(self, entropy: bytes):
pass
class FlatOverlay(EntropyOverlay):
def set_entropy(self, entropy: bytes):
self.entropy = entropy
def is_leader(self, _id: Id):
return _id == self.leader()
def leader(self) -> Id:
random.seed(a=self.entropy, version=2)
return random.choice(self.nodes)
def is_member_of_leaf_committee(self, _id: Id) -> bool:
return True
def is_member_of_root_committee(self, _id: Id) -> bool:
return True
def is_member_of_child_committee(self, parent: Id, child: Id) -> bool:
return False
def parent_committee(self, _id: Id) -> Optional[Committee]:
return None
def leaf_committees(self) -> Set[Committee]:
return {frozenset(self.nodes)}
def root_committee(self) -> Committee:
return set(self.nodes)
def is_child_of_root_committee(self, _id: Id) -> bool:
return True
def leader_super_majority_threshold(self, _id: Id) -> int:
return ((len(self.nodes) * 2) // 3) + 1
def super_majority_threshold(self, _id: Id) -> int:
return 0
def __init__(self, nodes: List[Id]):
self.nodes = nodes
self.entropy = None

View File

@ -0,0 +1,73 @@
from typing import Tuple
from unittest import TestCase
from beacon import *
from random import randint
class TestRandomBeaconVerification(TestCase):
@staticmethod
def happy_entropy_and_proof(view: View) -> Tuple[Beacon, Proof]:
sk = generate_random_sk()
beacon = NormalMode.generate_beacon(sk, view)
return bytes(beacon), bytes(sk.get_g1())
@staticmethod
def unhappy_entropy(last_beacon: Beacon, view: View) -> Beacon:
return RecoveryMode.generate_beacon(last_beacon, view)
def setUp(self):
entropy, proof = self.happy_entropy_and_proof(0)
self.beacon = RandomBeaconHandler(
beacon=RandomBeacon(
version=0,
context=0,
entropy=entropy,
proof=proof
)
)
def test_happy(self):
for i in range(3):
entropy, proof = self.happy_entropy_and_proof(i)
new_beacon = RandomBeacon(
version=0,
context=i,
entropy=entropy,
proof=proof
)
self.beacon.verify_happy(new_beacon)
self.assertEqual(self.beacon.last_beacon.context, 2)
def test_unhappy(self):
for i in range(1, 3):
entropy = self.unhappy_entropy(self.beacon.last_beacon.entropy, i)
new_beacon = RandomBeacon(
version=0,
context=i,
entropy=entropy,
proof=b""
)
self.beacon.verify_unhappy(new_beacon)
self.assertEqual(self.beacon.last_beacon.context, 2)
def test_mixed(self):
for i in range(1, 6, 2):
entropy, proof = self.happy_entropy_and_proof(i)
new_beacon = RandomBeacon(
version=0,
context=i,
entropy=entropy,
proof=proof
)
self.beacon.verify_happy(new_beacon)
entropy = self.unhappy_entropy(self.beacon.last_beacon.entropy, i+1)
new_beacon = RandomBeacon(
version=0,
context=i+1,
entropy=entropy,
proof=b""
)
self.beacon.verify_unhappy(new_beacon)
self.assertEqual(self.beacon.last_beacon.context, 6)

View File

@ -0,0 +1,184 @@
from typing import Dict, List
from unittest import TestCase
from itertools import chain
from blspy import PrivateKey
from carnot import Id, Carnot, Block, Overlay, Vote, StandardQc, NewView
from beacon import generate_random_sk, RandomBeacon, NormalMode
from beconized_carnot import BeaconizedCarnot, BeaconizedBlock
from overlay import FlatOverlay, EntropyOverlay
from test_unhappy_path import parents_from_childs
def gen_node(sk: PrivateKey, overlay: Overlay, entropy: bytes = b""):
node = BeaconizedCarnot(sk, overlay)
return node.id, node
def succeed(nodes: Dict[Id, BeaconizedCarnot], proposed_block: BeaconizedBlock) -> (List[Vote], EntropyOverlay):
overlay = FlatOverlay(list(nodes.keys()))
overlay.set_entropy(proposed_block.beacon.entropy)
# broadcast the block
for node in nodes.values():
node.receive_block(proposed_block)
votes = {}
childs_ids = list(chain.from_iterable(overlay.leaf_committees()))
leafs = [nodes[_id] for _id in childs_ids]
for node in leafs:
vote = node.approve_block(proposed_block, set()).payload
votes[node.id] = vote
while len(parents := parents_from_childs(overlay, childs_ids)) != 0:
for node_id in parents:
node = nodes[node_id]
child_votes = [votes[_id] for _id in votes.keys() if overlay.is_member_of_child_committee(node_id, _id)]
if len(child_votes) == overlay.super_majority_threshold(node_id) and node_id not in votes:
vote = node.approve_block(proposed_block, child_votes).payload
votes[node_id] = vote
childs_ids = list(set(parents))
root_votes = [
votes[node_id]
for node_id in nodes
if overlay.is_member_of_root_committee(node_id) or overlay.is_child_of_root_committee(node_id)
]
return root_votes, overlay
def fail(nodes: Dict[Id, BeaconizedCarnot], proposed_block: BeaconizedBlock) -> (List[NewView], EntropyOverlay):
overlay = FlatOverlay(list(nodes.keys()))
overlay.set_entropy(proposed_block.beacon.entropy)
# broadcast the block
for node in nodes.values():
node.receive_block(proposed_block)
node: BeaconizedCarnot
timeouts = []
for node in (nodes[_id] for _id in nodes if overlay.is_member_of_root_committee(_id) or overlay.is_child_of_root_committee(_id)):
timeout = node.local_timeout().payload
timeouts.append(timeout)
root_member = next(nodes[_id] for _id in nodes if overlay.is_member_of_root_committee(_id))
timeouts = [timeouts[i] for i in range(overlay.leader_super_majority_threshold(root_member.id))]
timeout_qc = root_member.timeout_detected(timeouts).payload
for node in nodes.values():
node.receive_timeout_qc(timeout_qc)
overlay = next(iter(nodes.values())).overlay
votes = {}
childs_ids = list(chain.from_iterable(overlay.leaf_committees()))
leafs = [nodes[_id] for _id in childs_ids]
for node in leafs:
vote = node.approve_new_view(timeout_qc, set()).payload
votes[node.id] = vote
while len(parents := parents_from_childs(overlay, childs_ids)) != 0:
for node_id in parents:
node = nodes[node_id]
child_votes = [votes[_id] for _id in votes.keys() if overlay.is_member_of_child_committee(node_id, _id)]
if len(child_votes) == overlay.super_majority_threshold(node_id) and node_id not in votes:
vote = node.approve_new_view(timeout_qc, child_votes).payload
votes[node_id] = vote
childs_ids = list(set(parents))
root_votes = [
votes[node_id]
for node_id in nodes
if overlay.is_member_of_root_committee(node_id) or overlay.is_child_of_root_committee(node_id)
]
return root_votes, overlay
def add_genesis_block(carnot: BeaconizedCarnot, sk: PrivateKey) -> Block:
entropy = NormalMode.generate_beacon(sk, -1)
genesis_block = BeaconizedBlock(
view=0,
qc=StandardQc(block=b"", view=0),
_id=b"",
beacon=RandomBeacon(
version=0,
context=-1,
entropy=entropy,
proof=bytes(sk.get_g1())
)
)
carnot.safe_blocks[genesis_block.id()] = genesis_block
carnot.receive_block(genesis_block)
carnot.local_high_qc = genesis_block.qc
carnot.current_view = 1
carnot.overlay.set_entropy(entropy)
return genesis_block
def setup_initial_setup(test_case: TestCase, size: int) -> (Dict[Id, Carnot], Carnot, Block, EntropyOverlay):
keys = [generate_random_sk() for _ in range(size)]
nodes_ids = [bytes(key.get_g1()) for key in keys]
genesis_sk = generate_random_sk()
nodes = dict(gen_node(key, FlatOverlay(nodes_ids), bytes(genesis_sk.get_g1())) for key in keys)
genesis_block = None
overlay = FlatOverlay(nodes_ids)
overlay.set_entropy(NormalMode.generate_beacon(genesis_sk, -1))
leader: Carnot = nodes[overlay.leader()]
for node in nodes.values():
genesis_block = add_genesis_block(node, genesis_sk)
# votes for genesis block
genesis_votes = set(
Vote(
block=genesis_block.id(),
view=0,
voter=nodes_ids[i],
qc=StandardQc(
block=genesis_block.id(),
view=0
),
) for i in range(overlay.leader_super_majority_threshold(overlay.leader()))
)
proposed_block = leader.propose_block(1, genesis_votes).payload
test_case.assertIsNotNone(proposed_block)
overlay = FlatOverlay(nodes_ids)
overlay.set_entropy(NormalMode.generate_beacon(genesis_sk, -1))
return nodes, leader, proposed_block, overlay
class TestBeaconizedCarnot(TestCase):
def test_interleave_success_fails(self):
"""
At the end of the timeout the highQC in the next leader's aggregatedQC should be the highestQC held by the
majority of nodes or a qc higher than th highestQC held by the majority of nodes.
Majority means more than two thirds of total number of nodes, randomly assigned to committees.
"""
leader: BeaconizedCarnot
nodes, leader, proposed_block, overlay = setup_initial_setup(self, 5)
for view in range(2, 5):
root_votes, overlay = succeed(nodes, proposed_block)
leader = nodes[overlay.leader()]
proposed_block = leader.propose_block(view, root_votes).payload
root_votes, overlay = fail(nodes, proposed_block)
leader = nodes[overlay.leader()]
proposed_block = leader.propose_block(6, root_votes).payload
for view in range(7, 8):
root_votes, overlay = succeed(nodes, proposed_block)
leader = nodes[overlay.leader()]
proposed_block = leader.propose_block(view, root_votes).payload
root_votes, overlay = fail(nodes, proposed_block)
leader = nodes[overlay.leader()]
proposed_block = leader.propose_block(9, root_votes).payload
for view in range(10, 15):
root_votes, overlay = succeed(nodes, proposed_block)
leader = nodes[overlay.leader()]
proposed_block = leader.propose_block(view, root_votes).payload
committed_blocks = [view for view in range(1, 11) if view not in (4, 5, 7, 8)]
for node in nodes.values():
for view in committed_blocks:
self.assertIn(view, [block.view for block in node.committed_blocks().values()])

View File

@ -126,7 +126,7 @@ def parents_from_childs(overlay: MockOverlay, childs: List[Id]) -> Set[Id]:
return set(possible_parents) if possible_parents else set() return set(possible_parents) if possible_parents else set()
def succeed(test_case: TestCase, overlay: MockOverlay, nodes: Dict[Id, MockCarnot], proposed_block: Block) -> List[Vote]: def succeed(test_case: TestCase, overlay: Overlay, nodes: Dict[Id, Carnot], proposed_block: Block) -> List[Vote]:
# broadcast the block # broadcast the block
for node in nodes.values(): for node in nodes.values():
node.receive_block(proposed_block) node.receive_block(proposed_block)
@ -155,7 +155,7 @@ def succeed(test_case: TestCase, overlay: MockOverlay, nodes: Dict[Id, MockCarno
return root_votes return root_votes
def fail(test_case: TestCase, overlay: MockOverlay, nodes: Dict[Id, MockCarnot], proposed_block: Block) -> List[NewView]: def fail(test_case: TestCase, overlay: Overlay, nodes: Dict[Id, Carnot], proposed_block: Block) -> List[NewView]:
# broadcast the block # broadcast the block
for node in nodes.values(): for node in nodes.values():
node.receive_block(proposed_block) node.receive_block(proposed_block)

1
requirements.txt Normal file
View File

@ -0,0 +1 @@
blspy~=1.0.16