diff --git a/cryptarchia/cryptarchia.py b/cryptarchia/cryptarchia.py index 63e5da5..c410251 100644 --- a/cryptarchia/cryptarchia.py +++ b/cryptarchia/cryptarchia.py @@ -4,18 +4,26 @@ from math import floor from copy import deepcopy from itertools import chain import functools - -# Please note this is still a work in progress from dataclasses import dataclass, field, replace +import logging + +import numpy as np + + +logger = logging.getLogger(__name__) + Id: TypeAlias = bytes -@dataclass +@dataclass(frozen=True) class Epoch: # identifier of the epoch, counting incrementally from 0 epoch: int + def prev(self) -> "Epoch": + return Epoch(self.epoch - 1) + @dataclass class TimeConfig: @@ -30,27 +38,34 @@ class Config: k: int # The depth of a block before it is considered immutable. active_slot_coeff: float # 'f', the rate of occupied slots - # The stake distribution is always taken at the beginning of the previous epoch. + # The stake distribution is taken at the beginning of the previous epoch. # This parameters controls how many slots to wait for it to be stabilized - # The value is computed as epoch_stake_distribution_stabilization * int(floor(k / f)) + # The value is computed as + # epoch_stake_distribution_stabilization * int(floor(k / f)) epoch_stake_distribution_stabilization: int - # This parameter controls how many slots we wait after the stake distribution - # snapshot has stabilized to take the nonce snapshot. + # This parameter controls how many `base periods` we wait after stake + # distribution snapshot has stabilized to take the nonce snapshot. epoch_period_nonce_buffer: int - # This parameter controls how many slots we wait for the nonce snapshot to be considered - # stabilized + # This parameter controls how many `base periods` we wait for the nonce + # snapshot to be considered stabilized epoch_period_nonce_stabilization: int + # -- Stake Relativization Params + initial_total_active_stake: int # D_0 + total_active_stake_learning_rate: int # beta + time: TimeConfig @staticmethod - def cryptarchia_v0_0_1() -> "Config": + def cryptarchia_v0_0_1(initial_total_active_stake) -> "Config": return Config( k=2160, active_slot_coeff=0.05, epoch_stake_distribution_stabilization=3, epoch_period_nonce_buffer=3, epoch_period_nonce_stabilization=4, + initial_total_active_stake=initial_total_active_stake, + total_active_stake_learning_rate=0.8, time=TimeConfig( slot_duration=1, chain_start_time=0, @@ -61,19 +76,24 @@ class Config: def base_period_length(self) -> int: return int(floor(self.k / self.active_slot_coeff)) + @property + def epoch_relative_nonce_slot(self) -> int: + return ( + self.epoch_stake_distribution_stabilization + self.epoch_period_nonce_buffer + ) * self.base_period_length + @property def epoch_length(self) -> int: return ( - self.epoch_stake_distribution_stabilization - + self.epoch_period_nonce_buffer - + self.epoch_period_nonce_stabilization - ) * self.base_period_length + self.epoch_relative_nonce_slot + + self.epoch_period_nonce_stabilization * self.base_period_length + ) @property def s(self): """ - The Security Paramater. This paramter controls how many slots one must wait before we - have high confidence that k blocks have been produced. + The Security Paramater. This paramter controls how many slots one must + wait before we have high confidence that k blocks have been produced. """ return self.base_period_length * 3 @@ -258,11 +278,14 @@ class LedgerState: """ block: Id = None - # This nonce is used to derive the seed for the slot leader lottery - # It's updated at every block by hashing the previous nonce with the nullifier - # Note that this does not prevent nonce grinding at the last slot before the nonce snapshot + + # This nonce is used to derive the seed for the slot leader lottery. + # It's updated at every block by hashing the previous nonce with the + # leader proof's nullifier. + # + # NOTE that this does not prevent nonce grinding at the last slot + # when the nonce snapshot is taken nonce: Id = None - total_stake: int = None # set of commitments commitments_spend: set[Id] = field(default_factory=set) @@ -273,16 +296,24 @@ class LedgerState: # set of nullified coins nullifiers: set[Id] = field(default_factory=set) + # -- Stake Relativization State + # The number of observed leaders (blocks + orphans), this measurement is + # used in inferring total active stake in the network. + leader_count: int = 0 + def copy(self): return LedgerState( block=self.block, nonce=self.nonce, - total_stake=self.total_stake, commitments_spend=deepcopy(self.commitments_spend), commitments_lead=deepcopy(self.commitments_lead), nullifiers=deepcopy(self.nullifiers), + leader_count=self.leader_count, ) + def replace(self, **kwarg) -> "LedgerState": + return replace(self, **kwarg) + def verify_eligible_to_spend(self, commitment: Id) -> bool: return commitment in self.commitments_spend @@ -304,10 +335,13 @@ class LedgerState: self.nonce = h.digest() self.block = block.id() for proof in chain(block.orphaned_proofs, [block]): - proof = proof.leader_proof - self.nullifiers.add(proof.nullifier) - self.commitments_spend.add(proof.evolved_commitment) - self.commitments_lead.add(proof.evolved_commitment) + self.apply_leader_proof(proof.leader_proof) + + def apply_leader_proof(self, proof: MockLeaderProof): + self.nullifiers.add(proof.nullifier) + self.commitments_spend.add(proof.evolved_commitment) + self.commitments_lead.add(proof.evolved_commitment) + self.leader_count += 1 @dataclass @@ -315,25 +349,33 @@ class EpochState: # for details of snapshot schedule please see: # https://github.com/IntersectMBO/ouroboros-consensus/blob/fe245ac1d8dbfb563ede2fdb6585055e12ce9738/docs/website/contents/for-developers/Glossary.md#epoch-structure - # The stake distribution snapshot is taken at the beginning of the previous epoch + # Stake distribution snapshot is taken at the start of the previous epoch stake_distribution_snapshot: LedgerState - # The nonce snapshot is taken 7k/f slots into the previous epoch + # Nonce snapshot is taken 6k/f slots into the previous epoch nonce_snapshot: LedgerState + # Total stake is inferred from watching block production rate over the past + # epoch. This inferred total stake is used to relativize stake values in the + # leadership lottery. + inferred_total_active_stake: int + def verify_eligible_to_lead_due_to_age(self, commitment: Id) -> bool: # A coin is eligible to lead if it was committed to before the the stake - # distribution snapshot was taken or it was produced by a leader proof since the snapshot was taken. + # distribution snapshot was taken or it was produced by a leader proof + # since the snapshot was taken. # # This verification is checking that first condition. # # NOTE: `ledger_state.commitments_spend` is a super-set of `ledger_state.commitments_lead` - return self.stake_distribution_snapshot.verify_eligible_to_spend(commitment) - def total_stake(self) -> int: - """Returns the total stake that will be used to reletivize leadership proofs during this epoch""" - return self.stake_distribution_snapshot.total_stake + def total_active_stake(self) -> int: + """ + Returns the inferred total stake participating in consensus. + Total active stake is used to reletivize a coin's value in leadership proofs. + """ + return self.inferred_total_active_stake def nonce(self) -> bytes: return self.nonce_snapshot.nonce @@ -346,53 +388,75 @@ class Follower: self.local_chain = Chain([], genesis=genesis_state.block) self.genesis_state = genesis_state self.ledger_state = {genesis_state.block: genesis_state.copy()} + self.epoch_state = {} def validate_header(self, block: BlockHeader, chain: Chain) -> bool: # TODO: verify blocks are not in the 'future' - current_state = self.ledger_state[chain.tip_id()].copy() - orphaned_commitments = set() + if block.parent != chain.tip_id(): + logger.warning("block parent is not chain tip") + return False + + current_state = self.ledger_state[block.parent].copy() + # first, we verify adopted leadership transactions - for proof in block.orphaned_proofs: - proof = proof.leader_proof - # each proof is validated against the last state of the ledger of the chain this block - # is being added to before that proof slot - parent_state = self.state_at_slot_beginning(chain, proof.slot).copy() - # we add effects of previous orphaned proofs to the ledger state - parent_state.commitments_lead |= orphaned_commitments - epoch_state = self.compute_epoch_state(proof.slot.epoch(self.config), chain) - if self.verify_slot_leader( - proof.slot, proof, epoch_state, parent_state, current_state - ): - # if an adopted leadership proof is valid we need to apply its effects to the ledger state - orphaned_commitments.add(proof.evolved_commitment) - current_state.nullifiers.add(proof.nullifier) - else: - # otherwise, the whole block is invalid + for orphan in block.orphaned_proofs: + # orphan proofs are checked in two ways + # 1. ensure they are valid locally in their original branch + # 2. ensure it does not conflict with current state + + # We take a shortcut for (1.) by restricting orphans to proofs we've + # already processed in other branches. + if orphan.id() not in self.ledger_state: + logger.warning("missing orphan proof") return False - parent_state = self.ledger_state[block.parent].copy() - parent_state.commitments_lead |= orphaned_commitments + # we use the proposed block epoch state here instead of the orphan's + # epoch state. For very old orphans, these states may be different. + epoch_state = self.compute_epoch_state(block.slot.epoch(self.config), chain) + + # (2.) is satisfied by verifying the proof against current state ensuring: + # - it is a valid proof + # - and the nullifier has not already been spent + if not self.verify_slot_leader( + orphan.slot, + orphan.parent, + orphan.leader_proof, + epoch_state, + current_state, + ): + logger.warning("invalid orphan proof") + return False + + # if an adopted leadership proof is valid we need to apply its + # effects to the ledger state + current_state.apply_leader_proof(orphan.leader_proof) + epoch_state = self.compute_epoch_state(block.slot.epoch(self.config), chain) # TODO: this is not the full block validation spec, only slot leader is verified return self.verify_slot_leader( - block.slot, block.leader_proof, epoch_state, parent_state, current_state + block.slot, + block.parent, + block.leader_proof, + epoch_state, + current_state, ) def verify_slot_leader( self, slot: Slot, + parent: Id, proof: MockLeaderProof, # coins are old enough if their commitment is in the stake distribution snapshot epoch_state: EpochState, - # commitments derived from leadership coin evolution are checked in the parent state - parent_state: LedgerState, - # nullifiers are checked in the current state + # nullifiers (and commitments) are checked against the current state. + # For now, we assume proof parent state and current state are identical. + # This will change once we start putting merkle roots in headers current_state: LedgerState, ) -> bool: return ( - proof.verify(slot, parent_state.block) # verify slot leader proof + proof.verify(slot, parent) # verify slot leader proof and ( - parent_state.verify_eligible_to_lead(proof.commitment) + current_state.verify_eligible_to_lead(proof.commitment) or epoch_state.verify_eligible_to_lead_due_to_age(proof.commitment) ) and current_state.verify_unspent(proof.nullifier) @@ -405,7 +469,7 @@ class Follower: return self.local_chain for chain in self.forks: - if chain.tip().id() == block.parent: + if chain.tip_id() == block.parent: return chain return None @@ -436,24 +500,51 @@ class Follower: if new_chain is not None: self.forks.append(new_chain) else: + logger.warning("missing parent block") # otherwise, we're missing the parent block # in that case, just ignore the block return if not self.validate_header(block, new_chain): + logger.warning("invalid header") return new_chain.blocks.append(block) # We may need to switch forks, lets run the fork choice rule to check. new_chain = self.fork_choice() - self.local_chain = new_chain + if new_chain != self.local_chain: + self.forks.remove(new_chain) + self.forks.append(self.local_chain) + self.local_chain = new_chain new_state = self.ledger_state[block.parent].copy() new_state.apply(block) self.ledger_state[block.id()] = new_state - # Evaluate the fork choice rule and return the block header of the block that should be the head of the chain + def unimported_orphans(self, tip: Id) -> list[BlockHeader]: + """ + Returns all unimported orphans w.r.t. the given tip's state. + Orphans are returned in the order that they should be imported. + """ + tip_state = self.ledger_state[tip].copy() + + orphans = [] + for fork in [self.local_chain, *self.forks]: + if fork.block_position(tip) is not None: + # the tip is a member of this fork, it doesn't make sense + # to take orphans from this fork as they are all already "imported" + continue + + for block in fork.blocks: + for b in [*block.orphaned_proofs, block]: + if b.leader_proof.nullifier not in tip_state.nullifiers: + tip_state.nullifiers.add(b.leader_proof.nullifier) + orphans += [b] + + return orphans + + # Evaluate the fork choice rule and return the chain we should be following def fork_choice(self) -> Chain: return maxvalid_bg( self.local_chain, self.forks, k=self.config.k, s=self.config.s @@ -463,10 +554,7 @@ class Follower: return self.local_chain.tip() def tip_id(self) -> Id: - if self.local_chain.length() > 0: - return self.local_chain.tip().id() - else: - return self.genesis_state.block + return self.local_chain.tip_id() def tip_state(self) -> LedgerState: return self.ledger_state[self.tip_id()] @@ -478,29 +566,82 @@ class Follower: return self.genesis_state - def compute_epoch_state(self, epoch: Epoch, chain: Chain) -> EpochState: + def epoch_start_slot(self, epoch) -> Slot: + return Slot(epoch.epoch * self.config.epoch_length) + + def stake_distribution_snapshot(self, epoch, chain): # stake distribution snapshot happens at the beginning of the previous epoch, # i.e. for epoch e, the snapshot is taken at the last block of epoch e-2 - stake_snapshot_slot = Slot((epoch.epoch - 1) * self.config.epoch_length) - stake_distribution_snapshot = self.state_at_slot_beginning( - chain, stake_snapshot_slot - ) + slot = Slot(epoch.prev().epoch * self.config.epoch_length) + return self.state_at_slot_beginning(chain, slot) - nonce_slot = Slot( - self.config.base_period_length - * ( - self.config.epoch_stake_distribution_stabilization - + self.config.epoch_period_nonce_buffer + def nonce_snapshot(self, epoch, chain): + # nonce snapshot happens partway through the previous epoch after the + # stake distribution has stabilized + slot = Slot( + self.config.epoch_relative_nonce_slot + + self.epoch_start_slot(epoch.prev()).absolute_slot + ) + return self.state_at_slot_beginning(chain, slot) + + def compute_epoch_state(self, epoch: Epoch, chain: Chain) -> EpochState: + if epoch.epoch == 0: + return EpochState( + stake_distribution_snapshot=self.genesis_state, + nonce_snapshot=self.genesis_state, + inferred_total_active_stake=self.config.initial_total_active_stake, ) - + stake_snapshot_slot.absolute_slot - ) - nonce_snapshot = self.state_at_slot_beginning(chain, nonce_slot) - return EpochState( + stake_distribution_snapshot = self.stake_distribution_snapshot(epoch, chain) + nonce_snapshot = self.nonce_snapshot(epoch, chain) + + # we memoize epoch states to avoid recursion killing our performance + memo_block_id = nonce_snapshot.block + if state := self.epoch_state.get((epoch, memo_block_id)): + return state + + # To update our inference of total stake, we need the prior estimate which + # was calculated last epoch. Thus we recurse here to retreive the previous + # estimate of total stake. + prev_epoch = self.compute_epoch_state(epoch.prev(), chain) + inferred_total_active_stake = self._infer_total_active_stake( + prev_epoch, nonce_snapshot, stake_distribution_snapshot + ) + + state = EpochState( stake_distribution_snapshot=stake_distribution_snapshot, nonce_snapshot=nonce_snapshot, + inferred_total_active_stake=inferred_total_active_stake, ) + self.epoch_state[(epoch, memo_block_id)] = state + return state + + def _infer_total_active_stake( + self, + prev_epoch: EpochState, + nonce_snapshot: LedgerState, + stake_distribution_snapshot: LedgerState, + ): + # Infer total stake from empirical block production rate in last epoch + + # Since we need a stable inference of total stake for the start of this epoch, + # we limit our look back period to the start of last epoch until when the nonce + # snapshot was taken. + block_proposals_last_epoch = ( + nonce_snapshot.leader_count - stake_distribution_snapshot.leader_count + ) + T = self.config.epoch_relative_nonce_slot + mean_blocks_per_slot = block_proposals_last_epoch / T + expected_blocks_per_slot = np.log(1 / (1 - self.config.active_slot_coeff)) + blocks_per_slot_err = expected_blocks_per_slot - mean_blocks_per_slot + h = ( + self.config.total_active_stake_learning_rate + * prev_epoch.inferred_total_active_stake + / expected_blocks_per_slot + ) + return int(prev_epoch.inferred_total_active_stake - h * blocks_per_slot_err) + def phi(f: float, alpha: float) -> float: """ @@ -514,7 +655,7 @@ def phi(f: float, alpha: float) -> float: class MOCK_LEADER_VRF: - """NOT SECURE: A mock VRF function where the sk and pk are assummed to be the same""" + """NOT SECURE: A mock VRF function""" ORDER = 2**256 @@ -545,15 +686,8 @@ class Leader: if self._is_slot_leader(epoch, slot): return MockLeaderProof.new(self.coin, slot, parent) - def propose_block( - self, slot: Slot, parent: BlockHeader, orphaned_proofs=[] - ) -> BlockHeader: - return BlockHeader( - parent=parent.id(), slot=slot, orphaned_proofs=orphaned_proofs - ) - def _is_slot_leader(self, epoch: EpochState, slot: Slot): - relative_stake = self.coin.value / epoch.total_stake() + relative_stake = self.coin.value / epoch.total_active_stake() r = MOCK_LEADER_VRF.vrf(self.coin, epoch.nonce(), slot) diff --git a/cryptarchia/test_common.py b/cryptarchia/test_common.py index a959962..4b3dbc0 100644 --- a/cryptarchia/test_common.py +++ b/cryptarchia/test_common.py @@ -7,13 +7,47 @@ from .cryptarchia import ( BlockHeader, LedgerState, MockLeaderProof, + Leader, + Follower, ) -def mk_config() -> Config: - return Config.cryptarchia_v0_0_1().replace( +class TestNode: + def __init__(self, config: Config, genesis: LedgerState, coin: Coin): + self.config = config + self.leader = Leader(coin=coin, config=config) + self.follower = Follower(genesis, config) + + def epoch_state(self, slot: Slot): + return self.follower.compute_epoch_state( + slot.epoch(self.config), self.follower.local_chain + ) + + def on_slot(self, slot: Slot) -> BlockHeader | None: + parent = self.follower.tip_id() + epoch_state = self.epoch_state(slot) + if leader_proof := self.leader.try_prove_slot_leader(epoch_state, slot, parent): + orphans = self.follower.unimported_orphans(parent) + self.leader.coin = self.leader.coin.evolve() + return BlockHeader( + parent=parent, + slot=slot, + orphaned_proofs=orphans, + leader_proof=leader_proof, + content_size=0, + content_id=bytes(32), + ) + return None + + def on_block(self, block: BlockHeader): + self.follower.on_block(block) + + +def mk_config(initial_stake_distribution: list[Coin]) -> Config: + initial_inferred_total_stake = sum(c.value for c in initial_stake_distribution) + return Config.cryptarchia_v0_0_1(initial_inferred_total_stake).replace( k=1, - active_slot_coeff=1.0, + active_slot_coeff=0.5, ) @@ -21,7 +55,6 @@ def mk_genesis_state(initial_stake_distribution: list[Coin]) -> LedgerState: return LedgerState( block=bytes(32), nonce=bytes(32), - total_stake=sum(c.value for c in initial_stake_distribution), commitments_spend={c.commitment() for c in initial_stake_distribution}, commitments_lead={c.commitment() for c in initial_stake_distribution}, nullifiers=set(), diff --git a/cryptarchia/test_fork_choice.py b/cryptarchia/test_fork_choice.py index 8129d95..ff048e1 100644 --- a/cryptarchia/test_fork_choice.py +++ b/cryptarchia/test_fork_choice.py @@ -12,9 +12,10 @@ from cryptarchia.cryptarchia import ( Id, MockLeaderProof, Coin, + Follower, ) -from .test_common import mk_chain +from .test_common import mk_chain, mk_config, mk_genesis_state, mk_block class TestForkChoice(TestCase): @@ -74,3 +75,50 @@ class TestForkChoice(TestCase): short_chain = Chain(short_chain, genesis=bytes(32)) long_chain = Chain(long_chain, genesis=bytes(32)) assert maxvalid_bg(short_chain, [long_chain], k, s) == long_chain + + def test_fork_choice_integration(self): + c_a, c_b = Coin(sk=0, value=10), Coin(sk=1, value=10) + coins = [c_a, c_b] + config = mk_config(coins) + genesis = mk_genesis_state(coins) + follower = Follower(genesis, config) + + b1, c_a = mk_block(genesis.block, 1, c_a), c_a.evolve() + + follower.on_block(b1) + + assert follower.tip_id() == b1.id() + assert follower.forks == [] + + # -- then we fork -- + # + # b2 == tip + # / + # b1 + # \ + # b3 + # + + b2, c_a = mk_block(b1.id(), 2, c_a), c_a.evolve() + b3, c_b = mk_block(b1.id(), 2, c_b), c_b.evolve() + + follower.on_block(b2) + follower.on_block(b3) + + assert follower.tip_id() == b2.id() + assert len(follower.forks) == 1 and follower.forks[0].tip_id() == b3.id() + + # -- extend the fork causing a re-org -- + # + # b2 + # / + # b1 + # \ + # b3 - b4 == tip + # + + b4, c_b = mk_block(b3.id(), 3, c_b), c_a.evolve() + follower.on_block(b4) + + assert follower.tip_id() == b4.id() + assert len(follower.forks) == 1 and follower.forks[0].tip_id() == b2.id() diff --git a/cryptarchia/test_leader.py b/cryptarchia/test_leader.py index b599833..3f6918c 100644 --- a/cryptarchia/test_leader.py +++ b/cryptarchia/test_leader.py @@ -18,22 +18,22 @@ from .test_common import mk_config class TestLeader(TestCase): def test_slot_leader_statistics(self): epoch = EpochState( - stake_distribution_snapshot=LedgerState( - total_stake=1000, - ), + stake_distribution_snapshot=LedgerState(), nonce_snapshot=LedgerState(nonce=b"1010101010"), + inferred_total_active_stake=1000, ) + coin = Coin(sk=0, value=10) f = 0.05 l = Leader( - config=mk_config().replace(active_slot_coeff=f), - coin=Coin(sk=0, value=10), + config=mk_config([coin]).replace(active_slot_coeff=f), + coin=coin, ) # We'll use the Margin of Error equation to decide how many samples we need. # https://en.wikipedia.org/wiki/Margin_of_error margin_of_error = 1e-4 - p = phi(f=f, alpha=10 / 1000) + p = phi(f=f, alpha=10 / epoch.total_active_stake()) std = np.sqrt(p * (1 - p)) Z = 3 # we want 3 std from the mean to be within the margin of error N = int((Z * std / margin_of_error) ** 2) diff --git a/cryptarchia/test_ledger_state_update.py b/cryptarchia/test_ledger_state_update.py index 8e50fb1..f724891 100644 --- a/cryptarchia/test_ledger_state_update.py +++ b/cryptarchia/test_ledger_state_update.py @@ -22,7 +22,7 @@ class TestLedgerStateUpdate(TestCase): leader_coin = Coin(sk=0, value=100) genesis = mk_genesis_state([leader_coin]) - follower = Follower(genesis, mk_config()) + follower = Follower(genesis, mk_config([leader_coin])) block = mk_block(slot=0, parent=genesis.block, coin=leader_coin) follower.on_block(block) @@ -42,24 +42,22 @@ class TestLedgerStateUpdate(TestCase): assert follower.tip() == block def test_ledger_state_is_properly_updated_on_reorg(self): - coin_1 = Coin(sk=0, value=100) - coin_2 = Coin(sk=1, value=100) - coin_3 = Coin(sk=2, value=100) + coin = [Coin(sk=0, value=100), Coin(sk=1, value=100), Coin(sk=2, value=100)] - genesis = mk_genesis_state([coin_1, coin_2, coin_3]) + genesis = mk_genesis_state(coin) - follower = Follower(genesis, mk_config()) + follower = Follower(genesis, mk_config(coin)) - # 1) coin_1 & coin_2 both concurrently win slot 0 + # 1) coin[0] & coin[1] both concurrently win slot 0 - block_1 = mk_block(parent=genesis.block, slot=0, coin=coin_1) - block_2 = mk_block(parent=genesis.block, slot=0, coin=coin_2) + block_1 = mk_block(parent=genesis.block, slot=0, coin=coin[0]) + block_2 = mk_block(parent=genesis.block, slot=0, coin=coin[1]) # 2) follower sees block 1 first follower.on_block(block_1) assert follower.tip() == block_1 - assert not follower.tip_state().verify_unspent(coin_1.nullifier()) + assert not follower.tip_state().verify_unspent(coin[0].nullifier()) # 3) then sees block 2, but sticks with block_1 as the tip @@ -67,21 +65,21 @@ class TestLedgerStateUpdate(TestCase): assert follower.tip() == block_1 assert len(follower.forks) == 1, f"{len(follower.forks)}" - # 4) then coin_3 wins slot 1 and chooses to extend from block_2 + # 4) then coin[2] wins slot 1 and chooses to extend from block_2 - block_3 = mk_block(parent=block_2.id(), slot=1, coin=coin_3) + block_3 = mk_block(parent=block_2.id(), slot=1, coin=coin[2]) follower.on_block(block_3) # the follower should have switched over to the block_2 fork assert follower.tip() == block_3 - # and the original coin_1 should now be removed from the spent pool - assert follower.tip_state().verify_unspent(coin_1.nullifier()) + # and the original coin[0] should now be removed from the spent pool + assert follower.tip_state().verify_unspent(coin[0].nullifier()) def test_fork_creation(self): coins = [Coin(sk=i, value=100) for i in range(7)] genesis = mk_genesis_state(coins) - follower = Follower(genesis, mk_config()) + follower = Follower(genesis, mk_config(coins)) # coin_0 & coin_1 both concurrently win slot 0 based on the genesis block # Both blocks are accepted, and a fork is created "from the genesis block" @@ -126,12 +124,12 @@ class TestLedgerStateUpdate(TestCase): def test_epoch_transition(self): leader_coins = [Coin(sk=i, value=100) for i in range(4)] genesis = mk_genesis_state(leader_coins) - config = mk_config() + config = mk_config(leader_coins) follower = Follower(genesis, config) # We assume an epoch length of 10 slots in this test. - assert config.epoch_length == 10, f"epoch len: {config.epoch_length}" + assert config.epoch_length == 20, f"epoch len: {config.epoch_length}" # ---- EPOCH 0 ---- @@ -140,14 +138,14 @@ class TestLedgerStateUpdate(TestCase): assert follower.tip() == block_1 assert follower.tip().slot.epoch(config).epoch == 0 - block_2 = mk_block(slot=9, parent=block_1.id(), coin=leader_coins[1]) + block_2 = mk_block(slot=19, parent=block_1.id(), coin=leader_coins[1]) follower.on_block(block_2) assert follower.tip() == block_2 assert follower.tip().slot.epoch(config).epoch == 0 # ---- EPOCH 1 ---- - block_3 = mk_block(slot=10, parent=block_2.id(), coin=leader_coins[2]) + block_3 = mk_block(slot=20, parent=block_2.id(), coin=leader_coins[2]) follower.on_block(block_3) assert follower.tip() == block_3 assert follower.tip().slot.epoch(config).epoch == 1 @@ -159,7 +157,7 @@ class TestLedgerStateUpdate(TestCase): # To ensure this is the case, we add a new coin just to the state associated with that slot, # so that the new block can be accepted only if that is the snapshot used # first, verify that if we don't change the state, the block is not accepted - block_4 = mk_block(slot=20, parent=block_3.id(), coin=Coin(sk=4, value=100)) + block_4 = mk_block(slot=40, parent=block_3.id(), coin=Coin(sk=4, value=100)) follower.on_block(block_4) assert follower.tip() == block_3 # then we add the coin to "spendable commitments" associated with slot 9 @@ -175,7 +173,7 @@ class TestLedgerStateUpdate(TestCase): genesis = mk_genesis_state([coin]) - follower = Follower(genesis, mk_config()) + follower = Follower(genesis, mk_config([coin])) # coin wins the first slot block_1 = mk_block(slot=0, parent=genesis.block, coin=coin) @@ -193,13 +191,13 @@ class TestLedgerStateUpdate(TestCase): assert follower.tip() == block_2_evolve def test_new_coins_becoming_eligible_after_stake_distribution_stabilizes(self): - config = mk_config() coin = Coin(sk=0, value=100) + config = mk_config([coin]) genesis = mk_genesis_state([coin]) follower = Follower(genesis, config) - # We assume an epoch length of 10 slots in this test. - assert config.epoch_length == 10 + # We assume an epoch length of 20 slots in this test. + assert config.epoch_length == 20 # ---- EPOCH 0 ---- @@ -228,7 +226,7 @@ class TestLedgerStateUpdate(TestCase): # The newly minted coin is still not eligible in the following epoch since the # stake distribution snapshot is taken at the beginning of the previous epoch - block_1_0 = mk_block(slot=10, parent=block_0_1.id(), coin=coin_new) + block_1_0 = mk_block(slot=20, parent=block_0_1.id(), coin=coin_new) follower.on_block(block_1_0) assert follower.tip() == block_0_1 @@ -237,7 +235,7 @@ class TestLedgerStateUpdate(TestCase): # The coin is finally eligible 2 epochs after it was first minted block_2_0 = mk_block( - slot=20, + slot=40, parent=block_0_1.id(), coin=coin_new, ) @@ -245,16 +243,15 @@ class TestLedgerStateUpdate(TestCase): assert follower.tip() == block_2_0 # And now the minted coin can freely use the evolved coin for subsequent blocks - - block_2_1 = mk_block(slot=20, parent=block_2_0.id(), coin=coin_new.evolve()) + block_2_1 = mk_block(slot=40, parent=block_2_0.id(), coin=coin_new.evolve()) follower.on_block(block_2_1) assert follower.tip() == block_2_1 def test_orphaned_proofs(self): - coin = Coin(sk=0, value=100) - genesis = mk_genesis_state([coin]) + coin, coin_orphan = Coin(sk=0, value=100), Coin(sk=1, value=100) + genesis = mk_genesis_state([coin, coin_orphan]) - follower = Follower(genesis, mk_config()) + follower = Follower(genesis, mk_config([coin, coin_orphan])) block_0_0 = mk_block(slot=0, parent=genesis.block, coin=coin) follower.on_block(block_0_0) @@ -267,24 +264,26 @@ class TestLedgerStateUpdate(TestCase): # the coin evolved twice should not be accepted as it is not in the lead commitments assert follower.tip() == block_0_0 - # an orphaned proof with an evolved coin for the same slot as the original coin - # should not be accepted as the evolved coin is not in the lead commitments at slot 0 + # An orphaned proof will not be accepted until a node first sees the corresponding block. + # + # Also, notice that the block is using the evolved orphan coin which is not present on the main + # branch. The evolved orphan commitment is added from the orphan prior to validating the block + # header as part of orphan importing process + orphan = mk_block(parent=genesis.block, slot=0, coin=coin_orphan) block_0_1 = mk_block( slot=1, parent=block_0_0.id(), - coin=coin_new_new, - orphaned_proofs=[mk_block(parent=genesis.block, slot=0, coin=coin_new)], + coin=coin_orphan.evolve(), + orphaned_proofs=[orphan], ) follower.on_block(block_0_1) + + # since follower had not seen this orphan prior to being included as + # an orphan proof, it will be rejected assert follower.tip() == block_0_0 - # the coin evolved twice should be accepted as the evolved coin is in the lead commitments - # at slot 1 and processed before that - block_0_2 = mk_block( - slot=2, - parent=block_0_0.id(), - coin=coin_new_new, - orphaned_proofs=[mk_block(parent=block_0_0.id(), slot=1, coin=coin_new)], - ) - follower.on_block(block_0_2) - assert follower.tip() == block_0_2 + # but all is fine if the follower first sees the orphan block, and then + # is imported into the main chain + follower.on_block(orphan) + follower.on_block(block_0_1) + assert follower.tip() == block_0_1 diff --git a/cryptarchia/test_orphaned_proofs.py b/cryptarchia/test_orphaned_proofs.py new file mode 100644 index 0000000..41a1ccb --- /dev/null +++ b/cryptarchia/test_orphaned_proofs.py @@ -0,0 +1,264 @@ +from unittest import TestCase +from itertools import repeat +import numpy as np +import hashlib + +from copy import deepcopy +from cryptarchia.cryptarchia import ( + maxvalid_bg, + Chain, + BlockHeader, + Slot, + Id, + MockLeaderProof, + Coin, + Follower, +) + +from .test_common import mk_chain, mk_config, mk_genesis_state, mk_block + + +class TestOrphanedProofs(TestCase): + def test_simple_orphan_import(self): + c_a, c_b = Coin(sk=0, value=10), Coin(sk=1, value=10) + coins = [c_a, c_b] + config = mk_config(coins) + genesis = mk_genesis_state(coins) + follower = Follower(genesis, config) + + # -- fork -- + # + # b2 == tip + # / + # b1 + # \ + # b3 + # + + b1, c_a = mk_block(genesis.block, 1, c_a), c_a.evolve() + b2, c_a = mk_block(b1.id(), 2, c_a), c_a.evolve() + b3, c_b = mk_block(b1.id(), 2, c_b), c_b.evolve() + + for b in [b1, b2, b3]: + follower.on_block(b) + + assert follower.tip() == b2 + assert [f.tip() for f in follower.forks] == [b3] + assert follower.unimported_orphans(follower.tip_id()) == [b3] + + # -- extend with import -- + # + # b2 - b4 + # / / + # b1 / + # \ / + # b3 + # + b4, c_a = mk_block(b2.id(), 3, c_a, orphaned_proofs=[b3]), c_a.evolve() + follower.on_block(b4) + + assert follower.tip() == b4 + assert [f.tip() for f in follower.forks] == [b3] + assert follower.unimported_orphans(follower.tip_id()) == [] + + def test_orphan_proof_import_from_long_running_fork(self): + c_a, c_b = Coin(sk=0, value=10), Coin(sk=1, value=10) + coins = [c_a, c_b] + config = mk_config(coins) + genesis = mk_genesis_state(coins) + follower = Follower(genesis, config) + + # -- fork -- + # + # b2 - b3 == tip + # / + # b1 + # \ + # b4 - b5 + # + + b1, c_a = mk_block(genesis.block, 1, c_a), c_a.evolve() + + b2, c_a = mk_block(b1.id(), 2, c_a), c_a.evolve() + b3, c_a = mk_block(b2.id(), 3, c_a), c_a.evolve() + + b4, c_b = mk_block(b1.id(), 2, c_b), c_b.evolve() + b5, c_b = mk_block(b4.id(), 3, c_b), c_b.evolve() + + for b in [b1, b2, b3, b4, b5]: + follower.on_block(b) + + assert follower.tip() == b3 + assert [f.tip() for f in follower.forks] == [b5] + assert follower.unimported_orphans(follower.tip_id()) == [b4, b5] + + # -- extend b3, importing the fork -- + # + # b2 - b3 - b6 == tip + # / ___/ + # b1 ___/ / + # \ / / + # b4 - b5 + + b6, c_a = mk_block(b3.id(), 4, c_a, orphaned_proofs=[b4, b5]), c_a.evolve() + follower.on_block(b6) + + assert follower.tip() == b6 + assert [f.tip() for f in follower.forks] == [b5] + + def test_orphan_proof_import_from_fork_without_direct_shared_parent(self): + coins = [Coin(sk=i, value=10) for i in range(2)] + c_a, c_b = coins + config = mk_config(coins) + genesis = mk_genesis_state(coins) + follower = Follower(genesis, config) + + # -- forks -- + # + # b2 - b3 - b4 == tip + # / + # b1 + # \ + # b5 - b6 - b7 + + b1, c_a = mk_block(genesis.block, 1, c_a), c_a.evolve() + + b2, c_a = mk_block(b1.id(), 2, c_a), c_a.evolve() + b3, c_a = mk_block(b2.id(), 3, c_a), c_a.evolve() + b4, c_a = mk_block(b3.id(), 4, c_a), c_a.evolve() + + b5, c_b = mk_block(b1.id(), 2, c_b), c_b.evolve() + b6, c_b = mk_block(b5.id(), 3, c_b), c_b.evolve() + b7, c_b = mk_block(b6.id(), 4, c_b), c_b.evolve() + + for b in [b1, b2, b3, b4, b5, b6, b7]: + follower.on_block(b) + + assert follower.tip() == b4 + assert [f.tip() for f in follower.forks] == [b7] + assert follower.unimported_orphans(follower.tip_id()) == [b5, b6, b7] + + # -- extend b4, importing the forks -- + # + # b2 - b3 - b4 - b8 == tip + # / _______/ + # b1 ____/______/ + # \ / / / + # b5 - b6 - b7 + # + # Earlier implementations of orphan proof validation failed to + # validate b7 as an orphan here. + + b8, c_a = mk_block(b4.id(), 5, c_a, orphaned_proofs=[b5, b6, b7]), c_a.evolve() + follower.on_block(b8) + + assert follower.tip() == b8 + assert [f.tip() for f in follower.forks] == [b7] + assert follower.unimported_orphans(follower.tip_id()) == [] + + def test_unimported_orphans(self): + # Given the following fork graph: + # + # b2 - b3 + # / + # b1 + # \ + # b4 - b5 + # \ + # -- b6 + # + # Orphans w.r.t. to b3 are b4..6, thus extending from b3 with b7 would + # give the following fork graph + # + # b2 - b3 --- b7== tip + # / ____/ + # b1 ____/ __/ + # \ / / / + # b4 - b5 / + # \ / + # -- b6 + # + + coins = [Coin(sk=i, value=10) for i in range(3)] + c_a, c_b, c_c = coins + config = mk_config(coins) + genesis = mk_genesis_state(coins) + follower = Follower(genesis, config) + + b1, c_a = mk_block(genesis.block, 1, c_a), c_a.evolve() + + b2, c_a = mk_block(b1.id(), 2, c_a), c_a.evolve() + b3, c_a = mk_block(b2.id(), 3, c_a), c_a.evolve() + + b4, c_b = mk_block(b1.id(), 2, c_b), c_b.evolve() + b5, c_b = mk_block(b4.id(), 3, c_b), c_b.evolve() + + b6, c_c = mk_block(b4.id(), 3, c_c), c_c.evolve() + + for b in [b1, b2, b3, b4, b5, b6]: + follower.on_block(b) + + assert follower.tip() == b3 + assert [f.tip() for f in follower.forks] == [b5, b6] + assert follower.unimported_orphans(follower.tip_id()) == [b4, b5, b6] + + b7, c_a = mk_block(b3.id(), 4, c_a, orphaned_proofs=[b4, b5, b6]), c_a.evolve() + + follower.on_block(b7) + assert follower.tip() == b7 + + def test_transitive_orphan_reimports(self): + # Two forks, one after the other, with some complicated orphan imports. + # I don't have different line colors to differentiate orphans from parents + # so I've added o=XX to differentiate orphans from parents. + # + # - The first fork at b3(a) is not too interesting. + # - The second fork at b4(b) has both b6 and b7 importing b5 + # - crucially b7 uses the evolved commitment from b5 + # - Then finally b8 imports b7. + # + # proper orphan proof importing will be able to deal with the fact that + # b7's commitment was produced outside of the main branch AND the commitment + # is not part of the current list of orphans in b8 + # (b5 had already been imported, therefore it is not included as an orphan in b8) + # + # b1(a) - b2(a) - b3(a) - b4(b) - b6(b, o=b5) - b8(b, o=b7) + # \ \___ __/ __/ + # \ _x_ __/ + # \ / \_ / + # -b5(a)-----\-b7(a, o=b5) + + coins = [Coin(sk=i, value=10) for i in range(2)] + c_a, c_b = coins + config = mk_config(coins) + genesis = mk_genesis_state(coins) + follower = Follower(genesis, config) + + b1, c_a = mk_block(genesis.block, 1, c_a), c_a.evolve() + b2, c_a = mk_block(b1.id(), 2, c_a), c_a.evolve() + b3, c_a = mk_block(b2.id(), 3, c_a), c_a.evolve() + + b4, c_b = mk_block(b3.id(), 4, c_b), c_b.evolve() + b5, c_a = mk_block(b3.id(), 4, c_a), c_a.evolve() + + b6, c_b = mk_block(b4.id(), 5, c_b, orphaned_proofs=[b5]), c_b.evolve() + b7, c_a = mk_block(b4.id(), 5, c_a, orphaned_proofs=[b5]), c_a.evolve() + + b8, c_b = mk_block(b6.id(), 6, c_b, orphaned_proofs=[b7]), c_b.evolve() + + for b in [b1, b2, b3, b4, b5]: + follower.on_block(b) + + assert follower.tip() == b4 + assert follower.unimported_orphans(follower.tip_id()) == [b5] + + for b in [b6, b7]: + follower.on_block(b) + + assert follower.tip() == b6 + assert follower.unimported_orphans(follower.tip_id()) == [b7] + + follower.on_block(b8) + + assert follower.tip() == b8 + assert follower.unimported_orphans(follower.tip_id()) == [] diff --git a/cryptarchia/test_stake_relativization.py b/cryptarchia/test_stake_relativization.py new file mode 100644 index 0000000..6cc3f36 --- /dev/null +++ b/cryptarchia/test_stake_relativization.py @@ -0,0 +1,170 @@ +from unittest import TestCase +from dataclasses import dataclass +import itertools + +import numpy as np + +from .cryptarchia import Config, Coin, Slot +from .test_common import mk_config, mk_genesis_state, mk_block, TestNode, Follower + + +class TestStakeRelativization(TestCase): + def test_ledger_leader_counting(self): + coins = [Coin(sk=i, value=10) for i in range(2)] + c_a, c_b = coins + + config = mk_config(coins) + genesis = mk_genesis_state(coins) + + follower = Follower(genesis, config) + + # initially, there are 0 leaders + assert follower.tip_state().leader_count == 0 + + # after a block, 1 leader has been observed + b1 = mk_block(genesis.block, slot=1, coin=c_a) + follower.on_block(b1) + assert follower.tip_state().leader_count == 1 + + # on fork, tip state is not updated + orphan = mk_block(genesis.block, slot=1, coin=c_b) + follower.on_block(orphan) + assert follower.tip_state().block == b1.id() + assert follower.tip_state().leader_count == 1 + + # after orphan is adopted, leader count should jumpy by 2 (each orphan counts as a leader) + b2 = mk_block(b1.id(), slot=2, coin=c_a.evolve(), orphaned_proofs=[orphan]) + follower.on_block(b2) + assert follower.tip_state().block == b2.id() + assert follower.tip_state().leader_count == 3 + + def test_inference_on_empty_genesis_epoch(self): + coin = Coin(sk=0, value=10) + config = mk_config([coin]).replace( + initial_total_active_stake=20, + total_active_stake_learning_rate=0.5, + active_slot_coeff=0.5, + ) + genesis = mk_genesis_state([coin]) + node = TestNode(config, genesis, coin) + + # -- epoch 0 -- + + # ..... silence + + # -- epoch 1 -- + # Given no blocks produced in epoch 0, + + epoch1_state = node.epoch_state(Slot(config.epoch_length)) + + # given learning rate of 0.5 and 0 occupied slots in epoch 0, we should see + # inferred total stake drop by half in epoch 1 + assert epoch1_state.inferred_total_active_stake == 10 + + # -- epoch 2 -- + epoch1_state = node.epoch_state(Slot(config.epoch_length * 2)) + + # and again, we should see inferred total stake drop by half in epoch 2 given + # no occupied slots in epoch 1 + assert epoch1_state.inferred_total_active_stake == 5 + + def test_inferred_total_active_stake_close_to_true_total_stake(self): + PRINT_DEBUG = False + + seed = 0 + N = 3 + EPOCHS = 2 + + np.random.seed(seed) + + stake = np.array((np.random.pareto(10, N) + 1) * 1000, dtype=np.int64) + coins = [Coin(sk=i, value=int(s)) for i, s in enumerate(stake)] + + config = Config.cryptarchia_v0_0_1(stake.sum() * 2).replace(k=40) + genesis = mk_genesis_state(coins) + + nodes = [TestNode(config, genesis, c) for c in coins] + + T = config.epoch_length * EPOCHS + slot_leaders = np.zeros(T, dtype=np.int32) + for slot in map(Slot, range(T)): + proposed_blocks = [n.on_slot(slot) for n in nodes] + slot_leaders[slot.absolute_slot] = N - proposed_blocks.count(None) + + # now deliver the proposed blocks + for n_idx, node in enumerate(nodes): + # shuffle proposed blocks to simulate random delivery + block_order = list(range(N)) + np.random.shuffle(block_order) + for block_idx in block_order: + if block := proposed_blocks[block_idx]: + node.on_block(block) + + # Instead of inspecting state of each node, we group the nodes by their + # tip, and select a representative for each group to inspect. + # + # This makes debugging with large number of nodes more maneagable. + + grouped_by_tip = _group_by(nodes, lambda n: n.follower.tip_id()) + for group in grouped_by_tip.values(): + ref_node = group[0] + ref_epoch_state = ref_node.epoch_state(Slot(T)) + for node in group: + assert node.follower.tip_state() == ref_node.follower.tip_state() + assert node.epoch_state(Slot(T)) == ref_epoch_state + + reps = [g[0] for g in grouped_by_tip.values()] + + if PRINT_DEBUG: + print() + print("seed", seed) + print(f"T={T}, EPOCHS={EPOCHS}") + print( + f"lottery stats", + f"mean={slot_leaders.mean():.3f}", + f"var={slot_leaders.var():.3f}", + ) + print("true total stake\t", stake.sum()) + print("D_0\t", config.initial_total_stake) + + inferred_stake_by_epoch_by_rep = [ + [ + r.epoch_state(Slot(e * config.epoch_length)).total_stake() + for e in range(EPOCHS + 1) + ] + for r in reps + ] + print( + f"D_{list(range(EPOCHS + 1))}\n\t", + "\n\t".join( + [ + f"Rep {i}: {stakes}" + for i, stakes in inferred_stake_by_epoch_by_rep + ] + ), + ) + print("true leader count\t", slot_leaders.sum()) + print( + "follower leader counts\t", + [r.follower.tip_state().leader_count for r in reps], + ) + + assert all( + slot_leaders.sum() + 1 == len(n.follower.ledger_state) for n in nodes + ), f"{slot_leaders.sum() + 1}!={[len(n.follower.ledger_state) for n in nodes]}" + + for node in reps: + inferred_stake = node.epoch_state(Slot(T)).total_active_stake() + pct_err = ( + abs(stake.sum() - inferred_stake) / config.initial_total_active_stake + ) + eps = (1 - config.total_active_stake_learning_rate) ** EPOCHS + assert pct_err < eps, f"pct_err={pct_err} < eps={eps}" + + +def _group_by(iterable, key): + import itertools + + return { + k: list(group) for k, group in itertools.groupby(sorted(iterable, key=key), key) + }