From f484b1e98c44564c33672d49da23a42ed38ccc22 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 23:18:22 -0600 Subject: [PATCH 01/12] some fixes to finality_12 --- .../test_process_justification_and_finalization.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 0afa4f307..b1d94c4fb 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -189,12 +189,14 @@ def finalize_on_12(spec, state, epoch, support): # c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root + state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c2 state.current_justified_checkpoint = c2 - bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - bits[3] = 1 # mock 3rd latest epoch as justified + state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + state.justification_bits[0] = 1 # mock latest epoch as justified # mock the 1st latest epoch as justifiable, with 2nd as source add_mock_attestations(spec, state, epoch=epoch - 1, From 1885e285e3b57a51450910e77e19cf6d505b34c6 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 23:28:36 -0600 Subject: [PATCH 02/12] fix source/target epochs in test_12 --- .../test_process_justification_and_finalization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index b1d94c4fb..afb25fdae 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -187,8 +187,8 @@ def finalize_on_12(spec, state, epoch, support): # checkpoints for the epochs ago: # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) # c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) - c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + c2 = spec.Checkpoint(epoch=epoch - 1, root=b'\xbb' * 32) + c1 = spec.Checkpoint(epoch=epoch, root=b'\xcc' * 32) state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root From bc5e947efc01141909aaf952167d6b20e6eb9d63 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 23:38:14 -0600 Subject: [PATCH 03/12] aggregation_bitfield - bits --- .../test_process_justification_and_finalization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index afb25fdae..879c982f0 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -49,10 +49,10 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. attesting_count = math.ceil(size * att_ratio) - aggregation_bitfield = ((1 << attesting_count) - 1).to_bytes(length=((size + 7) // 8), byteorder='big') + aggregation_bits = [i < attesting_count for i in range(size)] attestations.append(spec.PendingAttestation( - aggregation_bitfield=aggregation_bitfield, + aggregation_bits=aggregation_bits, data=spec.AttestationData( beacon_block_root=b'\xaa' * 32, source=source, From 022f1e7108a2d6971c147b22a3fef2b0aacebbc9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 28 Jun 2019 23:59:20 -0600 Subject: [PATCH 04/12] fix source/target --- .../test_process_justification_and_finalization.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 879c982f0..246f20c9e 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -56,7 +56,7 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): data=spec.AttestationData( beacon_block_root=b'\xaa' * 32, source=source, - target=source, + target=target, crosslink=spec.Crosslink() ), inclusion_delay=1, @@ -187,8 +187,8 @@ def finalize_on_12(spec, state, epoch, support): # checkpoints for the epochs ago: # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) # c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) - c2 = spec.Checkpoint(epoch=epoch - 1, root=b'\xbb' * 32) - c1 = spec.Checkpoint(epoch=epoch, root=b'\xcc' * 32) + c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) + c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root From 129fd6297e1218f0655dda2bf8eb2b49a75bd387 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Sat, 29 Jun 2019 00:03:06 -0600 Subject: [PATCH 05/12] add shard to mock crosslink to separate attestations from eachother --- .../test_process_justification_and_finalization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 246f20c9e..adcf6ea1d 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -57,7 +57,7 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): beacon_block_root=b'\xaa' * 32, source=source, target=target, - crosslink=spec.Crosslink() + crosslink=spec.Crosslink(shard=shard) ), inclusion_delay=1, )) From b05bebf45c1bfe31fdf9ee195283827be4f145e0 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 12:23:03 +0200 Subject: [PATCH 06/12] Fix list slicing --- ..._process_justification_and_finalization.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index adcf6ea1d..3e2f6e967 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -74,13 +74,15 @@ def finalize_on_234(spec, state, epoch, support): c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - # c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + state.block_roots[spec.get_epoch_start_slot(c4.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c4.root + state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root + state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c4 state.current_justified_checkpoint = c3 - bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - bits[3:4] = [1, 1] # mock 3rd and 4th latest epochs as justified + state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + state.justification_bits[1:3] = [1, 1] # mock 3rd and 4th latest epochs as justified # mock the 2nd latest epoch as justifiable, with 4th as source add_mock_attestations(spec, state, epoch=epoch - 2, @@ -109,16 +111,16 @@ def finalize_on_23(spec, state, epoch, support): # 3210x -- justification bitfield indices # 01*0. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) - # c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root + state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c3 state.current_justified_checkpoint = c3 - bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - bits[2] = 1 # mock 3rd latest epoch as justified + state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + state.justification_bits[1] = 1 # mock 2nd latest epoch as justified # mock the 2nd latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 2, @@ -147,16 +149,18 @@ def finalize_on_123(spec, state, epoch, support): # 3210x -- justification bitfield indices # 011*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) + state.block_roots[spec.get_epoch_start_slot(c3.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c3.root + state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root + state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root old_finalized = state.finalized_checkpoint state.previous_justified_checkpoint = c3 state.current_justified_checkpoint = c2 - bits = state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - bits[1:2] = [1, 1] # mock 2rd and 3th latest epochs as justified + state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() + state.justification_bits[0:2] = [1, 1] # mock 1st and 2nd latest epochs as justified # mock the 1st latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 1, @@ -185,8 +189,6 @@ def finalize_on_12(spec, state, epoch, support): # 3210 -- justification bitfield indices # 001*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: - # c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) - # c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) state.block_roots[spec.get_epoch_start_slot(c2.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c2.root From f9ca7c97c9bc4d994ba88ee721afcfc04336673b Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 13:01:59 +0200 Subject: [PATCH 07/12] Fix 123 finalisation --- .../test_process_justification_and_finalization.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 3e2f6e967..42ef73084 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -149,6 +149,7 @@ def finalize_on_123(spec, state, epoch, support): # 3210x -- justification bitfield indices # 011*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: + c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) c1 = spec.Checkpoint(epoch=epoch - 1, root=b'\xcc' * 32) @@ -157,7 +158,7 @@ def finalize_on_123(spec, state, epoch, support): state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root old_finalized = state.finalized_checkpoint - state.previous_justified_checkpoint = c3 + state.previous_justified_checkpoint = c4 state.current_justified_checkpoint = c2 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() state.justification_bits[0:2] = [1, 1] # mock 1st and 2nd latest epochs as justified From 2eca6ef09d6d94656863907ba2fed5213788f213 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 13:10:09 +0200 Subject: [PATCH 08/12] Corrects justification comments --- .../test_process_justification_and_finalization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 42ef73084..02439f664 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -82,7 +82,7 @@ def finalize_on_234(spec, state, epoch, support): state.previous_justified_checkpoint = c4 state.current_justified_checkpoint = c3 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[1:3] = [1, 1] # mock 3rd and 4th latest epochs as justified + state.justification_bits[1:3] = [1, 1] # mock 2nd and 3rd latest epochs as justified # mock the 2nd latest epoch as justifiable, with 4th as source add_mock_attestations(spec, state, epoch=epoch - 2, From 0680d8cc536f72a863d9e9ec1975a2fd5ff1729c Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 14:35:07 +0200 Subject: [PATCH 09/12] Makes justification ratios more marginal --- ..._process_justification_and_finalization.py | 56 +++++++++---------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 02439f664..406bdbf7f 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -25,7 +25,7 @@ def get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) return size -def add_mock_attestations(spec, state, epoch, att_ratio, source, target): +def add_mock_attestations(spec, state, epoch, source, target, sufficient_support=False): # we must be at the end of the epoch assert (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0 @@ -48,7 +48,8 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): size = get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. - attesting_count = math.ceil(size * att_ratio) + attesting_count = math.ceil(size * 2 /3) + attesting_count = attesting_count if sufficient_support else attesting_count - 1 aggregation_bits = [i < attesting_count for i in range(size)] attestations.append(spec.PendingAttestation( @@ -63,7 +64,7 @@ def add_mock_attestations(spec, state, epoch, att_ratio, source, target): )) -def finalize_on_234(spec, state, epoch, support): +def finalize_on_234(spec, state, epoch, sufficient_support): assert epoch > 4 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -86,14 +87,14 @@ def finalize_on_234(spec, state, epoch, support): # mock the 2nd latest epoch as justifiable, with 4th as source add_mock_attestations(spec, state, epoch=epoch - 2, - att_ratio=support, source=c4, - target=c2) + target=c2, + sufficient_support=sufficient_support) # process! yield from run_process_just_and_fin(spec, state) - if support >= (2 / 3): + if sufficient_support: assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c2 # changed to 2nd latest assert state.finalized_checkpoint == c4 # finalized old previous justified epoch @@ -103,7 +104,7 @@ def finalize_on_234(spec, state, epoch, support): assert state.finalized_checkpoint == old_finalized # no new finalized -def finalize_on_23(spec, state, epoch, support): +def finalize_on_23(spec, state, epoch, sufficient_support): assert epoch > 3 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -124,14 +125,14 @@ def finalize_on_23(spec, state, epoch, support): # mock the 2nd latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 2, - att_ratio=support, source=c3, - target=c2) + target=c2, + sufficient_support=sufficient_support) # process! yield from run_process_just_and_fin(spec, state) - if support >= (2 / 3): + if sufficient_support: assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c2 # changed to 2nd latest assert state.finalized_checkpoint == c3 # finalized old previous justified epoch @@ -141,7 +142,7 @@ def finalize_on_23(spec, state, epoch, support): assert state.finalized_checkpoint == old_finalized # no new finalized -def finalize_on_123(spec, state, epoch, support): +def finalize_on_123(spec, state, epoch, sufficient_support): assert epoch > 3 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -165,14 +166,14 @@ def finalize_on_123(spec, state, epoch, support): # mock the 1st latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 1, - att_ratio=support, source=c3, - target=c1) + target=c1, + sufficient_support=sufficient_support) # process! yield from run_process_just_and_fin(spec, state) - if support >= (2 / 3): + if sufficient_support: assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c1 # changed to 1st latest assert state.finalized_checkpoint == c2 # finalized old current @@ -182,7 +183,7 @@ def finalize_on_123(spec, state, epoch, support): assert state.finalized_checkpoint == old_finalized # no new finalized -def finalize_on_12(spec, state, epoch, support): +def finalize_on_12(spec, state, epoch, sufficient_support): assert epoch > 2 state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch @@ -203,14 +204,14 @@ def finalize_on_12(spec, state, epoch, support): # mock the 1st latest epoch as justifiable, with 2nd as source add_mock_attestations(spec, state, epoch=epoch - 1, - att_ratio=support, source=c2, - target=c1) + target=c1, + sufficient_support=sufficient_support) # process! yield from run_process_just_and_fin(spec, state) - if support >= (2 / 3): + if sufficient_support: assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c1 # changed to 1st latest assert state.finalized_checkpoint == c2 # finalized previous justified epoch @@ -223,49 +224,46 @@ def finalize_on_12(spec, state, epoch, support): @with_all_phases @spec_state_test def test_234_ok_support(spec, state): - yield from finalize_on_234(spec, state, 5, 1.0) + yield from finalize_on_234(spec, state, 5, True) @with_all_phases @spec_state_test def test_234_poor_support(spec, state): - yield from finalize_on_234(spec, state, 5, 0.6) + yield from finalize_on_234(spec, state, 5, False) @with_all_phases @spec_state_test def test_23_ok_support(spec, state): - yield from finalize_on_23(spec, state, 4, 1.0) + yield from finalize_on_23(spec, state, 4, True) @with_all_phases @spec_state_test def test_23_poor_support(spec, state): - yield from finalize_on_23(spec, state, 4, 0.6) + yield from finalize_on_23(spec, state, 4, False) @with_all_phases @spec_state_test def test_123_ok_support(spec, state): - yield from finalize_on_123(spec, state, 4, 1.0) + yield from finalize_on_123(spec, state, 4, True) @with_all_phases @spec_state_test def test_123_poor_support(spec, state): - yield from finalize_on_123(spec, state, 4, 0.6) + yield from finalize_on_123(spec, state, 4, False) @with_all_phases @spec_state_test def test_12_ok_support(spec, state): - yield from finalize_on_12(spec, state, 3, 1.0) + yield from finalize_on_12(spec, state, 3, True) @with_all_phases @spec_state_test def test_12_poor_support(spec, state): - yield from finalize_on_12(spec, state, 3, 0.6) - - -# TODO: bring ratios closer to 2/3 for edge case testing. + yield from finalize_on_12(spec, state, 3, False) From 4ed7af7bacb2602d97ea67596b8b21d4b019b150 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 15:48:39 +0200 Subject: [PATCH 10/12] mock attestation refactor --- ..._process_justification_and_finalization.py | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 406bdbf7f..66e2095b4 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -1,4 +1,3 @@ -import math from eth2spec.test.context import spec_state_test, with_all_phases from eth2spec.test.phase_0.epoch_processing.run_epoch_process_base import ( run_epoch_processing_with @@ -39,29 +38,37 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support else: raise Exception(f"cannot include attestations in epoch ${epoch} from epoch ${current_epoch}") - committee_count = spec.get_epoch_committee_count(state, epoch) - indices = spec.get_active_validator_indices(state, epoch) - epoch_start_shard = spec.get_epoch_start_shard(state, epoch) + total_balance = spec.get_total_active_balance(state) + remaining_balance = total_balance * 2 // 3 + epoch_start_slot = spec.get_epoch_start_slot(epoch) for slot in range(epoch_start_slot, epoch_start_slot + spec.SLOTS_PER_EPOCH): for shard in get_shards_for_slot(spec, state, slot): - size = get_committee_size(spec, epoch_start_shard, shard, committee_count, indices) + committee = spec.get_crosslink_committee(state, spec.slot_to_epoch(slot), shard) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. - attesting_count = math.ceil(size * 2 /3) - attesting_count = attesting_count if sufficient_support else attesting_count - 1 - aggregation_bits = [i < attesting_count for i in range(size)] - attestations.append(spec.PendingAttestation( - aggregation_bits=aggregation_bits, - data=spec.AttestationData( - beacon_block_root=b'\xaa' * 32, - source=source, - target=target, - crosslink=spec.Crosslink(shard=shard) - ), - inclusion_delay=1, - )) + aggregation_bits = [0] * len(committee) + for v in range(len(committee) * 2 // 3 + 1): + if remaining_balance > 0: + remaining_balance -= state.validators[v].effective_balance + aggregation_bits[v] = 1 + elif not sufficient_support: + aggregation_bits[v - 1] = 0 + break + else: + break + + attestations.append(spec.PendingAttestation( + aggregation_bits=aggregation_bits, + data=spec.AttestationData( + beacon_block_root=b'\xaa' * 32, + source=source, + target=target, + crosslink=spec.Crosslink(shard=shard) + ), + inclusion_delay=1, + )) def finalize_on_234(spec, state, epoch, sufficient_support): From 0c29c5125f0e890e5bdd7d9e14afd166d0b6b260 Mon Sep 17 00:00:00 2001 From: Carl Beekhuizen Date: Sat, 29 Jun 2019 17:10:12 +0200 Subject: [PATCH 11/12] Finnish refactor --- .../test_process_justification_and_finalization.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 66e2095b4..50a0a0cab 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -16,14 +16,6 @@ def get_shards_for_slot(spec, state, slot): return [shard + i for i in range(committees_per_slot)] -def get_committee_size(spec, epoch_start_shard, shard, committee_count, indices): - committee_index = (shard + spec.SHARD_COUNT - epoch_start_shard) % spec.SHARD_COUNT - start = (len(indices) * committee_index) // committee_count - end = (len(indices) * (committee_index + 1)) // committee_count - size = end - start - return size - - def add_mock_attestations(spec, state, epoch, source, target, sufficient_support=False): # we must be at the end of the epoch assert (state.slot + 1) % spec.SLOTS_PER_EPOCH == 0 From 5d633bfdf36c5b4a9e069e302cecd70bbd782b13 Mon Sep 17 00:00:00 2001 From: protolambda Date: Sat, 29 Jun 2019 18:14:17 +0200 Subject: [PATCH 12/12] bugfix attestation creation so that it works on mainnet with multiple committees per slot, and improve bitfield index descriptions --- ..._process_justification_and_finalization.py | 63 ++++++++++--------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py index 50a0a0cab..d043d8d97 100644 --- a/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py +++ b/test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py @@ -36,6 +36,11 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support epoch_start_slot = spec.get_epoch_start_slot(epoch) for slot in range(epoch_start_slot, epoch_start_slot + spec.SLOTS_PER_EPOCH): for shard in get_shards_for_slot(spec, state, slot): + # Check if we already have had sufficient balance. (and undone if we don't want it). + # If so, do not create more attestations. (we do not have empty pending attestations normally anyway) + if remaining_balance < 0: + return + committee = spec.get_crosslink_committee(state, spec.slot_to_epoch(slot), shard) # Create a bitfield filled with the given count per attestation, # exactly on the right-most part of the committee field. @@ -45,22 +50,23 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support if remaining_balance > 0: remaining_balance -= state.validators[v].effective_balance aggregation_bits[v] = 1 - elif not sufficient_support: - aggregation_bits[v - 1] = 0 - break else: break - attestations.append(spec.PendingAttestation( - aggregation_bits=aggregation_bits, - data=spec.AttestationData( - beacon_block_root=b'\xaa' * 32, - source=source, - target=target, - crosslink=spec.Crosslink(shard=shard) - ), - inclusion_delay=1, - )) + # remove just one attester to make the marginal support insufficient + if not sufficient_support: + aggregation_bits[aggregation_bits.index(1)] = 0 + + attestations.append(spec.PendingAttestation( + aggregation_bits=aggregation_bits, + data=spec.AttestationData( + beacon_block_root=b'\xaa' * 32, + source=source, + target=target, + crosslink=spec.Crosslink(shard=shard) + ), + inclusion_delay=1, + )) def finalize_on_234(spec, state, epoch, sufficient_support): @@ -82,7 +88,7 @@ def finalize_on_234(spec, state, epoch, sufficient_support): state.previous_justified_checkpoint = c4 state.current_justified_checkpoint = c3 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[1:3] = [1, 1] # mock 2nd and 3rd latest epochs as justified + state.justification_bits[1:3] = [1, 1] # mock 3rd and 4th latest epochs as justified (indices are pre-shift) # mock the 2nd latest epoch as justifiable, with 4th as source add_mock_attestations(spec, state, epoch=epoch - 2, @@ -93,12 +99,11 @@ def finalize_on_234(spec, state, epoch, sufficient_support): # process! yield from run_process_just_and_fin(spec, state) + assert state.previous_justified_checkpoint == c3 # changed to old current if sufficient_support: - assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c2 # changed to 2nd latest assert state.finalized_checkpoint == c4 # finalized old previous justified epoch else: - assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c3 # still old current assert state.finalized_checkpoint == old_finalized # no new finalized @@ -108,7 +113,8 @@ def finalize_on_23(spec, state, epoch, sufficient_support): state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch # 43210 -- epochs ago - # 3210x -- justification bitfield indices + # 210xx -- justification bitfield indices (pre shift) + # 3210x -- justification bitfield indices (post shift) # 01*0. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: c3 = spec.Checkpoint(epoch=epoch - 3, root=b'\xaa' * 32) @@ -120,7 +126,7 @@ def finalize_on_23(spec, state, epoch, sufficient_support): state.previous_justified_checkpoint = c3 state.current_justified_checkpoint = c3 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[1] = 1 # mock 2nd latest epoch as justified + state.justification_bits[1] = 1 # mock 3rd latest epoch as justified (indices are pre-shift) # mock the 2nd latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 2, @@ -131,12 +137,11 @@ def finalize_on_23(spec, state, epoch, sufficient_support): # process! yield from run_process_just_and_fin(spec, state) + assert state.previous_justified_checkpoint == c3 # changed to old current if sufficient_support: - assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c2 # changed to 2nd latest assert state.finalized_checkpoint == c3 # finalized old previous justified epoch else: - assert state.previous_justified_checkpoint == c3 # changed to old current assert state.current_justified_checkpoint == c3 # still old current assert state.finalized_checkpoint == old_finalized # no new finalized @@ -146,7 +151,8 @@ def finalize_on_123(spec, state, epoch, sufficient_support): state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch # 43210 -- epochs ago - # 3210x -- justification bitfield indices + # 210xx -- justification bitfield indices (pre shift) + # 3210x -- justification bitfield indices (post shift) # 011*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: c4 = spec.Checkpoint(epoch=epoch - 4, root=b'\xaa' * 32) @@ -158,10 +164,10 @@ def finalize_on_123(spec, state, epoch, sufficient_support): state.block_roots[spec.get_epoch_start_slot(c1.epoch) % spec.SLOTS_PER_HISTORICAL_ROOT] = c1.root old_finalized = state.finalized_checkpoint - state.previous_justified_checkpoint = c4 + state.previous_justified_checkpoint = c4 # not c3, otherwise finalize 23 would trigger. state.current_justified_checkpoint = c2 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[0:2] = [1, 1] # mock 1st and 2nd latest epochs as justified + state.justification_bits[0:2] = [1, 1] # mock 2nd and 3rd latest epochs as justified (indices are pre-shift) # mock the 1st latest epoch as justifiable, with 3rd as source add_mock_attestations(spec, state, epoch=epoch - 1, @@ -172,12 +178,11 @@ def finalize_on_123(spec, state, epoch, sufficient_support): # process! yield from run_process_just_and_fin(spec, state) + assert state.previous_justified_checkpoint == c2 # changed to old current if sufficient_support: - assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c1 # changed to 1st latest assert state.finalized_checkpoint == c2 # finalized old current else: - assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c2 # still old current assert state.finalized_checkpoint == old_finalized # no new finalized @@ -187,7 +192,8 @@ def finalize_on_12(spec, state, epoch, sufficient_support): state.slot = (spec.SLOTS_PER_EPOCH * epoch) - 1 # skip ahead to just before epoch # 43210 -- epochs ago - # 3210 -- justification bitfield indices + # 210xx -- justification bitfield indices (pre shift) + # 3210x -- justification bitfield indices (post shift) # 001*. -- justification bitfield contents, . = this epoch, * is being justified now # checkpoints for the epochs ago: c2 = spec.Checkpoint(epoch=epoch - 2, root=b'\xbb' * 32) @@ -199,7 +205,7 @@ def finalize_on_12(spec, state, epoch, sufficient_support): state.previous_justified_checkpoint = c2 state.current_justified_checkpoint = c2 state.justification_bits = spec.Bitvector[spec.JUSTIFICATION_BITS_LENGTH]() - state.justification_bits[0] = 1 # mock latest epoch as justified + state.justification_bits[0] = 1 # mock 2nd latest epoch as justified (this is pre-shift) # mock the 1st latest epoch as justifiable, with 2nd as source add_mock_attestations(spec, state, epoch=epoch - 1, @@ -210,12 +216,11 @@ def finalize_on_12(spec, state, epoch, sufficient_support): # process! yield from run_process_just_and_fin(spec, state) + assert state.previous_justified_checkpoint == c2 # changed to old current if sufficient_support: - assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c1 # changed to 1st latest assert state.finalized_checkpoint == c2 # finalized previous justified epoch else: - assert state.previous_justified_checkpoint == c2 # changed to old current assert state.current_justified_checkpoint == c2 # still old current assert state.finalized_checkpoint == old_finalized # no new finalized