Merge pull request #756 from mir-protocol/rlp_fixes

RLP related fixes

evm/Cargo.toml

@@ -24,6 +24,7 @@ rand = "0.8.5"
 rand_chacha = "0.3.1"
 ripemd = "0.1.3"
 rlp = "0.5.1"
+rlp-derive = "0.1.0"
 serde = { version = "1.0.144", features = ["derive"] }
 sha2 = "0.10.2"
 tiny-keccak = "2.0.2"

evm/src/cpu/kernel/asm/mpt/hash_trie_specific.asm

@@ -43,33 +43,40 @@ encode_account:
     // to determine their contribution, while the other two fields are fixed
     // 32-bytes integers.
     DUP2 %mload_trie_data // nonce = value[0]
-    %scalar_rlp_len
+    %rlp_scalar_len
     // stack: nonce_rlp_len, rlp_pos, value_ptr, retdest
     DUP3 %add_const(1) %mload_trie_data // balance = value[1]
-    %scalar_rlp_len
+    %rlp_scalar_len
-    // stack: balance_rlp_lenm, nonce_rlp_len, rlp_pos, value_ptr, retdest
+    // stack: balance_rlp_len, nonce_rlp_len, rlp_pos, value_ptr, retdest
     PUSH 66 // storage_root and code_hash fields each take 1 + 32 bytes
     ADD ADD
     // stack: payload_len, rlp_pos, value_ptr, retdest
     SWAP1
+    // stack: rlp_pos, payload_len, value_ptr, retdest
+    DUP2 %rlp_list_len
+    // stack: list_len, rlp_pos, payload_len, value_ptr, retdest
+    SWAP1
+    // stack: rlp_pos, list_len, payload_len, value_ptr, retdest
+    %encode_rlp_multi_byte_string_prefix
+    // stack: rlp_pos_2, payload_len, value_ptr, retdest
     %encode_rlp_list_prefix
-    // stack: rlp_pos', value_ptr, retdest
+    // stack: rlp_pos_3, value_ptr, retdest
     DUP2 %mload_trie_data // nonce = value[0]
-    // stack: nonce, rlp_pos', value_ptr, retdest
+    // stack: nonce, rlp_pos_3, value_ptr, retdest
     SWAP1 %encode_rlp_scalar
-    // stack: rlp_pos'', value_ptr, retdest
+    // stack: rlp_pos_4, value_ptr, retdest
     DUP2 %add_const(1) %mload_trie_data // balance = value[1]
-    // stack: balance, rlp_pos'', value_ptr, retdest
+    // stack: balance, rlp_pos_4, value_ptr, retdest
     SWAP1 %encode_rlp_scalar
-    // stack: rlp_pos''', value_ptr, retdest
+    // stack: rlp_pos_5, value_ptr, retdest
     DUP2 %add_const(2) %mload_trie_data // storage_root = value[2]
-    // stack: storage_root, rlp_pos''', value_ptr, retdest
+    // stack: storage_root, rlp_pos_5, value_ptr, retdest
     SWAP1 %encode_rlp_256
-    // stack: rlp_pos'''', value_ptr, retdest
+    // stack: rlp_pos_6, value_ptr, retdest
     SWAP1 %add_const(3) %mload_trie_data // code_hash = value[3]
-    // stack: code_hash, rlp_pos'''', retdest
+    // stack: code_hash, rlp_pos_6, retdest
     SWAP1 %encode_rlp_256
-    // stack: rlp_pos''''', retdest
+    // stack: rlp_pos_7, retdest
     SWAP1
     JUMP
 

evm/src/cpu/kernel/asm/rlp/encode.asm

@@ -12,7 +12,7 @@ global encode_rlp_scalar:
     // stack: pos, scalar, retdest
     %stack (pos, scalar) -> (pos, scalar, pos)
     // stack: pos, scalar, pos, retdest
-    %mstore_current(@SEGMENT_RLP_RAW)
+    %mstore_rlp
     // stack: pos, retdest
     %add_const(1)
     // stack: pos', retdest
@@ -73,7 +73,7 @@ encode_rlp_fixed:
     // stack: first_byte, len, pos, string, retdest
     DUP3
     // stack: pos, first_byte, len, pos, string, retdest
-    %mstore_current(@SEGMENT_RLP_RAW)
+    %mstore_rlp
     // stack: len, pos, string, retdest
     SWAP1
     %add_const(1) // increment pos
@@ -86,6 +86,64 @@ encode_rlp_fixed_finish:
     SWAP1
     JUMP
 
+// Writes the RLP prefix for a string of the given length. This does not handle
+// the trivial encoding of certain single-byte strings, as handling that would
+// require access to the actual string, while this method only accesses its
+// length. This method should generally be used only when we know a string
+// contains at least two bytes.
+//
+// Pre stack: pos, str_len, retdest
+// Post stack: pos'
+global encode_rlp_multi_byte_string_prefix:
+    // stack: pos, str_len, retdest
+    DUP2 %gt_const(55)
+    // stack: str_len > 55, pos, str_len, retdest
+    %jumpi(encode_rlp_multi_byte_string_prefix_large)
+    // Medium case; prefix is 0x80 + str_len.
+    // stack: pos, str_len, retdest
+    SWAP1 %add_const(0x80)
+    // stack: prefix, pos, retdest
+    DUP2
+    // stack: pos, prefix, pos, retdest
+    %mstore_rlp
+    // stack: pos, retdest
+    %increment
+    // stack: pos', retdest
+    SWAP1
+    JUMP
+encode_rlp_multi_byte_string_prefix_large:
+    // Large case; prefix is 0xb7 + len_of_len, followed by str_len.
+    // stack: pos, str_len, retdest
+    DUP2
+    %num_bytes
+    // stack: len_of_len, pos, str_len, retdest
+    SWAP1
+    DUP2 // len_of_len
+    %add_const(0xb7)
+    // stack: first_byte, pos, len_of_len, str_len, retdest
+    DUP2
+    // stack: pos, first_byte, pos, len_of_len, str_len, retdest
+    %mstore_rlp
+    // stack: pos, len_of_len, str_len, retdest
+    %increment
+    // stack: pos', len_of_len, str_len, retdest
+    %stack (pos, len_of_len, str_len)
+        -> (pos, str_len, len_of_len,
+            encode_rlp_multi_byte_string_prefix_large_done_writing_len)
+    %jump(mstore_unpacking_rlp)
+encode_rlp_multi_byte_string_prefix_large_done_writing_len:
+    // stack: pos'', retdest
+    SWAP1
+    JUMP
+
+%macro encode_rlp_multi_byte_string_prefix
+    %stack (pos, str_len) -> (pos, str_len, %%after)
+    %jump(encode_rlp_multi_byte_string_prefix)
+%%after:
+%endmacro
+
+// Writes the RLP prefix for a list with the given payload length.
+//
 // Pre stack: pos, payload_len, retdest
 // Post stack: pos'
 global encode_rlp_list_prefix:
@@ -116,10 +174,9 @@ encode_rlp_list_prefix_large:
     // stack: len_of_len, pos, payload_len, retdest
     SWAP1 %add_const(1)
     // stack: pos', len_of_len, payload_len, retdest
-    %stack (pos, len_of_len, payload_len, retdest)
+    %stack (pos, len_of_len, payload_len)
         -> (pos, payload_len, len_of_len,
-            encode_rlp_list_prefix_large_done_writing_len,
+            encode_rlp_list_prefix_large_done_writing_len)
-            retdest)
     %jump(mstore_unpacking_rlp)
 encode_rlp_list_prefix_large_done_writing_len:
     // stack: pos'', retdest
@@ -198,7 +255,7 @@ prepend_rlp_list_prefix_big_done_writing_len:
 
 // Given some scalar, compute the number of bytes used in its RLP encoding,
 // including any length prefix.
-%macro scalar_rlp_len
+%macro rlp_scalar_len
     // stack: scalar
     // Since the scalar fits in a word, we can't hit the large (>55 byte)
     // case, so we just check for small vs medium.
@@ -207,12 +264,36 @@ prepend_rlp_list_prefix_big_done_writing_len:
     %jumpi(%%medium)
     // Small case; result is 1.
     %stack (scalar) -> (1)
+    %jump(%%finish)
 %%medium:
     // stack: scalar
     %num_bytes
     // stack: scalar_bytes
     %add_const(1) // Account for the length prefix.
     // stack: rlp_len
+%%finish:
+%endmacro
+
+// Given some list with the given payload length, compute the number of bytes
+// used in its RLP encoding, including the list prefix.
+%macro rlp_list_len
+    // stack: payload_len
+    DUP1 %gt_const(55)
+    // stack: is_large, payload_len
+    %jumpi(%%large)
+    // Small case; prefix is a single byte.
+    %increment
+    // stack: 1 + payload_len
+    %jump(%%finish)
+%%large:
+    // Prefix is 1 byte containing len_of_len, followed by len_of_len bytes containing len.
+    // stack: payload_len
+    DUP1 %num_bytes
+    // stack: len_of_len, payload_len
+    %increment
+    // stack: prefix_len, payload_len
+    ADD
+%%finish:
 %endmacro
 
 // Like mstore_unpacking, but specifically for the RLP segment.

evm/src/cpu/kernel/tests/mpt/hash.rs

@@ -1,22 +1,21 @@
 use anyhow::Result;
 use eth_trie_utils::partial_trie::{Nibbles, PartialTrie};
 use ethereum_types::{BigEndianHash, H256, U256};
-use hex_literal::hex;
 
 use crate::cpu::kernel::aggregator::KERNEL;
 use crate::cpu::kernel::interpreter::Interpreter;
-use crate::generation::mpt::all_mpt_prover_inputs_reversed;
+use crate::generation::mpt::{all_mpt_prover_inputs_reversed, AccountRlp};
 use crate::generation::TrieInputs;
 
 #[test]
 fn mpt_hash() -> Result<()> {
-    let nonce = U256::from(1111);
+    let account = AccountRlp {
-    let balance = U256::from(2222);
+        nonce: U256::from(1111),
-    let storage_root = U256::from(3333);
+        balance: U256::from(2222),
-    let code_hash = U256::from(4444);
+        storage_root: H256::from_uint(&U256::from(3333)),
-
+        code_hash: H256::from_uint(&U256::from(4444)),
-    let account = &[nonce, balance, storage_root, code_hash];
+    };
-    let account_rlp = rlp::encode_list(account);
+    let account_rlp = rlp::encode(&account);
 
     // TODO: Try this more "advanced" trie.
     // let state_trie = state_trie_ext_to_account_leaf(account_rlp.to_vec());
@@ -27,11 +26,7 @@ fn mpt_hash() -> Result<()> {
         },
         value: account_rlp.to_vec(),
     };
-    // TODO: It seems like calc_hash isn't giving the expected hash yet, so for now, I'm using a
+    let state_trie_hash = state_trie.calc_hash();
-    // hardcoded hash obtained from py-evm.
-    // let state_trie_hash = state_trie.calc_hash();
-    let state_trie_hash =
-        hex!("e38d6053838fe057c865ec0c74a8f0de21865d74fac222a2d3241fe57c9c3a0f").into();
 
     let trie_inputs = TrieInputs {
         state_trie,

evm/src/cpu/kernel/tests/mpt/load.rs

@@ -1,22 +1,23 @@
 use anyhow::Result;
-use ethereum_types::U256;
+use ethereum_types::{BigEndianHash, H256, U256};
 
 use crate::cpu::kernel::aggregator::KERNEL;
 use crate::cpu::kernel::constants::global_metadata::GlobalMetadata;
 use crate::cpu::kernel::constants::trie_type::PartialTrieType;
 use crate::cpu::kernel::interpreter::Interpreter;
 use crate::cpu::kernel::tests::mpt::state_trie_ext_to_account_leaf;
-use crate::generation::mpt::all_mpt_prover_inputs_reversed;
+use crate::generation::mpt::{all_mpt_prover_inputs_reversed, AccountRlp};
 use crate::generation::TrieInputs;
 
 #[test]
 fn load_all_mpts() -> Result<()> {
-    let nonce = U256::from(1111);
+    let account = AccountRlp {
-    let balance = U256::from(2222);
+        nonce: U256::from(1111),
-    let storage_root = U256::from(3333);
+        balance: U256::from(2222),
-    let code_hash = U256::from(4444);
+        storage_root: H256::from_uint(&U256::from(3333)),
-
+        code_hash: H256::from_uint(&U256::from(4444)),
-    let account_rlp = rlp::encode_list(&[nonce, balance, storage_root, code_hash]);
+    };
+    let account_rlp = rlp::encode(&account);
 
     let trie_inputs = TrieInputs {
         state_trie: state_trie_ext_to_account_leaf(account_rlp.to_vec()),
@@ -47,10 +48,10 @@ fn load_all_mpts() -> Result<()> {
             type_leaf,
             3.into(),     // 3 nibbles
             0xDEF.into(), // key part
-            nonce,
+            account.nonce,
-            balance,
+            account.balance,
-            storage_root,
+            account.storage_root.into_uint(),
-            code_hash,
+            account.code_hash.into_uint(),
             type_empty,
             type_empty,
         ]

evm/src/cpu/kernel/tests/mpt/read.rs

@@ -1,22 +1,22 @@
 use anyhow::Result;
-use ethereum_types::U256;
+use ethereum_types::{BigEndianHash, H256, U256};
 
 use crate::cpu::kernel::aggregator::KERNEL;
 use crate::cpu::kernel::constants::global_metadata::GlobalMetadata;
 use crate::cpu::kernel::interpreter::Interpreter;
 use crate::cpu::kernel::tests::mpt::state_trie_ext_to_account_leaf;
-use crate::generation::mpt::all_mpt_prover_inputs_reversed;
+use crate::generation::mpt::{all_mpt_prover_inputs_reversed, AccountRlp};
 use crate::generation::TrieInputs;
 
 #[test]
 fn mpt_read() -> Result<()> {
-    let nonce = U256::from(1111);
+    let account = AccountRlp {
-    let balance = U256::from(2222);
+        nonce: U256::from(1111),
-    let storage_root = U256::from(3333);
+        balance: U256::from(2222),
-    let code_hash = U256::from(4444);
+        storage_root: H256::from_uint(&U256::from(3333)),
-
+        code_hash: H256::from_uint(&U256::from(4444)),
-    let account = &[nonce, balance, storage_root, code_hash];
+    };
-    let account_rlp = rlp::encode_list(account);
+    let account_rlp = rlp::encode(&account);
 
     let trie_inputs = TrieInputs {
         state_trie: state_trie_ext_to_account_leaf(account_rlp.to_vec()),
@@ -44,8 +44,11 @@ fn mpt_read() -> Result<()> {
 
     assert_eq!(interpreter.stack().len(), 1);
     let result_ptr = interpreter.stack()[0].as_usize();
-    let result = &interpreter.get_trie_data()[result_ptr..][..account.len()];
+    let result = &interpreter.get_trie_data()[result_ptr..][..4];
-    assert_eq!(result, account);
+    assert_eq!(result[0], account.nonce);
+    assert_eq!(result[1], account.balance);
+    assert_eq!(result[2], account.storage_root.into_uint());
+    assert_eq!(result[3], account.code_hash.into_uint());
 
     Ok(())
 }

evm/src/generation/mpt.rs

@@ -1,9 +1,18 @@
 use eth_trie_utils::partial_trie::PartialTrie;
-use ethereum_types::U256;
+use ethereum_types::{BigEndianHash, H256, U256};
+use rlp_derive::{RlpDecodable, RlpEncodable};
 
 use crate::cpu::kernel::constants::trie_type::PartialTrieType;
 use crate::generation::TrieInputs;
 
+#[derive(RlpEncodable, RlpDecodable, Debug)]
+pub(crate) struct AccountRlp {
+    pub(crate) nonce: U256,
+    pub(crate) balance: U256,
+    pub(crate) storage_root: H256,
+    pub(crate) code_hash: H256,
+}
+
 pub(crate) fn all_mpt_prover_inputs_reversed(trie_inputs: &TrieInputs) -> Vec<U256> {
     let mut inputs = all_mpt_prover_inputs(trie_inputs);
     inputs.reverse();
@@ -15,7 +24,13 @@ pub(crate) fn all_mpt_prover_inputs(trie_inputs: &TrieInputs) -> Vec<U256> {
     let mut prover_inputs = vec![];
 
     mpt_prover_inputs(&trie_inputs.state_trie, &mut prover_inputs, &|rlp| {
-        rlp::decode_list(rlp)
+        let account: AccountRlp = rlp::decode(rlp).expect("Decoding failed");
+        vec![
+            account.nonce,
+            account.balance,
+            account.storage_root.into_uint(),
+            account.code_hash.into_uint(),
+        ]
     });
 
     mpt_prover_inputs(&trie_inputs.transactions_trie, &mut prover_inputs, &|rlp| {