diff --git a/evm/src/memory/memory_stark.rs b/evm/src/memory/memory_stark.rs
index 187435ed..dab5ba2d 100644
--- a/evm/src/memory/memory_stark.rs
+++ b/evm/src/memory/memory_stark.rs
@@ -315,7 +315,7 @@ impl<F: RichField + Extendable<D>, const D: usize> MemoryStark<F, D> {
             &segment_first_change,
             &virtual_first_change,
         );
-        let to_pad_to = max_timestamp_diff.next_power_of_two();
+        let to_pad_to = max_timestamp_diff.next_power_of_two().max(num_trace_rows);
         let to_pad = to_pad_to - num_trace_rows;
 
         trace_cols[SORTED_TIMESTAMP] = sorted_timestamp;