Initial commit of risc0-circuit-rv32im-v2 (#2661)
This commit is contained in:
Frank Laub
@@ -445,6 +445,8 @@ jobs:
|
||||
--exclude fibonacci-methods \
|
||||
--exclude risc0-bigint2 \
|
||||
--exclude risc0-bigint2-methods \
|
||||
--exclude risc0-circuit-rv32im-v2 \
|
||||
--exclude risc0-circuit-rv32im-v2-sys \
|
||||
--exclude risc0-circuit-keccak-methods \
|
||||
--exclude risc0-zkvm-receipts \
|
||||
--exclude risc0-zkvm-methods
|
||||
|
||||
@@ -18,3 +18,4 @@ tools/smoke-test/Cargo.lock
|
||||
compile_commands.json
|
||||
.vercel
|
||||
node_modules
|
||||
*.code-workspace
|
||||
|
||||
Vendored
+1
-1
@@ -94,7 +94,7 @@
|
||||
"benchmarks/Cargo.toml",
|
||||
"examples/Cargo.toml",
|
||||
"risc0/bigint2/methods/guest/Cargo.toml",
|
||||
"risc0/cargo-risczero/tests/test_crate",
|
||||
"risc0/cargo-risczero/tests/test_crate/Cargo.toml",
|
||||
"risc0/zkvm/methods/guest/Cargo.toml",
|
||||
"risc0/zkvm/methods/std/Cargo.toml",
|
||||
"tools/crates-validator/Cargo.toml"
|
||||
|
||||
Generated
+41
@@ -4354,6 +4354,47 @@ dependencies = [
|
||||
"sppark",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "risc0-circuit-rv32im-v2"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bytemuck",
|
||||
"cfg-if",
|
||||
"clap 4.5.23",
|
||||
"derive_more 1.0.0",
|
||||
"flate2",
|
||||
"num-derive",
|
||||
"num-traits",
|
||||
"paste",
|
||||
"rand",
|
||||
"rayon",
|
||||
"risc0-binfmt",
|
||||
"risc0-circuit-rv32im-v2-sys",
|
||||
"risc0-core",
|
||||
"risc0-sys",
|
||||
"risc0-zkp",
|
||||
"serde",
|
||||
"tar",
|
||||
"test-log",
|
||||
"tracing",
|
||||
"tracing-subscriber 0.3.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "risc0-circuit-rv32im-v2-sys"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cust",
|
||||
"derive_more 1.0.0",
|
||||
"glob",
|
||||
"risc0-build-kernel",
|
||||
"risc0-core",
|
||||
"risc0-sys",
|
||||
"sppark",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "risc0-core"
|
||||
version = "1.3.0-alpha.1"
|
||||
|
||||
@@ -14,6 +14,8 @@ members = [
|
||||
"risc0/circuit/recursion-sys",
|
||||
"risc0/circuit/rv32im",
|
||||
"risc0/circuit/rv32im-sys",
|
||||
"risc0/circuit/rv32im-v2",
|
||||
"risc0/circuit/rv32im-v2-sys",
|
||||
"risc0/core",
|
||||
"risc0/groth16",
|
||||
"risc0/r0vm",
|
||||
@@ -52,6 +54,8 @@ risc0-circuit-recursion = { version = "1.3.0-alpha.1", default-features = false,
|
||||
risc0-circuit-recursion-sys = { version = "1.3.0-alpha.1", default-features = false, path = "risc0/circuit/recursion-sys" }
|
||||
risc0-circuit-rv32im = { version = "1.3.0-alpha.1", default-features = false, path = "risc0/circuit/rv32im" }
|
||||
risc0-circuit-rv32im-sys = { version = "1.3.0-alpha.1", default-features = false, path = "risc0/circuit/rv32im-sys" }
|
||||
risc0-circuit-rv32im-v2 = { version = "0.1.0", default-features = false, path = "risc0/circuit/rv32im-v2" }
|
||||
risc0-circuit-rv32im-v2-sys = { version = "0.1.0", default-features = false, path = "risc0/circuit/rv32im-v2-sys" }
|
||||
risc0-core = { version = "1.3.0-alpha.1", default-features = false, path = "risc0/core" }
|
||||
risc0-groth16 = { version = "1.3.0-alpha.1", default-features = false, path = "risc0/groth16" }
|
||||
risc0-r0vm = { version = "1.3.0-alpha.1", default-features = false, path = "risc0/r0vm" }
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
[package]
|
||||
name = "risc0-circuit-rv32im-v2-sys"
|
||||
description = "Generated HAL code for rv32im-v2 cicuit"
|
||||
version = "0.1.0"
|
||||
edition = { workspace = true }
|
||||
license = { workspace = true }
|
||||
homepage = { workspace = true }
|
||||
repository = { workspace = true }
|
||||
links = "risc0-circuit-rv32im-v2-sys"
|
||||
|
||||
[dependencies]
|
||||
cust = { version = "0.3", optional = true }
|
||||
derive_more = { version = "1.0", features = ["debug"] }
|
||||
risc0-core = { workspace = true }
|
||||
risc0-sys = { workspace = true }
|
||||
sppark = { workspace = true, optional = true }
|
||||
|
||||
[build-dependencies]
|
||||
cc = { version = "1.2.2", features = ["parallel"] }
|
||||
glob = "0.3"
|
||||
risc0-build-kernel = { workspace = true }
|
||||
|
||||
[features]
|
||||
default = []
|
||||
cuda = ["dep:cust", "dep:sppark", "risc0-sys/cuda"]
|
||||
metal = []
|
||||
@@ -0,0 +1,94 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
env,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use risc0_build_kernel::{KernelBuild, KernelType};
|
||||
|
||||
fn main() {
|
||||
if env::var("CARGO_FEATURE_CUDA").is_ok() {
|
||||
build_cuda_kernels();
|
||||
}
|
||||
|
||||
build_cpu_kernels();
|
||||
}
|
||||
|
||||
fn build_cpu_kernels() {
|
||||
rerun_if_changed("kernels/cxx");
|
||||
KernelBuild::new(KernelType::Cpp)
|
||||
.files(glob_paths("kernels/cxx/*.cpp"))
|
||||
.deps(glob_paths("kernels/cxx/*.h"))
|
||||
.deps(glob_paths("kernels/cxx/*.cpp.inc"))
|
||||
.deps(glob_paths("kernels/cxx/*.h.inc"))
|
||||
.include(env::var("DEP_RISC0_SYS_CXX_ROOT").unwrap())
|
||||
.compile("risc0_rv32im_v2_cpu");
|
||||
}
|
||||
|
||||
fn build_cuda_kernels() {
|
||||
let output = "risc0_rv32im_v2_cuda";
|
||||
|
||||
println!("cargo:rerun-if-env-changed=NVCC_APPEND_FLAGS");
|
||||
println!("cargo:rerun-if-env-changed=NVCC_PREPEND_FLAGS");
|
||||
println!("cargo:rerun-if-env-changed=SCCACHE_RECACHE");
|
||||
rerun_if_changed("kernels/cuda");
|
||||
|
||||
env::set_var("SCCACHE_IDLE_TIMEOUT", "0");
|
||||
|
||||
if env::var("RISC0_SKIP_BUILD_KERNELS").is_ok() {
|
||||
let out_dir = env::var("OUT_DIR").map(PathBuf::from).unwrap();
|
||||
let out_path = out_dir.join(format!("lib{output}-skip.a"));
|
||||
std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.write(true)
|
||||
.open(&out_path)
|
||||
.unwrap();
|
||||
println!("cargo:{}={}", output, out_path.display());
|
||||
return;
|
||||
}
|
||||
|
||||
let mut build = cc::Build::new();
|
||||
build
|
||||
.cuda(true)
|
||||
.cudart("static")
|
||||
.debug(false)
|
||||
.flag("-diag-suppress=177")
|
||||
.flag("-diag-suppress=550")
|
||||
.flag("-diag-suppress=2922")
|
||||
.flag("-std=c++17")
|
||||
.flag("-Xcompiler")
|
||||
.flag("-Wno-unused-function,-Wno-unused-parameter")
|
||||
.flag("-Xcompiler")
|
||||
.flag("-O3")
|
||||
.flag("-Xptxas")
|
||||
.flag("-O3")
|
||||
.include(env::var("DEP_RISC0_SYS_CUDA_ROOT").unwrap())
|
||||
.include(env::var("DEP_RISC0_SYS_CXX_ROOT").unwrap())
|
||||
.include(env::var("DEP_SPPARK_ROOT").unwrap());
|
||||
if env::var_os("NVCC_PREPEND_FLAGS").is_none() && env::var_os("NVCC_APPEND_FLAGS").is_none() {
|
||||
build.flag("-arch=native");
|
||||
}
|
||||
build.files(glob_paths("kernels/cuda/*.cu")).compile(output);
|
||||
}
|
||||
|
||||
fn rerun_if_changed<P: AsRef<Path>>(path: P) {
|
||||
println!("cargo:rerun-if-changed={}", path.as_ref().display());
|
||||
}
|
||||
|
||||
fn glob_paths(pattern: &str) -> Vec<PathBuf> {
|
||||
glob::glob(pattern).unwrap().map(|x| x.unwrap()).collect()
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "fp.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <cstdio>
|
||||
|
||||
namespace risc0::circuit::rv32im_v2::cuda {
|
||||
|
||||
struct Buffer {
|
||||
Fp* buf;
|
||||
size_t rows;
|
||||
size_t cols;
|
||||
bool checkedReads;
|
||||
|
||||
__device__ void set(size_t row, size_t col, Fp val) {
|
||||
Fp& elem = buf[col * rows + row];
|
||||
if (elem != Fp::invalid() && elem != val) {
|
||||
printf("set(row: %lu, col: %lu, val: 0x%08x) cur: 0x%08x\n",
|
||||
row,
|
||||
col,
|
||||
val.asUInt32(),
|
||||
elem.asUInt32());
|
||||
assert(false && "Inconsistent set");
|
||||
}
|
||||
// printf("set(row: %lu, col: %lu, val: 0x%08x)\n", row, col, val.asUInt32());
|
||||
elem = val;
|
||||
}
|
||||
|
||||
__device__ Fp get(size_t row, size_t col) {
|
||||
Fp ret = buf[col * rows + row];
|
||||
if (ret == Fp::invalid() && checkedReads) {
|
||||
printf("get(row: %lu, col: %lu) -> 0x%08x\n", row, col, ret.asRaw());
|
||||
assert(false && "Read of unset value");
|
||||
}
|
||||
// printf("get(row: %lu, col: %lu) -> 0x%08x\n", row, col, ret.asUInt32());
|
||||
return ret;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace risc0::circuit::rv32im_v2::cuda
|
||||
@@ -0,0 +1,7 @@
|
||||
SET_FIELD(BabyBear);
|
||||
constexpr size_t kRegCountAccum = 76;
|
||||
constexpr size_t kRegCountCode = 1;
|
||||
constexpr size_t kRegCountData = 192;
|
||||
constexpr size_t kRegCountGlobal = 73;
|
||||
constexpr size_t kRegCountMix = 32;
|
||||
constexpr size_t kRegCountTest = 192;
|
||||
@@ -0,0 +1,193 @@
|
||||
// This code is automatically generated
|
||||
|
||||
#include "supra/fp.h"
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace risc0::circuit::rv32im_v2::cuda {
|
||||
|
||||
extern __device__ FpExt rv32im_v2_12(uint32_t idx,
|
||||
uint32_t size,
|
||||
Fp* arg0,
|
||||
FpExt arg1,
|
||||
FpExt arg2,
|
||||
FpExt arg3,
|
||||
FpExt arg4,
|
||||
FpExt arg5,
|
||||
FpExt arg6,
|
||||
FpExt arg7,
|
||||
FpExt arg8,
|
||||
FpExt* arg9,
|
||||
const Fp* arg10,
|
||||
const Fp* arg11,
|
||||
const Fp* arg12,
|
||||
const Fp* arg13);
|
||||
extern __device__ FpExt rv32im_v2_11(uint32_t idx,
|
||||
uint32_t size,
|
||||
Fp* arg0,
|
||||
FpExt arg1,
|
||||
FpExt arg2,
|
||||
FpExt arg3,
|
||||
FpExt arg4,
|
||||
FpExt arg5,
|
||||
FpExt* arg6,
|
||||
const Fp* arg7,
|
||||
const Fp* arg8,
|
||||
const Fp* arg9,
|
||||
const Fp* arg10);
|
||||
extern __device__ FpExt rv32im_v2_10(uint32_t idx,
|
||||
uint32_t size,
|
||||
Fp* arg0,
|
||||
FpExt arg1,
|
||||
FpExt arg2,
|
||||
FpExt arg3,
|
||||
FpExt arg4,
|
||||
FpExt arg5,
|
||||
FpExt arg6,
|
||||
FpExt arg7,
|
||||
FpExt* arg8,
|
||||
FpExt arg9,
|
||||
const Fp* arg10,
|
||||
const Fp* arg11,
|
||||
const Fp* arg12,
|
||||
const Fp* arg13);
|
||||
extern __device__ FpExt rv32im_v2_9(uint32_t idx,
|
||||
uint32_t size,
|
||||
Fp* arg0,
|
||||
FpExt arg1,
|
||||
FpExt arg2,
|
||||
FpExt arg3,
|
||||
FpExt arg4,
|
||||
FpExt arg5,
|
||||
FpExt arg6,
|
||||
FpExt* arg7,
|
||||
FpExt arg8,
|
||||
const Fp* arg9,
|
||||
const Fp* arg10,
|
||||
const Fp* arg11,
|
||||
const Fp* arg12);
|
||||
extern __device__ FpExt rv32im_v2_8(uint32_t idx,
|
||||
uint32_t size,
|
||||
Fp* arg0,
|
||||
FpExt arg1,
|
||||
FpExt arg2,
|
||||
FpExt arg3,
|
||||
FpExt arg4,
|
||||
FpExt arg5,
|
||||
FpExt arg6,
|
||||
FpExt* arg7,
|
||||
FpExt arg8,
|
||||
const Fp* arg9,
|
||||
const Fp* arg10,
|
||||
const Fp* arg11,
|
||||
const Fp* arg12);
|
||||
extern __device__ FpExt rv32im_v2_7(uint32_t idx,
|
||||
uint32_t size,
|
||||
Fp* arg0,
|
||||
FpExt arg1,
|
||||
FpExt arg2,
|
||||
FpExt arg3,
|
||||
FpExt arg4,
|
||||
FpExt arg5,
|
||||
FpExt* arg6,
|
||||
FpExt arg7,
|
||||
const Fp* arg8,
|
||||
const Fp* arg9,
|
||||
const Fp* arg10,
|
||||
const Fp* arg11);
|
||||
extern __device__ FpExt rv32im_v2_6(uint32_t idx,
|
||||
uint32_t size,
|
||||
Fp* arg0,
|
||||
FpExt arg1,
|
||||
FpExt arg2,
|
||||
FpExt arg3,
|
||||
FpExt arg4,
|
||||
FpExt* arg5,
|
||||
FpExt arg6,
|
||||
FpExt arg7,
|
||||
FpExt arg8,
|
||||
FpExt arg9,
|
||||
FpExt arg10,
|
||||
FpExt arg11,
|
||||
const Fp* arg12,
|
||||
const Fp* arg13,
|
||||
const Fp* arg14,
|
||||
const Fp* arg15);
|
||||
extern __device__ FpExt rv32im_v2_5(uint32_t idx,
|
||||
uint32_t size,
|
||||
FpExt arg0,
|
||||
Fp* arg1,
|
||||
FpExt* arg2,
|
||||
FpExt arg3,
|
||||
FpExt arg4,
|
||||
FpExt arg5,
|
||||
FpExt arg6,
|
||||
FpExt arg7,
|
||||
const Fp* arg8,
|
||||
const Fp* arg9,
|
||||
const Fp* arg10);
|
||||
extern __device__ FpExt rv32im_v2_4(uint32_t idx,
|
||||
uint32_t size,
|
||||
Fp* arg0,
|
||||
FpExt arg1,
|
||||
FpExt* arg2,
|
||||
FpExt arg3,
|
||||
FpExt arg4,
|
||||
FpExt arg5,
|
||||
const Fp* arg6,
|
||||
const Fp* arg7,
|
||||
const Fp* arg8);
|
||||
extern __device__ FpExt rv32im_v2_3(uint32_t idx,
|
||||
uint32_t size,
|
||||
Fp* arg0,
|
||||
FpExt arg1,
|
||||
FpExt* arg2,
|
||||
FpExt arg3,
|
||||
FpExt arg4,
|
||||
FpExt arg5,
|
||||
FpExt arg6,
|
||||
const Fp* arg7,
|
||||
const Fp* arg8,
|
||||
const Fp* arg9);
|
||||
extern __device__ FpExt rv32im_v2_2(uint32_t idx,
|
||||
uint32_t size,
|
||||
Fp* arg0,
|
||||
FpExt arg1,
|
||||
FpExt* arg2,
|
||||
FpExt arg3,
|
||||
FpExt arg4,
|
||||
FpExt arg5,
|
||||
FpExt arg6,
|
||||
const Fp* arg7,
|
||||
const Fp* arg8,
|
||||
const Fp* arg9);
|
||||
extern __device__ FpExt rv32im_v2_1(uint32_t idx,
|
||||
uint32_t size,
|
||||
FpExt* arg0,
|
||||
FpExt arg1,
|
||||
FpExt arg2,
|
||||
FpExt arg3,
|
||||
const Fp* arg4,
|
||||
const Fp* arg5,
|
||||
const Fp* arg6);
|
||||
extern __device__ FpExt rv32im_v2_0(uint32_t idx,
|
||||
uint32_t size,
|
||||
FpExt* arg0,
|
||||
FpExt arg1,
|
||||
FpExt arg2,
|
||||
FpExt arg3,
|
||||
const Fp* arg4,
|
||||
const Fp* arg5);
|
||||
extern __device__ FpExt poly_fp(uint32_t idx,
|
||||
uint32_t size,
|
||||
const Fp* ctrl,
|
||||
const Fp* out,
|
||||
const Fp* data,
|
||||
const Fp* mix,
|
||||
const Fp* accum);
|
||||
|
||||
constexpr size_t INV_RATE = 4;
|
||||
constexpr size_t kNumPolyMixPows = 411;
|
||||
extern __constant__ FpExt poly_mix[kNumPolyMixPows];
|
||||
|
||||
} // namespace risc0::circuit::rv32im_v2::cuda
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,360 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "cuda.h"
|
||||
#include "steps.cuh"
|
||||
#include "witgen.h"
|
||||
|
||||
#include "vendor/nvtx3/nvtx3.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cuda/std/array>
|
||||
#include <string.h>
|
||||
|
||||
namespace risc0::circuit::rv32im_v2::cuda {
|
||||
|
||||
struct ExecBuffers {
|
||||
Buffer global;
|
||||
Buffer data;
|
||||
};
|
||||
|
||||
struct AccumBuffers {
|
||||
Buffer data;
|
||||
Buffer accum;
|
||||
Buffer mix;
|
||||
};
|
||||
|
||||
struct DeviceContext {
|
||||
Buffer* data;
|
||||
Buffer* global;
|
||||
PreflightTrace* preflight;
|
||||
LookupTables* tables;
|
||||
};
|
||||
|
||||
struct HostContext {
|
||||
DeviceContext* ctx;
|
||||
PreflightTrace d_preflight;
|
||||
LookupTables d_tables;
|
||||
|
||||
HostContext(ExecBuffers* buffers, PreflightTrace* preflight, size_t cycles) {
|
||||
CUDA_OK(cudaMallocManaged(&ctx, sizeof(DeviceContext)));
|
||||
|
||||
CUDA_OK(cudaMalloc(&ctx->data, sizeof(Buffer)));
|
||||
CUDA_OK(cudaMemcpy(ctx->data, &buffers->data, sizeof(Buffer), cudaMemcpyHostToDevice));
|
||||
|
||||
CUDA_OK(cudaMalloc(&ctx->global, sizeof(Buffer)));
|
||||
CUDA_OK(cudaMemcpy(ctx->global, &buffers->global, sizeof(Buffer), cudaMemcpyHostToDevice));
|
||||
|
||||
CUDA_OK(cudaMalloc(&d_preflight.cycles, cycles * sizeof(PreflightCycle)));
|
||||
CUDA_OK(cudaMemcpy(d_preflight.cycles,
|
||||
preflight->cycles,
|
||||
cycles * sizeof(PreflightCycle),
|
||||
cudaMemcpyHostToDevice));
|
||||
|
||||
CUDA_OK(cudaMalloc(&d_preflight.txns, preflight->txnsLen * sizeof(MemoryTransaction)));
|
||||
CUDA_OK(cudaMemcpy(d_preflight.txns,
|
||||
preflight->txns,
|
||||
preflight->txnsLen * sizeof(MemoryTransaction),
|
||||
cudaMemcpyHostToDevice));
|
||||
|
||||
d_preflight.txnsLen = preflight->txnsLen;
|
||||
d_preflight.tableSplitCycle = preflight->tableSplitCycle;
|
||||
|
||||
CUDA_OK(cudaMalloc(&ctx->preflight, sizeof(PreflightTrace)));
|
||||
CUDA_OK(
|
||||
cudaMemcpy(ctx->preflight, &d_preflight, sizeof(PreflightTrace), cudaMemcpyHostToDevice));
|
||||
|
||||
CUDA_OK(cudaMalloc(&d_tables.tableU8, (1 << 8) * sizeof(uint32_t)));
|
||||
CUDA_OK(cudaMemset(d_tables.tableU8, 0, (1 << 8) * sizeof(uint32_t)));
|
||||
|
||||
CUDA_OK(cudaMalloc(&d_tables.tableU16, (1 << 16) * sizeof(uint32_t)));
|
||||
CUDA_OK(cudaMemset(d_tables.tableU16, 0, (1 << 16) * sizeof(uint32_t)));
|
||||
|
||||
CUDA_OK(cudaMalloc(&ctx->tables, sizeof(LookupTables)));
|
||||
CUDA_OK(cudaMemcpy(ctx->tables, &d_tables, sizeof(LookupTables), cudaMemcpyHostToDevice));
|
||||
}
|
||||
|
||||
~HostContext() {
|
||||
cudaFree(d_tables.tableU16);
|
||||
cudaFree(d_tables.tableU8);
|
||||
cudaFree(ctx->tables);
|
||||
cudaFree(d_preflight.txns);
|
||||
cudaFree(d_preflight.cycles);
|
||||
cudaFree(ctx->preflight);
|
||||
cudaFree(ctx->global);
|
||||
cudaFree(ctx->data);
|
||||
cudaFree(ctx);
|
||||
}
|
||||
};
|
||||
|
||||
__device__ ::cuda::std::array<uint32_t, 2>
|
||||
divide_rv32im(uint32_t numer, uint32_t denom, uint32_t signType) {
|
||||
uint32_t onesComp = (signType == 2);
|
||||
bool negNumer = signType && int32_t(numer) < 0;
|
||||
bool negDenom = signType == 1 && int32_t(denom) < 0;
|
||||
if (negNumer) {
|
||||
numer = -numer - onesComp;
|
||||
}
|
||||
if (negDenom) {
|
||||
denom = -denom - onesComp;
|
||||
}
|
||||
uint32_t quot;
|
||||
uint32_t rem;
|
||||
if (denom == 0) {
|
||||
quot = 0xffffffff;
|
||||
rem = numer;
|
||||
} else {
|
||||
quot = numer / denom;
|
||||
rem = numer % denom;
|
||||
}
|
||||
uint32_t quotNegOut = (negNumer ^ negDenom) - ((denom == 0) * negNumer);
|
||||
uint32_t remNegOut = negNumer;
|
||||
if (quotNegOut) {
|
||||
quot = -quot - onesComp;
|
||||
}
|
||||
if (remNegOut) {
|
||||
rem = -rem - onesComp;
|
||||
}
|
||||
return {quot, rem};
|
||||
}
|
||||
|
||||
__device__ ::cuda::std::array<Val, 5> extern_getMemoryTxn(ExecContext& ctx, Val addrElem) {
|
||||
uint32_t addr = addrElem.asUInt32();
|
||||
size_t txnIdx = ctx.preflight.cycles[ctx.cycle].txnIdx++;
|
||||
const MemoryTransaction& txn = ctx.preflight.txns[txnIdx];
|
||||
// printf("getMemoryTxn(%lu, 0x%08x): txn(%u, 0x%08x, 0x%08x)\n",
|
||||
// ctx.cycle,
|
||||
// addr,
|
||||
// txn.cycle,
|
||||
// txn.addr,
|
||||
// txn.word);
|
||||
|
||||
if (txn.cycle != ctx.cycle) {
|
||||
printf("txn.cycle: %u, ctx.cycle: %zu\n", txn.cycle, ctx.cycle);
|
||||
assert(false && "txn cycle mismatch");
|
||||
}
|
||||
|
||||
if (txn.addr != addr) {
|
||||
printf("txn.addr: 0x%08x, addr: 0x%08x\n", txn.addr, addr);
|
||||
assert(false && "memory peek not in preflight");
|
||||
}
|
||||
return {
|
||||
txn.prevCycle,
|
||||
txn.prevWord & 0xffff,
|
||||
txn.prevWord >> 16,
|
||||
txn.word & 0xffff,
|
||||
txn.word >> 16,
|
||||
};
|
||||
}
|
||||
|
||||
__device__ void extern_lookupDelta(ExecContext& ctx, Val table, Val index, Val count) {
|
||||
// printf("lookupDelta(table: %u, index: %u, count: %u, P: %u)\n",
|
||||
// table.asUInt32(),
|
||||
// index.asUInt32(),
|
||||
// count.asUInt32(),
|
||||
// Fp::P);
|
||||
ctx.tables.lookupDelta(table, index, count);
|
||||
}
|
||||
|
||||
__device__ Val extern_lookupCurrent(ExecContext& ctx, Val table, Val index) {
|
||||
Val ret = ctx.tables.lookupCurrent(table, index);
|
||||
// printf("lookupCurrent(table: %u, index: %u): %u\n",
|
||||
// table.asUInt32(),
|
||||
// index.asUInt32(),
|
||||
// ret.asUInt32());
|
||||
return ret;
|
||||
}
|
||||
|
||||
__device__ void
|
||||
extern_memoryDelta(ExecContext& ctx, Val addr, Val cycle, Val dataLow, Val dataHigh, Val count) {
|
||||
// printf("memoryDelta\n");
|
||||
// ctx.tables.memoryDelta(
|
||||
// addr.asUInt32(), cycle.asUInt32(), dataLow.asUInt32() | (dataHigh.asUInt32() << 16),
|
||||
// count);
|
||||
}
|
||||
|
||||
__device__ uint32_t extern_getDiffCount(ExecContext& ctx, Val cycle) {
|
||||
// printf("getDiffCount\n");
|
||||
return ctx.preflight.cycles[cycle.asUInt32()].diffCount;
|
||||
}
|
||||
|
||||
__device__ Val extern_isFirstCycle_0(ExecContext& ctx) {
|
||||
// printf("isFirstCycle\n");
|
||||
return ctx.cycle == 0;
|
||||
}
|
||||
|
||||
__device__ Val extern_getCycle(ExecContext& ctx) {
|
||||
// printf("getCycle\n");
|
||||
return ctx.cycle;
|
||||
}
|
||||
|
||||
__device__ ::cuda::std::array<Val, 4> extern_divide(
|
||||
ExecContext& ctx, Val numerLow, Val numerHigh, Val denomLow, Val denomHigh, Val signType) {
|
||||
// printf("divide\n");
|
||||
uint32_t numer = numerLow.asUInt32() | (numerHigh.asUInt32() << 16);
|
||||
uint32_t denom = denomLow.asUInt32() | (denomHigh.asUInt32() << 16);
|
||||
auto [quot, rem] = divide_rv32im(numer, denom, signType.asUInt32());
|
||||
::cuda::std::array<Val, 4> ret;
|
||||
ret[0] = quot & 0xffff;
|
||||
ret[1] = quot >> 16;
|
||||
ret[2] = rem & 0xffff;
|
||||
ret[3] = rem >> 16;
|
||||
return ret;
|
||||
}
|
||||
|
||||
__device__ void extern_print(ExecContext& ctx, Val v) {
|
||||
// printf("LOG: %u\n", v.asUInt32());
|
||||
}
|
||||
|
||||
__device__ ::cuda::std::array<Val, 2> extern_getMajorMinor(ExecContext& ctx) {
|
||||
uint8_t major = ctx.preflight.cycles[ctx.cycle].major;
|
||||
uint8_t minor = ctx.preflight.cycles[ctx.cycle].minor;
|
||||
// printf("getMajorMinor: %u, %u\n", major, minor);
|
||||
return {major, minor};
|
||||
}
|
||||
|
||||
__device__ Val extern_hostReadPrepare(ExecContext& ctx, Val fp, Val len) {
|
||||
// printf("hostReadPrepare\n");
|
||||
assert(false && "extern_hostReadPrepare");
|
||||
// return ctx.stepHandler.readPrepare(fp.asUInt32(), len.asUInt32());
|
||||
return 0;
|
||||
}
|
||||
|
||||
__device__ Val
|
||||
extern_hostWrite(ExecContext& ctx, Val fdVal, Val addrLow, Val addrHigh, Val lenVal) {
|
||||
// printf("hostWrite\n");
|
||||
assert(false && "extern_hostWrite");
|
||||
// uint32_t fd = fdVal.asUInt32();
|
||||
// uint32_t addr = addrLow.asUInt32() | (addrHigh.asUInt32() << 16);
|
||||
// uint32_t len = lenVal.asUInt32();
|
||||
// return ctx.stepHandler.write(fd, addr, len);
|
||||
return 0;
|
||||
}
|
||||
|
||||
__device__ ::cuda::std::array<Val, 2> extern_nextPagingIdx(ExecContext& ctx) {
|
||||
uint32_t pagingIdx = ctx.preflight.cycles[ctx.cycle].pagingIdx;
|
||||
uint32_t machineMode = ctx.preflight.cycles[ctx.cycle].machineMode;
|
||||
// printf("nextPagingIdx: (0x%05x, %u)\n", pagingIdx, machineMode);
|
||||
return {pagingIdx, machineMode};
|
||||
}
|
||||
|
||||
// __device__ void
|
||||
// stepAccum(AccumBuffers& buffers, PreflightTrace& preflight, LookupTables& tables, size_t cycle) {
|
||||
// ExecContext ctx(preflight, tables, cycle);
|
||||
// MutableBufObj data(ctx, buffers.data);
|
||||
// MutableBufObj accum(ctx, buffers.accum);
|
||||
// GlobalBufObj mix(ctx, buffers.mix);
|
||||
// step_TopAccum(ctx, &accum, &data, &mix);
|
||||
// }
|
||||
|
||||
__device__ void nextStep(DeviceContext* ctx, uint32_t cycle) {
|
||||
// printf("nextStep: %u\n", cycle);
|
||||
ExecContext execCtx(*ctx->preflight, *ctx->tables, cycle);
|
||||
MutableBufObj data(*ctx->data);
|
||||
GlobalBufObj global(*ctx->global);
|
||||
step_Top(execCtx, &data, &global);
|
||||
}
|
||||
|
||||
__global__ void par_stepExec(DeviceContext* ctx, uint32_t start, uint32_t count) {
|
||||
uint32_t cycle = blockDim.x * blockIdx.x + threadIdx.x;
|
||||
if (cycle >= count) {
|
||||
return;
|
||||
}
|
||||
nextStep(ctx, start + cycle);
|
||||
}
|
||||
|
||||
__global__ void rev_stepExec(DeviceContext* ctx, uint32_t split, uint32_t lastCycle) {
|
||||
for (uint32_t cycle = split; cycle-- > 0;) {
|
||||
nextStep(ctx, cycle);
|
||||
}
|
||||
for (uint32_t cycle = lastCycle; cycle-- > split;) {
|
||||
nextStep(ctx, cycle);
|
||||
}
|
||||
}
|
||||
|
||||
__global__ void fwd_stepExec(DeviceContext* ctx, uint32_t count) {
|
||||
for (uint32_t cycle = 0; cycle < count; cycle++) {
|
||||
nextStep(ctx, cycle);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace risc0::circuit::rv32im_v2::cuda
|
||||
|
||||
constexpr size_t kStepModeParallel = 0;
|
||||
constexpr size_t kStepModeSeqForward = 1;
|
||||
constexpr size_t kStepModeSeqReverse = 2;
|
||||
|
||||
extern "C" {
|
||||
|
||||
using namespace risc0::circuit::rv32im_v2::cuda;
|
||||
|
||||
const char* risc0_circuit_rv32im_v2_cuda_witgen(uint32_t mode,
|
||||
ExecBuffers* buffers,
|
||||
PreflightTrace* preflight,
|
||||
uint32_t lastCycle) {
|
||||
try {
|
||||
HostContext ctx(buffers, preflight, lastCycle);
|
||||
CudaStream stream;
|
||||
size_t split = preflight->tableSplitCycle;
|
||||
|
||||
switch (mode) {
|
||||
case kStepModeParallel: {
|
||||
auto cfg1 = getSimpleConfig(split);
|
||||
size_t phase2Count = lastCycle - split;
|
||||
// printf("phase1: %zu, phase2: %zu\n", split, phase2Count);
|
||||
auto cfg2 = getSimpleConfig(phase2Count);
|
||||
{
|
||||
nvtx3::scoped_range range("phase1");
|
||||
par_stepExec<<<cfg1.grid, cfg1.block, 0, stream>>>(ctx.ctx, 0, split);
|
||||
CUDA_OK(cudaStreamSynchronize(stream));
|
||||
}
|
||||
{
|
||||
nvtx3::scoped_range range("phase2");
|
||||
par_stepExec<<<cfg2.grid, cfg2.block, 0, stream>>>(ctx.ctx, split, phase2Count);
|
||||
CUDA_OK(cudaStreamSynchronize(stream));
|
||||
}
|
||||
} break;
|
||||
case kStepModeSeqForward:
|
||||
fwd_stepExec<<<1, 1, 0, stream>>>(ctx.ctx, lastCycle);
|
||||
CUDA_OK(cudaStreamSynchronize(stream));
|
||||
break;
|
||||
case kStepModeSeqReverse:
|
||||
rev_stepExec<<<1, 1, 0, stream>>>(ctx.ctx, split, lastCycle);
|
||||
CUDA_OK(cudaStreamSynchronize(stream));
|
||||
break;
|
||||
}
|
||||
} catch (const std::exception& err) {
|
||||
return strdup(err.what());
|
||||
} catch (...) {
|
||||
return strdup("Generic exception");
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const char* risc0_circuit_rv32im_v2_cuda_accum(AccumBuffers* buffers,
|
||||
PreflightTrace* preflight,
|
||||
uint32_t lastCycle) {
|
||||
try {
|
||||
// LookupTables tables;
|
||||
// for (size_t cycle = 0; cycle < lastCycle; cycle++) {
|
||||
// stepAccum(*buffers, *preflight, tables, cycle);
|
||||
// }
|
||||
} catch (const std::exception& err) {
|
||||
return strdup(err.what());
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
} // extern "C"
|
||||
@@ -0,0 +1,81 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "eval_check.cuh"
|
||||
|
||||
#include "cuda.h"
|
||||
#include "supra/fp.h"
|
||||
|
||||
#include <exception>
|
||||
|
||||
namespace risc0::circuit::rv32im_v2::cuda {
|
||||
|
||||
__constant__ FpExt poly_mix[kNumPolyMixPows];
|
||||
|
||||
__global__ void eval_check(Fp* check,
|
||||
const Fp* ctrl,
|
||||
const Fp* data,
|
||||
const Fp* accum,
|
||||
const Fp* mix,
|
||||
const Fp* out,
|
||||
const Fp rou,
|
||||
uint32_t po2,
|
||||
uint32_t domain) {
|
||||
uint32_t cycle = blockDim.x * blockIdx.x + threadIdx.x;
|
||||
if (cycle < domain) {
|
||||
FpExt tot = poly_fp(cycle, domain, ctrl, out, data, mix, accum);
|
||||
Fp x = pow(rou, cycle);
|
||||
Fp y = pow(Fp(3) * x, 1 << po2);
|
||||
FpExt ret = tot * inv(y - Fp(1));
|
||||
check[domain * 0 + cycle] = ret[0];
|
||||
check[domain * 1 + cycle] = ret[1];
|
||||
check[domain * 2 + cycle] = ret[2];
|
||||
check[domain * 3 + cycle] = ret[3];
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace risc0::circuit::rv32im_v2::cuda
|
||||
|
||||
using namespace risc0::circuit::rv32im_v2::cuda;
|
||||
|
||||
extern "C" {
|
||||
|
||||
const char* risc0_circuit_rv32im_v2_cuda_eval_check(Fp* check,
|
||||
const Fp* ctrl,
|
||||
const Fp* data,
|
||||
const Fp* accum,
|
||||
const Fp* mix,
|
||||
const Fp* out,
|
||||
const Fp& rou,
|
||||
uint32_t po2,
|
||||
uint32_t domain,
|
||||
const FpExt* poly_mix_pows) {
|
||||
try {
|
||||
CUDA_OK(cudaDeviceSynchronize());
|
||||
|
||||
CudaStream stream;
|
||||
auto cfg = getSimpleConfig(domain);
|
||||
cudaMemcpyToSymbol(poly_mix, poly_mix_pows, sizeof(poly_mix));
|
||||
eval_check<<<cfg.grid, cfg.block, 0, stream>>>(
|
||||
check, ctrl, data, accum, mix, out, rou, po2, domain);
|
||||
CUDA_OK(cudaStreamSynchronize(stream));
|
||||
} catch (const std::exception& err) {
|
||||
return strdup(err.what());
|
||||
} catch (...) {
|
||||
return strdup("Generic exception");
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
} // extern "C"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,885 @@
|
||||
extern __device__ const NondetRegLayout8LayoutArray kLayout__3;
|
||||
extern __device__ const OneHot_8_Layout kLayout__2;
|
||||
extern __device__ const InstInputLayout kLayout__1;
|
||||
extern __device__ const NondetRegLayout11LayoutArray kLayout__5;
|
||||
extern __device__ const OneHot_11_Layout kLayout__4;
|
||||
extern __device__ const NondetU16RegLayout kLayout__10;
|
||||
extern __device__ const NondetU16RegLayout kLayout__11;
|
||||
extern __device__ const NormalizeU32Layout kLayout__9;
|
||||
extern __device__ const NondetU16RegLayout kLayout__13;
|
||||
extern __device__ const NondetU16RegLayout kLayout__14;
|
||||
extern __device__ const NormalizeU32Layout kLayout__12;
|
||||
extern __device__ const MemoryArgLayout kLayout__18;
|
||||
extern __device__ const MemoryArgLayout kLayout__19;
|
||||
extern __device__ const MemoryIOLayout kLayout__17;
|
||||
extern __device__ const IsCycleLayout kLayout__21;
|
||||
extern __device__ const IsForwardLayout kLayout__20;
|
||||
extern __device__ const MemoryWriteLayout kLayout__16;
|
||||
extern __device__ const WriteRdLayout kLayout__15;
|
||||
extern __device__ const FinalizeMiscLayout kLayout__8;
|
||||
extern __device__ const DecoderLayout kLayout__24;
|
||||
extern __device__ const NondetU16RegLayout kLayout__27;
|
||||
extern __device__ const U16RegLayout kLayout__26;
|
||||
extern __device__ const NondetU16RegLayout kLayout__28;
|
||||
extern __device__ const AddrDecomposeLayout kLayout__25;
|
||||
extern __device__ const MemoryArgLayout kLayout__31;
|
||||
extern __device__ const MemoryArgLayout kLayout__32;
|
||||
extern __device__ const MemoryIOLayout kLayout__30;
|
||||
extern __device__ const IsCycleLayout kLayout__34;
|
||||
extern __device__ const IsForwardLayout kLayout__33;
|
||||
extern __device__ const MemoryReadLayout kLayout__29;
|
||||
extern __device__ const DecodeInstLayout kLayout__23;
|
||||
extern __device__ const MemoryArgLayout kLayout__38;
|
||||
extern __device__ const MemoryArgLayout kLayout__39;
|
||||
extern __device__ const MemoryIOLayout kLayout__37;
|
||||
extern __device__ const IsCycleLayout kLayout__41;
|
||||
extern __device__ const IsForwardLayout kLayout__40;
|
||||
extern __device__ const MemoryReadLayout kLayout__36;
|
||||
extern __device__ const ReadRegLayout kLayout__35;
|
||||
extern __device__ const MemoryArgLayout kLayout__45;
|
||||
extern __device__ const MemoryArgLayout kLayout__46;
|
||||
extern __device__ const MemoryIOLayout kLayout__44;
|
||||
extern __device__ const IsCycleLayout kLayout__48;
|
||||
extern __device__ const IsForwardLayout kLayout__47;
|
||||
extern __device__ const MemoryReadLayout kLayout__43;
|
||||
extern __device__ const ReadRegLayout kLayout__42;
|
||||
extern __device__ const MiscInputLayout kLayout__22;
|
||||
extern __device__ const ArgU16Layout5LayoutArray kLayout__50;
|
||||
extern __device__ const _Arguments_Misc0MiscOutputLayout kLayout__49;
|
||||
extern __device__ const Misc0MiscOutputArm0Layout kLayout__52;
|
||||
extern __device__ const Misc0MiscOutputArm1Layout kLayout__53;
|
||||
extern __device__ const NondetRegLayout16LayoutArray kLayout__60;
|
||||
extern __device__ const ToBits_16_Layout kLayout__59;
|
||||
extern __device__ const NondetRegLayout16LayoutArray kLayout__62;
|
||||
extern __device__ const ToBits_16_Layout kLayout__61;
|
||||
extern __device__ const BitwiseAndU16Layout kLayout__58;
|
||||
extern __device__ const NondetRegLayout16LayoutArray kLayout__65;
|
||||
extern __device__ const ToBits_16_Layout kLayout__64;
|
||||
extern __device__ const NondetRegLayout16LayoutArray kLayout__67;
|
||||
extern __device__ const ToBits_16_Layout kLayout__66;
|
||||
extern __device__ const BitwiseAndU16Layout kLayout__63;
|
||||
extern __device__ const BitwiseAndLayout kLayout__57;
|
||||
extern __device__ const BitwiseXorLayout kLayout__56;
|
||||
extern __device__ const OpXORLayout kLayout__55;
|
||||
extern __device__ const Misc0MiscOutputArm2Layout kLayout__54;
|
||||
extern __device__ const BitwiseOrLayout kLayout__70;
|
||||
extern __device__ const OpORLayout kLayout__69;
|
||||
extern __device__ const Misc0MiscOutputArm3Layout kLayout__68;
|
||||
extern __device__ const OpANDLayout kLayout__72;
|
||||
extern __device__ const Misc0MiscOutputArm4Layout kLayout__71;
|
||||
extern __device__ const NondetU16RegLayout kLayout__76;
|
||||
extern __device__ const NondetU16RegLayout kLayout__77;
|
||||
extern __device__ const NormalizeU32Layout kLayout__75;
|
||||
extern __device__ const NondetU16RegLayout kLayout__79;
|
||||
extern __device__ const GetSignU32Layout kLayout__78;
|
||||
extern __device__ const NondetU16RegLayout kLayout__81;
|
||||
extern __device__ const GetSignU32Layout kLayout__80;
|
||||
extern __device__ const NondetU16RegLayout kLayout__83;
|
||||
extern __device__ const GetSignU32Layout kLayout__82;
|
||||
extern __device__ const CmpLessThanLayout kLayout__74;
|
||||
extern __device__ const OpSLTLayout kLayout__73;
|
||||
extern __device__ const CmpLessThanUnsignedLayout kLayout__86;
|
||||
extern __device__ const OpSLTULayout kLayout__85;
|
||||
extern __device__ const Misc0MiscOutputArm6Layout kLayout__84;
|
||||
extern __device__ const Misc0MiscOutputArm7Layout kLayout__87;
|
||||
extern __device__ const Misc0MiscOutputLayout kLayout__51;
|
||||
extern __device__ const Misc0Layout kLayout__7;
|
||||
extern __device__ const _Arguments_Misc1MiscOutputLayout kLayout__89;
|
||||
extern __device__ const OpXORILayout kLayout__92;
|
||||
extern __device__ const Misc1MiscOutputArm0Layout kLayout__91;
|
||||
extern __device__ const OpORILayout kLayout__94;
|
||||
extern __device__ const Misc1MiscOutputArm1Layout kLayout__93;
|
||||
extern __device__ const OpANDILayout kLayout__96;
|
||||
extern __device__ const Misc1MiscOutputArm2Layout kLayout__95;
|
||||
extern __device__ const OpSLTILayout kLayout__97;
|
||||
extern __device__ const OpSLTIULayout kLayout__99;
|
||||
extern __device__ const Misc1MiscOutputArm4Layout kLayout__98;
|
||||
extern __device__ const CmpEqualLayout kLayout__102;
|
||||
extern __device__ const OpBEQLayout kLayout__101;
|
||||
extern __device__ const Misc1MiscOutputArm5Layout kLayout__100;
|
||||
extern __device__ const OpBNELayout kLayout__104;
|
||||
extern __device__ const Misc1MiscOutputArm6Layout kLayout__103;
|
||||
extern __device__ const OpBLTLayout kLayout__105;
|
||||
extern __device__ const Misc1MiscOutputLayout kLayout__90;
|
||||
extern __device__ const Misc1Layout kLayout__88;
|
||||
extern __device__ const _Arguments_Misc2MiscOutputLayout kLayout__107;
|
||||
extern __device__ const OpBGELayout kLayout__109;
|
||||
extern __device__ const OpBLTULayout kLayout__111;
|
||||
extern __device__ const Misc2MiscOutputArm1Layout kLayout__110;
|
||||
extern __device__ const OpBGEULayout kLayout__113;
|
||||
extern __device__ const Misc2MiscOutputArm2Layout kLayout__112;
|
||||
extern __device__ const Misc2MiscOutputArm3Layout kLayout__114;
|
||||
extern __device__ const Misc2MiscOutputArm4Layout kLayout__115;
|
||||
extern __device__ const Misc2MiscOutputArm5Layout kLayout__116;
|
||||
extern __device__ const Misc2MiscOutputArm6Layout kLayout__117;
|
||||
extern __device__ const Misc2MiscOutputArm7Layout kLayout__118;
|
||||
extern __device__ const Misc2MiscOutputLayout kLayout__108;
|
||||
extern __device__ const Misc2Layout kLayout__106;
|
||||
extern __device__ const DecoderLayout kLayout__122;
|
||||
extern __device__ const NondetU16RegLayout kLayout__125;
|
||||
extern __device__ const U16RegLayout kLayout__124;
|
||||
extern __device__ const NondetU16RegLayout kLayout__126;
|
||||
extern __device__ const AddrDecomposeLayout kLayout__123;
|
||||
extern __device__ const MemoryArgLayout kLayout__129;
|
||||
extern __device__ const MemoryArgLayout kLayout__130;
|
||||
extern __device__ const MemoryIOLayout kLayout__128;
|
||||
extern __device__ const IsCycleLayout kLayout__132;
|
||||
extern __device__ const IsForwardLayout kLayout__131;
|
||||
extern __device__ const MemoryReadLayout kLayout__127;
|
||||
extern __device__ const DecodeInstLayout kLayout__121;
|
||||
extern __device__ const MemoryArgLayout kLayout__136;
|
||||
extern __device__ const MemoryArgLayout kLayout__137;
|
||||
extern __device__ const MemoryIOLayout kLayout__135;
|
||||
extern __device__ const IsCycleLayout kLayout__139;
|
||||
extern __device__ const IsForwardLayout kLayout__138;
|
||||
extern __device__ const MemoryReadLayout kLayout__134;
|
||||
extern __device__ const ReadRegLayout kLayout__133;
|
||||
extern __device__ const MemoryArgLayout kLayout__143;
|
||||
extern __device__ const MemoryArgLayout kLayout__144;
|
||||
extern __device__ const MemoryIOLayout kLayout__142;
|
||||
extern __device__ const IsCycleLayout kLayout__146;
|
||||
extern __device__ const IsForwardLayout kLayout__145;
|
||||
extern __device__ const MemoryReadLayout kLayout__141;
|
||||
extern __device__ const ReadRegLayout kLayout__140;
|
||||
extern __device__ const MulInputLayout kLayout__120;
|
||||
extern __device__ const ArgU16Layout6LayoutArray kLayout__148;
|
||||
extern __device__ const ArgU8Layout13LayoutArray kLayout__149;
|
||||
extern __device__ const _Arguments_Mul0MulOutputLayout kLayout__147;
|
||||
extern __device__ const NondetRegLayout5LayoutArray kLayout__154;
|
||||
extern __device__ const ToBits_5_Layout kLayout__153;
|
||||
extern __device__ const DynPo2Layout kLayout__152;
|
||||
extern __device__ const NondetU8RegLayout kLayout__158;
|
||||
extern __device__ const NondetU8RegLayout kLayout__159;
|
||||
extern __device__ const NondetU8RegLayout kLayout__160;
|
||||
extern __device__ const NondetU8RegLayout kLayout__161;
|
||||
extern __device__ const NondetU8RegLayout kLayout__162;
|
||||
extern __device__ const ExpandU32Layout kLayout__157;
|
||||
extern __device__ const NondetU8RegLayout kLayout__164;
|
||||
extern __device__ const NondetU8RegLayout kLayout__165;
|
||||
extern __device__ const NondetU8RegLayout kLayout__166;
|
||||
extern __device__ const NondetU8RegLayout kLayout__167;
|
||||
extern __device__ const NondetU8RegLayout kLayout__168;
|
||||
extern __device__ const ExpandU32Layout kLayout__163;
|
||||
extern __device__ const NondetU8RegLayout kLayout__170;
|
||||
extern __device__ const SplitTotalLayout kLayout__169;
|
||||
extern __device__ const NondetU8RegLayout kLayout__172;
|
||||
extern __device__ const SplitTotalLayout kLayout__171;
|
||||
extern __device__ const NondetU8RegLayout kLayout__174;
|
||||
extern __device__ const SplitTotalLayout kLayout__173;
|
||||
extern __device__ const MultiplyAccumulateLayout kLayout__156;
|
||||
extern __device__ const DoMulLayout kLayout__155;
|
||||
extern __device__ const OpSLLLayout kLayout__151;
|
||||
extern __device__ const OpSLLILayout kLayout__175;
|
||||
extern __device__ const ExpandU32Layout kLayout__180;
|
||||
extern __device__ const ExpandU32Layout kLayout__181;
|
||||
extern __device__ const SplitTotalLayout kLayout__182;
|
||||
extern __device__ const SplitTotalLayout kLayout__183;
|
||||
extern __device__ const SplitTotalLayout kLayout__184;
|
||||
extern __device__ const MultiplyAccumulateLayout kLayout__179;
|
||||
extern __device__ const DoMulLayout kLayout__178;
|
||||
extern __device__ const OpMULLayout kLayout__177;
|
||||
extern __device__ const Mul0MulOutputArm2Layout kLayout__176;
|
||||
extern __device__ const OpMULHLayout kLayout__186;
|
||||
extern __device__ const Mul0MulOutputArm3Layout kLayout__185;
|
||||
extern __device__ const OpMULHSULayout kLayout__188;
|
||||
extern __device__ const Mul0MulOutputArm4Layout kLayout__187;
|
||||
extern __device__ const OpMULHULayout kLayout__190;
|
||||
extern __device__ const Mul0MulOutputArm5Layout kLayout__189;
|
||||
extern __device__ const Mul0MulOutputArm6Layout kLayout__191;
|
||||
extern __device__ const Mul0MulOutputArm7Layout kLayout__192;
|
||||
extern __device__ const Mul0MulOutputLayout kLayout__150;
|
||||
extern __device__ const MemoryArgLayout kLayout__196;
|
||||
extern __device__ const MemoryArgLayout kLayout__197;
|
||||
extern __device__ const MemoryIOLayout kLayout__195;
|
||||
extern __device__ const IsCycleLayout kLayout__199;
|
||||
extern __device__ const IsForwardLayout kLayout__198;
|
||||
extern __device__ const MemoryWriteLayout kLayout__194;
|
||||
extern __device__ const WriteRdLayout kLayout__193;
|
||||
extern __device__ const NondetU16RegLayout kLayout__201;
|
||||
extern __device__ const NondetU16RegLayout kLayout__202;
|
||||
extern __device__ const NormalizeU32Layout kLayout__200;
|
||||
extern __device__ const Mul0Layout kLayout__119;
|
||||
extern __device__ const DecoderLayout kLayout__206;
|
||||
extern __device__ const NondetU16RegLayout kLayout__209;
|
||||
extern __device__ const U16RegLayout kLayout__208;
|
||||
extern __device__ const NondetU16RegLayout kLayout__210;
|
||||
extern __device__ const AddrDecomposeLayout kLayout__207;
|
||||
extern __device__ const MemoryArgLayout kLayout__213;
|
||||
extern __device__ const MemoryArgLayout kLayout__214;
|
||||
extern __device__ const MemoryIOLayout kLayout__212;
|
||||
extern __device__ const IsCycleLayout kLayout__216;
|
||||
extern __device__ const IsForwardLayout kLayout__215;
|
||||
extern __device__ const MemoryReadLayout kLayout__211;
|
||||
extern __device__ const DecodeInstLayout kLayout__205;
|
||||
extern __device__ const MemoryArgLayout kLayout__220;
|
||||
extern __device__ const MemoryArgLayout kLayout__221;
|
||||
extern __device__ const MemoryIOLayout kLayout__219;
|
||||
extern __device__ const IsCycleLayout kLayout__223;
|
||||
extern __device__ const IsForwardLayout kLayout__222;
|
||||
extern __device__ const MemoryReadLayout kLayout__218;
|
||||
extern __device__ const ReadRegLayout kLayout__217;
|
||||
extern __device__ const MemoryArgLayout kLayout__227;
|
||||
extern __device__ const MemoryArgLayout kLayout__228;
|
||||
extern __device__ const MemoryIOLayout kLayout__226;
|
||||
extern __device__ const IsCycleLayout kLayout__230;
|
||||
extern __device__ const IsForwardLayout kLayout__229;
|
||||
extern __device__ const MemoryReadLayout kLayout__225;
|
||||
extern __device__ const ReadRegLayout kLayout__224;
|
||||
extern __device__ const DivInputLayout kLayout__204;
|
||||
extern __device__ const ArgU16Layout9LayoutArray kLayout__232;
|
||||
extern __device__ const ArgU8Layout13LayoutArray kLayout__233;
|
||||
extern __device__ const _Arguments_Div0MulOutputLayout kLayout__231;
|
||||
extern __device__ const NondetRegLayout5LayoutArray kLayout__239;
|
||||
extern __device__ const ToBits_5_Layout kLayout__238;
|
||||
extern __device__ const DynPo2Layout kLayout__237;
|
||||
extern __device__ const ExpandU32Layout kLayout__242;
|
||||
extern __device__ const ExpandU32Layout kLayout__243;
|
||||
extern __device__ const NondetU8RegLayout kLayout__245;
|
||||
extern __device__ const SplitTotalLayout kLayout__244;
|
||||
extern __device__ const NondetU8RegLayout kLayout__247;
|
||||
extern __device__ const SplitTotalLayout kLayout__246;
|
||||
extern __device__ const NondetU16RegLayout kLayout__249;
|
||||
extern __device__ const NondetU8RegLayout kLayout__250;
|
||||
extern __device__ const SplitTotalLayout kLayout__248;
|
||||
extern __device__ const NondetU16RegLayout kLayout__251;
|
||||
extern __device__ const MultiplyAccumulateLayout kLayout__241;
|
||||
extern __device__ const DoDivLayout kLayout__240;
|
||||
extern __device__ const OpSRLLayout kLayout__236;
|
||||
extern __device__ const Div0MulOutputArm0Layout kLayout__235;
|
||||
extern __device__ const TopBitLayout kLayout__253;
|
||||
extern __device__ const ExpandU32Layout kLayout__256;
|
||||
extern __device__ const ExpandU32Layout kLayout__257;
|
||||
extern __device__ const SplitTotalLayout kLayout__258;
|
||||
extern __device__ const SplitTotalLayout kLayout__259;
|
||||
extern __device__ const SplitTotalLayout kLayout__260;
|
||||
extern __device__ const MultiplyAccumulateLayout kLayout__255;
|
||||
extern __device__ const DoDivLayout kLayout__254;
|
||||
extern __device__ const OpSRALayout kLayout__252;
|
||||
extern __device__ const OpSRLILayout kLayout__262;
|
||||
extern __device__ const Div0MulOutputArm2Layout kLayout__261;
|
||||
extern __device__ const OpSRAILayout kLayout__263;
|
||||
extern __device__ const ExpandU32Layout kLayout__268;
|
||||
extern __device__ const ExpandU32Layout kLayout__269;
|
||||
extern __device__ const SplitTotalLayout kLayout__270;
|
||||
extern __device__ const SplitTotalLayout kLayout__271;
|
||||
extern __device__ const SplitTotalLayout kLayout__272;
|
||||
extern __device__ const MultiplyAccumulateLayout kLayout__267;
|
||||
extern __device__ const DoDivLayout kLayout__266;
|
||||
extern __device__ const OpDIVLayout kLayout__265;
|
||||
extern __device__ const Div0MulOutputArm4Layout kLayout__264;
|
||||
extern __device__ const OpDIVULayout kLayout__274;
|
||||
extern __device__ const Div0MulOutputArm5Layout kLayout__273;
|
||||
extern __device__ const OpREMLayout kLayout__276;
|
||||
extern __device__ const Div0MulOutputArm6Layout kLayout__275;
|
||||
extern __device__ const OpREMULayout kLayout__278;
|
||||
extern __device__ const Div0MulOutputArm7Layout kLayout__277;
|
||||
extern __device__ const Div0MulOutputLayout kLayout__234;
|
||||
extern __device__ const MemoryArgLayout kLayout__282;
|
||||
extern __device__ const MemoryArgLayout kLayout__283;
|
||||
extern __device__ const MemoryIOLayout kLayout__281;
|
||||
extern __device__ const IsCycleLayout kLayout__285;
|
||||
extern __device__ const IsForwardLayout kLayout__284;
|
||||
extern __device__ const MemoryWriteLayout kLayout__280;
|
||||
extern __device__ const WriteRdLayout kLayout__279;
|
||||
extern __device__ const NondetU16RegLayout kLayout__287;
|
||||
extern __device__ const NondetU16RegLayout kLayout__288;
|
||||
extern __device__ const NormalizeU32Layout kLayout__286;
|
||||
extern __device__ const Div0Layout kLayout__203;
|
||||
extern __device__ const DecoderLayout kLayout__292;
|
||||
extern __device__ const NondetU16RegLayout kLayout__295;
|
||||
extern __device__ const U16RegLayout kLayout__294;
|
||||
extern __device__ const NondetU16RegLayout kLayout__296;
|
||||
extern __device__ const AddrDecomposeLayout kLayout__293;
|
||||
extern __device__ const MemoryArgLayout kLayout__299;
|
||||
extern __device__ const MemoryArgLayout kLayout__300;
|
||||
extern __device__ const MemoryIOLayout kLayout__298;
|
||||
extern __device__ const IsCycleLayout kLayout__302;
|
||||
extern __device__ const IsForwardLayout kLayout__301;
|
||||
extern __device__ const MemoryReadLayout kLayout__297;
|
||||
extern __device__ const DecodeInstLayout kLayout__291;
|
||||
extern __device__ const MemoryArgLayout kLayout__306;
|
||||
extern __device__ const MemoryArgLayout kLayout__307;
|
||||
extern __device__ const MemoryIOLayout kLayout__305;
|
||||
extern __device__ const IsCycleLayout kLayout__309;
|
||||
extern __device__ const IsForwardLayout kLayout__308;
|
||||
extern __device__ const MemoryReadLayout kLayout__304;
|
||||
extern __device__ const ReadRegLayout kLayout__303;
|
||||
extern __device__ const NormalizeU32Layout kLayout__310;
|
||||
extern __device__ const NondetU16RegLayout kLayout__313;
|
||||
extern __device__ const U16RegLayout kLayout__312;
|
||||
extern __device__ const NondetU16RegLayout kLayout__314;
|
||||
extern __device__ const AddrDecomposeBitsLayout kLayout__311;
|
||||
extern __device__ const MemoryArgLayout kLayout__317;
|
||||
extern __device__ const MemoryArgLayout kLayout__318;
|
||||
extern __device__ const MemoryIOLayout kLayout__316;
|
||||
extern __device__ const IsCycleLayout kLayout__320;
|
||||
extern __device__ const IsForwardLayout kLayout__319;
|
||||
extern __device__ const MemoryReadLayout kLayout__315;
|
||||
extern __device__ const MemLoadInputLayout kLayout__290;
|
||||
extern __device__ const ArgU8Layout3LayoutArray kLayout__322;
|
||||
extern __device__ const _Arguments_Mem0OutputLayout kLayout__321;
|
||||
extern __device__ const NondetU8RegLayout kLayout__327;
|
||||
extern __device__ const NondetU8RegLayout kLayout__328;
|
||||
extern __device__ const SplitWordLayout kLayout__326;
|
||||
extern __device__ const NondetU8RegLayout kLayout__329;
|
||||
extern __device__ const OpLBLayout kLayout__325;
|
||||
extern __device__ const Mem0OutputArm0Layout kLayout__324;
|
||||
extern __device__ const OpLHLayout kLayout__331;
|
||||
extern __device__ const Mem0OutputArm1Layout kLayout__330;
|
||||
extern __device__ const Mem0OutputArm2Layout kLayout__332;
|
||||
extern __device__ const OpLBULayout kLayout__334;
|
||||
extern __device__ const Mem0OutputArm3Layout kLayout__333;
|
||||
extern __device__ const Mem0OutputArm4Layout kLayout__335;
|
||||
extern __device__ const Mem0OutputArm5Layout kLayout__336;
|
||||
extern __device__ const Mem0OutputArm6Layout kLayout__337;
|
||||
extern __device__ const Mem0OutputArm7Layout kLayout__338;
|
||||
extern __device__ const Mem0OutputLayout kLayout__323;
|
||||
extern __device__ const MemoryArgLayout kLayout__342;
|
||||
extern __device__ const MemoryArgLayout kLayout__343;
|
||||
extern __device__ const MemoryIOLayout kLayout__341;
|
||||
extern __device__ const IsCycleLayout kLayout__345;
|
||||
extern __device__ const IsForwardLayout kLayout__344;
|
||||
extern __device__ const MemoryWriteLayout kLayout__340;
|
||||
extern __device__ const WriteRdLayout kLayout__339;
|
||||
extern __device__ const NondetU16RegLayout kLayout__347;
|
||||
extern __device__ const NondetU16RegLayout kLayout__348;
|
||||
extern __device__ const NormalizeU32Layout kLayout__346;
|
||||
extern __device__ const Mem0Layout kLayout__289;
|
||||
extern __device__ const MemoryArgLayout kLayout__354;
|
||||
extern __device__ const MemoryArgLayout kLayout__355;
|
||||
extern __device__ const MemoryIOLayout kLayout__353;
|
||||
extern __device__ const IsCycleLayout kLayout__357;
|
||||
extern __device__ const IsForwardLayout kLayout__356;
|
||||
extern __device__ const MemoryReadLayout kLayout__352;
|
||||
extern __device__ const ReadRegLayout kLayout__351;
|
||||
extern __device__ const NondetU16RegLayout kLayout__359;
|
||||
extern __device__ const NondetU16RegLayout kLayout__360;
|
||||
extern __device__ const NormalizeU32Layout kLayout__358;
|
||||
extern __device__ const NondetU16RegLayout kLayout__363;
|
||||
extern __device__ const U16RegLayout kLayout__362;
|
||||
extern __device__ const NondetU16RegLayout kLayout__364;
|
||||
extern __device__ const AddrDecomposeBitsLayout kLayout__361;
|
||||
extern __device__ const MemStoreInputLayout kLayout__350;
|
||||
extern __device__ const ArgU8Layout4LayoutArray kLayout__366;
|
||||
extern __device__ const _Arguments_Mem1OutputLayout kLayout__365;
|
||||
extern __device__ const NondetU8RegLayout kLayout__370;
|
||||
extern __device__ const SplitWordLayout kLayout__369;
|
||||
extern __device__ const OpSBLayout kLayout__368;
|
||||
extern __device__ const Mem1OutputArm1Layout kLayout__371;
|
||||
extern __device__ const Mem1OutputArm2Layout kLayout__372;
|
||||
extern __device__ const Mem1OutputArm3Layout kLayout__373;
|
||||
extern __device__ const Mem1OutputArm4Layout kLayout__374;
|
||||
extern __device__ const Mem1OutputArm5Layout kLayout__375;
|
||||
extern __device__ const Mem1OutputArm6Layout kLayout__376;
|
||||
extern __device__ const Mem1OutputArm7Layout kLayout__377;
|
||||
extern __device__ const Mem1OutputLayout kLayout__367;
|
||||
extern __device__ const MemoryArgLayout kLayout__381;
|
||||
extern __device__ const MemoryArgLayout kLayout__382;
|
||||
extern __device__ const MemoryIOLayout kLayout__380;
|
||||
extern __device__ const IsCycleLayout kLayout__384;
|
||||
extern __device__ const IsForwardLayout kLayout__383;
|
||||
extern __device__ const MemoryWriteLayout kLayout__379;
|
||||
extern __device__ const MemStoreFinalizeLayout kLayout__378;
|
||||
extern __device__ const NondetU16RegLayout kLayout__386;
|
||||
extern __device__ const NondetU16RegLayout kLayout__387;
|
||||
extern __device__ const NormalizeU32Layout kLayout__385;
|
||||
extern __device__ const Mem1Layout kLayout__349;
|
||||
extern __device__ const MemoryArgLayout kLayout__396;
|
||||
extern __device__ const MemoryArgLayout kLayout__397;
|
||||
extern __device__ const MemoryIOLayout kLayout__395;
|
||||
extern __device__ const MemoryPageInLayout kLayout__394;
|
||||
extern __device__ const ControlLoadRoot__0_SuperLayout kLayout__393;
|
||||
extern __device__ const MemoryArgLayout kLayout__401;
|
||||
extern __device__ const MemoryArgLayout kLayout__402;
|
||||
extern __device__ const MemoryIOLayout kLayout__400;
|
||||
extern __device__ const MemoryPageInLayout kLayout__399;
|
||||
extern __device__ const ControlLoadRoot__0_SuperLayout kLayout__398;
|
||||
extern __device__ const MemoryArgLayout kLayout__406;
|
||||
extern __device__ const MemoryArgLayout kLayout__407;
|
||||
extern __device__ const MemoryIOLayout kLayout__405;
|
||||
extern __device__ const MemoryPageInLayout kLayout__404;
|
||||
extern __device__ const ControlLoadRoot__0_SuperLayout kLayout__403;
|
||||
extern __device__ const MemoryArgLayout kLayout__411;
|
||||
extern __device__ const MemoryArgLayout kLayout__412;
|
||||
extern __device__ const MemoryIOLayout kLayout__410;
|
||||
extern __device__ const MemoryPageInLayout kLayout__409;
|
||||
extern __device__ const ControlLoadRoot__0_SuperLayout kLayout__408;
|
||||
extern __device__ const MemoryArgLayout kLayout__416;
|
||||
extern __device__ const MemoryArgLayout kLayout__417;
|
||||
extern __device__ const MemoryIOLayout kLayout__415;
|
||||
extern __device__ const MemoryPageInLayout kLayout__414;
|
||||
extern __device__ const ControlLoadRoot__0_SuperLayout kLayout__413;
|
||||
extern __device__ const MemoryArgLayout kLayout__421;
|
||||
extern __device__ const MemoryArgLayout kLayout__422;
|
||||
extern __device__ const MemoryIOLayout kLayout__420;
|
||||
extern __device__ const MemoryPageInLayout kLayout__419;
|
||||
extern __device__ const ControlLoadRoot__0_SuperLayout kLayout__418;
|
||||
extern __device__ const MemoryArgLayout kLayout__426;
|
||||
extern __device__ const MemoryArgLayout kLayout__427;
|
||||
extern __device__ const MemoryIOLayout kLayout__425;
|
||||
extern __device__ const MemoryPageInLayout kLayout__424;
|
||||
extern __device__ const ControlLoadRoot__0_SuperLayout kLayout__423;
|
||||
extern __device__ const MemoryArgLayout kLayout__431;
|
||||
extern __device__ const MemoryArgLayout kLayout__432;
|
||||
extern __device__ const MemoryIOLayout kLayout__430;
|
||||
extern __device__ const MemoryPageInLayout kLayout__429;
|
||||
extern __device__ const ControlLoadRoot__0_SuperLayout kLayout__428;
|
||||
extern __device__ const ControlLoadRoot__0_SuperLayout8LayoutArray kLayout__392;
|
||||
extern __device__ const ControlLoadRootLayout kLayout__391;
|
||||
extern __device__ const Control0_SuperArm0Layout kLayout__390;
|
||||
extern __device__ const IsCycleLayout kLayout__440;
|
||||
extern __device__ const IsForwardLayout kLayout__439;
|
||||
extern __device__ const MemoryReadLayout kLayout__438;
|
||||
extern __device__ const IsCycleLayout kLayout__443;
|
||||
extern __device__ const IsForwardLayout kLayout__442;
|
||||
extern __device__ const MemoryReadLayout kLayout__441;
|
||||
extern __device__ const ControlResume_SuperArm0_SuperLayout kLayout__437;
|
||||
extern __device__ const ControlResume_SuperArm0Layout kLayout__436;
|
||||
extern __device__ const MemoryWriteLayout kLayout__447;
|
||||
extern __device__ const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__446;
|
||||
extern __device__ const MemoryWriteLayout kLayout__449;
|
||||
extern __device__ const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__448;
|
||||
extern __device__ const IsCycleLayout kLayout__453;
|
||||
extern __device__ const IsForwardLayout kLayout__452;
|
||||
extern __device__ const MemoryWriteLayout kLayout__451;
|
||||
extern __device__ const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__450;
|
||||
extern __device__ const MemoryWriteLayout kLayout__455;
|
||||
extern __device__ const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__454;
|
||||
extern __device__ const IsCycleLayout kLayout__459;
|
||||
extern __device__ const IsForwardLayout kLayout__458;
|
||||
extern __device__ const MemoryWriteLayout kLayout__457;
|
||||
extern __device__ const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__456;
|
||||
extern __device__ const IsCycleLayout kLayout__463;
|
||||
extern __device__ const IsForwardLayout kLayout__462;
|
||||
extern __device__ const MemoryWriteLayout kLayout__461;
|
||||
extern __device__ const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__460;
|
||||
extern __device__ const MemoryWriteLayout kLayout__465;
|
||||
extern __device__ const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__464;
|
||||
extern __device__ const IsCycleLayout kLayout__469;
|
||||
extern __device__ const IsForwardLayout kLayout__468;
|
||||
extern __device__ const MemoryWriteLayout kLayout__467;
|
||||
extern __device__ const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__466;
|
||||
extern __device__ const ControlResume_SuperArm1_Super__0_SuperLayout8LayoutArray kLayout__445;
|
||||
extern __device__ const ControlResume_SuperArm1_SuperLayout kLayout__444;
|
||||
extern __device__ const ControlResume_SuperLayout kLayout__435;
|
||||
extern __device__ const MemoryArgLayout16LayoutArray kLayout__471;
|
||||
extern __device__ const CycleArgLayout8LayoutArray kLayout__472;
|
||||
extern __device__ const _Arguments_ControlResume_SuperLayout kLayout__470;
|
||||
extern __device__ const ControlResumeLayout kLayout__434;
|
||||
extern __device__ const Control0_SuperArm1Layout kLayout__433;
|
||||
extern __device__ const NondetU16RegLayout kLayout__477;
|
||||
extern __device__ const U16RegLayout kLayout__476;
|
||||
extern __device__ const NondetU16RegLayout kLayout__478;
|
||||
extern __device__ const AddrDecomposeBitsLayout kLayout__475;
|
||||
extern __device__ const NondetU16RegLayout kLayout__480;
|
||||
extern __device__ const U16RegLayout kLayout__479;
|
||||
extern __device__ const MemoryReadLayout kLayout__481;
|
||||
extern __device__ const ControlUserECALLLayout kLayout__474;
|
||||
extern __device__ const Control0_SuperArm2Layout kLayout__473;
|
||||
extern __device__ const NondetU16RegLayout kLayout__485;
|
||||
extern __device__ const NormalizeU32Layout kLayout__484;
|
||||
extern __device__ const ControlMRETLayout kLayout__483;
|
||||
extern __device__ const Control0_SuperArm3Layout kLayout__482;
|
||||
extern __device__ const MemoryReadLayout kLayout__491;
|
||||
extern __device__ const MemoryReadLayout kLayout__492;
|
||||
extern __device__ const MemoryReadLayout kLayout__493;
|
||||
extern __device__ const MemoryReadLayout kLayout__494;
|
||||
extern __device__ const MemoryReadLayout kLayout__495;
|
||||
extern __device__ const MemoryReadLayout8LayoutArray kLayout__490;
|
||||
extern __device__ const ControlSuspend_SuperArm0_SuperLayout kLayout__489;
|
||||
extern __device__ const ControlSuspend_SuperArm1_SuperLayout kLayout__497;
|
||||
extern __device__ const ControlSuspend_SuperArm1Layout kLayout__496;
|
||||
extern __device__ const ControlSuspend_SuperLayout kLayout__488;
|
||||
extern __device__ const _Arguments_ControlSuspend_SuperLayout kLayout__498;
|
||||
extern __device__ const ControlSuspendLayout kLayout__487;
|
||||
extern __device__ const Control0_SuperArm4Layout kLayout__486;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__502;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__503;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__504;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__505;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__506;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__507;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__508;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__509;
|
||||
extern __device__ const MemoryPageOutLayout8LayoutArray kLayout__501;
|
||||
extern __device__ const ControlStoreRootLayout kLayout__500;
|
||||
extern __device__ const Control0_SuperArm5Layout kLayout__499;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__516;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__517;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__518;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__519;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__520;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__521;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__522;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__523;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__524;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__525;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__526;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__527;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__528;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__529;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__530;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__531;
|
||||
extern __device__ const ControlTable_SuperArm0_Super__0_SuperLayout16LayoutArray kLayout__515;
|
||||
extern __device__ const ControlTable_SuperArm0_SuperLayout kLayout__514;
|
||||
extern __device__ const ControlTable_SuperArm0Layout kLayout__513;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__535;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__536;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__537;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__538;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__539;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__540;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__541;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__542;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__543;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__544;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__545;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__546;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__547;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__548;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__549;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__550;
|
||||
extern __device__ const ControlTable_SuperArm1_Super__0_SuperLayout16LayoutArray kLayout__534;
|
||||
extern __device__ const ControlTable_SuperArm1_SuperLayout kLayout__533;
|
||||
extern __device__ const ControlTable_SuperArm1Layout kLayout__532;
|
||||
extern __device__ const ControlTable_SuperLayout kLayout__512;
|
||||
extern __device__ const ArgU16Layout16LayoutArray kLayout__552;
|
||||
extern __device__ const ArgU8Layout16LayoutArray kLayout__553;
|
||||
extern __device__ const _Arguments_ControlTable_SuperLayout kLayout__551;
|
||||
extern __device__ const ControlTableLayout kLayout__511;
|
||||
extern __device__ const Control0_SuperArm6Layout kLayout__510;
|
||||
extern __device__ const Control0_SuperArm7Layout kLayout__554;
|
||||
extern __device__ const Control0_SuperLayout kLayout__389;
|
||||
extern __device__ const _Arguments_Control0_SuperLayout kLayout__555;
|
||||
extern __device__ const Control0Layout kLayout__388;
|
||||
extern __device__ const NondetU16RegLayout kLayout__559;
|
||||
extern __device__ const U16RegLayout kLayout__558;
|
||||
extern __device__ const AddrDecomposeBitsLayout kLayout__557;
|
||||
extern __device__ const MemoryArgLayout8LayoutArray kLayout__561;
|
||||
extern __device__ const CycleArgLayout4LayoutArray kLayout__562;
|
||||
extern __device__ const ArgU16Layout2LayoutArray kLayout__563;
|
||||
extern __device__ const _Arguments_ECall0OutputLayout kLayout__560;
|
||||
extern __device__ const IsCycleLayout kLayout__569;
|
||||
extern __device__ const IsForwardLayout kLayout__568;
|
||||
extern __device__ const MemoryReadLayout kLayout__567;
|
||||
extern __device__ const IsCycleLayout kLayout__572;
|
||||
extern __device__ const IsForwardLayout kLayout__571;
|
||||
extern __device__ const MemoryReadLayout kLayout__570;
|
||||
extern __device__ const NondetRegLayout4LayoutArray kLayout__574;
|
||||
extern __device__ const OneHot_4_Layout kLayout__573;
|
||||
extern __device__ const MachineECallLayout kLayout__566;
|
||||
extern __device__ const ECall0OutputArm0Layout kLayout__565;
|
||||
extern __device__ const ECallTerminateLayout kLayout__576;
|
||||
extern __device__ const ECall0OutputArm1Layout kLayout__575;
|
||||
extern __device__ const IsCycleLayout kLayout__580;
|
||||
extern __device__ const IsForwardLayout kLayout__579;
|
||||
extern __device__ const MemoryReadLayout kLayout__578;
|
||||
extern __device__ const NondetU16RegLayout kLayout__581;
|
||||
extern __device__ const NondetU16RegLayout kLayout__583;
|
||||
extern __device__ const U16RegLayout kLayout__582;
|
||||
extern __device__ const IsCycleLayout kLayout__586;
|
||||
extern __device__ const IsForwardLayout kLayout__585;
|
||||
extern __device__ const MemoryWriteLayout kLayout__584;
|
||||
extern __device__ const NondetRegLayout4LayoutArray kLayout__589;
|
||||
extern __device__ const OneHot_4_Layout kLayout__588;
|
||||
extern __device__ const DecomposeLow2Layout kLayout__587;
|
||||
extern __device__ const NondetRegLayout4LayoutArray kLayout__592;
|
||||
extern __device__ const OneHot_4_Layout kLayout__591;
|
||||
extern __device__ const DecomposeLow2Layout kLayout__590;
|
||||
extern __device__ const ECallHostReadSetupLayout kLayout__577;
|
||||
extern __device__ const ECallHostWriteLayout kLayout__593;
|
||||
extern __device__ const ECall0OutputArm4Layout kLayout__594;
|
||||
extern __device__ const MemoryWriteUnconstrainedLayout kLayout__599;
|
||||
extern __device__ const ECallHostReadWords__0_SuperLayout kLayout__598;
|
||||
extern __device__ const MemoryWriteUnconstrainedLayout kLayout__601;
|
||||
extern __device__ const ECallHostReadWords__0_SuperLayout kLayout__600;
|
||||
extern __device__ const MemoryWriteUnconstrainedLayout kLayout__603;
|
||||
extern __device__ const ECallHostReadWords__0_SuperLayout kLayout__602;
|
||||
extern __device__ const MemoryWriteUnconstrainedLayout kLayout__605;
|
||||
extern __device__ const ECallHostReadWords__0_SuperLayout kLayout__604;
|
||||
extern __device__ const ECallHostReadWords__0_SuperLayout4LayoutArray kLayout__597;
|
||||
extern __device__ const ECallHostReadWordsLayout kLayout__596;
|
||||
extern __device__ const ECall0OutputArm5Layout kLayout__595;
|
||||
extern __device__ const ECall0OutputArm6Layout kLayout__606;
|
||||
extern __device__ const ECall0OutputArm7Layout kLayout__607;
|
||||
extern __device__ const ECall0OutputLayout kLayout__564;
|
||||
extern __device__ const NondetU16RegLayout kLayout__609;
|
||||
extern __device__ const NormalizeU32Layout kLayout__608;
|
||||
extern __device__ const ECall0Layout kLayout__556;
|
||||
extern __device__ const NondetRegLayout24LayoutArray kLayout__612;
|
||||
extern __device__ const PoseidonStateLayout kLayout__611;
|
||||
extern __device__ const MemoryArgLayout kLayout__615;
|
||||
extern __device__ const MemoryArgLayout kLayout__616;
|
||||
extern __device__ const MemoryArgLayout kLayout__617;
|
||||
extern __device__ const MemoryArgLayout kLayout__618;
|
||||
extern __device__ const MemoryArgLayout kLayout__619;
|
||||
extern __device__ const MemoryArgLayout kLayout__620;
|
||||
extern __device__ const MemoryArgLayout kLayout__621;
|
||||
extern __device__ const MemoryArgLayout kLayout__622;
|
||||
extern __device__ const MemoryArgLayout kLayout__623;
|
||||
extern __device__ const MemoryArgLayout kLayout__624;
|
||||
extern __device__ const MemoryArgLayout kLayout__625;
|
||||
extern __device__ const MemoryArgLayout kLayout__626;
|
||||
extern __device__ const MemoryArgLayout kLayout__627;
|
||||
extern __device__ const MemoryArgLayout kLayout__628;
|
||||
extern __device__ const MemoryArgLayout kLayout__629;
|
||||
extern __device__ const MemoryArgLayout kLayout__630;
|
||||
extern __device__ const MemoryArgLayout16LayoutArray kLayout__614;
|
||||
extern __device__ const CycleArgLayout8LayoutArray kLayout__631;
|
||||
extern __device__ const ArgU16Layout16LayoutArray kLayout__632;
|
||||
extern __device__ const ArgU8Layout2LayoutArray kLayout__633;
|
||||
extern __device__ const _Arguments_Poseidon0StateLayout kLayout__613;
|
||||
extern __device__ const PoseidonEntry_SuperArm0Layout kLayout__638;
|
||||
extern __device__ const MemoryIOLayout kLayout__642;
|
||||
extern __device__ const IsCycleLayout kLayout__644;
|
||||
extern __device__ const IsForwardLayout kLayout__643;
|
||||
extern __device__ const MemoryReadLayout kLayout__641;
|
||||
extern __device__ const ReadAddrLayout kLayout__640;
|
||||
extern __device__ const MemoryIOLayout kLayout__647;
|
||||
extern __device__ const IsCycleLayout kLayout__649;
|
||||
extern __device__ const IsForwardLayout kLayout__648;
|
||||
extern __device__ const MemoryReadLayout kLayout__646;
|
||||
extern __device__ const ReadAddrLayout kLayout__645;
|
||||
extern __device__ const MemoryIOLayout kLayout__652;
|
||||
extern __device__ const IsCycleLayout kLayout__654;
|
||||
extern __device__ const IsForwardLayout kLayout__653;
|
||||
extern __device__ const MemoryReadLayout kLayout__651;
|
||||
extern __device__ const ReadAddrLayout kLayout__650;
|
||||
extern __device__ const MemoryIOLayout kLayout__656;
|
||||
extern __device__ const IsCycleLayout kLayout__658;
|
||||
extern __device__ const IsForwardLayout kLayout__657;
|
||||
extern __device__ const MemoryReadLayout kLayout__655;
|
||||
extern __device__ const PoseidonEcallLayout kLayout__639;
|
||||
extern __device__ const PoseidonEntry_SuperLayout kLayout__637;
|
||||
extern __device__ const MemoryArgLayout8LayoutArray kLayout__660;
|
||||
extern __device__ const CycleArgLayout4LayoutArray kLayout__661;
|
||||
extern __device__ const _Arguments_PoseidonEntry_SuperLayout kLayout__659;
|
||||
extern __device__ const PoseidonEntryLayout kLayout__636;
|
||||
extern __device__ const Poseidon0StateArm0Layout kLayout__635;
|
||||
extern __device__ const ReadElemLayout kLayout__665;
|
||||
extern __device__ const ReadElemLayout kLayout__666;
|
||||
extern __device__ const ReadElemLayout kLayout__667;
|
||||
extern __device__ const ReadElemLayout kLayout__668;
|
||||
extern __device__ const MemoryIOLayout kLayout__671;
|
||||
extern __device__ const IsCycleLayout kLayout__673;
|
||||
extern __device__ const IsForwardLayout kLayout__672;
|
||||
extern __device__ const MemoryReadLayout kLayout__670;
|
||||
extern __device__ const ReadElemLayout kLayout__669;
|
||||
extern __device__ const MemoryIOLayout kLayout__676;
|
||||
extern __device__ const IsCycleLayout kLayout__678;
|
||||
extern __device__ const IsForwardLayout kLayout__677;
|
||||
extern __device__ const MemoryReadLayout kLayout__675;
|
||||
extern __device__ const ReadElemLayout kLayout__674;
|
||||
extern __device__ const MemoryIOLayout kLayout__681;
|
||||
extern __device__ const IsCycleLayout kLayout__683;
|
||||
extern __device__ const IsForwardLayout kLayout__682;
|
||||
extern __device__ const MemoryReadLayout kLayout__680;
|
||||
extern __device__ const ReadElemLayout kLayout__679;
|
||||
extern __device__ const MemoryIOLayout kLayout__686;
|
||||
extern __device__ const IsCycleLayout kLayout__688;
|
||||
extern __device__ const IsForwardLayout kLayout__687;
|
||||
extern __device__ const MemoryReadLayout kLayout__685;
|
||||
extern __device__ const ReadElemLayout kLayout__684;
|
||||
extern __device__ const ReadElemLayout8LayoutArray kLayout__664;
|
||||
extern __device__ const PoseidonLoadStateLayout kLayout__663;
|
||||
extern __device__ const Poseidon0StateArm1Layout kLayout__662;
|
||||
extern __device__ const OneHot_3_Layout kLayout__693;
|
||||
extern __device__ const MemoryPageInLayout kLayout__698;
|
||||
extern __device__ const MemoryGet_SuperArm1Layout kLayout__697;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__699;
|
||||
extern __device__ const MemoryGet_SuperLayout kLayout__696;
|
||||
extern __device__ const MemoryArgLayout2LayoutArray kLayout__701;
|
||||
extern __device__ const _Arguments_MemoryGet_SuperLayout kLayout__700;
|
||||
extern __device__ const MemoryGetLayout kLayout__695;
|
||||
extern __device__ const MemoryPageInLayout kLayout__705;
|
||||
extern __device__ const MemoryGet_SuperArm1Layout kLayout__704;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__706;
|
||||
extern __device__ const MemoryGet_SuperLayout kLayout__703;
|
||||
extern __device__ const MemoryArgLayout2LayoutArray kLayout__708;
|
||||
extern __device__ const _Arguments_MemoryGet_SuperLayout kLayout__707;
|
||||
extern __device__ const MemoryGetLayout kLayout__702;
|
||||
extern __device__ const MemoryPageInLayout kLayout__712;
|
||||
extern __device__ const MemoryGet_SuperArm1Layout kLayout__711;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__713;
|
||||
extern __device__ const MemoryGet_SuperLayout kLayout__710;
|
||||
extern __device__ const MemoryArgLayout2LayoutArray kLayout__715;
|
||||
extern __device__ const _Arguments_MemoryGet_SuperLayout kLayout__714;
|
||||
extern __device__ const MemoryGetLayout kLayout__709;
|
||||
extern __device__ const MemoryPageInLayout kLayout__719;
|
||||
extern __device__ const MemoryGet_SuperArm1Layout kLayout__718;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__720;
|
||||
extern __device__ const MemoryGet_SuperLayout kLayout__717;
|
||||
extern __device__ const MemoryArgLayout2LayoutArray kLayout__722;
|
||||
extern __device__ const _Arguments_MemoryGet_SuperLayout kLayout__721;
|
||||
extern __device__ const MemoryGetLayout kLayout__716;
|
||||
extern __device__ const MemoryPageInLayout kLayout__726;
|
||||
extern __device__ const MemoryGet_SuperArm1Layout kLayout__725;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__727;
|
||||
extern __device__ const MemoryGet_SuperLayout kLayout__724;
|
||||
extern __device__ const MemoryArgLayout2LayoutArray kLayout__729;
|
||||
extern __device__ const _Arguments_MemoryGet_SuperLayout kLayout__728;
|
||||
extern __device__ const MemoryGetLayout kLayout__723;
|
||||
extern __device__ const MemoryPageInLayout kLayout__733;
|
||||
extern __device__ const MemoryGet_SuperArm1Layout kLayout__732;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__734;
|
||||
extern __device__ const MemoryGet_SuperLayout kLayout__731;
|
||||
extern __device__ const MemoryArgLayout2LayoutArray kLayout__736;
|
||||
extern __device__ const _Arguments_MemoryGet_SuperLayout kLayout__735;
|
||||
extern __device__ const MemoryGetLayout kLayout__730;
|
||||
extern __device__ const MemoryPageInLayout kLayout__740;
|
||||
extern __device__ const MemoryGet_SuperArm1Layout kLayout__739;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__741;
|
||||
extern __device__ const MemoryGet_SuperLayout kLayout__738;
|
||||
extern __device__ const MemoryArgLayout2LayoutArray kLayout__743;
|
||||
extern __device__ const _Arguments_MemoryGet_SuperLayout kLayout__742;
|
||||
extern __device__ const MemoryGetLayout kLayout__737;
|
||||
extern __device__ const MemoryPageInLayout kLayout__747;
|
||||
extern __device__ const MemoryGet_SuperArm1Layout kLayout__746;
|
||||
extern __device__ const MemoryPageOutLayout kLayout__748;
|
||||
extern __device__ const MemoryGet_SuperLayout kLayout__745;
|
||||
extern __device__ const MemoryArgLayout2LayoutArray kLayout__750;
|
||||
extern __device__ const _Arguments_MemoryGet_SuperLayout kLayout__749;
|
||||
extern __device__ const MemoryGetLayout kLayout__744;
|
||||
extern __device__ const MemoryGetLayout8LayoutArray kLayout__694;
|
||||
extern __device__ const PoseidonLoadInShortLayout kLayout__692;
|
||||
extern __device__ const PoseidonLoadInLowLayout kLayout__751;
|
||||
extern __device__ const PoseidonLoadInHighLayout kLayout__752;
|
||||
extern __device__ const PoseidonLoadIn_SuperLayout kLayout__691;
|
||||
extern __device__ const OneHot_3_Layout kLayout__753;
|
||||
extern __device__ const _Arguments_PoseidonLoadIn_SuperLayout kLayout__754;
|
||||
extern __device__ const PoseidonLoadInLayout kLayout__690;
|
||||
extern __device__ const Poseidon0StateArm2Layout kLayout__689;
|
||||
extern __device__ const Poseidon0StateArm3Layout kLayout__755;
|
||||
extern __device__ const Poseidon0StateArm4Layout kLayout__756;
|
||||
extern __device__ const PoseidonCheckOut__0_SuperLayout kLayout__763;
|
||||
extern __device__ const PoseidonCheckOut__0_SuperLayout kLayout__764;
|
||||
extern __device__ const PoseidonCheckOut__0_SuperLayout kLayout__765;
|
||||
extern __device__ const PoseidonCheckOut__0_SuperLayout kLayout__766;
|
||||
extern __device__ const PoseidonCheckOut__0_SuperLayout kLayout__767;
|
||||
extern __device__ const PoseidonCheckOut__0_SuperLayout kLayout__768;
|
||||
extern __device__ const PoseidonCheckOut__0_SuperLayout kLayout__769;
|
||||
extern __device__ const PoseidonCheckOut__0_SuperLayout kLayout__770;
|
||||
extern __device__ const PoseidonCheckOut__0_SuperLayout8LayoutArray kLayout__762;
|
||||
extern __device__ const PoseidonCheckOutLayout kLayout__761;
|
||||
extern __device__ const PoseidonDoOut_SuperArm0Layout kLayout__760;
|
||||
extern __device__ const NondetU16RegLayout kLayout__774;
|
||||
extern __device__ const NondetU16RegLayout kLayout__776;
|
||||
extern __device__ const U16RegLayout kLayout__775;
|
||||
extern __device__ const MemoryWriteLayout kLayout__777;
|
||||
extern __device__ const PoseidonStoreOut__0_SuperLayout kLayout__773;
|
||||
extern __device__ const NondetU16RegLayout kLayout__779;
|
||||
extern __device__ const NondetU16RegLayout kLayout__781;
|
||||
extern __device__ const U16RegLayout kLayout__780;
|
||||
extern __device__ const MemoryWriteLayout kLayout__782;
|
||||
extern __device__ const PoseidonStoreOut__0_SuperLayout kLayout__778;
|
||||
extern __device__ const NondetU16RegLayout kLayout__784;
|
||||
extern __device__ const U16RegLayout kLayout__785;
|
||||
extern __device__ const MemoryWriteLayout kLayout__786;
|
||||
extern __device__ const PoseidonStoreOut__0_SuperLayout kLayout__783;
|
||||
extern __device__ const NondetU16RegLayout kLayout__788;
|
||||
extern __device__ const NondetU16RegLayout kLayout__790;
|
||||
extern __device__ const U16RegLayout kLayout__789;
|
||||
extern __device__ const MemoryWriteLayout kLayout__791;
|
||||
extern __device__ const PoseidonStoreOut__0_SuperLayout kLayout__787;
|
||||
extern __device__ const NondetU16RegLayout kLayout__793;
|
||||
extern __device__ const NondetU16RegLayout kLayout__795;
|
||||
extern __device__ const U16RegLayout kLayout__794;
|
||||
extern __device__ const MemoryWriteLayout kLayout__796;
|
||||
extern __device__ const PoseidonStoreOut__0_SuperLayout kLayout__792;
|
||||
extern __device__ const NondetU16RegLayout kLayout__799;
|
||||
extern __device__ const U16RegLayout kLayout__798;
|
||||
extern __device__ const MemoryWriteLayout kLayout__800;
|
||||
extern __device__ const PoseidonStoreOut__0_SuperLayout kLayout__797;
|
||||
extern __device__ const NondetU16RegLayout kLayout__802;
|
||||
extern __device__ const NondetU16RegLayout kLayout__804;
|
||||
extern __device__ const U16RegLayout kLayout__803;
|
||||
extern __device__ const MemoryWriteLayout kLayout__805;
|
||||
extern __device__ const PoseidonStoreOut__0_SuperLayout kLayout__801;
|
||||
extern __device__ const NondetU16RegLayout kLayout__807;
|
||||
extern __device__ const NondetU16RegLayout kLayout__809;
|
||||
extern __device__ const U16RegLayout kLayout__808;
|
||||
extern __device__ const MemoryWriteLayout kLayout__810;
|
||||
extern __device__ const PoseidonStoreOut__0_SuperLayout kLayout__806;
|
||||
extern __device__ const PoseidonStoreOut__0_SuperLayout8LayoutArray kLayout__772;
|
||||
extern __device__ const PoseidonStoreOutLayout kLayout__771;
|
||||
extern __device__ const PoseidonDoOut_SuperLayout kLayout__759;
|
||||
extern __device__ const _Arguments_PoseidonDoOut_SuperLayout kLayout__811;
|
||||
extern __device__ const PoseidonDoOutLayout kLayout__758;
|
||||
extern __device__ const Poseidon0StateArm5Layout kLayout__757;
|
||||
extern __device__ const PoseidonPaging_SuperLayout kLayout__814;
|
||||
extern __device__ const NondetRegLayout6LayoutArray kLayout__816;
|
||||
extern __device__ const OneHot_6_Layout kLayout__815;
|
||||
extern __device__ const NondetU8RegLayout kLayout__819;
|
||||
extern __device__ const U8RegLayout kLayout__818;
|
||||
extern __device__ const IsU24Layout kLayout__817;
|
||||
extern __device__ const _Arguments_PoseidonPaging__1Layout kLayout__820;
|
||||
extern __device__ const NondetU8RegLayout kLayout__825;
|
||||
extern __device__ const U8RegLayout kLayout__824;
|
||||
extern __device__ const IsU24Layout kLayout__823;
|
||||
extern __device__ const PoseidonPaging__1Arm0_SuperLayout kLayout__822;
|
||||
extern __device__ const PoseidonPaging__1Arm1_SuperLayout kLayout__826;
|
||||
extern __device__ const PoseidonPaging__1Layout kLayout__821;
|
||||
extern __device__ const PoseidonPagingLayout kLayout__813;
|
||||
extern __device__ const Poseidon0StateArm6Layout kLayout__812;
|
||||
extern __device__ const PoseidonStoreState__0_SuperLayout kLayout__830;
|
||||
extern __device__ const PoseidonStoreState__0_SuperLayout kLayout__831;
|
||||
extern __device__ const PoseidonStoreState__0_SuperLayout kLayout__832;
|
||||
extern __device__ const PoseidonStoreState__0_SuperLayout kLayout__833;
|
||||
extern __device__ const PoseidonStoreState__0_SuperLayout kLayout__834;
|
||||
extern __device__ const PoseidonStoreState__0_SuperLayout kLayout__835;
|
||||
extern __device__ const PoseidonStoreState__0_SuperLayout kLayout__836;
|
||||
extern __device__ const PoseidonStoreState__0_SuperLayout kLayout__837;
|
||||
extern __device__ const PoseidonStoreState__0_SuperLayout8LayoutArray kLayout__829;
|
||||
extern __device__ const PoseidonStoreStateLayout kLayout__828;
|
||||
extern __device__ const Poseidon0StateArm7Layout kLayout__827;
|
||||
extern __device__ const Poseidon0StateLayout kLayout__634;
|
||||
extern __device__ const Poseidon0Layout kLayout__610;
|
||||
extern __device__ const SBoxLayout24LayoutArray kLayout__843;
|
||||
extern __device__ const DoExtRoundLayout kLayout__842;
|
||||
extern __device__ const NondetRegLayout8LayoutArray kLayout__845;
|
||||
extern __device__ const OneHot_8_Layout kLayout__844;
|
||||
extern __device__ const DoExtRoundByIdxLayout kLayout__841;
|
||||
extern __device__ const PoseidonExtRoundLayout kLayout__840;
|
||||
extern __device__ const DoIntRoundLayout kLayout__849;
|
||||
extern __device__ const DoIntRoundLayout kLayout__850;
|
||||
extern __device__ const DoIntRoundLayout kLayout__851;
|
||||
extern __device__ const DoIntRoundLayout kLayout__852;
|
||||
extern __device__ const DoIntRoundLayout kLayout__853;
|
||||
extern __device__ const DoIntRoundLayout kLayout__854;
|
||||
extern __device__ const DoIntRoundLayout kLayout__855;
|
||||
extern __device__ const DoIntRoundLayout kLayout__856;
|
||||
extern __device__ const DoIntRoundLayout kLayout__857;
|
||||
extern __device__ const DoIntRoundLayout kLayout__858;
|
||||
extern __device__ const DoIntRoundLayout kLayout__859;
|
||||
extern __device__ const DoIntRoundLayout kLayout__860;
|
||||
extern __device__ const DoIntRoundLayout kLayout__861;
|
||||
extern __device__ const DoIntRoundLayout kLayout__862;
|
||||
extern __device__ const DoIntRoundLayout kLayout__863;
|
||||
extern __device__ const DoIntRoundLayout kLayout__864;
|
||||
extern __device__ const DoIntRoundLayout kLayout__865;
|
||||
extern __device__ const DoIntRoundLayout kLayout__866;
|
||||
extern __device__ const DoIntRoundLayout kLayout__867;
|
||||
extern __device__ const DoIntRoundLayout kLayout__868;
|
||||
extern __device__ const DoIntRoundLayout kLayout__869;
|
||||
extern __device__ const DoIntRoundLayout21LayoutArray kLayout__848;
|
||||
extern __device__ const DoIntRoundsLayout kLayout__847;
|
||||
extern __device__ const PoseidonIntRoundsLayout kLayout__846;
|
||||
extern __device__ const Poseidon1StateLayout kLayout__839;
|
||||
extern __device__ const Poseidon1Layout kLayout__838;
|
||||
extern __device__ const TopInstResultLayout kLayout__6;
|
||||
extern __device__ const TopLayout kLayout__0;
|
||||
extern __device__ const DigestRegValues_SuperLayout8LayoutArray kLayout__871;
|
||||
extern __device__ const DigestRegLayout kLayout__870;
|
||||
extern __device__ const DigestRegValues_SuperLayout8LayoutArray kLayout__873;
|
||||
extern __device__ const DigestRegLayout kLayout__872;
|
||||
extern __device__ const DigestRegValues_SuperLayout8LayoutArray kLayout__875;
|
||||
extern __device__ const DigestRegLayout kLayout__874;
|
||||
extern __device__ const DigestRegValues_SuperLayout8LayoutArray kLayout__877;
|
||||
extern __device__ const DigestRegLayout kLayout__876;
|
||||
extern __device__ const _accumLayout kLayout__878;
|
||||
extern __device__ const LayoutAccumLayout kLayoutTestSuccRunAccum;
|
||||
extern __device__ const LayoutAccumLayout kLayout_TopAccum;
|
||||
extern __device__ const TestSuccRunLayout kLayoutTestSuccRun;
|
||||
extern __device__ const TopLayout kLayout_Top;
|
||||
extern __device__ const _globalLayout kLayoutGlobal;
|
||||
extern __device__ const _mixLayout kLayoutMix;
|
||||
@@ -0,0 +1,49 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace risc0::circuit::rv32im_v2::cuda {
|
||||
|
||||
struct MemoryTransaction {
|
||||
uint32_t addr;
|
||||
uint32_t cycle;
|
||||
uint32_t word;
|
||||
uint32_t prevCycle;
|
||||
uint32_t prevWord;
|
||||
};
|
||||
|
||||
struct PreflightCycle {
|
||||
uint32_t state;
|
||||
uint32_t pc;
|
||||
uint8_t major;
|
||||
uint8_t minor;
|
||||
uint8_t machineMode;
|
||||
uint8_t padding;
|
||||
uint32_t userCycle;
|
||||
uint32_t txnIdx;
|
||||
uint32_t pagingIdx;
|
||||
uint32_t diffCount;
|
||||
};
|
||||
|
||||
struct PreflightTrace {
|
||||
PreflightCycle* cycles;
|
||||
MemoryTransaction* txns;
|
||||
uint32_t txnsLen;
|
||||
uint32_t tableSplitCycle;
|
||||
};
|
||||
|
||||
} // namespace risc0::circuit::rv32im_v2::cuda
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,632 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "witgen.h"
|
||||
|
||||
namespace risc0::circuit::rv32im_v2::cuda {
|
||||
extern __device__ NondetRegStruct back_NondetReg(ExecContext& ctx,
|
||||
Index distance0,
|
||||
BoundLayout<NondetRegLayout> layout1);
|
||||
extern __device__ NondetRegStruct exec_NondetReg(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<NondetRegLayout> layout1);
|
||||
extern __device__ NondetExtRegStruct back_NondetExtReg(ExecContext& ctx,
|
||||
Index distance0,
|
||||
BoundLayout<NondetExtRegLayout> layout1);
|
||||
extern __device__ NondetExtRegStruct exec_NondetExtReg(ExecContext& ctx,
|
||||
ExtVal arg0,
|
||||
BoundLayout<NondetExtRegLayout> layout1);
|
||||
extern __device__ RegStruct back_Reg(ExecContext& ctx,
|
||||
Index distance0,
|
||||
BoundLayout<NondetRegLayout> layout1);
|
||||
extern __device__ RegStruct exec_Reg(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<NondetRegLayout> layout1);
|
||||
extern __device__ NondetExtRegStruct back_ExtReg(ExecContext& ctx,
|
||||
Index distance0,
|
||||
BoundLayout<NondetExtRegLayout> layout1);
|
||||
extern __device__ NondetExtRegStruct exec_ExtReg(ExecContext& ctx,
|
||||
ExtVal arg0,
|
||||
BoundLayout<NondetExtRegLayout> layout1);
|
||||
extern __device__ NondetRegStruct exec_NondetBitReg(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<NondetRegLayout> layout1);
|
||||
extern __device__ BitRegStruct exec_BitReg(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<NondetRegLayout> layout1);
|
||||
extern __device__ NondetRegStruct exec_NondetTwitReg(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<NondetRegLayout> layout1);
|
||||
extern __device__ NondetFakeTwitRegStruct
|
||||
exec_NondetFakeTwitReg(ExecContext& ctx, Val arg0, BoundLayout<NondetFakeTwitRegLayout> layout1);
|
||||
extern __device__ FakeTwitRegStruct exec_FakeTwitReg(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<NondetFakeTwitRegLayout> layout1);
|
||||
extern __device__ NondetRegStruct exec_IsZero(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<IsZeroLayout> layout1);
|
||||
extern __device__ ArgU8Struct exec_ArgU8(ExecContext& ctx,
|
||||
Val arg0,
|
||||
Val arg1,
|
||||
BoundLayout<ArgU8Layout> layout2);
|
||||
extern __device__ NondetRegStruct exec_NondetU8Reg(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<NondetU8RegLayout> layout1);
|
||||
extern __device__ U8RegStruct exec_U8Reg(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<U8RegLayout> layout1);
|
||||
extern __device__ ArgU16Struct exec_ArgU16(ExecContext& ctx,
|
||||
Val arg0,
|
||||
Val arg1,
|
||||
BoundLayout<ArgU16Layout> layout2);
|
||||
extern __device__ NondetRegStruct exec_NondetU16Reg(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<NondetU16RegLayout> layout1);
|
||||
extern __device__ U16RegStruct exec_U16Reg(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<U16RegLayout> layout1);
|
||||
extern __device__ ToBits_5_Struct exec_ToBits_5_(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<ToBits_5_Layout> layout1);
|
||||
extern __device__ ValU32Struct exec_DynPo2(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<DynPo2Layout> layout1);
|
||||
extern __device__ NormalizeU32Struct exec_NormalizeU32(ExecContext& ctx,
|
||||
DenormedValU32Struct arg0,
|
||||
BoundLayout<NormalizeU32Layout> layout1);
|
||||
extern __device__ AddrDecomposeStruct exec_AddrDecompose(ExecContext& ctx,
|
||||
ValU32Struct arg0,
|
||||
Val arg1,
|
||||
BoundLayout<AddrDecomposeLayout> layout2);
|
||||
extern __device__ AddrDecomposeBitsStruct exec_AddrDecomposeBits(
|
||||
ExecContext& ctx, ValU32Struct arg0, Val arg1, BoundLayout<AddrDecomposeBitsLayout> layout2);
|
||||
extern __device__ CmpEqualStruct exec_CmpEqual(ExecContext& ctx,
|
||||
ValU32Struct arg0,
|
||||
ValU32Struct arg1,
|
||||
BoundLayout<CmpEqualLayout> layout2);
|
||||
extern __device__ CmpLessThanUnsignedStruct
|
||||
exec_CmpLessThanUnsigned(ExecContext& ctx,
|
||||
ValU32Struct arg0,
|
||||
ValU32Struct arg1,
|
||||
BoundLayout<CmpLessThanUnsignedLayout> layout2);
|
||||
extern __device__ NondetRegStruct exec_GetSignU32(ExecContext& ctx,
|
||||
ValU32Struct arg0,
|
||||
BoundLayout<GetSignU32Layout> layout1);
|
||||
extern __device__ CmpLessThanStruct exec_CmpLessThan(ExecContext& ctx,
|
||||
ValU32Struct arg0,
|
||||
ValU32Struct arg1,
|
||||
BoundLayout<CmpLessThanLayout> layout2);
|
||||
extern __device__ ToBits_16_Struct exec_ToBits_16_(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<ToBits_16_Layout> layout1);
|
||||
extern __device__ FromBits_16_Struct exec_BitwiseAndU16(ExecContext& ctx,
|
||||
Val arg0,
|
||||
Val arg1,
|
||||
BoundLayout<BitwiseAndU16Layout> layout2);
|
||||
extern __device__ ValU32Struct exec_BitwiseAnd(ExecContext& ctx,
|
||||
ValU32Struct arg0,
|
||||
ValU32Struct arg1,
|
||||
BoundLayout<BitwiseAndLayout> layout2);
|
||||
extern __device__ ValU32Struct exec_BitwiseOr(ExecContext& ctx,
|
||||
ValU32Struct arg0,
|
||||
ValU32Struct arg1,
|
||||
BoundLayout<BitwiseOrLayout> layout2);
|
||||
extern __device__ ValU32Struct exec_BitwiseXor(ExecContext& ctx,
|
||||
ValU32Struct arg0,
|
||||
ValU32Struct arg1,
|
||||
BoundLayout<BitwiseXorLayout> layout2);
|
||||
extern __device__ DecoderStruct exec_Decoder(ExecContext& ctx,
|
||||
ValU32Struct arg0,
|
||||
BoundLayout<DecoderLayout> layout1);
|
||||
extern __device__ MemoryArgStruct exec_MemoryArg(ExecContext& ctx,
|
||||
Val arg0,
|
||||
Val arg1,
|
||||
Val arg2,
|
||||
ValU32Struct arg3,
|
||||
BoundLayout<MemoryArgLayout> layout4);
|
||||
extern __device__ CycleArgStruct exec_CycleArg(ExecContext& ctx,
|
||||
Val arg0,
|
||||
Val arg1,
|
||||
BoundLayout<CycleArgLayout> layout2);
|
||||
extern __device__ IsCycleStruct exec_IsCycle(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<IsCycleLayout> layout1);
|
||||
extern __device__ MemoryIOStruct exec_MemoryIO(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
Val arg1,
|
||||
BoundLayout<MemoryIOLayout> layout2);
|
||||
extern __device__ IsForwardStruct exec_IsForward(ExecContext& ctx,
|
||||
MemoryIOStruct arg0,
|
||||
BoundLayout<IsForwardLayout> layout1);
|
||||
extern __device__ GetDataStruct exec_MemoryRead(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
Val arg1,
|
||||
BoundLayout<MemoryReadLayout> layout2);
|
||||
extern __device__ MemoryWriteStruct exec_MemoryWrite(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
Val arg1,
|
||||
ValU32Struct arg2,
|
||||
BoundLayout<MemoryWriteLayout> layout3);
|
||||
extern __device__ MemoryWriteUnconstrainedStruct
|
||||
exec_MemoryWriteUnconstrained(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
Val arg1,
|
||||
BoundLayout<MemoryWriteUnconstrainedLayout> layout2);
|
||||
extern __device__ GetDataStruct exec_MemoryPageIn(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
Val arg1,
|
||||
BoundLayout<MemoryPageInLayout> layout2);
|
||||
extern __device__ GetDataStruct exec_MemoryPageOut(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
Val arg1,
|
||||
BoundLayout<MemoryPageOutLayout> layout2);
|
||||
extern __device__ OneHot_3_Struct exec_OneHot_3_(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<OneHot_3_Layout> layout1);
|
||||
extern __device__ GetDataStruct exec_MemoryGet(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
Val arg1,
|
||||
OneHot_3_Struct arg2,
|
||||
BoundLayout<MemoryGetLayout> layout3);
|
||||
extern __device__ OneHot_8_Struct exec_OneHot_8_(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<OneHot_8_Layout> layout1);
|
||||
extern __device__ InstInputStruct exec_InstInput(ExecContext& ctx,
|
||||
Val arg0,
|
||||
Val arg1,
|
||||
Val arg2,
|
||||
ValU32Struct arg3,
|
||||
Val arg4,
|
||||
Val arg5,
|
||||
BoundLayout<InstInputLayout> layout6);
|
||||
extern __device__ DecoderStruct exec_DecodeInst(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<DecodeInstLayout> layout2);
|
||||
extern __device__ GetDataStruct exec_ReadReg(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
Val arg2,
|
||||
BoundLayout<ReadRegLayout> layout3);
|
||||
extern __device__ WriteRdStruct exec_WriteRd(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
DecoderStruct arg2,
|
||||
Val arg3,
|
||||
ValU32Struct arg4,
|
||||
BoundLayout<WriteRdLayout> layout5);
|
||||
extern __device__ ExpandU32Struct exec_ExpandU32(ExecContext& ctx,
|
||||
ValU32Struct arg0,
|
||||
Val arg1,
|
||||
BoundLayout<ExpandU32Layout> layout2);
|
||||
extern __device__ SplitTotalStruct exec_SplitTotal(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<SplitTotalLayout> layout1);
|
||||
extern __device__ MultiplyAccumulateStruct
|
||||
exec_MultiplyAccumulate(ExecContext& ctx,
|
||||
ValU32Struct arg0,
|
||||
ValU32Struct arg1,
|
||||
ValU32Struct arg2,
|
||||
MultiplySettingsStruct arg3,
|
||||
BoundLayout<MultiplyAccumulateLayout> layout4);
|
||||
extern __device__ DivInputStruct exec_DivInput(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<DivInputLayout> layout2);
|
||||
extern __device__ DivideReturnStruct exec_DoDiv(ExecContext& ctx,
|
||||
ValU32Struct arg0,
|
||||
ValU32Struct arg1,
|
||||
Val arg2,
|
||||
Val arg3,
|
||||
BoundLayout<DoDivLayout> layout4);
|
||||
extern __device__ ValU32Struct exec_OpSRL(ExecContext& ctx,
|
||||
DivInputStruct arg0,
|
||||
BoundLayout<OpSRLLayout> layout1);
|
||||
extern __device__ NondetRegStruct exec_TopBit(ExecContext& ctx,
|
||||
ValU32Struct arg0,
|
||||
BoundLayout<TopBitLayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpSRA(ExecContext& ctx,
|
||||
DivInputStruct arg0,
|
||||
BoundLayout<OpSRALayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpSRLI(ExecContext& ctx,
|
||||
DivInputStruct arg0,
|
||||
BoundLayout<OpSRLILayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpSRAI(ExecContext& ctx,
|
||||
DivInputStruct arg0,
|
||||
BoundLayout<OpSRAILayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpDIV(ExecContext& ctx,
|
||||
DivInputStruct arg0,
|
||||
BoundLayout<OpDIVLayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpDIVU(ExecContext& ctx,
|
||||
DivInputStruct arg0,
|
||||
BoundLayout<OpDIVULayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpREM(ExecContext& ctx,
|
||||
DivInputStruct arg0,
|
||||
BoundLayout<OpREMLayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpREMU(ExecContext& ctx,
|
||||
DivInputStruct arg0,
|
||||
BoundLayout<OpREMULayout> layout1);
|
||||
extern __device__ InstOutputStruct exec_Div0(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<Div0Layout> layout2);
|
||||
extern __device__ MiscInputStruct exec_MiscInput(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<MiscInputLayout> layout2);
|
||||
extern __device__ InstOutputStruct exec_FinalizeMisc(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
MiscInputStruct arg1,
|
||||
MiscOutputStruct arg2,
|
||||
BoundLayout<FinalizeMiscLayout> layout3);
|
||||
extern __device__ MiscOutputStruct exec_OpXOR(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpXORLayout> layout1);
|
||||
extern __device__ MiscOutputStruct exec_OpOR(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpORLayout> layout1);
|
||||
extern __device__ MiscOutputStruct exec_OpAND(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpANDLayout> layout1);
|
||||
extern __device__ MiscOutputStruct exec_OpSLT(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpSLTLayout> layout1);
|
||||
extern __device__ MiscOutputStruct exec_OpSLTU(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpSLTULayout> layout1);
|
||||
extern __device__ InstOutputStruct exec_Misc0(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<Misc0Layout> layout2);
|
||||
extern __device__ MiscOutputStruct exec_OpXORI(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpXORILayout> layout1);
|
||||
extern __device__ MiscOutputStruct exec_OpORI(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpORILayout> layout1);
|
||||
extern __device__ MiscOutputStruct exec_OpANDI(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpANDILayout> layout1);
|
||||
extern __device__ MiscOutputStruct exec_OpSLTI(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpSLTILayout> layout1);
|
||||
extern __device__ MiscOutputStruct exec_OpSLTIU(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpSLTIULayout> layout1);
|
||||
extern __device__ MiscOutputStruct exec_OpBEQ(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpBEQLayout> layout1);
|
||||
extern __device__ MiscOutputStruct exec_OpBNE(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpBNELayout> layout1);
|
||||
extern __device__ MiscOutputStruct exec_OpBLT(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpBLTLayout> layout1);
|
||||
extern __device__ InstOutputStruct exec_Misc1(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<Misc1Layout> layout2);
|
||||
extern __device__ MiscOutputStruct exec_OpBGE(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpBGELayout> layout1);
|
||||
extern __device__ MiscOutputStruct exec_OpBLTU(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpBLTULayout> layout1);
|
||||
extern __device__ MiscOutputStruct exec_OpBGEU(ExecContext& ctx,
|
||||
MiscInputStruct arg0,
|
||||
BoundLayout<OpBGEULayout> layout1);
|
||||
extern __device__ InstOutputStruct exec_Misc2(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<Misc2Layout> layout2);
|
||||
extern __device__ MulInputStruct exec_MulInput(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<MulInputLayout> layout2);
|
||||
extern __device__ DoMulStruct exec_DoMul(ExecContext& ctx,
|
||||
ValU32Struct arg0,
|
||||
ValU32Struct arg1,
|
||||
Val arg2,
|
||||
Val arg3,
|
||||
BoundLayout<DoMulLayout> layout4);
|
||||
extern __device__ ValU32Struct exec_OpSLL(ExecContext& ctx,
|
||||
MulInputStruct arg0,
|
||||
BoundLayout<OpSLLLayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpSLLI(ExecContext& ctx,
|
||||
MulInputStruct arg0,
|
||||
BoundLayout<OpSLLILayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpMUL(ExecContext& ctx,
|
||||
MulInputStruct arg0,
|
||||
BoundLayout<OpMULLayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpMULH(ExecContext& ctx,
|
||||
MulInputStruct arg0,
|
||||
BoundLayout<OpMULHLayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpMULHSU(ExecContext& ctx,
|
||||
MulInputStruct arg0,
|
||||
BoundLayout<OpMULHSULayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpMULHU(ExecContext& ctx,
|
||||
MulInputStruct arg0,
|
||||
BoundLayout<OpMULHULayout> layout1);
|
||||
extern __device__ InstOutputStruct exec_Mul0(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<Mul0Layout> layout2);
|
||||
extern __device__ MemLoadInputStruct exec_MemLoadInput(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<MemLoadInputLayout> layout2);
|
||||
extern __device__ MemStoreInputStruct exec_MemStoreInput(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<MemStoreInputLayout> layout2);
|
||||
extern __device__ MemStoreFinalizeStruct
|
||||
exec_MemStoreFinalize(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
MemStoreInputStruct arg1,
|
||||
ValU32Struct arg2,
|
||||
BoundLayout<MemStoreFinalizeLayout> layout3);
|
||||
extern __device__ SplitWordStruct exec_SplitWord(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<SplitWordLayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpLB(ExecContext& ctx,
|
||||
MemLoadInputStruct arg0,
|
||||
BoundLayout<OpLBLayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpLH(ExecContext& ctx,
|
||||
MemLoadInputStruct arg0,
|
||||
BoundLayout<OpLHLayout> layout1);
|
||||
extern __device__ ValU32Struct exec_OpLBU(ExecContext& ctx,
|
||||
MemLoadInputStruct arg0,
|
||||
BoundLayout<OpLBULayout> layout1);
|
||||
extern __device__ InstOutputStruct exec_Mem0(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<Mem0Layout> layout2);
|
||||
extern __device__ ValU32Struct exec_OpSB(ExecContext& ctx,
|
||||
MemStoreInputStruct arg0,
|
||||
BoundLayout<OpSBLayout> layout1);
|
||||
extern __device__ InstOutputStruct exec_Mem1(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<Mem1Layout> layout2);
|
||||
extern __device__ DigestRegStruct back_DigestReg(ExecContext& ctx,
|
||||
Index distance0,
|
||||
BoundLayout<DigestRegLayout> layout1);
|
||||
extern __device__ DigestRegStruct exec_DigestReg(ExecContext& ctx,
|
||||
ValU32Struct8Array arg0,
|
||||
BoundLayout<DigestRegLayout> layout1);
|
||||
extern __device__ InstOutputStruct exec_ControlLoadRoot(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<ControlLoadRootLayout> layout2,
|
||||
GlobalBuf global3);
|
||||
extern __device__ InstOutputStruct exec_ControlResume(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<ControlResumeLayout> layout2,
|
||||
GlobalBuf global3);
|
||||
extern __device__ InstOutputStruct
|
||||
exec_ControlUserECALL(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<ControlUserECALLLayout> layout2);
|
||||
extern __device__ InstOutputStruct exec_ControlMRET(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<ControlMRETLayout> layout2);
|
||||
extern __device__ InstOutputStruct exec_ControlSuspend(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<ControlSuspendLayout> layout2,
|
||||
GlobalBuf global3);
|
||||
extern __device__ InstOutputStruct
|
||||
exec_ControlStoreRoot(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<ControlStoreRootLayout> layout2,
|
||||
GlobalBuf global3);
|
||||
extern __device__ InstOutputStruct exec_ControlTable(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<ControlTableLayout> layout2);
|
||||
extern __device__ InstOutputStruct exec_Control0(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<Control0Layout> layout2,
|
||||
GlobalBuf global3);
|
||||
extern __device__ OneHot_4_Struct exec_OneHot_4_(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<OneHot_4_Layout> layout1);
|
||||
extern __device__ ECallOutputStruct exec_MachineECall(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
Val arg2,
|
||||
BoundLayout<MachineECallLayout> layout3);
|
||||
extern __device__ ECallOutputStruct exec_ECallTerminate(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<ECallTerminateLayout> layout2,
|
||||
GlobalBuf global3);
|
||||
extern __device__ DecomposeLow2Struct exec_DecomposeLow2(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<DecomposeLow2Layout> layout1);
|
||||
extern __device__ ECallOutputStruct
|
||||
exec_ECallHostReadSetup(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<ECallHostReadSetupLayout> layout2);
|
||||
extern __device__ ECallOutputStruct exec_ECallHostWrite(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<ECallHostWriteLayout> layout2);
|
||||
extern __device__ ECallOutputStruct
|
||||
exec_ECallHostReadWords(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
Val arg2,
|
||||
Val arg3,
|
||||
BoundLayout<ECallHostReadWordsLayout> layout4);
|
||||
extern __device__ InstOutputStruct exec_ECall0(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<ECall0Layout> layout2,
|
||||
GlobalBuf global3);
|
||||
extern __device__ RegStruct exec_SBox(ExecContext& ctx, Val arg0, BoundLayout<SBoxLayout> layout1);
|
||||
extern __device__ MultiplyByMIntStruct exec_DoIntRound(ExecContext& ctx,
|
||||
Val24Array arg0,
|
||||
Val arg1,
|
||||
BoundLayout<DoIntRoundLayout> layout2);
|
||||
extern __device__ DoIntRoundsStruct exec_DoIntRounds(ExecContext& ctx,
|
||||
Val24Array arg0,
|
||||
BoundLayout<DoIntRoundsLayout> layout1);
|
||||
extern __device__ MultiplyByMExtStruct exec_DoExtRound(ExecContext& ctx,
|
||||
Val24Array arg0,
|
||||
Val24Array arg1,
|
||||
BoundLayout<DoExtRoundLayout> layout2);
|
||||
extern __device__ MultiplyByMExtStruct exec_DoExtRoundByIdx(
|
||||
ExecContext& ctx, Val24Array arg0, Val arg1, BoundLayout<DoExtRoundByIdxLayout> layout2);
|
||||
extern __device__ PoseidonStateStruct back_PoseidonState(ExecContext& ctx,
|
||||
Index distance0,
|
||||
BoundLayout<PoseidonStateLayout> layout1);
|
||||
extern __device__ PoseidonStateStruct exec_PoseidonState(ExecContext& ctx,
|
||||
PoseidonOpDefStruct arg0,
|
||||
Val arg1,
|
||||
Val arg2,
|
||||
Val arg3,
|
||||
Val arg4,
|
||||
Val arg5,
|
||||
Val24Array arg6,
|
||||
ExtVal arg7,
|
||||
BoundLayout<PoseidonStateLayout> layout8);
|
||||
extern __device__ PoseidonStateStruct
|
||||
exec_PoseidonInvalid(ExecContext& ctx, BoundLayout<PoseidonStateLayout> layout0);
|
||||
extern __device__ ReadAddrStruct exec_ReadAddr(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
Val arg1,
|
||||
BoundLayout<ReadAddrLayout> layout2);
|
||||
extern __device__ PoseidonStateStruct exec_PoseidonEcall(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
Val arg1,
|
||||
BoundLayout<PoseidonEcallLayout> layout2);
|
||||
extern __device__ PoseidonStateStruct exec_PoseidonPagingEntry(
|
||||
ExecContext& ctx, RegStruct arg0, Val arg1, BoundLayout<PoseidonStateLayout> layout2);
|
||||
extern __device__ PoseidonStateStruct exec_PoseidonEntry(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
ValU32Struct arg1,
|
||||
Val arg2,
|
||||
BoundLayout<PoseidonEntryLayout> layout3);
|
||||
extern __device__ ReadElemStruct exec_ReadElem(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
Val arg1,
|
||||
BoundLayout<ReadElemLayout> layout2);
|
||||
extern __device__ PoseidonStateStruct
|
||||
exec_PoseidonLoadState(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
PoseidonStateStruct arg1,
|
||||
BoundLayout<PoseidonLoadStateLayout> layout2);
|
||||
extern __device__ PoseidonStateStruct
|
||||
exec_PoseidonLoadInShort(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
PoseidonStateStruct arg1,
|
||||
BoundLayout<PoseidonLoadInShortLayout> layout2,
|
||||
GlobalBuf global3);
|
||||
extern __device__ PoseidonStateStruct
|
||||
exec_PoseidonLoadInLow(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
PoseidonStateStruct arg1,
|
||||
BoundLayout<PoseidonLoadInLowLayout> layout2,
|
||||
GlobalBuf global3);
|
||||
extern __device__ PoseidonStateStruct
|
||||
exec_PoseidonLoadInHigh(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
PoseidonStateStruct arg1,
|
||||
BoundLayout<PoseidonLoadInHighLayout> layout2,
|
||||
GlobalBuf global3);
|
||||
extern __device__ PoseidonStateStruct exec_PoseidonLoadIn(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
PoseidonStateStruct arg1,
|
||||
BoundLayout<PoseidonLoadInLayout> layout2,
|
||||
GlobalBuf global3);
|
||||
extern __device__ PoseidonStateStruct exec_PoseidonExtRound(
|
||||
ExecContext& ctx, PoseidonStateStruct arg0, BoundLayout<PoseidonExtRoundLayout> layout1);
|
||||
extern __device__ PoseidonStateStruct exec_PoseidonIntRounds(
|
||||
ExecContext& ctx, PoseidonStateStruct arg0, BoundLayout<PoseidonIntRoundsLayout> layout1);
|
||||
extern __device__ PoseidonStateStruct
|
||||
exec_PoseidonCheckOut(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
PoseidonStateStruct arg1,
|
||||
BoundLayout<PoseidonCheckOutLayout> layout2);
|
||||
extern __device__ PoseidonStateStruct
|
||||
exec_PoseidonStoreOut(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
PoseidonStateStruct arg1,
|
||||
BoundLayout<PoseidonStoreOutLayout> layout2);
|
||||
extern __device__ PoseidonStateStruct exec_PoseidonDoOut(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
PoseidonStateStruct arg1,
|
||||
BoundLayout<PoseidonDoOutLayout> layout2);
|
||||
extern __device__ PoseidonStateStruct
|
||||
exec_PoseidonStoreState(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
PoseidonStateStruct arg1,
|
||||
BoundLayout<PoseidonStoreStateLayout> layout2);
|
||||
extern __device__ IsU24Struct exec_IsU24(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<IsU24Layout> layout1);
|
||||
extern __device__ PoseidonStateStruct exec_PoseidonPagingLoadNode(
|
||||
ExecContext& ctx, RegStruct arg0, Val arg1, BoundLayout<PoseidonStateLayout> layout2);
|
||||
extern __device__ PoseidonStateStruct exec_PoseidonPagingLoadPage(
|
||||
ExecContext& ctx, RegStruct arg0, Val arg1, BoundLayout<PoseidonStateLayout> layout2);
|
||||
extern __device__ PoseidonStateStruct
|
||||
exec_PoseidonPagingLoadDone(ExecContext& ctx, BoundLayout<PoseidonStateLayout> layout0);
|
||||
extern __device__ PoseidonStateStruct exec_PoseidonPagingStoreNode(
|
||||
ExecContext& ctx, RegStruct arg0, Val arg1, BoundLayout<PoseidonStateLayout> layout2);
|
||||
extern __device__ PoseidonStateStruct exec_PoseidonPagingStorePage(
|
||||
ExecContext& ctx, RegStruct arg0, Val arg1, BoundLayout<PoseidonStateLayout> layout2);
|
||||
extern __device__ PoseidonStateStruct
|
||||
exec_PoseidonPagingStoreDone(ExecContext& ctx, BoundLayout<PoseidonStateLayout> layout0);
|
||||
extern __device__ OneHot_6_Struct exec_OneHot_6_(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<OneHot_6_Layout> layout1);
|
||||
extern __device__ PoseidonStateStruct
|
||||
exec_PoseidonPaging(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
Val arg1,
|
||||
PoseidonStateStruct arg2,
|
||||
BoundLayout<PoseidonPagingLayout> layout3);
|
||||
extern __device__ InstOutputStruct exec_Poseidon0(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<Poseidon0Layout> layout2,
|
||||
GlobalBuf global3);
|
||||
extern __device__ InstOutputStruct exec_Poseidon1(ExecContext& ctx,
|
||||
RegStruct arg0,
|
||||
InstInputStruct arg1,
|
||||
BoundLayout<Poseidon1Layout> layout2);
|
||||
extern __device__ OneHot_11_Struct exec_OneHot_11_(ExecContext& ctx,
|
||||
Val arg0,
|
||||
BoundLayout<OneHot_11_Layout> layout1);
|
||||
extern __device__ TopStruct exec_Top(ExecContext& ctx,
|
||||
BoundLayout<TopLayout> layout0,
|
||||
GlobalBuf global1);
|
||||
extern __device__ void step_Top(ExecContext& ctx, MutableBuf data0, GlobalBuf global1);
|
||||
extern __device__ ComponentStruct exec_TopAccum(ExecContext& ctx,
|
||||
BoundLayout<TopLayout> arg0,
|
||||
BoundLayout<LayoutAccumLayout> layout1,
|
||||
GlobalBuf mix2);
|
||||
extern __device__ void
|
||||
step_TopAccum(ExecContext& ctx, MutableBuf accum0, MutableBuf data1, GlobalBuf mix2);
|
||||
|
||||
} // namespace risc0::circuit::rv32im_v2::cuda
|
||||
@@ -0,0 +1,66 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "fp.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cuda/atomic>
|
||||
|
||||
namespace risc0::circuit::rv32im_v2::cuda {
|
||||
|
||||
struct LookupTables {
|
||||
::cuda::atomic<uint32_t>* tableU8;
|
||||
::cuda::atomic<uint32_t>* tableU16;
|
||||
|
||||
__device__ void lookupDelta(Fp table, Fp index, Fp count) {
|
||||
uint32_t tableU32 = table.asUInt32();
|
||||
uint32_t indexU32 = index.asUInt32();
|
||||
if (tableU32 == 0) {
|
||||
// tableCycle[index] += count;
|
||||
return;
|
||||
}
|
||||
if (tableU32 != 8 && tableU32 != 16) {
|
||||
assert(false && "Invalid lookup table");
|
||||
}
|
||||
if (indexU32 >= (1 << tableU32)) {
|
||||
printf("LOOKUP ERROR: table = %u, index = %u\n", tableU32, indexU32);
|
||||
assert(false && "u8/16 table error");
|
||||
}
|
||||
// printf("table = %u, index = %u\n", tableU32, indexU32);
|
||||
if (tableU32 == 8) {
|
||||
tableU8[indexU32]++;
|
||||
} else {
|
||||
tableU16[indexU32]++;
|
||||
}
|
||||
}
|
||||
|
||||
__device__ Fp lookupCurrent(Fp table, Fp index) {
|
||||
uint32_t tableU32 = table.asUInt32();
|
||||
if (tableU32 != 8 && tableU32 != 16) {
|
||||
assert(false && "Invalid lookup table");
|
||||
}
|
||||
uint32_t indexU32 = index.asUInt32();
|
||||
if (tableU32 == 8) {
|
||||
return Fp(tableU8[indexU32]);
|
||||
} else {
|
||||
return Fp(tableU16[indexU32]);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace risc0::circuit::rv32im_v2::cuda
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,280 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "buffers.h"
|
||||
#include "fp.h"
|
||||
#include "fpext.h"
|
||||
#include "preflight.h"
|
||||
#include "tables.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <cuda/std/array>
|
||||
|
||||
namespace risc0::circuit::rv32im_v2::cuda {
|
||||
|
||||
#if defined(__clang__)
|
||||
#pragma clang diagnostic ignored "-Wunused-parameter"
|
||||
#pragma clang diagnostic ignored "-Wunused-variable"
|
||||
#elif defined(__GNUC__)
|
||||
#pragma GCC diagnostic ignored "-Wunused-parameter"
|
||||
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
|
||||
#endif
|
||||
|
||||
using Val = Fp;
|
||||
using ExtVal = FpExt;
|
||||
|
||||
struct ExecContext {
|
||||
__device__ ExecContext(PreflightTrace& preflight, LookupTables& tables, size_t cycle)
|
||||
: preflight(preflight), tables(tables), cycle(cycle) {}
|
||||
PreflightTrace& preflight;
|
||||
LookupTables& tables;
|
||||
size_t cycle;
|
||||
};
|
||||
|
||||
struct BufferObj {
|
||||
__device__ virtual Val load(ExecContext& ctx, size_t col, size_t back) = 0;
|
||||
__device__ virtual void store(ExecContext& ctx, size_t col, Val val) = 0;
|
||||
};
|
||||
|
||||
struct MutableBufObj : public BufferObj {
|
||||
__device__ MutableBufObj(Buffer& buf) : buf(buf) {}
|
||||
|
||||
__device__ Val load(ExecContext& ctx, size_t col, size_t back) override {
|
||||
if (back > ctx.cycle) {
|
||||
return 0;
|
||||
}
|
||||
return buf.get(ctx.cycle - back, col);
|
||||
}
|
||||
|
||||
__device__ void store(ExecContext& ctx, size_t col, Val val) override {
|
||||
return buf.set(ctx.cycle, col, val);
|
||||
}
|
||||
|
||||
Buffer& buf;
|
||||
};
|
||||
|
||||
using MutableBuf = MutableBufObj*;
|
||||
|
||||
struct GlobalBufObj : public BufferObj {
|
||||
__device__ GlobalBufObj(Buffer& buf) : buf(buf) {}
|
||||
|
||||
__device__ Val load(ExecContext& ctx, size_t col, size_t back) override {
|
||||
assert(back == 0);
|
||||
return buf.get(0, col);
|
||||
}
|
||||
|
||||
__device__ void store(ExecContext& ctx, size_t col, Val val) override {
|
||||
return buf.set(0, col, val);
|
||||
}
|
||||
|
||||
Buffer& buf;
|
||||
};
|
||||
|
||||
using GlobalBuf = GlobalBufObj*;
|
||||
|
||||
template <typename T> struct BoundLayout {
|
||||
__device__ BoundLayout(const T& layout, BufferObj* buf) : layout(layout), buf(buf) {}
|
||||
|
||||
const T& layout;
|
||||
BufferObj* buf = nullptr;
|
||||
};
|
||||
|
||||
__device__ inline size_t to_size_t(Val v) {
|
||||
return v.asUInt32();
|
||||
}
|
||||
|
||||
__device__ inline Val mod(Val a, Val b) {
|
||||
return Val(a.asUInt32() % b.asUInt32());
|
||||
}
|
||||
|
||||
constexpr size_t EXT_SIZE = 4;
|
||||
|
||||
// Built in field operations
|
||||
__device__ inline Val isz(Val x) {
|
||||
return Val(x == Val(0));
|
||||
}
|
||||
|
||||
__device__ inline Val neg_0(Val x) {
|
||||
return -x;
|
||||
}
|
||||
|
||||
__device__ inline Val inv_0(Val x) {
|
||||
return inv(x);
|
||||
}
|
||||
|
||||
__device__ inline ExtVal inv_0(ExtVal x) {
|
||||
return inv(x);
|
||||
}
|
||||
|
||||
__device__ inline Val bitAnd(Val a, Val b) {
|
||||
return Val(a.asUInt32() & b.asUInt32());
|
||||
}
|
||||
|
||||
__device__ inline Val inRange(Val low, Val mid, Val high) {
|
||||
assert(low <= high);
|
||||
return Val(low <= mid && mid < high);
|
||||
}
|
||||
|
||||
__device__ inline void eqz(Val a, const char* loc) {
|
||||
if (a.asUInt32()) {
|
||||
printf("eqz failure at: %s\n", loc);
|
||||
assert(false && "eqz failure");
|
||||
}
|
||||
}
|
||||
|
||||
__device__ inline void eqz(ExtVal a, const char* loc) {
|
||||
for (size_t i = 0; i < EXT_SIZE; i++) {
|
||||
eqz(a.elems[i], loc);
|
||||
}
|
||||
}
|
||||
|
||||
// Define index type (used in back)
|
||||
using Index = size_t;
|
||||
|
||||
struct Reg {
|
||||
__device__ constexpr Reg(size_t col) : col(col) {}
|
||||
size_t col;
|
||||
};
|
||||
|
||||
#define BIND_LAYOUT(orig, buf) BoundLayout(orig, buf)
|
||||
#define LAYOUT_LOOKUP(orig, elem) BoundLayout(orig.layout.elem, orig.buf)
|
||||
#define LAYOUT_SUBSCRIPT(orig, index) BoundLayout(orig.layout[index], orig.buf)
|
||||
#define EQZ(val, loc) eqz(val, loc)
|
||||
|
||||
__device__ inline void store(ExecContext& ctx, BoundLayout<Reg> reg, Val val) {
|
||||
reg.buf->store(ctx, reg.layout.col, val);
|
||||
}
|
||||
|
||||
__device__ inline void set(ExecContext& ctx, BufferObj* buf, size_t offset, Val val) {
|
||||
static_cast<MutableBufObj*>(buf)->store(ctx, offset, val);
|
||||
}
|
||||
|
||||
__device__ inline void setGlobal(ExecContext& ctx, BufferObj* buf, size_t offset, Val val) {
|
||||
static_cast<GlobalBufObj*>(buf)->store(ctx, offset, val);
|
||||
}
|
||||
|
||||
__device__ inline void storeExt(ExecContext& ctx, BoundLayout<Reg> reg, ExtVal val) {
|
||||
for (size_t i = 0; i < EXT_SIZE; i++) {
|
||||
reg.buf->store(ctx, reg.layout.col + i, val.elems[i]);
|
||||
}
|
||||
}
|
||||
|
||||
__device__ inline Val load(ExecContext& ctx, BoundLayout<Reg> reg, size_t back) {
|
||||
return reg.buf->load(ctx, reg.layout.col, back);
|
||||
}
|
||||
|
||||
__device__ inline ExtVal loadExt(ExecContext& ctx, BoundLayout<Reg> reg, size_t back) {
|
||||
::cuda::std::array<Fp, EXT_SIZE> elems;
|
||||
for (size_t i = 0; i < EXT_SIZE; i++) {
|
||||
elems[i] = reg.buf->load(ctx, reg.layout.col + i, back);
|
||||
}
|
||||
return FpExt(elems[0], elems[1], elems[2], elems[3]);
|
||||
}
|
||||
|
||||
__device__ inline Val get(ExecContext& ctx, BufferObj* buf, size_t offset, size_t back) {
|
||||
return static_cast<MutableBufObj*>(buf)->load(ctx, offset, back);
|
||||
}
|
||||
|
||||
__device__ inline Val getGlobal(ExecContext& ctx, BufferObj* buf, size_t offset) {
|
||||
return static_cast<GlobalBufObj*>(buf)->load(ctx, offset, 0);
|
||||
}
|
||||
|
||||
#define LOAD(reg, back) load(ctx, reg, back)
|
||||
#define LOAD_EXT(reg, back) loadExt(ctx, reg, back)
|
||||
#define STORE(reg, val) store(ctx, reg, val)
|
||||
#define STORE_EXT(reg, val) storeExt(ctx, reg, val)
|
||||
|
||||
// Map + reduce support
|
||||
template <typename T1, typename F, size_t N>
|
||||
__device__ inline auto map(::cuda::std::array<T1, N> a, F f) {
|
||||
::cuda::std::array<decltype(f(a[0])), N> out;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
out[i] = f(a[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T1, typename T2, typename F, size_t N>
|
||||
__device__ inline auto map(::cuda::std::array<T1, N> a, ::cuda::std::array<T2, N> b, F f) {
|
||||
::cuda::std::array<decltype(f(a[0], b[0])), N> out;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
out[i] = f(a[i], b[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T1, typename T2, typename F, size_t N>
|
||||
__device__ inline auto map(::cuda::std::array<T1, N> a, const BoundLayout<T2>& b, F f) {
|
||||
::cuda::std::array<decltype(f(a[0], BoundLayout(b.layout[0], b.buf))), N> out;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
out[i] = f(a[i], BoundLayout(b.layout[i], b.buf));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T1, typename T2, typename F, size_t N>
|
||||
__device__ inline auto reduce(::cuda::std::array<T1, N> elems, T2 start, F f) {
|
||||
T2 cur = start;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
cur = f(cur, elems[i]);
|
||||
}
|
||||
return cur;
|
||||
}
|
||||
|
||||
template <typename T1, typename T2, typename T3, typename F, size_t N>
|
||||
__device__ inline auto
|
||||
reduce(::cuda::std::array<T1, N> elems, T2 start, const BoundLayout<T3>& b, F f) {
|
||||
T2 cur = start;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
cur = f(cur, elems[i], BoundLayout(b.layout[i], b.buf));
|
||||
}
|
||||
return cur;
|
||||
}
|
||||
|
||||
// All the extern handling
|
||||
#define INVOKE_EXTERN(ctx, name, ...) extern_##name(ctx, ##__VA_ARGS__)
|
||||
|
||||
__device__ ::cuda::std::array<Val, 5> extern_getMemoryTxn(ExecContext& ctx, Val addrElem);
|
||||
__device__ void extern_lookupDelta(ExecContext& ctx, Val table, Val index, Val count);
|
||||
__device__ Val extern_lookupCurrent(ExecContext& ctx, Val table, Val index);
|
||||
__device__ void
|
||||
extern_memoryDelta(ExecContext& ctx, Val addr, Val cycle, Val dataLow, Val dataHigh, Val count);
|
||||
__device__ uint32_t extern_getDiffCount(ExecContext& ctx, Val cycle);
|
||||
__device__ Val extern_isFirstCycle_0(ExecContext& ctx);
|
||||
__device__ Val extern_getCycle(ExecContext& ctx);
|
||||
__device__ ::cuda::std::array<Val, 4> extern_divide(
|
||||
ExecContext& ctx, Val numerLow, Val numerHigh, Val denomLow, Val denomHigh, Val signType);
|
||||
__device__ void extern_print(ExecContext& ctx, Val v);
|
||||
__device__ ::cuda::std::array<Val, 2> extern_getMajorMinor(ExecContext& ctx);
|
||||
__device__ Val extern_hostReadPrepare(ExecContext& ctx, Val fp, Val len);
|
||||
__device__ Val extern_hostWrite(ExecContext& ctx, Val fdVal, Val addrLow, Val addrHigh, Val lenVal);
|
||||
__device__ ::cuda::std::array<Val, 2> extern_nextPagingIdx(ExecContext& ctx);
|
||||
|
||||
template <typename T> __device__ void extern_log(ExecContext& ctx, const char* message, T vals) {
|
||||
// printf("%s\n", message);
|
||||
}
|
||||
|
||||
// Setup the basic field stuff
|
||||
#define SET_FIELD(x) /**/
|
||||
|
||||
#include "defs.cu.inc"
|
||||
|
||||
#include "types.cuh.inc"
|
||||
|
||||
#include "layout.cu.inc"
|
||||
|
||||
} // namespace risc0::circuit::rv32im_v2::cuda
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "fp.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace risc0 {
|
||||
|
||||
struct Buffer {
|
||||
Fp* buf;
|
||||
size_t rows;
|
||||
size_t cols;
|
||||
bool checkedReads;
|
||||
|
||||
void set(size_t row, size_t col, Fp val) {
|
||||
Fp& elem = buf[col * rows + row];
|
||||
if (elem != Fp::invalid() && elem != val) {
|
||||
printf("set(row: %zu, col: %zu, val: 0x%08x) cur: 0x%08x\n",
|
||||
row,
|
||||
col,
|
||||
val.asUInt32(),
|
||||
elem.asUInt32());
|
||||
throw std::runtime_error("Inconsistent set");
|
||||
}
|
||||
// printf("set(row: %zu, col: %zu, val: 0x%08x)\n", row, col, val.asUInt32());
|
||||
elem = val;
|
||||
}
|
||||
|
||||
Fp get(size_t row, size_t col) {
|
||||
Fp ret = buf[col * rows + row];
|
||||
if (ret == Fp::invalid() && checkedReads) {
|
||||
printf("get(row: %zu, col: %zu) -> 0x%08x\n", row, col, ret.asRaw());
|
||||
throw std::runtime_error("Read of unset value");
|
||||
}
|
||||
// printf("get(row: %zu, col: %zu) -> 0x%08x\n", row, col, ret.asUInt32());
|
||||
return ret;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace risc0
|
||||
@@ -0,0 +1,7 @@
|
||||
SET_FIELD(BabyBear);
|
||||
constexpr size_t kRegCountAccum = 76;
|
||||
constexpr size_t kRegCountCode = 1;
|
||||
constexpr size_t kRegCountData = 192;
|
||||
constexpr size_t kRegCountGlobal = 73;
|
||||
constexpr size_t kRegCountMix = 32;
|
||||
constexpr size_t kRegCountTest = 192;
|
||||
@@ -0,0 +1,39 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "fp.h"
|
||||
#include "fpext.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <exception>
|
||||
#include <string.h>
|
||||
|
||||
using namespace risc0;
|
||||
|
||||
namespace risc0::circuit::rv32im_v2 {
|
||||
|
||||
FpExt poly_fp(size_t cycle, size_t steps, FpExt* poly_mix, Fp** args);
|
||||
|
||||
} // namespace risc0::circuit::rv32im_v2
|
||||
|
||||
extern "C" const char* risc0_circuit_rv32im_v2_cpu_poly_fp(
|
||||
size_t cycle, size_t steps, FpExt* poly_mix, Fp** args, FpExt* result) {
|
||||
try {
|
||||
*result = circuit::rv32im_v2::poly_fp(cycle, steps, poly_mix, args);
|
||||
} catch (const std::exception& err) {
|
||||
return strdup(err.what());
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "buffers.h"
|
||||
#include "fp.h"
|
||||
#include "fpext.h"
|
||||
#include "preflight.h"
|
||||
#include "steps.h"
|
||||
#include "witgen.h"
|
||||
|
||||
#include "vendor/poolstl.hpp"
|
||||
|
||||
#include <array>
|
||||
#include <assert.h>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <string.h>
|
||||
#include <vector>
|
||||
|
||||
namespace risc0::circuit::rv32im_v2::cpu {
|
||||
|
||||
std::array<uint32_t, 2> divide_rv32im(uint32_t numer, uint32_t denom, uint32_t signType) {
|
||||
uint32_t onesComp = (signType == 2);
|
||||
bool negNumer = signType && int32_t(numer) < 0;
|
||||
bool negDenom = signType == 1 && int32_t(denom) < 0;
|
||||
if (negNumer) {
|
||||
numer = -numer - onesComp;
|
||||
}
|
||||
if (negDenom) {
|
||||
denom = -denom - onesComp;
|
||||
}
|
||||
uint32_t quot;
|
||||
uint32_t rem;
|
||||
if (denom == 0) {
|
||||
quot = 0xffffffff;
|
||||
rem = numer;
|
||||
} else {
|
||||
quot = numer / denom;
|
||||
rem = numer % denom;
|
||||
}
|
||||
uint32_t quotNegOut = (negNumer ^ negDenom) - ((denom == 0) * negNumer);
|
||||
uint32_t remNegOut = negNumer;
|
||||
if (quotNegOut) {
|
||||
quot = -quot - onesComp;
|
||||
}
|
||||
if (remNegOut) {
|
||||
rem = -rem - onesComp;
|
||||
}
|
||||
return {quot, rem};
|
||||
}
|
||||
|
||||
std::array<Val, 5> extern_getMemoryTxn(ExecContext& ctx, Val addrElem) {
|
||||
uint32_t addr = addrElem.asUInt32();
|
||||
size_t txnIdx = ctx.preflight.cycles[ctx.cycle].txnIdx++;
|
||||
const MemoryTransaction& txn = ctx.preflight.txns[txnIdx];
|
||||
// printf("getMemoryTxn(%lu, 0x%08x): txn(%u, 0x%08x, 0x%08x)\n",
|
||||
// ctx.cycle,
|
||||
// addr,
|
||||
// txn.cycle,
|
||||
// txn.addr,
|
||||
// txn.word);
|
||||
|
||||
if (txn.cycle != ctx.cycle) {
|
||||
printf("txn.cycle: %u, ctx.cycle: %zu\n", txn.cycle, ctx.cycle);
|
||||
throw std::runtime_error("txn cycle mismatch");
|
||||
}
|
||||
|
||||
if (txn.addr != addr) {
|
||||
printf("[%lu]: txn.addr: 0x%08x, addr: 0x%08x\n", ctx.cycle, txn.addr, addr);
|
||||
throw std::runtime_error("memory peek not in preflight");
|
||||
}
|
||||
|
||||
return {
|
||||
txn.prevCycle,
|
||||
txn.prevWord & 0xffff,
|
||||
txn.prevWord >> 16,
|
||||
txn.word & 0xffff,
|
||||
txn.word >> 16,
|
||||
};
|
||||
}
|
||||
|
||||
void extern_lookupDelta(ExecContext& ctx, Val table, Val index, Val count) {
|
||||
// printf("[%lu]: lookupDelta(table: %u, index: %u, count: %u)\n",
|
||||
// ctx.cycle,
|
||||
// table.asUInt32(),
|
||||
// index.asUInt32(),
|
||||
// count.asUInt32());
|
||||
ctx.tables.lookupDelta(ctx.cycle, table, index, count);
|
||||
}
|
||||
|
||||
Val extern_lookupCurrent(ExecContext& ctx, Val table, Val index) {
|
||||
Val ret = ctx.tables.lookupCurrent(table, index);
|
||||
// printf("lookupCurrent(table: %u, index: %u): %u\n",
|
||||
// table.asUInt32(),
|
||||
// index.asUInt32(),
|
||||
// ret.asUInt32());
|
||||
return ret;
|
||||
}
|
||||
|
||||
void extern_memoryDelta(
|
||||
ExecContext& ctx, Val addr, Val cycle, Val dataLow, Val dataHigh, Val count) {
|
||||
// printf("memoryDelta\n");
|
||||
// ctx.tables.memoryDelta(
|
||||
// addr.asUInt32(), cycle.asUInt32(), dataLow.asUInt32() | (dataHigh.asUInt32() << 16),
|
||||
// count);
|
||||
}
|
||||
|
||||
uint32_t extern_getDiffCount(ExecContext& ctx, Val cycle) {
|
||||
// printf("getDiffCount\n");
|
||||
return ctx.preflight.cycles[cycle.asUInt32()].diffCount;
|
||||
}
|
||||
|
||||
Val extern_isFirstCycle_0(ExecContext& ctx) {
|
||||
return ctx.cycle == 0;
|
||||
}
|
||||
|
||||
Val extern_getCycle(ExecContext& ctx) {
|
||||
return ctx.cycle;
|
||||
}
|
||||
|
||||
std::ostream& hex_word(std::ostream& os, uint32_t word) {
|
||||
std::cout << "0x" //
|
||||
<< std::hex << std::setw(8) << std::setfill('0') //
|
||||
<< word //
|
||||
<< std::dec << std::setw(0);
|
||||
return os;
|
||||
}
|
||||
|
||||
void extern_log(ExecContext& ctx, const std::string& message, std::vector<Val> vals) {
|
||||
// std::cout << "LOG: '" << message << "': ";
|
||||
// for (size_t i = 0; i < vals.size(); i++) {
|
||||
// if (i != 0) {
|
||||
// std::cout << ", ";
|
||||
// }
|
||||
// hex_word(std::cout, vals[i].asUInt32());
|
||||
// }
|
||||
// std::cout << "\n";
|
||||
}
|
||||
|
||||
std::array<Val, 4> extern_divide(
|
||||
ExecContext& ctx, Val numerLow, Val numerHigh, Val denomLow, Val denomHigh, Val signType) {
|
||||
uint32_t numer = numerLow.asUInt32() | (numerHigh.asUInt32() << 16);
|
||||
uint32_t denom = denomLow.asUInt32() | (denomHigh.asUInt32() << 16);
|
||||
auto [quot, rem] = divide_rv32im(numer, denom, signType.asUInt32());
|
||||
std::array<Val, 4> ret;
|
||||
ret[0] = quot & 0xffff;
|
||||
ret[1] = quot >> 16;
|
||||
ret[2] = rem & 0xffff;
|
||||
ret[3] = rem >> 16;
|
||||
return ret;
|
||||
}
|
||||
|
||||
// TODO: logging
|
||||
void extern_print(ExecContext& ctx, Val v) {
|
||||
std::cout << "LOG: " << v.asUInt32() << "\n";
|
||||
}
|
||||
|
||||
std::array<Val, 2> extern_getMajorMinor(ExecContext& ctx) {
|
||||
uint8_t major = ctx.preflight.cycles[ctx.cycle].major;
|
||||
uint8_t minor = ctx.preflight.cycles[ctx.cycle].minor;
|
||||
return {major, minor};
|
||||
}
|
||||
|
||||
Val extern_hostReadPrepare(ExecContext& ctx, Val fp, Val len) {
|
||||
std::cout << "hostReadPrepare\n";
|
||||
throw std::runtime_error("extern_hostReadPrepare");
|
||||
// return ctx.stepHandler.readPrepare(fp.asUInt32(), len.asUInt32());
|
||||
// return 0;
|
||||
}
|
||||
|
||||
Val extern_hostWrite(ExecContext& ctx, Val fdVal, Val addrLow, Val addrHigh, Val lenVal) {
|
||||
std::cout << "hostWrite\n";
|
||||
throw std::runtime_error("extern_hostWrite");
|
||||
// uint32_t fd = fdVal.asUInt32();
|
||||
// uint32_t addr = addrLow.asUInt32() | (addrHigh.asUInt32() << 16);
|
||||
// uint32_t len = lenVal.asUInt32();
|
||||
// return ctx.stepHandler.write(fd, addr, len);
|
||||
// return 0;
|
||||
}
|
||||
|
||||
std::array<Val, 2> extern_nextPagingIdx(ExecContext& ctx) {
|
||||
uint32_t pagingIdx = ctx.preflight.cycles[ctx.cycle].pagingIdx;
|
||||
uint32_t machineMode = ctx.preflight.cycles[ctx.cycle].machineMode;
|
||||
// printf("nextPagingIdx: (0x%05x, %u)\n", pagingIdx, machineMode);
|
||||
return {pagingIdx, machineMode};
|
||||
}
|
||||
|
||||
void stepExec(ExecBuffers& buffers, PreflightTrace& preflight, LookupTables& tables, size_t cycle) {
|
||||
ExecContext ctx(preflight, tables, cycle);
|
||||
MutableBufObj data(buffers.data);
|
||||
GlobalBufObj global(buffers.global);
|
||||
step_Top(ctx, &data, &global);
|
||||
}
|
||||
|
||||
void stepAccum(AccumBuffers& buffers,
|
||||
PreflightTrace& preflight,
|
||||
LookupTables& tables,
|
||||
size_t cycle) {
|
||||
ExecContext ctx(preflight, tables, cycle);
|
||||
MutableBufObj data(buffers.data);
|
||||
MutableBufObj accum(buffers.accum);
|
||||
GlobalBufObj mix(buffers.mix);
|
||||
step_TopAccum(ctx, &accum, &data, &mix);
|
||||
}
|
||||
|
||||
} // namespace risc0::circuit::rv32im_v2::cpu
|
||||
|
||||
constexpr size_t kStepModeParallel = 0;
|
||||
constexpr size_t kStepModeSeqForward = 1;
|
||||
constexpr size_t kStepModeSeqReverse = 2;
|
||||
|
||||
extern "C" {
|
||||
|
||||
using namespace risc0::circuit::rv32im_v2::cpu;
|
||||
|
||||
const char* risc0_circuit_rv32im_v2_cpu_witgen(uint32_t mode,
|
||||
ExecBuffers* buffers,
|
||||
PreflightTrace* preflight,
|
||||
uint32_t lastCycle) {
|
||||
LookupTables tables;
|
||||
size_t split = preflight->tableSplitCycle;
|
||||
try {
|
||||
switch (mode) {
|
||||
case kStepModeParallel: {
|
||||
auto begin1 = poolstl::iota_iter<uint32_t>(0);
|
||||
auto end1 = poolstl::iota_iter<uint32_t>(split);
|
||||
std::for_each(poolstl::par, begin1, end1, [&](uint32_t cycle) {
|
||||
stepExec(*buffers, *preflight, tables, cycle);
|
||||
});
|
||||
|
||||
auto begin2 = poolstl::iota_iter<uint32_t>(split);
|
||||
auto end2 = poolstl::iota_iter<uint32_t>(lastCycle);
|
||||
std::for_each(poolstl::par, begin2, end2, [&](uint32_t cycle) {
|
||||
stepExec(*buffers, *preflight, tables, cycle);
|
||||
});
|
||||
} break;
|
||||
case kStepModeSeqForward:
|
||||
for (size_t cycle = 0; cycle < lastCycle; cycle++) {
|
||||
stepExec(*buffers, *preflight, tables, cycle);
|
||||
}
|
||||
break;
|
||||
case kStepModeSeqReverse: {
|
||||
for (size_t i = split; i-- > 0;) {
|
||||
// printf("stepExec: %zu\n", i);
|
||||
stepExec(*buffers, *preflight, tables, i);
|
||||
}
|
||||
for (size_t i = lastCycle; i-- > split;) {
|
||||
// printf("stepExec: %zu\n", i);
|
||||
stepExec(*buffers, *preflight, tables, i);
|
||||
}
|
||||
} break;
|
||||
}
|
||||
} catch (const std::exception& err) {
|
||||
return strdup(err.what());
|
||||
} catch (...) {
|
||||
return strdup("Generic exception");
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const char* risc0_circuit_rv32im_v2_cpu_accum(AccumBuffers* buffers,
|
||||
PreflightTrace* preflight,
|
||||
uint32_t lastCycle) {
|
||||
try {
|
||||
LookupTables tables;
|
||||
for (size_t cycle = 0; cycle < lastCycle; cycle++) {
|
||||
stepAccum(*buffers, *preflight, tables, cycle);
|
||||
}
|
||||
} catch (const std::exception& err) {
|
||||
return strdup(err.what());
|
||||
} catch (...) {
|
||||
return strdup("Generic exception");
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
} // extern "C"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,885 @@
|
||||
extern const NondetRegLayout8LayoutArray kLayout__3;
|
||||
extern const OneHot_8_Layout kLayout__2;
|
||||
extern const InstInputLayout kLayout__1;
|
||||
extern const NondetRegLayout11LayoutArray kLayout__5;
|
||||
extern const OneHot_11_Layout kLayout__4;
|
||||
extern const NondetU16RegLayout kLayout__10;
|
||||
extern const NondetU16RegLayout kLayout__11;
|
||||
extern const NormalizeU32Layout kLayout__9;
|
||||
extern const NondetU16RegLayout kLayout__13;
|
||||
extern const NondetU16RegLayout kLayout__14;
|
||||
extern const NormalizeU32Layout kLayout__12;
|
||||
extern const MemoryArgLayout kLayout__18;
|
||||
extern const MemoryArgLayout kLayout__19;
|
||||
extern const MemoryIOLayout kLayout__17;
|
||||
extern const IsCycleLayout kLayout__21;
|
||||
extern const IsForwardLayout kLayout__20;
|
||||
extern const MemoryWriteLayout kLayout__16;
|
||||
extern const WriteRdLayout kLayout__15;
|
||||
extern const FinalizeMiscLayout kLayout__8;
|
||||
extern const DecoderLayout kLayout__24;
|
||||
extern const NondetU16RegLayout kLayout__27;
|
||||
extern const U16RegLayout kLayout__26;
|
||||
extern const NondetU16RegLayout kLayout__28;
|
||||
extern const AddrDecomposeLayout kLayout__25;
|
||||
extern const MemoryArgLayout kLayout__31;
|
||||
extern const MemoryArgLayout kLayout__32;
|
||||
extern const MemoryIOLayout kLayout__30;
|
||||
extern const IsCycleLayout kLayout__34;
|
||||
extern const IsForwardLayout kLayout__33;
|
||||
extern const MemoryReadLayout kLayout__29;
|
||||
extern const DecodeInstLayout kLayout__23;
|
||||
extern const MemoryArgLayout kLayout__38;
|
||||
extern const MemoryArgLayout kLayout__39;
|
||||
extern const MemoryIOLayout kLayout__37;
|
||||
extern const IsCycleLayout kLayout__41;
|
||||
extern const IsForwardLayout kLayout__40;
|
||||
extern const MemoryReadLayout kLayout__36;
|
||||
extern const ReadRegLayout kLayout__35;
|
||||
extern const MemoryArgLayout kLayout__45;
|
||||
extern const MemoryArgLayout kLayout__46;
|
||||
extern const MemoryIOLayout kLayout__44;
|
||||
extern const IsCycleLayout kLayout__48;
|
||||
extern const IsForwardLayout kLayout__47;
|
||||
extern const MemoryReadLayout kLayout__43;
|
||||
extern const ReadRegLayout kLayout__42;
|
||||
extern const MiscInputLayout kLayout__22;
|
||||
extern const ArgU16Layout5LayoutArray kLayout__50;
|
||||
extern const _Arguments_Misc0MiscOutputLayout kLayout__49;
|
||||
extern const Misc0MiscOutputArm0Layout kLayout__52;
|
||||
extern const Misc0MiscOutputArm1Layout kLayout__53;
|
||||
extern const NondetRegLayout16LayoutArray kLayout__60;
|
||||
extern const ToBits_16_Layout kLayout__59;
|
||||
extern const NondetRegLayout16LayoutArray kLayout__62;
|
||||
extern const ToBits_16_Layout kLayout__61;
|
||||
extern const BitwiseAndU16Layout kLayout__58;
|
||||
extern const NondetRegLayout16LayoutArray kLayout__65;
|
||||
extern const ToBits_16_Layout kLayout__64;
|
||||
extern const NondetRegLayout16LayoutArray kLayout__67;
|
||||
extern const ToBits_16_Layout kLayout__66;
|
||||
extern const BitwiseAndU16Layout kLayout__63;
|
||||
extern const BitwiseAndLayout kLayout__57;
|
||||
extern const BitwiseXorLayout kLayout__56;
|
||||
extern const OpXORLayout kLayout__55;
|
||||
extern const Misc0MiscOutputArm2Layout kLayout__54;
|
||||
extern const BitwiseOrLayout kLayout__70;
|
||||
extern const OpORLayout kLayout__69;
|
||||
extern const Misc0MiscOutputArm3Layout kLayout__68;
|
||||
extern const OpANDLayout kLayout__72;
|
||||
extern const Misc0MiscOutputArm4Layout kLayout__71;
|
||||
extern const NondetU16RegLayout kLayout__76;
|
||||
extern const NondetU16RegLayout kLayout__77;
|
||||
extern const NormalizeU32Layout kLayout__75;
|
||||
extern const NondetU16RegLayout kLayout__79;
|
||||
extern const GetSignU32Layout kLayout__78;
|
||||
extern const NondetU16RegLayout kLayout__81;
|
||||
extern const GetSignU32Layout kLayout__80;
|
||||
extern const NondetU16RegLayout kLayout__83;
|
||||
extern const GetSignU32Layout kLayout__82;
|
||||
extern const CmpLessThanLayout kLayout__74;
|
||||
extern const OpSLTLayout kLayout__73;
|
||||
extern const CmpLessThanUnsignedLayout kLayout__86;
|
||||
extern const OpSLTULayout kLayout__85;
|
||||
extern const Misc0MiscOutputArm6Layout kLayout__84;
|
||||
extern const Misc0MiscOutputArm7Layout kLayout__87;
|
||||
extern const Misc0MiscOutputLayout kLayout__51;
|
||||
extern const Misc0Layout kLayout__7;
|
||||
extern const _Arguments_Misc1MiscOutputLayout kLayout__89;
|
||||
extern const OpXORILayout kLayout__92;
|
||||
extern const Misc1MiscOutputArm0Layout kLayout__91;
|
||||
extern const OpORILayout kLayout__94;
|
||||
extern const Misc1MiscOutputArm1Layout kLayout__93;
|
||||
extern const OpANDILayout kLayout__96;
|
||||
extern const Misc1MiscOutputArm2Layout kLayout__95;
|
||||
extern const OpSLTILayout kLayout__97;
|
||||
extern const OpSLTIULayout kLayout__99;
|
||||
extern const Misc1MiscOutputArm4Layout kLayout__98;
|
||||
extern const CmpEqualLayout kLayout__102;
|
||||
extern const OpBEQLayout kLayout__101;
|
||||
extern const Misc1MiscOutputArm5Layout kLayout__100;
|
||||
extern const OpBNELayout kLayout__104;
|
||||
extern const Misc1MiscOutputArm6Layout kLayout__103;
|
||||
extern const OpBLTLayout kLayout__105;
|
||||
extern const Misc1MiscOutputLayout kLayout__90;
|
||||
extern const Misc1Layout kLayout__88;
|
||||
extern const _Arguments_Misc2MiscOutputLayout kLayout__107;
|
||||
extern const OpBGELayout kLayout__109;
|
||||
extern const OpBLTULayout kLayout__111;
|
||||
extern const Misc2MiscOutputArm1Layout kLayout__110;
|
||||
extern const OpBGEULayout kLayout__113;
|
||||
extern const Misc2MiscOutputArm2Layout kLayout__112;
|
||||
extern const Misc2MiscOutputArm3Layout kLayout__114;
|
||||
extern const Misc2MiscOutputArm4Layout kLayout__115;
|
||||
extern const Misc2MiscOutputArm5Layout kLayout__116;
|
||||
extern const Misc2MiscOutputArm6Layout kLayout__117;
|
||||
extern const Misc2MiscOutputArm7Layout kLayout__118;
|
||||
extern const Misc2MiscOutputLayout kLayout__108;
|
||||
extern const Misc2Layout kLayout__106;
|
||||
extern const DecoderLayout kLayout__122;
|
||||
extern const NondetU16RegLayout kLayout__125;
|
||||
extern const U16RegLayout kLayout__124;
|
||||
extern const NondetU16RegLayout kLayout__126;
|
||||
extern const AddrDecomposeLayout kLayout__123;
|
||||
extern const MemoryArgLayout kLayout__129;
|
||||
extern const MemoryArgLayout kLayout__130;
|
||||
extern const MemoryIOLayout kLayout__128;
|
||||
extern const IsCycleLayout kLayout__132;
|
||||
extern const IsForwardLayout kLayout__131;
|
||||
extern const MemoryReadLayout kLayout__127;
|
||||
extern const DecodeInstLayout kLayout__121;
|
||||
extern const MemoryArgLayout kLayout__136;
|
||||
extern const MemoryArgLayout kLayout__137;
|
||||
extern const MemoryIOLayout kLayout__135;
|
||||
extern const IsCycleLayout kLayout__139;
|
||||
extern const IsForwardLayout kLayout__138;
|
||||
extern const MemoryReadLayout kLayout__134;
|
||||
extern const ReadRegLayout kLayout__133;
|
||||
extern const MemoryArgLayout kLayout__143;
|
||||
extern const MemoryArgLayout kLayout__144;
|
||||
extern const MemoryIOLayout kLayout__142;
|
||||
extern const IsCycleLayout kLayout__146;
|
||||
extern const IsForwardLayout kLayout__145;
|
||||
extern const MemoryReadLayout kLayout__141;
|
||||
extern const ReadRegLayout kLayout__140;
|
||||
extern const MulInputLayout kLayout__120;
|
||||
extern const ArgU16Layout6LayoutArray kLayout__148;
|
||||
extern const ArgU8Layout13LayoutArray kLayout__149;
|
||||
extern const _Arguments_Mul0MulOutputLayout kLayout__147;
|
||||
extern const NondetRegLayout5LayoutArray kLayout__154;
|
||||
extern const ToBits_5_Layout kLayout__153;
|
||||
extern const DynPo2Layout kLayout__152;
|
||||
extern const NondetU8RegLayout kLayout__158;
|
||||
extern const NondetU8RegLayout kLayout__159;
|
||||
extern const NondetU8RegLayout kLayout__160;
|
||||
extern const NondetU8RegLayout kLayout__161;
|
||||
extern const NondetU8RegLayout kLayout__162;
|
||||
extern const ExpandU32Layout kLayout__157;
|
||||
extern const NondetU8RegLayout kLayout__164;
|
||||
extern const NondetU8RegLayout kLayout__165;
|
||||
extern const NondetU8RegLayout kLayout__166;
|
||||
extern const NondetU8RegLayout kLayout__167;
|
||||
extern const NondetU8RegLayout kLayout__168;
|
||||
extern const ExpandU32Layout kLayout__163;
|
||||
extern const NondetU8RegLayout kLayout__170;
|
||||
extern const SplitTotalLayout kLayout__169;
|
||||
extern const NondetU8RegLayout kLayout__172;
|
||||
extern const SplitTotalLayout kLayout__171;
|
||||
extern const NondetU8RegLayout kLayout__174;
|
||||
extern const SplitTotalLayout kLayout__173;
|
||||
extern const MultiplyAccumulateLayout kLayout__156;
|
||||
extern const DoMulLayout kLayout__155;
|
||||
extern const OpSLLLayout kLayout__151;
|
||||
extern const OpSLLILayout kLayout__175;
|
||||
extern const ExpandU32Layout kLayout__180;
|
||||
extern const ExpandU32Layout kLayout__181;
|
||||
extern const SplitTotalLayout kLayout__182;
|
||||
extern const SplitTotalLayout kLayout__183;
|
||||
extern const SplitTotalLayout kLayout__184;
|
||||
extern const MultiplyAccumulateLayout kLayout__179;
|
||||
extern const DoMulLayout kLayout__178;
|
||||
extern const OpMULLayout kLayout__177;
|
||||
extern const Mul0MulOutputArm2Layout kLayout__176;
|
||||
extern const OpMULHLayout kLayout__186;
|
||||
extern const Mul0MulOutputArm3Layout kLayout__185;
|
||||
extern const OpMULHSULayout kLayout__188;
|
||||
extern const Mul0MulOutputArm4Layout kLayout__187;
|
||||
extern const OpMULHULayout kLayout__190;
|
||||
extern const Mul0MulOutputArm5Layout kLayout__189;
|
||||
extern const Mul0MulOutputArm6Layout kLayout__191;
|
||||
extern const Mul0MulOutputArm7Layout kLayout__192;
|
||||
extern const Mul0MulOutputLayout kLayout__150;
|
||||
extern const MemoryArgLayout kLayout__196;
|
||||
extern const MemoryArgLayout kLayout__197;
|
||||
extern const MemoryIOLayout kLayout__195;
|
||||
extern const IsCycleLayout kLayout__199;
|
||||
extern const IsForwardLayout kLayout__198;
|
||||
extern const MemoryWriteLayout kLayout__194;
|
||||
extern const WriteRdLayout kLayout__193;
|
||||
extern const NondetU16RegLayout kLayout__201;
|
||||
extern const NondetU16RegLayout kLayout__202;
|
||||
extern const NormalizeU32Layout kLayout__200;
|
||||
extern const Mul0Layout kLayout__119;
|
||||
extern const DecoderLayout kLayout__206;
|
||||
extern const NondetU16RegLayout kLayout__209;
|
||||
extern const U16RegLayout kLayout__208;
|
||||
extern const NondetU16RegLayout kLayout__210;
|
||||
extern const AddrDecomposeLayout kLayout__207;
|
||||
extern const MemoryArgLayout kLayout__213;
|
||||
extern const MemoryArgLayout kLayout__214;
|
||||
extern const MemoryIOLayout kLayout__212;
|
||||
extern const IsCycleLayout kLayout__216;
|
||||
extern const IsForwardLayout kLayout__215;
|
||||
extern const MemoryReadLayout kLayout__211;
|
||||
extern const DecodeInstLayout kLayout__205;
|
||||
extern const MemoryArgLayout kLayout__220;
|
||||
extern const MemoryArgLayout kLayout__221;
|
||||
extern const MemoryIOLayout kLayout__219;
|
||||
extern const IsCycleLayout kLayout__223;
|
||||
extern const IsForwardLayout kLayout__222;
|
||||
extern const MemoryReadLayout kLayout__218;
|
||||
extern const ReadRegLayout kLayout__217;
|
||||
extern const MemoryArgLayout kLayout__227;
|
||||
extern const MemoryArgLayout kLayout__228;
|
||||
extern const MemoryIOLayout kLayout__226;
|
||||
extern const IsCycleLayout kLayout__230;
|
||||
extern const IsForwardLayout kLayout__229;
|
||||
extern const MemoryReadLayout kLayout__225;
|
||||
extern const ReadRegLayout kLayout__224;
|
||||
extern const DivInputLayout kLayout__204;
|
||||
extern const ArgU16Layout9LayoutArray kLayout__232;
|
||||
extern const ArgU8Layout13LayoutArray kLayout__233;
|
||||
extern const _Arguments_Div0MulOutputLayout kLayout__231;
|
||||
extern const NondetRegLayout5LayoutArray kLayout__239;
|
||||
extern const ToBits_5_Layout kLayout__238;
|
||||
extern const DynPo2Layout kLayout__237;
|
||||
extern const ExpandU32Layout kLayout__242;
|
||||
extern const ExpandU32Layout kLayout__243;
|
||||
extern const NondetU8RegLayout kLayout__245;
|
||||
extern const SplitTotalLayout kLayout__244;
|
||||
extern const NondetU8RegLayout kLayout__247;
|
||||
extern const SplitTotalLayout kLayout__246;
|
||||
extern const NondetU16RegLayout kLayout__249;
|
||||
extern const NondetU8RegLayout kLayout__250;
|
||||
extern const SplitTotalLayout kLayout__248;
|
||||
extern const NondetU16RegLayout kLayout__251;
|
||||
extern const MultiplyAccumulateLayout kLayout__241;
|
||||
extern const DoDivLayout kLayout__240;
|
||||
extern const OpSRLLayout kLayout__236;
|
||||
extern const Div0MulOutputArm0Layout kLayout__235;
|
||||
extern const TopBitLayout kLayout__253;
|
||||
extern const ExpandU32Layout kLayout__256;
|
||||
extern const ExpandU32Layout kLayout__257;
|
||||
extern const SplitTotalLayout kLayout__258;
|
||||
extern const SplitTotalLayout kLayout__259;
|
||||
extern const SplitTotalLayout kLayout__260;
|
||||
extern const MultiplyAccumulateLayout kLayout__255;
|
||||
extern const DoDivLayout kLayout__254;
|
||||
extern const OpSRALayout kLayout__252;
|
||||
extern const OpSRLILayout kLayout__262;
|
||||
extern const Div0MulOutputArm2Layout kLayout__261;
|
||||
extern const OpSRAILayout kLayout__263;
|
||||
extern const ExpandU32Layout kLayout__268;
|
||||
extern const ExpandU32Layout kLayout__269;
|
||||
extern const SplitTotalLayout kLayout__270;
|
||||
extern const SplitTotalLayout kLayout__271;
|
||||
extern const SplitTotalLayout kLayout__272;
|
||||
extern const MultiplyAccumulateLayout kLayout__267;
|
||||
extern const DoDivLayout kLayout__266;
|
||||
extern const OpDIVLayout kLayout__265;
|
||||
extern const Div0MulOutputArm4Layout kLayout__264;
|
||||
extern const OpDIVULayout kLayout__274;
|
||||
extern const Div0MulOutputArm5Layout kLayout__273;
|
||||
extern const OpREMLayout kLayout__276;
|
||||
extern const Div0MulOutputArm6Layout kLayout__275;
|
||||
extern const OpREMULayout kLayout__278;
|
||||
extern const Div0MulOutputArm7Layout kLayout__277;
|
||||
extern const Div0MulOutputLayout kLayout__234;
|
||||
extern const MemoryArgLayout kLayout__282;
|
||||
extern const MemoryArgLayout kLayout__283;
|
||||
extern const MemoryIOLayout kLayout__281;
|
||||
extern const IsCycleLayout kLayout__285;
|
||||
extern const IsForwardLayout kLayout__284;
|
||||
extern const MemoryWriteLayout kLayout__280;
|
||||
extern const WriteRdLayout kLayout__279;
|
||||
extern const NondetU16RegLayout kLayout__287;
|
||||
extern const NondetU16RegLayout kLayout__288;
|
||||
extern const NormalizeU32Layout kLayout__286;
|
||||
extern const Div0Layout kLayout__203;
|
||||
extern const DecoderLayout kLayout__292;
|
||||
extern const NondetU16RegLayout kLayout__295;
|
||||
extern const U16RegLayout kLayout__294;
|
||||
extern const NondetU16RegLayout kLayout__296;
|
||||
extern const AddrDecomposeLayout kLayout__293;
|
||||
extern const MemoryArgLayout kLayout__299;
|
||||
extern const MemoryArgLayout kLayout__300;
|
||||
extern const MemoryIOLayout kLayout__298;
|
||||
extern const IsCycleLayout kLayout__302;
|
||||
extern const IsForwardLayout kLayout__301;
|
||||
extern const MemoryReadLayout kLayout__297;
|
||||
extern const DecodeInstLayout kLayout__291;
|
||||
extern const MemoryArgLayout kLayout__306;
|
||||
extern const MemoryArgLayout kLayout__307;
|
||||
extern const MemoryIOLayout kLayout__305;
|
||||
extern const IsCycleLayout kLayout__309;
|
||||
extern const IsForwardLayout kLayout__308;
|
||||
extern const MemoryReadLayout kLayout__304;
|
||||
extern const ReadRegLayout kLayout__303;
|
||||
extern const NormalizeU32Layout kLayout__310;
|
||||
extern const NondetU16RegLayout kLayout__313;
|
||||
extern const U16RegLayout kLayout__312;
|
||||
extern const NondetU16RegLayout kLayout__314;
|
||||
extern const AddrDecomposeBitsLayout kLayout__311;
|
||||
extern const MemoryArgLayout kLayout__317;
|
||||
extern const MemoryArgLayout kLayout__318;
|
||||
extern const MemoryIOLayout kLayout__316;
|
||||
extern const IsCycleLayout kLayout__320;
|
||||
extern const IsForwardLayout kLayout__319;
|
||||
extern const MemoryReadLayout kLayout__315;
|
||||
extern const MemLoadInputLayout kLayout__290;
|
||||
extern const ArgU8Layout3LayoutArray kLayout__322;
|
||||
extern const _Arguments_Mem0OutputLayout kLayout__321;
|
||||
extern const NondetU8RegLayout kLayout__327;
|
||||
extern const NondetU8RegLayout kLayout__328;
|
||||
extern const SplitWordLayout kLayout__326;
|
||||
extern const NondetU8RegLayout kLayout__329;
|
||||
extern const OpLBLayout kLayout__325;
|
||||
extern const Mem0OutputArm0Layout kLayout__324;
|
||||
extern const OpLHLayout kLayout__331;
|
||||
extern const Mem0OutputArm1Layout kLayout__330;
|
||||
extern const Mem0OutputArm2Layout kLayout__332;
|
||||
extern const OpLBULayout kLayout__334;
|
||||
extern const Mem0OutputArm3Layout kLayout__333;
|
||||
extern const Mem0OutputArm4Layout kLayout__335;
|
||||
extern const Mem0OutputArm5Layout kLayout__336;
|
||||
extern const Mem0OutputArm6Layout kLayout__337;
|
||||
extern const Mem0OutputArm7Layout kLayout__338;
|
||||
extern const Mem0OutputLayout kLayout__323;
|
||||
extern const MemoryArgLayout kLayout__342;
|
||||
extern const MemoryArgLayout kLayout__343;
|
||||
extern const MemoryIOLayout kLayout__341;
|
||||
extern const IsCycleLayout kLayout__345;
|
||||
extern const IsForwardLayout kLayout__344;
|
||||
extern const MemoryWriteLayout kLayout__340;
|
||||
extern const WriteRdLayout kLayout__339;
|
||||
extern const NondetU16RegLayout kLayout__347;
|
||||
extern const NondetU16RegLayout kLayout__348;
|
||||
extern const NormalizeU32Layout kLayout__346;
|
||||
extern const Mem0Layout kLayout__289;
|
||||
extern const MemoryArgLayout kLayout__354;
|
||||
extern const MemoryArgLayout kLayout__355;
|
||||
extern const MemoryIOLayout kLayout__353;
|
||||
extern const IsCycleLayout kLayout__357;
|
||||
extern const IsForwardLayout kLayout__356;
|
||||
extern const MemoryReadLayout kLayout__352;
|
||||
extern const ReadRegLayout kLayout__351;
|
||||
extern const NondetU16RegLayout kLayout__359;
|
||||
extern const NondetU16RegLayout kLayout__360;
|
||||
extern const NormalizeU32Layout kLayout__358;
|
||||
extern const NondetU16RegLayout kLayout__363;
|
||||
extern const U16RegLayout kLayout__362;
|
||||
extern const NondetU16RegLayout kLayout__364;
|
||||
extern const AddrDecomposeBitsLayout kLayout__361;
|
||||
extern const MemStoreInputLayout kLayout__350;
|
||||
extern const ArgU8Layout4LayoutArray kLayout__366;
|
||||
extern const _Arguments_Mem1OutputLayout kLayout__365;
|
||||
extern const NondetU8RegLayout kLayout__370;
|
||||
extern const SplitWordLayout kLayout__369;
|
||||
extern const OpSBLayout kLayout__368;
|
||||
extern const Mem1OutputArm1Layout kLayout__371;
|
||||
extern const Mem1OutputArm2Layout kLayout__372;
|
||||
extern const Mem1OutputArm3Layout kLayout__373;
|
||||
extern const Mem1OutputArm4Layout kLayout__374;
|
||||
extern const Mem1OutputArm5Layout kLayout__375;
|
||||
extern const Mem1OutputArm6Layout kLayout__376;
|
||||
extern const Mem1OutputArm7Layout kLayout__377;
|
||||
extern const Mem1OutputLayout kLayout__367;
|
||||
extern const MemoryArgLayout kLayout__381;
|
||||
extern const MemoryArgLayout kLayout__382;
|
||||
extern const MemoryIOLayout kLayout__380;
|
||||
extern const IsCycleLayout kLayout__384;
|
||||
extern const IsForwardLayout kLayout__383;
|
||||
extern const MemoryWriteLayout kLayout__379;
|
||||
extern const MemStoreFinalizeLayout kLayout__378;
|
||||
extern const NondetU16RegLayout kLayout__386;
|
||||
extern const NondetU16RegLayout kLayout__387;
|
||||
extern const NormalizeU32Layout kLayout__385;
|
||||
extern const Mem1Layout kLayout__349;
|
||||
extern const MemoryArgLayout kLayout__396;
|
||||
extern const MemoryArgLayout kLayout__397;
|
||||
extern const MemoryIOLayout kLayout__395;
|
||||
extern const MemoryPageInLayout kLayout__394;
|
||||
extern const ControlLoadRoot__0_SuperLayout kLayout__393;
|
||||
extern const MemoryArgLayout kLayout__401;
|
||||
extern const MemoryArgLayout kLayout__402;
|
||||
extern const MemoryIOLayout kLayout__400;
|
||||
extern const MemoryPageInLayout kLayout__399;
|
||||
extern const ControlLoadRoot__0_SuperLayout kLayout__398;
|
||||
extern const MemoryArgLayout kLayout__406;
|
||||
extern const MemoryArgLayout kLayout__407;
|
||||
extern const MemoryIOLayout kLayout__405;
|
||||
extern const MemoryPageInLayout kLayout__404;
|
||||
extern const ControlLoadRoot__0_SuperLayout kLayout__403;
|
||||
extern const MemoryArgLayout kLayout__411;
|
||||
extern const MemoryArgLayout kLayout__412;
|
||||
extern const MemoryIOLayout kLayout__410;
|
||||
extern const MemoryPageInLayout kLayout__409;
|
||||
extern const ControlLoadRoot__0_SuperLayout kLayout__408;
|
||||
extern const MemoryArgLayout kLayout__416;
|
||||
extern const MemoryArgLayout kLayout__417;
|
||||
extern const MemoryIOLayout kLayout__415;
|
||||
extern const MemoryPageInLayout kLayout__414;
|
||||
extern const ControlLoadRoot__0_SuperLayout kLayout__413;
|
||||
extern const MemoryArgLayout kLayout__421;
|
||||
extern const MemoryArgLayout kLayout__422;
|
||||
extern const MemoryIOLayout kLayout__420;
|
||||
extern const MemoryPageInLayout kLayout__419;
|
||||
extern const ControlLoadRoot__0_SuperLayout kLayout__418;
|
||||
extern const MemoryArgLayout kLayout__426;
|
||||
extern const MemoryArgLayout kLayout__427;
|
||||
extern const MemoryIOLayout kLayout__425;
|
||||
extern const MemoryPageInLayout kLayout__424;
|
||||
extern const ControlLoadRoot__0_SuperLayout kLayout__423;
|
||||
extern const MemoryArgLayout kLayout__431;
|
||||
extern const MemoryArgLayout kLayout__432;
|
||||
extern const MemoryIOLayout kLayout__430;
|
||||
extern const MemoryPageInLayout kLayout__429;
|
||||
extern const ControlLoadRoot__0_SuperLayout kLayout__428;
|
||||
extern const ControlLoadRoot__0_SuperLayout8LayoutArray kLayout__392;
|
||||
extern const ControlLoadRootLayout kLayout__391;
|
||||
extern const Control0_SuperArm0Layout kLayout__390;
|
||||
extern const IsCycleLayout kLayout__440;
|
||||
extern const IsForwardLayout kLayout__439;
|
||||
extern const MemoryReadLayout kLayout__438;
|
||||
extern const IsCycleLayout kLayout__443;
|
||||
extern const IsForwardLayout kLayout__442;
|
||||
extern const MemoryReadLayout kLayout__441;
|
||||
extern const ControlResume_SuperArm0_SuperLayout kLayout__437;
|
||||
extern const ControlResume_SuperArm0Layout kLayout__436;
|
||||
extern const MemoryWriteLayout kLayout__447;
|
||||
extern const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__446;
|
||||
extern const MemoryWriteLayout kLayout__449;
|
||||
extern const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__448;
|
||||
extern const IsCycleLayout kLayout__453;
|
||||
extern const IsForwardLayout kLayout__452;
|
||||
extern const MemoryWriteLayout kLayout__451;
|
||||
extern const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__450;
|
||||
extern const MemoryWriteLayout kLayout__455;
|
||||
extern const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__454;
|
||||
extern const IsCycleLayout kLayout__459;
|
||||
extern const IsForwardLayout kLayout__458;
|
||||
extern const MemoryWriteLayout kLayout__457;
|
||||
extern const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__456;
|
||||
extern const IsCycleLayout kLayout__463;
|
||||
extern const IsForwardLayout kLayout__462;
|
||||
extern const MemoryWriteLayout kLayout__461;
|
||||
extern const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__460;
|
||||
extern const MemoryWriteLayout kLayout__465;
|
||||
extern const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__464;
|
||||
extern const IsCycleLayout kLayout__469;
|
||||
extern const IsForwardLayout kLayout__468;
|
||||
extern const MemoryWriteLayout kLayout__467;
|
||||
extern const ControlResume_SuperArm1_Super__0_SuperLayout kLayout__466;
|
||||
extern const ControlResume_SuperArm1_Super__0_SuperLayout8LayoutArray kLayout__445;
|
||||
extern const ControlResume_SuperArm1_SuperLayout kLayout__444;
|
||||
extern const ControlResume_SuperLayout kLayout__435;
|
||||
extern const MemoryArgLayout16LayoutArray kLayout__471;
|
||||
extern const CycleArgLayout8LayoutArray kLayout__472;
|
||||
extern const _Arguments_ControlResume_SuperLayout kLayout__470;
|
||||
extern const ControlResumeLayout kLayout__434;
|
||||
extern const Control0_SuperArm1Layout kLayout__433;
|
||||
extern const NondetU16RegLayout kLayout__477;
|
||||
extern const U16RegLayout kLayout__476;
|
||||
extern const NondetU16RegLayout kLayout__478;
|
||||
extern const AddrDecomposeBitsLayout kLayout__475;
|
||||
extern const NondetU16RegLayout kLayout__480;
|
||||
extern const U16RegLayout kLayout__479;
|
||||
extern const MemoryReadLayout kLayout__481;
|
||||
extern const ControlUserECALLLayout kLayout__474;
|
||||
extern const Control0_SuperArm2Layout kLayout__473;
|
||||
extern const NondetU16RegLayout kLayout__485;
|
||||
extern const NormalizeU32Layout kLayout__484;
|
||||
extern const ControlMRETLayout kLayout__483;
|
||||
extern const Control0_SuperArm3Layout kLayout__482;
|
||||
extern const MemoryReadLayout kLayout__491;
|
||||
extern const MemoryReadLayout kLayout__492;
|
||||
extern const MemoryReadLayout kLayout__493;
|
||||
extern const MemoryReadLayout kLayout__494;
|
||||
extern const MemoryReadLayout kLayout__495;
|
||||
extern const MemoryReadLayout8LayoutArray kLayout__490;
|
||||
extern const ControlSuspend_SuperArm0_SuperLayout kLayout__489;
|
||||
extern const ControlSuspend_SuperArm1_SuperLayout kLayout__497;
|
||||
extern const ControlSuspend_SuperArm1Layout kLayout__496;
|
||||
extern const ControlSuspend_SuperLayout kLayout__488;
|
||||
extern const _Arguments_ControlSuspend_SuperLayout kLayout__498;
|
||||
extern const ControlSuspendLayout kLayout__487;
|
||||
extern const Control0_SuperArm4Layout kLayout__486;
|
||||
extern const MemoryPageOutLayout kLayout__502;
|
||||
extern const MemoryPageOutLayout kLayout__503;
|
||||
extern const MemoryPageOutLayout kLayout__504;
|
||||
extern const MemoryPageOutLayout kLayout__505;
|
||||
extern const MemoryPageOutLayout kLayout__506;
|
||||
extern const MemoryPageOutLayout kLayout__507;
|
||||
extern const MemoryPageOutLayout kLayout__508;
|
||||
extern const MemoryPageOutLayout kLayout__509;
|
||||
extern const MemoryPageOutLayout8LayoutArray kLayout__501;
|
||||
extern const ControlStoreRootLayout kLayout__500;
|
||||
extern const Control0_SuperArm5Layout kLayout__499;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__516;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__517;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__518;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__519;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__520;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__521;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__522;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__523;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__524;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__525;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__526;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__527;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__528;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__529;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__530;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout kLayout__531;
|
||||
extern const ControlTable_SuperArm0_Super__0_SuperLayout16LayoutArray kLayout__515;
|
||||
extern const ControlTable_SuperArm0_SuperLayout kLayout__514;
|
||||
extern const ControlTable_SuperArm0Layout kLayout__513;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__535;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__536;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__537;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__538;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__539;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__540;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__541;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__542;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__543;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__544;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__545;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__546;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__547;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__548;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__549;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout kLayout__550;
|
||||
extern const ControlTable_SuperArm1_Super__0_SuperLayout16LayoutArray kLayout__534;
|
||||
extern const ControlTable_SuperArm1_SuperLayout kLayout__533;
|
||||
extern const ControlTable_SuperArm1Layout kLayout__532;
|
||||
extern const ControlTable_SuperLayout kLayout__512;
|
||||
extern const ArgU16Layout16LayoutArray kLayout__552;
|
||||
extern const ArgU8Layout16LayoutArray kLayout__553;
|
||||
extern const _Arguments_ControlTable_SuperLayout kLayout__551;
|
||||
extern const ControlTableLayout kLayout__511;
|
||||
extern const Control0_SuperArm6Layout kLayout__510;
|
||||
extern const Control0_SuperArm7Layout kLayout__554;
|
||||
extern const Control0_SuperLayout kLayout__389;
|
||||
extern const _Arguments_Control0_SuperLayout kLayout__555;
|
||||
extern const Control0Layout kLayout__388;
|
||||
extern const NondetU16RegLayout kLayout__559;
|
||||
extern const U16RegLayout kLayout__558;
|
||||
extern const AddrDecomposeBitsLayout kLayout__557;
|
||||
extern const MemoryArgLayout8LayoutArray kLayout__561;
|
||||
extern const CycleArgLayout4LayoutArray kLayout__562;
|
||||
extern const ArgU16Layout2LayoutArray kLayout__563;
|
||||
extern const _Arguments_ECall0OutputLayout kLayout__560;
|
||||
extern const IsCycleLayout kLayout__569;
|
||||
extern const IsForwardLayout kLayout__568;
|
||||
extern const MemoryReadLayout kLayout__567;
|
||||
extern const IsCycleLayout kLayout__572;
|
||||
extern const IsForwardLayout kLayout__571;
|
||||
extern const MemoryReadLayout kLayout__570;
|
||||
extern const NondetRegLayout4LayoutArray kLayout__574;
|
||||
extern const OneHot_4_Layout kLayout__573;
|
||||
extern const MachineECallLayout kLayout__566;
|
||||
extern const ECall0OutputArm0Layout kLayout__565;
|
||||
extern const ECallTerminateLayout kLayout__576;
|
||||
extern const ECall0OutputArm1Layout kLayout__575;
|
||||
extern const IsCycleLayout kLayout__580;
|
||||
extern const IsForwardLayout kLayout__579;
|
||||
extern const MemoryReadLayout kLayout__578;
|
||||
extern const NondetU16RegLayout kLayout__581;
|
||||
extern const NondetU16RegLayout kLayout__583;
|
||||
extern const U16RegLayout kLayout__582;
|
||||
extern const IsCycleLayout kLayout__586;
|
||||
extern const IsForwardLayout kLayout__585;
|
||||
extern const MemoryWriteLayout kLayout__584;
|
||||
extern const NondetRegLayout4LayoutArray kLayout__589;
|
||||
extern const OneHot_4_Layout kLayout__588;
|
||||
extern const DecomposeLow2Layout kLayout__587;
|
||||
extern const NondetRegLayout4LayoutArray kLayout__592;
|
||||
extern const OneHot_4_Layout kLayout__591;
|
||||
extern const DecomposeLow2Layout kLayout__590;
|
||||
extern const ECallHostReadSetupLayout kLayout__577;
|
||||
extern const ECallHostWriteLayout kLayout__593;
|
||||
extern const ECall0OutputArm4Layout kLayout__594;
|
||||
extern const MemoryWriteUnconstrainedLayout kLayout__599;
|
||||
extern const ECallHostReadWords__0_SuperLayout kLayout__598;
|
||||
extern const MemoryWriteUnconstrainedLayout kLayout__601;
|
||||
extern const ECallHostReadWords__0_SuperLayout kLayout__600;
|
||||
extern const MemoryWriteUnconstrainedLayout kLayout__603;
|
||||
extern const ECallHostReadWords__0_SuperLayout kLayout__602;
|
||||
extern const MemoryWriteUnconstrainedLayout kLayout__605;
|
||||
extern const ECallHostReadWords__0_SuperLayout kLayout__604;
|
||||
extern const ECallHostReadWords__0_SuperLayout4LayoutArray kLayout__597;
|
||||
extern const ECallHostReadWordsLayout kLayout__596;
|
||||
extern const ECall0OutputArm5Layout kLayout__595;
|
||||
extern const ECall0OutputArm6Layout kLayout__606;
|
||||
extern const ECall0OutputArm7Layout kLayout__607;
|
||||
extern const ECall0OutputLayout kLayout__564;
|
||||
extern const NondetU16RegLayout kLayout__609;
|
||||
extern const NormalizeU32Layout kLayout__608;
|
||||
extern const ECall0Layout kLayout__556;
|
||||
extern const NondetRegLayout24LayoutArray kLayout__612;
|
||||
extern const PoseidonStateLayout kLayout__611;
|
||||
extern const MemoryArgLayout kLayout__615;
|
||||
extern const MemoryArgLayout kLayout__616;
|
||||
extern const MemoryArgLayout kLayout__617;
|
||||
extern const MemoryArgLayout kLayout__618;
|
||||
extern const MemoryArgLayout kLayout__619;
|
||||
extern const MemoryArgLayout kLayout__620;
|
||||
extern const MemoryArgLayout kLayout__621;
|
||||
extern const MemoryArgLayout kLayout__622;
|
||||
extern const MemoryArgLayout kLayout__623;
|
||||
extern const MemoryArgLayout kLayout__624;
|
||||
extern const MemoryArgLayout kLayout__625;
|
||||
extern const MemoryArgLayout kLayout__626;
|
||||
extern const MemoryArgLayout kLayout__627;
|
||||
extern const MemoryArgLayout kLayout__628;
|
||||
extern const MemoryArgLayout kLayout__629;
|
||||
extern const MemoryArgLayout kLayout__630;
|
||||
extern const MemoryArgLayout16LayoutArray kLayout__614;
|
||||
extern const CycleArgLayout8LayoutArray kLayout__631;
|
||||
extern const ArgU16Layout16LayoutArray kLayout__632;
|
||||
extern const ArgU8Layout2LayoutArray kLayout__633;
|
||||
extern const _Arguments_Poseidon0StateLayout kLayout__613;
|
||||
extern const PoseidonEntry_SuperArm0Layout kLayout__638;
|
||||
extern const MemoryIOLayout kLayout__642;
|
||||
extern const IsCycleLayout kLayout__644;
|
||||
extern const IsForwardLayout kLayout__643;
|
||||
extern const MemoryReadLayout kLayout__641;
|
||||
extern const ReadAddrLayout kLayout__640;
|
||||
extern const MemoryIOLayout kLayout__647;
|
||||
extern const IsCycleLayout kLayout__649;
|
||||
extern const IsForwardLayout kLayout__648;
|
||||
extern const MemoryReadLayout kLayout__646;
|
||||
extern const ReadAddrLayout kLayout__645;
|
||||
extern const MemoryIOLayout kLayout__652;
|
||||
extern const IsCycleLayout kLayout__654;
|
||||
extern const IsForwardLayout kLayout__653;
|
||||
extern const MemoryReadLayout kLayout__651;
|
||||
extern const ReadAddrLayout kLayout__650;
|
||||
extern const MemoryIOLayout kLayout__656;
|
||||
extern const IsCycleLayout kLayout__658;
|
||||
extern const IsForwardLayout kLayout__657;
|
||||
extern const MemoryReadLayout kLayout__655;
|
||||
extern const PoseidonEcallLayout kLayout__639;
|
||||
extern const PoseidonEntry_SuperLayout kLayout__637;
|
||||
extern const MemoryArgLayout8LayoutArray kLayout__660;
|
||||
extern const CycleArgLayout4LayoutArray kLayout__661;
|
||||
extern const _Arguments_PoseidonEntry_SuperLayout kLayout__659;
|
||||
extern const PoseidonEntryLayout kLayout__636;
|
||||
extern const Poseidon0StateArm0Layout kLayout__635;
|
||||
extern const ReadElemLayout kLayout__665;
|
||||
extern const ReadElemLayout kLayout__666;
|
||||
extern const ReadElemLayout kLayout__667;
|
||||
extern const ReadElemLayout kLayout__668;
|
||||
extern const MemoryIOLayout kLayout__671;
|
||||
extern const IsCycleLayout kLayout__673;
|
||||
extern const IsForwardLayout kLayout__672;
|
||||
extern const MemoryReadLayout kLayout__670;
|
||||
extern const ReadElemLayout kLayout__669;
|
||||
extern const MemoryIOLayout kLayout__676;
|
||||
extern const IsCycleLayout kLayout__678;
|
||||
extern const IsForwardLayout kLayout__677;
|
||||
extern const MemoryReadLayout kLayout__675;
|
||||
extern const ReadElemLayout kLayout__674;
|
||||
extern const MemoryIOLayout kLayout__681;
|
||||
extern const IsCycleLayout kLayout__683;
|
||||
extern const IsForwardLayout kLayout__682;
|
||||
extern const MemoryReadLayout kLayout__680;
|
||||
extern const ReadElemLayout kLayout__679;
|
||||
extern const MemoryIOLayout kLayout__686;
|
||||
extern const IsCycleLayout kLayout__688;
|
||||
extern const IsForwardLayout kLayout__687;
|
||||
extern const MemoryReadLayout kLayout__685;
|
||||
extern const ReadElemLayout kLayout__684;
|
||||
extern const ReadElemLayout8LayoutArray kLayout__664;
|
||||
extern const PoseidonLoadStateLayout kLayout__663;
|
||||
extern const Poseidon0StateArm1Layout kLayout__662;
|
||||
extern const OneHot_3_Layout kLayout__693;
|
||||
extern const MemoryPageInLayout kLayout__698;
|
||||
extern const MemoryGet_SuperArm1Layout kLayout__697;
|
||||
extern const MemoryPageOutLayout kLayout__699;
|
||||
extern const MemoryGet_SuperLayout kLayout__696;
|
||||
extern const MemoryArgLayout2LayoutArray kLayout__701;
|
||||
extern const _Arguments_MemoryGet_SuperLayout kLayout__700;
|
||||
extern const MemoryGetLayout kLayout__695;
|
||||
extern const MemoryPageInLayout kLayout__705;
|
||||
extern const MemoryGet_SuperArm1Layout kLayout__704;
|
||||
extern const MemoryPageOutLayout kLayout__706;
|
||||
extern const MemoryGet_SuperLayout kLayout__703;
|
||||
extern const MemoryArgLayout2LayoutArray kLayout__708;
|
||||
extern const _Arguments_MemoryGet_SuperLayout kLayout__707;
|
||||
extern const MemoryGetLayout kLayout__702;
|
||||
extern const MemoryPageInLayout kLayout__712;
|
||||
extern const MemoryGet_SuperArm1Layout kLayout__711;
|
||||
extern const MemoryPageOutLayout kLayout__713;
|
||||
extern const MemoryGet_SuperLayout kLayout__710;
|
||||
extern const MemoryArgLayout2LayoutArray kLayout__715;
|
||||
extern const _Arguments_MemoryGet_SuperLayout kLayout__714;
|
||||
extern const MemoryGetLayout kLayout__709;
|
||||
extern const MemoryPageInLayout kLayout__719;
|
||||
extern const MemoryGet_SuperArm1Layout kLayout__718;
|
||||
extern const MemoryPageOutLayout kLayout__720;
|
||||
extern const MemoryGet_SuperLayout kLayout__717;
|
||||
extern const MemoryArgLayout2LayoutArray kLayout__722;
|
||||
extern const _Arguments_MemoryGet_SuperLayout kLayout__721;
|
||||
extern const MemoryGetLayout kLayout__716;
|
||||
extern const MemoryPageInLayout kLayout__726;
|
||||
extern const MemoryGet_SuperArm1Layout kLayout__725;
|
||||
extern const MemoryPageOutLayout kLayout__727;
|
||||
extern const MemoryGet_SuperLayout kLayout__724;
|
||||
extern const MemoryArgLayout2LayoutArray kLayout__729;
|
||||
extern const _Arguments_MemoryGet_SuperLayout kLayout__728;
|
||||
extern const MemoryGetLayout kLayout__723;
|
||||
extern const MemoryPageInLayout kLayout__733;
|
||||
extern const MemoryGet_SuperArm1Layout kLayout__732;
|
||||
extern const MemoryPageOutLayout kLayout__734;
|
||||
extern const MemoryGet_SuperLayout kLayout__731;
|
||||
extern const MemoryArgLayout2LayoutArray kLayout__736;
|
||||
extern const _Arguments_MemoryGet_SuperLayout kLayout__735;
|
||||
extern const MemoryGetLayout kLayout__730;
|
||||
extern const MemoryPageInLayout kLayout__740;
|
||||
extern const MemoryGet_SuperArm1Layout kLayout__739;
|
||||
extern const MemoryPageOutLayout kLayout__741;
|
||||
extern const MemoryGet_SuperLayout kLayout__738;
|
||||
extern const MemoryArgLayout2LayoutArray kLayout__743;
|
||||
extern const _Arguments_MemoryGet_SuperLayout kLayout__742;
|
||||
extern const MemoryGetLayout kLayout__737;
|
||||
extern const MemoryPageInLayout kLayout__747;
|
||||
extern const MemoryGet_SuperArm1Layout kLayout__746;
|
||||
extern const MemoryPageOutLayout kLayout__748;
|
||||
extern const MemoryGet_SuperLayout kLayout__745;
|
||||
extern const MemoryArgLayout2LayoutArray kLayout__750;
|
||||
extern const _Arguments_MemoryGet_SuperLayout kLayout__749;
|
||||
extern const MemoryGetLayout kLayout__744;
|
||||
extern const MemoryGetLayout8LayoutArray kLayout__694;
|
||||
extern const PoseidonLoadInShortLayout kLayout__692;
|
||||
extern const PoseidonLoadInLowLayout kLayout__751;
|
||||
extern const PoseidonLoadInHighLayout kLayout__752;
|
||||
extern const PoseidonLoadIn_SuperLayout kLayout__691;
|
||||
extern const OneHot_3_Layout kLayout__753;
|
||||
extern const _Arguments_PoseidonLoadIn_SuperLayout kLayout__754;
|
||||
extern const PoseidonLoadInLayout kLayout__690;
|
||||
extern const Poseidon0StateArm2Layout kLayout__689;
|
||||
extern const Poseidon0StateArm3Layout kLayout__755;
|
||||
extern const Poseidon0StateArm4Layout kLayout__756;
|
||||
extern const PoseidonCheckOut__0_SuperLayout kLayout__763;
|
||||
extern const PoseidonCheckOut__0_SuperLayout kLayout__764;
|
||||
extern const PoseidonCheckOut__0_SuperLayout kLayout__765;
|
||||
extern const PoseidonCheckOut__0_SuperLayout kLayout__766;
|
||||
extern const PoseidonCheckOut__0_SuperLayout kLayout__767;
|
||||
extern const PoseidonCheckOut__0_SuperLayout kLayout__768;
|
||||
extern const PoseidonCheckOut__0_SuperLayout kLayout__769;
|
||||
extern const PoseidonCheckOut__0_SuperLayout kLayout__770;
|
||||
extern const PoseidonCheckOut__0_SuperLayout8LayoutArray kLayout__762;
|
||||
extern const PoseidonCheckOutLayout kLayout__761;
|
||||
extern const PoseidonDoOut_SuperArm0Layout kLayout__760;
|
||||
extern const NondetU16RegLayout kLayout__774;
|
||||
extern const NondetU16RegLayout kLayout__776;
|
||||
extern const U16RegLayout kLayout__775;
|
||||
extern const MemoryWriteLayout kLayout__777;
|
||||
extern const PoseidonStoreOut__0_SuperLayout kLayout__773;
|
||||
extern const NondetU16RegLayout kLayout__779;
|
||||
extern const NondetU16RegLayout kLayout__781;
|
||||
extern const U16RegLayout kLayout__780;
|
||||
extern const MemoryWriteLayout kLayout__782;
|
||||
extern const PoseidonStoreOut__0_SuperLayout kLayout__778;
|
||||
extern const NondetU16RegLayout kLayout__784;
|
||||
extern const U16RegLayout kLayout__785;
|
||||
extern const MemoryWriteLayout kLayout__786;
|
||||
extern const PoseidonStoreOut__0_SuperLayout kLayout__783;
|
||||
extern const NondetU16RegLayout kLayout__788;
|
||||
extern const NondetU16RegLayout kLayout__790;
|
||||
extern const U16RegLayout kLayout__789;
|
||||
extern const MemoryWriteLayout kLayout__791;
|
||||
extern const PoseidonStoreOut__0_SuperLayout kLayout__787;
|
||||
extern const NondetU16RegLayout kLayout__793;
|
||||
extern const NondetU16RegLayout kLayout__795;
|
||||
extern const U16RegLayout kLayout__794;
|
||||
extern const MemoryWriteLayout kLayout__796;
|
||||
extern const PoseidonStoreOut__0_SuperLayout kLayout__792;
|
||||
extern const NondetU16RegLayout kLayout__799;
|
||||
extern const U16RegLayout kLayout__798;
|
||||
extern const MemoryWriteLayout kLayout__800;
|
||||
extern const PoseidonStoreOut__0_SuperLayout kLayout__797;
|
||||
extern const NondetU16RegLayout kLayout__802;
|
||||
extern const NondetU16RegLayout kLayout__804;
|
||||
extern const U16RegLayout kLayout__803;
|
||||
extern const MemoryWriteLayout kLayout__805;
|
||||
extern const PoseidonStoreOut__0_SuperLayout kLayout__801;
|
||||
extern const NondetU16RegLayout kLayout__807;
|
||||
extern const NondetU16RegLayout kLayout__809;
|
||||
extern const U16RegLayout kLayout__808;
|
||||
extern const MemoryWriteLayout kLayout__810;
|
||||
extern const PoseidonStoreOut__0_SuperLayout kLayout__806;
|
||||
extern const PoseidonStoreOut__0_SuperLayout8LayoutArray kLayout__772;
|
||||
extern const PoseidonStoreOutLayout kLayout__771;
|
||||
extern const PoseidonDoOut_SuperLayout kLayout__759;
|
||||
extern const _Arguments_PoseidonDoOut_SuperLayout kLayout__811;
|
||||
extern const PoseidonDoOutLayout kLayout__758;
|
||||
extern const Poseidon0StateArm5Layout kLayout__757;
|
||||
extern const PoseidonPaging_SuperLayout kLayout__814;
|
||||
extern const NondetRegLayout6LayoutArray kLayout__816;
|
||||
extern const OneHot_6_Layout kLayout__815;
|
||||
extern const NondetU8RegLayout kLayout__819;
|
||||
extern const U8RegLayout kLayout__818;
|
||||
extern const IsU24Layout kLayout__817;
|
||||
extern const _Arguments_PoseidonPaging__1Layout kLayout__820;
|
||||
extern const NondetU8RegLayout kLayout__825;
|
||||
extern const U8RegLayout kLayout__824;
|
||||
extern const IsU24Layout kLayout__823;
|
||||
extern const PoseidonPaging__1Arm0_SuperLayout kLayout__822;
|
||||
extern const PoseidonPaging__1Arm1_SuperLayout kLayout__826;
|
||||
extern const PoseidonPaging__1Layout kLayout__821;
|
||||
extern const PoseidonPagingLayout kLayout__813;
|
||||
extern const Poseidon0StateArm6Layout kLayout__812;
|
||||
extern const PoseidonStoreState__0_SuperLayout kLayout__830;
|
||||
extern const PoseidonStoreState__0_SuperLayout kLayout__831;
|
||||
extern const PoseidonStoreState__0_SuperLayout kLayout__832;
|
||||
extern const PoseidonStoreState__0_SuperLayout kLayout__833;
|
||||
extern const PoseidonStoreState__0_SuperLayout kLayout__834;
|
||||
extern const PoseidonStoreState__0_SuperLayout kLayout__835;
|
||||
extern const PoseidonStoreState__0_SuperLayout kLayout__836;
|
||||
extern const PoseidonStoreState__0_SuperLayout kLayout__837;
|
||||
extern const PoseidonStoreState__0_SuperLayout8LayoutArray kLayout__829;
|
||||
extern const PoseidonStoreStateLayout kLayout__828;
|
||||
extern const Poseidon0StateArm7Layout kLayout__827;
|
||||
extern const Poseidon0StateLayout kLayout__634;
|
||||
extern const Poseidon0Layout kLayout__610;
|
||||
extern const SBoxLayout24LayoutArray kLayout__843;
|
||||
extern const DoExtRoundLayout kLayout__842;
|
||||
extern const NondetRegLayout8LayoutArray kLayout__845;
|
||||
extern const OneHot_8_Layout kLayout__844;
|
||||
extern const DoExtRoundByIdxLayout kLayout__841;
|
||||
extern const PoseidonExtRoundLayout kLayout__840;
|
||||
extern const DoIntRoundLayout kLayout__849;
|
||||
extern const DoIntRoundLayout kLayout__850;
|
||||
extern const DoIntRoundLayout kLayout__851;
|
||||
extern const DoIntRoundLayout kLayout__852;
|
||||
extern const DoIntRoundLayout kLayout__853;
|
||||
extern const DoIntRoundLayout kLayout__854;
|
||||
extern const DoIntRoundLayout kLayout__855;
|
||||
extern const DoIntRoundLayout kLayout__856;
|
||||
extern const DoIntRoundLayout kLayout__857;
|
||||
extern const DoIntRoundLayout kLayout__858;
|
||||
extern const DoIntRoundLayout kLayout__859;
|
||||
extern const DoIntRoundLayout kLayout__860;
|
||||
extern const DoIntRoundLayout kLayout__861;
|
||||
extern const DoIntRoundLayout kLayout__862;
|
||||
extern const DoIntRoundLayout kLayout__863;
|
||||
extern const DoIntRoundLayout kLayout__864;
|
||||
extern const DoIntRoundLayout kLayout__865;
|
||||
extern const DoIntRoundLayout kLayout__866;
|
||||
extern const DoIntRoundLayout kLayout__867;
|
||||
extern const DoIntRoundLayout kLayout__868;
|
||||
extern const DoIntRoundLayout kLayout__869;
|
||||
extern const DoIntRoundLayout21LayoutArray kLayout__848;
|
||||
extern const DoIntRoundsLayout kLayout__847;
|
||||
extern const PoseidonIntRoundsLayout kLayout__846;
|
||||
extern const Poseidon1StateLayout kLayout__839;
|
||||
extern const Poseidon1Layout kLayout__838;
|
||||
extern const TopInstResultLayout kLayout__6;
|
||||
extern const TopLayout kLayout__0;
|
||||
extern const DigestRegValues_SuperLayout8LayoutArray kLayout__871;
|
||||
extern const DigestRegLayout kLayout__870;
|
||||
extern const DigestRegValues_SuperLayout8LayoutArray kLayout__873;
|
||||
extern const DigestRegLayout kLayout__872;
|
||||
extern const DigestRegValues_SuperLayout8LayoutArray kLayout__875;
|
||||
extern const DigestRegLayout kLayout__874;
|
||||
extern const DigestRegValues_SuperLayout8LayoutArray kLayout__877;
|
||||
extern const DigestRegLayout kLayout__876;
|
||||
extern const _accumLayout kLayout__878;
|
||||
extern const LayoutAccumLayout kLayoutTestSuccRunAccum;
|
||||
extern const LayoutAccumLayout kLayout_TopAccum;
|
||||
extern const TestSuccRunLayout kLayoutTestSuccRun;
|
||||
extern const TopLayout kLayout_Top;
|
||||
extern const _globalLayout kLayoutGlobal;
|
||||
extern const _mixLayout kLayoutMix;
|
||||
@@ -0,0 +1,49 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace risc0::circuit::rv32im_v2::cpu {
|
||||
|
||||
struct MemoryTransaction {
|
||||
uint32_t addr;
|
||||
uint32_t cycle;
|
||||
uint32_t word;
|
||||
uint32_t prevCycle;
|
||||
uint32_t prevWord;
|
||||
};
|
||||
|
||||
struct PreflightCycle {
|
||||
uint32_t state;
|
||||
uint32_t pc;
|
||||
uint8_t major;
|
||||
uint8_t minor;
|
||||
uint8_t machineMode;
|
||||
uint8_t padding;
|
||||
uint32_t userCycle;
|
||||
uint32_t txnIdx;
|
||||
uint32_t pagingIdx;
|
||||
uint32_t diffCount;
|
||||
};
|
||||
|
||||
struct PreflightTrace {
|
||||
PreflightCycle* cycles;
|
||||
MemoryTransaction* txns;
|
||||
uint32_t txnsLen;
|
||||
uint32_t tableSplitCycle;
|
||||
};
|
||||
|
||||
} // namespace risc0::circuit::rv32im_v2::cpu
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,190 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "buffers.h"
|
||||
#include "fp.h"
|
||||
#include "fpext.h"
|
||||
#include "witgen.h"
|
||||
|
||||
namespace risc0::circuit::rv32im_v2::cpu {
|
||||
|
||||
extern NondetRegStruct back_NondetReg(ExecContext& ctx,Index distance0, BoundLayout<NondetRegLayout> layout1) ;
|
||||
extern NondetRegStruct exec_NondetReg(ExecContext& ctx,Val arg0, BoundLayout<NondetRegLayout> layout1) ;
|
||||
extern NondetExtRegStruct back_NondetExtReg(ExecContext& ctx,Index distance0, BoundLayout<NondetExtRegLayout> layout1) ;
|
||||
extern NondetExtRegStruct exec_NondetExtReg(ExecContext& ctx,ExtVal arg0, BoundLayout<NondetExtRegLayout> layout1) ;
|
||||
extern RegStruct back_Reg(ExecContext& ctx,Index distance0, BoundLayout<NondetRegLayout> layout1) ;
|
||||
extern RegStruct exec_Reg(ExecContext& ctx,Val arg0, BoundLayout<NondetRegLayout> layout1) ;
|
||||
extern NondetExtRegStruct back_ExtReg(ExecContext& ctx,Index distance0, BoundLayout<NondetExtRegLayout> layout1) ;
|
||||
extern NondetExtRegStruct exec_ExtReg(ExecContext& ctx,ExtVal arg0, BoundLayout<NondetExtRegLayout> layout1) ;
|
||||
extern NondetRegStruct exec_NondetBitReg(ExecContext& ctx,Val arg0, BoundLayout<NondetRegLayout> layout1) ;
|
||||
extern BitRegStruct exec_BitReg(ExecContext& ctx,Val arg0, BoundLayout<NondetRegLayout> layout1) ;
|
||||
extern NondetRegStruct exec_NondetTwitReg(ExecContext& ctx,Val arg0, BoundLayout<NondetRegLayout> layout1) ;
|
||||
extern NondetFakeTwitRegStruct exec_NondetFakeTwitReg(ExecContext& ctx,Val arg0, BoundLayout<NondetFakeTwitRegLayout> layout1) ;
|
||||
extern FakeTwitRegStruct exec_FakeTwitReg(ExecContext& ctx,Val arg0, BoundLayout<NondetFakeTwitRegLayout> layout1) ;
|
||||
extern NondetRegStruct exec_IsZero(ExecContext& ctx,Val arg0, BoundLayout<IsZeroLayout> layout1) ;
|
||||
extern ArgU8Struct exec_ArgU8(ExecContext& ctx,Val arg0, Val arg1, BoundLayout<ArgU8Layout> layout2) ;
|
||||
extern NondetRegStruct exec_NondetU8Reg(ExecContext& ctx,Val arg0, BoundLayout<NondetU8RegLayout> layout1) ;
|
||||
extern U8RegStruct exec_U8Reg(ExecContext& ctx,Val arg0, BoundLayout<U8RegLayout> layout1) ;
|
||||
extern ArgU16Struct exec_ArgU16(ExecContext& ctx,Val arg0, Val arg1, BoundLayout<ArgU16Layout> layout2) ;
|
||||
extern NondetRegStruct exec_NondetU16Reg(ExecContext& ctx,Val arg0, BoundLayout<NondetU16RegLayout> layout1) ;
|
||||
extern U16RegStruct exec_U16Reg(ExecContext& ctx,Val arg0, BoundLayout<U16RegLayout> layout1) ;
|
||||
extern ToBits_5_Struct exec_ToBits_5_(ExecContext& ctx,Val arg0, BoundLayout<ToBits_5_Layout> layout1) ;
|
||||
extern ValU32Struct exec_DynPo2(ExecContext& ctx,Val arg0, BoundLayout<DynPo2Layout> layout1) ;
|
||||
extern NormalizeU32Struct exec_NormalizeU32(ExecContext& ctx,DenormedValU32Struct arg0, BoundLayout<NormalizeU32Layout> layout1) ;
|
||||
extern AddrDecomposeStruct exec_AddrDecompose(ExecContext& ctx,ValU32Struct arg0, Val arg1, BoundLayout<AddrDecomposeLayout> layout2) ;
|
||||
extern AddrDecomposeBitsStruct exec_AddrDecomposeBits(ExecContext& ctx,ValU32Struct arg0, Val arg1, BoundLayout<AddrDecomposeBitsLayout> layout2) ;
|
||||
extern CmpEqualStruct exec_CmpEqual(ExecContext& ctx,ValU32Struct arg0, ValU32Struct arg1, BoundLayout<CmpEqualLayout> layout2) ;
|
||||
extern CmpLessThanUnsignedStruct exec_CmpLessThanUnsigned(ExecContext& ctx,ValU32Struct arg0, ValU32Struct arg1, BoundLayout<CmpLessThanUnsignedLayout> layout2) ;
|
||||
extern NondetRegStruct exec_GetSignU32(ExecContext& ctx,ValU32Struct arg0, BoundLayout<GetSignU32Layout> layout1) ;
|
||||
extern CmpLessThanStruct exec_CmpLessThan(ExecContext& ctx,ValU32Struct arg0, ValU32Struct arg1, BoundLayout<CmpLessThanLayout> layout2) ;
|
||||
extern ToBits_16_Struct exec_ToBits_16_(ExecContext& ctx,Val arg0, BoundLayout<ToBits_16_Layout> layout1) ;
|
||||
extern FromBits_16_Struct exec_BitwiseAndU16(ExecContext& ctx,Val arg0, Val arg1, BoundLayout<BitwiseAndU16Layout> layout2) ;
|
||||
extern ValU32Struct exec_BitwiseAnd(ExecContext& ctx,ValU32Struct arg0, ValU32Struct arg1, BoundLayout<BitwiseAndLayout> layout2) ;
|
||||
extern ValU32Struct exec_BitwiseOr(ExecContext& ctx,ValU32Struct arg0, ValU32Struct arg1, BoundLayout<BitwiseOrLayout> layout2) ;
|
||||
extern ValU32Struct exec_BitwiseXor(ExecContext& ctx,ValU32Struct arg0, ValU32Struct arg1, BoundLayout<BitwiseXorLayout> layout2) ;
|
||||
extern DecoderStruct exec_Decoder(ExecContext& ctx,ValU32Struct arg0, BoundLayout<DecoderLayout> layout1) ;
|
||||
extern MemoryArgStruct exec_MemoryArg(ExecContext& ctx,Val arg0, Val arg1, Val arg2, ValU32Struct arg3, BoundLayout<MemoryArgLayout> layout4) ;
|
||||
extern CycleArgStruct exec_CycleArg(ExecContext& ctx,Val arg0, Val arg1, BoundLayout<CycleArgLayout> layout2) ;
|
||||
extern IsCycleStruct exec_IsCycle(ExecContext& ctx,Val arg0, BoundLayout<IsCycleLayout> layout1) ;
|
||||
extern MemoryIOStruct exec_MemoryIO(ExecContext& ctx,RegStruct arg0, Val arg1, BoundLayout<MemoryIOLayout> layout2) ;
|
||||
extern IsForwardStruct exec_IsForward(ExecContext& ctx,MemoryIOStruct arg0, BoundLayout<IsForwardLayout> layout1) ;
|
||||
extern GetDataStruct exec_MemoryRead(ExecContext& ctx,RegStruct arg0, Val arg1, BoundLayout<MemoryReadLayout> layout2) ;
|
||||
extern MemoryWriteStruct exec_MemoryWrite(ExecContext& ctx,RegStruct arg0, Val arg1, ValU32Struct arg2, BoundLayout<MemoryWriteLayout> layout3) ;
|
||||
extern MemoryWriteUnconstrainedStruct exec_MemoryWriteUnconstrained(ExecContext& ctx,RegStruct arg0, Val arg1, BoundLayout<MemoryWriteUnconstrainedLayout> layout2) ;
|
||||
extern GetDataStruct exec_MemoryPageIn(ExecContext& ctx,RegStruct arg0, Val arg1, BoundLayout<MemoryPageInLayout> layout2) ;
|
||||
extern GetDataStruct exec_MemoryPageOut(ExecContext& ctx,RegStruct arg0, Val arg1, BoundLayout<MemoryPageOutLayout> layout2) ;
|
||||
extern OneHot_3_Struct exec_OneHot_3_(ExecContext& ctx,Val arg0, BoundLayout<OneHot_3_Layout> layout1) ;
|
||||
extern GetDataStruct exec_MemoryGet(ExecContext& ctx,RegStruct arg0, Val arg1, OneHot_3_Struct arg2, BoundLayout<MemoryGetLayout> layout3) ;
|
||||
extern OneHot_8_Struct exec_OneHot_8_(ExecContext& ctx,Val arg0, BoundLayout<OneHot_8_Layout> layout1) ;
|
||||
extern InstInputStruct exec_InstInput(ExecContext& ctx,Val arg0, Val arg1, Val arg2, ValU32Struct arg3, Val arg4, Val arg5, BoundLayout<InstInputLayout> layout6) ;
|
||||
extern DecoderStruct exec_DecodeInst(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<DecodeInstLayout> layout2) ;
|
||||
extern GetDataStruct exec_ReadReg(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, Val arg2, BoundLayout<ReadRegLayout> layout3) ;
|
||||
extern WriteRdStruct exec_WriteRd(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, DecoderStruct arg2, Val arg3, ValU32Struct arg4, BoundLayout<WriteRdLayout> layout5) ;
|
||||
extern ExpandU32Struct exec_ExpandU32(ExecContext& ctx,ValU32Struct arg0, Val arg1, BoundLayout<ExpandU32Layout> layout2) ;
|
||||
extern SplitTotalStruct exec_SplitTotal(ExecContext& ctx,Val arg0, BoundLayout<SplitTotalLayout> layout1) ;
|
||||
extern MultiplyAccumulateStruct exec_MultiplyAccumulate(ExecContext& ctx,ValU32Struct arg0, ValU32Struct arg1, ValU32Struct arg2, MultiplySettingsStruct arg3, BoundLayout<MultiplyAccumulateLayout> layout4) ;
|
||||
extern DivInputStruct exec_DivInput(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<DivInputLayout> layout2) ;
|
||||
extern DivideReturnStruct exec_DoDiv(ExecContext& ctx,ValU32Struct arg0, ValU32Struct arg1, Val arg2, Val arg3, BoundLayout<DoDivLayout> layout4) ;
|
||||
extern ValU32Struct exec_OpSRL(ExecContext& ctx,DivInputStruct arg0, BoundLayout<OpSRLLayout> layout1) ;
|
||||
extern NondetRegStruct exec_TopBit(ExecContext& ctx,ValU32Struct arg0, BoundLayout<TopBitLayout> layout1) ;
|
||||
extern ValU32Struct exec_OpSRA(ExecContext& ctx,DivInputStruct arg0, BoundLayout<OpSRALayout> layout1) ;
|
||||
extern ValU32Struct exec_OpSRLI(ExecContext& ctx,DivInputStruct arg0, BoundLayout<OpSRLILayout> layout1) ;
|
||||
extern ValU32Struct exec_OpSRAI(ExecContext& ctx,DivInputStruct arg0, BoundLayout<OpSRAILayout> layout1) ;
|
||||
extern ValU32Struct exec_OpDIV(ExecContext& ctx,DivInputStruct arg0, BoundLayout<OpDIVLayout> layout1) ;
|
||||
extern ValU32Struct exec_OpDIVU(ExecContext& ctx,DivInputStruct arg0, BoundLayout<OpDIVULayout> layout1) ;
|
||||
extern ValU32Struct exec_OpREM(ExecContext& ctx,DivInputStruct arg0, BoundLayout<OpREMLayout> layout1) ;
|
||||
extern ValU32Struct exec_OpREMU(ExecContext& ctx,DivInputStruct arg0, BoundLayout<OpREMULayout> layout1) ;
|
||||
extern InstOutputStruct exec_Div0(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<Div0Layout> layout2) ;
|
||||
extern MiscInputStruct exec_MiscInput(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<MiscInputLayout> layout2) ;
|
||||
extern InstOutputStruct exec_FinalizeMisc(ExecContext& ctx,RegStruct arg0, MiscInputStruct arg1, MiscOutputStruct arg2, BoundLayout<FinalizeMiscLayout> layout3) ;
|
||||
extern MiscOutputStruct exec_OpXOR(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpXORLayout> layout1) ;
|
||||
extern MiscOutputStruct exec_OpOR(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpORLayout> layout1) ;
|
||||
extern MiscOutputStruct exec_OpAND(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpANDLayout> layout1) ;
|
||||
extern MiscOutputStruct exec_OpSLT(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpSLTLayout> layout1) ;
|
||||
extern MiscOutputStruct exec_OpSLTU(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpSLTULayout> layout1) ;
|
||||
extern InstOutputStruct exec_Misc0(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<Misc0Layout> layout2) ;
|
||||
extern MiscOutputStruct exec_OpXORI(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpXORILayout> layout1) ;
|
||||
extern MiscOutputStruct exec_OpORI(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpORILayout> layout1) ;
|
||||
extern MiscOutputStruct exec_OpANDI(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpANDILayout> layout1) ;
|
||||
extern MiscOutputStruct exec_OpSLTI(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpSLTILayout> layout1) ;
|
||||
extern MiscOutputStruct exec_OpSLTIU(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpSLTIULayout> layout1) ;
|
||||
extern MiscOutputStruct exec_OpBEQ(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpBEQLayout> layout1) ;
|
||||
extern MiscOutputStruct exec_OpBNE(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpBNELayout> layout1) ;
|
||||
extern MiscOutputStruct exec_OpBLT(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpBLTLayout> layout1) ;
|
||||
extern InstOutputStruct exec_Misc1(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<Misc1Layout> layout2) ;
|
||||
extern MiscOutputStruct exec_OpBGE(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpBGELayout> layout1) ;
|
||||
extern MiscOutputStruct exec_OpBLTU(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpBLTULayout> layout1) ;
|
||||
extern MiscOutputStruct exec_OpBGEU(ExecContext& ctx,MiscInputStruct arg0, BoundLayout<OpBGEULayout> layout1) ;
|
||||
extern InstOutputStruct exec_Misc2(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<Misc2Layout> layout2) ;
|
||||
extern MulInputStruct exec_MulInput(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<MulInputLayout> layout2) ;
|
||||
extern DoMulStruct exec_DoMul(ExecContext& ctx,ValU32Struct arg0, ValU32Struct arg1, Val arg2, Val arg3, BoundLayout<DoMulLayout> layout4) ;
|
||||
extern ValU32Struct exec_OpSLL(ExecContext& ctx,MulInputStruct arg0, BoundLayout<OpSLLLayout> layout1) ;
|
||||
extern ValU32Struct exec_OpSLLI(ExecContext& ctx,MulInputStruct arg0, BoundLayout<OpSLLILayout> layout1) ;
|
||||
extern ValU32Struct exec_OpMUL(ExecContext& ctx,MulInputStruct arg0, BoundLayout<OpMULLayout> layout1) ;
|
||||
extern ValU32Struct exec_OpMULH(ExecContext& ctx,MulInputStruct arg0, BoundLayout<OpMULHLayout> layout1) ;
|
||||
extern ValU32Struct exec_OpMULHSU(ExecContext& ctx,MulInputStruct arg0, BoundLayout<OpMULHSULayout> layout1) ;
|
||||
extern ValU32Struct exec_OpMULHU(ExecContext& ctx,MulInputStruct arg0, BoundLayout<OpMULHULayout> layout1) ;
|
||||
extern InstOutputStruct exec_Mul0(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<Mul0Layout> layout2) ;
|
||||
extern MemLoadInputStruct exec_MemLoadInput(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<MemLoadInputLayout> layout2) ;
|
||||
extern MemStoreInputStruct exec_MemStoreInput(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<MemStoreInputLayout> layout2) ;
|
||||
extern MemStoreFinalizeStruct exec_MemStoreFinalize(ExecContext& ctx,RegStruct arg0, MemStoreInputStruct arg1, ValU32Struct arg2, BoundLayout<MemStoreFinalizeLayout> layout3) ;
|
||||
extern SplitWordStruct exec_SplitWord(ExecContext& ctx,Val arg0, BoundLayout<SplitWordLayout> layout1) ;
|
||||
extern ValU32Struct exec_OpLB(ExecContext& ctx,MemLoadInputStruct arg0, BoundLayout<OpLBLayout> layout1) ;
|
||||
extern ValU32Struct exec_OpLH(ExecContext& ctx,MemLoadInputStruct arg0, BoundLayout<OpLHLayout> layout1) ;
|
||||
extern ValU32Struct exec_OpLBU(ExecContext& ctx,MemLoadInputStruct arg0, BoundLayout<OpLBULayout> layout1) ;
|
||||
extern InstOutputStruct exec_Mem0(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<Mem0Layout> layout2) ;
|
||||
extern ValU32Struct exec_OpSB(ExecContext& ctx,MemStoreInputStruct arg0, BoundLayout<OpSBLayout> layout1) ;
|
||||
extern InstOutputStruct exec_Mem1(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<Mem1Layout> layout2) ;
|
||||
extern DigestRegStruct back_DigestReg(ExecContext& ctx,Index distance0, BoundLayout<DigestRegLayout> layout1) ;
|
||||
extern DigestRegStruct exec_DigestReg(ExecContext& ctx,ValU32Struct8Array arg0, BoundLayout<DigestRegLayout> layout1) ;
|
||||
extern InstOutputStruct exec_ControlLoadRoot(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<ControlLoadRootLayout> layout2, GlobalBuf global3) ;
|
||||
extern InstOutputStruct exec_ControlResume(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<ControlResumeLayout> layout2, GlobalBuf global3) ;
|
||||
extern InstOutputStruct exec_ControlUserECALL(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<ControlUserECALLLayout> layout2) ;
|
||||
extern InstOutputStruct exec_ControlMRET(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<ControlMRETLayout> layout2) ;
|
||||
extern InstOutputStruct exec_ControlSuspend(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<ControlSuspendLayout> layout2, GlobalBuf global3) ;
|
||||
extern InstOutputStruct exec_ControlStoreRoot(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<ControlStoreRootLayout> layout2, GlobalBuf global3) ;
|
||||
extern InstOutputStruct exec_ControlTable(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<ControlTableLayout> layout2) ;
|
||||
extern InstOutputStruct exec_Control0(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<Control0Layout> layout2, GlobalBuf global3) ;
|
||||
extern OneHot_4_Struct exec_OneHot_4_(ExecContext& ctx,Val arg0, BoundLayout<OneHot_4_Layout> layout1) ;
|
||||
extern ECallOutputStruct exec_MachineECall(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, Val arg2, BoundLayout<MachineECallLayout> layout3) ;
|
||||
extern ECallOutputStruct exec_ECallTerminate(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<ECallTerminateLayout> layout2, GlobalBuf global3) ;
|
||||
extern DecomposeLow2Struct exec_DecomposeLow2(ExecContext& ctx,Val arg0, BoundLayout<DecomposeLow2Layout> layout1) ;
|
||||
extern ECallOutputStruct exec_ECallHostReadSetup(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<ECallHostReadSetupLayout> layout2) ;
|
||||
extern ECallOutputStruct exec_ECallHostWrite(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<ECallHostWriteLayout> layout2) ;
|
||||
extern ECallOutputStruct exec_ECallHostReadWords(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, Val arg2, Val arg3, BoundLayout<ECallHostReadWordsLayout> layout4) ;
|
||||
extern InstOutputStruct exec_ECall0(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<ECall0Layout> layout2, GlobalBuf global3) ;
|
||||
extern RegStruct exec_SBox(ExecContext& ctx,Val arg0, BoundLayout<SBoxLayout> layout1) ;
|
||||
extern MultiplyByMIntStruct exec_DoIntRound(ExecContext& ctx,Val24Array arg0, Val arg1, BoundLayout<DoIntRoundLayout> layout2) ;
|
||||
extern DoIntRoundsStruct exec_DoIntRounds(ExecContext& ctx,Val24Array arg0, BoundLayout<DoIntRoundsLayout> layout1) ;
|
||||
extern MultiplyByMExtStruct exec_DoExtRound(ExecContext& ctx,Val24Array arg0, Val24Array arg1, BoundLayout<DoExtRoundLayout> layout2) ;
|
||||
extern MultiplyByMExtStruct exec_DoExtRoundByIdx(ExecContext& ctx,Val24Array arg0, Val arg1, BoundLayout<DoExtRoundByIdxLayout> layout2) ;
|
||||
extern PoseidonStateStruct back_PoseidonState(ExecContext& ctx,Index distance0, BoundLayout<PoseidonStateLayout> layout1) ;
|
||||
extern PoseidonStateStruct exec_PoseidonState(ExecContext& ctx,PoseidonOpDefStruct arg0, Val arg1, Val arg2, Val arg3, Val arg4, Val arg5, Val24Array arg6, ExtVal arg7, BoundLayout<PoseidonStateLayout> layout8) ;
|
||||
extern PoseidonStateStruct exec_PoseidonInvalid(ExecContext& ctx,BoundLayout<PoseidonStateLayout> layout0) ;
|
||||
extern ReadAddrStruct exec_ReadAddr(ExecContext& ctx,RegStruct arg0, Val arg1, BoundLayout<ReadAddrLayout> layout2) ;
|
||||
extern PoseidonStateStruct exec_PoseidonEcall(ExecContext& ctx,RegStruct arg0, Val arg1, BoundLayout<PoseidonEcallLayout> layout2) ;
|
||||
extern PoseidonStateStruct exec_PoseidonPagingEntry(ExecContext& ctx,RegStruct arg0, Val arg1, BoundLayout<PoseidonStateLayout> layout2) ;
|
||||
extern PoseidonStateStruct exec_PoseidonEntry(ExecContext& ctx,RegStruct arg0, ValU32Struct arg1, Val arg2, BoundLayout<PoseidonEntryLayout> layout3) ;
|
||||
extern ReadElemStruct exec_ReadElem(ExecContext& ctx,RegStruct arg0, Val arg1, BoundLayout<ReadElemLayout> layout2) ;
|
||||
extern PoseidonStateStruct exec_PoseidonLoadState(ExecContext& ctx,RegStruct arg0, PoseidonStateStruct arg1, BoundLayout<PoseidonLoadStateLayout> layout2) ;
|
||||
extern PoseidonStateStruct exec_PoseidonLoadInShort(ExecContext& ctx,RegStruct arg0, PoseidonStateStruct arg1, BoundLayout<PoseidonLoadInShortLayout> layout2, GlobalBuf global3) ;
|
||||
extern PoseidonStateStruct exec_PoseidonLoadInLow(ExecContext& ctx,RegStruct arg0, PoseidonStateStruct arg1, BoundLayout<PoseidonLoadInLowLayout> layout2, GlobalBuf global3) ;
|
||||
extern PoseidonStateStruct exec_PoseidonLoadInHigh(ExecContext& ctx,RegStruct arg0, PoseidonStateStruct arg1, BoundLayout<PoseidonLoadInHighLayout> layout2, GlobalBuf global3) ;
|
||||
extern PoseidonStateStruct exec_PoseidonLoadIn(ExecContext& ctx,RegStruct arg0, PoseidonStateStruct arg1, BoundLayout<PoseidonLoadInLayout> layout2, GlobalBuf global3) ;
|
||||
extern PoseidonStateStruct exec_PoseidonExtRound(ExecContext& ctx,PoseidonStateStruct arg0, BoundLayout<PoseidonExtRoundLayout> layout1) ;
|
||||
extern PoseidonStateStruct exec_PoseidonIntRounds(ExecContext& ctx,PoseidonStateStruct arg0, BoundLayout<PoseidonIntRoundsLayout> layout1) ;
|
||||
extern PoseidonStateStruct exec_PoseidonCheckOut(ExecContext& ctx,RegStruct arg0, PoseidonStateStruct arg1, BoundLayout<PoseidonCheckOutLayout> layout2) ;
|
||||
extern PoseidonStateStruct exec_PoseidonStoreOut(ExecContext& ctx,RegStruct arg0, PoseidonStateStruct arg1, BoundLayout<PoseidonStoreOutLayout> layout2) ;
|
||||
extern PoseidonStateStruct exec_PoseidonDoOut(ExecContext& ctx,RegStruct arg0, PoseidonStateStruct arg1, BoundLayout<PoseidonDoOutLayout> layout2) ;
|
||||
extern PoseidonStateStruct exec_PoseidonStoreState(ExecContext& ctx,RegStruct arg0, PoseidonStateStruct arg1, BoundLayout<PoseidonStoreStateLayout> layout2) ;
|
||||
extern IsU24Struct exec_IsU24(ExecContext& ctx,Val arg0, BoundLayout<IsU24Layout> layout1) ;
|
||||
extern PoseidonStateStruct exec_PoseidonPagingLoadNode(ExecContext& ctx,RegStruct arg0, Val arg1, BoundLayout<PoseidonStateLayout> layout2) ;
|
||||
extern PoseidonStateStruct exec_PoseidonPagingLoadPage(ExecContext& ctx,RegStruct arg0, Val arg1, BoundLayout<PoseidonStateLayout> layout2) ;
|
||||
extern PoseidonStateStruct exec_PoseidonPagingLoadDone(ExecContext& ctx,BoundLayout<PoseidonStateLayout> layout0) ;
|
||||
extern PoseidonStateStruct exec_PoseidonPagingStoreNode(ExecContext& ctx,RegStruct arg0, Val arg1, BoundLayout<PoseidonStateLayout> layout2) ;
|
||||
extern PoseidonStateStruct exec_PoseidonPagingStorePage(ExecContext& ctx,RegStruct arg0, Val arg1, BoundLayout<PoseidonStateLayout> layout2) ;
|
||||
extern PoseidonStateStruct exec_PoseidonPagingStoreDone(ExecContext& ctx,BoundLayout<PoseidonStateLayout> layout0) ;
|
||||
extern OneHot_6_Struct exec_OneHot_6_(ExecContext& ctx,Val arg0, BoundLayout<OneHot_6_Layout> layout1) ;
|
||||
extern PoseidonStateStruct exec_PoseidonPaging(ExecContext& ctx,RegStruct arg0, Val arg1, PoseidonStateStruct arg2, BoundLayout<PoseidonPagingLayout> layout3) ;
|
||||
extern InstOutputStruct exec_Poseidon0(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<Poseidon0Layout> layout2, GlobalBuf global3) ;
|
||||
extern InstOutputStruct exec_Poseidon1(ExecContext& ctx,RegStruct arg0, InstInputStruct arg1, BoundLayout<Poseidon1Layout> layout2) ;
|
||||
extern OneHot_11_Struct exec_OneHot_11_(ExecContext& ctx,Val arg0, BoundLayout<OneHot_11_Layout> layout1) ;
|
||||
extern TopStruct exec_Top(ExecContext& ctx,BoundLayout<TopLayout> layout0, GlobalBuf global1) ;
|
||||
extern void step_Top(ExecContext& ctx,MutableBuf data0, GlobalBuf global1) ;
|
||||
extern ComponentStruct exec_TopAccum(ExecContext& ctx,BoundLayout<TopLayout> arg0, BoundLayout<LayoutAccumLayout> layout1, GlobalBuf mix2) ;
|
||||
extern void step_TopAccum(ExecContext& ctx,MutableBuf accum0, MutableBuf data1, GlobalBuf mix2) ;
|
||||
|
||||
} // namespace risc0::circuit::rv32im_v2::cpu
|
||||
@@ -0,0 +1,69 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "fp.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <stdexcept>
|
||||
#include <vector>
|
||||
|
||||
namespace risc0::circuit::rv32im_v2::cpu {
|
||||
|
||||
struct LookupTables {
|
||||
std::vector<std::atomic_uint32_t> tableU8;
|
||||
std::vector<std::atomic_uint32_t> tableU16;
|
||||
|
||||
LookupTables() : tableU8(1 << 8), tableU16(1 << 16) {}
|
||||
|
||||
void lookupDelta(size_t cycle, Fp table, Fp index, Fp /*count*/) {
|
||||
uint32_t tableU32 = table.asUInt32();
|
||||
uint32_t indexU32 = index.asUInt32();
|
||||
if (tableU32 == 0) {
|
||||
// tableCycle[index] += count;
|
||||
return;
|
||||
}
|
||||
if (tableU32 != 8 && tableU32 != 16) {
|
||||
throw std::runtime_error("Invalid lookup table");
|
||||
}
|
||||
if (indexU32 >= (1u << tableU32)) {
|
||||
printf("[%lu]: LOOKUP ERROR: table = %u, index = %u\n", cycle, tableU32, indexU32);
|
||||
throw std::runtime_error("u8/16 table error");
|
||||
}
|
||||
// printf("table = %u, index = %u\n", tableU32, indexU32);
|
||||
if (tableU32 == 8) {
|
||||
tableU8[indexU32]++;
|
||||
} else {
|
||||
tableU16[indexU32]++;
|
||||
}
|
||||
}
|
||||
|
||||
Fp lookupCurrent(Fp table, Fp index) {
|
||||
uint32_t tableU32 = table.asUInt32();
|
||||
if (tableU32 != 8 && tableU32 != 16) {
|
||||
throw std::runtime_error("Invalid lookup table");
|
||||
}
|
||||
uint32_t indexU32 = index.asUInt32();
|
||||
if (tableU32 == 8) {
|
||||
return Fp(tableU8[indexU32]);
|
||||
} else {
|
||||
return Fp(tableU16[indexU32]);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace risc0::circuit::rv32im_v2::cpu
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,293 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "buffers.h"
|
||||
#include "fp.h"
|
||||
#include "fpext.h"
|
||||
#include "preflight.h"
|
||||
#include "tables.h"
|
||||
|
||||
#include <array>
|
||||
#include <cassert>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <functional>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <string.h>
|
||||
#include <vector>
|
||||
|
||||
namespace risc0::circuit::rv32im_v2::cpu {
|
||||
|
||||
struct ExecBuffers {
|
||||
Buffer global;
|
||||
Buffer data;
|
||||
};
|
||||
|
||||
struct AccumBuffers {
|
||||
Buffer data;
|
||||
Buffer accum;
|
||||
Buffer mix;
|
||||
};
|
||||
|
||||
#if defined(__clang__)
|
||||
#pragma clang diagnostic ignored "-Wunused-parameter"
|
||||
#pragma clang diagnostic ignored "-Wunused-variable"
|
||||
#elif defined(__GNUC__)
|
||||
#pragma GCC diagnostic ignored "-Wunused-parameter"
|
||||
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
|
||||
#endif
|
||||
|
||||
using Val = risc0::Fp;
|
||||
using ExtVal = risc0::FpExt;
|
||||
|
||||
inline size_t to_size_t(Val v) {
|
||||
return v.asUInt32();
|
||||
}
|
||||
|
||||
inline Val mod(Val a, Val b) {
|
||||
return Val(a.asUInt32() % b.asUInt32());
|
||||
}
|
||||
|
||||
constexpr size_t EXT_SIZE = 4;
|
||||
|
||||
// Built in field operations
|
||||
inline Val isz(Val x) {
|
||||
return Val(x == Val(0));
|
||||
}
|
||||
|
||||
inline Val neg_0(Val x) {
|
||||
return -x;
|
||||
}
|
||||
|
||||
inline Val inv_0(Val x) {
|
||||
return inv(x);
|
||||
}
|
||||
|
||||
inline ExtVal inv_0(ExtVal x) {
|
||||
return inv(x);
|
||||
}
|
||||
|
||||
inline Val bitAnd(Val a, Val b) {
|
||||
return Val(a.asUInt32() & b.asUInt32());
|
||||
}
|
||||
|
||||
inline Val inRange(Val low, Val mid, Val high) {
|
||||
assert(low <= high);
|
||||
return Val(low <= mid && mid < high);
|
||||
}
|
||||
|
||||
inline void eqz(Val a, const char* loc) {
|
||||
if (a.asUInt32()) {
|
||||
printf("eqz failure at: %s\n", loc);
|
||||
throw std::runtime_error("eqz failure");
|
||||
}
|
||||
}
|
||||
|
||||
inline void eqz(ExtVal a, const char* loc) {
|
||||
for (size_t i = 0; i < EXT_SIZE; i++) {
|
||||
eqz(a.elems[i], loc);
|
||||
}
|
||||
}
|
||||
|
||||
struct ExecContext {
|
||||
ExecContext(PreflightTrace& preflight, LookupTables& tables, size_t cycle)
|
||||
: preflight(preflight), tables(tables), cycle(cycle) {}
|
||||
PreflightTrace& preflight;
|
||||
LookupTables& tables;
|
||||
size_t cycle;
|
||||
};
|
||||
|
||||
// Define index type (used in back)
|
||||
using Index = size_t;
|
||||
|
||||
struct Reg {
|
||||
constexpr Reg(size_t col) : col(col) {}
|
||||
size_t col;
|
||||
};
|
||||
|
||||
struct BufferObj {
|
||||
virtual Val load(ExecContext& ctx, size_t col, size_t back) = 0;
|
||||
virtual void store(ExecContext& ctx, size_t col, Val val) = 0;
|
||||
};
|
||||
|
||||
struct MutableBufObj : public BufferObj {
|
||||
MutableBufObj(Buffer& buf) : buf(buf) {}
|
||||
|
||||
Val load(ExecContext& ctx, size_t col, size_t back) override {
|
||||
if (back > ctx.cycle) {
|
||||
return 0;
|
||||
}
|
||||
return buf.get(ctx.cycle - back, col);
|
||||
}
|
||||
|
||||
void store(ExecContext& ctx, size_t col, Val val) override {
|
||||
return buf.set(ctx.cycle, col, val);
|
||||
}
|
||||
|
||||
Buffer& buf;
|
||||
};
|
||||
|
||||
using MutableBuf = MutableBufObj*;
|
||||
|
||||
struct GlobalBufObj : public BufferObj {
|
||||
GlobalBufObj(Buffer& buf) : buf(buf) {}
|
||||
|
||||
Val load(ExecContext& ctx, size_t col, size_t back) override {
|
||||
assert(back == 0);
|
||||
return buf.get(0, col);
|
||||
}
|
||||
|
||||
void store(ExecContext& ctx, size_t col, Val val) override { return buf.set(0, col, val); }
|
||||
|
||||
Buffer& buf;
|
||||
};
|
||||
|
||||
using GlobalBuf = GlobalBufObj*;
|
||||
|
||||
template <typename T> struct BoundLayout {
|
||||
BoundLayout(const T& layout, BufferObj* buf) : layout(layout), buf(buf) {}
|
||||
BoundLayout() = default;
|
||||
BoundLayout(const BoundLayout&) = default;
|
||||
|
||||
const T& layout;
|
||||
BufferObj* buf = nullptr;
|
||||
};
|
||||
|
||||
#define BIND_LAYOUT(orig, buf) BoundLayout(orig, buf)
|
||||
#define LAYOUT_LOOKUP(orig, elem) BoundLayout(orig.layout.elem, orig.buf)
|
||||
#define LAYOUT_SUBSCRIPT(orig, index) BoundLayout(orig.layout[index], orig.buf)
|
||||
#define EQZ(val, loc) eqz(val, loc)
|
||||
|
||||
inline void store(ExecContext& ctx, BoundLayout<Reg> reg, Val val) {
|
||||
reg.buf->store(ctx, reg.layout.col, val);
|
||||
}
|
||||
|
||||
inline void set(ExecContext& ctx, BufferObj* buf, size_t offset, Val val) {
|
||||
static_cast<MutableBufObj*>(buf)->store(ctx, offset, val);
|
||||
}
|
||||
|
||||
inline void setGlobal(ExecContext& ctx, BufferObj* buf, size_t offset, Val val) {
|
||||
static_cast<GlobalBufObj*>(buf)->store(ctx, offset, val);
|
||||
}
|
||||
|
||||
inline void storeExt(ExecContext& ctx, BoundLayout<Reg> reg, ExtVal val) {
|
||||
for (size_t i = 0; i < EXT_SIZE; i++) {
|
||||
reg.buf->store(ctx, reg.layout.col + i, val.elems[i]);
|
||||
}
|
||||
}
|
||||
|
||||
inline Val load(ExecContext& ctx, BoundLayout<Reg> reg, size_t back) {
|
||||
return reg.buf->load(ctx, reg.layout.col, back);
|
||||
}
|
||||
|
||||
inline ExtVal loadExt(ExecContext& ctx, BoundLayout<Reg> reg, size_t back) {
|
||||
std::array<Fp, EXT_SIZE> elems;
|
||||
for (size_t i = 0; i < EXT_SIZE; i++) {
|
||||
elems[i] = reg.buf->load(ctx, reg.layout.col + i, back);
|
||||
}
|
||||
return FpExt(elems[0], elems[1], elems[2], elems[3]);
|
||||
}
|
||||
|
||||
inline Val get(ExecContext& ctx, BufferObj* buf, size_t offset, size_t back) {
|
||||
return static_cast<MutableBufObj*>(buf)->load(ctx, offset, back);
|
||||
}
|
||||
|
||||
inline Val getGlobal(ExecContext& ctx, BufferObj* buf, size_t offset) {
|
||||
return static_cast<GlobalBufObj*>(buf)->load(ctx, offset, 0);
|
||||
}
|
||||
|
||||
#define LOAD(reg, back) load(ctx, reg, back)
|
||||
#define LOAD_EXT(reg, back) loadExt(ctx, reg, back)
|
||||
#define STORE(reg, val) store(ctx, reg, val)
|
||||
#define STORE_EXT(reg, val) storeExt(ctx, reg, val)
|
||||
|
||||
// Map + reduce support
|
||||
template <typename T1, typename F, size_t N> inline auto map(std::array<T1, N> a, F f) {
|
||||
std::array<decltype(f(a[0])), N> out;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
out[i] = f(a[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T1, typename T2, typename F, size_t N>
|
||||
inline auto map(std::array<T1, N> a, std::array<T2, N> b, F f) {
|
||||
std::array<decltype(f(a[0], b[0])), N> out;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
out[i] = f(a[i], b[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T1, typename T2, typename F, size_t N>
|
||||
inline auto map(std::array<T1, N> a, const BoundLayout<T2>& b, F f) {
|
||||
std::array<decltype(f(a[0], BoundLayout(b.layout[0], b.buf))), N> out;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
out[i] = f(a[i], BoundLayout(b.layout[i], b.buf));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T1, typename T2, typename F, size_t N>
|
||||
inline auto reduce(std::array<T1, N> elems, T2 start, F f) {
|
||||
T2 cur = start;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
cur = f(cur, elems[i]);
|
||||
}
|
||||
return cur;
|
||||
}
|
||||
|
||||
template <typename T1, typename T2, typename T3, typename F, size_t N>
|
||||
inline auto reduce(std::array<T1, N> elems, T2 start, const BoundLayout<T3>& b, F f) {
|
||||
T2 cur = start;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
cur = f(cur, elems[i], BoundLayout(b.layout[i], b.buf));
|
||||
}
|
||||
return cur;
|
||||
}
|
||||
|
||||
// All the extern handling
|
||||
#define INVOKE_EXTERN(ctx, name, ...) extern_##name(ctx, ##__VA_ARGS__)
|
||||
|
||||
std::array<Val, 5> extern_getMemoryTxn(ExecContext& ctx, Val addrElem);
|
||||
void extern_lookupDelta(ExecContext& ctx, Val table, Val index, Val count);
|
||||
Val extern_lookupCurrent(ExecContext& ctx, Val table, Val index);
|
||||
void extern_memoryDelta(
|
||||
ExecContext& ctx, Val addr, Val cycle, Val dataLow, Val dataHigh, Val count);
|
||||
uint32_t extern_getDiffCount(ExecContext& ctx, Val cycle);
|
||||
Val extern_isFirstCycle_0(ExecContext& ctx);
|
||||
Val extern_getCycle(ExecContext& ctx);
|
||||
void extern_log(ExecContext& ctx, const std::string& message, std::vector<Val> vals);
|
||||
std::array<Val, 4> extern_divide(
|
||||
ExecContext& ctx, Val numerLow, Val numerHigh, Val denomLow, Val denomHigh, Val signType);
|
||||
void extern_print(ExecContext& ctx, Val v);
|
||||
std::array<Val, 2> extern_getMajorMinor(ExecContext& ctx);
|
||||
Val extern_hostReadPrepare(ExecContext& ctx, Val fp, Val len);
|
||||
Val extern_hostWrite(ExecContext& ctx, Val fdVal, Val addrLow, Val addrHigh, Val lenVal);
|
||||
std::array<Val, 2> extern_nextPagingIdx(ExecContext& ctx);
|
||||
|
||||
// Setup the basic field stuff
|
||||
#define SET_FIELD(x) /**/
|
||||
|
||||
#include "defs.cpp.inc"
|
||||
|
||||
#include "types.h.inc"
|
||||
|
||||
#include "layout.cpp.inc"
|
||||
|
||||
} // namespace risc0::circuit::rv32im_v2::cpu
|
||||
@@ -0,0 +1,129 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(feature = "cuda")]
|
||||
use cust::memory::DevicePointer;
|
||||
use derive_more::Debug;
|
||||
use risc0_core::field::baby_bear::{BabyBearElem, BabyBearExtElem};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
#[repr(C)]
|
||||
pub struct RawMemoryTransaction {
|
||||
#[debug("{addr:#010x}")]
|
||||
pub addr: u32,
|
||||
pub cycle: u32,
|
||||
#[debug("{word:#010x}")]
|
||||
pub word: u32,
|
||||
pub prev_cycle: u32,
|
||||
#[debug("{word:#010x}")]
|
||||
pub prev_word: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
#[repr(C)]
|
||||
pub struct RawPreflightCycle {
|
||||
pub state: u32,
|
||||
#[debug("{pc:#010x}")]
|
||||
pub pc: u32,
|
||||
pub major: u8,
|
||||
pub minor: u8,
|
||||
pub machine_mode: u8,
|
||||
#[debug(skip)]
|
||||
pub padding: u8,
|
||||
pub user_cycle: u32,
|
||||
pub txn_idx: u32,
|
||||
pub paging_idx: u32,
|
||||
pub diff_count: u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
pub struct RawPreflightTrace {
|
||||
pub cycles: *const RawPreflightCycle,
|
||||
pub txns: *const RawMemoryTransaction,
|
||||
pub txns_len: u32,
|
||||
pub table_split_cycle: u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
pub struct RawBuffer {
|
||||
pub buf: *const BabyBearElem,
|
||||
pub rows: usize,
|
||||
pub cols: usize,
|
||||
pub checked_reads: bool,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
pub struct RawExecBuffers {
|
||||
pub global: RawBuffer,
|
||||
pub data: RawBuffer,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
pub struct RawAccumBuffers {
|
||||
pub data: RawBuffer,
|
||||
pub accum: RawBuffer,
|
||||
pub mix: RawBuffer,
|
||||
}
|
||||
|
||||
extern "C" {
|
||||
pub fn risc0_circuit_rv32im_v2_cpu_witgen(
|
||||
mode: u32,
|
||||
buffers: *const RawExecBuffers,
|
||||
preflight: *const RawPreflightTrace,
|
||||
cycles: u32,
|
||||
) -> *const std::os::raw::c_char;
|
||||
|
||||
pub fn risc0_circuit_rv32im_v2_cpu_accum(
|
||||
buffers: *const RawAccumBuffers,
|
||||
preflight: *const RawPreflightTrace,
|
||||
cycles: u32,
|
||||
) -> *const std::os::raw::c_char;
|
||||
|
||||
pub fn risc0_circuit_rv32im_v2_cpu_poly_fp(
|
||||
cycle: usize,
|
||||
steps: usize,
|
||||
poly_mixs: *const BabyBearExtElem,
|
||||
args_ptr: *const *const BabyBearElem,
|
||||
result: *mut BabyBearExtElem,
|
||||
) -> *const std::os::raw::c_char;
|
||||
}
|
||||
|
||||
#[cfg(feature = "cuda")]
|
||||
extern "C" {
|
||||
pub fn risc0_circuit_rv32im_v2_cuda_witgen(
|
||||
mode: u32,
|
||||
buffers: *const RawExecBuffers,
|
||||
preflight: *const RawPreflightTrace,
|
||||
cycles: u32,
|
||||
) -> *const std::os::raw::c_char;
|
||||
|
||||
pub fn risc0_circuit_rv32im_v2_cuda_accum(
|
||||
buffers: *const RawAccumBuffers,
|
||||
preflight: *const RawPreflightTrace,
|
||||
cycles: u32,
|
||||
) -> *const std::os::raw::c_char;
|
||||
|
||||
pub fn risc0_circuit_rv32im_v2_cuda_eval_check(
|
||||
check: DevicePointer<u8>,
|
||||
ctrl: DevicePointer<u8>,
|
||||
data: DevicePointer<u8>,
|
||||
accum: DevicePointer<u8>,
|
||||
mix: DevicePointer<u8>,
|
||||
out: DevicePointer<u8>,
|
||||
rou: *const BabyBearElem,
|
||||
po2: u32,
|
||||
domain: u32,
|
||||
poly_mix_pows: *const u32,
|
||||
) -> *const std::os::raw::c_char;
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
[package]
|
||||
name = "risc0-circuit-rv32im-v2"
|
||||
description = "RISC Zero circuit for rv32im-v2"
|
||||
version = "0.1.0"
|
||||
edition = { workspace = true }
|
||||
license = { workspace = true }
|
||||
homepage = { workspace = true }
|
||||
repository = { workspace = true }
|
||||
|
||||
[[example]]
|
||||
name = "rv32im_v2"
|
||||
required-features = ["prove"]
|
||||
|
||||
[dependencies]
|
||||
anyhow = { version = "1.0", features = ["backtrace"] }
|
||||
num-derive = "0.4.2"
|
||||
num-traits = "0.2.19"
|
||||
risc0-binfmt = { workspace = true }
|
||||
risc0-core = { workspace = true }
|
||||
risc0-zkp = { workspace = true, default-features = false }
|
||||
serde = { version = "1.0", default-features = false, features = [
|
||||
"derive",
|
||||
"alloc",
|
||||
] }
|
||||
tracing = "0.1"
|
||||
|
||||
[target.'cfg(not(target_os = "zkvm"))'.dependencies]
|
||||
bytemuck = { version = "1.13", optional = true }
|
||||
cfg-if = { version = "1.0", optional = true }
|
||||
derive_more = { version = "1.0", features = [
|
||||
"add",
|
||||
"add_assign",
|
||||
"debug",
|
||||
], optional = true }
|
||||
paste = { version = "1.0", optional = true }
|
||||
rand = { version = "0.8", optional = true }
|
||||
rayon = { version = "1.5", optional = true }
|
||||
risc0-circuit-rv32im-v2-sys = { workspace = true, optional = true }
|
||||
risc0-sys = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
clap = { version = "4.5", features = ["derive"] }
|
||||
flate2 = "1.0"
|
||||
tar = "0.4"
|
||||
test-log = { version = "0.2", default-features = false, features = ["trace"] }
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
|
||||
[features]
|
||||
cuda = [
|
||||
"prove",
|
||||
"risc0-circuit-rv32im-v2-sys/cuda",
|
||||
"risc0-sys/cuda",
|
||||
"risc0-zkp/cuda",
|
||||
]
|
||||
default = ["prove"]
|
||||
execute = ["dep:bytemuck", "dep:derive_more", "std"]
|
||||
prove = [
|
||||
"dep:cfg-if",
|
||||
"dep:paste",
|
||||
"dep:rand",
|
||||
"dep:rayon",
|
||||
"dep:risc0-circuit-rv32im-v2-sys",
|
||||
"dep:risc0-sys",
|
||||
"execute",
|
||||
"risc0-core/perf",
|
||||
"risc0-zkp/prove",
|
||||
"std",
|
||||
]
|
||||
std = ["risc0-zkp/std", "serde/std"]
|
||||
@@ -0,0 +1,92 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::time::Instant;
|
||||
|
||||
use clap::Parser;
|
||||
use risc0_circuit_rv32im_v2::{
|
||||
execute::{platform::LOOKUP_TABLE_CYCLES, testutil, MemoryImage2, DEFAULT_SEGMENT_LIMIT_PO2},
|
||||
prove::segment_prover,
|
||||
};
|
||||
|
||||
/// keccak prover benchmarking tool
|
||||
#[derive(Parser)]
|
||||
#[command(about, version, author)]
|
||||
struct Cli {
|
||||
/// Circuit PO2
|
||||
#[arg(long, default_value_t = DEFAULT_SEGMENT_LIMIT_PO2)]
|
||||
po2: usize,
|
||||
|
||||
/// Number of proofs to run
|
||||
#[arg(long, default_value_t = 1)]
|
||||
count: usize,
|
||||
|
||||
/// Don't verify the seal
|
||||
#[arg(long)]
|
||||
skip_verification: bool,
|
||||
}
|
||||
|
||||
const PAGING_CYCLES: usize = 1821;
|
||||
const NON_LOOP_CYCLES: usize = 8;
|
||||
const RESERVED_CYCLES: usize = LOOKUP_TABLE_CYCLES + PAGING_CYCLES + NON_LOOP_CYCLES;
|
||||
|
||||
fn main() {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(tracing_subscriber::filter::EnvFilter::from_default_env())
|
||||
.init();
|
||||
|
||||
let args = Cli::parse();
|
||||
|
||||
let po2 = args.po2;
|
||||
let cycles = 1 << po2;
|
||||
assert!(cycles > RESERVED_CYCLES);
|
||||
let iterations = (cycles - RESERVED_CYCLES) / 2;
|
||||
|
||||
let program = testutil::simple_loop(iterations as u32);
|
||||
let image = MemoryImage2::new(program);
|
||||
let result = testutil::execute(
|
||||
image,
|
||||
args.po2,
|
||||
testutil::DEFAULT_SESSION_LIMIT,
|
||||
&testutil::NullSyscall,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let prover = segment_prover().unwrap();
|
||||
let segments = result.segments;
|
||||
let segment = segments.first().unwrap();
|
||||
assert_eq!(args.po2, segment.po2 as usize);
|
||||
|
||||
let mut tot_time: f64 = 0.0;
|
||||
for i in 0..args.count {
|
||||
let start_time = Instant::now();
|
||||
let seal = prover.prove(segment).unwrap();
|
||||
if !args.skip_verification {
|
||||
prover.verify(&seal).expect("Verification failed");
|
||||
}
|
||||
let run_time = start_time.elapsed().as_secs_f64();
|
||||
println!(
|
||||
"PO2={po2} Run #{i}: {run_time:.3}s, {:.3} cycles/sec",
|
||||
cycles as f64 / run_time
|
||||
);
|
||||
tot_time += run_time;
|
||||
}
|
||||
println!(
|
||||
"{} runs of PO2={po2} completed in {tot_time:.3}s, avg={:.3}s, {:.3} cycles/sec",
|
||||
args.count,
|
||||
tot_time / (args.count as f64),
|
||||
(args.count * cycles) as f64 / tot_time,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::ops;
|
||||
|
||||
use derive_more::{Add, AddAssign, Debug, Sub};
|
||||
|
||||
use super::{pager::PAGE_WORDS, platform::WORD_SIZE};
|
||||
|
||||
#[derive(Add, AddAssign, Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Sub)]
|
||||
#[debug("{_0:#010x}")]
|
||||
pub struct ByteAddr(pub u32);
|
||||
|
||||
#[derive(Add, AddAssign, Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Sub)]
|
||||
#[debug("${_0:#010x}")]
|
||||
pub struct WordAddr(pub u32);
|
||||
|
||||
impl ByteAddr {
|
||||
pub const fn waddr(self) -> WordAddr {
|
||||
WordAddr(self.0 / WORD_SIZE as u32)
|
||||
}
|
||||
|
||||
pub const fn is_aligned(&self) -> bool {
|
||||
self.0 % WORD_SIZE as u32 == 0
|
||||
}
|
||||
|
||||
pub const fn is_null(&self) -> bool {
|
||||
self.0 == 0
|
||||
}
|
||||
|
||||
pub fn wrapping_add(self, rhs: u32) -> Self {
|
||||
Self(self.0.wrapping_add(rhs))
|
||||
}
|
||||
|
||||
pub fn subaddr(&self) -> u32 {
|
||||
self.0 % WORD_SIZE as u32
|
||||
}
|
||||
}
|
||||
|
||||
impl WordAddr {
|
||||
pub const fn baddr(self) -> ByteAddr {
|
||||
ByteAddr(self.0 * WORD_SIZE as u32)
|
||||
}
|
||||
|
||||
pub fn page_idx(&self) -> u32 {
|
||||
self.0 / PAGE_WORDS as u32
|
||||
}
|
||||
|
||||
pub fn page_subaddr(&self) -> WordAddr {
|
||||
Self(self.0 % PAGE_WORDS as u32)
|
||||
}
|
||||
|
||||
pub fn postfix_inc(&mut self) -> Self {
|
||||
let cur = *self;
|
||||
self.0 += 1;
|
||||
cur
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Add<usize> for WordAddr {
|
||||
type Output = WordAddr;
|
||||
|
||||
fn add(self, rhs: usize) -> Self::Output {
|
||||
Self(self.0 + rhs as u32)
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Add<u32> for WordAddr {
|
||||
type Output = WordAddr;
|
||||
|
||||
fn add(self, rhs: u32) -> Self::Output {
|
||||
Self(self.0 + rhs)
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Sub<u32> for WordAddr {
|
||||
type Output = WordAddr;
|
||||
|
||||
fn sub(self, rhs: u32) -> Self::Output {
|
||||
Self(self.0 - rhs)
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::AddAssign<usize> for WordAddr {
|
||||
fn add_assign(&mut self, rhs: usize) {
|
||||
self.0 += rhs as u32;
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::AddAssign<u32> for WordAddr {
|
||||
fn add_assign(&mut self, rhs: u32) {
|
||||
self.0 += rhs;
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Add<usize> for ByteAddr {
|
||||
type Output = ByteAddr;
|
||||
|
||||
fn add(self, rhs: usize) -> Self::Output {
|
||||
Self(self.0 + rhs as u32)
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::Add<u32> for ByteAddr {
|
||||
type Output = ByteAddr;
|
||||
|
||||
fn add(self, rhs: u32) -> Self::Output {
|
||||
Self(self.0 + rhs)
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::AddAssign<usize> for ByteAddr {
|
||||
fn add_assign(&mut self, rhs: usize) {
|
||||
self.0 += rhs as u32;
|
||||
}
|
||||
}
|
||||
|
||||
impl ops::AddAssign<u32> for ByteAddr {
|
||||
fn add_assign(&mut self, rhs: u32) {
|
||||
self.0 += rhs;
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ByteAddr> for WordAddr {
|
||||
fn from(addr: ByteAddr) -> Self {
|
||||
addr.waddr()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<WordAddr> for ByteAddr {
|
||||
fn from(addr: WordAddr) -> Self {
|
||||
addr.baddr()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{cell::RefCell, rc::Rc};
|
||||
|
||||
use anyhow::{bail, Result};
|
||||
use risc0_binfmt::ExitCode;
|
||||
use risc0_zkp::core::{digest::Digest, log2_ceil};
|
||||
|
||||
use super::{
|
||||
addr::{ByteAddr, WordAddr},
|
||||
image::MemoryImage2,
|
||||
pager::PagedMemory,
|
||||
platform::{CycleState, LOOKUP_TABLE_CYCLES},
|
||||
r0vm::{Risc0Context, Risc0Machine},
|
||||
rv32im::{disasm, DecodedInstruction, Emulator, Instruction},
|
||||
segment::Segment,
|
||||
syscall::Syscall,
|
||||
trace::{TraceCallback, TraceEvent},
|
||||
};
|
||||
|
||||
pub struct Executor<'a, 'b, S: Syscall> {
|
||||
pc: ByteAddr,
|
||||
machine_mode: u32,
|
||||
user_cycles: u32,
|
||||
phys_cycles: u32,
|
||||
pager: PagedMemory,
|
||||
exit_code: Option<ExitCode>,
|
||||
read_record: Vec<Vec<u8>>,
|
||||
write_record: Vec<u32>,
|
||||
syscall_handler: &'a S,
|
||||
input_digest: Digest,
|
||||
output_digest: Option<Digest>,
|
||||
trace: Vec<Rc<RefCell<dyn TraceCallback + 'b>>>,
|
||||
cycles: SessionCycles,
|
||||
}
|
||||
|
||||
pub struct ExecutorResult {
|
||||
pub segments: u64,
|
||||
pub exit_code: ExitCode,
|
||||
pub post_image: MemoryImage2,
|
||||
pub user_cycles: u64,
|
||||
pub total_cycles: u64,
|
||||
pub pre_digest: Digest,
|
||||
pub post_digest: Digest,
|
||||
pub output_digest: Option<Digest>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct SessionCycles {
|
||||
user: u64,
|
||||
total: u64,
|
||||
}
|
||||
|
||||
pub struct SimpleSession {
|
||||
pub segments: Vec<Segment>,
|
||||
pub result: ExecutorResult,
|
||||
}
|
||||
|
||||
impl<'a, 'b, S: Syscall> Executor<'a, 'b, S> {
|
||||
pub fn new(
|
||||
image: MemoryImage2,
|
||||
syscall_handler: &'a S,
|
||||
input_digest: Option<Digest>,
|
||||
trace: Vec<Rc<RefCell<dyn TraceCallback + 'b>>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
pc: ByteAddr(0),
|
||||
machine_mode: 0,
|
||||
user_cycles: 0,
|
||||
phys_cycles: 0,
|
||||
pager: PagedMemory::new(image),
|
||||
exit_code: None,
|
||||
read_record: Vec::new(),
|
||||
write_record: Vec::new(),
|
||||
syscall_handler,
|
||||
input_digest: input_digest.unwrap_or_default(),
|
||||
output_digest: None,
|
||||
trace,
|
||||
cycles: SessionCycles::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run<F: FnMut(Segment) -> Result<()>>(
|
||||
&mut self,
|
||||
segment_po2: usize,
|
||||
max_cycles: Option<u64>,
|
||||
mut callback: F,
|
||||
) -> Result<ExecutorResult> {
|
||||
let segment_limit = 1 << segment_po2;
|
||||
let mut segment_counter = 0u64;
|
||||
|
||||
self.reset();
|
||||
|
||||
let mut emu = Emulator::new();
|
||||
Risc0Machine::resume(self)?;
|
||||
let initial_digest = *self.pager.image.image_id();
|
||||
|
||||
while self.exit_code.is_none() {
|
||||
if let Some(max_cycles) = max_cycles {
|
||||
if self.cycles.user >= max_cycles {
|
||||
bail!("Session limit exceeded");
|
||||
}
|
||||
}
|
||||
|
||||
if self.segment_cycles() >= segment_limit {
|
||||
Risc0Machine::suspend(self)?;
|
||||
|
||||
let (pre_digest, partial_image, post_digest) = self.pager.commit()?;
|
||||
callback(Segment {
|
||||
partial_image,
|
||||
pre_digest,
|
||||
post_digest,
|
||||
read_record: std::mem::take(&mut self.read_record),
|
||||
write_record: std::mem::take(&mut self.write_record),
|
||||
user_cycles: self.user_cycles,
|
||||
suspend_cycle: self.phys_cycles,
|
||||
paging_cycles: self.pager.cycles,
|
||||
po2: segment_po2 as u32,
|
||||
exit_code: ExitCode::SystemSplit,
|
||||
index: segment_counter,
|
||||
input_digest: self.input_digest,
|
||||
output_digest: self.output_digest,
|
||||
})?;
|
||||
|
||||
segment_counter += 1;
|
||||
self.cycles.total += 1 << segment_po2;
|
||||
self.user_cycles = 0;
|
||||
self.phys_cycles = 0;
|
||||
self.pager.reset();
|
||||
|
||||
Risc0Machine::resume(self)?;
|
||||
}
|
||||
|
||||
Risc0Machine::step(&mut emu, self)?;
|
||||
}
|
||||
|
||||
Risc0Machine::suspend(self)?;
|
||||
|
||||
let (pre_digest, partial_image, post_digest) = self.pager.commit()?;
|
||||
let last_po2 = log2_ceil(self.segment_cycles().next_power_of_two() as usize);
|
||||
let exit_code = self.exit_code.unwrap();
|
||||
|
||||
callback(Segment {
|
||||
partial_image,
|
||||
pre_digest,
|
||||
post_digest,
|
||||
read_record: std::mem::take(&mut self.read_record),
|
||||
write_record: std::mem::take(&mut self.write_record),
|
||||
user_cycles: self.user_cycles,
|
||||
suspend_cycle: self.phys_cycles,
|
||||
paging_cycles: self.pager.cycles,
|
||||
po2: last_po2 as u32,
|
||||
exit_code,
|
||||
index: segment_counter,
|
||||
input_digest: self.input_digest,
|
||||
output_digest: self.output_digest,
|
||||
})?;
|
||||
|
||||
self.cycles.total += 1 << last_po2;
|
||||
|
||||
Ok(ExecutorResult {
|
||||
segments: segment_counter + 1,
|
||||
exit_code,
|
||||
post_image: self.pager.image.clone(),
|
||||
user_cycles: self.cycles.user,
|
||||
total_cycles: self.cycles.total,
|
||||
pre_digest: initial_digest,
|
||||
post_digest,
|
||||
output_digest: self.output_digest,
|
||||
})
|
||||
}
|
||||
|
||||
fn reset(&mut self) {
|
||||
self.pager.reset();
|
||||
self.exit_code = None;
|
||||
self.read_record.clear();
|
||||
self.write_record.clear();
|
||||
self.output_digest = None;
|
||||
self.machine_mode = 0;
|
||||
self.user_cycles = 0;
|
||||
self.phys_cycles = 0;
|
||||
self.cycles.user = 0;
|
||||
self.cycles.total = 0;
|
||||
self.pc = ByteAddr(0);
|
||||
}
|
||||
|
||||
fn segment_cycles(&self) -> u32 {
|
||||
self.phys_cycles + self.pager.cycles + LOOKUP_TABLE_CYCLES as u32
|
||||
}
|
||||
|
||||
fn trace(&mut self, event: TraceEvent) -> Result<()> {
|
||||
for trace in self.trace.iter() {
|
||||
trace.borrow_mut().trace_callback(event.clone())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, 'b, S: Syscall> Risc0Context for Executor<'a, 'b, S> {
|
||||
fn get_pc(&self) -> ByteAddr {
|
||||
self.pc
|
||||
}
|
||||
|
||||
fn set_pc(&mut self, addr: ByteAddr) {
|
||||
self.pc = addr;
|
||||
}
|
||||
|
||||
fn get_machine_mode(&self) -> u32 {
|
||||
self.machine_mode
|
||||
}
|
||||
|
||||
fn set_machine_mode(&mut self, mode: u32) {
|
||||
self.machine_mode = mode;
|
||||
}
|
||||
|
||||
fn on_insn_start(&mut self, insn: &Instruction, decoded: &DecodedInstruction) -> Result<()> {
|
||||
let cycle = self.cycles.user;
|
||||
self.cycles.user += 1;
|
||||
tracing::trace!(
|
||||
"[{}:{}:{cycle}] {:?}> {:#010x} {}",
|
||||
self.user_cycles + 1,
|
||||
self.segment_cycles() + 1,
|
||||
self.pc,
|
||||
decoded.insn,
|
||||
disasm(insn, decoded)
|
||||
);
|
||||
self.trace(TraceEvent::InstructionStart {
|
||||
cycle,
|
||||
pc: self.pc.0,
|
||||
insn: decoded.insn,
|
||||
})
|
||||
}
|
||||
|
||||
fn on_insn_end(&mut self, _insn: &Instruction, _decoded: &DecodedInstruction) -> Result<()> {
|
||||
self.user_cycles += 1;
|
||||
self.phys_cycles += 1;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn on_ecall_cycle(
|
||||
&mut self,
|
||||
_cur: CycleState,
|
||||
_next: CycleState,
|
||||
_s0: u32,
|
||||
_s1: u32,
|
||||
_s2: u32,
|
||||
) -> Result<()> {
|
||||
self.phys_cycles += 1;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn peek_u32(&mut self, addr: WordAddr) -> Result<u32> {
|
||||
self.pager.peek(addr)
|
||||
}
|
||||
|
||||
fn load_u32(&mut self, addr: WordAddr) -> Result<u32> {
|
||||
let word = self.pager.load(addr)?;
|
||||
// tracing::trace!("load_mem({:?}) -> {word:#010x}", addr.baddr());
|
||||
Ok(word)
|
||||
}
|
||||
|
||||
fn store_u32(&mut self, addr: WordAddr, word: u32) -> Result<()> {
|
||||
// tracing::trace!("store_mem({:?}, {word:#010x})", addr.baddr());
|
||||
self.trace(TraceEvent::MemorySet {
|
||||
addr: addr.baddr().0,
|
||||
region: word.to_be_bytes().to_vec(),
|
||||
})?;
|
||||
self.pager.store(addr, word)
|
||||
}
|
||||
|
||||
fn on_terminate(&mut self, a0: u32, _a1: u32) {
|
||||
self.user_cycles += 1;
|
||||
self.exit_code = Some(ExitCode::Halted(a0));
|
||||
}
|
||||
|
||||
fn host_read(&mut self, fd: u32, buf: &mut [u8]) -> Result<u32> {
|
||||
let rlen = self.syscall_handler.host_read(fd, buf)?;
|
||||
let slice = &buf[..rlen as usize];
|
||||
self.read_record.push(slice.to_vec());
|
||||
Ok(rlen)
|
||||
}
|
||||
|
||||
fn host_write(&mut self, fd: u32, buf: &[u8]) -> Result<u32> {
|
||||
let rlen = self.syscall_handler.host_write(fd, buf)?;
|
||||
self.write_record.push(rlen);
|
||||
Ok(rlen)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,363 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{collections::BTreeMap, sync::LazyLock};
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use derive_more::Debug;
|
||||
use risc0_binfmt::Program;
|
||||
use risc0_zkp::{
|
||||
core::{
|
||||
digest::{Digest, DIGEST_WORDS},
|
||||
hash::poseidon2::{poseidon2_mix, CELLS},
|
||||
},
|
||||
field::Elem as _,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::zirgen::circuit::Val;
|
||||
|
||||
use super::{
|
||||
addr::{ByteAddr, WordAddr},
|
||||
pager::PAGE_WORDS,
|
||||
platform::*,
|
||||
};
|
||||
|
||||
static ZERO_CACHE: LazyLock<ZeroCache> = LazyLock::new(ZeroCache::new);
|
||||
|
||||
struct ZeroCache {
|
||||
pub page: Page,
|
||||
pub digests: Vec<Digest>,
|
||||
}
|
||||
|
||||
impl ZeroCache {
|
||||
fn new() -> Self {
|
||||
let page = Page::default();
|
||||
let mut digest = page.digest();
|
||||
let mut digests = vec![Digest::ZERO; MERKLE_TREE_DEPTH + 1];
|
||||
for depth in (0..MERKLE_TREE_DEPTH + 1).rev() {
|
||||
digests[depth] = digest;
|
||||
digest = DigestPair {
|
||||
lhs: digest,
|
||||
rhs: digest,
|
||||
}
|
||||
.digest();
|
||||
}
|
||||
Self { page, digests }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
pub struct Page(Vec<u8>);
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct MemoryImage2 {
|
||||
#[debug("{}", pages.len())]
|
||||
// #[debug("{:#010x?}", pages.keys())]
|
||||
pub pages: BTreeMap<u32, Page>,
|
||||
#[debug("{}", digests.len())]
|
||||
// #[debug("{:#010x?}", digests.keys())]
|
||||
pub digests: BTreeMap<u32, Digest>,
|
||||
}
|
||||
|
||||
impl Default for MemoryImage2 {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
pages: Default::default(),
|
||||
digests: BTreeMap::from([(1, ZERO_CACHE.digests[0])]),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MemoryImage2 {
|
||||
pub fn new(program: Program) -> Self {
|
||||
let mut this = Self::default();
|
||||
let mut cur_page_idx = 0xffffffff_u32;
|
||||
let mut cur_page: Option<Page> = None;
|
||||
|
||||
let mut image = program.image;
|
||||
image.insert(SUSPEND_PC_ADDR.0, program.entry);
|
||||
image.insert(SUSPEND_MODE_ADDR.0, 1);
|
||||
|
||||
for (&addr, &word) in image.iter() {
|
||||
let addr = ByteAddr(addr).waddr();
|
||||
let page_idx = addr.page_idx();
|
||||
if page_idx != cur_page_idx {
|
||||
if let Some(page) = cur_page.take() {
|
||||
this.set_page(cur_page_idx, page);
|
||||
}
|
||||
cur_page = Some(Page::default());
|
||||
cur_page_idx = page_idx;
|
||||
}
|
||||
|
||||
cur_page.as_mut().unwrap().store(addr, word);
|
||||
}
|
||||
|
||||
if let Some(page) = cur_page.take() {
|
||||
this.set_page(cur_page_idx, page);
|
||||
}
|
||||
|
||||
this
|
||||
}
|
||||
|
||||
pub fn with_kernel(mut user: Program, mut kernel: Program) -> Self {
|
||||
kernel.image.append(&mut user.image);
|
||||
kernel
|
||||
.image
|
||||
.insert(MEPC_ADDR.0, user.entry - WORD_SIZE as u32);
|
||||
// .insert(MEPC_ADDR.waddr().0, user.entry - WORD_SIZE as u32);
|
||||
Self::new(kernel)
|
||||
}
|
||||
|
||||
/// Return the page data, fails if unavailable
|
||||
pub fn get_page(&mut self, page_idx: u32) -> Result<Page> {
|
||||
// If page exists, return it
|
||||
if let Some(page) = self.pages.get(&page_idx) {
|
||||
return Ok(page.clone());
|
||||
}
|
||||
|
||||
// Otherwise try an expand
|
||||
let digest_idx = MEMORY_PAGES as u32 + page_idx;
|
||||
if self.expand_if_zero(digest_idx) {
|
||||
let zero_page = &ZERO_CACHE.page;
|
||||
self.pages.insert(page_idx, zero_page.clone());
|
||||
return Ok(zero_page.clone());
|
||||
}
|
||||
|
||||
// Otherwise fail
|
||||
bail!("Unavailable page: {page_idx}")
|
||||
}
|
||||
|
||||
/// Set the data for a page
|
||||
pub fn set_page(&mut self, page_idx: u32, page: Page) {
|
||||
// tracing::trace!("set_page({page_idx:#08x})");
|
||||
let digest_idx = MEMORY_PAGES as u32 + page_idx;
|
||||
self.expand_if_zero(digest_idx);
|
||||
self.digests.insert(digest_idx, page.digest());
|
||||
self.pages.insert(page_idx, page);
|
||||
self.fixup_digests(digest_idx);
|
||||
}
|
||||
|
||||
/// Get a digest, fails if unavailable
|
||||
pub fn get_digest(&mut self, digest_idx: u32) -> Result<&Digest> {
|
||||
// Expand if needed
|
||||
self.expand_if_zero(digest_idx);
|
||||
self.digests
|
||||
.get(&digest_idx)
|
||||
.ok_or(anyhow!("Unavailable digest: {digest_idx}"))
|
||||
}
|
||||
|
||||
/// Set a digest
|
||||
pub fn set_digest(&mut self, digest_idx: u32, digest: Digest) {
|
||||
// If digest is in a zero region, reify for proper uncles
|
||||
self.expand_if_zero(digest_idx);
|
||||
// Set the digest value
|
||||
self.digests.insert(digest_idx, digest);
|
||||
// Fixup digest values
|
||||
self.fixup_digests(digest_idx);
|
||||
}
|
||||
|
||||
/// Return the root digest
|
||||
pub fn image_id(&mut self) -> &Digest {
|
||||
self.get_digest(1).unwrap()
|
||||
}
|
||||
|
||||
/// Expand if digest at `digest_idx` is a zero, return if expanded
|
||||
fn expand_if_zero(&mut self, digest_idx: u32) -> bool {
|
||||
let ret = self.is_zero(digest_idx);
|
||||
if ret {
|
||||
self.expand_zero(digest_idx);
|
||||
}
|
||||
ret
|
||||
}
|
||||
|
||||
/// Check if given MT node is a zero
|
||||
fn is_zero(&self, mut digest_idx: u32) -> bool {
|
||||
// Compute the depth in the tree of this node
|
||||
let mut depth = digest_idx.ilog2() as usize;
|
||||
// Go up until we hit a valid node or get past the root
|
||||
while !self.digests.contains_key(&digest_idx) && digest_idx > 0 {
|
||||
digest_idx /= 2;
|
||||
depth -= 1;
|
||||
}
|
||||
if digest_idx == 0 {
|
||||
false
|
||||
} else {
|
||||
self.digests[&digest_idx] == ZERO_CACHE.digests[depth]
|
||||
}
|
||||
}
|
||||
|
||||
/// Expand zero MT node.
|
||||
///
|
||||
/// Presumes `is_zero(digest_idx)` returned true.
|
||||
fn expand_zero(&mut self, mut digest_idx: u32) {
|
||||
// Compute the depth in the tree of this node
|
||||
let mut depth = digest_idx.ilog2() as usize;
|
||||
// Go up until we hit the valid zero node
|
||||
while !self.digests.contains_key(&digest_idx) {
|
||||
let parent_idx = digest_idx / 2;
|
||||
let lhs_idx = parent_idx * 2;
|
||||
let rhs_idx = parent_idx * 2 + 1;
|
||||
self.digests.insert(lhs_idx, ZERO_CACHE.digests[depth]);
|
||||
self.digests.insert(rhs_idx, ZERO_CACHE.digests[depth]);
|
||||
digest_idx = parent_idx;
|
||||
depth -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// Fixup digests after a change
|
||||
fn fixup_digests(&mut self, mut digest_idx: u32) {
|
||||
while digest_idx != 1 {
|
||||
let parent_idx = digest_idx / 2;
|
||||
let lhs_idx = parent_idx * 2;
|
||||
let rhs_idx = parent_idx * 2 + 1;
|
||||
let lhs = self.digests.get(&lhs_idx);
|
||||
let rhs = self.digests.get(&rhs_idx);
|
||||
if let (Some(&lhs), Some(&rhs)) = (lhs, rhs) {
|
||||
let parent_digest = DigestPair { lhs, rhs }.digest();
|
||||
self.digests.insert(parent_idx, parent_digest);
|
||||
digest_idx = parent_idx;
|
||||
} else {
|
||||
break;
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Page {
|
||||
fn default() -> Self {
|
||||
Self(vec![0; PAGE_BYTES])
|
||||
}
|
||||
}
|
||||
|
||||
impl Page {
|
||||
pub fn digest(&self) -> Digest {
|
||||
let mut cells = [Val::ZERO; CELLS];
|
||||
for i in 0..PAGE_WORDS / DIGEST_WORDS {
|
||||
for j in 0..DIGEST_WORDS {
|
||||
let addr = WordAddr((i * DIGEST_WORDS + j) as u32);
|
||||
let word = self.load(addr);
|
||||
cells[2 * j] = Val::new(word & 0xffff);
|
||||
cells[2 * j + 1] = Val::new(word >> 16);
|
||||
}
|
||||
poseidon2_mix(&mut cells);
|
||||
}
|
||||
cells_to_digest(&cells)
|
||||
}
|
||||
|
||||
pub fn load(&self, addr: WordAddr) -> u32 {
|
||||
let byte_addr = addr.page_subaddr().baddr().0 as usize;
|
||||
let mut bytes = [0u8; WORD_SIZE];
|
||||
bytes.clone_from_slice(&self.0[byte_addr..byte_addr + WORD_SIZE]);
|
||||
#[allow(clippy::let_and_return)] // easier to toggle optional tracing
|
||||
let word = u32::from_le_bytes(bytes);
|
||||
// tracing::trace!("load({addr:?}) -> {word:#010x}");
|
||||
word
|
||||
}
|
||||
|
||||
pub fn store(&mut self, addr: WordAddr, word: u32) {
|
||||
let byte_addr = addr.page_subaddr().baddr().0 as usize;
|
||||
// tracing::trace!("store({addr:?}, {byte_addr:#05x}, {word:#010x})");
|
||||
self.0[byte_addr..byte_addr + WORD_SIZE].clone_from_slice(&word.to_le_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
struct DigestPair {
|
||||
lhs: Digest,
|
||||
rhs: Digest,
|
||||
}
|
||||
|
||||
impl DigestPair {
|
||||
pub fn digest(&self) -> Digest {
|
||||
let mut cells = [Val::ZERO; CELLS];
|
||||
for i in 0..DIGEST_WORDS {
|
||||
cells[i] = Val::new(self.rhs.as_words()[i]);
|
||||
cells[DIGEST_WORDS + i] = Val::new(self.lhs.as_words()[i]);
|
||||
}
|
||||
poseidon2_mix(&mut cells);
|
||||
cells_to_digest(&cells)
|
||||
}
|
||||
}
|
||||
|
||||
fn cells_to_digest(cells: &[Val; CELLS]) -> Digest {
|
||||
Digest::new([
|
||||
cells[0].as_u32(),
|
||||
cells[1].as_u32(),
|
||||
cells[2].as_u32(),
|
||||
cells[3].as_u32(),
|
||||
cells[4].as_u32(),
|
||||
cells[5].as_u32(),
|
||||
cells[6].as_u32(),
|
||||
cells[7].as_u32(),
|
||||
])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use risc0_binfmt::Program;
|
||||
use risc0_zkp::digest;
|
||||
use test_log::test;
|
||||
|
||||
use super::{MemoryImage2, ZERO_CACHE};
|
||||
|
||||
#[test]
|
||||
fn poseidon2_zeros() {
|
||||
let expected = [
|
||||
digest!("f85c5a32ccc45c22f9686b08d710d4597d7ce256cdcd63146426270d9432c644"),
|
||||
digest!("2ce7714c40af126c2e86f320b10de417eddd8f51d2b9133d3105c3541a154812"),
|
||||
digest!("889c443e0c55734c0212fe6c400f00423c421f2070b1340351e77826e4918274"),
|
||||
digest!("53ea92273a7dfb7622de685c49f4ce1bd69db1696cd6846e9f5de56c89098b01"),
|
||||
digest!("82db13229831cb2ad63df0476dc1f217c702503d46770c283b6ecc1520fff074"),
|
||||
digest!("45cba5321f90c34b780d5d1790f23612fb834b3d21dc1e53594826470719ba34"),
|
||||
digest!("132689262568ae5ac27a4b65018aef0b2e4345578a16453acd874973a61c6350"),
|
||||
digest!("9fc9626e87aa3614eb38b44d9d832712fb2ea32427c6fd49281ca225f1fefd0d"),
|
||||
digest!("70947164fe9a4353fa33fb024f09ea0df24be40d88b6025278a3472ac49e6715"),
|
||||
digest!("4b707f15d9941c0168d630618cdcc05ccae5d84ab9674a6666123a0039915173"),
|
||||
digest!("97fb1325724ddb74b1446b5bfa13f02c2ecb1b2b2a2f5b1334a04c5c76335d12"),
|
||||
digest!("adba743a459eb5357487a1238a0c4c238b8313458283900447e9b8540adfb042"),
|
||||
digest!("a16e68725fe981434dcca548e972214b2dd85e017c3a4e03909a0f4c31a08741"),
|
||||
digest!("fb94f356397279703f12c24da7aa371e192294347af15d46f10ab512708cdb68"),
|
||||
digest!("30a2fe1aa5c2ae0e10b91074e34b06742be91e450a9bc10f28ab082263c48750"),
|
||||
digest!("2347f636d9a0ea45bbe8bf519f39d3127f72b625e2e5495f26a6dd583eb2965d"),
|
||||
digest!("e43d140e71e366521152d932e846c73535674921576711023deaee06de3b091e"),
|
||||
digest!("35500a740d3a8b4e5a0ca06a8362f3444456e3206826102dd9e9bc3e5a1a5a18"),
|
||||
digest!("7c650c1a2000ef1a9baf4f56c2d66e76a3a0b4510175b171268d156a25d8dd45"),
|
||||
digest!("d73a1e0997a00543afd8de5261f316704215ce384e3ea13df3f87e000f04fb5f"),
|
||||
digest!("5b77f60275cb272fa0a3d267bdf1fc15021dbe7185ed6a3c94e45d70bbd70148"),
|
||||
digest!("e053c93b359c8905c5d8523139988b0ed4ef3426864a80498dfcb91d9b813364"),
|
||||
digest!("242ce034cc4e9326f8b7071124454b2be1a1cd5d21b6483c7ff81d4ba5ac9566"),
|
||||
];
|
||||
assert_eq!(ZERO_CACHE.digests, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_circuit_match() {
|
||||
let entry = 0x10000;
|
||||
let program = Program {
|
||||
entry,
|
||||
image: BTreeMap::from([(entry, 0x1234b337)]),
|
||||
};
|
||||
let mut image = MemoryImage2::new(program);
|
||||
assert_eq!(
|
||||
*image.get_digest(0x0040_0100).unwrap(),
|
||||
digest!("242ce034cc4e9326f8b7071124454b2be1a1cd5d21b6483c7ff81d4ba5ac9566")
|
||||
);
|
||||
assert_eq!(
|
||||
*image.image_id(),
|
||||
digest!("9d41290fa400705127c0240cb646586cc6ea8a23d560aa57cfa86c1369d9d53f")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod addr;
|
||||
mod executor;
|
||||
pub(crate) mod image;
|
||||
pub(crate) mod pager;
|
||||
pub mod platform;
|
||||
pub(crate) mod r0vm;
|
||||
pub(crate) mod rv32im;
|
||||
pub(crate) mod segment;
|
||||
mod syscall;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
pub mod testutil;
|
||||
mod trace;
|
||||
|
||||
use self::platform::MEMORY_PAGES;
|
||||
|
||||
pub use self::{
|
||||
executor::{Executor, ExecutorResult, SimpleSession},
|
||||
image::MemoryImage2,
|
||||
};
|
||||
|
||||
pub const DEFAULT_SEGMENT_LIMIT_PO2: usize = 20;
|
||||
|
||||
pub(crate) fn node_idx(page_idx: u32) -> u32 {
|
||||
MEMORY_PAGES as u32 + page_idx
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use anyhow::{bail, Result};
|
||||
use derive_more::Debug;
|
||||
use risc0_zkp::core::digest::Digest;
|
||||
|
||||
use super::{
|
||||
addr::WordAddr,
|
||||
image::{MemoryImage2, Page},
|
||||
node_idx,
|
||||
platform::*,
|
||||
};
|
||||
|
||||
pub const PAGE_WORDS: usize = PAGE_BYTES / WORD_SIZE;
|
||||
|
||||
const LOAD_ROOT_CYCLES: u32 = 1;
|
||||
const RESUME_CYCLES: u32 = 2;
|
||||
const SUSPEND_CYCLES: u32 = 2;
|
||||
const STORE_ROOT_CYCLES: u32 = 1;
|
||||
|
||||
const POSEIDON_PAGING: u32 = 1;
|
||||
const POSEIDON_LOAD_IN: u32 = 2;
|
||||
const POSEIDON_DO_OUT: u32 = 1;
|
||||
const POSEIDON_EXTERNAL: u32 = 8;
|
||||
const POSEIDON_INTERNAL: u32 = 1;
|
||||
const POSEIDON_ENTRY: u32 = 1;
|
||||
pub(crate) const POSEIDON_BLOCK_WORDS: u32 = 8;
|
||||
pub(crate) const POSEIDON_PAGE_ROUNDS: u32 = PAGE_WORDS as u32 / POSEIDON_BLOCK_WORDS;
|
||||
|
||||
const CYCLE_COST_PAGE: u32 = POSEIDON_PAGING + 10 * POSEIDON_PAGE_ROUNDS + POSEIDON_DO_OUT;
|
||||
|
||||
const CYCLE_COST_NODE: u32 =
|
||||
POSEIDON_PAGING + POSEIDON_LOAD_IN + POSEIDON_EXTERNAL + POSEIDON_INTERNAL + POSEIDON_DO_OUT;
|
||||
|
||||
const CYCLE_COST_RESERVED: u32 = LOAD_ROOT_CYCLES
|
||||
+ POSEIDON_ENTRY
|
||||
+ POSEIDON_PAGING
|
||||
+ RESUME_CYCLES
|
||||
+ SUSPEND_CYCLES
|
||||
+ POSEIDON_ENTRY
|
||||
+ POSEIDON_PAGING
|
||||
+ STORE_ROOT_CYCLES;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, PartialOrd)]
|
||||
enum PageState {
|
||||
Unloaded,
|
||||
Loaded,
|
||||
Dirty,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Debug)]
|
||||
pub(crate) struct PagingActivity {
|
||||
pub pages: BTreeSet<u32>,
|
||||
pub nodes: BTreeSet<u32>,
|
||||
}
|
||||
|
||||
impl PagingActivity {
|
||||
fn new(pages: BTreeSet<u32>) -> Self {
|
||||
let mut nodes = BTreeSet::new();
|
||||
for &page_idx in pages.iter() {
|
||||
let mut node_idx = node_idx(page_idx);
|
||||
while node_idx != 1 {
|
||||
let parent_idx = node_idx / 2;
|
||||
// tracing::trace!("add node: {node_idx:#010x}, parent_idx: {parent_idx:#010x}");
|
||||
if !nodes.insert(parent_idx) {
|
||||
break;
|
||||
}
|
||||
node_idx = parent_idx;
|
||||
}
|
||||
}
|
||||
Self { pages, nodes }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct PagedMemory {
|
||||
pub image: MemoryImage2,
|
||||
#[debug("{:#x?}", page_cache.keys())]
|
||||
page_cache: BTreeMap<u32, Page>,
|
||||
#[debug("{page_states:#x?}")]
|
||||
page_states: BTreeMap<u32, PageState>,
|
||||
pub cycles: u32,
|
||||
}
|
||||
|
||||
impl PagedMemory {
|
||||
pub(crate) fn new(image: MemoryImage2) -> Self {
|
||||
Self {
|
||||
image,
|
||||
page_cache: BTreeMap::new(),
|
||||
page_states: BTreeMap::new(),
|
||||
cycles: CYCLE_COST_RESERVED,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn reset(&mut self) {
|
||||
self.page_cache.clear();
|
||||
self.page_states.clear();
|
||||
self.cycles = CYCLE_COST_RESERVED;
|
||||
}
|
||||
|
||||
pub(crate) fn loaded_pages(&self) -> PagingActivity {
|
||||
tracing::trace!("loaded_pages: {:#010x?}", self.image.pages.keys());
|
||||
PagingActivity::new(self.image.pages.keys().copied().collect())
|
||||
}
|
||||
|
||||
pub(crate) fn dirty_pages(&self) -> PagingActivity {
|
||||
let pages = self
|
||||
.page_cache
|
||||
.keys()
|
||||
.filter(|page_idx| self.page_states[&node_idx(**page_idx)] == PageState::Dirty)
|
||||
.copied()
|
||||
.collect();
|
||||
PagingActivity::new(pages)
|
||||
}
|
||||
|
||||
pub(crate) fn peek(&mut self, addr: WordAddr) -> Result<u32> {
|
||||
if addr >= MEMORY_END_ADDR {
|
||||
bail!("Invalid peek address: {addr:?}");
|
||||
}
|
||||
let page_idx = addr.page_idx();
|
||||
let node_idx = node_idx(page_idx);
|
||||
if self.page_states.contains_key(&node_idx) {
|
||||
// Loaded, get from cache
|
||||
Ok(self.page_cache[&page_idx].load(addr))
|
||||
} else {
|
||||
// Unloaded, peek into image
|
||||
Ok(self.image.get_page(page_idx)?.load(addr))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn load(&mut self, addr: WordAddr) -> Result<u32> {
|
||||
if addr >= MEMORY_END_ADDR {
|
||||
bail!("Invalid load address: {addr:?}");
|
||||
}
|
||||
let page_idx = addr.page_idx();
|
||||
// tracing::trace!("load: {addr:?}, page: {page_idx:#08x}");
|
||||
let node_idx = node_idx(page_idx);
|
||||
#[allow(clippy::map_entry)] // lifetime issues
|
||||
if !self.page_states.contains_key(&node_idx) {
|
||||
self.load_page(page_idx)?;
|
||||
self.page_states.insert(node_idx, PageState::Loaded);
|
||||
}
|
||||
Ok(self.page_cache[&page_idx].load(addr))
|
||||
}
|
||||
|
||||
pub(crate) fn store(&mut self, addr: WordAddr, word: u32) -> Result<()> {
|
||||
if addr >= MEMORY_END_ADDR {
|
||||
bail!("Invalid store address: {addr:?}");
|
||||
}
|
||||
let page_idx = addr.page_idx();
|
||||
// tracing::trace!("store: {addr:?}, page: {page_idx:#08x}, word: {word:#010x}");
|
||||
let node_idx = node_idx(page_idx);
|
||||
let state = if let Some(state) = self.page_states.get(&node_idx) {
|
||||
*state
|
||||
} else {
|
||||
self.load_page(page_idx)?;
|
||||
PageState::Loaded
|
||||
};
|
||||
if state == PageState::Loaded {
|
||||
self.cycles += CYCLE_COST_PAGE;
|
||||
self.fixup_costs(node_idx, PageState::Dirty);
|
||||
self.page_states.insert(node_idx, PageState::Dirty);
|
||||
}
|
||||
self.page_cache
|
||||
.get_mut(&page_idx)
|
||||
.unwrap()
|
||||
.store(addr, word);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn commit(&mut self) -> Result<(Digest, MemoryImage2, Digest)> {
|
||||
// tracing::trace!("commit: {self:#?}");
|
||||
|
||||
let pre_state = *self.image.image_id();
|
||||
|
||||
let mut image = MemoryImage2::default();
|
||||
|
||||
// Gather the original pages
|
||||
for (&page_idx, page) in self.page_cache.iter() {
|
||||
let page_state = self.page_states[&node_idx(page_idx)];
|
||||
tracing::trace!("commit: {page_idx:#08x}, state: {page_state:?}");
|
||||
|
||||
// Copy original state of all pages accessed in this segment.
|
||||
image.set_page(page_idx, self.image.get_page(page_idx)?);
|
||||
|
||||
// Update dirty pages into the image that accumulates over a session.
|
||||
if page_state == PageState::Dirty {
|
||||
self.image.set_page(page_idx, page.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Add minimal needed 'uncles'
|
||||
for &node_idx in self.page_states.keys() {
|
||||
// If this is a leaf, break
|
||||
if node_idx >= MEMORY_PAGES as u32 {
|
||||
break;
|
||||
}
|
||||
|
||||
let lhs_idx = node_idx * 2;
|
||||
let rhs_idx = node_idx * 2 + 1;
|
||||
|
||||
// Otherwise, add whichever child digest (if any) is not loaded
|
||||
if !self.page_states.contains_key(&lhs_idx) {
|
||||
image.set_digest(lhs_idx, *self.image.get_digest(lhs_idx)?);
|
||||
}
|
||||
if !self.page_states.contains_key(&rhs_idx) {
|
||||
image.set_digest(rhs_idx, *self.image.get_digest(rhs_idx)?);
|
||||
}
|
||||
}
|
||||
|
||||
let post_state = *self.image.image_id();
|
||||
|
||||
Ok((pre_state, image, post_state))
|
||||
}
|
||||
|
||||
fn load_page(&mut self, page_idx: u32) -> Result<()> {
|
||||
tracing::trace!("load_page: {page_idx:#08x}");
|
||||
self.page_cache
|
||||
.insert(page_idx, self.image.get_page(page_idx)?);
|
||||
self.cycles += CYCLE_COST_PAGE;
|
||||
self.fixup_costs(node_idx(page_idx), PageState::Loaded);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn fixup_costs(&mut self, mut node_idx: u32, goal: PageState) {
|
||||
tracing::trace!("fixup: {node_idx:#010x}: {goal:?}");
|
||||
while node_idx != 0 {
|
||||
let state = *self
|
||||
.page_states
|
||||
.get(&node_idx)
|
||||
.unwrap_or(&PageState::Unloaded);
|
||||
if goal > state {
|
||||
if node_idx < MEMORY_PAGES as u32 {
|
||||
if state == PageState::Unloaded {
|
||||
// tracing::trace!("fixup: {state:?}: {node_idx:#010x}");
|
||||
self.cycles += CYCLE_COST_NODE;
|
||||
}
|
||||
if goal == PageState::Dirty {
|
||||
// tracing::trace!("fixup: {goal:?}: {node_idx:#010x}");
|
||||
self.cycles += CYCLE_COST_NODE;
|
||||
}
|
||||
}
|
||||
self.page_states.insert(node_idx, goal);
|
||||
}
|
||||
node_idx /= 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![allow(unused)]
|
||||
|
||||
use num_derive::FromPrimitive;
|
||||
|
||||
use super::addr::{ByteAddr, WordAddr};
|
||||
|
||||
pub const WORD_SIZE: usize = 4;
|
||||
pub const PAGE_BYTES: usize = 1024;
|
||||
pub const MEMORY_BYTES: usize = 1 << 32; // TODO: this only works on 64-bit machines
|
||||
pub const MEMORY_PAGES: usize = MEMORY_BYTES / PAGE_BYTES;
|
||||
pub const MERKLE_TREE_DEPTH: usize = MEMORY_PAGES.ilog2() as usize;
|
||||
pub const LOOKUP_TABLE_CYCLES: usize = ((1 << 8) + (1 << 16)) / 16;
|
||||
|
||||
pub const ZERO_PAGE_START_ADDR: ByteAddr = ByteAddr(0x0000_0000);
|
||||
pub const ZERO_PAGE_END_ADDR: ByteAddr = ByteAddr(0x0001_0000);
|
||||
pub const USER_START_ADDR: ByteAddr = ZERO_PAGE_END_ADDR;
|
||||
pub const USER_END_ADDR: ByteAddr = ByteAddr(0xc000_0000);
|
||||
pub const KERNEL_START_ADDR: ByteAddr = USER_END_ADDR;
|
||||
pub const KERNEL_END_ADDR: ByteAddr = ByteAddr(0xff00_0000);
|
||||
pub const MACHINE_REGS_ADDR: ByteAddr = ByteAddr(0xffff_0000);
|
||||
pub const USER_REGS_ADDR: ByteAddr = ByteAddr(0xffff_0080);
|
||||
pub const SAFE_WRITE_ADDR: ByteAddr = ByteAddr(0xffff_0100);
|
||||
pub const MEPC_ADDR: ByteAddr = ByteAddr(0xffff_0200);
|
||||
pub const SUSPEND_PC_ADDR: ByteAddr = ByteAddr(0xffff_0210);
|
||||
pub const SUSPEND_MODE_ADDR: ByteAddr = ByteAddr(0xffff_0214);
|
||||
pub const SUSPEND_CYCLE_LOW_ADDR: ByteAddr = ByteAddr(0xffff_0218);
|
||||
pub const SUSPEND_CYCLE_HIGH_ADDR: ByteAddr = ByteAddr(0xffff_021c);
|
||||
pub const GLOBAL_OUTPUT_ADDR: ByteAddr = ByteAddr(0xffff_0240);
|
||||
pub const GLOBAL_INPUT_ADDR: ByteAddr = ByteAddr(0xffff_0260);
|
||||
|
||||
pub const ECALL_DISPATCH_ADDR: ByteAddr = ByteAddr(0xffff_1000);
|
||||
pub const TRAP_DISPATCH_ADDR: ByteAddr = ByteAddr(0xffff_2000);
|
||||
|
||||
pub const MEMORY_END_ADDR: WordAddr = WordAddr(0x4000_0000);
|
||||
pub const MERKLE_TREE_START_ADDR: WordAddr = WordAddr(0x4000_0000);
|
||||
pub const MERKLE_TREE_END_ADDR: WordAddr = WordAddr(0x4400_0000);
|
||||
|
||||
pub const REG_ZERO: usize = 0; // zero constant
|
||||
pub const REG_RA: usize = 1; // return address
|
||||
pub const REG_SP: usize = 2; // stack pointer
|
||||
pub const REG_GP: usize = 3; // global pointer
|
||||
pub const REG_TP: usize = 4; // thread pointer
|
||||
pub const REG_T0: usize = 5; // temporary
|
||||
pub const REG_T1: usize = 6; // temporary
|
||||
pub const REG_T2: usize = 7; // temporary
|
||||
pub const REG_S0: usize = 8; // saved register
|
||||
pub const REG_FP: usize = 8; // frame pointer
|
||||
pub const REG_S1: usize = 9; // saved register
|
||||
pub const REG_A0: usize = 10; // fn arg / return value
|
||||
pub const REG_A1: usize = 11; // fn arg / return value
|
||||
pub const REG_A2: usize = 12; // fn arg
|
||||
pub const REG_A3: usize = 13; // fn arg
|
||||
pub const REG_A4: usize = 14; // fn arg
|
||||
pub const REG_A5: usize = 15; // fn arg
|
||||
pub const REG_A6: usize = 16; // fn arg
|
||||
pub const REG_A7: usize = 17; // fn arg
|
||||
pub const REG_S2: usize = 18; // saved register
|
||||
pub const REG_S3: usize = 19; // saved register
|
||||
pub const REG_S4: usize = 20; // saved register
|
||||
pub const REG_S5: usize = 21; // saved register
|
||||
pub const REG_S6: usize = 22; // saved register
|
||||
pub const REG_S7: usize = 23; // saved register
|
||||
pub const REG_S8: usize = 24; // saved register
|
||||
pub const REG_S9: usize = 25; // saved register
|
||||
pub const REG_S10: usize = 26; // saved register
|
||||
pub const REG_S11: usize = 27; // saved register
|
||||
pub const REG_T3: usize = 28; // temporary
|
||||
pub const REG_T4: usize = 29; // temporary
|
||||
pub const REG_T5: usize = 30; // temporary
|
||||
pub const REG_T6: usize = 31; // temporary
|
||||
pub const REG_MAX: usize = 32; // maximum number of registers
|
||||
|
||||
pub const HOST_ECALL_TERMINATE: u32 = 0;
|
||||
pub const HOST_ECALL_READ: u32 = 1;
|
||||
pub const HOST_ECALL_WRITE: u32 = 2;
|
||||
pub const HOST_ECALL_POSEIDON2: u32 = 3;
|
||||
|
||||
pub const PFLAG_IS_ELEM: u32 = 0x8000_0000;
|
||||
pub const PFLAG_CHECK_OUT: u32 = 0x4000_0000;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, FromPrimitive, PartialEq)]
|
||||
pub enum CycleState {
|
||||
#[default]
|
||||
LoadRoot = 0,
|
||||
Resume = 1,
|
||||
Suspend = 4,
|
||||
StoreRoot = 5,
|
||||
ControlTable = 6,
|
||||
ControlDone = 7,
|
||||
MachineEcall = 8,
|
||||
Terminate = 9,
|
||||
HostReadSetup = 10,
|
||||
HostWrite = 11,
|
||||
HostReadBytes = 12,
|
||||
HostReadWords = 13,
|
||||
PoseidonEntry = 16,
|
||||
PoseidonLoadState = 17,
|
||||
PoseidonLoadIn = 18,
|
||||
PoseidonDoOut = 21,
|
||||
PoseidonPaging = 22,
|
||||
PoseidonStoreState = 23,
|
||||
PoseidonExtRound = 24,
|
||||
PoseidonIntRound = 25,
|
||||
Decode = 32,
|
||||
}
|
||||
|
||||
pub const SYSCALL_MAX: u32 = 512;
|
||||
|
||||
pub const MAX_IO_BYTES: u32 = 1024;
|
||||
pub const MAX_IO_WORDS: u32 = 4;
|
||||
|
||||
/// Returns whether `addr` is within user memory bounds.
|
||||
pub fn is_user_memory(addr: ByteAddr) -> bool {
|
||||
addr >= USER_START_ADDR && addr < USER_END_ADDR
|
||||
}
|
||||
|
||||
/// Returns whether `addr` is within user memory bounds.
|
||||
pub fn is_kernel_memory(addr: ByteAddr) -> bool {
|
||||
addr >= KERNEL_START_ADDR && addr < KERNEL_END_ADDR
|
||||
}
|
||||
|
||||
pub mod major {
|
||||
pub const MISC0: u8 = 0;
|
||||
pub const MISC1: u8 = 1;
|
||||
pub const MISC2: u8 = 2;
|
||||
pub const MUL0: u8 = 3;
|
||||
pub const DIV0: u8 = 4;
|
||||
pub const MEM0: u8 = 5;
|
||||
pub const MEM1: u8 = 6;
|
||||
pub const CONTROL0: u8 = 7;
|
||||
pub const ECALL0: u8 = 8;
|
||||
pub const POSEIDON0: u8 = 9;
|
||||
pub const POSEIDON1: u8 = 10;
|
||||
}
|
||||
|
||||
pub mod control_minor {
|
||||
pub const RESUME: u8 = 1;
|
||||
pub const USER_ECALL: u8 = 2;
|
||||
pub const MRET: u8 = 3;
|
||||
}
|
||||
|
||||
pub mod ecall_minor {
|
||||
pub const MACHINE_ECALL: u8 = 0;
|
||||
pub const TERMINATE: u8 = 1;
|
||||
pub const HOST_READ_SETUP: u8 = 2;
|
||||
pub const HOST_WRITE: u8 = 3;
|
||||
pub const HOST_READ_BYTES: u8 = 4;
|
||||
pub const HOST_READ_WORDS: u8 = 5;
|
||||
}
|
||||
|
||||
pub mod poseidon_minor {
|
||||
pub const LOAD_STATE: u8 = 0;
|
||||
pub const LOAD_DATA: u8 = 1;
|
||||
pub const EXT_ROUND: u8 = 2;
|
||||
pub const INT_ROUNDS: u8 = 3;
|
||||
pub const STORE_STATE: u8 = 4;
|
||||
}
|
||||
|
||||
pub mod tx {
|
||||
pub const READ: u32 = 0;
|
||||
pub const PAGE_IN: u32 = 1;
|
||||
pub const PAGE_OUT: u32 = 2;
|
||||
}
|
||||
@@ -0,0 +1,392 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::cmp::min;
|
||||
|
||||
use anyhow::{bail, Result};
|
||||
|
||||
use super::{
|
||||
addr::{ByteAddr, WordAddr},
|
||||
platform::*,
|
||||
rv32im::{DecodedInstruction, EmuContext, Emulator, Instruction, TrapCause},
|
||||
};
|
||||
|
||||
pub trait Risc0Context {
|
||||
/// Get the program counter
|
||||
fn get_pc(&self) -> ByteAddr;
|
||||
|
||||
/// Set the program counter
|
||||
fn set_pc(&mut self, addr: ByteAddr);
|
||||
|
||||
/// Get the machine mode
|
||||
fn get_machine_mode(&self) -> u32;
|
||||
|
||||
/// Set the machine mode
|
||||
fn set_machine_mode(&mut self, mode: u32);
|
||||
|
||||
fn on_insn_start(&mut self, insn: &Instruction, decoded: &DecodedInstruction) -> Result<()>;
|
||||
|
||||
fn on_insn_end(&mut self, insn: &Instruction, decoded: &DecodedInstruction) -> Result<()>;
|
||||
|
||||
fn peek_u32(&mut self, addr: WordAddr) -> Result<u32>;
|
||||
|
||||
fn load_u32(&mut self, addr: WordAddr) -> Result<u32>;
|
||||
|
||||
fn store_u32(&mut self, addr: WordAddr, word: u32) -> Result<()>;
|
||||
|
||||
fn on_ecall_cycle(
|
||||
&mut self,
|
||||
cur: CycleState,
|
||||
next: CycleState,
|
||||
s0: u32,
|
||||
s1: u32,
|
||||
s2: u32,
|
||||
) -> Result<()>;
|
||||
|
||||
fn on_terminate(&mut self, a0: u32, a1: u32);
|
||||
|
||||
fn suspend(&mut self) -> Result<()> {
|
||||
// default no-op
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resume(&mut self) -> Result<()> {
|
||||
// default no-op
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn trap_rewind(&mut self) {
|
||||
// default no-op
|
||||
}
|
||||
|
||||
fn trap(&mut self, _cause: TrapCause) {
|
||||
// default no-op
|
||||
}
|
||||
|
||||
/// Record what was read during execution so we can replay
|
||||
fn host_read(&mut self, fd: u32, buf: &mut [u8]) -> Result<u32>;
|
||||
|
||||
/// For writes, just pass through, record rlen only
|
||||
fn host_write(&mut self, fd: u32, buf: &[u8]) -> Result<u32>;
|
||||
}
|
||||
|
||||
pub struct Risc0Machine<'a> {
|
||||
ctx: &'a mut dyn Risc0Context,
|
||||
}
|
||||
|
||||
impl<'a> Risc0Machine<'a> {
|
||||
pub fn step(emu: &mut Emulator, ctx: &'a mut dyn Risc0Context) -> Result<()> {
|
||||
emu.step(&mut Risc0Machine { ctx })
|
||||
}
|
||||
|
||||
pub fn suspend(ctx: &'a mut dyn Risc0Context) -> Result<()> {
|
||||
let mut this = Risc0Machine { ctx };
|
||||
this.store_memory(SUSPEND_PC_ADDR.waddr(), this.ctx.get_pc().0)?;
|
||||
this.store_memory(SUSPEND_MODE_ADDR.waddr(), this.ctx.get_machine_mode())?;
|
||||
this.ctx.suspend()
|
||||
}
|
||||
|
||||
pub fn resume(ctx: &'a mut dyn Risc0Context) -> Result<()> {
|
||||
let mut this = Risc0Machine { ctx };
|
||||
let pc = ByteAddr(this.load_memory(SUSPEND_PC_ADDR.waddr())?);
|
||||
let machine_mode = this.load_memory(SUSPEND_MODE_ADDR.waddr())?;
|
||||
// tracing::debug!("resume(entry: {pc:?}, mode: {machine_mode})");
|
||||
this.ctx.set_pc(pc);
|
||||
this.ctx.set_machine_mode(machine_mode);
|
||||
this.ctx.resume()
|
||||
}
|
||||
|
||||
fn is_machine_mode(&self) -> bool {
|
||||
self.ctx.get_machine_mode() != 0
|
||||
}
|
||||
|
||||
fn next_pc(&mut self) {
|
||||
self.ctx.set_pc(self.ctx.get_pc() + WORD_SIZE);
|
||||
}
|
||||
|
||||
fn machine_ecall(&mut self) -> Result<bool> {
|
||||
match self.load_register(REG_A7)? {
|
||||
HOST_ECALL_TERMINATE => self.ecall_terminate(),
|
||||
HOST_ECALL_READ => self.ecall_read(),
|
||||
HOST_ECALL_WRITE => self.ecall_write(),
|
||||
HOST_ECALL_POSEIDON2 => self.ecall_poseidon2(),
|
||||
_ => unimplemented!(),
|
||||
}
|
||||
}
|
||||
|
||||
fn user_ecall(&mut self) -> Result<bool> {
|
||||
let dispatch_idx = self.load_register(REG_A7)?;
|
||||
if dispatch_idx >= SYSCALL_MAX {
|
||||
return self.trap(TrapCause::EnvironmentCallFromUserMode);
|
||||
}
|
||||
|
||||
let dispatch_addr = ByteAddr(self.load_memory(ECALL_DISPATCH_ADDR.waddr() + dispatch_idx)?);
|
||||
if dispatch_addr.is_aligned() || dispatch_addr < KERNEL_START_ADDR {
|
||||
return self.trap(TrapCause::EnvironmentCallFromUserMode);
|
||||
}
|
||||
|
||||
self.enter_trap(dispatch_addr)?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn ecall_terminate(&mut self) -> Result<bool> {
|
||||
self.ctx
|
||||
.on_ecall_cycle(CycleState::MachineEcall, CycleState::Terminate, 0, 0, 0)?;
|
||||
let a0 = self.load_memory(USER_REGS_ADDR.waddr() + REG_A0)?;
|
||||
let a1 = self.load_memory(USER_REGS_ADDR.waddr() + REG_A1)?;
|
||||
self.ctx.on_terminate(a0, a1);
|
||||
self.ctx
|
||||
.on_ecall_cycle(CycleState::Terminate, CycleState::Suspend, 0, 0, 0)?;
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
fn ecall_read(&mut self) -> Result<bool> {
|
||||
self.ctx
|
||||
.on_ecall_cycle(CycleState::MachineEcall, CycleState::HostReadSetup, 0, 0, 0)?;
|
||||
let mut cur_state = CycleState::HostReadSetup;
|
||||
let fd = self.load_register(REG_A0)?;
|
||||
let mut ptr = ByteAddr(self.load_register(REG_A1)?);
|
||||
let len = self.load_register(REG_A2)?;
|
||||
if ptr + len < ptr {
|
||||
bail!("Invalid length in host read: {len}");
|
||||
}
|
||||
if len > MAX_IO_BYTES {
|
||||
bail!("Invalid length (too big) in host read: {len}");
|
||||
}
|
||||
let mut bytes = vec![0u8; len as usize];
|
||||
let mut rlen = self.ctx.host_read(fd, &mut bytes)?;
|
||||
self.store_register(REG_A0, rlen)?;
|
||||
if rlen == 0 {
|
||||
self.next_pc();
|
||||
}
|
||||
|
||||
fn next_io_state(ptr: ByteAddr, rlen: u32) -> CycleState {
|
||||
if rlen == 0 {
|
||||
return CycleState::Decode;
|
||||
}
|
||||
if !ptr.is_aligned() || rlen < WORD_SIZE as u32 {
|
||||
return CycleState::HostReadBytes;
|
||||
}
|
||||
CycleState::HostReadWords
|
||||
}
|
||||
|
||||
let next_state = next_io_state(ptr, rlen);
|
||||
self.ctx
|
||||
.on_ecall_cycle(cur_state, next_state, ptr.waddr().0, ptr.subaddr(), rlen)?;
|
||||
cur_state = next_state;
|
||||
|
||||
let mut i = 0;
|
||||
|
||||
while rlen > 0 && !ptr.is_aligned() {
|
||||
self.store_u8(ptr, bytes[i])?;
|
||||
ptr += 1u32;
|
||||
i += 1;
|
||||
rlen -= 1;
|
||||
}
|
||||
|
||||
while rlen >= MAX_IO_WORDS {
|
||||
let words = min(rlen / MAX_IO_WORDS, MAX_IO_WORDS);
|
||||
for j in 0..MAX_IO_WORDS {
|
||||
if j < words {
|
||||
let word = u32::from_le_bytes(bytes[i..i + WORD_SIZE].try_into()?);
|
||||
self.store_memory(ptr.waddr(), word)?;
|
||||
} else {
|
||||
self.store_memory(SAFE_WRITE_ADDR.waddr(), 0)?;
|
||||
}
|
||||
ptr += words;
|
||||
i += words as usize;
|
||||
rlen -= words;
|
||||
}
|
||||
|
||||
if rlen == 0 {
|
||||
self.next_pc();
|
||||
}
|
||||
|
||||
let next_state = next_io_state(ptr, rlen);
|
||||
self.ctx
|
||||
.on_ecall_cycle(cur_state, next_state, ptr.waddr().0, ptr.subaddr(), rlen)?;
|
||||
cur_state = next_state;
|
||||
}
|
||||
|
||||
while rlen > 0 && !ptr.is_aligned() {
|
||||
self.store_u8(ptr, bytes[i])?;
|
||||
ptr += 1u32;
|
||||
i += 1;
|
||||
rlen -= 1;
|
||||
}
|
||||
|
||||
// Ok(true)
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
fn ecall_write(&mut self) -> Result<bool> {
|
||||
self.ctx
|
||||
.on_ecall_cycle(CycleState::MachineEcall, CycleState::HostWrite, 0, 0, 0)?;
|
||||
let fd = self.load_register(REG_A0)?;
|
||||
let ptr = ByteAddr(self.load_register(REG_A1)?);
|
||||
let len = self.load_register(REG_A2)?;
|
||||
if ptr + len < ptr {
|
||||
bail!("Invalid length in host write: {len}");
|
||||
}
|
||||
if len > MAX_IO_BYTES {
|
||||
bail!("Invalid length (too big) in host write: {len}");
|
||||
}
|
||||
let bytes = self.peek(ptr, len as usize)?;
|
||||
let rlen = self.ctx.host_write(fd, &bytes)?;
|
||||
self.store_register(REG_A0, rlen)?;
|
||||
self.next_pc();
|
||||
self.ctx
|
||||
.on_ecall_cycle(CycleState::HostWrite, CycleState::Decode, 0, 0, 0)?;
|
||||
// Ok(true)
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
fn ecall_poseidon2(&mut self) -> Result<bool> {
|
||||
self.next_pc();
|
||||
self.ctx
|
||||
.on_ecall_cycle(CycleState::MachineEcall, CycleState::PoseidonEntry, 0, 0, 0)?;
|
||||
// Ok(true)
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
fn enter_trap(&mut self, dispatch_addr: ByteAddr) -> Result<()> {
|
||||
if self.is_machine_mode() {
|
||||
bail!("Illegal trap in machine mode");
|
||||
}
|
||||
let pc = self.ctx.get_pc();
|
||||
self.store_memory(MEPC_ADDR.waddr(), pc.0)?;
|
||||
self.ctx.set_pc(dispatch_addr);
|
||||
self.ctx.set_machine_mode(1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn peek(&mut self, ptr: ByteAddr, len: usize) -> Result<Vec<u8>> {
|
||||
let mut bytes = vec![0u8; len];
|
||||
for (i, byte) in bytes.iter_mut().enumerate().take(len) {
|
||||
*byte = self.peek_u8(ptr + i)?;
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn peek_u8(&mut self, ptr: ByteAddr) -> Result<u8> {
|
||||
let word = self.ctx.peek_u32(ptr.waddr())?;
|
||||
let bytes = word.to_le_bytes();
|
||||
let offset = ptr.subaddr() as usize;
|
||||
Ok(bytes[offset])
|
||||
}
|
||||
|
||||
fn store_u8(&mut self, addr: ByteAddr, byte: u8) -> Result<()> {
|
||||
let byte_offset = addr.subaddr() as usize;
|
||||
let word = self.load_memory(addr.waddr())?;
|
||||
let mut bytes = word.to_le_bytes();
|
||||
bytes[byte_offset] = byte;
|
||||
let word = u32::from_le_bytes(bytes);
|
||||
self.store_memory(addr.waddr(), word)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> EmuContext for Risc0Machine<'a> {
|
||||
fn ecall(&mut self) -> Result<bool> {
|
||||
if self.is_machine_mode() {
|
||||
self.machine_ecall()
|
||||
} else {
|
||||
self.user_ecall()
|
||||
}
|
||||
}
|
||||
|
||||
fn mret(&mut self) -> Result<bool> {
|
||||
if !self.is_machine_mode() {
|
||||
bail!("Illegal mret in user mode");
|
||||
}
|
||||
let dispatch_addr = ByteAddr(self.load_memory(MEPC_ADDR.waddr())?);
|
||||
self.ctx.set_pc(dispatch_addr + WORD_SIZE);
|
||||
self.ctx.set_machine_mode(0);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn trap(&mut self, cause: TrapCause) -> Result<bool> {
|
||||
self.ctx.trap_rewind();
|
||||
let dispatch_addr =
|
||||
ByteAddr(self.load_memory(TRAP_DISPATCH_ADDR.waddr() + cause.as_u32())?);
|
||||
if !dispatch_addr.is_aligned() || !is_kernel_memory(dispatch_addr) {
|
||||
bail!("Invalid trap address: {dispatch_addr:?}, cause: {cause:?}");
|
||||
}
|
||||
self.enter_trap(dispatch_addr)?;
|
||||
self.ctx.trap(cause);
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
fn on_insn_decoded(&mut self, insn: &Instruction, decoded: &DecodedInstruction) -> Result<()> {
|
||||
self.ctx.on_insn_start(insn, decoded)
|
||||
}
|
||||
|
||||
fn on_normal_end(&mut self, insn: &Instruction, decoded: &DecodedInstruction) -> Result<()> {
|
||||
self.ctx.on_insn_end(insn, decoded)
|
||||
}
|
||||
|
||||
fn get_pc(&self) -> ByteAddr {
|
||||
self.ctx.get_pc()
|
||||
}
|
||||
|
||||
fn set_pc(&mut self, addr: ByteAddr) {
|
||||
self.ctx.set_pc(addr);
|
||||
}
|
||||
|
||||
fn load_register(&mut self, idx: usize) -> Result<u32> {
|
||||
// tracing::trace!("load_reg: x{idx}");
|
||||
let base = if self.is_machine_mode() {
|
||||
MACHINE_REGS_ADDR.waddr()
|
||||
} else {
|
||||
USER_REGS_ADDR.waddr()
|
||||
};
|
||||
self.ctx.load_u32(base + idx)
|
||||
}
|
||||
|
||||
fn store_register(&mut self, idx: usize, word: u32) -> Result<()> {
|
||||
// tracing::trace!("store_reg: x{idx} <= {word:#010x}");
|
||||
let mut base = if self.is_machine_mode() {
|
||||
MACHINE_REGS_ADDR.waddr()
|
||||
} else {
|
||||
USER_REGS_ADDR.waddr()
|
||||
};
|
||||
|
||||
// To avoid the use of a degree in the circuit, all writes to REG_ZERO
|
||||
// are shunted to a memory location that is never read from.
|
||||
if idx == REG_ZERO {
|
||||
base += REG_MAX * 2;
|
||||
}
|
||||
|
||||
self.ctx.store_u32(base + idx, word)
|
||||
}
|
||||
|
||||
fn load_memory(&mut self, addr: WordAddr) -> Result<u32> {
|
||||
self.ctx.load_u32(addr)
|
||||
}
|
||||
|
||||
fn store_memory(&mut self, addr: WordAddr, word: u32) -> Result<()> {
|
||||
self.ctx.store_u32(addr, word)
|
||||
}
|
||||
|
||||
fn check_insn_load(&self, addr: ByteAddr) -> bool {
|
||||
!(addr < ZERO_PAGE_END_ADDR || (!self.is_machine_mode() && addr >= KERNEL_START_ADDR))
|
||||
}
|
||||
|
||||
fn check_data_load(&self, addr: ByteAddr) -> bool {
|
||||
self.is_machine_mode() || is_user_memory(addr)
|
||||
}
|
||||
|
||||
fn check_data_store(&self, addr: ByteAddr) -> bool {
|
||||
self.check_data_load(addr)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,711 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
use super::{
|
||||
addr::{ByteAddr, WordAddr},
|
||||
platform::WORD_SIZE,
|
||||
};
|
||||
|
||||
pub trait EmuContext {
|
||||
// Handle environment call
|
||||
fn ecall(&mut self) -> Result<bool>;
|
||||
|
||||
// Handle a machine return
|
||||
fn mret(&mut self) -> Result<bool>;
|
||||
|
||||
// Handle a trap
|
||||
fn trap(&mut self, cause: TrapCause) -> Result<bool>;
|
||||
|
||||
// Callback when instructions are decoded
|
||||
fn on_insn_decoded(&mut self, insn: &Instruction, decoded: &DecodedInstruction) -> Result<()>;
|
||||
|
||||
// Callback when instructions end normally
|
||||
fn on_normal_end(&mut self, insn: &Instruction, decoded: &DecodedInstruction) -> Result<()>;
|
||||
|
||||
// Get the program counter
|
||||
fn get_pc(&self) -> ByteAddr;
|
||||
|
||||
// Set the program counter
|
||||
fn set_pc(&mut self, addr: ByteAddr);
|
||||
|
||||
// Load from a register
|
||||
fn load_register(&mut self, idx: usize) -> Result<u32>;
|
||||
|
||||
// Store to a register
|
||||
fn store_register(&mut self, idx: usize, word: u32) -> Result<()>;
|
||||
|
||||
// Load from memory
|
||||
fn load_memory(&mut self, addr: WordAddr) -> Result<u32>;
|
||||
|
||||
// Store to memory
|
||||
fn store_memory(&mut self, addr: WordAddr, word: u32) -> Result<()>;
|
||||
|
||||
// Check access for instruction load
|
||||
fn check_insn_load(&self, _addr: ByteAddr) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
// Check access for data load
|
||||
fn check_data_load(&self, _addr: ByteAddr) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
// Check access for data store
|
||||
fn check_data_store(&self, _addr: ByteAddr) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Emulator {
|
||||
table: FastDecodeTable,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[repr(u32)]
|
||||
pub enum TrapCause {
|
||||
InstructionAddressMisaligned = 0,
|
||||
InstructionAccessFault,
|
||||
#[allow(dead_code)]
|
||||
IllegalInstruction(u32),
|
||||
Breakpoint,
|
||||
LoadAddressMisaligned,
|
||||
#[allow(dead_code)]
|
||||
LoadAccessFault(ByteAddr),
|
||||
#[allow(dead_code)]
|
||||
StoreAddressMisaligned(ByteAddr),
|
||||
StoreAccessFault,
|
||||
EnvironmentCallFromUserMode,
|
||||
}
|
||||
|
||||
impl TrapCause {
|
||||
pub fn as_u32(&self) -> u32 {
|
||||
unsafe { *(self as *const Self as *const u32) }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct DecodedInstruction {
|
||||
pub insn: u32,
|
||||
top_bit: u32,
|
||||
func7: u32,
|
||||
rs2: u32,
|
||||
rs1: u32,
|
||||
func3: u32,
|
||||
rd: u32,
|
||||
opcode: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum InsnCategory {
|
||||
Compute,
|
||||
Load,
|
||||
Store,
|
||||
System,
|
||||
Invalid,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub enum InsnKind {
|
||||
Add = 0, // major: 0, minor: 0
|
||||
Sub = 1, // major: 0, minor: 1
|
||||
Xor = 2, // major: 0, minor: 2
|
||||
Or = 3, // major: 0, minor: 3
|
||||
And = 4, // major: 0, minor: 4
|
||||
Slt = 5, // major: 0, minor: 5
|
||||
SltU = 6, // major: 0, minor: 6
|
||||
AddI = 7, // major: 0, minor: 7
|
||||
|
||||
XorI = 8, // major: 1, minor: 0
|
||||
OrI = 9, // major: 1, minor: 1
|
||||
AndI = 10, // major: 1, minor: 2
|
||||
SltI = 11, // major: 1, minor: 3
|
||||
SltIU = 12, // major: 1, minor: 4
|
||||
Beq = 13, // major: 1, minor: 5
|
||||
Bne = 14, // major: 1, minor: 6
|
||||
Blt = 15, // major: 1, minor: 7
|
||||
|
||||
Bge = 16, // major: 2, minor: 0
|
||||
BltU = 17, // major: 2, minor: 1
|
||||
BgeU = 18, // major: 2, minor: 2
|
||||
Jal = 19, // major: 2, minor: 3
|
||||
JalR = 20, // major: 2, minor: 4
|
||||
Lui = 21, // major: 2, minor: 5
|
||||
Auipc = 22, // major: 2, minor: 6
|
||||
|
||||
Sll = 24, // major: 3, minor: 0
|
||||
SllI = 25, // major: 3, minor: 1
|
||||
Mul = 26, // major: 3, minor: 2
|
||||
MulH = 27, // major: 3, minor: 3
|
||||
MulHSU = 28, // major: 3, minor: 4
|
||||
MulHU = 29, // major: 3, minor: 5
|
||||
|
||||
Srl = 32, // major: 4, minor: 0
|
||||
Sra = 33, // major: 4, minor: 1
|
||||
SrlI = 34, // major: 4, minor: 2
|
||||
SraI = 35, // major: 4, minor: 3
|
||||
Div = 36, // major: 4, minor: 4
|
||||
DivU = 37, // major: 4, minor: 5
|
||||
Rem = 38, // major: 4, minor: 6
|
||||
RemU = 39, // major: 4, minor: 7
|
||||
|
||||
Lb = 40, // major: 5, minor: 0
|
||||
Lh = 41, // major: 5, minor: 1
|
||||
Lw = 42, // major: 5, minor: 2
|
||||
LbU = 43, // major: 5, minor: 3
|
||||
LhU = 44, // major: 5, minor: 4
|
||||
|
||||
Sb = 48, // major: 6, minor: 0
|
||||
Sh = 49, // major: 6, minor: 1
|
||||
Sw = 50, // major: 6, minor: 2
|
||||
|
||||
Eany = 56, // major: 7, minor: 0
|
||||
Mret = 57, // major: 7, minor: 1
|
||||
|
||||
Invalid = 255,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Instruction {
|
||||
pub kind: InsnKind,
|
||||
category: InsnCategory,
|
||||
pub opcode: u32,
|
||||
pub func3: u32,
|
||||
pub func7: u32,
|
||||
}
|
||||
|
||||
impl DecodedInstruction {
|
||||
fn new(insn: u32) -> Self {
|
||||
Self {
|
||||
insn,
|
||||
top_bit: (insn & 0x80000000) >> 31,
|
||||
func7: (insn & 0xfe000000) >> 25,
|
||||
rs2: (insn & 0x01f00000) >> 20,
|
||||
rs1: (insn & 0x000f8000) >> 15,
|
||||
func3: (insn & 0x00007000) >> 12,
|
||||
rd: (insn & 0x00000f80) >> 7,
|
||||
opcode: insn & 0x0000007f,
|
||||
}
|
||||
}
|
||||
|
||||
fn imm_b(&self) -> u32 {
|
||||
(self.top_bit * 0xfffff000)
|
||||
| ((self.rd & 1) << 11)
|
||||
| ((self.func7 & 0x3f) << 5)
|
||||
| (self.rd & 0x1e)
|
||||
}
|
||||
|
||||
fn imm_i(&self) -> u32 {
|
||||
(self.top_bit * 0xfffff000) | (self.func7 << 5) | self.rs2
|
||||
}
|
||||
|
||||
fn imm_s(&self) -> u32 {
|
||||
(self.top_bit * 0xfffff000) | (self.func7 << 5) | self.rd
|
||||
}
|
||||
|
||||
fn imm_j(&self) -> u32 {
|
||||
(self.top_bit * 0xfff00000)
|
||||
| (self.rs1 << 15)
|
||||
| (self.func3 << 12)
|
||||
| ((self.rs2 & 1) << 11)
|
||||
| ((self.func7 & 0x3f) << 5)
|
||||
| (self.rs2 & 0x1e)
|
||||
}
|
||||
|
||||
fn imm_u(&self) -> u32 {
|
||||
self.insn & 0xfffff000
|
||||
}
|
||||
}
|
||||
|
||||
const fn insn(
|
||||
kind: InsnKind,
|
||||
category: InsnCategory,
|
||||
opcode: u32,
|
||||
func3: i32,
|
||||
func7: i32,
|
||||
) -> Instruction {
|
||||
Instruction {
|
||||
kind,
|
||||
category,
|
||||
opcode,
|
||||
func3: func3 as u32,
|
||||
func7: func7 as u32,
|
||||
}
|
||||
}
|
||||
|
||||
type InstructionTable = [Instruction; 48];
|
||||
type FastInstructionTable = [u8; 1 << 10];
|
||||
|
||||
const RV32IM_ISA: InstructionTable = [
|
||||
insn(InsnKind::Invalid, InsnCategory::Invalid, 0x00, 0x0, 0x00),
|
||||
insn(InsnKind::Add, InsnCategory::Compute, 0x33, 0x0, 0x00),
|
||||
insn(InsnKind::Sub, InsnCategory::Compute, 0x33, 0x0, 0x20),
|
||||
insn(InsnKind::Xor, InsnCategory::Compute, 0x33, 0x4, 0x00),
|
||||
insn(InsnKind::Or, InsnCategory::Compute, 0x33, 0x6, 0x00),
|
||||
insn(InsnKind::And, InsnCategory::Compute, 0x33, 0x7, 0x00),
|
||||
insn(InsnKind::Sll, InsnCategory::Compute, 0x33, 0x1, 0x00),
|
||||
insn(InsnKind::Srl, InsnCategory::Compute, 0x33, 0x5, 0x00),
|
||||
insn(InsnKind::Sra, InsnCategory::Compute, 0x33, 0x5, 0x20),
|
||||
insn(InsnKind::Slt, InsnCategory::Compute, 0x33, 0x2, 0x00),
|
||||
insn(InsnKind::SltU, InsnCategory::Compute, 0x33, 0x3, 0x00),
|
||||
insn(InsnKind::AddI, InsnCategory::Compute, 0x13, 0x0, -1),
|
||||
insn(InsnKind::XorI, InsnCategory::Compute, 0x13, 0x4, -1),
|
||||
insn(InsnKind::OrI, InsnCategory::Compute, 0x13, 0x6, -1),
|
||||
insn(InsnKind::AndI, InsnCategory::Compute, 0x13, 0x7, -1),
|
||||
insn(InsnKind::SllI, InsnCategory::Compute, 0x13, 0x1, 0x00),
|
||||
insn(InsnKind::SrlI, InsnCategory::Compute, 0x13, 0x5, 0x00),
|
||||
insn(InsnKind::SraI, InsnCategory::Compute, 0x13, 0x5, 0x20),
|
||||
insn(InsnKind::SltI, InsnCategory::Compute, 0x13, 0x2, -1),
|
||||
insn(InsnKind::SltIU, InsnCategory::Compute, 0x13, 0x3, -1),
|
||||
insn(InsnKind::Beq, InsnCategory::Compute, 0x63, 0x0, -1),
|
||||
insn(InsnKind::Bne, InsnCategory::Compute, 0x63, 0x1, -1),
|
||||
insn(InsnKind::Blt, InsnCategory::Compute, 0x63, 0x4, -1),
|
||||
insn(InsnKind::Bge, InsnCategory::Compute, 0x63, 0x5, -1),
|
||||
insn(InsnKind::BltU, InsnCategory::Compute, 0x63, 0x6, -1),
|
||||
insn(InsnKind::BgeU, InsnCategory::Compute, 0x63, 0x7, -1),
|
||||
insn(InsnKind::Jal, InsnCategory::Compute, 0x6f, -1, -1),
|
||||
insn(InsnKind::JalR, InsnCategory::Compute, 0x67, 0x0, -1),
|
||||
insn(InsnKind::Lui, InsnCategory::Compute, 0x37, -1, -1),
|
||||
insn(InsnKind::Auipc, InsnCategory::Compute, 0x17, -1, -1),
|
||||
insn(InsnKind::Mul, InsnCategory::Compute, 0x33, 0x0, 0x01),
|
||||
insn(InsnKind::MulH, InsnCategory::Compute, 0x33, 0x1, 0x01),
|
||||
insn(InsnKind::MulHSU, InsnCategory::Compute, 0x33, 0x2, 0x01),
|
||||
insn(InsnKind::MulHU, InsnCategory::Compute, 0x33, 0x3, 0x01),
|
||||
insn(InsnKind::Div, InsnCategory::Compute, 0x33, 0x4, 0x01),
|
||||
insn(InsnKind::DivU, InsnCategory::Compute, 0x33, 0x5, 0x01),
|
||||
insn(InsnKind::Rem, InsnCategory::Compute, 0x33, 0x6, 0x01),
|
||||
insn(InsnKind::RemU, InsnCategory::Compute, 0x33, 0x7, 0x01),
|
||||
insn(InsnKind::Lb, InsnCategory::Load, 0x03, 0x0, -1),
|
||||
insn(InsnKind::Lh, InsnCategory::Load, 0x03, 0x1, -1),
|
||||
insn(InsnKind::Lw, InsnCategory::Load, 0x03, 0x2, -1),
|
||||
insn(InsnKind::LbU, InsnCategory::Load, 0x03, 0x4, -1),
|
||||
insn(InsnKind::LhU, InsnCategory::Load, 0x03, 0x5, -1),
|
||||
insn(InsnKind::Sb, InsnCategory::Store, 0x23, 0x0, -1),
|
||||
insn(InsnKind::Sh, InsnCategory::Store, 0x23, 0x1, -1),
|
||||
insn(InsnKind::Sw, InsnCategory::Store, 0x23, 0x2, -1),
|
||||
insn(InsnKind::Eany, InsnCategory::System, 0x73, 0x0, 0x00),
|
||||
insn(InsnKind::Mret, InsnCategory::System, 0x73, 0x0, 0x18),
|
||||
];
|
||||
|
||||
// RISC-V instruction are determined by 3 parts:
|
||||
// - Opcode: 7 bits
|
||||
// - Func3: 3 bits
|
||||
// - Func7: 7 bits
|
||||
// In many cases, func7 and/or func3 is ignored. A standard trick is to decode
|
||||
// via a table, but a 17 bit lookup table destroys L1 cache. Luckily for us,
|
||||
// in practice the low 2 bits of opcode are always 11, so we can drop them, and
|
||||
// also func7 is always either 0, 1, 0x20 or don't care, so we can reduce func7
|
||||
// to 2 bits, which gets us to 10 bits, which is only 1k.
|
||||
struct FastDecodeTable {
|
||||
table: FastInstructionTable,
|
||||
}
|
||||
|
||||
impl Default for FastDecodeTable {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl FastDecodeTable {
|
||||
fn new() -> Self {
|
||||
let mut table: FastInstructionTable = [InsnKind::Invalid as u8; 1 << 10];
|
||||
for (isa_idx, insn) in RV32IM_ISA.iter().enumerate() {
|
||||
Self::add_insn(&mut table, insn, isa_idx);
|
||||
}
|
||||
Self { table }
|
||||
}
|
||||
|
||||
// Map to 10 bit format
|
||||
fn map10(opcode: u32, func3: u32, func7: u32) -> usize {
|
||||
let op_high = opcode >> 2;
|
||||
// Map 0 -> 0, 1 -> 1, 0x20 -> 2, everything else to 3
|
||||
let func72bits = if func7 <= 1 {
|
||||
func7
|
||||
} else if func7 == 0x20 {
|
||||
2
|
||||
} else {
|
||||
3
|
||||
};
|
||||
((op_high << 5) | (func72bits << 3) | func3) as usize
|
||||
}
|
||||
|
||||
fn add_insn(table: &mut FastInstructionTable, insn: &Instruction, isa_idx: usize) {
|
||||
let op_high = insn.opcode >> 2;
|
||||
if (insn.func3 as i32) < 0 {
|
||||
for f3 in 0..8 {
|
||||
for f7b in 0..4 {
|
||||
let idx = (op_high << 5) | (f7b << 3) | f3;
|
||||
table[idx as usize] = isa_idx as u8;
|
||||
}
|
||||
}
|
||||
} else if (insn.func7 as i32) < 0 {
|
||||
for f7b in 0..4 {
|
||||
let idx = (op_high << 5) | (f7b << 3) | insn.func3;
|
||||
table[idx as usize] = isa_idx as u8;
|
||||
}
|
||||
} else {
|
||||
table[Self::map10(insn.opcode, insn.func3, insn.func7)] = isa_idx as u8;
|
||||
}
|
||||
}
|
||||
|
||||
fn lookup(&self, decoded: &DecodedInstruction) -> Instruction {
|
||||
let isa_idx = self.table[Self::map10(decoded.opcode, decoded.func3, decoded.func7)];
|
||||
RV32IM_ISA[isa_idx as usize]
|
||||
}
|
||||
}
|
||||
|
||||
impl Emulator {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
table: FastDecodeTable::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn step<C: EmuContext>(&mut self, ctx: &mut C) -> Result<()> {
|
||||
let pc = ctx.get_pc();
|
||||
|
||||
if !ctx.check_insn_load(pc) {
|
||||
ctx.trap(TrapCause::InstructionAccessFault)?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let word = ctx.load_memory(pc.waddr())?;
|
||||
if word & 0x03 != 0x03 {
|
||||
ctx.trap(TrapCause::IllegalInstruction(word))?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let decoded = DecodedInstruction::new(word);
|
||||
let insn = self.table.lookup(&decoded);
|
||||
ctx.on_insn_decoded(&insn, &decoded)?;
|
||||
|
||||
if match insn.category {
|
||||
InsnCategory::Compute => self.step_compute(ctx, insn.kind, &decoded)?,
|
||||
InsnCategory::Load => self.step_load(ctx, insn.kind, &decoded)?,
|
||||
InsnCategory::Store => self.step_store(ctx, insn.kind, &decoded)?,
|
||||
InsnCategory::System => self.step_system(ctx, insn.kind, &decoded)?,
|
||||
InsnCategory::Invalid => ctx.trap(TrapCause::IllegalInstruction(word))?,
|
||||
} {
|
||||
ctx.on_normal_end(&insn, &decoded)?;
|
||||
};
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn step_compute<M: EmuContext>(
|
||||
&mut self,
|
||||
ctx: &mut M,
|
||||
kind: InsnKind,
|
||||
decoded: &DecodedInstruction,
|
||||
) -> Result<bool> {
|
||||
let pc = ctx.get_pc();
|
||||
let mut new_pc = pc + WORD_SIZE;
|
||||
let mut rd = decoded.rd;
|
||||
let rs1 = ctx.load_register(decoded.rs1 as usize)?;
|
||||
let rs2 = ctx.load_register(decoded.rs2 as usize)?;
|
||||
let imm_i = decoded.imm_i();
|
||||
let mut br_cond = |cond| -> u32 {
|
||||
rd = 0;
|
||||
if cond {
|
||||
new_pc = pc.wrapping_add(decoded.imm_b());
|
||||
}
|
||||
0
|
||||
};
|
||||
let out = match kind {
|
||||
InsnKind::Add => rs1.wrapping_add(rs2),
|
||||
InsnKind::Sub => rs1.wrapping_sub(rs2),
|
||||
InsnKind::Xor => rs1 ^ rs2,
|
||||
InsnKind::Or => rs1 | rs2,
|
||||
InsnKind::And => rs1 & rs2,
|
||||
InsnKind::Sll => rs1 << (rs2 & 0x1f),
|
||||
InsnKind::Srl => rs1 >> (rs2 & 0x1f),
|
||||
InsnKind::Sra => ((rs1 as i32) >> (rs2 & 0x1f)) as u32,
|
||||
InsnKind::Slt => {
|
||||
if (rs1 as i32) < (rs2 as i32) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
InsnKind::SltU => {
|
||||
if rs1 < rs2 {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
InsnKind::AddI => rs1.wrapping_add(imm_i),
|
||||
InsnKind::XorI => rs1 ^ imm_i,
|
||||
InsnKind::OrI => rs1 | imm_i,
|
||||
InsnKind::AndI => rs1 & imm_i,
|
||||
InsnKind::SllI => rs1 << (imm_i & 0x1f),
|
||||
InsnKind::SrlI => rs1 >> (imm_i & 0x1f),
|
||||
InsnKind::SraI => ((rs1 as i32) >> (imm_i & 0x1f)) as u32,
|
||||
InsnKind::SltI => {
|
||||
if (rs1 as i32) < (imm_i as i32) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
InsnKind::SltIU => {
|
||||
if rs1 < imm_i {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
InsnKind::Beq => br_cond(rs1 == rs2),
|
||||
InsnKind::Bne => br_cond(rs1 != rs2),
|
||||
InsnKind::Blt => br_cond((rs1 as i32) < (rs2 as i32)),
|
||||
InsnKind::Bge => br_cond((rs1 as i32) >= (rs2 as i32)),
|
||||
InsnKind::BltU => br_cond(rs1 < rs2),
|
||||
InsnKind::BgeU => br_cond(rs1 >= rs2),
|
||||
InsnKind::Jal => {
|
||||
new_pc = pc.wrapping_add(decoded.imm_j());
|
||||
(pc + WORD_SIZE).0
|
||||
}
|
||||
InsnKind::JalR => {
|
||||
new_pc = ByteAddr(rs1.wrapping_add(imm_i) & 0xfffffffe);
|
||||
(pc + WORD_SIZE).0
|
||||
}
|
||||
InsnKind::Lui => decoded.imm_u(),
|
||||
InsnKind::Auipc => (pc.wrapping_add(decoded.imm_u())).0,
|
||||
InsnKind::Mul => rs1.wrapping_mul(rs2),
|
||||
InsnKind::MulH => {
|
||||
(sign_extend_u32(rs1).wrapping_mul(sign_extend_u32(rs2)) >> 32) as u32
|
||||
}
|
||||
InsnKind::MulHSU => (sign_extend_u32(rs1).wrapping_mul(rs2 as i64) >> 32) as u32,
|
||||
InsnKind::MulHU => (((rs1 as u64).wrapping_mul(rs2 as u64)) >> 32) as u32,
|
||||
InsnKind::Div => {
|
||||
if rs2 == 0 {
|
||||
u32::MAX
|
||||
} else {
|
||||
((rs1 as i32).wrapping_div(rs2 as i32)) as u32
|
||||
}
|
||||
}
|
||||
InsnKind::DivU => {
|
||||
if rs2 == 0 {
|
||||
u32::MAX
|
||||
} else {
|
||||
rs1 / rs2
|
||||
}
|
||||
}
|
||||
InsnKind::Rem => {
|
||||
if rs2 == 0 {
|
||||
rs1
|
||||
} else {
|
||||
((rs1 as i32).wrapping_rem(rs2 as i32)) as u32
|
||||
}
|
||||
}
|
||||
InsnKind::RemU => {
|
||||
if rs2 == 0 {
|
||||
rs1
|
||||
} else {
|
||||
rs1 % rs2
|
||||
}
|
||||
}
|
||||
_ => unreachable!(),
|
||||
};
|
||||
if !new_pc.is_aligned() {
|
||||
return ctx.trap(TrapCause::InstructionAddressMisaligned);
|
||||
}
|
||||
ctx.store_register(rd as usize, out)?;
|
||||
ctx.set_pc(new_pc);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn step_load<M: EmuContext>(
|
||||
&mut self,
|
||||
ctx: &mut M,
|
||||
kind: InsnKind,
|
||||
decoded: &DecodedInstruction,
|
||||
) -> Result<bool> {
|
||||
let rs1 = ctx.load_register(decoded.rs1 as usize)?;
|
||||
let addr = ByteAddr(rs1.wrapping_add(decoded.imm_i()));
|
||||
if !ctx.check_data_load(addr) {
|
||||
return ctx.trap(TrapCause::LoadAccessFault(addr));
|
||||
}
|
||||
let data = ctx.load_memory(addr.waddr())?;
|
||||
let shift = 8 * (addr.0 & 3);
|
||||
let out = match kind {
|
||||
InsnKind::Lb => {
|
||||
let mut out = (data >> shift) & 0xff;
|
||||
if out & 0x80 != 0 {
|
||||
out |= 0xffffff00;
|
||||
}
|
||||
out
|
||||
}
|
||||
InsnKind::Lh => {
|
||||
if addr.0 & 0x01 != 0 {
|
||||
return ctx.trap(TrapCause::LoadAddressMisaligned);
|
||||
}
|
||||
let mut out = (data >> shift) & 0xffff;
|
||||
if out & 0x8000 != 0 {
|
||||
out |= 0xffff0000;
|
||||
}
|
||||
out
|
||||
}
|
||||
InsnKind::Lw => {
|
||||
if addr.0 & 0x03 != 0 {
|
||||
return ctx.trap(TrapCause::LoadAddressMisaligned);
|
||||
}
|
||||
data
|
||||
}
|
||||
InsnKind::LbU => (data >> shift) & 0xff,
|
||||
InsnKind::LhU => {
|
||||
if addr.0 & 0x01 != 0 {
|
||||
return ctx.trap(TrapCause::LoadAddressMisaligned);
|
||||
}
|
||||
(data >> shift) & 0xffff
|
||||
}
|
||||
_ => unreachable!(),
|
||||
};
|
||||
ctx.store_register(decoded.rd as usize, out)?;
|
||||
ctx.set_pc(ctx.get_pc() + WORD_SIZE);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn step_store<M: EmuContext>(
|
||||
&mut self,
|
||||
ctx: &mut M,
|
||||
kind: InsnKind,
|
||||
decoded: &DecodedInstruction,
|
||||
) -> Result<bool> {
|
||||
let rs1 = ctx.load_register(decoded.rs1 as usize)?;
|
||||
let rs2 = ctx.load_register(decoded.rs2 as usize)?;
|
||||
let addr = ByteAddr(rs1.wrapping_add(decoded.imm_s()));
|
||||
let shift = 8 * (addr.0 & 3);
|
||||
if !ctx.check_data_store(addr) {
|
||||
return ctx.trap(TrapCause::StoreAccessFault);
|
||||
}
|
||||
let mut data = ctx.load_memory(addr.waddr())?;
|
||||
match kind {
|
||||
InsnKind::Sb => {
|
||||
data ^= data & (0xff << shift);
|
||||
data |= (rs2 & 0xff) << shift;
|
||||
}
|
||||
InsnKind::Sh => {
|
||||
if addr.0 & 0x01 != 0 {
|
||||
tracing::debug!("Misaligned SH");
|
||||
return ctx.trap(TrapCause::StoreAddressMisaligned(addr));
|
||||
}
|
||||
data ^= data & (0xffff << shift);
|
||||
data |= (rs2 & 0xffff) << shift;
|
||||
}
|
||||
InsnKind::Sw => {
|
||||
if addr.0 & 0x03 != 0 {
|
||||
tracing::debug!("Misaligned SW");
|
||||
return ctx.trap(TrapCause::StoreAddressMisaligned(addr));
|
||||
}
|
||||
data = rs2;
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
ctx.store_memory(addr.waddr(), data)?;
|
||||
ctx.set_pc(ctx.get_pc() + WORD_SIZE);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn step_system<M: EmuContext>(
|
||||
&mut self,
|
||||
ctx: &mut M,
|
||||
kind: InsnKind,
|
||||
decoded: &DecodedInstruction,
|
||||
) -> Result<bool> {
|
||||
match kind {
|
||||
InsnKind::Eany => match decoded.rs2 {
|
||||
0 => ctx.ecall(),
|
||||
1 => ctx.trap(TrapCause::Breakpoint),
|
||||
_ => ctx.trap(TrapCause::IllegalInstruction(decoded.insn)),
|
||||
},
|
||||
InsnKind::Mret => ctx.mret(),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn sign_extend_u32(x: u32) -> i64 {
|
||||
(x as i32) as i64
|
||||
}
|
||||
|
||||
pub fn disasm(insn: &Instruction, decoded: &DecodedInstruction) -> String {
|
||||
let (rd, rs1, rs2) = (decoded.rd, decoded.rs1, decoded.rs2);
|
||||
match insn.kind {
|
||||
InsnKind::Invalid => "illegal".to_string(),
|
||||
InsnKind::Add => format!("add x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::Sub => format!("sub x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::Xor => format!("xor x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::Or => format!("or x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::And => format!("and x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::Sll => format!("sll x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::Srl => format!("srl x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::Sra => format!("sra x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::Slt => format!("slt x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::SltU => format!("sltu x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::AddI => format!("addi x{rd}, x{rs1}, {}", decoded.imm_i() as i32),
|
||||
InsnKind::XorI => format!("xori x{rd}, x{rs1}, {}", decoded.imm_i() as i32),
|
||||
InsnKind::OrI => format!("ori x{rd}, x{rs1}, {}", decoded.imm_i() as i32),
|
||||
InsnKind::AndI => format!("andi x{rd}, x{rs1}, {}", decoded.imm_i() as i32),
|
||||
InsnKind::SllI => format!("slli x{rd}, x{rs1}, {}", decoded.imm_i() as i32),
|
||||
InsnKind::SrlI => format!("srli x{rd}, x{rs1}, {}", decoded.imm_i() as i32),
|
||||
InsnKind::SraI => format!("srai x{rd}, x{rs1}, {}", decoded.imm_i() as i32),
|
||||
InsnKind::SltI => format!("slti x{rd}, x{rs1}, {}", decoded.imm_i() as i32),
|
||||
InsnKind::SltIU => format!("sltiu x{rd}, x{rs1}, {}", decoded.imm_i() as i32),
|
||||
InsnKind::Beq => format!("beq x{rs1}, x{rs2}, {}", decoded.imm_b() as i32),
|
||||
InsnKind::Bne => format!("bne x{rs1}, x{rs2}, {}", decoded.imm_b() as i32),
|
||||
InsnKind::Blt => format!("blt x{rs1}, x{rs2}, {}", decoded.imm_b() as i32),
|
||||
InsnKind::Bge => format!("bge x{rs1}, x{rs2}, {}", decoded.imm_b() as i32),
|
||||
InsnKind::BltU => format!("bltu x{rs1}, x{rs2}, {}", decoded.imm_b() as i32),
|
||||
InsnKind::BgeU => format!("bgeu x{rs1}, x{rs2}, {}", decoded.imm_b() as i32),
|
||||
InsnKind::Jal => format!("jal x{rd}, {}", decoded.imm_j() as i32),
|
||||
InsnKind::JalR => format!("jalr x{rd}, x{rs1}, {}", decoded.imm_i() as i32),
|
||||
InsnKind::Lui => format!("lui x{rd}, {:#010x}", decoded.imm_u() >> 12),
|
||||
InsnKind::Auipc => format!("auipc x{rd}, {:#010x}", decoded.imm_u() >> 12),
|
||||
InsnKind::Mul => format!("mul x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::MulH => format!("mulh x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::MulHSU => format!("mulhsu x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::MulHU => format!("mulhu x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::Div => format!("div x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::DivU => format!("divu x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::Rem => format!("rem x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::RemU => format!("remu x{rd}, x{rs1}, x{rs2}"),
|
||||
InsnKind::Lb => format!("lb x{rd}, {}(x{rs1})", decoded.imm_i() as i32),
|
||||
InsnKind::Lh => format!("lh x{rd}, {}(x{rs1})", decoded.imm_i() as i32),
|
||||
InsnKind::Lw => format!("lw x{rd}, {}(x{rs1})", decoded.imm_i() as i32),
|
||||
InsnKind::LbU => format!("lbu x{rd}, {}(x{rs1})", decoded.imm_i() as i32),
|
||||
InsnKind::LhU => format!("lhu x{rd}, {}(x{rs1})", decoded.imm_i() as i32),
|
||||
InsnKind::Sb => format!("sb x{rs2}, {}(x{rs1})", decoded.imm_i() as i32),
|
||||
InsnKind::Sh => format!("sh x{rs2}, {}(x{rs1})", decoded.imm_i() as i32),
|
||||
InsnKind::Sw => format!("sw x{rs2}, {}(x{rs1})", decoded.imm_i() as i32),
|
||||
InsnKind::Eany => match decoded.rs2 {
|
||||
0 => "ecall".to_string(),
|
||||
1 => "ebreak".to_string(),
|
||||
_ => "illegal eany".to_string(),
|
||||
},
|
||||
InsnKind::Mret => "mret".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
impl InsnKind {
|
||||
pub fn major(&self) -> u8 {
|
||||
(*self as u32 / 8) as u8
|
||||
}
|
||||
|
||||
pub fn minor(&self) -> u8 {
|
||||
(*self as u32 % 8) as u8
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use derive_more::Debug;
|
||||
use risc0_binfmt::ExitCode;
|
||||
use risc0_zkp::core::digest::Digest;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::image::MemoryImage2;
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct Segment {
|
||||
/// Initial sparse memory state for the segment
|
||||
pub partial_image: MemoryImage2,
|
||||
|
||||
pub pre_digest: Digest,
|
||||
|
||||
pub post_digest: Digest,
|
||||
|
||||
/// Recorded host->guest IO, one entry per read
|
||||
#[debug(skip)]
|
||||
pub read_record: Vec<Vec<u8>>,
|
||||
|
||||
/// Recorded rlen of guest->host IO, one entry per write
|
||||
#[debug(skip)]
|
||||
pub write_record: Vec<u32>,
|
||||
|
||||
pub user_cycles: u32,
|
||||
|
||||
/// Cycle at which we suspend
|
||||
pub suspend_cycle: u32,
|
||||
|
||||
/// Total paging cycles
|
||||
pub paging_cycles: u32,
|
||||
|
||||
pub po2: u32,
|
||||
|
||||
pub exit_code: ExitCode,
|
||||
|
||||
pub index: u64,
|
||||
|
||||
pub input_digest: Digest,
|
||||
|
||||
pub output_digest: Option<Digest>,
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
/// A host-side implementation of a system call.
|
||||
pub trait Syscall {
|
||||
/// Reads from the host.
|
||||
fn host_read(&self, fd: u32, buf: &mut [u8]) -> Result<u32>;
|
||||
|
||||
/// Writes to the host.
|
||||
fn host_write(&self, fd: u32, buf: &[u8]) -> Result<u32>;
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use risc0_binfmt::ExitCode;
|
||||
use test_log::test;
|
||||
|
||||
use super::{image::MemoryImage2, testutil, DEFAULT_SEGMENT_LIMIT_PO2};
|
||||
|
||||
// impl Syscall for BasicSyscall {
|
||||
// fn syscall(
|
||||
// &self,
|
||||
// syscall: &str,
|
||||
// ctx: &mut dyn SyscallContext,
|
||||
// guest_buf: &mut [u32],
|
||||
// ) -> Result<(u32, u32)> {
|
||||
// self.state.borrow_mut().syscall = syscall.to_string();
|
||||
// let buf_ptr = ByteAddr(ctx.peek_register(REG_A4)?);
|
||||
// let buf_len = ctx.peek_register(REG_A5)?;
|
||||
// self.state.borrow_mut().from_guest = ctx.peek_region(buf_ptr, buf_len)?;
|
||||
// let guest_buf_bytes: &mut [u8] = bytemuck::cast_slice_mut(guest_buf);
|
||||
// let into_guest = &self.state.borrow().into_guest;
|
||||
// guest_buf_bytes[..into_guest.len()].clone_from_slice(into_guest);
|
||||
// Ok((0, 0))
|
||||
// }
|
||||
// }
|
||||
|
||||
#[test]
|
||||
fn basic() {
|
||||
let program = testutil::basic();
|
||||
let expected_cycles = program.image.len();
|
||||
let mut image = MemoryImage2::new(program);
|
||||
let pre_image_id = *image.image_id();
|
||||
|
||||
println!("image_id: {pre_image_id}");
|
||||
|
||||
let result = testutil::execute(
|
||||
image,
|
||||
DEFAULT_SEGMENT_LIMIT_PO2,
|
||||
testutil::DEFAULT_SESSION_LIMIT,
|
||||
&testutil::NullSyscall,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let segments = result.segments;
|
||||
assert_eq!(segments.len(), 1);
|
||||
let segment = segments.first().unwrap();
|
||||
assert_eq!(segment.pre_digest, pre_image_id);
|
||||
assert_ne!(segment.post_digest, pre_image_id);
|
||||
assert!(segment.read_record.is_empty());
|
||||
assert!(segment.write_record.is_empty());
|
||||
assert_eq!(segment.user_cycles, expected_cycles as u32);
|
||||
assert_eq!(segment.exit_code, ExitCode::Halted(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_split() {
|
||||
let program = testutil::simple_loop(2000);
|
||||
let mut image = MemoryImage2::new(program);
|
||||
let pre_image_id = *image.image_id();
|
||||
|
||||
let result = testutil::execute(
|
||||
image,
|
||||
13,
|
||||
testutil::DEFAULT_SESSION_LIMIT,
|
||||
&testutil::NullSyscall,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let segments = result.segments;
|
||||
assert_eq!(segments.len(), 2);
|
||||
assert_eq!(segments[0].exit_code, ExitCode::SystemSplit);
|
||||
assert_eq!(segments[0].pre_digest, pre_image_id);
|
||||
assert_ne!(segments[0].post_digest, pre_image_id);
|
||||
assert!(segments[0].read_record.is_empty());
|
||||
assert!(segments[0].write_record.is_empty());
|
||||
assert_eq!(segments[1].exit_code, ExitCode::Halted(0));
|
||||
assert_eq!(segments[1].pre_digest, segments[0].post_digest);
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use anyhow::{bail, Result};
|
||||
use risc0_binfmt::Program;
|
||||
use risc0_zkp::{core::digest::Digest, MAX_CYCLES_PO2};
|
||||
|
||||
use super::{image::MemoryImage2, platform::*, syscall::Syscall, Executor, SimpleSession};
|
||||
|
||||
pub const DEFAULT_SESSION_LIMIT: Option<u64> = Some(1 << 24);
|
||||
pub const MIN_CYCLES_PO2: usize = 8;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct NullSyscall;
|
||||
|
||||
impl Syscall for NullSyscall {
|
||||
fn host_read(&self, _fd: u32, _buf: &mut [u8]) -> Result<u32> {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn host_write(&self, _fd: u32, _buf: &[u8]) -> Result<u32> {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn execute<S: Syscall>(
|
||||
image: MemoryImage2,
|
||||
segment_limit_po2: usize,
|
||||
max_cycles: Option<u64>,
|
||||
syscall_handler: &S,
|
||||
input_digest: Option<Digest>,
|
||||
) -> Result<SimpleSession> {
|
||||
if !(MIN_CYCLES_PO2..=MAX_CYCLES_PO2).contains(&segment_limit_po2) {
|
||||
bail!("Invalid segment_limit_po2: {segment_limit_po2}");
|
||||
}
|
||||
|
||||
let mut segments = Vec::new();
|
||||
let trace = Vec::new();
|
||||
let result = Executor::new(image, syscall_handler, input_digest, trace).run(
|
||||
segment_limit_po2,
|
||||
max_cycles,
|
||||
|segment| {
|
||||
tracing::trace!("{segment:#?}");
|
||||
segments.push(segment);
|
||||
Ok(())
|
||||
},
|
||||
)?;
|
||||
|
||||
Ok(SimpleSession { segments, result })
|
||||
}
|
||||
|
||||
/// Constructs a program from an iterator of instructions starting from an entrypoint.
|
||||
fn program_from_instructions(entry: u32, instructions: impl IntoIterator<Item = u32>) -> Program {
|
||||
let mut pc = entry;
|
||||
|
||||
Program {
|
||||
entry,
|
||||
image: instructions
|
||||
.into_iter()
|
||||
.map(|instr| {
|
||||
let result = (pc, instr);
|
||||
pc += WORD_SIZE as u32;
|
||||
result
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn basic() -> Program {
|
||||
program_from_instructions(
|
||||
USER_START_ADDR.0,
|
||||
[
|
||||
lui(REG_T1, 0x1234b),
|
||||
lui(REG_T2, 0xf387e),
|
||||
add(REG_T0, REG_T1, REG_T2),
|
||||
lui(REG_A1, 0x4),
|
||||
ecall(),
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
pub fn simple_loop(count: u32) -> Program {
|
||||
// loop.asm:
|
||||
//
|
||||
// .global _boot
|
||||
// .text
|
||||
//
|
||||
// _boot:
|
||||
// li a4,0
|
||||
// li a5,100
|
||||
// loop:
|
||||
// addi a4,a4,1
|
||||
// blt a4,a5,loop
|
||||
// lui a1,0x1000
|
||||
// ecall
|
||||
//
|
||||
// riscv32-unknown-elf-as loop.asm -o loop; riscv32-unknown-elf-objdump -d loop
|
||||
|
||||
// sign extend low 12 bits
|
||||
let low = ((count as i32) << 20) >> 20;
|
||||
// upper 20 bits
|
||||
let high = (count as i32 - low) >> 12;
|
||||
tracing::debug!("{count:#010x}: ({high:#010x}, {low:#010x})");
|
||||
|
||||
program_from_instructions(
|
||||
USER_START_ADDR.0,
|
||||
[
|
||||
addi(REG_A4, REG_ZERO, 0),
|
||||
lui(REG_A5, high as u32),
|
||||
addi(REG_A5, REG_A5, low as u32),
|
||||
// loop:
|
||||
addi(REG_A4, REG_A4, 1),
|
||||
blt(REG_A4, REG_A5, -4 /*loop: */),
|
||||
lui(REG_A1, 0x1000),
|
||||
ecall(),
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
// 31 25 | 24 20 | 19 15 | 14 12 | 11 7 | 6 0 |
|
||||
// funct7 | rs2 | rs1 | funct3 | rd | opcode |
|
||||
fn insn_r(funct7: u32, rs2: u32, rs1: u32, funct3: u32, rd: u32, opcode: u32) -> u32 {
|
||||
(funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | opcode
|
||||
}
|
||||
|
||||
// 31 20 | 19 15 | 14 12 | 11 7 | 6 0 |
|
||||
// imm[11:0] | rs1 | funct3 | rd | opcode |
|
||||
fn insn_i(imm: u32, rs1: u32, funct3: u32, rd: u32, opcode: u32) -> u32 {
|
||||
(imm << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | opcode
|
||||
}
|
||||
|
||||
// 31 25 | 24 20 | 19 15 | 14 12 | 11 7 | 6 0 |
|
||||
// imm[12|10:5] | rs2 | rs1 | funct3 | imm[4:1|11] | opcode |
|
||||
fn insn_b(imm: u32, rs2: u32, rs1: u32, funct3: u32, opcode: u32) -> u32 {
|
||||
let imm_12 = (imm >> 12) & 0b1;
|
||||
let imm_10_5 = (imm >> 5) & 0b111111;
|
||||
let imm_11 = (imm >> 11) & 0b1;
|
||||
let imm_4_1 = (imm >> 1) & 0b1111;
|
||||
((imm_12 << 6 | imm_10_5) << 25)
|
||||
| (rs2 << 20)
|
||||
| (rs1 << 15)
|
||||
| (funct3 << 12)
|
||||
| ((imm_4_1 << 1 | imm_11) << 7)
|
||||
| opcode
|
||||
}
|
||||
|
||||
// 31 12 | 11 7 | 6 0 |
|
||||
// imm[31:12] | rd | opcode |
|
||||
fn insn_u(imm: u32, rd: u32, opcode: u32) -> u32 {
|
||||
(imm << 12) | rd << 7 | opcode
|
||||
}
|
||||
|
||||
fn add(rd: usize, rs1: usize, rs2: usize) -> u32 {
|
||||
insn_r(0x00, rs2 as u32, rs1 as u32, 0x0, rd as u32, 0b0110011)
|
||||
}
|
||||
|
||||
fn addi(rd: usize, rs1: usize, imm: u32) -> u32 {
|
||||
insn_i(imm, rs1 as u32, 0x0, rd as u32, 0b0010011)
|
||||
}
|
||||
|
||||
fn blt(rs1: usize, rs2: usize, offset: i32) -> u32 {
|
||||
insn_b(offset as u32, rs2 as u32, rs1 as u32, 0x4, 0b1100011)
|
||||
}
|
||||
|
||||
fn ecall() -> u32 {
|
||||
insn_i(0x0, 0x0, 0x0, 0x0, 0b1110011)
|
||||
}
|
||||
|
||||
fn lui(rd: usize, imm: u32) -> u32 {
|
||||
insn_u(imm, rd as u32, 0b0110111)
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use anyhow::Result;
|
||||
use derive_more::Debug;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// An event traced from the running VM.
|
||||
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
|
||||
pub enum TraceEvent {
|
||||
/// An instruction has started at the given program counter
|
||||
InstructionStart {
|
||||
/// Cycle number since startup
|
||||
cycle: u64,
|
||||
|
||||
/// Program counter of the instruction being executed
|
||||
#[debug("{pc:#010x}")]
|
||||
pc: u32,
|
||||
|
||||
/// Encoded instruction being executed.
|
||||
#[debug("{pc:#010x}")]
|
||||
insn: u32,
|
||||
},
|
||||
|
||||
/// A register has been set
|
||||
RegisterSet {
|
||||
/// Register ID (0-16)
|
||||
idx: usize,
|
||||
|
||||
/// New value in the register
|
||||
#[debug("{value:#010x}")]
|
||||
value: u32,
|
||||
},
|
||||
|
||||
/// A memory location has been written
|
||||
MemorySet {
|
||||
/// Address of memory that's been written
|
||||
#[debug("{addr:#010x}")]
|
||||
addr: u32,
|
||||
|
||||
/// Data that's been written
|
||||
#[debug("{region:#04x?}")]
|
||||
region: Vec<u8>,
|
||||
},
|
||||
}
|
||||
|
||||
/// A callback used to collect [TraceEvent]s.
|
||||
pub trait TraceCallback {
|
||||
fn trace_callback(&mut self, event: TraceEvent) -> Result<()>;
|
||||
}
|
||||
|
||||
impl<F: FnMut(TraceEvent) -> Result<()>> TraceCallback for F {
|
||||
fn trace_callback(&mut self, event: TraceEvent) -> Result<()> {
|
||||
self(event)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(feature = "execute")]
|
||||
pub mod execute;
|
||||
#[cfg(feature = "prove")]
|
||||
pub mod prove;
|
||||
mod zirgen;
|
||||
@@ -0,0 +1,218 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::rc::Rc;
|
||||
|
||||
use anyhow::Result;
|
||||
use rayon::prelude::*;
|
||||
use risc0_circuit_rv32im_v2_sys::{
|
||||
risc0_circuit_rv32im_v2_cpu_accum, risc0_circuit_rv32im_v2_cpu_poly_fp,
|
||||
risc0_circuit_rv32im_v2_cpu_witgen, RawAccumBuffers, RawBuffer, RawExecBuffers,
|
||||
RawPreflightTrace,
|
||||
};
|
||||
use risc0_core::scope;
|
||||
use risc0_sys::ffi_wrap;
|
||||
use risc0_zkp::{
|
||||
core::{hash::poseidon2::Poseidon2HashSuite, log2_ceil},
|
||||
field::{map_pow, Elem, ExtElem as _, RootsOfUnity as _},
|
||||
hal::{cpu::CpuBuffer, AccumPreflight, CircuitHal},
|
||||
INV_RATE,
|
||||
};
|
||||
|
||||
use super::{
|
||||
CircuitAccumulator, CircuitWitnessGenerator, MetaBuffer, SegmentProver, SegmentProverImpl,
|
||||
StepMode,
|
||||
};
|
||||
use crate::{
|
||||
prove::{witgen::preflight::PreflightTrace, GLOBAL_MIX, GLOBAL_OUT},
|
||||
zirgen::{
|
||||
circuit::{CircuitField, ExtVal, Val, REGISTER_GROUP_ACCUM, REGISTER_GROUP_DATA},
|
||||
info::POLY_MIX_POWERS,
|
||||
},
|
||||
};
|
||||
|
||||
type CpuHal = risc0_zkp::hal::cpu::CpuHal<CircuitField>;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct CpuCircuitHal;
|
||||
|
||||
impl CircuitWitnessGenerator<CpuHal> for CpuCircuitHal {
|
||||
fn generate_witness(
|
||||
&self,
|
||||
mode: StepMode,
|
||||
preflight: &PreflightTrace,
|
||||
global: &MetaBuffer<CpuHal>,
|
||||
data: &MetaBuffer<CpuHal>,
|
||||
) -> Result<()> {
|
||||
scope!("cpu_witgen");
|
||||
let cycles = preflight.cycles.len();
|
||||
tracing::debug!("witgen: {cycles}");
|
||||
let global_buf = global.buf.as_slice();
|
||||
let data_buf = data.buf.as_slice();
|
||||
let buffers = RawExecBuffers {
|
||||
global: RawBuffer {
|
||||
buf: global_buf.as_ptr(),
|
||||
rows: global.rows,
|
||||
cols: global.cols,
|
||||
checked_reads: global.checked_reads,
|
||||
},
|
||||
data: RawBuffer {
|
||||
buf: data_buf.as_ptr(),
|
||||
rows: data.rows,
|
||||
cols: data.cols,
|
||||
checked_reads: data.checked_reads,
|
||||
},
|
||||
};
|
||||
let preflight = RawPreflightTrace {
|
||||
cycles: preflight.cycles.as_ptr(),
|
||||
txns: preflight.txns.as_ptr(),
|
||||
txns_len: preflight.txns.len() as u32,
|
||||
table_split_cycle: preflight.table_split_cycle,
|
||||
};
|
||||
ffi_wrap(|| unsafe {
|
||||
risc0_circuit_rv32im_v2_cpu_witgen(mode as u32, &buffers, &preflight, cycles as u32)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl CircuitAccumulator<CpuHal> for CpuCircuitHal {
|
||||
fn step_accum(
|
||||
&self,
|
||||
preflight: &PreflightTrace,
|
||||
data: &MetaBuffer<CpuHal>,
|
||||
accum: &MetaBuffer<CpuHal>,
|
||||
mix: &MetaBuffer<CpuHal>,
|
||||
) -> Result<()> {
|
||||
scope!("accumulate");
|
||||
let cycles = preflight.cycles.len();
|
||||
tracing::debug!("accumulate: {cycles}");
|
||||
let data_buf = data.buf.as_slice();
|
||||
let accum_buf = accum.buf.as_slice();
|
||||
let mix_buf = mix.buf.as_slice();
|
||||
let buffers = RawAccumBuffers {
|
||||
data: RawBuffer {
|
||||
buf: data_buf.as_ptr(),
|
||||
rows: data.rows,
|
||||
cols: data.cols,
|
||||
checked_reads: data.checked_reads,
|
||||
},
|
||||
accum: RawBuffer {
|
||||
buf: accum_buf.as_ptr(),
|
||||
rows: accum.rows,
|
||||
cols: accum.cols,
|
||||
checked_reads: accum.checked_reads,
|
||||
},
|
||||
mix: RawBuffer {
|
||||
buf: mix_buf.as_ptr(),
|
||||
rows: mix.rows,
|
||||
cols: mix.cols,
|
||||
checked_reads: mix.checked_reads,
|
||||
},
|
||||
};
|
||||
let preflight = RawPreflightTrace {
|
||||
cycles: preflight.cycles.as_ptr(),
|
||||
txns: preflight.txns.as_ptr(),
|
||||
txns_len: preflight.txns.len() as u32,
|
||||
table_split_cycle: preflight.table_split_cycle,
|
||||
};
|
||||
ffi_wrap(|| unsafe {
|
||||
risc0_circuit_rv32im_v2_cpu_accum(&buffers, &preflight, cycles as u32)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl CircuitHal<CpuHal> for CpuCircuitHal {
|
||||
fn eval_check(
|
||||
&self,
|
||||
check: &CpuBuffer<Val>,
|
||||
groups: &[&CpuBuffer<Val>],
|
||||
globals: &[&CpuBuffer<Val>],
|
||||
poly_mix: ExtVal,
|
||||
po2: usize,
|
||||
steps: usize,
|
||||
) {
|
||||
scope!("eval_check");
|
||||
|
||||
const EXP_PO2: usize = log2_ceil(INV_RATE);
|
||||
let domain = steps * INV_RATE;
|
||||
let poly_mix_pows = map_pow(poly_mix, POLY_MIX_POWERS);
|
||||
|
||||
// SAFETY: Convert a borrow of a cell into a raw const slice so that we can pass
|
||||
// it over the thread boundary. This should be safe because the scope of the
|
||||
// usage is within this function and each thread access will not overlap with
|
||||
// each other.
|
||||
|
||||
let data = groups[REGISTER_GROUP_DATA].as_slice();
|
||||
let accum = groups[REGISTER_GROUP_ACCUM].as_slice();
|
||||
let mix = globals[GLOBAL_MIX].as_slice();
|
||||
let out = globals[GLOBAL_OUT].as_slice();
|
||||
let check = check.as_slice();
|
||||
|
||||
let data = unsafe { std::slice::from_raw_parts(data.as_ptr(), data.len()) };
|
||||
let accum = unsafe { std::slice::from_raw_parts(accum.as_ptr(), accum.len()) };
|
||||
let mix = unsafe { std::slice::from_raw_parts(mix.as_ptr(), mix.len()) };
|
||||
let out = unsafe { std::slice::from_raw_parts(out.as_ptr(), out.len()) };
|
||||
let check = unsafe { std::slice::from_raw_parts(check.as_ptr(), check.len()) };
|
||||
let poly_mix_pows = poly_mix_pows.as_slice();
|
||||
|
||||
let args: &[&[Val]] = &[accum, data, out, mix];
|
||||
|
||||
(0..domain).into_par_iter().for_each(|cycle| {
|
||||
let args: Vec<*const Val> = args.iter().map(|x| (*x).as_ptr()).collect();
|
||||
let mut tot = ExtVal::ZERO;
|
||||
unsafe {
|
||||
risc0_circuit_rv32im_v2_cpu_poly_fp(
|
||||
cycle,
|
||||
domain,
|
||||
poly_mix_pows.as_ptr(),
|
||||
args.as_ptr(),
|
||||
&mut tot,
|
||||
)
|
||||
};
|
||||
let x = Val::ROU_FWD[po2 + EXP_PO2].pow(cycle);
|
||||
// TODO: what is this magic number 3?
|
||||
let y = (Val::new(3) * x).pow(1 << po2);
|
||||
let ret = tot * (y - Val::new(1)).inv();
|
||||
|
||||
// SAFETY: This conversion is to make the check slice mutable, which should be
|
||||
// safe because each thread access will not overlap with each other.
|
||||
let check =
|
||||
unsafe { std::slice::from_raw_parts_mut(check.as_ptr() as *mut Val, check.len()) };
|
||||
for i in 0..ExtVal::EXT_SIZE {
|
||||
check[i * domain + cycle] = ret.elems()[i];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn accumulate(
|
||||
&self,
|
||||
_preflight: &AccumPreflight,
|
||||
_ctrl: &CpuBuffer<Val>,
|
||||
_global: &CpuBuffer<Val>,
|
||||
_data: &CpuBuffer<Val>,
|
||||
_mix: &CpuBuffer<Val>,
|
||||
_accum: &CpuBuffer<Val>,
|
||||
_steps: usize,
|
||||
) {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn segment_prover() -> Result<Box<dyn SegmentProver>> {
|
||||
let suite = Poseidon2HashSuite::new_suite();
|
||||
let hal = Rc::new(CpuHal::new(suite));
|
||||
let circuit_hal = Rc::new(CpuCircuitHal);
|
||||
Ok(Box::new(SegmentProverImpl::new(hal, circuit_hal)))
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::rc::Rc;
|
||||
|
||||
use anyhow::Result;
|
||||
use risc0_circuit_rv32im_v2_sys::{
|
||||
risc0_circuit_rv32im_v2_cpu_accum, risc0_circuit_rv32im_v2_cuda_eval_check,
|
||||
risc0_circuit_rv32im_v2_cuda_witgen, RawAccumBuffers, RawBuffer, RawExecBuffers,
|
||||
RawPreflightTrace,
|
||||
};
|
||||
use risc0_core::{
|
||||
field::{map_pow, Elem, ExtElem as _, RootsOfUnity},
|
||||
scope,
|
||||
};
|
||||
use risc0_sys::ffi_wrap;
|
||||
use risc0_zkp::{
|
||||
core::log2_ceil,
|
||||
hal::{
|
||||
cuda::{BufferImpl as CudaBuffer, CudaHal, CudaHalPoseidon2, CudaHash, CudaHashPoseidon2},
|
||||
AccumPreflight, Buffer, CircuitHal,
|
||||
},
|
||||
INV_RATE,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
prove::{SegmentProver, GLOBAL_MIX, GLOBAL_OUT},
|
||||
zirgen::{
|
||||
circuit::{ExtVal, Val, REGISTER_GROUP_ACCUM, REGISTER_GROUP_CODE, REGISTER_GROUP_DATA},
|
||||
info::{NUM_POLY_MIX_POWERS, POLY_MIX_POWERS},
|
||||
},
|
||||
};
|
||||
|
||||
use super::{
|
||||
CircuitAccumulator, CircuitWitnessGenerator, MetaBuffer, PreflightTrace, SegmentProverImpl,
|
||||
StepMode,
|
||||
};
|
||||
|
||||
pub struct CudaCircuitHal<CH: CudaHash> {
|
||||
_hal: Rc<CudaHal<CH>>, // retain a reference to ensure the context remains valid
|
||||
}
|
||||
|
||||
impl<CH: CudaHash> CudaCircuitHal<CH> {
|
||||
pub fn new(_hal: Rc<CudaHal<CH>>) -> Self {
|
||||
Self { _hal }
|
||||
}
|
||||
}
|
||||
|
||||
impl<CH: CudaHash> CircuitWitnessGenerator<CudaHal<CH>> for CudaCircuitHal<CH> {
|
||||
fn generate_witness(
|
||||
&self,
|
||||
mode: StepMode,
|
||||
preflight: &PreflightTrace,
|
||||
global: &MetaBuffer<CudaHal<CH>>,
|
||||
data: &MetaBuffer<CudaHal<CH>>,
|
||||
) -> Result<()> {
|
||||
scope!("witgen");
|
||||
|
||||
let cycles = preflight.cycles.len();
|
||||
assert_eq!(cycles, data.rows);
|
||||
tracing::debug!("witgen: {cycles}");
|
||||
|
||||
let global_ptr = global.buf.as_device_ptr();
|
||||
let data_ptr = data.buf.as_device_ptr();
|
||||
let buffers = RawExecBuffers {
|
||||
global: RawBuffer {
|
||||
buf: global_ptr.as_ptr() as *const Val,
|
||||
rows: global.rows,
|
||||
cols: global.cols,
|
||||
checked_reads: global.checked_reads,
|
||||
},
|
||||
data: RawBuffer {
|
||||
buf: data_ptr.as_ptr() as *const Val,
|
||||
rows: data.rows,
|
||||
cols: data.cols,
|
||||
checked_reads: data.checked_reads,
|
||||
},
|
||||
};
|
||||
|
||||
let preflight = RawPreflightTrace {
|
||||
cycles: preflight.cycles.as_ptr(),
|
||||
txns: preflight.txns.as_ptr(),
|
||||
txns_len: preflight.txns.len() as u32,
|
||||
table_split_cycle: preflight.table_split_cycle,
|
||||
};
|
||||
ffi_wrap(|| unsafe {
|
||||
risc0_circuit_rv32im_v2_cuda_witgen(mode as u32, &buffers, &preflight, cycles as u32)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<CH: CudaHash> CircuitAccumulator<CudaHal<CH>> for CudaCircuitHal<CH> {
|
||||
fn step_accum(
|
||||
&self,
|
||||
preflight: &PreflightTrace,
|
||||
data: &MetaBuffer<CudaHal<CH>>,
|
||||
accum: &MetaBuffer<CudaHal<CH>>,
|
||||
mix: &MetaBuffer<CudaHal<CH>>,
|
||||
) -> Result<()> {
|
||||
scope!("accumulate");
|
||||
|
||||
let cycles = preflight.cycles.len();
|
||||
tracing::debug!("accumulate: {cycles}");
|
||||
|
||||
let data_vec = data.buf.to_vec();
|
||||
let accum_vec = accum.buf.to_vec();
|
||||
let mix_vec = mix.buf.to_vec();
|
||||
let buffers = RawAccumBuffers {
|
||||
data: RawBuffer {
|
||||
buf: data_vec.as_ptr(),
|
||||
rows: data.rows,
|
||||
cols: data.cols,
|
||||
checked_reads: data.checked_reads,
|
||||
},
|
||||
accum: RawBuffer {
|
||||
buf: accum_vec.as_ptr(),
|
||||
rows: accum.rows,
|
||||
cols: accum.cols,
|
||||
checked_reads: accum.checked_reads,
|
||||
},
|
||||
mix: RawBuffer {
|
||||
buf: mix_vec.as_ptr(),
|
||||
rows: mix.rows,
|
||||
cols: mix.cols,
|
||||
checked_reads: mix.checked_reads,
|
||||
},
|
||||
};
|
||||
let preflight = RawPreflightTrace {
|
||||
cycles: preflight.cycles.as_ptr(),
|
||||
txns: preflight.txns.as_ptr(),
|
||||
txns_len: preflight.txns.len() as u32,
|
||||
table_split_cycle: preflight.table_split_cycle,
|
||||
};
|
||||
let result = ffi_wrap(|| unsafe {
|
||||
risc0_circuit_rv32im_v2_cpu_accum(&buffers, &preflight, cycles as u32)
|
||||
});
|
||||
data.buf.view_mut(|view| {
|
||||
view.copy_from_slice(&data_vec);
|
||||
});
|
||||
accum.buf.view_mut(|view| {
|
||||
view.copy_from_slice(&accum_vec);
|
||||
});
|
||||
mix.buf.view_mut(|view| {
|
||||
view.copy_from_slice(&mix_vec);
|
||||
});
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl<CH: CudaHash> CircuitHal<CudaHal<CH>> for CudaCircuitHal<CH> {
|
||||
fn accumulate(
|
||||
&self,
|
||||
_preflight: &AccumPreflight,
|
||||
_ctrl: &CudaBuffer<Val>,
|
||||
_io: &CudaBuffer<Val>,
|
||||
_data: &CudaBuffer<Val>,
|
||||
_mix: &CudaBuffer<Val>,
|
||||
_accum: &CudaBuffer<Val>,
|
||||
_steps: usize,
|
||||
) {
|
||||
}
|
||||
|
||||
fn eval_check(
|
||||
&self,
|
||||
check: &CudaBuffer<Val>,
|
||||
groups: &[&CudaBuffer<Val>],
|
||||
globals: &[&CudaBuffer<Val>],
|
||||
poly_mix: ExtVal,
|
||||
po2: usize,
|
||||
steps: usize,
|
||||
) {
|
||||
scope!("eval_check");
|
||||
|
||||
let accum = groups[REGISTER_GROUP_ACCUM];
|
||||
let ctrl = groups[REGISTER_GROUP_CODE];
|
||||
let data = groups[REGISTER_GROUP_DATA];
|
||||
let mix = globals[GLOBAL_MIX];
|
||||
let out = globals[GLOBAL_OUT];
|
||||
tracing::debug!(
|
||||
"check: {}, ctrl: {}, data: {}, accum: {}, mix: {} out: {}",
|
||||
check.size(),
|
||||
ctrl.size(),
|
||||
data.size(),
|
||||
accum.size(),
|
||||
mix.size(),
|
||||
out.size()
|
||||
);
|
||||
tracing::debug!(
|
||||
"total: {}",
|
||||
(check.size() + ctrl.size() + data.size() + accum.size() + mix.size() + out.size()) * 4
|
||||
);
|
||||
|
||||
const EXP_PO2: usize = log2_ceil(INV_RATE);
|
||||
let domain = steps * INV_RATE;
|
||||
let rou = Val::ROU_FWD[po2 + EXP_PO2];
|
||||
|
||||
tracing::debug!("steps: {steps}, domain: {domain}, po2: {po2}, rou: {rou:?}");
|
||||
let poly_mix_pows = map_pow(poly_mix, POLY_MIX_POWERS);
|
||||
let poly_mix_pows: &[u32; ExtVal::EXT_SIZE * NUM_POLY_MIX_POWERS] =
|
||||
ExtVal::as_u32_slice(poly_mix_pows.as_slice())
|
||||
.try_into()
|
||||
.unwrap();
|
||||
|
||||
ffi_wrap(|| unsafe {
|
||||
risc0_circuit_rv32im_v2_cuda_eval_check(
|
||||
check.as_device_ptr(),
|
||||
ctrl.as_device_ptr(),
|
||||
data.as_device_ptr(),
|
||||
accum.as_device_ptr(),
|
||||
mix.as_device_ptr(),
|
||||
out.as_device_ptr(),
|
||||
&rou as *const Val,
|
||||
po2 as u32,
|
||||
domain as u32,
|
||||
poly_mix_pows.as_ptr(),
|
||||
)
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
pub type CudaCircuitHalPoseidon2 = CudaCircuitHal<CudaHashPoseidon2>;
|
||||
|
||||
pub fn segment_prover() -> Result<Box<dyn SegmentProver>> {
|
||||
let hal = Rc::new(CudaHalPoseidon2::new());
|
||||
let circuit_hal = Rc::new(CudaCircuitHalPoseidon2::new(hal.clone()));
|
||||
Ok(Box::new(SegmentProverImpl::new(hal, circuit_hal)))
|
||||
}
|
||||
|
||||
// #[cfg(test)]
|
||||
// mod tests {
|
||||
// use std::rc::Rc;
|
||||
|
||||
// use risc0_core::field::baby_bear::BabyBear;
|
||||
// use risc0_zkp::{
|
||||
// core::hash::sha::Sha256HashSuite,
|
||||
// hal::{cpu::CpuHal, cuda::CudaHalSha256},
|
||||
// };
|
||||
// use test_log::test;
|
||||
|
||||
// use crate::prove::hal::cpu::CpuCircuitHal;
|
||||
|
||||
// #[test]
|
||||
// fn eval_check() {
|
||||
// const PO2: usize = 4;
|
||||
// let cpu_hal: CpuHal<BabyBear> = CpuHal::new(Sha256HashSuite::new_suite());
|
||||
// let cpu_eval = CpuCircuitHal;
|
||||
// let gpu_hal = Rc::new(CudaHalSha256::new());
|
||||
// let gpu_eval = super::CudaCircuitHal::new(gpu_hal.clone());
|
||||
// crate::prove::testutil::eval_check(&cpu_hal, cpu_eval, gpu_hal.as_ref(), gpu_eval, PO2);
|
||||
// }
|
||||
// }
|
||||
@@ -0,0 +1,217 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod cpu;
|
||||
#[cfg(feature = "cuda")]
|
||||
pub(crate) mod cuda;
|
||||
|
||||
use std::rc::Rc;
|
||||
|
||||
use anyhow::Result;
|
||||
use rand::thread_rng;
|
||||
use risc0_core::scope;
|
||||
use risc0_zkp::{
|
||||
adapter::{CircuitInfo as _, PROOF_SYSTEM_INFO},
|
||||
field::Elem as _,
|
||||
hal::{Buffer, CircuitHal, Hal},
|
||||
prove::Prover,
|
||||
};
|
||||
|
||||
use super::{
|
||||
witgen::{preflight::PreflightTrace, WitnessGenerator},
|
||||
Seal, SegmentProver,
|
||||
};
|
||||
use crate::{
|
||||
execute::segment::Segment,
|
||||
zirgen::{
|
||||
circuit::{
|
||||
CircuitField, ExtVal, Val, REGCOUNT_ACCUM, REGCOUNT_MIX, REGISTER_GROUP_ACCUM,
|
||||
REGISTER_GROUP_CODE, REGISTER_GROUP_DATA,
|
||||
},
|
||||
taps::TAPSET,
|
||||
CircuitImpl,
|
||||
},
|
||||
};
|
||||
|
||||
pub(crate) struct MetaBuffer<H: Hal> {
|
||||
pub buf: H::Buffer<H::Elem>,
|
||||
pub rows: usize,
|
||||
pub cols: usize,
|
||||
pub checked_reads: bool,
|
||||
}
|
||||
|
||||
impl<H> MetaBuffer<H>
|
||||
where
|
||||
H: Hal<Field = CircuitField, Elem = Val, ExtElem = ExtVal>,
|
||||
{
|
||||
pub fn new(name: &'static str, hal: &H, rows: usize, cols: usize, checked_reads: bool) -> Self {
|
||||
let buf = hal.alloc_elem_init(name, rows * cols, Val::INVALID);
|
||||
Self {
|
||||
buf,
|
||||
rows,
|
||||
cols,
|
||||
checked_reads,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn to_vec(&self) -> Vec<Val> {
|
||||
self.buf.to_vec()
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
pub(crate) enum StepMode {
|
||||
Parallel,
|
||||
// #[cfg(test)]
|
||||
SeqForward,
|
||||
#[cfg(test)]
|
||||
SeqReverse,
|
||||
}
|
||||
|
||||
pub(crate) trait CircuitWitnessGenerator<H: Hal> {
|
||||
fn generate_witness(
|
||||
&self,
|
||||
mode: StepMode,
|
||||
preflight: &PreflightTrace,
|
||||
global: &MetaBuffer<H>,
|
||||
data: &MetaBuffer<H>,
|
||||
) -> Result<()>;
|
||||
}
|
||||
|
||||
pub(crate) trait CircuitAccumulator<H: Hal> {
|
||||
fn step_accum(
|
||||
&self,
|
||||
preflight: &PreflightTrace,
|
||||
data: &MetaBuffer<H>,
|
||||
accum: &MetaBuffer<H>,
|
||||
mix: &MetaBuffer<H>,
|
||||
) -> Result<()>;
|
||||
}
|
||||
|
||||
pub(crate) struct SegmentProverImpl<H, C>
|
||||
where
|
||||
H: Hal<Field = CircuitField, Elem = Val, ExtElem = ExtVal>,
|
||||
C: CircuitHal<H> + CircuitWitnessGenerator<H>,
|
||||
{
|
||||
hal: Rc<H>,
|
||||
circuit_hal: Rc<C>,
|
||||
}
|
||||
|
||||
impl<H, C> SegmentProverImpl<H, C>
|
||||
where
|
||||
H: Hal<Field = CircuitField, Elem = Val, ExtElem = ExtVal>,
|
||||
C: CircuitHal<H> + CircuitWitnessGenerator<H>,
|
||||
{
|
||||
pub fn new(hal: Rc<H>, circuit_hal: Rc<C>) -> Self {
|
||||
Self { hal, circuit_hal }
|
||||
}
|
||||
}
|
||||
|
||||
impl<H, C> SegmentProver for SegmentProverImpl<H, C>
|
||||
where
|
||||
H: Hal<Field = CircuitField, Elem = Val, ExtElem = ExtVal>,
|
||||
C: CircuitHal<H> + CircuitWitnessGenerator<H> + CircuitAccumulator<H>,
|
||||
{
|
||||
fn prove(&self, segment: &Segment) -> Result<Seal> {
|
||||
scope!("prove");
|
||||
|
||||
let mut rng = thread_rng();
|
||||
let rand_z = ExtVal::random(&mut rng);
|
||||
|
||||
let witgen = WitnessGenerator::new(
|
||||
self.hal.as_ref(),
|
||||
self.circuit_hal.as_ref(),
|
||||
segment,
|
||||
// StepMode::Parallel,
|
||||
StepMode::SeqForward,
|
||||
rand_z,
|
||||
)?;
|
||||
|
||||
let code = &witgen.code.buf;
|
||||
let data = &witgen.data.buf;
|
||||
let global = &witgen.global.buf;
|
||||
|
||||
Ok(scope!("prove", {
|
||||
tracing::debug!("prove");
|
||||
|
||||
let mut prover = Prover::new(self.hal.as_ref(), TAPSET);
|
||||
let hashfn = &self.hal.get_hash_suite().hashfn;
|
||||
|
||||
let mix = scope!("main", {
|
||||
// At the start of the protocol, seed the Fiat-Shamir transcript with context information
|
||||
// about the proof system and circuit.
|
||||
prover
|
||||
.iop()
|
||||
.commit(&hashfn.hash_elem_slice(&PROOF_SYSTEM_INFO.encode()));
|
||||
prover
|
||||
.iop()
|
||||
.commit(&hashfn.hash_elem_slice(&CircuitImpl::CIRCUIT_INFO.encode()));
|
||||
|
||||
// Concat globals and po2 into a vector.
|
||||
let global_len = global.size();
|
||||
let mut header = vec![Val::ZERO; global_len + 1];
|
||||
global.view_mut(|view| {
|
||||
for (i, elem) in view.iter_mut().enumerate() {
|
||||
*elem = elem.valid_or_zero();
|
||||
header[i] = *elem;
|
||||
}
|
||||
header[global_len] = Val::new_raw(segment.po2);
|
||||
});
|
||||
|
||||
let header_digest = hashfn.hash_elem_slice(&header);
|
||||
prover.iop().commit(&header_digest);
|
||||
prover.iop().write_field_elem_slice(header.as_slice());
|
||||
prover.set_po2(segment.po2 as usize);
|
||||
|
||||
prover.commit_group(REGISTER_GROUP_CODE, code);
|
||||
prover.commit_group(REGISTER_GROUP_DATA, data);
|
||||
|
||||
// Make the mixing values
|
||||
let mix: [Val; REGCOUNT_MIX] = std::array::from_fn(|_| prover.iop().random_elem());
|
||||
let mix = MetaBuffer {
|
||||
buf: self.hal.copy_from_elem("mix", mix.as_slice()),
|
||||
rows: 1,
|
||||
cols: REGCOUNT_MIX,
|
||||
checked_reads: true,
|
||||
};
|
||||
|
||||
let accum = scope!(
|
||||
"alloc(accum)",
|
||||
MetaBuffer::new(
|
||||
"accum",
|
||||
self.hal.as_ref(),
|
||||
witgen.cycles,
|
||||
REGCOUNT_ACCUM,
|
||||
true
|
||||
)
|
||||
);
|
||||
|
||||
self.circuit_hal
|
||||
.step_accum(&witgen.trace, &witgen.data, &accum, &mix)?;
|
||||
|
||||
scope!("zeroize(accum)", {
|
||||
self.hal.eltwise_zeroize_elem(&accum.buf);
|
||||
});
|
||||
|
||||
prover.commit_group(REGISTER_GROUP_ACCUM, &accum.buf);
|
||||
|
||||
mix
|
||||
});
|
||||
|
||||
prover.finalize(&[&mix.buf, global], self.circuit_hal.as_ref())
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod hal;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
mod witgen;
|
||||
|
||||
use anyhow::Result;
|
||||
use cfg_if::cfg_if;
|
||||
use risc0_zkp::core::{digest::Digest, hash::poseidon2::Poseidon2HashSuite};
|
||||
|
||||
use crate::{execute::segment::Segment, zirgen::CircuitImpl};
|
||||
|
||||
const GLOBAL_MIX: usize = 0;
|
||||
const GLOBAL_OUT: usize = 1;
|
||||
|
||||
pub type Seal = Vec<u32>;
|
||||
|
||||
pub trait SegmentProver {
|
||||
fn prove(&self, segment: &Segment) -> Result<Seal>;
|
||||
|
||||
fn verify(&self, seal: &Seal) -> Result<()> {
|
||||
let hash_suite = Poseidon2HashSuite::new_suite();
|
||||
|
||||
// We don't have a `code' buffer to verify.
|
||||
let check_code_fn = |_: u32, _: &Digest| Ok(());
|
||||
|
||||
Ok(risc0_zkp::verify::verify(
|
||||
&CircuitImpl,
|
||||
&hash_suite,
|
||||
seal,
|
||||
check_code_fn,
|
||||
)?)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn segment_prover() -> Result<Box<dyn SegmentProver>> {
|
||||
cfg_if! {
|
||||
if #[cfg(feature = "cuda")] {
|
||||
self::hal::cuda::segment_prover()
|
||||
// } else if #[cfg(any(all(target_os = "macos", target_arch = "aarch64"), target_os = "ios"))] {
|
||||
// self::hal::metal::segment_prover(hashfn)
|
||||
} else {
|
||||
self::hal::cpu::segment_prover()
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,135 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use risc0_binfmt::Program;
|
||||
use test_log::test;
|
||||
|
||||
use super::segment_prover;
|
||||
use crate::execute::{image::MemoryImage2, testutil, DEFAULT_SEGMENT_LIMIT_PO2};
|
||||
|
||||
fn run_program(program: Program) {
|
||||
let image = MemoryImage2::new(program);
|
||||
let result = testutil::execute(
|
||||
image,
|
||||
DEFAULT_SEGMENT_LIMIT_PO2,
|
||||
testutil::DEFAULT_SESSION_LIMIT,
|
||||
&testutil::NullSyscall,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let segments = result.segments;
|
||||
let segment = segments.first().unwrap();
|
||||
|
||||
let prover = segment_prover().unwrap();
|
||||
let seal = prover.prove(segment).unwrap();
|
||||
prover.verify(&seal).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn basic() {
|
||||
run_program(testutil::basic());
|
||||
}
|
||||
|
||||
// These tests come from:
|
||||
// https://github.com/riscv-software-src/riscv-tests
|
||||
// They were built using the toolchain from:
|
||||
// https://github.com/risc0/toolchain/releases/tag/2022.03.25
|
||||
mod riscv {
|
||||
use risc0_binfmt::Program;
|
||||
|
||||
use super::run_program;
|
||||
|
||||
fn run_test(test_name: &str) {
|
||||
use std::io::Read;
|
||||
|
||||
use flate2::read::GzDecoder;
|
||||
use tar::Archive;
|
||||
|
||||
let bytes = include_bytes!("testdata/riscv-tests.tgz");
|
||||
let gz = GzDecoder::new(&bytes[..]);
|
||||
let mut tar = Archive::new(gz);
|
||||
for entry in tar.entries().unwrap() {
|
||||
let mut entry = entry.unwrap();
|
||||
if !entry.header().entry_type().is_file() {
|
||||
continue;
|
||||
}
|
||||
let path = entry.path().unwrap();
|
||||
let filename = path.file_name().unwrap().to_str().unwrap();
|
||||
if filename != test_name {
|
||||
continue;
|
||||
}
|
||||
let mut elf = Vec::new();
|
||||
entry.read_to_end(&mut elf).unwrap();
|
||||
|
||||
let program = Program::load_elf(&elf, u32::MAX).unwrap();
|
||||
run_program(program);
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! test_case {
|
||||
($func_name:ident) => {
|
||||
#[test_log::test]
|
||||
fn $func_name() {
|
||||
run_test(stringify!($func_name));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
test_case!(add);
|
||||
test_case!(addi);
|
||||
test_case!(and);
|
||||
test_case!(andi);
|
||||
test_case!(auipc);
|
||||
test_case!(beq);
|
||||
test_case!(bge);
|
||||
test_case!(bgeu);
|
||||
test_case!(blt);
|
||||
test_case!(bltu);
|
||||
test_case!(bne);
|
||||
test_case!(div);
|
||||
test_case!(divu);
|
||||
test_case!(jal);
|
||||
test_case!(jalr);
|
||||
test_case!(lb);
|
||||
test_case!(lbu);
|
||||
test_case!(lh);
|
||||
test_case!(lhu);
|
||||
test_case!(lui);
|
||||
test_case!(lw);
|
||||
test_case!(mul);
|
||||
test_case!(mulh);
|
||||
test_case!(mulhsu);
|
||||
test_case!(mulhu);
|
||||
test_case!(or);
|
||||
test_case!(ori);
|
||||
test_case!(rem);
|
||||
test_case!(remu);
|
||||
test_case!(sb);
|
||||
test_case!(sh);
|
||||
test_case!(simple);
|
||||
test_case!(sll);
|
||||
test_case!(slli);
|
||||
test_case!(slt);
|
||||
test_case!(slti);
|
||||
test_case!(sltiu);
|
||||
test_case!(sltu);
|
||||
test_case!(sra);
|
||||
test_case!(srai);
|
||||
test_case!(srl);
|
||||
test_case!(srli);
|
||||
test_case!(sub);
|
||||
test_case!(sw);
|
||||
test_case!(xor);
|
||||
test_case!(xori);
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod poseidon2;
|
||||
pub(crate) mod preflight;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use std::iter::zip;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use preflight::PreflightTrace;
|
||||
use risc0_circuit_rv32im_v2_sys::RawPreflightCycle;
|
||||
use risc0_core::scope;
|
||||
use risc0_zkp::{core::digest::DIGEST_WORDS, field::Elem as _, hal::Hal};
|
||||
|
||||
use self::{poseidon2::Poseidon2State, preflight::Back};
|
||||
use super::hal::{CircuitWitnessGenerator, MetaBuffer, StepMode};
|
||||
use crate::{
|
||||
execute::{addr::WordAddr, platform::MERKLE_TREE_END_ADDR, segment::Segment},
|
||||
zirgen::circuit::{
|
||||
CircuitField, ExtVal, Val, LAYOUT_GLOBAL, LAYOUT_TOP, REGCOUNT_CODE, REGCOUNT_DATA,
|
||||
REGCOUNT_GLOBAL,
|
||||
},
|
||||
};
|
||||
|
||||
pub(crate) struct WitnessGenerator<H: Hal> {
|
||||
pub cycles: usize,
|
||||
pub global: MetaBuffer<H>,
|
||||
pub code: MetaBuffer<H>,
|
||||
pub data: MetaBuffer<H>,
|
||||
pub trace: PreflightTrace,
|
||||
}
|
||||
|
||||
impl<H: Hal> WitnessGenerator<H> {
|
||||
pub fn new<C>(
|
||||
hal: &H,
|
||||
circuit_hal: &C,
|
||||
segment: &Segment,
|
||||
mode: StepMode,
|
||||
rand_z: ExtVal,
|
||||
) -> Result<Self>
|
||||
where
|
||||
H: Hal<Field = CircuitField, Elem = Val, ExtElem = ExtVal>,
|
||||
C: CircuitWitnessGenerator<H>,
|
||||
{
|
||||
scope!("witgen");
|
||||
|
||||
let trace = segment.preflight(rand_z)?;
|
||||
let cycles = trace.cycles.len();
|
||||
|
||||
tracing::trace!("{segment:#?}");
|
||||
tracing::trace!("{trace:#?}");
|
||||
|
||||
// assert_eq!(
|
||||
// segment.suspend_cycle + segment.paging_cycles + LOOKUP_TABLE_CYCLES as u32 + 1,
|
||||
// cycles as u32,
|
||||
// "suspend_cycle: {} + paging_cycles: {} + {LOOKUP_TABLE_CYCLES} + 1 == trace.cycles",
|
||||
// segment.suspend_cycle,
|
||||
// segment.paging_cycles
|
||||
// );
|
||||
// assert_eq!(cycles, 1 << segment.po2, "cycles == 1 << segment.po2");
|
||||
assert!(cycles <= 1 << segment.po2, "cycles <= 1 << segment.po2");
|
||||
let cycles = 1 << segment.po2;
|
||||
|
||||
let mut global = vec![Val::INVALID; REGCOUNT_GLOBAL];
|
||||
|
||||
for i in 0..DIGEST_WORDS {
|
||||
// state in
|
||||
let low = segment.pre_digest.as_words()[i] & 0xffff;
|
||||
let high = segment.pre_digest.as_words()[i] >> 16;
|
||||
global[LAYOUT_GLOBAL.state_in.values[i].low._super.offset] = low.into();
|
||||
global[LAYOUT_GLOBAL.state_in.values[i].high._super.offset] = high.into();
|
||||
|
||||
// input digest
|
||||
let low = 0u32;
|
||||
let high = 0u32;
|
||||
global[LAYOUT_GLOBAL.input.values[i].low._super.offset] = low.into();
|
||||
global[LAYOUT_GLOBAL.input.values[i].high._super.offset] = high.into();
|
||||
}
|
||||
|
||||
// rand_z
|
||||
for (i, &elem) in trace.rand_z.elems().iter().enumerate() {
|
||||
global[LAYOUT_GLOBAL.rng._super.offset + i] = elem;
|
||||
}
|
||||
|
||||
// is_terminate
|
||||
global[LAYOUT_GLOBAL.is_terminate._super.offset] = 1u32.into();
|
||||
|
||||
let global = MetaBuffer {
|
||||
buf: hal.copy_from_elem("global", &global),
|
||||
rows: 1,
|
||||
cols: REGCOUNT_GLOBAL,
|
||||
checked_reads: true,
|
||||
};
|
||||
|
||||
let code = MetaBuffer::new("code", hal, cycles, REGCOUNT_CODE, false);
|
||||
|
||||
let data = scope!(
|
||||
"alloc(data)",
|
||||
MetaBuffer::new("data", hal, cycles, REGCOUNT_DATA, true)
|
||||
);
|
||||
|
||||
// Set stateful columns from 'top'
|
||||
let mut injector = Injector::new(cycles);
|
||||
for (row, back) in trace.backs.iter().enumerate() {
|
||||
let cycle = &trace.cycles[row];
|
||||
// tracing::trace!(
|
||||
// "[{row}] pc: {:#010x}, state: {:?}",
|
||||
// cycle.pc,
|
||||
// CycleState::from_u32(cycle.state).unwrap()
|
||||
// );
|
||||
match back {
|
||||
Back::None => {}
|
||||
Back::Ecall(s0, s1, s2) => {
|
||||
const ECALL_S0: usize = LAYOUT_TOP.inst_result.arm8.s0._super.offset;
|
||||
const ECALL_S1: usize = LAYOUT_TOP.inst_result.arm8.s1._super.offset;
|
||||
const ECALL_S2: usize = LAYOUT_TOP.inst_result.arm8.s2._super.offset;
|
||||
injector.set(row, ECALL_S0, *s0);
|
||||
injector.set(row, ECALL_S1, *s1);
|
||||
injector.set(row, ECALL_S2, *s2);
|
||||
}
|
||||
Back::Poseidon2(p2_state) => {
|
||||
for (col, value) in zip(Poseidon2State::offsets(), p2_state.as_array()) {
|
||||
injector.set(row, col, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
injector.set_cycle(row, cycle);
|
||||
}
|
||||
|
||||
hal.scatter(
|
||||
&data.buf,
|
||||
&injector.index,
|
||||
&injector.offsets,
|
||||
&injector.values,
|
||||
);
|
||||
|
||||
circuit_hal
|
||||
.generate_witness(mode, &trace, &global, &data)
|
||||
.context("witness generation failure")?;
|
||||
|
||||
// Zero out 'invalid' entries in data and output.
|
||||
scope!("zeroize", {
|
||||
hal.eltwise_zeroize_elem(&global.buf);
|
||||
hal.eltwise_zeroize_elem(&code.buf);
|
||||
hal.eltwise_zeroize_elem(&data.buf);
|
||||
});
|
||||
|
||||
Ok(Self {
|
||||
cycles,
|
||||
global,
|
||||
code,
|
||||
data,
|
||||
trace,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Injector {
|
||||
rows: usize,
|
||||
offsets: Vec<u32>,
|
||||
values: Vec<Val>,
|
||||
index: Vec<u32>,
|
||||
}
|
||||
|
||||
impl Injector {
|
||||
fn new(rows: usize) -> Self {
|
||||
let mut index = Vec::with_capacity(rows + 1);
|
||||
index.push(0);
|
||||
Self {
|
||||
rows,
|
||||
offsets: vec![],
|
||||
values: vec![],
|
||||
index,
|
||||
}
|
||||
}
|
||||
|
||||
fn set_cycle(&mut self, row: usize, cycle: &RawPreflightCycle) {
|
||||
const NEXT_PC_LOW: usize = LAYOUT_TOP.next_pc_low._super.offset;
|
||||
const NEXT_PC_HIGH: usize = LAYOUT_TOP.next_pc_high._super.offset;
|
||||
const NEXT_STATE: usize = LAYOUT_TOP.next_state_0._super.offset;
|
||||
const MACHINE_MODE: usize = LAYOUT_TOP.next_machine_mode._super.offset;
|
||||
self.set(row, NEXT_PC_LOW, cycle.pc & 0xffff);
|
||||
self.set(row, NEXT_PC_HIGH, cycle.pc >> 16);
|
||||
self.set(row, NEXT_STATE, cycle.state);
|
||||
self.set(row, MACHINE_MODE, cycle.machine_mode as u32);
|
||||
self.index.push(self.offsets.len() as u32);
|
||||
}
|
||||
|
||||
fn set(&mut self, row: usize, col: usize, value: u32) {
|
||||
let idx = col * self.rows + row;
|
||||
self.offsets.push(idx as u32);
|
||||
self.values.push(value.into());
|
||||
}
|
||||
}
|
||||
|
||||
fn node_idx_to_addr(idx: u32) -> WordAddr {
|
||||
MERKLE_TREE_END_ADDR - idx * DIGEST_WORDS as u32
|
||||
}
|
||||
|
||||
fn node_addr_to_idx(addr: WordAddr) -> u32 {
|
||||
(MERKLE_TREE_END_ADDR - addr).0 / DIGEST_WORDS as u32
|
||||
}
|
||||
@@ -0,0 +1,541 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use anyhow::{bail, Result};
|
||||
use risc0_circuit_rv32im_v2_sys::RawMemoryTransaction;
|
||||
use risc0_zkp::{
|
||||
core::{
|
||||
digest::DIGEST_WORDS,
|
||||
hash::poseidon2::{
|
||||
CELLS, M_INT_DIAG_HZN, ROUNDS_HALF_FULL, ROUNDS_PARTIAL, ROUND_CONSTANTS,
|
||||
},
|
||||
},
|
||||
field::{
|
||||
baby_bear::{self},
|
||||
Elem,
|
||||
},
|
||||
};
|
||||
|
||||
use crate::{
|
||||
execute::{
|
||||
addr::WordAddr,
|
||||
node_idx,
|
||||
pager::{PAGE_WORDS, POSEIDON_PAGE_ROUNDS},
|
||||
platform::*,
|
||||
r0vm::Risc0Context as _,
|
||||
},
|
||||
zirgen::circuit::{ExtVal, PoseidonStateLayout, LAYOUT_TOP},
|
||||
};
|
||||
|
||||
use super::{node_idx_to_addr, preflight::Preflight};
|
||||
|
||||
const P2_STATE_LAYOUT: &PoseidonStateLayout = LAYOUT_TOP.inst_result.arm9.state;
|
||||
|
||||
const BABY_BEAR_P_U32: u32 = baby_bear::P;
|
||||
const BABY_BEAR_P_U64: u64 = baby_bear::P as u64;
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct Poseidon2State {
|
||||
pub has_state: u32,
|
||||
pub state_addr: u32,
|
||||
pub buf_out_addr: u32,
|
||||
pub is_elem: u32,
|
||||
pub check_out: u32,
|
||||
pub load_tx_type: u32,
|
||||
pub next_state: CycleState,
|
||||
pub sub_state: u32,
|
||||
pub buf_in_addr: u32,
|
||||
pub count: u32,
|
||||
pub mode: u32,
|
||||
pub inner: [u32; CELLS],
|
||||
pub zcheck: ExtVal,
|
||||
}
|
||||
|
||||
const P2_STATE_SIZE: usize = std::mem::size_of::<Poseidon2State>() / WORD_SIZE;
|
||||
|
||||
impl Poseidon2State {
|
||||
fn new_start(mode: u32) -> Self {
|
||||
Self {
|
||||
buf_out_addr: if mode == 0 {
|
||||
MERKLE_TREE_END_ADDR.0
|
||||
} else {
|
||||
MERKLE_TREE_START_ADDR.0
|
||||
},
|
||||
is_elem: 1,
|
||||
check_out: 1,
|
||||
load_tx_type: tx::PAGE_IN,
|
||||
next_state: CycleState::PoseidonPaging,
|
||||
mode,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn new_done(buf_out_addr: u32, next_state: CycleState, mode: u32) -> Self {
|
||||
Self {
|
||||
buf_out_addr,
|
||||
next_state,
|
||||
mode,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn new_node(node_idx: u32, is_read: bool) -> Self {
|
||||
Self {
|
||||
buf_out_addr: node_idx_to_addr(node_idx).0,
|
||||
is_elem: 1,
|
||||
check_out: if is_read { 1 } else { 0 },
|
||||
load_tx_type: if is_read { tx::PAGE_IN } else { tx::PAGE_OUT },
|
||||
next_state: CycleState::PoseidonPaging,
|
||||
buf_in_addr: node_idx_to_addr(2 * node_idx + 1).0,
|
||||
count: 1,
|
||||
mode: if is_read { 0 } else { 4 },
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn new_page(page_idx: u32, is_read: bool) -> Self {
|
||||
let node_idx = node_idx(page_idx);
|
||||
Self {
|
||||
buf_out_addr: node_idx_to_addr(node_idx).0,
|
||||
check_out: if is_read { 1 } else { 0 },
|
||||
load_tx_type: if is_read { tx::PAGE_IN } else { tx::PAGE_OUT },
|
||||
next_state: CycleState::PoseidonPaging,
|
||||
buf_in_addr: page_idx * PAGE_WORDS as u32,
|
||||
count: POSEIDON_PAGE_ROUNDS,
|
||||
mode: if is_read { 1 } else { 3 },
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn new_ecall(state_addr: u32, buf_in_addr: u32, buf_out_addr: u32, bits_count: u32) -> Self {
|
||||
let is_elem = bits_count & PFLAG_IS_ELEM;
|
||||
let check_out = bits_count & PFLAG_CHECK_OUT;
|
||||
Self {
|
||||
state_addr,
|
||||
buf_in_addr,
|
||||
buf_out_addr,
|
||||
has_state: if state_addr == 0 { 0 } else { 1 },
|
||||
is_elem: if is_elem == 0 { 0 } else { 1 },
|
||||
check_out: if check_out == 0 { 0 } else { 1 },
|
||||
count: bits_count & 0xffff,
|
||||
mode: 1,
|
||||
load_tx_type: tx::READ,
|
||||
next_state: CycleState::PoseidonEntry,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn offsets() -> [usize; P2_STATE_SIZE] {
|
||||
[
|
||||
P2_STATE_LAYOUT.has_state._super.offset,
|
||||
P2_STATE_LAYOUT.state_addr._super.offset,
|
||||
P2_STATE_LAYOUT.buf_out_addr._super.offset,
|
||||
P2_STATE_LAYOUT.is_elem._super.offset,
|
||||
P2_STATE_LAYOUT.check_out._super.offset,
|
||||
P2_STATE_LAYOUT.load_tx_type._super.offset,
|
||||
P2_STATE_LAYOUT.next_state._super.offset,
|
||||
P2_STATE_LAYOUT.sub_state._super.offset,
|
||||
P2_STATE_LAYOUT.buf_in_addr._super.offset,
|
||||
P2_STATE_LAYOUT.count._super.offset,
|
||||
P2_STATE_LAYOUT.mode._super.offset,
|
||||
P2_STATE_LAYOUT.inner[0]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[1]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[2]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[3]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[4]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[5]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[6]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[7]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[8]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[9]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[10]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[11]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[12]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[13]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[14]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[15]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[16]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[17]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[18]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[19]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[20]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[21]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[22]._super.offset,
|
||||
P2_STATE_LAYOUT.inner[23]._super.offset,
|
||||
P2_STATE_LAYOUT.zcheck._super.offset,
|
||||
P2_STATE_LAYOUT.zcheck._super.offset + 1,
|
||||
P2_STATE_LAYOUT.zcheck._super.offset + 2,
|
||||
P2_STATE_LAYOUT.zcheck._super.offset + 3,
|
||||
]
|
||||
}
|
||||
|
||||
pub(crate) fn as_array(&self) -> [u32; P2_STATE_SIZE] {
|
||||
let zcheck = self.zcheck.elems();
|
||||
[
|
||||
self.has_state,
|
||||
self.state_addr,
|
||||
self.buf_out_addr,
|
||||
self.is_elem,
|
||||
self.check_out,
|
||||
self.load_tx_type,
|
||||
self.next_state as u32,
|
||||
self.sub_state,
|
||||
self.buf_in_addr,
|
||||
self.count,
|
||||
self.mode,
|
||||
self.inner[0],
|
||||
self.inner[1],
|
||||
self.inner[2],
|
||||
self.inner[3],
|
||||
self.inner[4],
|
||||
self.inner[5],
|
||||
self.inner[6],
|
||||
self.inner[7],
|
||||
self.inner[8],
|
||||
self.inner[9],
|
||||
self.inner[10],
|
||||
self.inner[11],
|
||||
self.inner[12],
|
||||
self.inner[13],
|
||||
self.inner[14],
|
||||
self.inner[15],
|
||||
self.inner[16],
|
||||
self.inner[17],
|
||||
self.inner[18],
|
||||
self.inner[19],
|
||||
self.inner[20],
|
||||
self.inner[21],
|
||||
self.inner[22],
|
||||
self.inner[23],
|
||||
zcheck[0].into(),
|
||||
zcheck[1].into(),
|
||||
zcheck[2].into(),
|
||||
zcheck[3].into(),
|
||||
]
|
||||
}
|
||||
|
||||
fn step(
|
||||
&mut self,
|
||||
ctx: &mut Preflight,
|
||||
cur_state: &mut CycleState,
|
||||
next_state: CycleState,
|
||||
sub_state: u32,
|
||||
) {
|
||||
self.next_state = next_state;
|
||||
self.sub_state = sub_state;
|
||||
ctx.on_poseidon2_cycle(*cur_state, self);
|
||||
*cur_state = next_state;
|
||||
}
|
||||
|
||||
fn rest(&mut self, ctx: &mut Preflight, final_state: CycleState) -> Result<()> {
|
||||
let mut cur_state = self.next_state;
|
||||
let state_addr = WordAddr(self.state_addr);
|
||||
|
||||
// If we have state, load it
|
||||
if self.has_state == 1 {
|
||||
// tracing::trace!("has_state");
|
||||
self.step(ctx, &mut cur_state, CycleState::PoseidonLoadState, 0);
|
||||
for i in 0..DIGEST_WORDS {
|
||||
self.inner[DIGEST_WORDS * 2 + i] = ctx.load_u32(state_addr + i)?;
|
||||
}
|
||||
}
|
||||
|
||||
// While we have data to process
|
||||
let mut buf_in_addr = WordAddr(self.buf_in_addr);
|
||||
// tracing::debug!("buf_in_addr: {buf_in_addr:?}");
|
||||
while self.count > 0 {
|
||||
// Do load
|
||||
self.step(ctx, &mut cur_state, CycleState::PoseidonLoadIn, 0);
|
||||
|
||||
if self.is_elem != 0 {
|
||||
for i in 0..DIGEST_WORDS {
|
||||
self.inner[i] = ctx.load_u32(buf_in_addr.postfix_inc())?;
|
||||
}
|
||||
self.buf_in_addr = buf_in_addr.0;
|
||||
self.step(ctx, &mut cur_state, CycleState::PoseidonLoadIn, 1);
|
||||
for i in 0..DIGEST_WORDS {
|
||||
self.inner[DIGEST_WORDS + i] = ctx.load_u32(buf_in_addr.postfix_inc())?;
|
||||
}
|
||||
self.buf_in_addr = buf_in_addr.0;
|
||||
} else {
|
||||
for i in 0..DIGEST_WORDS {
|
||||
let word = ctx.load_u32(buf_in_addr.postfix_inc())?;
|
||||
self.inner[2 * i] = word & 0xffff;
|
||||
self.inner[2 * i + 1] = word >> 16;
|
||||
}
|
||||
self.buf_in_addr = buf_in_addr.0;
|
||||
}
|
||||
|
||||
// Do the mix
|
||||
self.multiply_by_m_ext();
|
||||
for i in 0..ROUNDS_HALF_FULL {
|
||||
self.step(ctx, &mut cur_state, CycleState::PoseidonExtRound, i as u32);
|
||||
self.do_ext_round(i);
|
||||
}
|
||||
self.step(ctx, &mut cur_state, CycleState::PoseidonIntRound, 0);
|
||||
self.do_int_rounds();
|
||||
for i in ROUNDS_HALF_FULL..ROUNDS_HALF_FULL * 2 {
|
||||
self.step(ctx, &mut cur_state, CycleState::PoseidonExtRound, i as u32);
|
||||
self.do_ext_round(i);
|
||||
}
|
||||
self.count -= 1;
|
||||
}
|
||||
|
||||
self.step(ctx, &mut cur_state, CycleState::PoseidonDoOut, 0);
|
||||
|
||||
let buf_out_addr = WordAddr(self.buf_out_addr);
|
||||
if self.check_out != 0 {
|
||||
for i in 0..DIGEST_WORDS {
|
||||
let addr = buf_out_addr + i;
|
||||
let word = ctx.load_u32(addr)?;
|
||||
let cell = self.inner[i];
|
||||
if word != cell {
|
||||
tracing::warn!(
|
||||
"buf_in_addr: {:?}, buf_out_addr: {buf_out_addr:?}, cell: {i}",
|
||||
WordAddr(self.buf_in_addr)
|
||||
);
|
||||
bail!("poseidon2 check failed: {word:#010x} != {cell:#010x}");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for i in 0..DIGEST_WORDS {
|
||||
ctx.store_u32(buf_out_addr + i, self.inner[i])?;
|
||||
}
|
||||
}
|
||||
|
||||
self.buf_in_addr = 0;
|
||||
|
||||
if self.has_state == 1 {
|
||||
self.step(ctx, &mut cur_state, CycleState::PoseidonStoreState, 0);
|
||||
for i in 0..DIGEST_WORDS {
|
||||
ctx.store_u32(state_addr + i, self.inner[DIGEST_WORDS * 2 + i])?;
|
||||
}
|
||||
}
|
||||
|
||||
self.step(ctx, &mut cur_state, final_state, 0);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Optimized method for multiplication by M_EXT.
|
||||
// See appendix B of Poseidon2 paper for additional details.
|
||||
fn multiply_by_m_ext(&mut self) {
|
||||
let mut out = [0; CELLS];
|
||||
let mut tmp_sums = [0; 4];
|
||||
|
||||
for i in 0..CELLS / 4 {
|
||||
let chunk = multiply_by_4x4_circulant(&[
|
||||
self.inner[i * 4],
|
||||
self.inner[i * 4 + 1],
|
||||
self.inner[i * 4 + 2],
|
||||
self.inner[i * 4 + 3],
|
||||
]);
|
||||
for j in 0..4 {
|
||||
let to_add = chunk[j] as u64;
|
||||
let to_add = (to_add % BABY_BEAR_P_U64) as u32;
|
||||
tmp_sums[j] += to_add;
|
||||
tmp_sums[j] %= BABY_BEAR_P_U32;
|
||||
out[i * 4 + j] += to_add;
|
||||
out[i * 4 + j] %= BABY_BEAR_P_U32;
|
||||
}
|
||||
}
|
||||
for i in 0..CELLS {
|
||||
self.inner[i] = (out[i] + tmp_sums[i % 4]) % BABY_BEAR_P_U32;
|
||||
}
|
||||
}
|
||||
|
||||
// Exploit the fact that off-diagonal entries of M_INT are all 1.
|
||||
fn multiply_by_m_int(&mut self) {
|
||||
let mut sum = 0u64;
|
||||
for i in 0..CELLS {
|
||||
sum += self.inner[i] as u64;
|
||||
}
|
||||
sum %= BABY_BEAR_P_U64;
|
||||
for (i, diag) in M_INT_DIAG_HZN.iter().enumerate().take(CELLS) {
|
||||
let diag = diag.as_u32() as u64;
|
||||
let cell = self.inner[i] as u64;
|
||||
self.inner[i] = ((sum + diag * cell) % BABY_BEAR_P_U64) as u32;
|
||||
}
|
||||
}
|
||||
|
||||
fn do_ext_round(&mut self, mut idx: usize) {
|
||||
if idx >= ROUNDS_HALF_FULL {
|
||||
idx += ROUNDS_PARTIAL;
|
||||
}
|
||||
|
||||
self.add_round_constants_full(idx);
|
||||
for i in 0..CELLS {
|
||||
self.inner[i] = sbox2(self.inner[i]);
|
||||
}
|
||||
|
||||
self.multiply_by_m_ext();
|
||||
}
|
||||
|
||||
fn do_int_rounds(&mut self) {
|
||||
for i in 0..ROUNDS_PARTIAL {
|
||||
self.add_round_constants_partial(ROUNDS_HALF_FULL + i);
|
||||
self.inner[0] = sbox2(self.inner[0]);
|
||||
self.multiply_by_m_int();
|
||||
}
|
||||
}
|
||||
|
||||
fn add_round_constants_full(&mut self, round: usize) {
|
||||
for i in 0..CELLS {
|
||||
self.inner[i] += ROUND_CONSTANTS[round * CELLS + i].as_u32();
|
||||
self.inner[i] %= BABY_BEAR_P_U32;
|
||||
}
|
||||
}
|
||||
|
||||
fn add_round_constants_partial(&mut self, round: usize) {
|
||||
self.inner[0] += ROUND_CONSTANTS[round * CELLS].as_u32();
|
||||
self.inner[0] %= BABY_BEAR_P_U32;
|
||||
}
|
||||
}
|
||||
|
||||
fn multiply_by_4x4_circulant(x: &[u32; 4]) -> [u32; 4] {
|
||||
// See appendix B of Poseidon2 paper.
|
||||
const CIRC_FACTOR_2: u64 = 2;
|
||||
const CIRC_FACTOR_4: u64 = 4;
|
||||
let t0 = (x[0] as u64 + x[1] as u64) % BABY_BEAR_P_U64;
|
||||
let t1 = (x[2] as u64 + x[3] as u64) % BABY_BEAR_P_U64;
|
||||
let t2 = (CIRC_FACTOR_2 * x[1] as u64 + t1) % BABY_BEAR_P_U64;
|
||||
let t3 = (CIRC_FACTOR_2 * x[3] as u64 + t0) % BABY_BEAR_P_U64;
|
||||
let t4 = (CIRC_FACTOR_4 * t1 + t3) % BABY_BEAR_P_U64;
|
||||
let t5 = (CIRC_FACTOR_4 * t0 + t2) % BABY_BEAR_P_U64;
|
||||
let t6 = (t3 + t5) % BABY_BEAR_P_U64;
|
||||
let t7 = (t2 + t4) % BABY_BEAR_P_U64;
|
||||
[t6 as u32, t5 as u32, t7 as u32, t4 as u32]
|
||||
}
|
||||
|
||||
fn sbox2(x: u32) -> u32 {
|
||||
let x = x as u64;
|
||||
let x2 = (x * x) % BABY_BEAR_P_U64;
|
||||
let x4 = (x2 * x2) % BABY_BEAR_P_U64;
|
||||
let x6 = (x4 * x2) % BABY_BEAR_P_U64;
|
||||
let x7 = (x6 * x) % BABY_BEAR_P_U64;
|
||||
x7 as u32
|
||||
}
|
||||
|
||||
pub fn read_start(ctx: &mut Preflight) -> Result<()> {
|
||||
// tracing::trace!("read_start");
|
||||
let p2 = Poseidon2State::new_start(0);
|
||||
ctx.on_poseidon2_cycle(CycleState::PoseidonEntry, &p2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn read_node(ctx: &mut Preflight, node_idx: u32) -> Result<()> {
|
||||
// tracing::trace!("read_node: {node_idx:#010x}");
|
||||
let mut p2 = Poseidon2State::new_node(node_idx, true);
|
||||
p2.rest(ctx, CycleState::PoseidonPaging)
|
||||
}
|
||||
|
||||
pub fn read_page(ctx: &mut Preflight, page_idx: u32) -> Result<()> {
|
||||
// tracing::trace!("read_page: {page_idx:#010x}");
|
||||
let mut p2 = Poseidon2State::new_page(page_idx, true);
|
||||
p2.rest(ctx, CycleState::PoseidonPaging)
|
||||
}
|
||||
|
||||
pub fn read_done(ctx: &mut Preflight) -> Result<()> {
|
||||
// tracing::trace!("read_done");
|
||||
let p2 = Poseidon2State::new_done(MERKLE_TREE_START_ADDR.0, CycleState::Resume, 2);
|
||||
ctx.on_poseidon2_cycle(CycleState::PoseidonPaging, &p2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn write_start(ctx: &mut Preflight) -> Result<()> {
|
||||
// tracing::trace!("write_start");
|
||||
let p2 = Poseidon2State::new_start(3);
|
||||
ctx.on_poseidon2_cycle(CycleState::PoseidonEntry, &p2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn write_node(ctx: &mut Preflight, node_idx: u32) -> Result<()> {
|
||||
// tracing::trace!("write_node: {node_idx:#010x}");
|
||||
let mut p2 = Poseidon2State::new_node(node_idx, false);
|
||||
p2.rest(ctx, CycleState::PoseidonPaging)
|
||||
}
|
||||
|
||||
pub fn write_page(ctx: &mut Preflight, page_idx: u32) -> Result<()> {
|
||||
// tracing::trace!("write_page: {page_idx:#010x}");
|
||||
let mut p2 = Poseidon2State::new_page(page_idx, false);
|
||||
p2.rest(ctx, CycleState::PoseidonPaging)
|
||||
}
|
||||
|
||||
pub fn write_done(ctx: &mut Preflight) -> Result<()> {
|
||||
// tracing::trace!("write_done");
|
||||
let p2 = Poseidon2State::new_done(MERKLE_TREE_END_ADDR.0, CycleState::StoreRoot, 5);
|
||||
ctx.on_poseidon2_cycle(CycleState::PoseidonPaging, &p2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn ecall(ctx: &mut Preflight) -> Result<()> {
|
||||
tracing::trace!("ecall");
|
||||
let state_addr = ctx.load_u32(MACHINE_REGS_ADDR.waddr() + REG_A0)?;
|
||||
let buf_in_addr = ctx.load_u32(MACHINE_REGS_ADDR.waddr() + REG_A1)?;
|
||||
let buf_out_addr = ctx.load_u32(MACHINE_REGS_ADDR.waddr() + REG_A2)?;
|
||||
let bits_count = ctx.load_u32(MACHINE_REGS_ADDR.waddr() + REG_A3)?;
|
||||
let mut p2 = Poseidon2State::new_ecall(state_addr, buf_in_addr, buf_out_addr, bits_count);
|
||||
p2.rest(ctx, CycleState::Decode)
|
||||
}
|
||||
|
||||
pub(crate) struct Checksum {
|
||||
powers: [ExtVal; DIGEST_WORDS * 2 + 1],
|
||||
pub zcheck: ExtVal,
|
||||
}
|
||||
|
||||
impl Checksum {
|
||||
pub(crate) fn new(rand_z: &ExtVal) -> Self {
|
||||
let mut cur = ExtVal::ONE;
|
||||
let mut powers = [ExtVal::ZERO; DIGEST_WORDS * 2 + 1];
|
||||
for power in powers.iter_mut() {
|
||||
*power = cur;
|
||||
cur *= *rand_z;
|
||||
}
|
||||
// tracing::trace!("powers: {powers:?}");
|
||||
|
||||
Self {
|
||||
powers,
|
||||
zcheck: ExtVal::ZERO,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn start(&mut self) {
|
||||
self.zcheck *= self.powers[16];
|
||||
}
|
||||
|
||||
pub(crate) fn clear(&mut self) {
|
||||
self.zcheck = ExtVal::ZERO;
|
||||
}
|
||||
|
||||
pub(crate) fn add(&mut self, tx_kind: u32, idx: usize, txn: &RawMemoryTransaction) {
|
||||
let mut coeffs = match tx_kind {
|
||||
tx::READ => (0, 1),
|
||||
tx::PAGE_IN => (0, txn.cycle as i32 - txn.prev_cycle as i32),
|
||||
tx::PAGE_OUT => (
|
||||
(txn.word & 0xffff) as i32 - (txn.prev_word & 0xffff) as i32,
|
||||
(txn.word >> 16) as i32 - (txn.prev_word >> 16) as i32,
|
||||
),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
if coeffs.0 < 0 {
|
||||
coeffs.0 += baby_bear::P as i32;
|
||||
}
|
||||
if coeffs.1 < 0 {
|
||||
coeffs.1 += baby_bear::P as i32;
|
||||
}
|
||||
let coeffs = (coeffs.0 as u32, coeffs.1 as u32);
|
||||
self.zcheck += self.powers[2 * idx] * ExtVal::from_u32(coeffs.0);
|
||||
self.zcheck += self.powers[2 * idx + 1] * ExtVal::from_u32(coeffs.1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,590 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use derive_more::Debug;
|
||||
use num_traits::FromPrimitive as _;
|
||||
use risc0_circuit_rv32im_v2_sys::{RawMemoryTransaction, RawPreflightCycle};
|
||||
use risc0_core::scope;
|
||||
use risc0_zkp::core::digest::DIGEST_WORDS;
|
||||
|
||||
use crate::{
|
||||
execute::{
|
||||
addr::{ByteAddr, WordAddr},
|
||||
pager::PagedMemory,
|
||||
platform::*,
|
||||
r0vm::{Risc0Context, Risc0Machine},
|
||||
rv32im::{DecodedInstruction, Emulator, InsnKind, Instruction},
|
||||
segment::Segment,
|
||||
},
|
||||
zirgen::circuit::ExtVal,
|
||||
};
|
||||
|
||||
use self::poseidon2::Checksum;
|
||||
|
||||
use super::{node_addr_to_idx, node_idx_to_addr, poseidon2, Poseidon2State};
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) enum Back {
|
||||
#[default]
|
||||
None,
|
||||
Ecall(u32, u32, u32),
|
||||
#[debug("Poseidon2")]
|
||||
Poseidon2(Poseidon2State),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct PreflightTrace {
|
||||
#[debug("{}", cycles.len())]
|
||||
pub cycles: Vec<RawPreflightCycle>,
|
||||
#[debug("{}", txns.len())]
|
||||
pub txns: Vec<RawMemoryTransaction>,
|
||||
#[debug("{}", backs.len())]
|
||||
pub backs: Vec<Back>,
|
||||
pub table_split_cycle: u32,
|
||||
pub rand_z: ExtVal,
|
||||
}
|
||||
|
||||
pub(crate) struct Preflight<'a> {
|
||||
pub trace: PreflightTrace,
|
||||
segment: &'a Segment,
|
||||
pager: PagedMemory,
|
||||
pc: ByteAddr,
|
||||
machine_mode: u32,
|
||||
cur_write: usize,
|
||||
cur_read: usize,
|
||||
user_cycle: u32,
|
||||
txn_idx: u32,
|
||||
phys_cycles: u32,
|
||||
orig_words: BTreeMap<WordAddr, u32>,
|
||||
prev_cycle: BTreeMap<WordAddr, u32>,
|
||||
page_memory: BTreeMap<WordAddr, u32>,
|
||||
}
|
||||
|
||||
impl Segment {
|
||||
pub(crate) fn preflight(&self, rand_z: ExtVal) -> Result<PreflightTrace> {
|
||||
scope!("preflight");
|
||||
tracing::debug!("preflight: {self:#?}");
|
||||
|
||||
let mut preflight = Preflight::new(self, rand_z);
|
||||
preflight.read_pages()?;
|
||||
preflight.body()?;
|
||||
preflight.write_pages()?;
|
||||
preflight.generate_tables()?;
|
||||
preflight.padding()?;
|
||||
preflight.wrap_memory_txns()?;
|
||||
preflight.update_p2_zcheck()?;
|
||||
|
||||
tracing::trace!("paging_cycles: {}", preflight.pager.cycles);
|
||||
|
||||
Ok(preflight.trace)
|
||||
}
|
||||
}
|
||||
|
||||
fn get_digest_addr(idx: u32) -> WordAddr {
|
||||
MERKLE_TREE_START_ADDR + DIGEST_WORDS as u32 * (2 * MEMORY_PAGES as u32 - idx)
|
||||
}
|
||||
|
||||
impl<'a> Preflight<'a> {
|
||||
fn new(segment: &'a Segment, rand_z: ExtVal) -> Self {
|
||||
tracing::debug!("po2: {}", segment.po2);
|
||||
|
||||
let mut page_memory = BTreeMap::new();
|
||||
for (&node_idx, digest) in segment.partial_image.digests.iter() {
|
||||
let node_addr = node_idx_to_addr(node_idx);
|
||||
for i in 0..DIGEST_WORDS {
|
||||
page_memory.insert(node_addr + i, digest.as_words()[i]);
|
||||
}
|
||||
}
|
||||
Self {
|
||||
trace: PreflightTrace {
|
||||
rand_z,
|
||||
..Default::default()
|
||||
},
|
||||
segment,
|
||||
pager: PagedMemory::new(segment.partial_image.clone()),
|
||||
pc: ByteAddr(0),
|
||||
machine_mode: 0,
|
||||
cur_write: 0,
|
||||
cur_read: 0,
|
||||
txn_idx: 0,
|
||||
user_cycle: 0,
|
||||
phys_cycles: 0,
|
||||
orig_words: BTreeMap::new(),
|
||||
prev_cycle: BTreeMap::new(),
|
||||
page_memory,
|
||||
}
|
||||
}
|
||||
|
||||
// Do page in
|
||||
pub fn read_pages(&mut self) -> Result<()> {
|
||||
self.read_root()?;
|
||||
let activity = self.pager.loaded_pages();
|
||||
poseidon2::read_start(self)?;
|
||||
for node_idx in activity.nodes {
|
||||
poseidon2::read_node(self, node_idx)?;
|
||||
}
|
||||
self.machine_mode = 1;
|
||||
for page_idx in activity.pages {
|
||||
poseidon2::read_page(self, page_idx)?;
|
||||
}
|
||||
self.machine_mode = 2;
|
||||
poseidon2::read_done(self)?;
|
||||
self.phys_cycles = 0;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Run main execution
|
||||
pub fn body(&mut self) -> Result<()> {
|
||||
let mut emu = Emulator::new();
|
||||
Risc0Machine::resume(self)?;
|
||||
while self.phys_cycles < self.segment.suspend_cycle {
|
||||
Risc0Machine::step(&mut emu, self)?;
|
||||
}
|
||||
Risc0Machine::suspend(self)
|
||||
}
|
||||
|
||||
// Do page out
|
||||
pub fn write_pages(&mut self) -> Result<()> {
|
||||
let activity = self.pager.dirty_pages();
|
||||
self.pager.commit()?;
|
||||
poseidon2::write_start(self)?;
|
||||
for &page_idx in activity.pages.iter().rev() {
|
||||
poseidon2::write_page(self, page_idx)?;
|
||||
}
|
||||
self.machine_mode = 4;
|
||||
for &node_idx in activity.nodes.iter().rev() {
|
||||
poseidon2::write_node(self, node_idx)?;
|
||||
}
|
||||
self.machine_mode = 5;
|
||||
poseidon2::write_done(self)?;
|
||||
self.machine_mode = 0;
|
||||
self.write_root()
|
||||
}
|
||||
|
||||
// Do table reification
|
||||
pub fn generate_tables(&mut self) -> Result<()> {
|
||||
self.trace.table_split_cycle = self.trace.cycles.len() as u32;
|
||||
self.fini();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn padding(&mut self) -> Result<()> {
|
||||
let last_cycle = 1 << self.segment.po2;
|
||||
for _ in self.trace.cycles.len()..last_cycle {
|
||||
self.add_cycle_special(
|
||||
CycleState::ControlDone,
|
||||
CycleState::ControlDone,
|
||||
0,
|
||||
0,
|
||||
Back::None,
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Now, go back and update memory transactions to wrap around
|
||||
fn wrap_memory_txns(&mut self) -> Result<()> {
|
||||
for txn in self.trace.txns.iter_mut() {
|
||||
// tracing::trace!("{txn:?}");
|
||||
let addr = WordAddr(txn.addr);
|
||||
if txn.prev_cycle == u32::MAX {
|
||||
// If first cycle for a particular address, set 'prev_cycle' to final cycle
|
||||
txn.prev_cycle = self.prev_cycle[&addr];
|
||||
} else {
|
||||
// Otherwise, compute cycle diff and another diff
|
||||
let diff = txn.cycle - txn.prev_cycle;
|
||||
self.trace.cycles[diff as usize].diff_count += 1;
|
||||
}
|
||||
|
||||
// If last cycle, set final value to original value
|
||||
if txn.cycle == self.prev_cycle[&addr] {
|
||||
txn.word = self.orig_words[&addr];
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn update_p2_zcheck(&mut self) -> Result<()> {
|
||||
let mut checksum = Checksum::new(&self.trace.rand_z);
|
||||
for (row, back) in self.trace.backs.iter_mut().enumerate() {
|
||||
if let Back::Poseidon2(p2_state) = back {
|
||||
let cycle = &self.trace.cycles[row];
|
||||
let next_cycle = &self.trace.cycles[row + 1];
|
||||
let state = CycleState::from_u32((cycle.major as u32 - 7) * 8 + cycle.minor as u32)
|
||||
.unwrap();
|
||||
if state == CycleState::PoseidonLoadIn {
|
||||
checksum.start();
|
||||
for (i, txn_idx) in (cycle.txn_idx..next_cycle.txn_idx).enumerate() {
|
||||
let txn = &self.trace.txns[txn_idx as usize];
|
||||
checksum.add(p2_state.load_tx_type, i, txn);
|
||||
}
|
||||
}
|
||||
match state {
|
||||
CycleState::PoseidonLoadIn
|
||||
| CycleState::PoseidonExtRound
|
||||
| CycleState::PoseidonIntRound => {
|
||||
p2_state.zcheck = checksum.zcheck;
|
||||
}
|
||||
_ => {
|
||||
checksum.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn fini(&mut self) {
|
||||
for i in (16..256).step_by(16) {
|
||||
self.add_cycle_special(
|
||||
CycleState::ControlTable,
|
||||
CycleState::ControlTable,
|
||||
i,
|
||||
0,
|
||||
Back::None,
|
||||
);
|
||||
}
|
||||
self.machine_mode = 1;
|
||||
for i in (0..64 * 1024).step_by(16) {
|
||||
self.add_cycle_special(
|
||||
CycleState::ControlTable,
|
||||
CycleState::ControlTable,
|
||||
i,
|
||||
0,
|
||||
Back::None,
|
||||
);
|
||||
}
|
||||
self.machine_mode = 0;
|
||||
self.add_cycle_special(
|
||||
CycleState::ControlTable,
|
||||
CycleState::ControlDone,
|
||||
0,
|
||||
0,
|
||||
Back::None,
|
||||
);
|
||||
self.add_cycle_special(
|
||||
CycleState::ControlDone,
|
||||
CycleState::ControlDone,
|
||||
0,
|
||||
0,
|
||||
Back::None,
|
||||
);
|
||||
}
|
||||
|
||||
fn read_root(&mut self) -> Result<()> {
|
||||
let addr = get_digest_addr(1);
|
||||
for i in 0..DIGEST_WORDS {
|
||||
self.load_u32(addr + i)?;
|
||||
}
|
||||
self.add_cycle_special(
|
||||
CycleState::LoadRoot,
|
||||
CycleState::PoseidonEntry,
|
||||
0,
|
||||
0,
|
||||
Back::None,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_root(&mut self) -> Result<()> {
|
||||
let addr = get_digest_addr(1);
|
||||
for i in 0..DIGEST_WORDS {
|
||||
self.load_u32(addr + i)?;
|
||||
}
|
||||
self.add_cycle_special(
|
||||
CycleState::StoreRoot,
|
||||
CycleState::ControlTable,
|
||||
0,
|
||||
0,
|
||||
Back::None,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn add_cycle(
|
||||
&mut self,
|
||||
state: CycleState,
|
||||
pc: u32,
|
||||
major: u8,
|
||||
minor: u8,
|
||||
paging_idx: u32,
|
||||
back: Back,
|
||||
) {
|
||||
let cycle = RawPreflightCycle {
|
||||
state: state as u32,
|
||||
pc,
|
||||
major,
|
||||
minor,
|
||||
machine_mode: self.machine_mode as u8,
|
||||
padding: 0,
|
||||
user_cycle: self.user_cycle,
|
||||
txn_idx: self.txn_idx,
|
||||
paging_idx,
|
||||
diff_count: 0,
|
||||
};
|
||||
// tracing::trace!("[{}]: {cycle:?}", self.trace.cycles.len());
|
||||
self.trace.cycles.push(cycle);
|
||||
self.trace.backs.push(back);
|
||||
self.txn_idx = self.trace.txns.len() as u32;
|
||||
}
|
||||
|
||||
fn add_cycle_insn(&mut self, state: CycleState, pc: u32, insn: InsnKind) {
|
||||
tracing::trace!("[{}]: {pc:#010x}> {insn:?}", self.trace.cycles.len());
|
||||
match insn {
|
||||
InsnKind::Eany => {
|
||||
// Technically we need to switch on the machine mode *entering* the EANY
|
||||
if self.trace.cycles.last().unwrap().machine_mode != 0 {
|
||||
self.add_cycle(
|
||||
state,
|
||||
pc,
|
||||
major::ECALL0,
|
||||
ecall_minor::MACHINE_ECALL,
|
||||
0,
|
||||
Back::None,
|
||||
);
|
||||
} else {
|
||||
self.add_cycle(
|
||||
state,
|
||||
pc,
|
||||
major::CONTROL0,
|
||||
control_minor::USER_ECALL,
|
||||
0,
|
||||
Back::None,
|
||||
);
|
||||
}
|
||||
}
|
||||
InsnKind::Mret => {
|
||||
self.add_cycle(
|
||||
state,
|
||||
pc,
|
||||
major::CONTROL0,
|
||||
control_minor::MRET,
|
||||
0,
|
||||
Back::None,
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
self.add_cycle(state, pc, insn.major(), insn.minor(), 0, Back::None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn add_cycle_special(
|
||||
&mut self,
|
||||
cur_state: CycleState,
|
||||
next_state: CycleState,
|
||||
pc: u32,
|
||||
paging_idx: u32,
|
||||
back: Back,
|
||||
) {
|
||||
let cur_state = cur_state as u32;
|
||||
let major = (7 + cur_state / 8) as u8;
|
||||
let minor = (cur_state % 8) as u8;
|
||||
// tracing::trace!("add_cycle_special(cur_state: {cur_state}, next_state: {next_state}, major: {major}, minor: {minor})");
|
||||
self.add_cycle(next_state, pc, major, minor, paging_idx, back);
|
||||
}
|
||||
|
||||
pub(crate) fn on_poseidon2_cycle(&mut self, cur_state: CycleState, p2: &Poseidon2State) {
|
||||
self.add_cycle_special(
|
||||
cur_state,
|
||||
p2.next_state,
|
||||
self.pc.0,
|
||||
node_addr_to_idx(WordAddr(p2.buf_out_addr)),
|
||||
Back::Poseidon2(p2.clone()),
|
||||
);
|
||||
self.phys_cycles += 1;
|
||||
}
|
||||
|
||||
pub(crate) fn load_u32_with_txn(
|
||||
&mut self,
|
||||
addr: WordAddr,
|
||||
) -> Result<(u32, RawMemoryTransaction)> {
|
||||
let cycle = self.trace.cycles.len();
|
||||
let word = if addr >= MERKLE_TREE_START_ADDR {
|
||||
*self
|
||||
.page_memory
|
||||
.get(&addr)
|
||||
.ok_or(anyhow!("Invalid load from page memory"))?
|
||||
} else {
|
||||
self.pager.load(addr)?
|
||||
};
|
||||
self.orig_words.entry(addr).or_insert(word);
|
||||
let prev_cycle = *self.prev_cycle.get(&addr).unwrap_or(&u32::MAX);
|
||||
let txn = RawMemoryTransaction {
|
||||
addr: addr.0,
|
||||
cycle: cycle as u32,
|
||||
word,
|
||||
prev_cycle,
|
||||
prev_word: word,
|
||||
};
|
||||
self.prev_cycle.insert(addr, txn.cycle);
|
||||
self.trace.txns.push(txn.clone());
|
||||
Ok((word, txn))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Risc0Context for Preflight<'a> {
|
||||
fn get_pc(&self) -> ByteAddr {
|
||||
self.pc
|
||||
}
|
||||
|
||||
fn set_pc(&mut self, addr: ByteAddr) {
|
||||
self.pc = addr;
|
||||
}
|
||||
|
||||
fn get_machine_mode(&self) -> u32 {
|
||||
self.machine_mode
|
||||
}
|
||||
|
||||
fn set_machine_mode(&mut self, mode: u32) {
|
||||
self.machine_mode = mode;
|
||||
}
|
||||
|
||||
fn resume(&mut self) -> Result<()> {
|
||||
self.add_cycle_special(
|
||||
CycleState::Resume,
|
||||
CycleState::Resume,
|
||||
self.pc.0,
|
||||
0,
|
||||
Back::None,
|
||||
);
|
||||
for i in 0..DIGEST_WORDS {
|
||||
self.store_u32(GLOBAL_INPUT_ADDR.waddr() + i, 0)?; // FIXME!
|
||||
}
|
||||
self.add_cycle_special(
|
||||
CycleState::Resume,
|
||||
CycleState::Decode,
|
||||
self.pc.0,
|
||||
0,
|
||||
Back::None,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn suspend(&mut self) -> Result<()> {
|
||||
self.pc = ByteAddr(0);
|
||||
self.add_cycle_special(CycleState::Suspend, CycleState::Suspend, 0, 0, Back::None);
|
||||
for i in 0..DIGEST_WORDS {
|
||||
self.load_u32(GLOBAL_OUTPUT_ADDR.waddr() + i)?;
|
||||
}
|
||||
self.machine_mode = 3;
|
||||
self.add_cycle_special(
|
||||
CycleState::Suspend,
|
||||
CycleState::PoseidonEntry,
|
||||
0,
|
||||
0,
|
||||
Back::None,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn on_insn_start(&mut self, _insn: &Instruction, _decoded: &DecodedInstruction) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn on_insn_end(&mut self, insn: &Instruction, _decoded: &DecodedInstruction) -> Result<()> {
|
||||
self.add_cycle_insn(CycleState::Decode, self.pc.0, insn.kind);
|
||||
self.user_cycle += 1;
|
||||
self.phys_cycles += 1;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn trap_rewind(&mut self) {
|
||||
self.trace.txns.truncate(self.txn_idx as usize);
|
||||
}
|
||||
|
||||
fn peek_u32(&mut self, _addr: WordAddr) -> Result<u32> {
|
||||
// no-op is OK
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
// Pass memory ops to pager + record
|
||||
fn load_u32(&mut self, addr: WordAddr) -> Result<u32> {
|
||||
// tracing::trace!("load_u32: {addr:?}");
|
||||
let (word, _) = self.load_u32_with_txn(addr)?;
|
||||
Ok(word)
|
||||
}
|
||||
|
||||
fn store_u32(&mut self, addr: WordAddr, word: u32) -> Result<()> {
|
||||
let cycle = self.trace.cycles.len();
|
||||
let prev_word = if addr >= MEMORY_END_ADDR {
|
||||
let prev_word = *self
|
||||
.page_memory
|
||||
.get(&addr)
|
||||
.ok_or(anyhow!("Invalid store to page memory"))?;
|
||||
self.page_memory.insert(addr, word);
|
||||
prev_word
|
||||
} else {
|
||||
let prev_word = self.pager.load(addr)?;
|
||||
self.pager.store(addr, word)?;
|
||||
prev_word
|
||||
};
|
||||
let prev_cycle = *self.prev_cycle.get(&addr).unwrap_or(&u32::MAX);
|
||||
let txn = RawMemoryTransaction {
|
||||
addr: addr.0,
|
||||
cycle: cycle as u32,
|
||||
word,
|
||||
prev_cycle,
|
||||
prev_word,
|
||||
};
|
||||
self.prev_cycle.insert(addr, txn.cycle);
|
||||
self.trace.txns.push(txn);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn on_ecall_cycle(
|
||||
&mut self,
|
||||
cur_state: CycleState,
|
||||
next_state: CycleState,
|
||||
s0: u32,
|
||||
s1: u32,
|
||||
s2: u32,
|
||||
) -> Result<()> {
|
||||
self.add_cycle_special(cur_state, next_state, self.pc.0, 0, Back::Ecall(s0, s1, s2));
|
||||
self.phys_cycles += 1;
|
||||
if next_state == CycleState::PoseidonEntry {
|
||||
poseidon2::ecall(self)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn on_terminate(&mut self, _a0: u32, _a1: u32) {
|
||||
// no-op
|
||||
}
|
||||
|
||||
fn host_read(&mut self, _fd: u32, buf: &mut [u8]) -> Result<u32> {
|
||||
if self.cur_read >= self.segment.read_record.len() {
|
||||
bail!("Invalid segment: unexpected read record");
|
||||
}
|
||||
let record = &self.segment.read_record[self.cur_read];
|
||||
let rlen = record.len();
|
||||
if rlen > buf.len() {
|
||||
bail!("Invalid segment: truncated read record");
|
||||
}
|
||||
buf[..rlen].copy_from_slice(record);
|
||||
Ok(rlen as u32)
|
||||
}
|
||||
|
||||
fn host_write(&mut self, _fd: u32, _buf: &[u8]) -> Result<u32> {
|
||||
if self.cur_write >= self.segment.write_record.len() {
|
||||
bail!("Invalid segment: unexpected write record");
|
||||
}
|
||||
self.cur_write += 1;
|
||||
Ok(self.segment.write_record[self.cur_write])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::rc::Rc;
|
||||
|
||||
use rand::thread_rng;
|
||||
use risc0_binfmt::Program;
|
||||
use risc0_zkp::field::Elem;
|
||||
use test_log::test;
|
||||
|
||||
use crate::{
|
||||
execute::{
|
||||
image::MemoryImage2,
|
||||
testutil::{self, NullSyscall, DEFAULT_SESSION_LIMIT},
|
||||
DEFAULT_SEGMENT_LIMIT_PO2,
|
||||
},
|
||||
prove::{hal::StepMode, witgen::WitnessGenerator},
|
||||
zirgen::circuit::{ExtVal, REGCOUNT_DATA},
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn basic() {
|
||||
let program = testutil::basic();
|
||||
let image = MemoryImage2::new(program);
|
||||
|
||||
let result = testutil::execute(
|
||||
image,
|
||||
DEFAULT_SEGMENT_LIMIT_PO2,
|
||||
DEFAULT_SESSION_LIMIT,
|
||||
&NullSyscall,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let segments = result.segments;
|
||||
let segment = segments.first().unwrap();
|
||||
|
||||
let mut rng = thread_rng();
|
||||
let rand_z = ExtVal::random(&mut rng);
|
||||
|
||||
segment.preflight(rand_z).unwrap();
|
||||
}
|
||||
|
||||
fn fwd_rev_ab_test(program: Program) {
|
||||
let image = MemoryImage2::new(program);
|
||||
|
||||
let session = testutil::execute(
|
||||
image,
|
||||
DEFAULT_SEGMENT_LIMIT_PO2,
|
||||
testutil::DEFAULT_SESSION_LIMIT,
|
||||
&testutil::NullSyscall,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
cfg_if::cfg_if! {
|
||||
if #[cfg(feature = "cuda")] {
|
||||
use risc0_zkp::hal::cuda::CudaHalPoseidon2;
|
||||
use crate::prove::hal::cuda::CudaCircuitHalPoseidon2;
|
||||
let hal = Rc::new(CudaHalPoseidon2::new());
|
||||
let circuit_hal = CudaCircuitHalPoseidon2::new(hal.clone());
|
||||
// } else if #[cfg(any(all(target_os = "macos", target_arch = "aarch64"), target_os = "ios"))] {
|
||||
// use risc0_zkp::hal::metal::MetalHalSha256;
|
||||
// use crate::prove::hal::metal::MetalCircuitHal;
|
||||
// let hal = Rc::new(MetalHalSha256::new());
|
||||
// let circuit_hal = MetalCircuitHal::new(hal.clone());
|
||||
} else {
|
||||
let suite = risc0_zkp::core::hash::poseidon2::Poseidon2HashSuite::new_suite();
|
||||
let hal = Rc::new(risc0_zkp::hal::cpu::CpuHal::new(suite));
|
||||
let circuit_hal = crate::prove::hal::cpu::CpuCircuitHal;
|
||||
}
|
||||
}
|
||||
|
||||
let mut rng = thread_rng();
|
||||
let rand_z = ExtVal::random(&mut rng);
|
||||
|
||||
let segments = session.segments;
|
||||
for segment in segments {
|
||||
tracing::debug!("fwd");
|
||||
let fwd_witgen = WitnessGenerator::new(
|
||||
hal.as_ref(),
|
||||
&circuit_hal,
|
||||
&segment,
|
||||
StepMode::SeqForward,
|
||||
rand_z,
|
||||
)
|
||||
.unwrap();
|
||||
tracing::debug!("rev");
|
||||
let rev_witgen = WitnessGenerator::new(
|
||||
hal.as_ref(),
|
||||
&circuit_hal,
|
||||
&segment,
|
||||
StepMode::SeqReverse,
|
||||
rand_z,
|
||||
)
|
||||
.unwrap();
|
||||
let cycles = 1 << segment.po2;
|
||||
let fwd_vec = fwd_witgen.data.to_vec();
|
||||
let rev_vec = rev_witgen.data.to_vec();
|
||||
for row in 0..cycles {
|
||||
let fwd_row = &fwd_vec[row * REGCOUNT_DATA..row * REGCOUNT_DATA + REGCOUNT_DATA];
|
||||
let rev_row = &rev_vec[row * REGCOUNT_DATA..row * REGCOUNT_DATA + REGCOUNT_DATA];
|
||||
assert_eq!(fwd_row, rev_row, "cycle: {row}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fwd_rev_ab_basic() {
|
||||
fwd_rev_ab_test(testutil::basic());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fwd_rev_ab_split() {
|
||||
fwd_rev_ab_test(testutil::simple_loop(2000));
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
set_field!(BabyBear);
|
||||
define_buffer_list! {
|
||||
all: [accum,code,data,global,mix,test,],
|
||||
rows: [accum,code,data,test,],
|
||||
taps: [accum,code,data,],
|
||||
globals: [global,mix,],}
|
||||
define_tap_buffer! {accum, /*count=*/76, /*groupId=*/0}
|
||||
define_tap_buffer! {code, /*count=*/1, /*groupId=*/1}
|
||||
define_tap_buffer! {data, /*count=*/192, /*groupId=*/2}
|
||||
define_global_buffer! {global, /*count=*/73}
|
||||
define_global_buffer! {mix, /*count=*/32}
|
||||
define_buffer! {test, /*count=*/192}
|
||||
@@ -0,0 +1,58 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// This code is automatically generated
|
||||
|
||||
use risc0_zkp::adapter::{CircuitInfo, ProtocolInfo};
|
||||
|
||||
use super::CircuitImpl;
|
||||
|
||||
impl CircuitInfo for CircuitImpl {
|
||||
#[rustfmt::skip]
|
||||
const CIRCUIT_INFO: ProtocolInfo = ProtocolInfo(*b"ZIRGEN_TEST_____");
|
||||
|
||||
#[rustfmt::skip]
|
||||
const OUTPUT_SIZE: usize = 73;
|
||||
|
||||
#[rustfmt::skip]
|
||||
const MIX_SIZE: usize = 32;
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub const NUM_POLY_MIX_POWERS: usize = 411;
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub const POLY_MIX_POWERS: &[usize] = &[
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
|
||||
26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
|
||||
50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73,
|
||||
74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97,
|
||||
98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116,
|
||||
117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135,
|
||||
136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154,
|
||||
155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 173, 182, 183, 184, 188, 190,
|
||||
197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215,
|
||||
216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 235, 236,
|
||||
237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255,
|
||||
256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274,
|
||||
280, 284, 291, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310,
|
||||
311, 312, 314, 323, 330, 332, 335, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352,
|
||||
353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371,
|
||||
372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390,
|
||||
391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 406, 411, 443, 463, 471, 515, 516, 517, 518,
|
||||
519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, 611, 704, 705,
|
||||
706, 707, 708, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721, 783, 815, 942,
|
||||
1013, 1092, 1171, 1343, 1446, 1719, 1853, 1878, 2028, 2176, 3048, 3361, 5215, 5622, 5623, 5624,
|
||||
5625, 5626, 5635, 5644, 5653, 5666, 5680, 5690, 5701, 5720, 5728, 5744,
|
||||
];
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,90 @@
|
||||
// Copyright 2024 RISC Zero, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use risc0_zkp::{
|
||||
adapter::{CircuitCoreDef, TapsProvider},
|
||||
field::baby_bear::BabyBear,
|
||||
taps::TapSet,
|
||||
};
|
||||
|
||||
pub(crate) mod info;
|
||||
pub(crate) mod poly_ext;
|
||||
pub(crate) mod taps;
|
||||
|
||||
pub(crate) struct CircuitImpl;
|
||||
|
||||
#[allow(unused)]
|
||||
#[allow(non_camel_case_types)]
|
||||
#[allow(non_snake_case)]
|
||||
pub(crate) mod circuit {
|
||||
use risc0_zkp::layout::Reg;
|
||||
|
||||
macro_rules! set_field {
|
||||
($field:ident) => {
|
||||
paste::paste! {
|
||||
pub type CircuitField = risc0_core::field::[<$field:snake>]::$field;
|
||||
pub type Val = <CircuitField as risc0_core::field::Field>::Elem;
|
||||
pub type ExtVal = <CircuitField as risc0_core::field::Field>::ExtElem;
|
||||
pub type MixState = risc0_zkp::adapter::MixState<ExtVal>;
|
||||
pub type PolyMix = ExtVal;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! define_buffer_list {
|
||||
(
|
||||
all: [ $( $name:ident ,)* ],
|
||||
rows: [ $( $rows_name:ident ,)* ],
|
||||
taps: [ $( $taps_name:ident ,)* ],
|
||||
globals: [ $( $globals_name:ident ,)* ],
|
||||
) => {};
|
||||
}
|
||||
|
||||
macro_rules! define_tap_buffer {
|
||||
($name:ident, $size:literal, $reg_group_id:literal) => {
|
||||
paste::paste! {
|
||||
pub const [< REGCOUNT_ $name:upper >] : usize = $size;
|
||||
pub const [< REGISTER_GROUP_ $name:upper >] : usize = $reg_group_id;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! define_global_buffer {
|
||||
($name:ident, $size:literal) => {
|
||||
paste::paste! {
|
||||
pub const [< REGCOUNT_ $name:upper >] : usize = $size;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! define_buffer {
|
||||
($name:ident, $size:literal) => {
|
||||
paste::paste! {
|
||||
pub const [< REGCOUNT_ $name:upper >] : usize = $size;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
include! {"types.rs.inc"}
|
||||
include! {"defs.rs.inc"}
|
||||
include! {"layout.rs.inc"}
|
||||
}
|
||||
|
||||
impl CircuitCoreDef<BabyBear> for CircuitImpl {}
|
||||
|
||||
impl TapsProvider for CircuitImpl {
|
||||
fn get_taps(&self) -> &'static TapSet<'static> {
|
||||
self::taps::TAPSET
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user