logos-storage/logos-storage-proofs-circuits
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-proofs-circuits.git synced 2026-01-07 16:03:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

make cellSize and blockSize global parameters

Browse Source
This commit is contained in:
Balazs Komuves 2023-12-15 12:08:28 +01:00
parent cbcf9b476c
commit fea3d520b4
No known key found for this signature in database
GPG Key ID: F63B7AEF18435562
7 changed files with 74 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
reference/nim/proof_input/src/blocks.nim
View File

@ -12,45 +12,47 @@ import merkle
#-------------------------------------------------------------------------------
func hashCellOpen( cellData: openArray[byte] ): Hash =
assert( cellData.len == cellSize , "cells are expected to be exactly 2048 bytes" )
func hashCellOpen( globcfg: GlobalConfig, cellData: openArray[byte] ): Hash =
assert( cellData.len == globcfg.cellSize , ("cells are expected to be exactly " & $globcfg.cellSize & " bytes") )
return Sponge.digest( cellData, rate=2 )
func hashCell*(cellData: Cell): Hash = hashCellOpen(cellData)
func hashCell*( globcfg: GlobalConfig, cellData: Cell): Hash = hashCellOpen(globcfg, cellData)
#-------------------------------------------------------------------------------
func splitBlockIntoCells( blockData: openArray[byte] ): seq[Cell] =
assert( blockData.len == blockSize , "network blocks are expected to be exactly 65536 bytes" )
func splitBlockIntoCells( globcfg: GlobalConfig, blockData: openArray[byte] ): seq[Cell] =
assert( blockData.len == globcfg.blockSize , ("network blocks are expected to be exactly" & $globcfg.blockSize & " bytes" ) )
var cells : seq[seq[byte]] = newSeq[seq[byte]]( cellsPerBlock )
var cells : seq[seq[byte]] = newSeq[seq[byte]]( cellsPerBlock(globcfg) )
let start = low(blockData)
var leaves : seq[Hash] = newSeq[Hash]( cellsPerBlock )
for i in 0..<cellsPerBlock:
let a = start + i * cellSize
let b = start + (i+1) * cellSize
var leaves : seq[Hash] = newSeq[Hash]( cellsPerBlock(globcfg) )
for i in 0..<cellsPerBlock(globcfg):
let a = start + i * globcfg.cellSize
let b = start + (i+1) * globcfg.cellSize
cells[i] = blockData[a..<b].toSeq()
return cells
# returns the special hash of a network block (this is a Merkle root built on the
# top of the hashes of the 32 cells inside the block)
func hashNetworkBlockOpen( blockData: openArray[byte] ): Hash =
let cells = splitBlockIntoCells(blockData)
let leaves = collect( newSeq , (for i in 0..<cellsPerBlock: hashCell(cells[i]) ))
func hashNetworkBlockOpen( globcfg: GlobalConfig, blockData: openArray[byte] ): Hash =
let cells = splitBlockIntoCells(globcfg, blockData)
let leaves = collect( newSeq , (for i in 0..<cellsPerBlock(globcfg): hashCell(globcfg, cells[i]) ))
return merkleRoot(leaves)
func hashNetworkBlock*(blockData: Block): Hash = hashNetworkBlockOpen(blockData)
func hashNetworkBlock*(globcfg: GlobalConfig, blockData: Block): Hash =
hashNetworkBlockOpen(globcfg, blockData)
#-------------------------------------------------------------------------------
# returns the mini Merkle tree built on the 32 cells inside a network block
func networkBlockTreeOpen( blockData: openArray[byte] ): MerkleTree =
let cells = splitBlockIntoCells(blockData)
let leaves = collect( newSeq , (for i in 0..<cellsPerBlock: hashCell(cells[i]) ))
func networkBlockTreeOpen( globcfg: GlobalConfig, blockData: openArray[byte] ): MerkleTree =
let cells = splitBlockIntoCells( globcfg, blockData)
let leaves = collect( newSeq , (for i in 0..<cellsPerBlock(globcfg): hashCell( globcfg, cells[i]) ))
return merkleTree(leaves)
func networkBlockTree*(blockData: Block): MerkleTree = networkBlockTreeOpen(blockData)
func networkBlockTree*( globcfg: GlobalConfig, blockData: Block): MerkleTree =
networkBlockTreeOpen(globcfg, blockData)
#-------------------------------------------------------------------------------

12
reference/nim/proof_input/src/dataset.nim
View File

@ -21,7 +21,9 @@ const exDataSetCfg* =
const exGlobalCfg* =
GlobalConfig( maxDepth: 16
, maxLog2NSlots: 5
, maxLog2NSlots: 5
, cellSize: 256
, blockSize: 4096
)
#-------------------------------------------------------------------------------
@ -50,12 +52,12 @@ func slotCfgFromDataSetCfg*( dsetcfg: DataSetConfig, idx: SlotIdx ): SlotConfig
#-------------------------------------------------------------------------------
proc dataSetLoadCellData*(dsetCfg: DataSetConfig, slotIdx: SlotIdx, cellIdx: CellIdx): Cell =
proc dataSetLoadCellData*(globCfg: GlobalConfig, dsetCfg: DataSetConfig, slotIdx: SlotIdx, cellIdx: CellIdx): Cell =
let slotCfg = slotCfgFromDataSetCfg( dsetCfg, slotIdx )
return slotLoadCellData(slotCfg, cellIdx)
return slotLoadCellData(globCfg, slotCfg, cellIdx)
proc dataSetLoadBlockData*(dsetCfg: DataSetConfig, slotIdx: SlotIdx, blockIdx: BlockIdx): Block =
proc dataSetLoadBlockData*(globCfg: GlobalConfig, dsetCfg: DataSetConfig, slotIdx: SlotIdx, blockIdx: BlockIdx): Block =
let slotCfg = slotCfgFromDataSetCfg( dsetCfg, slotIdx )
return slotLoadBlockData(slotCfg, blockIdx)
return slotLoadBlockData(globCfg, slotCfg, blockIdx)
#-------------------------------------------------------------------------------

28
reference/nim/proof_input/src/gen_input.nim
View File

@ -16,27 +16,27 @@ import types
#-------------------------------------------------------------------------------
proc buildSlotTreeFull( slotCfg: SlotConfig ): (seq[MerkleTree], MerkleTree) =
proc buildSlotTreeFull( globcfg: GlobalConfig, slotCfg: SlotConfig ): (seq[MerkleTree], MerkleTree) =
let ncells = slotCfg.nCells
let nblocks = ncells div cellsPerBlock
assert( nblocks * cellsPerBlock == ncells )
let blocks : seq[Block] = collect( newSeq, (for i in 0..<nblocks: slotLoadBlockData(slotCfg, i) ))
let miniTrees : seq[MerkleTree] = map( blocks , networkBlockTree )
let nblocks = ncells div cellsPerBlock(globcfg)
assert( nblocks * cellsPerBlock(globcfg) == ncells )
let blocks : seq[Block] = collect( newSeq, (for i in 0..<nblocks: slotLoadBlockData(globcfg, slotCfg, i) ))
let miniTrees : seq[MerkleTree] = collect( newSeq, (for blk in blocks: networkBlockTree(globcfg, blk) ))
let blockHashes : seq[Root] = map( miniTrees , treeRoot )
let bigTree = merkleTree( blockHashes )
return (miniTrees, bigTree)
proc buildSlotTree( slotCfg: SlotConfig ): MerkleTree =
return buildSlotTreeFull(slotCfg)[1]
proc buildSlotTree( globcfg: GlobalConfig, slotCfg: SlotConfig ): MerkleTree =
return buildSlotTreeFull(globcfg, slotCfg)[1]
proc generateProofInput*( globCfg: GlobalConfig, dsetCfg: DataSetConfig, slotIdx: SlotIdx, entropy: Entropy ): SlotProofInput =
let nslots = dsetCfg.nSlots
let ncells = dsetCfg.nCells
let nblocks = ncells div cellsPerBlock
assert( nblocks * cellsPerBlock == ncells )
let nblocks = ncells div cellsPerBlock(globCfg)
assert( nblocks * cellsPerBlock(globcfg) == ncells )
let slotCfgs = collect( newSeq , (for i in 0..<nslots: slotCfgFromDataSetCfg(dsetcfg, i) ))
let slotTrees = map( slotCfgs, buildSlotTree )
let slotTrees = collect( newSeq , (for scfg in slotcfgs: buildSlotTree(globCfg, scfg) ))
let slotRoots = map( slotTrees, treeRoot )
let ourSlotCfg = slotCfgs[slotIdx]
@ -51,11 +51,11 @@ proc generateProofInput*( globCfg: GlobalConfig, dsetCfg: DataSetConfig, slotIdx
var inputs : seq[CellProofInput]
for cellIdx in indices:
let (miniTrees, bigTree) = buildSlotTreeFull( ourSlotCfg )
let blockIdx = cellIdx div cellsPerBlock
let (miniTrees, bigTree) = buildSlotTreeFull( globCfg, ourSlotCfg )
let blockIdx = cellIdx div cellsPerBlock(globcfg)
let blockTree = miniTrees[ blockIdx ]
let cellData = slotLoadCellData( ourSlotCfg, cellIdx )
let botProof = merkleProof( blockTree , cellIdx mod cellsPerBlock )
let cellData = slotLoadCellData( globCfg, ourSlotCfg, cellIdx )
let botProof = merkleProof( blockTree , cellIdx mod cellsPerBlock(globcfg) )
let topProof = merkleProof( bigTree , blockIdx )
let prf = padMerkleProof( mergeMerkleProofs( botProof, topProof ), globCfg.maxDepth )
inputs.add( CellProofInput(cellData: cellData, merkleProof: prf) )

11
reference/nim/proof_input/src/misc.nim
View File

@ -3,9 +3,11 @@
# helper functions
#
import std/math
#-------------------------------------------------------------------------------
func floorLog2* (x : int) : int =
func floorLog2* (x: int): int =
var k = -1
var y = x
while (y > 0):
@ -13,10 +15,15 @@ func floorLog2* (x : int) : int =
y = y shr 1
return k
func ceilingLog2* (x : int) : int =
func ceilingLog2* (x: int): int =
if (x==0):
return -1
else:
return (floorLog2(x-1) + 1)
func exactLog2( x: int): int =
let k = ceilingLog2(x)
assert( x == 2^k, "exactLog2: not a power of two" )
return k
#-------------------------------------------------------------------------------

25
reference/nim/proof_input/src/slot.nim
View File

@ -22,12 +22,12 @@ const exSlotCfg =
# (10852671575406741732, 3735945064, 2557891771)
{.overflowChecks: off.}
proc genFakeCell(cfg: SlotConfig, seed: Seed, idx: CellIdx): Cell =
proc genFakeCell(globcfg: GlobalConfig, cfg: SlotConfig, seed: Seed, idx: CellIdx): Cell =
let seed1 : uint64 = uint64(seed) + 0xdeadcafe'u64
let seed2 : uint64 = uint64(idx) + 0x98765432'u64
var cell : seq[byte] = newSeq[byte](cellSize)
var cell : seq[byte] = newSeq[byte](globcfg.cellSize)
var state : uint64 = 1
for i in 0..<cellSize:
for i in 0..<globcfg.cellSize:
state = state*(state + seed1)*(state + seed2) + state*(state xor 0x5a5a5a5a) + seed1*state + (seed2 + 17)
state = state mod 1698428844001831'u64
cell[i] = byte(state)
@ -48,25 +48,28 @@ genFakeCell cfg (Seed seed) (CellIdx idx) = (mkCellData cfg $ B.pack list) where
#-------------------------------------------------------------------------------
proc slotLoadCellData*(cfg: SlotConfig, idx: CellIdx): Cell =
proc slotLoadCellData*(globcfg: GlobalConfig, cfg: SlotConfig, idx: CellIdx): Cell =
case cfg.dataSrc.kind
of FakeData:
return genFakeCell(cfg, cfg.dataSrc.seed, idx)
return genFakeCell(globcfg, cfg, cfg.dataSrc.seed, idx)
of SlotFile:
let stream = newFileStream(cfg.dataSrc.filename, mode = fmRead)
defer: stream.close()
stream.setPosition( cellSize * idx )
var arr : array[cellSize, byte]
discard stream.readData( addr(arr), cellSize )
const maxcellsize = 16384
assert( globcfg.cellSize <= maxcellsize )
stream.setPosition( globcfg.cellSize * idx )
var arr : array[maxcellsize, byte]
discard stream.readData( addr(arr), globcfg.cellSize )
return arr.toSeq()
let cell: seq[byte] = collect( newSeq, (for i in 0..<globcfg.cellSize: arr[i] ))
return cell
proc slotLoadBlockData*(cfg: SlotConfig, idx: BlockIdx): Block =
proc slotLoadBlockData*(globcfg: GlobalConfig, cfg: SlotConfig, idx: BlockIdx): Block =
let cells : seq[seq[byte]] =
collect( newSeq , (for i in 0..<cellsPerBlock: slotLoadCellData(cfg, idx*cellsPerBlock+i) ))
collect( newSeq , (for i in 0..<cellsPerBlock(globcfg): slotLoadCellData(globcfg, cfg, idx*cellsPerBlock(globcfg)+i) ))
return concat(cells)
#-------------------------------------------------------------------------------

2
reference/nim/proof_input/src/testmain.nim
View File

@ -50,7 +50,7 @@ when isMainModule:
let slotIdx = 3
let fakedata = DataSource(kind: FakeData, seed: 12345)
let globcfg = GlobalConfig( maxDepth: 16, maxLog2NSlots: 5)
let globcfg = GlobalConfig( maxDepth: 16, maxLog2NSlots: 5, cellSize: 128, blockSize: 4096)
let dsetcfg = DataSetConfig( nCells: 256, nSlots: 5, nSamples: 10, dataSrc: fakedata)
let entropy = toF( 1234567 )
let prfInput = generateProofInput(globcfg, dsetcfg, slotIdx, entropy)

16
reference/nim/proof_input/src/types.nim
View File

@ -9,13 +9,6 @@ export types
#-------------------------------------------------------------------------------
const cellSize* : int = 128 # 2048 # size of the cells we prove
const blockSize* : int = 4096 # 65536 # size of the network block
const cellsPerBlock* : int = blockSize div cellSize
#-------------------------------------------------------------------------------
type Entropy* = F
type Hash* = F
type Root* = Hash
@ -110,5 +103,14 @@ type
GlobalConfig* = object
maxDepth* : int # maximum depth of the big merkle tree (log2 of maximum numbers of cells per slot)
maxLog2NSlots* : int # log2 of maximum number of slots per dataset
cellSize* : int # size of the cells we prove (2048)
blockSize* : int # size of the network block (65536)
#-------------------------------------------------------------------------------
func cellsPerBlock*(glob: GlobalConfig): int =
let k = (glob.blockSize div glob.cellSize)
assert( k * glob.cellSize == glob.blockSize , "block size is not divisible by cell size" )
return k
#-------------------------------------------------------------------------------