mirror of
https://github.com/logos-messaging/nim-chat-sdk.git
synced 2026-01-05 07:33:07 +00:00
chore: clear log
This commit is contained in:
kaichaosun
parent
7b71248a80
commit
baa41c9e81
@ -47,7 +47,6 @@ proc saveMessageSegment*(
|
||||
sigPubKeyBlob: seq[byte],
|
||||
timestamp: int64,
|
||||
): Result[void, string] =
|
||||
echo "size.....", segment.payload.len
|
||||
try:
|
||||
self.db.exec(
|
||||
sql"""
|
||||
@ -70,8 +69,6 @@ proc saveMessageSegment*(
|
||||
segment.entireMessageHash.toHex, sigPubKeyBlob, segment.index
|
||||
)
|
||||
let payloadBytes = nimcrypto.fromHex(storedPayload)
|
||||
# let lenBytes = payloadBytes.get()
|
||||
echo "Stored payload length: ", payloadBytes.len
|
||||
|
||||
ok()
|
||||
except DbError as e:
|
||||
@ -102,7 +99,6 @@ proc getMessageSegments*(self: SegmentationPersistence, hash: seq[byte], sigPubK
|
||||
paritySegmentsCount: uint32(parseInt(row[4])),
|
||||
payload: nimcrypto.fromHex(row[5])
|
||||
)
|
||||
echo "read size.....", segment.payload.len
|
||||
segments.add(segment)
|
||||
return ok(segments)
|
||||
except CatchableError as e:
|
||||
|
||||
@ -12,7 +12,6 @@ import db
|
||||
|
||||
import_proto3 "segment_message.proto"
|
||||
|
||||
# Placeholder types (unchanged)
|
||||
type
|
||||
Chunk* = object
|
||||
payload*: seq[byte]
|
||||
@ -63,16 +62,11 @@ proc segmentMessage*(s: SegmentationHander, newMessage: Chunk): Result[seq[Chunk
|
||||
info "segmenting message",
|
||||
"payloadSize" = newMessage.payload.len,
|
||||
"segmentSize" = s.segmentSize
|
||||
info "segment payload in", "saaa" = newMessage.payload.toHex
|
||||
let entireMessageHash = keccak256.digest(newMessage.payload)
|
||||
let entirePayloadSize = newMessage.payload.len
|
||||
|
||||
let segmentsCount = int(ceil(entirePayloadSize.float / s.segmentSize.float))
|
||||
let paritySegmentsCount = int(floor(segmentsCount.float * SegmentsParityRate))
|
||||
info "parity count", paritySegmentsCount
|
||||
info "segmentsCount", segmentsCount
|
||||
info "entirePayloadSize", entirePayloadSize
|
||||
info "segmentSize", "segmentSize" = s.segmentSize
|
||||
|
||||
var segmentPayloads = newSeq[seq[byte]](segmentsCount)
|
||||
var segmentMessages = newSeq[Chunk](segmentsCount)
|
||||
@ -91,8 +85,6 @@ proc segmentMessage*(s: SegmentationHander, newMessage: Chunk): Result[seq[Chunk
|
||||
payload: segmentPayload
|
||||
)
|
||||
|
||||
info "entireMessageHash", "entireMessageHash" = entireMessageHash.data.toHex
|
||||
|
||||
let marshaledSegment = Protobuf.encode(segmentWithMetadata)
|
||||
let segmentMessage = Chunk(payload: marshaledSegment)
|
||||
|
||||
@ -109,45 +101,22 @@ proc segmentMessage*(s: SegmentationHander, newMessage: Chunk): Result[seq[Chunk
|
||||
segmentPayloads[segmentsCount-1] = newSeq[byte](s.segmentSize)
|
||||
segmentPayloads[segmentsCount-1][0..<lastSegmentPayload.len] = lastSegmentPayload
|
||||
|
||||
for i, shard in segmentPayloads:
|
||||
info "segment payload before encoding", "index" = i, "len" = shard.len, "payload" = shard
|
||||
|
||||
# Allocate space for parity shards
|
||||
# for i in segmentsCount..<(segmentsCount + paritySegmentsCount):
|
||||
# segmentPayloads[i] = newSeq[byte](s.segmentSize)
|
||||
|
||||
# Use nim-leopard for Reed-Solomon encoding
|
||||
# var data = segmentPayloads[0..<segmentsCount]
|
||||
var parity = newSeq[seq[byte]](paritySegmentsCount)
|
||||
for i in 0..<paritySegmentsCount:
|
||||
newSeq(parity[i], s.segmentSize)
|
||||
|
||||
# var parity = segmentPayloads[segmentsCount..<(segmentsCount + paritySegmentsCount)]
|
||||
|
||||
info "parity segments",
|
||||
"paritySegmentsCount" = paritySegmentsCount,
|
||||
"parity" = parity
|
||||
info "initializing encoder",
|
||||
"segmentSize" = s.segmentSize,
|
||||
"segmentsCount" = segmentsCount,
|
||||
"paritySegmentsCount" = paritySegmentsCount
|
||||
var encoderRes = LeoEncoder.init(s.segmentSize, segmentsCount, paritySegmentsCount)
|
||||
if encoderRes.isErr:
|
||||
return err("failed to initialize encoder: " & $encoderRes.error)
|
||||
|
||||
var encoder = encoderRes.get
|
||||
let encodeResult = encoder.encode(segmentPayloads, parity)
|
||||
info "parity segments",
|
||||
"paritySegmentsCount" = paritySegmentsCount,
|
||||
"parity" = parity
|
||||
info "encoding result", "encodeResult" = encodeResult
|
||||
if encodeResult.isErr:
|
||||
return err("failed to encode segments with leopard: " & $encodeResult.error)
|
||||
|
||||
# Create parity messages
|
||||
for i in 0..<paritySegmentsCount:
|
||||
info "parity segment index", "index" = i
|
||||
info "parity segment payload", "payload" = parity[i]
|
||||
let segmentWithMetadata = SegmentMessageProto(
|
||||
entireMessageHash: entireMessageHash.data.toSeq,
|
||||
segmentsCount: 0,
|
||||
@ -194,7 +163,6 @@ proc handleSegmentationLayer*(handler: SegmentationHander, message: var Message)
|
||||
paritySegmentsCount: segmentMessageProto.paritySegmentsCount,
|
||||
payload: segmentMessageProto.payload
|
||||
)
|
||||
info "segment payload", "len" = segmentMessage.payload.len, "segment payload" = segmentMessage.payload.toHex
|
||||
let saveResult = handler.persistence.saveMessageSegment(segmentMessage, message.sigPubKey, getTime().toUnix)
|
||||
if saveResult.isErr:
|
||||
return err(saveResult.error)
|
||||
@ -209,14 +177,14 @@ proc handleSegmentationLayer*(handler: SegmentationHander, message: var Message)
|
||||
let firstSegmentMessage = segments.get()[0]
|
||||
let lastSegmentMessage = segments.get()[^1]
|
||||
|
||||
info "first segment",
|
||||
debug "first segment",
|
||||
"index" = firstSegmentMessage.index,
|
||||
"segmentsCount" = firstSegmentMessage.segmentsCount,
|
||||
"paritySegmentIndex" = firstSegmentMessage.paritySegmentIndex,
|
||||
"paritySegmentsCount" = firstSegmentMessage.paritySegmentsCount,
|
||||
"len" = firstSegmentMessage.payload.len
|
||||
|
||||
info "last segment",
|
||||
debug "last segment",
|
||||
"index" = lastSegmentMessage.index,
|
||||
"segmentsCount" = lastSegmentMessage.segmentsCount,
|
||||
"paritySegmentIndex" = lastSegmentMessage.paritySegmentIndex,
|
||||
@ -231,18 +199,13 @@ proc handleSegmentationLayer*(handler: SegmentationHander, message: var Message)
|
||||
let payloadSize = firstSegmentMessage.payload.len
|
||||
|
||||
let restoreUsingParityData = lastSegmentMessage.isParityMessage()
|
||||
info "restoreUsingParityData", restoreUsingParityData
|
||||
if not restoreUsingParityData:
|
||||
for i, segment in segments.get():
|
||||
payloads[i] = segment.payload
|
||||
info "segment payload=====", "index" = i, "len" = segment.payload.len, "payload" = segment.payload.toHex
|
||||
else:
|
||||
info "restoring using parity data...", "payloadSize" = payloadSize
|
||||
var lastNonParitySegmentPayload: seq[byte]
|
||||
for segment in segments.get():
|
||||
info "segment after read", "segment"= segment.entireMessageHash.toHex, "index" = segment.index, "paritySegmentIndex" = segment.paritySegmentIndex, "paritySegmentsCount" = segment.paritySegmentsCount, "len" = segment.payload.len
|
||||
if not segment.isParityMessage():
|
||||
info "segment index for non parity", "index" = segment.index
|
||||
if segment.index == firstSegmentMessage.segmentsCount - 1:
|
||||
payloads[segment.index] = newSeq[byte](payloadSize)
|
||||
payloads[segment.index][0..<segment.payload.len] = segment.payload
|
||||
@ -250,21 +213,13 @@ proc handleSegmentationLayer*(handler: SegmentationHander, message: var Message)
|
||||
else:
|
||||
payloads[segment.index] = segment.payload
|
||||
else:
|
||||
info "parity segment index.....", "index" = segment.paritySegmentIndex, "hhhh" = firstSegmentMessage.segmentsCount + segment.paritySegmentIndex
|
||||
info "parity segment index payload.....", "payload" = segment.payload
|
||||
parity[segment.paritySegmentIndex] = segment.payload
|
||||
|
||||
# Use nim-leopard for Reed-Solomon reconstruction
|
||||
info "leo decoder init",
|
||||
"payloadSize" = payloadSize,
|
||||
"segmentsCount" = firstSegmentMessage.segmentsCount,
|
||||
"paritySegmentsCount" = lastSegmentMessage.paritySegmentsCount
|
||||
let decoderRes = LeoDecoder.init(payloadSize, int(firstSegmentMessage.segmentsCount), int(lastSegmentMessage.paritySegmentsCount))
|
||||
if decoderRes.isErr:
|
||||
return err("failed to initialize LeoDecoder: " & $decoderRes.error)
|
||||
var decoder = decoderRes.get()
|
||||
info "decode data", "data" = payloads
|
||||
info "decode data parity", "parityData" = parity
|
||||
var recovered = newSeq[seq[byte]](int(firstSegmentMessage.segmentsCount)) # Allocate for recovered shards
|
||||
for i in 0..<int(firstSegmentMessage.segmentsCount):
|
||||
recovered[i] = newSeq[byte](payloadSize)
|
||||
@ -272,8 +227,6 @@ proc handleSegmentationLayer*(handler: SegmentationHander, message: var Message)
|
||||
if reconstructResult.isErr:
|
||||
return err("failed to reconstruct payloads with leopard: " & $reconstructResult.error)
|
||||
|
||||
info "recovered segments", "recoveredCount" = recovered.len, "recovered" = recovered
|
||||
|
||||
for i in 0..<firstSegmentMessage.segmentsCount:
|
||||
if payloads[i].len == 0:
|
||||
payloads[i] = recovered[i]
|
||||
@ -287,8 +240,6 @@ proc handleSegmentationLayer*(handler: SegmentationHander, message: var Message)
|
||||
entirePayload.add(payloads[i])
|
||||
|
||||
# Sanity check
|
||||
info "entirePayloadSize", "entirePayloadSize" = entirePayload.len
|
||||
info "entire payload", "entirePayloadSize" = entirePayload.toHex
|
||||
let entirePayloadHash = keccak256.digest(entirePayload)
|
||||
if entirePayloadHash.data != segmentMessage.entireMessageHash:
|
||||
return err(ErrMessageSegmentsHashMismatch)
|
||||
|
||||
@ -118,13 +118,10 @@ suite "message Segmentation":
|
||||
let chunk = Chunk(payload: testPayload)
|
||||
sender.segmentSize = int(ceil(testPayload.len.float / tc.segmentsCount.float))
|
||||
let segmentedMessagesRes = sender.segmentMessage(chunk)
|
||||
echo "segmentedMessagesRes: ", segmentedMessagesRes
|
||||
require(segmentedMessagesRes.isOk)
|
||||
let segmentedMessages = segmentedMessagesRes.get()
|
||||
check(segmentedMessages.len == tc.segmentsCount + tc.expectedParitySegmentsCount)
|
||||
|
||||
echo "segmet result: ", tc.segmentsCount, " ", tc.expectedParitySegmentsCount, " ", segmentedMessages.len
|
||||
|
||||
var message = Message(
|
||||
sigPubKey: keccak256.digest("testkey").data.toSeq,
|
||||
hash: keccak256.digest(testPayload).data.toSeq
|
||||
@ -134,17 +131,9 @@ suite "message Segmentation":
|
||||
var handledSegments: seq[int] = @[]
|
||||
|
||||
for i, segmentIndex in tc.retrievedSegments:
|
||||
echo "i=", i, " segmentIndex=", segmentIndex
|
||||
message.payload = segmentedMessages[segmentIndex].payload
|
||||
let err = sender.handleSegmentationLayer(message)
|
||||
|
||||
echo "handle err: ", err
|
||||
|
||||
handledSegments.add(segmentIndex)
|
||||
echo "handledSegments: ", handledSegments
|
||||
|
||||
echo "handledSegments result: ", handledSegments.len, " ", tc.segmentsCount
|
||||
|
||||
if handledSegments.len < tc.segmentsCount:
|
||||
check(err.isErr and err.error == ErrMessageSegmentsIncomplete)
|
||||
elif handledSegments.len == tc.segmentsCount:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user