mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-01-03 05:53:07 +00:00
127 lines
4.0 KiB
Nim
127 lines
4.0 KiB
Nim
import std/streams
|
|
|
|
import pkg/chronos
|
|
import pkg/libp2p/cid
|
|
import pkg/questionable/results
|
|
|
|
import ../node
|
|
import ../blocktype
|
|
import ../manifest
|
|
import ../stores/blockstore
|
|
|
|
import ./tarballs
|
|
import ./stdstreamwrapper
|
|
import ./directorymanifest
|
|
import ./encoding
|
|
import ./decoding
|
|
|
|
proc fetchDirectoryManifest*(
|
|
self: CodexNodeRef, cid: Cid
|
|
): Future[?!DirectoryManifest] {.async: (raises: [CancelledError]).} =
|
|
## Fetch and decode a manifest block
|
|
##
|
|
|
|
# we only need try/catch here because PR for checked exceptions is
|
|
# not yet merged
|
|
try:
|
|
if err =? cid.isManifest.errorOption:
|
|
return failure "CID has invalid content type for manifest {$cid}"
|
|
|
|
trace "Retrieving directory manifest for cid", cid
|
|
|
|
without blk =? await self.blockStore.getBlock(BlockAddress.init(cid)), err:
|
|
trace "Error retrieving directory manifest block", cid, err = err.msg
|
|
return failure err
|
|
|
|
trace "Decoding directory manifest for cid", cid
|
|
|
|
without manifest =? DirectoryManifest.decode(blk), err:
|
|
trace "Unable to decode as directory manifest", err = err.msg
|
|
return failure("Unable to decode as directory manifest")
|
|
|
|
trace "Decoded directory manifest", cid
|
|
|
|
return manifest.success
|
|
except CancelledError as e:
|
|
raise e
|
|
except CatchableError as e:
|
|
trace "Error fetching directory manifest", cid, err = e.msg
|
|
return failure(e.msg)
|
|
|
|
proc storeDirectoryManifest*(
|
|
self: CodexNodeRef, manifest: DirectoryManifest
|
|
): Future[?!Block] {.async.} =
|
|
let encodedManifest = manifest.encode()
|
|
|
|
without blk =? Block.new(data = encodedManifest, codec = ManifestCodec), error:
|
|
trace "Unable to create block from manifest"
|
|
return failure(error)
|
|
|
|
if err =? (await self.blockStore.putBlock(blk)).errorOption:
|
|
trace "Unable to store manifest block", cid = blk.cid, err = err.msg
|
|
return failure(err)
|
|
|
|
success blk
|
|
|
|
proc storeTarball*(
|
|
self: CodexNodeRef, stream: AsyncStreamReader
|
|
): Future[?!string] {.async.} =
|
|
info "Storing tarball data"
|
|
|
|
# Just as a proof of concept, we process tar bar in memory
|
|
# Later to see how to do actual streaming to either store
|
|
# tarball locally in some tmp folder, or to process the
|
|
# tarball incrementally
|
|
let tarballBytes = await stream.read()
|
|
let stream = newStringStream(string.fromBytes(tarballBytes))
|
|
|
|
proc onProcessedTarFile(
|
|
stream: Stream, fileName: string
|
|
): Future[?!Cid] {.gcsafe, async: (raises: [CancelledError]).} =
|
|
try:
|
|
echo "onProcessedTarFile:name: ", fileName
|
|
let stream = newStdStreamWrapper(stream)
|
|
await self.store(stream, filename = some fileName, pad = false)
|
|
except CancelledError as e:
|
|
raise e
|
|
except CatchableError as e:
|
|
error "Error processing tar file", fileName, exc = e.msg
|
|
return failure(e.msg)
|
|
|
|
proc onProcessedTarDir(
|
|
name: string, cids: seq[Cid]
|
|
): Future[?!Cid] {.gcsafe, async: (raises: [CancelledError]).} =
|
|
try:
|
|
echo "onProcessedTarDir:name: ", name
|
|
echo "onProcessedTarDir:cids: ", cids
|
|
let directoryManifest = newDirectoryManifest(name = name, cids = cids)
|
|
without manifestBlk =? await self.storeDirectoryManifest(directoryManifest), err:
|
|
error "Unable to store manifest"
|
|
return failure(err)
|
|
manifestBlk.cid.success
|
|
except CancelledError as e:
|
|
raise e
|
|
except CatchableError as e:
|
|
error "Error processing tar dir", name, exc = e.msg
|
|
return failure(e.msg)
|
|
|
|
let tarball = Tarball()
|
|
if err =? (await tarball.open(stream, onProcessedTarFile)).errorOption:
|
|
error "Unable to open tarball", err = err.msg
|
|
return failure(err)
|
|
echo "tarball = ", $tarball
|
|
without root =? tarball.findRootDir(), err:
|
|
return failure(err.msg)
|
|
echo "root = ", root
|
|
let dirs = processDirEntries(tarball)
|
|
echo "dirs = ", dirs
|
|
without tree =? (await buildTree(root = root, dirs = dirs, onProcessedTarDir)), err:
|
|
error "Unable to build tree", err = err.msg
|
|
return failure(err)
|
|
echo ""
|
|
echo "preorderTraversal:"
|
|
let json = newJArray()
|
|
preorderTraversal(tree, json)
|
|
echo "json = ", json
|
|
success($json)
|