mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-01-05 15:03:07 +00:00
283 lines
8.8 KiB
Nim
283 lines
8.8 KiB
Nim
{.push raises: [].}
|
|
|
|
import std/os
|
|
import std/times
|
|
import std/strutils
|
|
import std/strformat
|
|
import std/sequtils
|
|
import std/streams
|
|
import std/tables
|
|
|
|
import std/random
|
|
|
|
import pkg/chronos
|
|
import pkg/questionable/results
|
|
import pkg/libp2p/[cid, multicodec, multihash]
|
|
import pkg/serde/json
|
|
|
|
import ../blocktype
|
|
import ../manifest
|
|
import ./directorymanifest
|
|
|
|
proc example2*(_: type Block, size: int = 4096): ?!Block =
|
|
let length = rand(size)
|
|
let bytes = newSeqWith(length, rand(uint8))
|
|
Block.new(bytes)
|
|
|
|
proc example2*(_: type Cid): ?!Cid =
|
|
Block.example2 .? cid
|
|
|
|
const
|
|
TUREAD* = 0o00400'u32 # read by owner */
|
|
TUWRITE* = 0o00200'u32 # write by owner */
|
|
TUEXEC* = 0o00100'u32 # execute/search by owner */
|
|
TGREAD* = 0o00040'u32 # read by group */
|
|
TGWRITE* = 0o00020'u32 # write by group */
|
|
TGEXEC* = 0o00010'u32 # execute/search by group */
|
|
TOREAD* = 0o00004'u32 # read by other */
|
|
TOWRITE* = 0o00002'u32 # write by other */
|
|
TOEXEC* = 0o00001'u32 # execute/search by other */
|
|
|
|
type
|
|
EntryKind* = enum
|
|
ekNormalFile = '0'
|
|
ekDirectory = '5'
|
|
|
|
TarballEntry* = object
|
|
kind*: EntryKind
|
|
name*: string
|
|
cid*: Cid
|
|
contentLength*: int
|
|
lastModified*: times.Time
|
|
permissions*: set[FilePermission]
|
|
|
|
Tarball* = ref object
|
|
contents*: OrderedTable[string, TarballEntry]
|
|
|
|
TarballError* = object of ValueError
|
|
|
|
TarballTree* = ref object
|
|
name*: string
|
|
cid*: Cid
|
|
children*: seq[TarballTree]
|
|
|
|
# ToDo: make sure we also record files permissions, modification time, etc...
|
|
# For now, only fileName so that we do not have to change the Codex manifest
|
|
# right away
|
|
OnProcessedTarFile* = proc(stream: Stream, fileName: string): Future[?!Cid] {.
|
|
gcsafe, async: (raises: [CancelledError])
|
|
.}
|
|
|
|
OnProcessedTarDir* = proc(name: string, cids: seq[Cid]): Future[?!Cid] {.
|
|
gcsafe, async: (raises: [CancelledError])
|
|
.}
|
|
|
|
proc `$`*(tarball: Tarball): string =
|
|
result = "Tarball with " & $tarball.contents.len & " entries"
|
|
for name, entry in tarball.contents.pairs:
|
|
var lastModified: string = "(unknown)"
|
|
try:
|
|
let lastModified = $entry.lastModified
|
|
except TimeFormatParseError:
|
|
discard
|
|
result.add(
|
|
"\n " &
|
|
fmt"{name}: name = {entry.name}, {entry.kind} ({entry.contentLength} bytes) @ {lastModified} [{entry.cid}]"
|
|
)
|
|
|
|
proc `$`*(tarballEntry: TarballEntry): string =
|
|
## Returns a string representation of the tarball entry.
|
|
result = fmt"({tarballEntry.kind}, {tarballEntry.name})"
|
|
|
|
proc parseFilePermissions*(permissions: uint32): set[FilePermission] =
|
|
if defined(windows) or permissions == 0:
|
|
# Ignore file permissions on Windows. If they are absent (.zip made on
|
|
# Windows for example), set default permissions.
|
|
result.incl fpUserRead
|
|
result.incl fpUserWrite
|
|
result.incl fpGroupRead
|
|
result.incl fpOthersRead
|
|
else:
|
|
if (permissions and TUREAD) != 0:
|
|
result.incl(fpUserRead)
|
|
if (permissions and TUWRITE) != 0:
|
|
result.incl(fpUserWrite)
|
|
if (permissions and TUEXEC) != 0:
|
|
result.incl(fpUserExec)
|
|
if (permissions and TGREAD) != 0:
|
|
result.incl(fpGroupRead)
|
|
if (permissions and TGWRITE) != 0:
|
|
result.incl(fpGroupWrite)
|
|
if (permissions and TGEXEC) != 0:
|
|
result.incl(fpGroupExec)
|
|
if (permissions and TOREAD) != 0:
|
|
result.incl(fpOthersRead)
|
|
if (permissions and TOWRITE) != 0:
|
|
result.incl(fpOthersWrite)
|
|
if (permissions and TOEXEC) != 0:
|
|
result.incl(fpOthersExec)
|
|
|
|
proc toUnixPath(path: string): string =
|
|
path.replace('\\', '/')
|
|
|
|
proc clear*(tarball: Tarball) =
|
|
tarball.contents.clear()
|
|
|
|
proc openStreamImpl(
|
|
tarball: Tarball, stream: Stream, onProcessedTarFile: OnProcessedTarFile = nil
|
|
): Future[?!void] {.async: (raises: []).} =
|
|
tarball.clear()
|
|
|
|
proc trim(s: string): string =
|
|
for i in 0 ..< s.len:
|
|
if s[i] == '\0':
|
|
return s[0 ..< i]
|
|
s
|
|
|
|
try:
|
|
var data = stream.readAll() # TODO: actually treat as a stream
|
|
|
|
var pos: int
|
|
while pos < data.len:
|
|
if pos + 512 > data.len:
|
|
return failure("Attempted to read past end of file, corrupted tarball?")
|
|
|
|
let
|
|
header = data[pos ..< pos + 512]
|
|
fileName = header[0 ..< 100].trim()
|
|
|
|
pos += 512
|
|
|
|
if fileName.len == 0:
|
|
continue
|
|
|
|
let
|
|
fileSize =
|
|
try:
|
|
parseOctInt(header[124 .. 134])
|
|
except ValueError:
|
|
raise newException(TarballError, "Unexpected error while opening tarball")
|
|
lastModified =
|
|
try:
|
|
parseOctInt(header[136 .. 146])
|
|
except ValueError:
|
|
raise newException(TarballError, "Unexpected error while opening tarball")
|
|
typeFlag = header[156]
|
|
fileMode =
|
|
try:
|
|
parseOctInt(header[100 ..< 106])
|
|
except ValueError:
|
|
raise newException(
|
|
TarballError, "Unexpected error while opening tarball (mode)"
|
|
)
|
|
fileNamePrefix =
|
|
if header[257 ..< 263] == "ustar\0":
|
|
header[345 ..< 500].trim()
|
|
else:
|
|
""
|
|
|
|
if pos + fileSize > data.len:
|
|
return failure("Attempted to read past end of file, corrupted tarball?")
|
|
|
|
let normalizedFileName = normalizePathEnd(fileName)
|
|
|
|
if typeFlag == '0' or typeFlag == '\0':
|
|
if not onProcessedTarFile.isNil:
|
|
let stream = newStringStream(data[pos ..< pos + fileSize])
|
|
without cid =?
|
|
await onProcessedTarFile(stream, normalizedFileName.lastPathPart), err:
|
|
return failure(err.msg)
|
|
tarball.contents[(fileNamePrefix / fileName).toUnixPath()] = TarballEntry(
|
|
kind: ekNormalFile,
|
|
name: normalizedFileName,
|
|
contentLength: fileSize,
|
|
cid: cid,
|
|
lastModified: initTime(lastModified, 0),
|
|
permissions: parseFilePermissions(cast[uint32](fileMode)),
|
|
)
|
|
elif typeFlag == '5':
|
|
tarball.contents[normalizePathEnd((fileNamePrefix / fileName).toUnixPath())] = TarballEntry(
|
|
kind: ekDirectory,
|
|
name: normalizedFileName,
|
|
lastModified: initTime(lastModified, 0),
|
|
permissions: parseFilePermissions(cast[uint32](fileMode)),
|
|
)
|
|
|
|
# Move pos by fileSize, where fileSize is 512 byte aligned
|
|
pos += (fileSize + 511) and not 511
|
|
success()
|
|
except CatchableError as e:
|
|
return failure(e.msg)
|
|
|
|
proc open*(
|
|
tarball: Tarball, bytes: string, onProcessedTarFile: OnProcessedTarFile = nil
|
|
): Future[?!void] {.async: (raw: true, raises: []).} =
|
|
let stream = newStringStream(bytes)
|
|
tarball.openStreamImpl(stream, onProcessedTarFile)
|
|
|
|
proc open*(
|
|
tarball: Tarball, stream: Stream, onProcessedTarFile: OnProcessedTarFile = nil
|
|
): Future[?!void] {.async: (raw: true, raises: []).} =
|
|
tarball.openStreamImpl(stream, onProcessedTarFile)
|
|
|
|
proc processDirEntries*(tarball: Tarball): Table[string, seq[TarballEntry]] =
|
|
result = initTable[string, seq[TarballEntry]]()
|
|
for name, entry in tarball.contents.pairs:
|
|
let path = normalizePathEnd(name)
|
|
if not isRootDir(path):
|
|
let (head, _) = splitPath(path)
|
|
result.withValue(head, value):
|
|
value[].add(entry)
|
|
do:
|
|
result[head] = @[entry]
|
|
|
|
proc findRootDir*(tarball: Tarball): ?!string =
|
|
var rootDir = ""
|
|
for entry in tarball.contents.values:
|
|
if entry.kind == ekDirectory:
|
|
if isRootDir(entry.name):
|
|
return success(entry.name)
|
|
failure("No root directory found in tarball")
|
|
|
|
proc buildTree*(
|
|
root: string,
|
|
dirs: Table[string, seq[TarballEntry]],
|
|
onProcessedTarDir: OnProcessedTarDir = nil,
|
|
): Future[?!TarballTree] {.async: (raises: [CancelledError]).} =
|
|
let tree = TarballTree(name: root.lastPathPart, children: @[])
|
|
let entries = dirs.getOrDefault(root)
|
|
for entry in entries:
|
|
if entry.kind == ekDirectory:
|
|
without subTree =?
|
|
await buildTree(root = entry.name, dirs = dirs, onProcessedTarDir), err:
|
|
return failure(err.msg)
|
|
# compute Cid for the subtree
|
|
# let cids = subTree.children.mapIt(it.cid)
|
|
# if not onProcessedTarDir.isNil:
|
|
# without cid =? await onProcessedTarDir(subTree.name, cids), err:
|
|
# return failure(err.msg)
|
|
# subTree.cid = cid
|
|
tree.children.add(subTree)
|
|
else:
|
|
let child =
|
|
TarballTree(name: entry.name.lastPathPart, children: @[], cid: entry.cid)
|
|
tree.children.add(child)
|
|
let cids = tree.children.mapIt(it.cid)
|
|
if not onProcessedTarDir.isNil:
|
|
without cid =? await onProcessedTarDir(tree.name, cids), err:
|
|
return failure(err.msg)
|
|
tree.cid = cid
|
|
success(tree)
|
|
|
|
proc preorderTraversal*(root: TarballTree, json: JsonNode) =
|
|
echo root.name
|
|
let jsonObj = newJObject()
|
|
jsonObj["name"] = newJString(root.name)
|
|
jsonObj["cid"] = newJString($root.cid)
|
|
json.add(jsonObj)
|
|
if root.children.len > 0:
|
|
let jsonArray = newJArray()
|
|
jsonObj["children"] = jsonArray
|
|
for child in root.children:
|
|
preorderTraversal(child, jsonArray)
|