From 8a40a9832251840d96e121abdc9e6bd760bc5afd Mon Sep 17 00:00:00 2001 From: Dario Lipicar Date: Sun, 26 Jul 2026 13:58:14 -0300 Subject: [PATCH] =?UTF-8?q?feat(cdylib):=20support=20[bstr]=20=E2=80=94=20?= =?UTF-8?q?arrays=20of=20byte=20strings=20(#111)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A universal module could take or return a single blob (`bstr`) but not a list of them: `std::vector>` parsed to `[bstr]`, which the cdylib gate rejected by name. Authors had to flatten to hex or base64 strings by hand. logos-execution-zone-module hit this on send_generic_private_transaction(..., program_dependencies), whose dependency ELFs are exactly a list of blobs. The gate excluded `[bstr]` because the array path decodes with a blanket `expr.get()`, and `[bstr]` elements arrive as the canonical tagged {"_bytes": base64url} OBJECT — nlohmann refuses that, and a number-array element would silently bypass the base64 decode. So admitting the type needed a per-element codec, not just a whitelist entry. Adds one, on top of the scalar codecs already emitted: - lidlBytesListFromJson: element-wise lidlBytesFromJson, so each element may independently be tagged, a plain string or a number array; a non-array arg yields an empty list instead of throwing, matching the scalar decoder. - lidlBytesListToJson: element-wise lidlBytesToJson, so a returned or emitted list carries the tagged form per element instead of nested number arrays that no consumer decodes as bytes. Wired into all three places the type can appear — method params, method returns, event payloads — and both helpers are gated (usesBytesArray / hasBytesArrayEventParam) so a module that never carries a byte-string array gains no unused static function, matching how the scalar encoder is gated. Nothing outside this generator needed changing: lidlTypeToStd already spelled `[bstr]` as std::vector>, the wire form is protocol's existing tagged-bytes encoding, and consumers see `[bstr]` as QVariantList exactly like `[int]` — with nested QByteArray preserved through qvariantToNlohmann since logos-protocol#23. Tests: 171/171. New coverage for the param decode, the return encode, the event encode, and the unused-helper gating; the test that enshrined the rejection is now an eligibility + tagging assertion. Verified end to end, not just as generated text — a module with `[bstr]` as param, return and event payload, driven through logoscore over the real transport: param : json:[{"_bytes":"AH-A_w"},{"_bytes":""},{"_bytes":"3q2-7w"}] -> "3|4:0:255:510|0:-1:-1:0|4:222:239:824" (byte-exact; the 0x80 and 0xff bytes survive, and the empty element stays an element) return : [{"_bytes":"AH-A_w"},{"_bytes":""},{"_bytes":"3q2-7w"}] event : {"arg0":[{"_bytes":"AH-A_w"},{"_bytes":""},{"_bytes":"3q2-7w"}]} And lez_core's real header now generates, decoding program_elf with the scalar codec and program_dependencies with the list one. Co-authored-by: Claude Opus 5 --- .../experimental/impl_header_parser.cpp | 7 +- .../experimental/lidl_gen_cdylib.cpp | 96 ++++++++++++++++- tests/experimental/test_lidl_gen_cdylib.cpp | 102 ++++++++++++++++-- 3 files changed, 190 insertions(+), 15 deletions(-) diff --git a/cpp-generator/experimental/impl_header_parser.cpp b/cpp-generator/experimental/impl_header_parser.cpp index d77bb2c..9f961e9 100644 --- a/cpp-generator/experimental/impl_header_parser.cpp +++ b/cpp-generator/experimental/impl_header_parser.cpp @@ -67,9 +67,10 @@ static TypeExpr cppTypeToLidl(const QString& raw) } // std::vector> — an array of byte strings. Spelled // out so it lands on `[bstr]` rather than the opaque `any` fallback - // below: `any` would make the cdylib gate admit it and then emit a bare - // QVariant into the Qt-free TU. As `[bstr]` the gate rejects it with a - // message naming the offending parameter. + // below, which would emit a bare QVariant into the Qt-free TU. As + // `[bstr]` it goes through the cdylib list codec + // (lidlBytesListFromJson / lidlBytesListToJson), so each element keeps + // the canonical tagged {"_bytes": base64url} form on the wire. if (inner == "std::vector") { TypeExpr elem = { TypeExpr::Primitive, "bstr", {} }; return { TypeExpr::Array, "", { elem } }; diff --git a/cpp-generator/experimental/lidl_gen_cdylib.cpp b/cpp-generator/experimental/lidl_gen_cdylib.cpp index 3cc2c59..fe85431 100644 --- a/cpp-generator/experimental/lidl_gen_cdylib.cpp +++ b/cpp-generator/experimental/lidl_gen_cdylib.cpp @@ -29,9 +29,14 @@ bool typeSupported(const TypeExpr& te, bool isReturn) } if (te.kind == TypeExpr::Array && te.elements.size() == 1) { const TypeExpr& e = te.elements[0]; + // `bstr` elements are the one array kind that cannot ride nlohmann's + // blanket get<>(): each element is the tagged {"_bytes": base64url} + // OBJECT, not a number array, so it needs the emitted per-element + // codec (lidlBytesListFromJson / lidlBytesListToJson) below. return e.kind == TypeExpr::Primitive && (e.name == "tstr" || e.name == "int" || e.name == "uint" - || e.name == "float64" || e.name == "bool" || e.name == "any"); + || e.name == "float64" || e.name == "bool" || e.name == "any" + || e.name == "bstr"); } // Maps ({k: v}, i.e. LogosMap) round-trip through nlohmann too. if (te.kind == TypeExpr::Map) @@ -39,6 +44,36 @@ bool typeSupported(const TypeExpr& te, bool isReturn) return false; } +// `[bstr]` — an array of byte strings (std::vector>). +// Every place scalar `bstr` needs the tagged-bytes codec, this needs the list +// form of it, so the check lives in one place. +bool isBytesArray(const TypeExpr& te) +{ + return te.kind == TypeExpr::Array && te.elements.size() == 1 + && te.elements[0].kind == TypeExpr::Primitive + && te.elements[0].name == "bstr"; +} + +// True when the module mentions `[bstr]` anywhere the emitted TU has to encode +// or decode it — method params, method returns, or event payloads. Gates the +// list codec so modules that never use it don't carry an unused static function +// (the same reason hasBytesEventParam() gates the scalar encoder). +bool usesBytesArray(const ModuleDecl& module) +{ + for (const MethodDecl& md : module.methods) { + if (isBytesArray(md.returnType)) + return true; + for (const ParamDecl& pd : md.params) + if (isBytesArray(pd.type)) + return true; + } + for (const EventDecl& ed : module.events) + for (const ParamDecl& pd : ed.params) + if (isBytesArray(pd.type)) + return true; + return false; +} + // Qt-free spelling of a LIDL type (defined below). Forward-declared so the // method-param decoder can spell composite `any` containers as their nlohmann // aliases instead of Qt containers in this Qt-free TU. @@ -56,6 +91,12 @@ QString jsonArgToStd(const TypeExpr& te, const QString& expr) if (te.name == "bool") return expr + ".get()"; } if (te.kind == TypeExpr::Array && te.elements.size() == 1) { + // `[bstr]` cannot go through get>>(): + // its elements arrive as tagged {"_bytes": …} objects, which nlohmann + // would refuse (and a raw number-array element would silently bypass + // the base64 decode). Route it through the per-element codec instead. + if (isBytesArray(te)) + return "lidlBytesListFromJson(" + expr + ")"; // Qt-free spelling: `[any]` must decode as LogosList, NOT QVariantList // (lidlTypeToStd's Qt fallback), which is undeclared in this TU. Typed // scalar arrays ([tstr]/[int]/…) are unaffected — Cdylib defers to the @@ -82,6 +123,10 @@ QString stdReturnToJson(const MethodDecl& md, const QString& var) if (te.name == "bstr") return "lidlBytesToJson(" + var + ")"; return "nlohmann::json(" + var + ")"; } + // `[bstr]`: nlohmann::json(std::vector>) would emit + // nested number arrays, which no consumer decodes as bytes. Tag each element. + if (isBytesArray(te)) + return "lidlBytesListToJson(" + var + ")"; return "nlohmann::json(" + var + ")"; } @@ -112,7 +157,19 @@ bool hasBytesEventParam(const ModuleDecl& module) { for (const EventDecl& ed : module.events) for (const ParamDecl& pd : ed.params) - if (pd.type.kind == TypeExpr::Primitive && pd.type.name == "bstr") + if ((pd.type.kind == TypeExpr::Primitive && pd.type.name == "bstr") + || isBytesArray(pd.type)) + return true; + return false; +} + +// True when an EVENT carries `[bstr]`, so the sidecar needs the list encoder on +// top of the scalar one. Method params/returns are handled in the impl TU. +bool hasBytesArrayEventParam(const ModuleDecl& module) +{ + for (const EventDecl& ed : module.events) + for (const ParamDecl& pd : ed.params) + if (isBytesArray(pd.type)) return true; return false; } @@ -130,7 +187,10 @@ bool hasJsonEventParam(const ModuleDecl& module) return false; } -void emitBytesEncodeHelpers(QTextStream& s) +// `withList` additionally emits lidlBytesListToJson for `[bstr]`. Off by +// default so a module that never carries a byte-string array does not gain an +// unused static function. +void emitBytesEncodeHelpers(QTextStream& s, bool withList = false) { s << "// Canonical tagged bytes form {\"_bytes\": base64url} (see logos_protocol.h)\n"; s << "std::string lidlB64UrlEncode(const std::vector& bytes)\n{\n"; @@ -151,6 +211,14 @@ void emitBytesEncodeHelpers(QTextStream& s) s << "nlohmann::json lidlBytesToJson(const std::vector& bytes)\n{\n"; s << " return nlohmann::json{{\"_bytes\", lidlB64UrlEncode(bytes)}};\n}\n\n"; + + if (withList) { + s << "nlohmann::json lidlBytesListToJson(const std::vector>& list)\n{\n"; + s << " nlohmann::json out = nlohmann::json::array();\n"; + s << " for (const std::vector& bytes : list)\n"; + s << " out.push_back(lidlBytesToJson(bytes));\n"; + s << " return out;\n}\n\n"; + } } void emitInterfaceJson(QTextStream& s, const ModuleDecl& module) @@ -305,7 +373,8 @@ QString lidlMakeModuleImplExports(const ModuleDecl& module, s << " if (out) std::memcpy(out, str.data(), str.size() + 1);\n"; s << " return out;\n}\n\n"; - emitBytesEncodeHelpers(s); + const bool bytesList = usesBytesArray(module); + emitBytesEncodeHelpers(s, bytesList); s << "int lidlB64Idx(char ch)\n{\n"; s << " if (ch >= 'A' && ch <= 'Z') return ch - 'A';\n"; @@ -362,6 +431,21 @@ QString lidlMakeModuleImplExports(const ModuleDecl& module, s << " out.push_back((n >> 8) & 0xff);\n"; s << " }\n }\n return out;\n}\n\n"; + if (bytesList) { + s << "std::vector> lidlBytesListFromJson(const nlohmann::json& j)\n{\n"; + s << " std::vector> out;\n"; + s << " // Each element runs through the lenient scalar decode above, so a\n"; + s << " // caller may send tagged {\"_bytes\": base64url} objects, plain\n"; + s << " // strings or number arrays — element by element. A non-array arg\n"; + s << " // yields an empty list rather than throwing, matching the scalar\n"; + s << " // decoder's behaviour on an unexpected shape.\n"; + s << " if (!j.is_array()) return out;\n"; + s << " out.reserve(j.size());\n"; + s << " for (const auto& e : j)\n"; + s << " out.push_back(lidlBytesFromJson(e));\n"; + s << " return out;\n}\n\n"; + } + s << "nlohmann::json lidlResultToJson(const StdLogosResult& r)\n{\n"; s << " nlohmann::json obj;\n"; s << " obj[\"success\"] = r.success;\n"; @@ -548,7 +632,7 @@ QString lidlMakeEventsSourceCdylib(const ModuleDecl& module, // encoder; emitting it everywhere would leave it unused (and warned about). if (hasBytesEventParam(module)) { s << "namespace {\n\n"; - emitBytesEncodeHelpers(s); + emitBytesEncodeHelpers(s, hasBytesArrayEventParam(module)); s << "} // namespace\n\n"; } @@ -570,6 +654,8 @@ QString lidlMakeEventsSourceCdylib(const ModuleDecl& module, for (const ParamDecl& pd : ed.params) { if (pd.type.kind == TypeExpr::Primitive && pd.type.name == "bstr") s << " args.push_back(lidlBytesToJson(" << pd.name << "));\n"; + else if (isBytesArray(pd.type)) + s << " args.push_back(lidlBytesListToJson(" << pd.name << "));\n"; else s << " args.push_back(" << pd.name << ");\n"; } diff --git a/tests/experimental/test_lidl_gen_cdylib.cpp b/tests/experimental/test_lidl_gen_cdylib.cpp index 20ab89c..07fa290 100644 --- a/tests/experimental/test_lidl_gen_cdylib.cpp +++ b/tests/experimental/test_lidl_gen_cdylib.cpp @@ -44,6 +44,29 @@ QString eventsSourceFor(const ModuleDecl& m) return lidlMakeEventsSourceCdylib(m, "DeliveryModuleImpl", "delivery_module_plugin.h"); } +MethodDecl method(const char* name, const TypeExpr& returnType, + const std::vector& params) +{ + MethodDecl md; + md.name = name; + md.returnType = returnType; + md.params = params; + return md; +} + +ModuleDecl moduleWithMethod(const MethodDecl& md) +{ + ModuleDecl m; + m.name = "delivery_module"; + m.methods.push_back(md); + return m; +} + +QString implSourceFor(const ModuleDecl& m) +{ + return lidlMakeModuleImplExports(m, "DeliveryModuleImpl", "delivery_module_plugin.h"); +} + } // namespace // logos-cpp-sdk#99: `payload` was replaced by an empty tagged value, so a module @@ -118,19 +141,84 @@ TEST(LidlGenCdylib, JsonEventPayloadIsQtFree) EXPECT_FALSE(source.contains("QVariant")); } -// An array of byte strings is outside the cdylib-supported subset. It must be -// rejected by name at generation time rather than admitted and then emitted as -// a QVariant that fails to compile. -TEST(LidlGenCdylib, ArrayOfBytesEventParamIsRejected) +// `[bstr]` is in the supported subset: each element carries the canonical tagged +// form, so a module can take or return a list of blobs (e.g. a program plus its +// dependency ELFs) instead of hand-encoding them as hex strings. +TEST(LidlGenCdylib, ArrayOfBytesEventParamIsEligibleAndTagsEachElement) { const ModuleDecl m = moduleWithEvent("batchReceived", { param("payloads", TypeExpr{TypeExpr::Array, "", {prim("bstr")}}), }); QString error; - EXPECT_FALSE(lidlCdylibSupported(m, &error)); - EXPECT_TRUE(error.contains("batchReceived")); - EXPECT_TRUE(error.contains("payloads")); + EXPECT_TRUE(lidlCdylibSupported(m, &error)) << error.toStdString(); + + const QString source = eventsSourceFor(m); + + // Each element is tagged, not emitted as a nested number array — which is + // what nlohmann::json(std::vector>) would have produced + // and no consumer decodes as bytes. + EXPECT_TRUE(source.contains("args.push_back(lidlBytesListToJson(payloads));")); + EXPECT_TRUE(source.contains("nlohmann::json lidlBytesListToJson")); + EXPECT_TRUE(source.contains("out.push_back(lidlBytesToJson(bytes));")); + + // Qt-free, and taken by const-ref like the other composite payloads. + EXPECT_TRUE(source.contains("const std::vector>& payloads")); + EXPECT_FALSE(source.contains("QVariant")); +} + +// The method path: a `[bstr]` parameter must be DECODED per element, never via +// nlohmann's blanket get<>(). get>>() throws on +// the tagged {"_bytes": …} object form, and would silently skip the base64 +// decode for a number-array element. +TEST(LidlGenCdylib, ArrayOfBytesMethodParamDecodesPerElement) +{ + const ModuleDecl m = moduleWithMethod(method("send", prim("tstr"), { + param("program_elf", prim("bstr")), + param("program_dependencies", TypeExpr{TypeExpr::Array, "", {prim("bstr")}}), + })); + + QString error; + ASSERT_TRUE(lidlCdylibSupported(m, &error)) << error.toStdString(); + + const QString source = implSourceFor(m); + + EXPECT_TRUE(source.contains("lidlBytesListFromJson(")); + EXPECT_TRUE(source.contains("std::vector> lidlBytesListFromJson")); + EXPECT_TRUE(source.contains("out.push_back(lidlBytesFromJson(e));")); + // The scalar param still uses the scalar decoder. + EXPECT_TRUE(source.contains("lidlBytesFromJson(")); + // The blanket container decode must not be used for this type. + EXPECT_FALSE(source.contains(".get>>()")); +} + +// The return path: nlohmann::json(std::vector>) would emit +// nested number arrays, which no consumer decodes as bytes. +TEST(LidlGenCdylib, ArrayOfBytesReturnTagsEachElement) +{ + const ModuleDecl m = moduleWithMethod( + method("dependencies", TypeExpr{TypeExpr::Array, "", {prim("bstr")}}, {})); + + QString error; + ASSERT_TRUE(lidlCdylibSupported(m, &error)) << error.toStdString(); + + const QString source = implSourceFor(m); + EXPECT_TRUE(source.contains("lidlBytesListToJson(")); + EXPECT_TRUE(source.contains("nlohmann::json lidlBytesListToJson")); +} + +// The list encoder is gated the same way the scalar one is: a module whose +// events carry only a single blob must not gain an unused static function. +TEST(LidlGenCdylib, BytesListEncoderOmittedWhenNoEventCarriesAnArray) +{ + const ModuleDecl m = moduleWithEvent("messageReceived", { + param("payload", prim("bstr")), + }); + + const QString source = eventsSourceFor(m); + + EXPECT_TRUE(source.contains("nlohmann::json lidlBytesToJson")); + EXPECT_FALSE(source.contains("lidlBytesListToJson")); } // The supported scalar / bytes payloads stay eligible.