Merge pull request #27 from zarvox/parse-packed

Add support for parsing structures in Cap'n Proto's packed encoding

src/node-capnp/capnp-test.js

@@ -26,12 +26,15 @@ var Promise = require("es6-promise").Promise;
 var spawn = require("child_process").spawn;
 
 var goldenBinary;
+var goldenPackedBinary;
 try {
   // Works in Ekam build.
   goldenBinary = fs.readFileSync("node-capnp/testdata/binary");
+  goldenPackedBinary = fs.readFileSync("node-capnp/testdata/packedbinary");
 } catch (ex) {
   // Works in npm build.
   goldenBinary = fs.readFileSync("src/node-capnp/testdata/binary");
+  goldenPackedBinary = fs.readFileSync("src/node-capnp/testdata/packedbinary");
 }
 
 var test = require("./test.capnp");
@@ -54,6 +57,19 @@ assert.equal("foo", test.TestConstants.textConst);
 assert.equal("baz", test.TestConstants.structConst.textField);
 assert.equal("xyzzy", test.TestConstants.textListConst[1]);
 
+// Test packed serialization/deserialization
+
+var parsedPacked = capnp.parsePacked(test.TestAllTypes, goldenPackedBinary);
+var roundTrippedPacked = capnp.serializePacked(test.TestAllTypes, parsedPacked);
+assert.equal(goldenPackedBinary.length, roundTrippedPacked.length, "Round trip changed size?");
+assert.equal(goldenPackedBinary.toString("base64"), roundTrippedPacked.toString("base64"), "Round trip lost data?");
+
+// TODO(someday): do a more thorough deep equality comparison of parsed and parsedPacked
+var keys = ["voidField", "boolField", "int8Field", "int16Field", "int32Field", "int64Field"];
+for (var key in keys) {
+  assert.equal(parsed[key], parsedPacked[key]);
+}
+
 // =======================================================================================
 
 /*

src/node-capnp/capnp.cc

@@ -34,12 +34,14 @@
 #include <uv.h>
 #include <kj/async.h>
 #include <kj/async-io.h>
+#include <kj/io.h>
 #include <kj/vector.h>
 #include <errno.h>
 #include <unistd.h>
 #include <capnp/rpc-twoparty.h>
 #include <capnp/rpc.capnp.h>
 #include <capnp/serialize.h>
+#include <capnp/serialize-packed.h>
 #include <unordered_map>
 #include <inttypes.h>
 #include <set>
@@ -1112,6 +1114,14 @@ struct StructReader {
       : message(data), root(message.getRoot<capnp::DynamicStruct>(schema)) {}
 };
 
+struct PackedStructReader {
+  capnp::PackedMessageReader message;
+  capnp::DynamicStruct::Reader root;
+
+  PackedStructReader(kj::BufferedInputStream& inputStream, capnp::StructSchema schema)
+      : message(inputStream), root(message.getRoot<capnp::DynamicStruct>(schema)) {}
+};
+
 struct ServerRequest {
   kj::Own<kj::PromiseFulfiller<void>> fulfiller;
   // Fulfill to complete the call.  You must null out the pointers below, as well as
@@ -1137,6 +1147,8 @@ kj::Maybe<capnp::DynamicStruct::Reader> unwrapReader(v8::Handle<v8::Value> handl
     });
   } else KJ_IF_MAYBE(reader, Wrapper::tryUnwrap<StructReader>(handle)) {
     return reader->root;
+  } else KJ_IF_MAYBE(reader, Wrapper::tryUnwrap<PackedStructReader>(handle)) {
+    return reader->root;
   } else KJ_IF_MAYBE(request, Wrapper::tryUnwrap<ServerRequest>(handle)) {
     return request->params;
   } else KJ_IF_MAYBE(builder, unwrapBuilder(handle)) {
@@ -1728,6 +1740,27 @@ v8::Handle<v8::Value> fromBytes(const v8::Arguments& args) {
   });
 }
 
+v8::Handle<v8::Value> fromBytesPacked(const v8::Arguments& args) {
+  // fromBytesPacked(buffer, schema) -> reader
+  KJV8_UNWRAP(CapnpContext, context, args.Data());
+
+  v8::Handle<v8::Value> bufferHandle = args[0];
+  KJV8_UNWRAP_BUFFER(buffer, bufferHandle);
+
+  KJV8_UNWRAP(capnp::Schema, schema, args[1]);
+  if (!schema.getProto().isStruct()) {
+    KJV8_TYPE_ERROR(schema, capnp::StructSchema);
+    return emptyHandle;
+  }
+
+  return liftKj([&]() -> v8::Handle<v8::Value> {
+    kj::ArrayInputStream inputStream(kj::arrayPtr(buffer.begin(), buffer.size()));
+    auto wrapper = context.wrapper.wrap(new PackedStructReader(inputStream, schema.asStruct()));
+    wrapper->SetHiddenValue(v8::String::NewSymbol("buffer"), bufferHandle);
+    return wrapper;
+  });
+}
+
 v8::Handle<v8::Value> toBytes(const v8::Arguments& args) {
   // toBytes(builder) -> buffer
 
@@ -1738,6 +1771,24 @@ v8::Handle<v8::Value> toBytes(const v8::Arguments& args) {
   });
 }
 
+v8::Handle<v8::Value> toBytesPacked(const v8::Arguments& args) {
+  // toBytes(builder) -> buffer
+
+  KJV8_UNWRAP(StructBuilder, builder, args[0]);
+
+  return liftKj([&]() -> v8::Handle<v8::Value> {
+    auto unpackedSize = computeSerializedSizeInWords(builder.message) * sizeof(capnp::word);
+    // Packed encoding can add 2 bytes of overhead per 2k segment reached.
+    auto packedOverhead = ((unpackedSize + 2047) / 2048) * 2;
+    auto packedSizeUpperBound = unpackedSize + packedOverhead;
+    auto oversizedBuf = kj::heapArray<byte>(packedSizeUpperBound);
+    kj::ArrayOutputStream outputStream(oversizedBuf);
+    capnp::writePackedMessage(outputStream, builder.message);
+    auto packedMessage = heapArray(outputStream.getArray());
+    return wrapBuffer(kj::mv(packedMessage));
+  });
+}
+
 // -----------------------------------------------------------------------------
 
 class RpcConnection: public kj::Refcounted {
@@ -2385,7 +2436,9 @@ void init(v8::Handle<v8::Object> exports) {
     mapFunction("toJs", toJs);
     mapFunction("toJsParams", toJsParams);
     mapFunction("fromBytes", fromBytes);
+    mapFunction("fromBytesPacked", fromBytesPacked);
     mapFunction("toBytes", toBytes);
+    mapFunction("toBytesPacked", toBytesPacked);
     mapFunction("connect", connect);
     mapFunction("disconnect", disconnect);
     mapFunction("restore", restore);

src/node-capnp/capnp.js

@@ -261,12 +261,23 @@ exports.parse = function (schema, buffer) {
   return v8capnp.toJs(reader, Capability);
 }
 
+exports.parsePacked = function (schema, buffer) {
+  var reader = v8capnp.fromBytesPacked(buffer, schema);
+  return v8capnp.toJs(reader, Capability);
+}
+
 exports.serialize = function (schema, value) {
   var builder = v8capnp.newBuilder(schema);
   v8capnp.fromJs(builder, value, LocalCapWrapper);
   return v8capnp.toBytes(builder);
 }
 
+exports.serializePacked = function (schema, value) {
+  var builder = v8capnp.newBuilder(schema);
+  v8capnp.fromJs(builder, value, LocalCapWrapper);
+  return v8capnp.toBytesPacked(builder);
+}
+
 exports.Capability = Capability;
 
 exports.bytesToPreorder = function(schema, buf) {