Merge pull request #32 from kentonv/powerbox-query

Implement Sandstorm powerbox query matching algorithm as a library function.

src/node-capnp/capnp-test.js

@@ -87,10 +87,109 @@ for (var key in keys) {
   assert.equal(parsed[key], parsedPackedFlat[key]);
 }
 
-// =======================================================================================
-
 console.log("serialization: pass");
 
+// =======================================================================================
+// Test matchPowerboxQuery
+
+var tag1 = capnp.serialize(test.TestAllTypes, {int32Field: 123});
+var tag2 = capnp.serialize(test.TestAllTypes, {int32Field: 321});
+var tag3 = capnp.serialize(test.TestAllTypes, {});
+
+assert(capnp.matchPowerboxQuery(tag1, tag1));
+assert(capnp.matchPowerboxQuery(tag2, tag2));
+assert(capnp.matchPowerboxQuery(tag3, tag3));
+
+assert(!capnp.matchPowerboxQuery(tag1, tag2));
+assert(!capnp.matchPowerboxQuery(tag2, tag3));
+assert(!capnp.matchPowerboxQuery(tag3, tag1));
+
+var emptyTag = capnp.serialize(test.TestEmptyStruct, {});
+assert(!capnp.matchPowerboxQuery(emptyTag, tag1));
+assert(!capnp.matchPowerboxQuery(emptyTag, tag2));
+assert(capnp.matchPowerboxQuery(emptyTag, tag3));
+assert(!capnp.matchPowerboxQuery(tag1, emptyTag));
+assert(!capnp.matchPowerboxQuery(tag2, emptyTag));
+assert(capnp.matchPowerboxQuery(tag3, emptyTag));
+
+// Test comparing pointers.
+var tagStr1 = capnp.serialize(test.TestAllTypes, {textField: "foo"});
+var tagStr2 = capnp.serialize(test.TestAllTypes, {textField: "bar"});
+var tagStr3 = capnp.serialize(test.TestAllTypes, {textField: ""});
+assert(capnp.matchPowerboxQuery(tagStr1, tagStr1));
+assert(capnp.matchPowerboxQuery(tagStr2, tagStr2));
+assert(capnp.matchPowerboxQuery(tagStr3, tagStr3));
+assert(!capnp.matchPowerboxQuery(tagStr1, tagStr2));
+assert(!capnp.matchPowerboxQuery(tagStr2, tagStr3));
+assert(!capnp.matchPowerboxQuery(tagStr3, tagStr1));
+assert(capnp.matchPowerboxQuery(tagStr1, emptyTag));
+assert(capnp.matchPowerboxQuery(tagStr2, emptyTag));
+assert(capnp.matchPowerboxQuery(tagStr3, emptyTag));
+assert(capnp.matchPowerboxQuery(emptyTag, tagStr1));
+assert(capnp.matchPowerboxQuery(emptyTag, tagStr2));
+assert(capnp.matchPowerboxQuery(emptyTag, tagStr3));
+
+// Note that string comparisons are actually list comparisons. But let's make sure an exact match
+// is required (which is not the case for list-of-structs).
+var tagStr4 = capnp.serialize(test.TestAllTypes, {textField: "oof"});
+assert(!capnp.matchPowerboxQuery(tagStr1, tagStr4));
+
+// Test list-of-structs.
+var tagFooBar = capnp.serialize(test.TestAllTypes,
+    {structList: [{textField: "foo"}, {textField: "bar"}]});
+var tagBarFoo = capnp.serialize(test.TestAllTypes,
+    {structList: [{textField: "bar"}, {textField: "foo"}]});
+var tagFooOnly = capnp.serialize(test.TestAllTypes,
+    {structList: [{textField: "foo"}]});
+var tagBarOnly = capnp.serialize(test.TestAllTypes,
+    {structList: [{textField: "bar"}]});
+var tagEmptyList = capnp.serialize(test.TestAllTypes,
+    {structList: []});
+
+assert(capnp.matchPowerboxQuery(tagFooBar, tagFooBar));
+assert(capnp.matchPowerboxQuery(tagFooBar, tagBarFoo));
+assert(capnp.matchPowerboxQuery(tagBarFoo, tagFooBar));
+assert(capnp.matchPowerboxQuery(tagFooOnly, tagFooBar));
+assert(capnp.matchPowerboxQuery(tagBarOnly, tagFooBar));
+assert(capnp.matchPowerboxQuery(tagFooOnly, tagBarFoo));
+assert(capnp.matchPowerboxQuery(tagBarOnly, tagBarFoo));
+assert(!capnp.matchPowerboxQuery(tagFooBar, tagFooOnly));
+assert(!capnp.matchPowerboxQuery(tagFooBar, tagBarOnly));
+assert(!capnp.matchPowerboxQuery(tagBarFoo, tagFooOnly));
+assert(!capnp.matchPowerboxQuery(tagBarFoo, tagBarOnly));
+
+assert(capnp.matchPowerboxQuery(tagFooBar, emptyTag));
+assert(capnp.matchPowerboxQuery(tagFooOnly, emptyTag));
+assert(capnp.matchPowerboxQuery(tagBarOnly, emptyTag));
+assert(capnp.matchPowerboxQuery(emptyTag, tagFooBar));
+assert(capnp.matchPowerboxQuery(emptyTag, tagFooOnly));
+assert(capnp.matchPowerboxQuery(emptyTag, tagBarOnly));
+
+assert(!capnp.matchPowerboxQuery(tagFooBar, tagEmptyList));
+assert(!capnp.matchPowerboxQuery(tagFooOnly, tagEmptyList));
+assert(!capnp.matchPowerboxQuery(tagBarOnly, tagEmptyList));
+assert(capnp.matchPowerboxQuery(tagEmptyList, tagFooBar));
+assert(capnp.matchPowerboxQuery(tagEmptyList, tagFooOnly));
+assert(capnp.matchPowerboxQuery(tagEmptyList, tagBarOnly));
+
+// Test list-of-pointers.
+var tagStrList1 = capnp.serialize(test.TestAllTypes, {textList: ["foo", "bar"]});
+var tagStrList2 = capnp.serialize(test.TestAllTypes, {textList: ["bar", "foo"]});
+var tagStrList3 = capnp.serialize(test.TestAllTypes, {textList: ["foo"]});
+var tagStrList4 = capnp.serialize(test.TestAllTypes, {textList: ["foo", null]});
+
+assert(capnp.matchPowerboxQuery(tagStrList1, tagStrList1));
+assert(capnp.matchPowerboxQuery(tagStrList2, tagStrList2));
+assert(!capnp.matchPowerboxQuery(tagStrList1, tagStrList2));
+assert(!capnp.matchPowerboxQuery(tagStrList2, tagStrList1));
+assert(!capnp.matchPowerboxQuery(tagStrList3, tagStrList1));
+assert(capnp.matchPowerboxQuery(tagStrList4, tagStrList1));
+
+console.log("matchPowerboxQuery: pass");
+
+// =======================================================================================
+// Test RPC, if possible.
+
 if (!fs.existsSync("capnp-samples")) {
   console.warn("skipping RPC because capnp-samples not present");
   process.exit(0);

src/node-capnp/capnp.cc

@@ -1368,12 +1368,19 @@ struct FromJsConverter {
             }
             if (error) break;
           } else {
+            bool isPointerList = builder.as<capnp::AnyList>().getElementSize() ==
+                                 capnp::ElementSize::POINTER;
             for (uint i: kj::indices(builder)) {
-              auto element = orphanFromJs(field, orphanage, elementType, jsArray->Get(i));
-              if (element.getType() == capnp::DynamicValue::UNKNOWN) {
-                return nullptr;
+              auto jsElement = jsArray->Get(i);
+              if (isPointerList && (jsElement->IsNull() || jsElement->IsUndefined())) {
+                // Skip null element.
+              } else {
+                auto element = orphanFromJs(field, orphanage, elementType, jsElement);
+                if (element.getType() == capnp::DynamicValue::UNKNOWN) {
+                  return nullptr;
+                }
+                builder.adopt(i, kj::mv(element));
               }
-              builder.adopt(i, kj::mv(element));
             }
           }
           return kj::mv(orphan);
@@ -2521,6 +2528,174 @@ void throw_(const v8::FunctionCallbackInfo<v8::Value>& args) {
 
 // -----------------------------------------------------------------------------
 
+class AlignedWords {
+public:
+  AlignedWords(kj::ArrayPtr<const kj::byte> bytes) {
+    if (reinterpret_cast<uintptr_t>(bytes.begin()) % sizeof(capnp::word) != 0) {
+      // Array is not aligned.  We have to make a copy.  :(
+      copy = kj::heapArray<capnp::word>(bytes.size() / sizeof(capnp::word));
+      memcpy(copy.begin(), bytes.begin(), copy.asBytes().size());
+      words = copy;
+    } else {
+      // Yay, array is aligned.
+      words = kj::arrayPtr(reinterpret_cast<const capnp::word*>(bytes.begin()),
+                           bytes.size() / sizeof(capnp::word));
+    }
+  }
+
+  inline const kj::ArrayPtr<const capnp::word>* operator->() const { return &words; }
+  inline const kj::ArrayPtr<const capnp::word>& operator*() const { return words; }
+
+private:
+  kj::ArrayPtr<const capnp::word> words;
+  kj::Array<capnp::word> copy;
+};
+
+bool matchPowerboxQuery(capnp::AnyPointer::Reader query, capnp::AnyPointer::Reader candidate);
+
+bool matchPowerboxQuery(capnp::AnyStruct::Reader query, capnp::AnyStruct::Reader candidate) {
+  {
+    // Compare data.
+    auto queryData = query.getDataSection();
+    auto candidateData = candidate.getDataSection();
+
+    auto commonSize = kj::min(queryData.size(), candidateData.size());
+    if (memcmp(queryData.begin(), candidateData.begin(), commonSize) != 0) {
+      // Data sections don't match.
+      return false;
+    }
+
+    // Non-matched parts of data sections must be all-zero.
+    kj::byte accum = 0;
+    for (kj::byte b: queryData.slice(commonSize, queryData.size())) {
+      accum |= b;
+    }
+    for (kj::byte b: candidateData.slice(commonSize, candidateData.size())) {
+      accum |= b;
+    }
+    if (accum != 0) return false;
+  }
+
+  {
+    // Compare pointers.
+    auto queryPointers = query.getPointerSection();
+    auto candidatePointers = candidate.getPointerSection();
+
+    auto commonSize = kj::min(queryPointers.size(), candidatePointers.size());
+    for (auto i: kj::range<decltype(commonSize)>(0, commonSize)) {
+      if (!matchPowerboxQuery(queryPointers[i], candidatePointers[i])) {
+        return false;
+      }
+    }
+
+    // No need to compare the non-overlapping range since null pointers match anything.
+  }
+
+  return true;
+}
+
+bool matchPowerboxQuery(capnp::AnyList::Reader query, capnp::AnyList::Reader candidate) {
+  auto elementSize = query.getElementSize();
+  if (candidate.getElementSize() != elementSize) return false;
+
+  switch (elementSize) {
+    case capnp::ElementSize::VOID:
+    case capnp::ElementSize::BIT:
+    case capnp::ElementSize::BYTE:
+    case capnp::ElementSize::TWO_BYTES:
+    case capnp::ElementSize::FOUR_BYTES:
+    case capnp::ElementSize::EIGHT_BYTES:
+      return query == candidate;
+
+    case capnp::ElementSize::POINTER: {
+      auto queryPtrs = query.as<capnp::List<capnp::AnyPointer>>();
+      auto candidatePtrs = candidate.as<capnp::List<capnp::AnyPointer>>();
+
+      if (queryPtrs.size() != candidatePtrs.size()) return false;
+      for (auto i: kj::indices(queryPtrs)) {
+        if (!matchPowerboxQuery(queryPtrs[i], candidatePtrs[i])) return false;
+      }
+      return true;
+    }
+
+    case capnp::ElementSize::INLINE_COMPOSITE: {
+      // Every element in the query must be matched by at least one element in the candidate.
+      //
+      // TODO(perf): O(m*n), can we do better?
+
+      auto candidateStructs = candidate.as<capnp::List<capnp::AnyStruct>>();
+      for (auto queryElement: query.as<capnp::List<capnp::AnyStruct>>()) {
+        bool matched = false;
+        for (auto candidateElement: candidateStructs) {
+          if (matchPowerboxQuery(queryElement, candidateElement)) {
+            matched = true;
+            break;
+          }
+        }
+        if (!matched) return false;
+      }
+      return true;
+    }
+  }
+}
+
+bool matchPowerboxQuery(capnp::AnyPointer::Reader query, capnp::AnyPointer::Reader candidate) {
+  auto queryType = query.getPointerType();
+  auto candidateType = candidate.getPointerType();
+
+  if (queryType != candidateType) {
+    // Different types -> no match, unless one is null, which is a wildcard.
+    return queryType == capnp::PointerType::NULL_ ||
+           candidateType == capnp::PointerType::NULL_;
+  }
+
+  switch (queryType) {
+    case capnp::PointerType::NULL_:
+      return true;
+    case capnp::PointerType::STRUCT:
+      return matchPowerboxQuery(query.getAs<capnp::AnyStruct>(),
+                                candidate.getAs<capnp::AnyStruct>());
+    case capnp::PointerType::LIST:
+      return matchPowerboxQuery(query.getAs<capnp::AnyList>(),
+                                candidate.getAs<capnp::AnyList>());
+    case capnp::PointerType::CAPABILITY:
+      // TODO(someday): Support matching capabilities?
+      return false;
+  }
+}
+
+void matchPowerboxQuery(const v8::FunctionCallbackInfo<v8::Value>& args) {
+  // matchPowerboxQuery(queryBuffer, candidateBuffer) -> bool
+  //
+  // Decodes queryBuffer and candidateBuffer as capnp messages and returns true if the query
+  // "matches" the candidate according to Sandstorm's powerbox query tag matching algorithm. See
+  // PowerboxDescriptor::Tag::value in sandstorm/powerbox.capnp for full details of the algorithm.
+  // The idea here is that the two parameters are AnyPointer values from Tag::value, which in
+  // node-capnp are normally represented as byte arrays. Note that the order of the parameters
+  // is important, particularly in the case of struct list matching.
+
+  KJV8_UNWRAP(CapnpContext, context, args.Data());
+  KJV8_UNWRAP_BUFFER(queryBuffer, args[0]);
+  KJV8_UNWRAP_BUFFER(candidateBuffer, args[1]);
+
+  liftKj(args, [&]() -> v8::Handle<v8::Value> {
+    AlignedWords queryWords(queryBuffer);
+    AlignedWords candidateWords(candidateBuffer);
+
+    capnp::FlatArrayMessageReader queryReader(*queryWords);
+    capnp::FlatArrayMessageReader candidateReader(*candidateWords);
+
+    auto queryRoot = queryReader.getRoot<capnp::AnyPointer>();
+    auto candidateRoot = candidateReader.getRoot<capnp::AnyPointer>();
+
+    bool result = matchPowerboxQuery(queryRoot, candidateRoot);
+
+    return v8::Boolean::New(args.GetIsolate(), result);
+  });
+}
+
+// -----------------------------------------------------------------------------
+
 void init(v8::Handle<v8::Object> exports) {
   CapnpContext* context = new CapnpContext;
   auto wrappedContext = context->wrapper.wrap(context);
@@ -2565,6 +2740,8 @@ void init(v8::Handle<v8::Object> exports) {
   mapFunction("return_", return_);
   mapFunction("throw_", throw_);
 
+  mapFunction("matchPowerboxQuery", matchPowerboxQuery);
+
   mapFunction("dumpLocalCapTypeCounts", dumpLocalCapTypeCounts);
 }
 

src/node-capnp/capnp.js

@@ -291,4 +291,6 @@ exports.bytesToPreorder = function(schema, buf, options) {
   return v8capnp.toBytes(builder, options || {});
 }
 
+exports.matchPowerboxQuery = v8capnp.matchPowerboxQuery;
+
 exports.dumpLocalCapTypeCounts = v8capnp.dumpLocalCapTypeCounts;