JSONDocument

Summary:
After evaluating options for JSON storage, I decided to implement our own. The reason is that we'll be able to optimize it better and we get to reduce unnecessary dependencies (which is what we'd get with folly).

I also plan to write a serializer/deserializer for JSONDocument with our own binary format similar to BSON. That way we'll store binary JSON format in RocksDB instead of the plain-text JSON. This means less storage and faster deserialization.

There are still some inefficiencies left here. I plan to optimize them after we develop a functioning DocumentDB. That way we can move and iterate faster.

Test Plan: added a unit test

Reviewers: dhruba, haobo, sdong, ljin, yhchiang

Reviewed By: haobo

Subscribers: leveldb

Differential Revision: https://reviews.facebook.net/D18831

Makefile

@@ -104,6 +104,7 @@ TESTS = \
 	stringappend_test \
 	ttl_test \
 	backupable_db_test \
+	json_document_test \
 	version_edit_test \
 	version_set_test \
 	file_indexer_test \
@@ -343,6 +344,9 @@ prefix_test: db/prefix_test.o $(LIBOBJECTS) $(TESTHARNESS)
 backupable_db_test: utilities/backupable/backupable_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) utilities/backupable/backupable_db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
 
+json_document_test: utilities/document/json_document_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) utilities/document/json_document_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
+
 ttl_test: utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
 

include/utilities/json_document.h

@@ -0,0 +1,172 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+#pragma once
+#ifndef ROCKSDB_LITE
+
+#include <string>
+#include <map>
+#include <unordered_map>
+#include <vector>
+
+#include "rocksdb/slice.h"
+
+// We use JSONDocument for DocumentDB API
+// Implementation inspired by folly::dynamic and rapidjson
+
+namespace rocksdb {
+
+// NOTE: none of this is thread-safe
+class JSONDocument {
+ public:
+  // return nullptr on parse failure
+  static JSONDocument* ParseJSON(const char* json);
+
+  enum Type {
+    kNull,
+    kArray,
+    kBool,
+    kDouble,
+    kInt64,
+    kObject,
+    kString,
+  };
+
+  JSONDocument();  // null
+  /* implicit */ JSONDocument(bool b);
+  /* implicit */ JSONDocument(double d);
+  /* implicit */ JSONDocument(int64_t i);
+  /* implicit */ JSONDocument(const std::string& s);
+  /* implicit */ JSONDocument(const char* s);
+  // constructs JSONDocument of specific type with default value
+  explicit JSONDocument(Type type);
+
+  // copy constructor
+  JSONDocument(const JSONDocument& json_document);
+
+  ~JSONDocument();
+
+  Type type() const;
+
+  // REQUIRES: IsObject()
+  bool Contains(const std::string& key) const;
+  // Returns nullptr if !Contains()
+  // don't delete the returned pointer
+  // REQUIRES: IsObject()
+  const JSONDocument* Get(const std::string& key) const;
+  // REQUIRES: IsObject()
+  JSONDocument& operator[](const std::string& key);
+  // REQUIRES: IsObject()
+  const JSONDocument& operator[](const std::string& key) const;
+  // returns `this`, so you can chain operations.
+  // Copies value
+  // REQUIRES: IsObject()
+  JSONDocument* Set(const std::string& key, const JSONDocument& value);
+
+  // REQUIRES: IsArray() == true || IsObject() == true
+  size_t Count() const;
+
+  // REQUIRES: IsArray()
+  const JSONDocument* GetFromArray(size_t i) const;
+  // REQUIRES: IsArray()
+  JSONDocument& operator[](size_t i);
+  // REQUIRES: IsArray()
+  const JSONDocument& operator[](size_t i) const;
+  // returns `this`, so you can chain operations.
+  // Copies the value
+  // REQUIRES: IsArray() && i < Count()
+  JSONDocument* SetInArray(size_t i, const JSONDocument& value);
+  // REQUIRES: IsArray()
+  JSONDocument* PushBack(const JSONDocument& value);
+
+  bool IsNull() const;
+  bool IsArray() const;
+  bool IsBool() const;
+  bool IsDouble() const;
+  bool IsInt64() const;
+  bool IsObject() const;
+  bool IsString() const;
+
+  // REQUIRES: IsBool() == true
+  bool GetBool() const;
+  // REQUIRES: IsDouble() == true
+  double GetDouble() const;
+  // REQUIRES: IsInt64() == true
+  int64_t GetInt64() const;
+  // REQUIRES: IsString() == true
+  const std::string& GetString() const;
+
+  bool operator==(const JSONDocument& rhs) const;
+
+ private:
+  class ItemsIteratorGenerator;
+
+ public:
+  // REQUIRES: IsObject()
+  ItemsIteratorGenerator Items() const;
+
+  // appends serialized object to dst
+  void Serialize(std::string* dst) const;
+  // returns nullptr if Slice doesn't represent valid serialized JSONDocument
+  static JSONDocument* Deserialize(const Slice& src);
+
+ private:
+  void SerializeInternal(std::string* dst, bool type_prefix) const;
+  // returns false if Slice doesn't represent valid serialized JSONDocument.
+  // Otherwise, true
+  bool DeserializeInternal(Slice* input);
+
+  typedef std::vector<JSONDocument*> Array;
+  typedef std::unordered_map<std::string, JSONDocument*> Object;
+
+  // iteration on objects
+  class const_item_iterator {
+   public:
+    typedef Object::const_iterator It;
+    typedef Object::value_type value_type;
+    /* implicit */ const_item_iterator(It it) : it_(it) {}
+    It& operator++() { return ++it_; }
+    bool operator!=(const const_item_iterator& other) {
+      return it_ != other.it_;
+    }
+    value_type operator*() { return *it_; }
+
+   private:
+    It it_;
+  };
+  class ItemsIteratorGenerator {
+   public:
+    /* implicit */ ItemsIteratorGenerator(const Object& object)
+        : object_(object) {}
+    const_item_iterator begin() { return object_.begin(); }
+    const_item_iterator end() { return object_.end(); }
+
+   private:
+    const Object& object_;
+  };
+
+  union Data {
+    Data() : n(nullptr) {}
+    ~Data() {}
+
+    void* n;
+    Array a;
+    bool b;
+    double d;
+    int64_t i;
+    std::string s;
+    Object o;
+  } data_;
+  const Type type_;
+
+  // Our serialization format's first byte specifies the encoding version. That
+  // way, we can easily change our format while providing backwards
+  // compatibility. This constant specifies the current version of the
+  // serialization format
+  static const char kSerializationFormatVersion;
+};
+
+}  // namespace rocksdb
+
+#endif  // ROCKSDB_LITE

util/coding.h

@@ -38,6 +38,7 @@ extern void PutLengthPrefixedSliceParts(std::string* dst,
 
 // Standard Get... routines parse a value from the beginning of a Slice
 // and advance the slice past the parsed value.
+extern bool GetFixed64(Slice* input, uint64_t* value);
 extern bool GetVarint32(Slice* input, uint32_t* value);
 extern bool GetVarint64(Slice* input, uint64_t* value);
 extern bool GetLengthPrefixedSlice(Slice* input, Slice* result);
@@ -228,6 +229,15 @@ inline int VarintLength(uint64_t v) {
   return len;
 }
 
+inline bool GetFixed64(Slice* input, uint64_t* value) {
+  if (input->size() < sizeof(uint64_t)) {
+    return false;
+  }
+  *value = DecodeFixed64(input->data());
+  input->remove_prefix(sizeof(uint64_t));
+  return true;
+}
+
 inline bool GetVarint32(Slice* input, uint32_t* value) {
   const char* p = input->data();
   const char* limit = p + input->size();

utilities/document/json_document.cc

@@ -0,0 +1,561 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+#ifndef ROCKSDB_LITE
+
+#include "utilities/json_document.h"
+
+#include <cassert>
+#include <string>
+#include <map>
+#include <vector>
+
+#include "third-party/rapidjson/reader.h"
+#include "util/coding.h"
+
+namespace rocksdb {
+
+JSONDocument::JSONDocument() : type_(kNull) {}
+JSONDocument::JSONDocument(bool b) : type_(kBool) { data_.b = b; }
+JSONDocument::JSONDocument(double d) : type_(kDouble) { data_.d = d; }
+JSONDocument::JSONDocument(int64_t i) : type_(kInt64) { data_.i = i; }
+JSONDocument::JSONDocument(const std::string& s) : type_(kString) {
+  new (&data_.s) std::string(s);
+}
+JSONDocument::JSONDocument(const char* s) : type_(kString) {
+  new (&data_.s) std::string(s);
+}
+JSONDocument::JSONDocument(Type type) : type_(type) {
+  // TODO(icanadi) make all of this better by using templates
+  switch (type) {
+    case kNull:
+      break;
+    case kObject:
+      new (&data_.o) Object;
+      break;
+    case kBool:
+      data_.b = false;
+      break;
+    case kDouble:
+      data_.d = 0.0;
+      break;
+    case kArray:
+      new (&data_.a) Array;
+      break;
+    case kInt64:
+      data_.i = 0;
+      break;
+    case kString:
+      new (&data_.s) std::string();
+      break;
+    default:
+      assert(false);
+  }
+}
+
+JSONDocument::JSONDocument(const JSONDocument& json_document)
+    : JSONDocument(json_document.type_) {
+  switch (json_document.type_) {
+    case kNull:
+      break;
+    case kArray:
+      data_.a.reserve(json_document.data_.a.size());
+      for (const auto& iter : json_document.data_.a) {
+        // deep copy
+        data_.a.push_back(new JSONDocument(*iter));
+      }
+      break;
+    case kBool:
+      data_.b = json_document.data_.b;
+      break;
+    case kDouble:
+      data_.d = json_document.data_.d;
+      break;
+    case kInt64:
+      data_.i = json_document.data_.i;
+      break;
+    case kObject: {
+      for (const auto& iter : json_document.data_.o) {
+        // deep copy
+        data_.o.insert({iter.first, new JSONDocument(*iter.second)});
+      }
+      break;
+    }
+    case kString:
+      data_.s = json_document.data_.s;
+      break;
+    default:
+      assert(false);
+  }
+}
+
+JSONDocument::~JSONDocument() {
+  switch (type_) {
+    case kObject:
+      for (auto iter : data_.o) {
+        delete iter.second;
+      }
+      (&data_.o)->~Object();
+      break;
+    case kArray:
+      for (auto iter : data_.a) {
+        delete iter;
+      }
+      (&data_.a)->~Array();
+      break;
+    case kString:
+      using std::string;
+      (&data_.s)->~string();
+      break;
+    default:
+      // we're cool, no need for destructors for others
+      break;
+  }
+}
+
+JSONDocument::Type JSONDocument::type() const { return type_; }
+
+bool JSONDocument::Contains(const std::string& key) const {
+  assert(type_ == kObject);
+  auto iter = data_.o.find(key);
+  return iter != data_.o.end();
+}
+
+const JSONDocument* JSONDocument::Get(const std::string& key) const {
+  assert(type_ == kObject);
+  auto iter = data_.o.find(key);
+  if (iter == data_.o.end()) {
+    return nullptr;
+  }
+  return iter->second;
+}
+
+JSONDocument& JSONDocument::operator[](const std::string& key) {
+  assert(type_ == kObject);
+  auto iter = data_.o.find(key);
+  assert(iter != data_.o.end());
+  return *(iter->second);
+}
+
+const JSONDocument& JSONDocument::operator[](const std::string& key) const {
+  assert(type_ == kObject);
+  auto iter = data_.o.find(key);
+  assert(iter != data_.o.end());
+  return *(iter->second);
+}
+
+JSONDocument* JSONDocument::Set(const std::string& key, const JSONDocument& value) {
+  assert(type_ == kObject);
+  auto itr = data_.o.find(key);
+  if (itr == data_.o.end()) {
+    // insert
+    data_.o.insert({key, new JSONDocument(value)});
+  } else {
+    // overwrite
+    delete itr->second;
+    itr->second = new JSONDocument(value);
+  }
+  return this;
+}
+
+size_t JSONDocument::Count() const {
+  assert(type_ == kArray || type_ == kObject);
+  if (type_ == kArray) {
+    return data_.a.size();
+  } else if (type_ == kObject) {
+    return data_.o.size();
+  }
+  assert(false);
+  return 0;
+}
+
+const JSONDocument* JSONDocument::GetFromArray(size_t i) const {
+  assert(type_ == kArray);
+  return data_.a[i];
+}
+
+JSONDocument& JSONDocument::operator[](size_t i) {
+  assert(type_ == kArray && i < data_.a.size());
+  return *data_.a[i];
+}
+
+const JSONDocument& JSONDocument::operator[](size_t i) const {
+  assert(type_ == kArray && i < data_.a.size());
+  return *data_.a[i];
+}
+
+JSONDocument* JSONDocument::SetInArray(size_t i, const JSONDocument& value) {
+  assert(IsArray() && i < data_.a.size());
+  delete data_.a[i];
+  data_.a[i] = new JSONDocument(value);
+  return this;
+}
+
+JSONDocument* JSONDocument::PushBack(const JSONDocument& value) {
+  assert(IsArray());
+  data_.a.push_back(new JSONDocument(value));
+  return this;
+}
+
+bool JSONDocument::IsNull() const { return type() == kNull; }
+bool JSONDocument::IsArray() const { return type() == kArray; }
+bool JSONDocument::IsBool() const { return type() == kBool; }
+bool JSONDocument::IsDouble() const { return type() == kDouble; }
+bool JSONDocument::IsInt64() const { return type() == kInt64; }
+bool JSONDocument::IsObject() const { return type() == kObject; }
+bool JSONDocument::IsString() const { return type() == kString; }
+
+bool JSONDocument::GetBool() const {
+  assert(IsBool());
+  return data_.b;
+}
+double JSONDocument::GetDouble() const {
+  assert(IsDouble());
+  return data_.d;
+}
+int64_t JSONDocument::GetInt64() const {
+  assert(IsInt64());
+  return data_.i;
+}
+const std::string& JSONDocument::GetString() const {
+  assert(IsString());
+  return data_.s;
+}
+
+bool JSONDocument::operator==(const JSONDocument& rhs) const {
+  if (type_ != rhs.type_) {
+    return false;
+  }
+  switch (type_) {
+    case kNull:
+      return true;  // null == null
+    case kArray:
+      if (data_.a.size() != rhs.data_.a.size()) {
+        return false;
+      }
+      for (size_t i = 0; i < data_.a.size(); ++i) {
+        if (!(*data_.a[i] == *rhs.data_.a[i])) {
+          return false;
+        }
+      }
+      return true;
+    case kBool:
+      return data_.b == rhs.data_.b;
+    case kDouble:
+      return data_.d == rhs.data_.d;
+    case kInt64:
+      return data_.i == rhs.data_.i;
+    case kObject:
+      if (data_.o.size() != rhs.data_.o.size()) {
+        return false;
+      }
+      for (const auto& iter : data_.o) {
+        auto rhs_iter = rhs.data_.o.find(iter.first);
+        if (rhs_iter == rhs.data_.o.end() ||
+            !(*(rhs_iter->second) == *iter.second)) {
+          return false;
+        }
+      }
+      return true;
+    case kString:
+      return data_.s == rhs.data_.s;
+    default:
+      assert(false);
+  }
+}
+
+JSONDocument::ItemsIteratorGenerator JSONDocument::Items() const {
+  assert(type_ == kObject);
+  return data_.o;
+}
+
+// parsing with rapidjson
+// TODO(icanadi) (perf) allocate objects with arena
+JSONDocument* JSONDocument::ParseJSON(const char* json) {
+  class JSONDocumentBuilder {
+   public:
+    JSONDocumentBuilder() {}
+
+    void Null() { stack_.push_back(new JSONDocument()); }
+    void Bool(bool b) { stack_.push_back(new JSONDocument(b)); }
+    void Int(int i) { Int64(static_cast<int64_t>(i)); }
+    void Uint(unsigned i) { Int64(static_cast<int64_t>(i)); }
+    void Int64(int64_t i) { stack_.push_back(new JSONDocument(i)); }
+    void Uint64(uint64_t i) { Int64(static_cast<int64_t>(i)); }
+    void Double(double d) { stack_.push_back(new JSONDocument(d)); }
+    void String(const char* str, size_t length, bool copy) {
+      assert(copy);
+      stack_.push_back(new JSONDocument(std::string(str, length)));
+    }
+    void StartObject() { stack_.push_back(new JSONDocument(kObject)); }
+    void EndObject(size_t member_count) {
+      assert(stack_.size() > 2 * member_count);
+      auto object_base_iter = stack_.end() - member_count * 2 - 1;
+      assert((*object_base_iter)->type_ == kObject);
+      auto& object_map = (*object_base_iter)->data_.o;
+      // iter will always be stack_.end() at some point (i.e. will not advance
+      // past it) because of the way we calculate object_base_iter
+      for (auto iter = object_base_iter + 1; iter != stack_.end(); iter += 2) {
+        assert((*iter)->type_ == kString);
+        object_map.insert({(*iter)->data_.s, *(iter + 1)});
+        delete *iter;
+      }
+      stack_.erase(object_base_iter + 1, stack_.end());
+    }
+    void StartArray() { stack_.push_back(new JSONDocument(kArray)); }
+    void EndArray(size_t element_count) {
+      assert(stack_.size() > element_count);
+      auto array_base_iter = stack_.end() - element_count - 1;
+      assert((*array_base_iter)->type_ == kArray);
+      (*array_base_iter)->data_.a.assign(array_base_iter + 1, stack_.end());
+      stack_.erase(array_base_iter + 1, stack_.end());
+    }
+
+    JSONDocument* GetDocument() {
+      if (stack_.size() != 1) {
+        return nullptr;
+      }
+      return stack_.back();
+    }
+
+    void DeleteAllDocumentsOnStack() {
+      for (auto document : stack_) {
+        delete document;
+      }
+      stack_.clear();
+    }
+
+   private:
+    std::vector<JSONDocument*> stack_;
+  };
+
+  rapidjson::StringStream stream(json);
+  rapidjson::Reader reader;
+  JSONDocumentBuilder handler;
+  bool ok = reader.Parse<0>(stream, handler);
+  if (!ok) {
+    handler.DeleteAllDocumentsOnStack();
+    return nullptr;
+  }
+  auto document = handler.GetDocument();
+  assert(document != nullptr);
+  return document;
+}
+
+// serialization and deserialization
+// format:
+// ------
+// document  ::= header(char) object
+// object    ::= varint32(n) key_value*(n times)
+// key_value ::= string element
+// element   ::= 0x01                     (kNull)
+//            |  0x02 array               (kArray)
+//            |  0x03 byte                (kBool)
+//            |  0x04 double              (kDouble)
+//            |  0x05 int64               (kInt64)
+//            |  0x06 object              (kObject)
+//            |  0x07 string              (kString)
+// array ::= varint32(n) element*(n times)
+// TODO(icanadi) evaluate string vs cstring format
+// string ::= varint32(n) byte*(n times)
+// double ::= 64-bit IEEE 754 floating point (8 bytes)
+// int64  ::= 8 bytes, 64-bit signed integer, little endian
+
+namespace {
+inline char GetPrefixFromType(JSONDocument::Type type) {
+  static char types[] = {0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7};
+  return types[type];
+}
+
+inline bool GetNextType(Slice* input, JSONDocument::Type* type) {
+  if (input->size() == 0) {
+    return false;
+  }
+  static JSONDocument::Type prefixes[] = {
+      JSONDocument::kNull,   JSONDocument::kArray, JSONDocument::kBool,
+      JSONDocument::kDouble, JSONDocument::kInt64, JSONDocument::kObject,
+      JSONDocument::kString};
+  size_t prefix = static_cast<size_t>((*input)[0]);
+  if (prefix == 0 || prefix >= 0x8) {
+    return false;
+  }
+  input->remove_prefix(1);
+  *type = prefixes[static_cast<size_t>(prefix - 1)];
+  return true;
+}
+
+// TODO(icanadi): Make sure this works on all platforms we support. Some
+// platforms may store double in different binary format (our specification says
+// we need IEEE 754)
+inline void PutDouble(std::string* dst, double d) {
+  dst->append(reinterpret_cast<char*>(&d), sizeof(d));
+}
+
+bool DecodeDouble(Slice* input, double* d) {
+  if (input->size() < sizeof(double)) {
+    return false;
+  }
+  memcpy(d, input->data(), sizeof(double));
+  input->remove_prefix(sizeof(double));
+
+  return true;
+}
+}  // namespace
+
+void JSONDocument::Serialize(std::string* dst) const {
+  // first byte is reserved for header
+  // currently, header is only version number. that will help us provide
+  // backwards compatility. we might also store more information here if
+  // necessary
+  dst->push_back(kSerializationFormatVersion);
+  SerializeInternal(dst, false);
+}
+
+void JSONDocument::SerializeInternal(std::string* dst, bool type_prefix) const {
+  if (type_prefix) {
+    dst->push_back(GetPrefixFromType(type_));
+  }
+  switch (type_) {
+    case kNull:
+      // just the prefix is all we need
+      break;
+    case kArray:
+      PutVarint32(dst, static_cast<uint32_t>(data_.a.size()));
+      for (const auto& element : data_.a) {
+        element->SerializeInternal(dst, true);
+      }
+      break;
+    case kBool:
+      dst->push_back(static_cast<char>(data_.b));
+      break;
+    case kDouble:
+      PutDouble(dst, data_.d);
+      break;
+    case kInt64:
+      PutFixed64(dst, static_cast<uint64_t>(data_.i));
+      break;
+    case kObject: {
+      PutVarint32(dst, static_cast<uint32_t>(data_.o.size()));
+      for (const auto& iter : data_.o) {
+        PutLengthPrefixedSlice(dst, Slice(iter.first));
+        iter.second->SerializeInternal(dst, true);
+      }
+      break;
+    }
+    case kString:
+      PutLengthPrefixedSlice(dst, Slice(data_.s));
+      break;
+    default:
+      assert(false);
+  }
+}
+
+const char JSONDocument::kSerializationFormatVersion = 1;
+
+JSONDocument* JSONDocument::Deserialize(const Slice& src) {
+  Slice input(src);
+  if (src.size() == 0) {
+    return nullptr;
+  }
+  char header = input[0];
+  if (header != kSerializationFormatVersion) {
+    // don't understand this header (possibly newer version format and we don't
+    // support downgrade)
+    return nullptr;
+  }
+  input.remove_prefix(1);
+  auto root = new JSONDocument(kObject);
+  bool ok = root->DeserializeInternal(&input);
+  if (!ok || input.size() > 0) {
+    // parsing failure :(
+    delete root;
+    return nullptr;
+  }
+  return root;
+}
+
+bool JSONDocument::DeserializeInternal(Slice* input) {
+  switch (type_) {
+    case kNull:
+      break;
+    case kArray: {
+      uint32_t size;
+      if (!GetVarint32(input, &size)) {
+        return false;
+      }
+      data_.a.resize(size);
+      for (size_t i = 0; i < size; ++i) {
+        Type type;
+        if (!GetNextType(input, &type)) {
+          return false;
+        }
+        data_.a[i] = new JSONDocument(type);
+        if (!data_.a[i]->DeserializeInternal(input)) {
+          return false;
+        }
+      }
+      break;
+    }
+    case kBool:
+      if (input->size() < 1) {
+        return false;
+      }
+      data_.b = static_cast<bool>((*input)[0]);
+      input->remove_prefix(1);
+      break;
+    case kDouble:
+      if (!DecodeDouble(input, &data_.d)) {
+        return false;
+      }
+      break;
+    case kInt64: {
+      uint64_t tmp;
+      if (!GetFixed64(input, &tmp)) {
+        return false;
+      }
+      data_.i = static_cast<int64_t>(tmp);
+      break;
+    }
+    case kObject: {
+      uint32_t num_elements;
+      bool ok = GetVarint32(input, &num_elements);
+      for (uint32_t i = 0; ok && i < num_elements; ++i) {
+        Slice key;
+        ok = GetLengthPrefixedSlice(input, &key);
+        Type type;
+        ok = ok && GetNextType(input, &type);
+        if (ok) {
+          std::unique_ptr<JSONDocument> value(new JSONDocument(type));
+          ok = value->DeserializeInternal(input);
+          if (ok) {
+            data_.o.insert({key.ToString(), value.get()});
+            value.release();
+          }
+        }
+      }
+      if (!ok) {
+        return false;
+      }
+      break;
+    }
+    case kString: {
+      Slice key;
+      if (!GetLengthPrefixedSlice(input, &key)) {
+        return false;
+      }
+      data_.s = key.ToString();
+      break;
+    }
+    default:
+      // this is an assert and not a return because DeserializeInternal() will
+      // always be called with a valid type_. In case there has been data
+      // corruption, GetNextType() is the function that will detect that and
+      // return corruption
+      assert(false);
+  }
+  return true;
+}
+
+}  // namespace rocksdb
+#endif  // ROCKSDB_LITE

utilities/document/json_document_test.cc

@@ -0,0 +1,168 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#include <set>
+
+#include "util/testutil.h"
+#include "util/testharness.h"
+#include "utilities/json_document.h"
+
+namespace rocksdb {
+namespace {
+void AssertField(const JSONDocument& json, const std::string& field) {
+  ASSERT_TRUE(json.Contains(field));
+  ASSERT_TRUE(json[field].IsNull());
+}
+
+void AssertField(const JSONDocument& json, const std::string& field,
+                 const std::string& expected) {
+  ASSERT_TRUE(json.Contains(field));
+  ASSERT_TRUE(json[field].IsString());
+  ASSERT_EQ(expected, json[field].GetString());
+}
+
+void AssertField(const JSONDocument& json, const std::string& field,
+                 int64_t expected) {
+  ASSERT_TRUE(json.Contains(field));
+  ASSERT_TRUE(json[field].IsInt64());
+  ASSERT_EQ(expected, json[field].GetInt64());
+}
+
+void AssertField(const JSONDocument& json, const std::string& field,
+                 bool expected) {
+  ASSERT_TRUE(json.Contains(field));
+  ASSERT_TRUE(json[field].IsBool());
+  ASSERT_EQ(expected, json[field].GetBool());
+}
+
+void AssertField(const JSONDocument& json, const std::string& field,
+                 double expected) {
+  ASSERT_TRUE(json.Contains(field));
+  ASSERT_TRUE(json[field].IsDouble());
+  ASSERT_EQ(expected, json[field].GetDouble());
+}
+}  // namespace
+
+class JSONDocumentTest {
+ public:
+  void AssertSampleJSON(const JSONDocument& json) {
+    AssertField(json, "title", std::string("json"));
+    AssertField(json, "type", std::string("object"));
+    // properties
+    ASSERT_TRUE(json.Contains("properties"));
+    ASSERT_TRUE(json["properties"].Contains("flags"));
+    ASSERT_TRUE(json["properties"]["flags"].IsArray());
+    ASSERT_EQ(3, json["properties"]["flags"].Count());
+    ASSERT_TRUE(json["properties"]["flags"][0].IsInt64());
+    ASSERT_EQ(10, json["properties"]["flags"][0].GetInt64());
+    ASSERT_TRUE(json["properties"]["flags"][1].IsString());
+    ASSERT_EQ("parse", json["properties"]["flags"][1].GetString());
+    ASSERT_TRUE(json["properties"]["flags"][2].IsObject());
+    AssertField(json["properties"]["flags"][2], "tag", std::string("no"));
+    AssertField(json["properties"]["flags"][2], std::string("status"));
+    AssertField(json["properties"], "age", 110.5e-4);
+    AssertField(json["properties"], "depth", static_cast<int64_t>(-10));
+    // test iteration
+    std::set<std::string> expected({"flags", "age", "depth"});
+    for (auto item : json["properties"].Items()) {
+      auto iter = expected.find(item.first);
+      ASSERT_TRUE(iter != expected.end());
+      expected.erase(iter);
+    }
+    ASSERT_EQ(0U, expected.size());
+    ASSERT_TRUE(json.Contains("latlong"));
+    ASSERT_TRUE(json["latlong"].IsArray());
+    ASSERT_EQ(2, json["latlong"].Count());
+    ASSERT_TRUE(json["latlong"][0].IsDouble());
+    ASSERT_EQ(53.25, json["latlong"][0].GetDouble());
+    ASSERT_TRUE(json["latlong"][1].IsDouble());
+    ASSERT_EQ(43.75, json["latlong"][1].GetDouble());
+    AssertField(json, "enabled", true);
+  }
+
+  const std::string kSampleJSON =
+      "{ \"title\" : \"json\", \"type\" : \"object\", \"properties\" : { "
+      "\"flags\": [10, \"parse\", {\"tag\": \"no\", \"status\": null}], "
+      "\"age\": 110.5e-4, \"depth\": -10 }, \"latlong\": [53.25, 43.75], "
+      "\"enabled\": true }";
+
+  const std::string kSampleJSONDifferent =
+      "{ \"title\" : \"json\", \"type\" : \"object\", \"properties\" : { "
+      "\"flags\": [10, \"parse\", {\"tag\": \"no\", \"status\": 2}], "
+      "\"age\": 110.5e-4, \"depth\": -10 }, \"latlong\": [53.25, 43.75], "
+      "\"enabled\": true }";
+};
+
+TEST(JSONDocumentTest, Parsing) {
+  JSONDocument x(static_cast<int64_t>(5));
+  ASSERT_TRUE(x.IsInt64());
+
+  // make sure it's correctly parsed
+  auto parsed_json = JSONDocument::ParseJSON(kSampleJSON.c_str());
+  ASSERT_TRUE(parsed_json != nullptr);
+  AssertSampleJSON(*parsed_json);
+
+  // test deep copying
+  JSONDocument copied_json_document(*parsed_json);
+  AssertSampleJSON(copied_json_document);
+  ASSERT_TRUE(copied_json_document == *parsed_json);
+  delete parsed_json;
+
+  auto parsed_different_sample =
+      JSONDocument::ParseJSON(kSampleJSONDifferent.c_str());
+  ASSERT_TRUE(parsed_different_sample != nullptr);
+  ASSERT_TRUE(!(*parsed_different_sample == copied_json_document));
+  delete parsed_different_sample;
+
+  // parse error
+  const std::string kFaultyJSON =
+      kSampleJSON.substr(0, kSampleJSON.size() - 10);
+  ASSERT_TRUE(JSONDocument::ParseJSON(kFaultyJSON.c_str()) == nullptr);
+}
+
+TEST(JSONDocumentTest, Serialization) {
+  auto parsed_json = JSONDocument::ParseJSON(kSampleJSON.c_str());
+  ASSERT_TRUE(parsed_json != nullptr);
+  std::string serialized;
+  parsed_json->Serialize(&serialized);
+  delete parsed_json;
+
+  auto deserialized_json = JSONDocument::Deserialize(Slice(serialized));
+  ASSERT_TRUE(deserialized_json != nullptr);
+  AssertSampleJSON(*deserialized_json);
+
+  // deserialization failure
+  ASSERT_TRUE(JSONDocument::Deserialize(
+                  Slice(serialized.data(), serialized.size() - 10)) == nullptr);
+}
+
+TEST(JSONDocumentTest, Mutation) {
+  auto sample_json = JSONDocument::ParseJSON(kSampleJSON.c_str());
+  ASSERT_TRUE(sample_json != nullptr);
+  auto different_json = JSONDocument::ParseJSON(kSampleJSONDifferent.c_str());
+  ASSERT_TRUE(different_json != nullptr);
+
+  (*different_json)["properties"]["flags"][2].Set("status", JSONDocument());
+
+  ASSERT_TRUE(*different_json == *sample_json);
+
+  delete different_json;
+  delete sample_json;
+
+  auto json1 = JSONDocument::ParseJSON("{\"a\": [1, 2, 3]}");
+  auto json2 = JSONDocument::ParseJSON("{\"a\": [2, 2, 3, 4]}");
+  ASSERT_TRUE(json1 != nullptr && json2 != nullptr);
+
+  (*json1)["a"].SetInArray(0, static_cast<int64_t>(2))->PushBack(
+      static_cast<int64_t>(4));
+  ASSERT_TRUE(*json1 == *json2);
+
+  delete json1;
+  delete json2;
+}
+
+}  //  namespace rocksdb
+
+int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }