Remove some components (#4101)

Summary:
Remove some components that we never heard people using them.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4101

Differential Revision: D8825431

Pulled By: siying

fbshipit-source-id: 97a12ad3cad4ab12c82741a5ba49669aaa854180

CMakeLists.txt

@@ -643,18 +643,10 @@ set(SOURCES
         utilities/cassandra/format.cc
         utilities/cassandra/merge_operator.cc
         utilities/checkpoint/checkpoint_impl.cc
-        utilities/col_buf_decoder.cc
-        utilities/col_buf_encoder.cc
-        utilities/column_aware_encoding_util.cc
         utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
-        utilities/date_tiered/date_tiered_db_impl.cc
         utilities/debug.cc
-        utilities/document/document_db.cc
-        utilities/document/json_document.cc
-        utilities/document/json_document_builder.cc
         utilities/env_mirror.cc
         utilities/env_timed.cc
-        utilities/geodb/geodb_impl.cc
         utilities/leveldb_options/leveldb_options.cc
         utilities/lua/rocks_lua_compaction_filter.cc
         utilities/memory/memory_util.cc
@@ -671,9 +663,7 @@ set(SOURCES
         utilities/persistent_cache/block_cache_tier_metadata.cc
         utilities/persistent_cache/persistent_cache_tier.cc
         utilities/persistent_cache/volatile_tier_impl.cc
-        utilities/redis/redis_lists.cc
         utilities/simulator_cache/sim_cache.cc
-        utilities/spatialdb/spatial_db.cc
         utilities/table_properties_collectors/compact_on_deletion_collector.cc
         utilities/trace/file_trace_reader_writer.cc
         utilities/transactions/optimistic_transaction_db_impl.cc
@@ -975,11 +965,6 @@ if(WITH_TESTS)
         utilities/cassandra/cassandra_row_merge_test.cc
         utilities/cassandra/cassandra_serialize_test.cc
         utilities/checkpoint/checkpoint_test.cc
-        utilities/column_aware_encoding_test.cc
-        utilities/date_tiered/date_tiered_test.cc
-        utilities/document/document_db_test.cc
-        utilities/document/json_document_test.cc
-        utilities/geodb/geodb_test.cc
         utilities/lua/rocks_lua_test.cc
         utilities/memory/memory_test.cc
         utilities/merge_operators/string_append/stringappend_test.cc
@@ -988,8 +973,6 @@ if(WITH_TESTS)
         utilities/options/options_util_test.cc
         utilities/persistent_cache/hash_table_test.cc
         utilities/persistent_cache/persistent_cache_test.cc
-        utilities/redis/redis_lists_test.cc
-        utilities/spatialdb/spatial_db_test.cc
         utilities/simulator_cache/sim_cache_test.cc
         utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
         utilities/transactions/optimistic_transaction_test.cc
@@ -1009,7 +992,6 @@ if(WITH_TESTS)
     db/range_del_aggregator_bench.cc
     tools/db_bench.cc
     table/table_reader_bench.cc
-    utilities/column_aware_encoding_exp.cc
     utilities/persistent_cache/hash_table_bench.cc)
   add_library(testharness OBJECT util/testharness.cc)
   foreach(sourcefile ${BENCHMARKS})

HISTORY.md

@@ -8,6 +8,7 @@
 ### Public API Change
 * Transaction::GetForUpdate is extended with a do_validate parameter with default value of true. If false it skips validating the snapshot before doing the read. Similarly ::Merge, ::Put, ::Delete, and ::SingleDelete are extended with assume_tracked with default value of false. If true it indicates that call is assumed to be after a ::GetForUpdate.
 * `TableProperties::num_entries` and `TableProperties::num_deletions` now also account for number of range tombstones.
+* Remove geodb, spatial_db, document_db, json_document, date_tiered_db, and redis_lists.
 
 ### Bug Fixes
 * Fix a deadlock caused by compaction and file ingestion waiting for each other in the event of write stalls.

Makefile

@@ -403,7 +403,7 @@ BENCHTOOLOBJECTS = $(BENCH_LIB_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
 
 ANALYZETOOLOBJECTS = $(ANALYZER_LIB_SOURCES:.cc=.o)
 
-EXPOBJECTS = $(EXP_LIB_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
+EXPOBJECTS = $(LIBOBJECTS) $(TESTUTIL)
 
 TESTS = \
 	db_basic_test \
@@ -482,7 +482,6 @@ TESTS = \
 	merger_test \
 	util_merge_operators_test \
 	options_file_test \
-	redis_test \
 	reduce_levels_test \
 	plain_table_db_test \
 	comparator_db_test \
@@ -496,12 +495,8 @@ TESTS = \
 	cassandra_row_merge_test \
 	cassandra_serialize_test \
 	ttl_test \
-	date_tiered_test \
 	backupable_db_test \
-	document_db_test \
-	json_document_test \
 	sim_cache_test \
-	spatial_db_test \
 	version_edit_test \
 	version_set_test \
 	compaction_picker_test \
@@ -513,7 +508,6 @@ TESTS = \
 	deletefile_test \
 	obsolete_files_test \
 	table_test \
-	geodb_test \
 	delete_scheduler_test \
 	options_test \
 	options_settable_test \
@@ -530,7 +524,6 @@ TESTS = \
 	compaction_job_test \
 	thread_list_test \
 	sst_dump_test \
-	column_aware_encoding_test \
 	compact_files_test \
 	optimistic_transaction_test \
 	write_callback_test \
@@ -604,7 +597,7 @@ TEST_LIBS = \
 	librocksdb_env_basic_test.a
 
 # TODO: add back forward_iterator_bench, after making it build in all environemnts.
-BENCHMARKS = db_bench table_reader_bench cache_bench memtablerep_bench column_aware_encoding_exp persistent_cache_bench range_del_aggregator_bench
+BENCHMARKS = db_bench table_reader_bench cache_bench memtablerep_bench persistent_cache_bench range_del_aggregator_bench
 
 # if user didn't config LIBNAME, set the default
 ifeq ($(LIBNAME),)
@@ -1153,9 +1146,6 @@ cassandra_row_merge_test: utilities/cassandra/cassandra_row_merge_test.o utiliti
 cassandra_serialize_test: utilities/cassandra/cassandra_serialize_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 hash_table_test: utilities/persistent_cache/hash_table_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1294,18 +1284,9 @@ backupable_db_test: utilities/backupable/backupable_db_test.o $(LIBOBJECTS) $(TE
 checkpoint_test: utilities/checkpoint/checkpoint_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-document_db_test: utilities/document/document_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-json_document_test: utilities/document/json_document_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 sim_cache_test: utilities/simulator_cache/sim_cache_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-spatial_db_test: utilities/spatialdb/spatial_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 env_mirror_test: utilities/env_mirror_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1323,9 +1304,6 @@ object_registry_test: utilities/object_registry_test.o $(LIBOBJECTS) $(TESTHARNE
 ttl_test: utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-date_tiered_test: utilities/date_tiered/date_tiered_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 write_batch_with_index_test: utilities/write_batch_with_index/write_batch_with_index_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1452,9 +1430,6 @@ deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS)
 obsolete_files_test: db/obsolete_files_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-geodb_test: utilities/geodb/geodb_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 rocksdb_dump: tools/dump/rocksdb_dump.o $(LIBOBJECTS)
 	$(AM_LINK)
 
@@ -1503,9 +1478,6 @@ timer_queue_test: util/timer_queue_test.o $(LIBOBJECTS) $(TESTHARNESS)
 sst_dump_test: tools/sst_dump_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-column_aware_encoding_test: utilities/column_aware_encoding_test.o $(TESTHARNESS) $(EXPOBJECTS)
-	$(AM_LINK)
-
 optimistic_transaction_test: utilities/transactions/optimistic_transaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1545,9 +1517,6 @@ sst_dump: tools/sst_dump.o $(LIBOBJECTS)
 blob_dump: tools/blob_dump.o $(LIBOBJECTS)
 	$(AM_LINK)
 
-column_aware_encoding_exp: utilities/column_aware_encoding_exp.o $(EXPOBJECTS)
-	$(AM_LINK)
-
 repair_test: db/repair_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1973,7 +1942,7 @@ endif
 #  	Source files dependencies detection
 # ---------------------------------------------------------------------------
 
-all_sources = $(LIB_SOURCES) $(MAIN_SOURCES) $(MOCK_LIB_SOURCES) $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(TEST_LIB_SOURCES) $(EXP_LIB_SOURCES) $(ANALYZER_LIB_SOURCES)
+all_sources = $(LIB_SOURCES) $(MAIN_SOURCES) $(MOCK_LIB_SOURCES) $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(TEST_LIB_SOURCES) $(ANALYZER_LIB_SOURCES)
 DEPFILES = $(all_sources:.cc=.cc.d)
 
 # Add proper dependency support so changing a .h file forces a .cc file to

TARGETS

@@ -255,14 +255,9 @@ cpp_library(
         "utilities/checkpoint/checkpoint_impl.cc",
         "utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
         "utilities/convenience/info_log_finder.cc",
-        "utilities/date_tiered/date_tiered_db_impl.cc",
         "utilities/debug.cc",
-        "utilities/document/document_db.cc",
-        "utilities/document/json_document.cc",
-        "utilities/document/json_document_builder.cc",
         "utilities/env_mirror.cc",
         "utilities/env_timed.cc",
-        "utilities/geodb/geodb_impl.cc",
         "utilities/leveldb_options/leveldb_options.cc",
         "utilities/lua/rocks_lua_compaction_filter.cc",
         "utilities/memory/memory_util.cc",
@@ -279,9 +274,7 @@ cpp_library(
         "utilities/persistent_cache/block_cache_tier_metadata.cc",
         "utilities/persistent_cache/persistent_cache_tier.cc",
         "utilities/persistent_cache/volatile_tier_impl.cc",
-        "utilities/redis/redis_lists.cc",
         "utilities/simulator_cache/sim_cache.cc",
-        "utilities/spatialdb/spatial_db.cc",
         "utilities/table_properties_collectors/compact_on_deletion_collector.cc",
         "utilities/trace/file_trace_reader_writer.cc",
         "utilities/transactions/optimistic_transaction.cc",
@@ -319,9 +312,6 @@ cpp_library(
         "util/testharness.cc",
         "util/testutil.cc",
         "utilities/cassandra/test_utils.cc",
-        "utilities/col_buf_decoder.cc",
-        "utilities/col_buf_encoder.cc",
-        "utilities/column_aware_encoding_util.cc",
     ],
     auto_headers = AutoHeaders.RECURSIVE_GLOB,
     arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
@@ -444,11 +434,6 @@ ROCKS_TESTS = [
         "util/coding_test.cc",
         "serial",
     ],
-    [
-        "column_aware_encoding_test",
-        "utilities/column_aware_encoding_test.cc",
-        "serial",
-    ],
     [
         "column_family_test",
         "db/column_family_test.cc",
@@ -519,11 +504,6 @@ ROCKS_TESTS = [
         "table/data_block_hash_index_test.cc",
         "serial",
     ],
-    [
-        "date_tiered_test",
-        "utilities/date_tiered/date_tiered_test.cc",
-        "serial",
-    ],
     [
         "db_basic_test",
         "db/db_basic_test.cc",
@@ -684,11 +664,6 @@ ROCKS_TESTS = [
         "db/deletefile_test.cc",
         "serial",
     ],
-    [
-        "document_db_test",
-        "utilities/document/document_db_test.cc",
-        "serial",
-    ],
     [
         "dynamic_bloom_test",
         "util/dynamic_bloom_test.cc",
@@ -764,11 +739,6 @@ ROCKS_TESTS = [
         "table/full_filter_block_test.cc",
         "serial",
     ],
-    [
-        "geodb_test",
-        "utilities/geodb/geodb_test.cc",
-        "serial",
-    ],
     [
         "hash_table_test",
         "utilities/persistent_cache/hash_table_test.cc",
@@ -799,11 +769,6 @@ ROCKS_TESTS = [
         "monitoring/iostats_context_test.cc",
         "serial",
     ],
-    [
-        "json_document_test",
-        "utilities/document/json_document_test.cc",
-        "serial",
-    ],
     [
         "ldb_cmd_test",
         "tools/ldb_cmd_test.cc",
@@ -969,11 +934,6 @@ ROCKS_TESTS = [
         "util/slice_transform_test.cc",
         "serial",
     ],
-    [
-        "spatial_db_test",
-        "utilities/spatialdb/spatial_db_test.cc",
-        "serial",
-    ],
     [
         "sst_dump_test",
         "tools/sst_dump_test.cc",

include/rocksdb/utilities/date_tiered_db.h

@@ -1,108 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-#ifndef ROCKSDB_LITE
-
-#include <map>
-#include <string>
-#include <vector>
-
-#include "rocksdb/db.h"
-
-namespace rocksdb {
-
-// Date tiered database is a wrapper of DB that implements
-// a simplified DateTieredCompactionStrategy by using multiple column famillies
-// as time windows.
-//
-// DateTieredDB provides an interface similar to DB, but it assumes that user
-// provides keys with last 8 bytes encoded as timestamp in seconds. DateTieredDB
-// is assigned with a TTL to declare when data should be deleted.
-//
-// DateTieredDB hides column families layer from standard RocksDB instance. It
-// uses multiple column families to manage time series data, each containing a
-// specific range of time. Column families are named by its maximum possible
-// timestamp. A column family is created automatically when data newer than
-// latest timestamp of all existing column families. The time range of a column
-// family is configurable by `column_family_interval`. By doing this, we
-// guarantee that compaction will only happen in a column family.
-//
-// DateTieredDB is assigned with a TTL. When all data in a column family are
-// expired (CF_Timestamp <= CUR_Timestamp - TTL), we directly drop the whole
-// column family.
-//
-// TODO(jhli): This is only a simplified version of DTCS. In a complete DTCS,
-// time windows can be merged over time, so that older time windows will have
-// larger time range. Also, compaction are executed only for adjacent SST files
-// to guarantee there is no time overlap between SST files.
-
-class DateTieredDB {
- public:
-  // Open a DateTieredDB whose name is `dbname`.
-  // Similar to DB::Open(), created database object is stored in dbptr.
-  //
-  // Two parameters can be configured: `ttl` to specify the length of time that
-  // keys should exist in the database, and `column_family_interval` to specify
-  // the time range of a column family interval.
-  //
-  // Open a read only database if read only is set as true.
-  // TODO(jhli): Should use an option object that includes ttl and
-  // column_family_interval.
-  static Status Open(const Options& options, const std::string& dbname,
-                     DateTieredDB** dbptr, int64_t ttl,
-                     int64_t column_family_interval, bool read_only = false);
-
-  explicit DateTieredDB() {}
-
-  virtual ~DateTieredDB() {}
-
-  // Wrapper for Put method. Similar to DB::Put(), but column family to be
-  // inserted is decided by the timestamp in keys, i.e. the last 8 bytes of user
-  // key. If key is already obsolete, it will not be inserted.
-  //
-  // When client put a key value pair in DateTieredDB, it assumes last 8 bytes
-  // of keys are encoded as timestamp. Timestamp is a 64-bit signed integer
-  // encoded as the number of seconds since 1970-01-01 00:00:00 (UTC) (Same as
-  // Env::GetCurrentTime()). Timestamp should be encoded in big endian.
-  virtual Status Put(const WriteOptions& options, const Slice& key,
-                     const Slice& val) = 0;
-
-  // Wrapper for Get method. Similar to DB::Get() but column family is decided
-  // by timestamp in keys. If key is already obsolete, it will not be found.
-  virtual Status Get(const ReadOptions& options, const Slice& key,
-                     std::string* value) = 0;
-
-  // Wrapper for Delete method. Similar to DB::Delete() but column family is
-  // decided by timestamp in keys. If key is already obsolete, return NotFound
-  // status.
-  virtual Status Delete(const WriteOptions& options, const Slice& key) = 0;
-
-  // Wrapper for KeyMayExist method. Similar to DB::KeyMayExist() but column
-  // family is decided by timestamp in keys. Return false when key is already
-  // obsolete.
-  virtual bool KeyMayExist(const ReadOptions& options, const Slice& key,
-                           std::string* value, bool* value_found = nullptr) = 0;
-
-  // Wrapper for Merge method. Similar to DB::Merge() but column family is
-  // decided by timestamp in keys.
-  virtual Status Merge(const WriteOptions& options, const Slice& key,
-                       const Slice& value) = 0;
-
-  // Create an iterator that hides low level details. This iterator internally
-  // merge results from all active time series column families. Note that
-  // column families are not deleted until all data are obsolete, so this
-  // iterator can possibly access obsolete key value pairs.
-  virtual Iterator* NewIterator(const ReadOptions& opts) = 0;
-
-  // Explicitly drop column families in which all keys are obsolete. This
-  // process is also inplicitly done in Put() operation.
-  virtual Status DropObsoleteColumnFamilies() = 0;
-
-  static const uint64_t kTSLength = sizeof(int64_t);  // size of timestamp
-};
-
-}  // namespace rocksdb
-#endif  // ROCKSDB_LITE

include/rocksdb/utilities/document_db.h

@@ -1,149 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-#ifndef ROCKSDB_LITE
-
-#include <string>
-#include <vector>
-
-#include "rocksdb/utilities/stackable_db.h"
-#include "rocksdb/utilities/json_document.h"
-#include "rocksdb/db.h"
-
-namespace rocksdb {
-
-// IMPORTANT: DocumentDB is a work in progress. It is unstable and we might
-// change the API without warning. Talk to RocksDB team before using this in
-// production ;)
-
-// DocumentDB is a layer on top of RocksDB that provides a very simple JSON API.
-// When creating a DB, you specify a list of indexes you want to keep on your
-// data. You can insert a JSON document to the DB, which is automatically
-// indexed. Every document added to the DB needs to have "_id" field which is
-// automatically indexed and is an unique primary key. All other indexes are
-// non-unique.
-
-// NOTE: field names in the JSON are NOT allowed to start with '$' or
-// contain '.'. We don't currently enforce that rule, but will start behaving
-// badly.
-
-// Cursor is what you get as a result of executing query. To get all
-// results from a query, call Next() on a Cursor while  Valid() returns true
-class Cursor {
- public:
-  Cursor() = default;
-  virtual ~Cursor() {}
-
-  virtual bool Valid() const = 0;
-  virtual void Next() = 0;
-  // Lifecycle of the returned JSONDocument is until the next Next() call
-  virtual const JSONDocument& document() const = 0;
-  virtual Status status() const = 0;
-
- private:
-  // No copying allowed
-  Cursor(const Cursor&);
-  void operator=(const Cursor&);
-};
-
-struct DocumentDBOptions {
-  int background_threads = 4;
-  uint64_t memtable_size = 128 * 1024 * 1024;    // 128 MB
-  uint64_t cache_size = 1 * 1024 * 1024 * 1024;  // 1 GB
-};
-
-// TODO(icanadi) Add `JSONDocument* info` parameter to all calls that can be
-// used by the caller to get more information about the call execution (number
-// of dropped records, number of updated records, etc.)
-class DocumentDB : public StackableDB {
- public:
-  struct IndexDescriptor {
-    // Currently, you can only define an index on a single field. To specify an
-    // index on a field X, set index description to JSON "{X: 1}"
-    // Currently the value needs to be 1, which means ascending.
-    // In the future, we plan to also support indexes on multiple keys, where
-    // you could mix ascending sorting (1) with descending sorting indexes (-1)
-    JSONDocument* description;
-    std::string name;
-  };
-
-  // Open DocumentDB with specified indexes. The list of indexes has to be
-  // complete, i.e. include all indexes present in the DB, except the primary
-  // key index.
-  // Otherwise, Open() will return an error
-  static Status Open(const DocumentDBOptions& options, const std::string& name,
-                     const std::vector<IndexDescriptor>& indexes,
-                     DocumentDB** db, bool read_only = false);
-
-  explicit DocumentDB(DB* db) : StackableDB(db) {}
-
-  // Create a new index. It will stop all writes for the duration of the call.
-  // All current documents in the DB are scanned and corresponding index entries
-  // are created
-  virtual Status CreateIndex(const WriteOptions& write_options,
-                             const IndexDescriptor& index) = 0;
-
-  // Drop an index. Client is responsible to make sure that index is not being
-  // used by currently executing queries
-  virtual Status DropIndex(const std::string& name) = 0;
-
-  // Insert a document to the DB. The document needs to have a primary key "_id"
-  // which can either be a string or an integer. Otherwise the write will fail
-  // with InvalidArgument.
-  virtual Status Insert(const WriteOptions& options,
-                        const JSONDocument& document) = 0;
-
-  // Deletes all documents matching a filter atomically
-  virtual Status Remove(const ReadOptions& read_options,
-                        const WriteOptions& write_options,
-                        const JSONDocument& query) = 0;
-
-  // Does this sequence of operations:
-  // 1. Find all documents matching a filter
-  // 2. For all documents, atomically:
-  // 2.1. apply the update operators
-  // 2.2. update the secondary indexes
-  //
-  // Currently only $set update operator is supported.
-  // Syntax is: {$set: {key1: value1, key2: value2, etc...}}
-  // This operator will change a document's key1 field to value1, key2 to
-  // value2, etc. New values will be set even if a document didn't have an entry
-  // for the specified key.
-  //
-  // You can not change a primary key of a document.
-  //
-  // Update example: Update({id: {$gt: 5}, $index: id}, {$set: {enabled: true}})
-  virtual Status Update(const ReadOptions& read_options,
-                        const WriteOptions& write_options,
-                        const JSONDocument& filter,
-                        const JSONDocument& updates) = 0;
-
-  // query has to be an array in which every element is an operator. Currently
-  // only $filter operator is supported. Syntax of $filter operator is:
-  // {$filter: {key1: condition1, key2: condition2, etc.}} where conditions can
-  // be either:
-  // 1) a single value in which case the condition is equality condition, or
-  // 2) a defined operators, like {$gt: 4}, which will match all documents that
-  // have key greater than 4.
-  //
-  // Supported operators are:
-  // 1) $gt -- greater than
-  // 2) $gte -- greater than or equal
-  // 3) $lt -- less than
-  // 4) $lte -- less than or equal
-  // If you want the filter to use an index, you need to specify it like this:
-  // {$filter: {...(conditions)..., $index: index_name}}
-  //
-  // Example query:
-  // * [{$filter: {name: John, age: {$gte: 18}, $index: age}}]
-  // will return all Johns whose age is greater or equal to 18 and it will use
-  // index "age" to satisfy the query.
-  virtual Cursor* Query(const ReadOptions& read_options,
-                        const JSONDocument& query) = 0;
-};
-
-}  // namespace rocksdb
-#endif  // ROCKSDB_LITE

include/rocksdb/utilities/geo_db.h

@@ -1,114 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-
-#ifndef ROCKSDB_LITE
-#pragma once
-#include <string>
-#include <vector>
-
-#include "rocksdb/utilities/stackable_db.h"
-#include "rocksdb/status.h"
-
-namespace rocksdb {
-
-//
-// Configurable options needed for setting up a Geo database
-//
-struct GeoDBOptions {
-  // Backup info and error messages will be written to info_log
-  // if non-nullptr.
-  // Default: nullptr
-  Logger* info_log;
-
-  explicit GeoDBOptions(Logger* _info_log = nullptr):info_log(_info_log) { }
-};
-
-//
-// A position in the earth's geoid
-//
-class GeoPosition {
- public:
-  double latitude;
-  double longitude;
-
-  explicit GeoPosition(double la = 0, double lo = 0) :
-    latitude(la), longitude(lo) {
-  }
-};
-
-//
-// Description of an object on the Geoid. It is located by a GPS location,
-// and is identified by the id. The value associated with this object is
-// an opaque string 'value'. Different objects identified by unique id's
-// can have the same gps-location associated with them.
-//
-class GeoObject {
- public:
-  GeoPosition position;
-  std::string id;
-  std::string value;
-
-  GeoObject() {}
-
-  GeoObject(const GeoPosition& pos, const std::string& i,
-            const std::string& val) :
-    position(pos), id(i), value(val) {
-  }
-};
-
-class GeoIterator {
- public:
-  GeoIterator() = default;
-  virtual ~GeoIterator() {}
-  virtual void Next() = 0;
-  virtual bool Valid() const = 0;
-  virtual const GeoObject& geo_object() = 0;
-  virtual Status status() const = 0;
-};
-
-//
-// Stack your DB with GeoDB to be able to get geo-spatial support
-//
-class GeoDB : public StackableDB {
- public:
-  // GeoDBOptions have to be the same as the ones used in a previous
-  // incarnation of the DB
-  //
-  // GeoDB owns the pointer `DB* db` now. You should not delete it or
-  // use it after the invocation of GeoDB
-  // GeoDB(DB* db, const GeoDBOptions& options) : StackableDB(db) {}
-  GeoDB(DB* db, const GeoDBOptions& /*options*/) : StackableDB(db) {}
-  virtual ~GeoDB() {}
-
-  // Insert a new object into the location database. The object is
-  // uniquely identified by the id. If an object with the same id already
-  // exists in the db, then the old one is overwritten by the new
-  // object being inserted here.
-  virtual Status Insert(const GeoObject& object) = 0;
-
-  // Retrieve the value of the object located at the specified GPS
-  // location and is identified by the 'id'.
-  virtual Status GetByPosition(const GeoPosition& pos,
-                               const Slice& id, std::string* value) = 0;
-
-  // Retrieve the value of the object identified by the 'id'. This method
-  // could be potentially slower than GetByPosition
-  virtual Status GetById(const Slice& id, GeoObject*  object) = 0;
-
-  // Delete the specified object
-  virtual Status Remove(const Slice& id) = 0;
-
-  // Returns an iterator for the items within a circular radius from the
-  // specified gps location. If 'number_of_values' is specified,
-  // then the iterator is capped to that number of objects.
-  // The radius is specified in 'meters'.
-  virtual GeoIterator* SearchRadial(const GeoPosition& pos,
-                                    double radius,
-                                    int number_of_values = INT_MAX) = 0;
-};
-
-}  // namespace rocksdb
-#endif  // ROCKSDB_LITE

include/rocksdb/utilities/json_document.h

@@ -1,195 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-#pragma once
-#ifndef ROCKSDB_LITE
-
-#include <deque>
-#include <map>
-#include <memory>
-#include <string>
-#include <unordered_map>
-#include <utility>
-#include <vector>
-
-#include "rocksdb/slice.h"
-
-// We use JSONDocument for DocumentDB API
-// Implementation inspired by folly::dynamic, rapidjson and fbson
-
-namespace fbson {
-  class FbsonValue;
-  class ObjectVal;
-  template <typename T>
-  class FbsonWriterT;
-  class FbsonOutStream;
-  typedef FbsonWriterT<FbsonOutStream> FbsonWriter;
-}  // namespace fbson
-
-namespace rocksdb {
-
-// NOTE: none of this is thread-safe
-class JSONDocument {
- public:
-  // return nullptr on parse failure
-  static JSONDocument* ParseJSON(const char* json);
-
-  enum Type {
-    kNull,
-    kArray,
-    kBool,
-    kDouble,
-    kInt64,
-    kObject,
-    kString,
-  };
-
-  /* implicit */ JSONDocument();  // null
-  /* implicit */ JSONDocument(bool b);
-  /* implicit */ JSONDocument(double d);
-  /* implicit */ JSONDocument(int8_t i);
-  /* implicit */ JSONDocument(int16_t i);
-  /* implicit */ JSONDocument(int32_t i);
-  /* implicit */ JSONDocument(int64_t i);
-  /* implicit */ JSONDocument(const std::string& s);
-  /* implicit */ JSONDocument(const char* s);
-  // constructs JSONDocument of specific type with default value
-  explicit JSONDocument(Type _type);
-
-  JSONDocument(const JSONDocument& json_document);
-
-  JSONDocument(JSONDocument&& json_document);
-
-  Type type() const;
-
-  // REQUIRES: IsObject()
-  bool Contains(const std::string& key) const;
-  // REQUIRES: IsObject()
-  // Returns non-owner object
-  JSONDocument operator[](const std::string& key) const;
-
-  // REQUIRES: IsArray() == true || IsObject() == true
-  size_t Count() const;
-
-  // REQUIRES: IsArray()
-  // Returns non-owner object
-  JSONDocument operator[](size_t i) const;
-
-  JSONDocument& operator=(JSONDocument jsonDocument);
-
-  bool IsNull() const;
-  bool IsArray() const;
-  bool IsBool() const;
-  bool IsDouble() const;
-  bool IsInt64() const;
-  bool IsObject() const;
-  bool IsString() const;
-
-  // REQUIRES: IsBool() == true
-  bool GetBool() const;
-  // REQUIRES: IsDouble() == true
-  double GetDouble() const;
-  // REQUIRES: IsInt64() == true
-  int64_t GetInt64() const;
-  // REQUIRES: IsString() == true
-  std::string GetString() const;
-
-  bool operator==(const JSONDocument& rhs) const;
-
-  bool operator!=(const JSONDocument& rhs) const;
-
-  JSONDocument Copy() const;
-
-  bool IsOwner() const;
-
-  std::string DebugString() const;
-
- private:
-  class ItemsIteratorGenerator;
-
- public:
-  // REQUIRES: IsObject()
-  ItemsIteratorGenerator Items() const;
-
-  // appends serialized object to dst
-  void Serialize(std::string* dst) const;
-  // returns nullptr if Slice doesn't represent valid serialized JSONDocument
-  static JSONDocument* Deserialize(const Slice& src);
-
- private:
-  friend class JSONDocumentBuilder;
-
-  JSONDocument(fbson::FbsonValue* val, bool makeCopy);
-
-  void InitFromValue(const fbson::FbsonValue* val);
-
-  // iteration on objects
-  class const_item_iterator {
-   private:
-    class Impl;
-   public:
-    typedef std::pair<std::string, JSONDocument> value_type;
-    explicit const_item_iterator(Impl* impl);
-    const_item_iterator(const_item_iterator&&);
-    const_item_iterator& operator++();
-    bool operator!=(const const_item_iterator& other);
-    value_type operator*();
-    ~const_item_iterator();
-   private:
-    friend class ItemsIteratorGenerator;
-    std::unique_ptr<Impl> it_;
-  };
-
-  class ItemsIteratorGenerator {
-   public:
-    explicit ItemsIteratorGenerator(const fbson::ObjectVal& object);
-    const_item_iterator begin() const;
-
-    const_item_iterator end() const;
-
-   private:
-    const fbson::ObjectVal& object_;
-  };
-
-  std::unique_ptr<char[]> data_;
-  mutable fbson::FbsonValue* value_;
-
-  // Our serialization format's first byte specifies the encoding version. That
-  // way, we can easily change our format while providing backwards
-  // compatibility. This constant specifies the current version of the
-  // serialization format
-  static const char kSerializationFormatVersion;
-};
-
-class JSONDocumentBuilder {
- public:
-  JSONDocumentBuilder();
-
-  explicit JSONDocumentBuilder(fbson::FbsonOutStream* out);
-
-  void Reset();
-
-  bool WriteStartArray();
-
-  bool WriteEndArray();
-
-  bool WriteStartObject();
-
-  bool WriteEndObject();
-
-  bool WriteKeyValue(const std::string& key, const JSONDocument& value);
-
-  bool WriteJSONDocument(const JSONDocument& value);
-
-  JSONDocument GetJSONDocument();
-
-  ~JSONDocumentBuilder();
-
- private:
-  std::unique_ptr<fbson::FbsonWriter> writer_;
-};
-
-}  // namespace rocksdb
-
-#endif  // ROCKSDB_LITE

include/rocksdb/utilities/spatial_db.h

@@ -1,261 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-#ifndef ROCKSDB_LITE
-
-#include <string>
-#include <vector>
-
-#include "rocksdb/db.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/utilities/stackable_db.h"
-
-namespace rocksdb {
-namespace spatial {
-
-// NOTE: SpatialDB is experimental and we might change its API without warning.
-// Please talk to us before developing against SpatialDB API.
-//
-// SpatialDB is a support for spatial indexes built on top of RocksDB.
-// When creating a new SpatialDB, clients specifies a list of spatial indexes to
-// build on their data. Each spatial index is defined by the area and
-// granularity. If you're storing map data, different spatial index
-// granularities can be used for different zoom levels.
-//
-// Each element inserted into SpatialDB has:
-// * a bounding box, which determines how will the element be indexed
-// * string blob, which will usually be WKB representation of the polygon
-// (http://en.wikipedia.org/wiki/Well-known_text)
-// * feature set, which is a map of key-value pairs, where value can be null,
-// int, double, bool, string
-// * a list of indexes to insert the element in
-//
-// Each query is executed on a single spatial index. Query guarantees that it
-// will return all elements intersecting the specified bounding box, but it
-// might also return some extra non-intersecting elements.
-
-// Variant is a class that can be many things: null, bool, int, double or string
-// It is used to store different value types in FeatureSet (see below)
-struct Variant {
-  // Don't change the values here, they are persisted on disk
-  enum Type {
-    kNull = 0x0,
-    kBool = 0x1,
-    kInt = 0x2,
-    kDouble = 0x3,
-    kString = 0x4,
-  };
-
-  Variant() : type_(kNull) {}
-  /* implicit */ Variant(bool b) : type_(kBool) { data_.b = b; }
-  /* implicit */ Variant(uint64_t i) : type_(kInt) { data_.i = i; }
-  /* implicit */ Variant(double d) : type_(kDouble) { data_.d = d; }
-  /* implicit */ Variant(const std::string& s) : type_(kString) {
-    new (&data_.s) std::string(s);
-  }
-
-  Variant(const Variant& v) : type_(v.type_) { Init(v, data_); }
-
-  Variant& operator=(const Variant& v);
-
-  Variant(Variant&& rhs) : type_(kNull) { *this = std::move(rhs); }
-
-  Variant& operator=(Variant&& v);
-
-  ~Variant() { Destroy(type_, data_); }
-
-  Type type() const { return type_; }
-  bool get_bool() const { return data_.b; }
-  uint64_t get_int() const { return data_.i; }
-  double get_double() const { return data_.d; }
-  const std::string& get_string() const { return *GetStringPtr(data_); }
-
-  bool operator==(const Variant& other) const;
-  bool operator!=(const Variant& other) const { return !(*this == other); }
-
- private:
-  Type type_;
-
-  union Data {
-    bool b;
-    uint64_t i;
-    double d;
-    // Current version of MS compiler not C++11 compliant so can not put
-    // std::string
-    // however, even then we still need the rest of the maintenance.
-    char s[sizeof(std::string)];
-  } data_;
-
-  // Avoid type_punned aliasing problem
-  static std::string* GetStringPtr(Data& d) {
-    void* p = d.s;
-    return reinterpret_cast<std::string*>(p);
-  }
-
-  static const std::string* GetStringPtr(const Data& d) {
-    const void* p = d.s;
-    return reinterpret_cast<const std::string*>(p);
-  }
-
-  static void Init(const Variant&, Data&);
-
-  static void Destroy(Type t, Data& d) {
-    if (t == kString) {
-      using std::string;
-      GetStringPtr(d)->~string();
-    }
-  }
-};
-
-// FeatureSet is a map of key-value pairs. One feature set is associated with
-// each element in SpatialDB. It can be used to add rich data about the element.
-class FeatureSet {
- private:
-  typedef std::unordered_map<std::string, Variant> map;
-
- public:
-  class iterator {
-   public:
-    /* implicit */ iterator(const map::const_iterator itr) : itr_(itr) {}
-    iterator& operator++() {
-      ++itr_;
-      return *this;
-    }
-    bool operator!=(const iterator& other) { return itr_ != other.itr_; }
-    bool operator==(const iterator& other) { return itr_ == other.itr_; }
-    map::value_type operator*() { return *itr_; }
-
-   private:
-    map::const_iterator itr_;
-  };
-  FeatureSet() = default;
-
-  FeatureSet* Set(const std::string& key, const Variant& value);
-  bool Contains(const std::string& key) const;
-  // REQUIRES: Contains(key)
-  const Variant& Get(const std::string& key) const;
-  iterator Find(const std::string& key) const;
-
-  iterator begin() const { return map_.begin(); }
-  iterator end() const { return map_.end(); }
-
-  void Clear();
-  size_t Size() const { return map_.size(); }
-
-  void Serialize(std::string* output) const;
-  // REQUIRED: empty FeatureSet
-  bool Deserialize(const Slice& input);
-
-  std::string DebugString() const;
-
- private:
-  map map_;
-};
-
-// BoundingBox is a helper structure for defining rectangles representing
-// bounding boxes of spatial elements.
-template <typename T>
-struct BoundingBox {
-  T min_x, min_y, max_x, max_y;
-  BoundingBox() = default;
-  BoundingBox(T _min_x, T _min_y, T _max_x, T _max_y)
-      : min_x(_min_x), min_y(_min_y), max_x(_max_x), max_y(_max_y) {}
-
-  bool Intersects(const BoundingBox<T>& a) const {
-    return !(min_x > a.max_x || min_y > a.max_y || a.min_x > max_x ||
-             a.min_y > max_y);
-  }
-};
-
-struct SpatialDBOptions {
-  uint64_t cache_size = 1 * 1024 * 1024 * 1024LL;  // 1GB
-  int num_threads = 16;
-  bool bulk_load = true;
-};
-
-// Cursor is used to return data from the query to the client. To get all the
-// data from the query, just call Next() while Valid() is true
-class Cursor {
- public:
-  Cursor() = default;
-  virtual ~Cursor() {}
-
-  virtual bool Valid() const = 0;
-  // REQUIRES: Valid()
-  virtual void Next() = 0;
-
-  // Lifetime of the underlying storage until the next call to Next()
-  // REQUIRES: Valid()
-  virtual const Slice blob() = 0;
-  // Lifetime of the underlying storage until the next call to Next()
-  // REQUIRES: Valid()
-  virtual const FeatureSet& feature_set() = 0;
-
-  virtual Status status() const = 0;
-
- private:
-  // No copying allowed
-  Cursor(const Cursor&);
-  void operator=(const Cursor&);
-};
-
-// SpatialIndexOptions defines a spatial index that will be built on the data
-struct SpatialIndexOptions {
-  // Spatial indexes are referenced by names
-  std::string name;
-  // An area that is indexed. If the element is not intersecting with spatial
-  // index's bbox, it will not be inserted into the index
-  BoundingBox<double> bbox;
-  // tile_bits control the granularity of the spatial index. Each dimension of
-  // the bbox will be split into (1 << tile_bits) tiles, so there will be a
-  // total of (1 << tile_bits)^2 tiles. It is recommended to configure a size of
-  // each  tile to be approximately the size of the query on that spatial index
-  uint32_t tile_bits;
-  SpatialIndexOptions() {}
-  SpatialIndexOptions(const std::string& _name,
-                      const BoundingBox<double>& _bbox, uint32_t _tile_bits)
-      : name(_name), bbox(_bbox), tile_bits(_tile_bits) {}
-};
-
-class SpatialDB : public StackableDB {
- public:
-  // Creates the SpatialDB with specified list of indexes.
-  // REQUIRED: db doesn't exist
-  static Status Create(const SpatialDBOptions& options, const std::string& name,
-                       const std::vector<SpatialIndexOptions>& spatial_indexes);
-
-  // Open the existing SpatialDB.  The resulting db object will be returned
-  // through db parameter.
-  // REQUIRED: db was created using SpatialDB::Create
-  static Status Open(const SpatialDBOptions& options, const std::string& name,
-                     SpatialDB** db, bool read_only = false);
-
-  explicit SpatialDB(DB* db) : StackableDB(db) {}
-
-  // Insert the element into the DB. Element will be inserted into specified
-  // spatial_indexes, based on specified bbox.
-  // REQUIRES: spatial_indexes.size() > 0
-  virtual Status Insert(const WriteOptions& write_options,
-                        const BoundingBox<double>& bbox, const Slice& blob,
-                        const FeatureSet& feature_set,
-                        const std::vector<std::string>& spatial_indexes) = 0;
-
-  // Calling Compact() after inserting a bunch of elements should speed up
-  // reading. This is especially useful if you use SpatialDBOptions::bulk_load
-  // Num threads determines how many threads we'll use for compactions. Setting
-  // this to bigger number will use more IO and CPU, but finish faster
-  virtual Status Compact(int num_threads = 1) = 0;
-
-  // Query the specified spatial_index. Query will return all elements that
-  // intersect bbox, but it may also return some extra elements.
-  virtual Cursor* Query(const ReadOptions& read_options,
-                        const BoundingBox<double>& bbox,
-                        const std::string& spatial_index) = 0;
-};
-
-}  // namespace spatial
-}  // namespace rocksdb
-#endif  // ROCKSDB_LITE

src.mk

@@ -177,14 +177,9 @@ LIB_SOURCES =                                                   \
   utilities/checkpoint/checkpoint_impl.cc                       \
   utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc    \
   utilities/convenience/info_log_finder.cc                      \
-  utilities/date_tiered/date_tiered_db_impl.cc                  \
   utilities/debug.cc                                            \
-  utilities/document/document_db.cc                             \
-  utilities/document/json_document.cc                           \
-  utilities/document/json_document_builder.cc                   \
   utilities/env_mirror.cc                                       \
   utilities/env_timed.cc                                        \
-  utilities/geodb/geodb_impl.cc                                 \
   utilities/leveldb_options/leveldb_options.cc                  \
   utilities/lua/rocks_lua_compaction_filter.cc                  \
   utilities/memory/memory_util.cc                               \
@@ -201,9 +196,7 @@ LIB_SOURCES =                                                   \
   utilities/persistent_cache/block_cache_tier_metadata.cc       \
   utilities/persistent_cache/persistent_cache_tier.cc           \
   utilities/persistent_cache/volatile_tier_impl.cc              \
-  utilities/redis/redis_lists.cc                                \
   utilities/simulator_cache/sim_cache.cc                        \
-  utilities/spatialdb/spatial_db.cc                             \
   utilities/table_properties_collectors/compact_on_deletion_collector.cc \
   utilities/trace/file_trace_reader_writer.cc                   \
   utilities/transactions/optimistic_transaction.cc              \
@@ -249,11 +242,6 @@ MOCK_LIB_SOURCES = \
 BENCH_LIB_SOURCES = \
   tools/db_bench_tool.cc                                        \
 
-EXP_LIB_SOURCES = \
-  utilities/col_buf_decoder.cc                                  \
-  utilities/col_buf_encoder.cc                                  \
-  utilities/column_aware_encoding_util.cc
-
 TEST_LIB_SOURCES = \
   db/db_test_util.cc                                            \
   util/testharness.cc                                           \
@@ -330,7 +318,6 @@ MAIN_SOURCES =                                                          \
   db/persistent_cache_test.cc                                           \
   db/plain_table_db_test.cc                                             \
   db/prefix_test.cc                                                     \
-  db/redis_test.cc                                                      \
   db/repair_test.cc                                                     \
   db/range_del_aggregator_test.cc                                       \
   db/range_del_aggregator_bench.cc                                      \
@@ -397,21 +384,13 @@ MAIN_SOURCES =                                                          \
   utilities/cassandra/cassandra_row_merge_test.cc                       \
   utilities/cassandra/cassandra_serialize_test.cc                       \
   utilities/checkpoint/checkpoint_test.cc                               \
-  utilities/column_aware_encoding_exp.cc                                \
-  utilities/column_aware_encoding_test.cc                               \
-  utilities/date_tiered/date_tiered_test.cc                             \
-  utilities/document/document_db_test.cc                                \
-  utilities/document/json_document_test.cc                              \
-  utilities/geodb/geodb_test.cc                                         \
   utilities/lua/rocks_lua_test.cc                                       \
   utilities/memory/memory_test.cc                                       \
   utilities/merge_operators/string_append/stringappend_test.cc          \
   utilities/object_registry_test.cc                                     \
   utilities/option_change_migration/option_change_migration_test.cc     \
   utilities/options/options_util_test.cc                                \
-  utilities/redis/redis_lists_test.cc                                   \
   utilities/simulator_cache/sim_cache_test.cc                           \
-  utilities/spatialdb/spatial_db_test.cc                                \
   utilities/table_properties_collectors/compact_on_deletion_collector_test.cc  \
   utilities/transactions/optimistic_transaction_test.cc                 \
   utilities/transactions/transaction_test.cc                            \

utilities/col_buf_decoder.cc

@@ -1,240 +0,0 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "utilities/col_buf_decoder.h"
-#include <cstring>
-#include <string>
-#include "port/port.h"
-
-namespace rocksdb {
-
-ColBufDecoder::~ColBufDecoder() {}
-
-namespace {
-
-inline uint64_t EncodeFixed64WithEndian(uint64_t val, bool big_endian,
-                                        size_t size) {
-  if (big_endian && port::kLittleEndian) {
-    val = EndianTransform(val, size);
-  } else if (!big_endian && !port::kLittleEndian) {
-    val = EndianTransform(val, size);
-  }
-  return val;
-}
-
-}  // namespace
-
-ColBufDecoder* ColBufDecoder::NewColBufDecoder(
-    const ColDeclaration& col_declaration) {
-  if (col_declaration.col_type == "FixedLength") {
-    return new FixedLengthColBufDecoder(
-        col_declaration.size, col_declaration.col_compression_type,
-        col_declaration.nullable, col_declaration.big_endian);
-  } else if (col_declaration.col_type == "VariableLength") {
-    return new VariableLengthColBufDecoder();
-  } else if (col_declaration.col_type == "VariableChunk") {
-    return new VariableChunkColBufDecoder(col_declaration.col_compression_type);
-  } else if (col_declaration.col_type == "LongFixedLength") {
-    return new LongFixedLengthColBufDecoder(col_declaration.size,
-                                            col_declaration.nullable);
-  }
-  // Unrecognized column type
-  return nullptr;
-}
-
-namespace {
-
-void ReadVarint64(const char** src_ptr, uint64_t* val_ptr) {
-  const char* q = GetVarint64Ptr(*src_ptr, *src_ptr + 10, val_ptr);
-  assert(q != nullptr);
-  *src_ptr = q;
-}
-}  // namespace
-
-size_t FixedLengthColBufDecoder::Init(const char* src) {
-  remain_runs_ = 0;
-  last_val_ = 0;
-  // Dictionary initialization
-  dict_vec_.clear();
-  const char* orig_src = src;
-  if (col_compression_type_ == kColDict ||
-      col_compression_type_ == kColRleDict) {
-    const char* q;
-    uint64_t dict_size;
-    // Bypass limit
-    q = GetVarint64Ptr(src, src + 10, &dict_size);
-    assert(q != nullptr);
-    src = q;
-
-    uint64_t dict_key;
-    for (uint64_t i = 0; i < dict_size; ++i) {
-      // Bypass limit
-      ReadVarint64(&src, &dict_key);
-
-      dict_key = EncodeFixed64WithEndian(dict_key, big_endian_, size_);
-      dict_vec_.push_back(dict_key);
-    }
-  }
-  return src - orig_src;
-}
-
-size_t FixedLengthColBufDecoder::Decode(const char* src, char** dest) {
-  uint64_t read_val = 0;
-  const char* orig_src = src;
-  const char* src_limit = src + 20;
-  if (nullable_) {
-    bool not_null;
-    not_null = *src;
-    src += 1;
-    if (!not_null) {
-      return 1;
-    }
-  }
-  if (IsRunLength(col_compression_type_)) {
-    if (remain_runs_ == 0) {
-      const char* q;
-      run_val_ = 0;
-      if (col_compression_type_ == kColRle) {
-        memcpy(&run_val_, src, size_);
-        src += size_;
-      } else {
-        q = GetVarint64Ptr(src, src_limit, &run_val_);
-        assert(q != nullptr);
-        src = q;
-      }
-
-      q = GetVarint64Ptr(src, src_limit, &remain_runs_);
-      assert(q != nullptr);
-      src = q;
-
-      if (col_compression_type_ != kColRleDeltaVarint &&
-          col_compression_type_ != kColRleDict) {
-        run_val_ = EncodeFixed64WithEndian(run_val_, big_endian_, size_);
-      }
-    }
-    read_val = run_val_;
-  } else {
-    if (col_compression_type_ == kColNoCompression) {
-      memcpy(&read_val, src, size_);
-      src += size_;
-    } else {
-      // Assume a column does not exceed 8 bytes here
-      const char* q = GetVarint64Ptr(src, src_limit, &read_val);
-      assert(q != nullptr);
-      src = q;
-    }
-    if (col_compression_type_ != kColDeltaVarint &&
-        col_compression_type_ != kColDict) {
-      read_val = EncodeFixed64WithEndian(read_val, big_endian_, size_);
-    }
-  }
-
-  uint64_t write_val = read_val;
-  if (col_compression_type_ == kColDeltaVarint ||
-      col_compression_type_ == kColRleDeltaVarint) {
-    // does not support 64 bit
-
-    uint64_t mask = (write_val & 1) ? (~uint64_t(0)) : 0;
-    int64_t delta = (write_val >> 1) ^ mask;
-    write_val = last_val_ + delta;
-
-    uint64_t tmp = write_val;
-    write_val = EncodeFixed64WithEndian(write_val, big_endian_, size_);
-    last_val_ = tmp;
-  } else if (col_compression_type_ == kColRleDict ||
-             col_compression_type_ == kColDict) {
-    uint64_t dict_val = read_val;
-    assert(dict_val < dict_vec_.size());
-    write_val = dict_vec_[static_cast<size_t>(dict_val)];
-  }
-
-  // dest->append(reinterpret_cast<char*>(&write_val), size_);
-  memcpy(*dest, reinterpret_cast<char*>(&write_val), size_);
-  *dest += size_;
-  if (IsRunLength(col_compression_type_)) {
-    --remain_runs_;
-  }
-  return src - orig_src;
-}
-
-size_t LongFixedLengthColBufDecoder::Decode(const char* src, char** dest) {
-  if (nullable_) {
-    bool not_null;
-    not_null = *src;
-    src += 1;
-    if (!not_null) {
-      return 1;
-    }
-  }
-  memcpy(*dest, src, size_);
-  *dest += size_;
-  return size_ + 1;
-}
-
-size_t VariableLengthColBufDecoder::Decode(const char* src, char** dest) {
-  uint8_t len;
-  len = *src;
-  memcpy(dest, reinterpret_cast<char*>(&len), 1);
-  *dest += 1;
-  src += 1;
-  memcpy(*dest, src, len);
-  *dest += len;
-  return len + 1;
-}
-
-size_t VariableChunkColBufDecoder::Init(const char* src) {
-  // Dictionary initialization
-  dict_vec_.clear();
-  const char* orig_src = src;
-  if (col_compression_type_ == kColDict) {
-    const char* q;
-    uint64_t dict_size;
-    // Bypass limit
-    q = GetVarint64Ptr(src, src + 10, &dict_size);
-    assert(q != nullptr);
-    src = q;
-
-    uint64_t dict_key;
-    for (uint64_t i = 0; i < dict_size; ++i) {
-      // Bypass limit
-      ReadVarint64(&src, &dict_key);
-      dict_vec_.push_back(dict_key);
-    }
-  }
-  return src - orig_src;
-}
-
-size_t VariableChunkColBufDecoder::Decode(const char* src, char** dest) {
-  const char* orig_src = src;
-  uint64_t size = 0;
-  ReadVarint64(&src, &size);
-  int64_t full_chunks = size / 8;
-  uint64_t chunk_buf;
-  size_t chunk_size = 8;
-  for (int64_t i = 0; i < full_chunks + 1; ++i) {
-    chunk_buf = 0;
-    if (i == full_chunks) {
-      chunk_size = size % 8;
-    }
-    if (col_compression_type_ == kColDict) {
-      uint64_t dict_val;
-      ReadVarint64(&src, &dict_val);
-      assert(dict_val < dict_vec_.size());
-      chunk_buf = dict_vec_[static_cast<size_t>(dict_val)];
-    } else {
-      memcpy(&chunk_buf, src, chunk_size);
-      src += chunk_size;
-    }
-    memcpy(*dest, reinterpret_cast<char*>(&chunk_buf), 8);
-    *dest += 8;
-    uint8_t mask = ((0xFF - 8) + chunk_size) & 0xFF;
-    memcpy(*dest, reinterpret_cast<char*>(&mask), 1);
-    *dest += 1;
-  }
-
-  return src - orig_src;
-}
-
-}  // namespace rocksdb

utilities/col_buf_decoder.h

@@ -1,119 +0,0 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-#include <cstdio>
-#include <cstring>
-#include <memory>
-#include <string>
-#include <unordered_map>
-#include <vector>
-#include "util/coding.h"
-#include "utilities/col_buf_encoder.h"
-
-namespace rocksdb {
-
-struct ColDeclaration;
-
-// ColBufDecoder is a class to decode column buffers. It can be populated from a
-// ColDeclaration. Before starting decoding, a Init() method should be called.
-// Each time it takes a column value into Decode() method.
-class ColBufDecoder {
- public:
-  virtual ~ColBufDecoder() = 0;
-  virtual size_t Init(const char* /*src*/) { return 0; }
-  virtual size_t Decode(const char* src, char** dest) = 0;
-  static ColBufDecoder* NewColBufDecoder(const ColDeclaration& col_declaration);
-
- protected:
-  std::string buffer_;
-  static inline bool IsRunLength(ColCompressionType type) {
-    return type == kColRle || type == kColRleVarint ||
-           type == kColRleDeltaVarint || type == kColRleDict;
-  }
-};
-
-class FixedLengthColBufDecoder : public ColBufDecoder {
- public:
-  explicit FixedLengthColBufDecoder(
-      size_t size, ColCompressionType col_compression_type = kColNoCompression,
-      bool nullable = false, bool big_endian = false)
-      : size_(size),
-        col_compression_type_(col_compression_type),
-        nullable_(nullable),
-        big_endian_(big_endian),
-        remain_runs_(0),
-        run_val_(0),
-        last_val_(0) {}
-
-  size_t Init(const char* src) override;
-  size_t Decode(const char* src, char** dest) override;
-  ~FixedLengthColBufDecoder() {}
-
- private:
-  size_t size_;
-  ColCompressionType col_compression_type_;
-  bool nullable_;
-  bool big_endian_;
-
-  // for decoding
-  std::vector<uint64_t> dict_vec_;
-  uint64_t remain_runs_;
-  uint64_t run_val_;
-  uint64_t last_val_;
-};
-
-class LongFixedLengthColBufDecoder : public ColBufDecoder {
- public:
-  LongFixedLengthColBufDecoder(size_t size, bool nullable)
-      : size_(size), nullable_(nullable) {}
-
-  size_t Decode(const char* src, char** dest) override;
-  ~LongFixedLengthColBufDecoder() {}
-
- private:
-  size_t size_;
-  bool nullable_;
-};
-
-class VariableLengthColBufDecoder : public ColBufDecoder {
- public:
-  size_t Decode(const char* src, char** dest) override;
-  ~VariableLengthColBufDecoder() {}
-};
-
-class VariableChunkColBufDecoder : public VariableLengthColBufDecoder {
- public:
-  size_t Init(const char* src) override;
-  size_t Decode(const char* src, char** dest) override;
-  explicit VariableChunkColBufDecoder(ColCompressionType col_compression_type)
-      : col_compression_type_(col_compression_type) {}
-  VariableChunkColBufDecoder() : col_compression_type_(kColNoCompression) {}
-
- private:
-  ColCompressionType col_compression_type_;
-  std::unordered_map<uint64_t, uint64_t> dictionary_;
-  std::vector<uint64_t> dict_vec_;
-};
-
-struct KVPairColBufDecoders {
-  std::vector<std::unique_ptr<ColBufDecoder>> key_col_bufs;
-  std::vector<std::unique_ptr<ColBufDecoder>> value_col_bufs;
-  std::unique_ptr<ColBufDecoder> value_checksum_buf;
-
-  explicit KVPairColBufDecoders(const KVPairColDeclarations& kvp_cd) {
-    for (auto kcd : *kvp_cd.key_col_declarations) {
-      key_col_bufs.emplace_back(
-          std::move(ColBufDecoder::NewColBufDecoder(kcd)));
-    }
-    for (auto vcd : *kvp_cd.value_col_declarations) {
-      value_col_bufs.emplace_back(
-          std::move(ColBufDecoder::NewColBufDecoder(vcd)));
-    }
-    value_checksum_buf.reset(
-        ColBufDecoder::NewColBufDecoder(*kvp_cd.value_checksum_declaration));
-  }
-};
-}  // namespace rocksdb

utilities/col_buf_encoder.cc

@@ -1,210 +0,0 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "utilities/col_buf_encoder.h"
-#include <cstring>
-#include <string>
-#include "port/port.h"
-
-namespace rocksdb {
-
-ColBufEncoder::~ColBufEncoder() {}
-
-namespace {
-
-inline uint64_t DecodeFixed64WithEndian(uint64_t val, bool big_endian,
-                                        size_t size) {
-  if (big_endian && port::kLittleEndian) {
-    val = EndianTransform(val, size);
-  } else if (!big_endian && !port::kLittleEndian) {
-    val = EndianTransform(val, size);
-  }
-  return val;
-}
-
-}  // namespace
-
-const std::string &ColBufEncoder::GetData() { return buffer_; }
-
-ColBufEncoder *ColBufEncoder::NewColBufEncoder(
-    const ColDeclaration &col_declaration) {
-  if (col_declaration.col_type == "FixedLength") {
-    return new FixedLengthColBufEncoder(
-        col_declaration.size, col_declaration.col_compression_type,
-        col_declaration.nullable, col_declaration.big_endian);
-  } else if (col_declaration.col_type == "VariableLength") {
-    return new VariableLengthColBufEncoder();
-  } else if (col_declaration.col_type == "VariableChunk") {
-    return new VariableChunkColBufEncoder(col_declaration.col_compression_type);
-  } else if (col_declaration.col_type == "LongFixedLength") {
-    return new LongFixedLengthColBufEncoder(col_declaration.size,
-                                            col_declaration.nullable);
-  }
-  // Unrecognized column type
-  return nullptr;
-}
-
-size_t FixedLengthColBufEncoder::Append(const char *buf) {
-  if (nullable_) {
-    if (buf == nullptr) {
-      buffer_.append(1, 0);
-      return 0;
-    } else {
-      buffer_.append(1, 1);
-    }
-  }
-  uint64_t read_val = 0;
-  memcpy(&read_val, buf, size_);
-  read_val = DecodeFixed64WithEndian(read_val, big_endian_, size_);
-
-  // Determine write value
-  uint64_t write_val = read_val;
-  if (col_compression_type_ == kColDeltaVarint ||
-      col_compression_type_ == kColRleDeltaVarint) {
-    int64_t delta = read_val - last_val_;
-    // Encode signed delta value
-    delta = (static_cast<uint64_t>(delta) << 1) ^ (delta >> 63);
-    write_val = delta;
-    last_val_ = read_val;
-  } else if (col_compression_type_ == kColDict ||
-             col_compression_type_ == kColRleDict) {
-    auto iter = dictionary_.find(read_val);
-    uint64_t dict_val;
-    if (iter == dictionary_.end()) {
-      // Add new entry to dictionary
-      dict_val = dictionary_.size();
-      dictionary_.insert(std::make_pair(read_val, dict_val));
-      dict_vec_.push_back(read_val);
-    } else {
-      dict_val = iter->second;
-    }
-    write_val = dict_val;
-  }
-
-  // Write into buffer
-  if (IsRunLength(col_compression_type_)) {
-    if (run_length_ == -1) {
-      // First element
-      run_val_ = write_val;
-      run_length_ = 1;
-    } else if (write_val != run_val_) {
-      // End of run
-      // Write run value
-      if (col_compression_type_ == kColRle) {
-        buffer_.append(reinterpret_cast<char *>(&run_val_), size_);
-      } else {
-        PutVarint64(&buffer_, run_val_);
-      }
-      // Write run length
-      PutVarint64(&buffer_, run_length_);
-      run_val_ = write_val;
-      run_length_ = 1;
-    } else {
-      run_length_++;
-    }
-  } else {  // non run-length encodings
-    if (col_compression_type_ == kColNoCompression) {
-      buffer_.append(reinterpret_cast<char *>(&write_val), size_);
-    } else {
-      PutVarint64(&buffer_, write_val);
-    }
-  }
-  return size_;
-}
-
-void FixedLengthColBufEncoder::Finish() {
-  if (col_compression_type_ == kColDict ||
-      col_compression_type_ == kColRleDict) {
-    std::string header;
-    PutVarint64(&header, dict_vec_.size());
-    // Put dictionary in the header
-    for (auto item : dict_vec_) {
-      PutVarint64(&header, item);
-    }
-    buffer_ = header + buffer_;
-  }
-  if (IsRunLength(col_compression_type_)) {
-    // Finish last run value
-    if (col_compression_type_ == kColRle) {
-      buffer_.append(reinterpret_cast<char *>(&run_val_), size_);
-    } else {
-      PutVarint64(&buffer_, run_val_);
-    }
-    PutVarint64(&buffer_, run_length_);
-  }
-}
-
-size_t LongFixedLengthColBufEncoder::Append(const char *buf) {
-  if (nullable_) {
-    if (buf == nullptr) {
-      buffer_.append(1, 0);
-      return 0;
-    } else {
-      buffer_.append(1, 1);
-    }
-  }
-  buffer_.append(buf, size_);
-  return size_;
-}
-
-void LongFixedLengthColBufEncoder::Finish() {}
-
-size_t VariableLengthColBufEncoder::Append(const char *buf) {
-  uint8_t length = 0;
-  length = *buf;
-  buffer_.append(buf, 1);
-  buf += 1;
-  buffer_.append(buf, length);
-  return length + 1;
-}
-
-void VariableLengthColBufEncoder::Finish() {}
-
-size_t VariableChunkColBufEncoder::Append(const char *buf) {
-  const char *orig_buf = buf;
-  uint8_t mark = 0xFF;
-  size_t length = 0;
-  std::string tmp_buffer;
-  while (mark == 0xFF) {
-    uint64_t val;
-    memcpy(&val, buf, 8);
-    buf += 8;
-    mark = *buf;
-    buf += 1;
-    int8_t chunk_size = 8 - (0xFF - mark);
-    if (col_compression_type_ == kColDict) {
-      auto iter = dictionary_.find(val);
-      uint64_t dict_val;
-      if (iter == dictionary_.end()) {
-        dict_val = dictionary_.size();
-        dictionary_.insert(std::make_pair(val, dict_val));
-        dict_vec_.push_back(val);
-      } else {
-        dict_val = iter->second;
-      }
-      PutVarint64(&tmp_buffer, dict_val);
-    } else {
-      tmp_buffer.append(reinterpret_cast<char *>(&val), chunk_size);
-    }
-    length += chunk_size;
-  }
-
-  PutVarint64(&buffer_, length);
-  buffer_.append(tmp_buffer);
-  return buf - orig_buf;
-}
-
-void VariableChunkColBufEncoder::Finish() {
-  if (col_compression_type_ == kColDict) {
-    std::string header;
-    PutVarint64(&header, dict_vec_.size());
-    for (auto item : dict_vec_) {
-      PutVarint64(&header, item);
-    }
-    buffer_ = header + buffer_;
-  }
-}
-
-}  // namespace rocksdb

utilities/col_buf_encoder.h

@@ -1,219 +0,0 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-#include <cstdio>
-#include <cstring>
-#include <memory>
-#include <string>
-#include <unordered_map>
-#include <vector>
-#include "util/coding.h"
-
-namespace rocksdb {
-
-enum ColCompressionType {
-  kColNoCompression,
-  kColRle,
-  kColVarint,
-  kColRleVarint,
-  kColDeltaVarint,
-  kColRleDeltaVarint,
-  kColDict,
-  kColRleDict
-};
-
-struct ColDeclaration;
-
-// ColBufEncoder is a class to encode column buffers. It can be populated from a
-// ColDeclaration. Each time it takes a column value into Append() method to
-// encode the column and store it into an internal buffer. After all rows for
-// this column are consumed, a Finish() should be called to add header and
-// remaining data.
-class ColBufEncoder {
- public:
-  // Read a column, encode data and append into internal buffer.
-  virtual size_t Append(const char *buf) = 0;
-  virtual ~ColBufEncoder() = 0;
-  // Get the internal column buffer. Should only be called after Finish().
-  const std::string &GetData();
-  // Finish encoding. Add header and remaining data.
-  virtual void Finish() = 0;
-  // Populate a ColBufEncoder from ColDeclaration.
-  static ColBufEncoder *NewColBufEncoder(const ColDeclaration &col_declaration);
-
- protected:
-  std::string buffer_;
-  static inline bool IsRunLength(ColCompressionType type) {
-    return type == kColRle || type == kColRleVarint ||
-           type == kColRleDeltaVarint || type == kColRleDict;
-  }
-};
-
-// Encoder for fixed length column buffer. In fixed length column buffer, the
-// size of the column should not exceed 8 bytes.
-// The following encodings are supported:
-// Varint: Variable length integer. See util/coding.h for more details
-// Rle (Run length encoding): encode a sequence of contiguous value as
-// [run_value][run_length]. Can be combined with Varint
-// Delta: Encode value to its delta with its adjacent entry. Use varint to
-// possibly reduce stored bytes. Can be combined with Rle.
-// Dictionary: Use a dictionary to record all possible values in the block and
-// encode them with an ID started from 0. IDs are encoded as varint. A column
-// with dictionary encoding will have a header to store all actual values,
-// ordered by their dictionary value, and the data will be replaced by
-// dictionary value. Can be combined with Rle.
-class FixedLengthColBufEncoder : public ColBufEncoder {
- public:
-  explicit FixedLengthColBufEncoder(
-      size_t size, ColCompressionType col_compression_type = kColNoCompression,
-      bool nullable = false, bool big_endian = false)
-      : size_(size),
-        col_compression_type_(col_compression_type),
-        nullable_(nullable),
-        big_endian_(big_endian),
-        last_val_(0),
-        run_length_(-1),
-        run_val_(0) {}
-
-  size_t Append(const char *buf) override;
-  void Finish() override;
-  ~FixedLengthColBufEncoder() {}
-
- private:
-  size_t size_;
-  ColCompressionType col_compression_type_;
-  // If set as true, the input value can be null (represented as nullptr). When
-  // nullable is true, use one more byte before actual value to indicate if the
-  // current value is null.
-  bool nullable_;
-  // If set as true, input value will be treated as big endian encoded.
-  bool big_endian_;
-
-  // for encoding
-  uint64_t last_val_;
-  int16_t run_length_;
-  uint64_t run_val_;
-  // Map to store dictionary for dictionary encoding
-  std::unordered_map<uint64_t, uint64_t> dictionary_;
-  // Vector of dictionary keys.
-  std::vector<uint64_t> dict_vec_;
-};
-
-// Long fixed length column buffer is a variant of fixed length buffer to hold
-// fixed length buffer with more than 8 bytes. We do not support any special
-// encoding schemes in LongFixedLengthColBufEncoder.
-class LongFixedLengthColBufEncoder : public ColBufEncoder {
- public:
-  LongFixedLengthColBufEncoder(size_t size, bool nullable)
-      : size_(size), nullable_(nullable) {}
-  size_t Append(const char *buf) override;
-  void Finish() override;
-
-  ~LongFixedLengthColBufEncoder() {}
-
- private:
-  size_t size_;
-  bool nullable_;
-};
-
-// Variable length column buffer holds a format of variable length column. In
-// this format, a column is composed of one byte length k, followed by data with
-// k bytes long data.
-class VariableLengthColBufEncoder : public ColBufEncoder {
- public:
-  size_t Append(const char *buf) override;
-  void Finish() override;
-
-  ~VariableLengthColBufEncoder() {}
-};
-
-// Variable chunk column buffer holds another format of variable length column.
-// In this format, a column contains multiple chunks of data, each of which is
-// composed of 8 bytes long data, and one byte as a mask to indicate whether we
-// have more data to come. If no more data coming, the mask is set as 0xFF. If
-// the chunk is the last chunk and has only k valid bytes, the mask is set as
-// 0xFF - (8 - k).
-class VariableChunkColBufEncoder : public VariableLengthColBufEncoder {
- public:
-  size_t Append(const char *buf) override;
-  void Finish() override;
-  explicit VariableChunkColBufEncoder(ColCompressionType col_compression_type)
-      : col_compression_type_(col_compression_type) {}
-  VariableChunkColBufEncoder() : col_compression_type_(kColNoCompression) {}
-
- private:
-  ColCompressionType col_compression_type_;
-  // Map to store dictionary for dictionary encoding
-  std::unordered_map<uint64_t, uint64_t> dictionary_;
-  // Vector of dictionary keys.
-  std::vector<uint64_t> dict_vec_;
-};
-
-// ColDeclaration declares a column's type, algorithm of column-aware encoding,
-// and other column data like endian and nullability.
-struct ColDeclaration {
-  explicit ColDeclaration(
-      std::string _col_type,
-      ColCompressionType _col_compression_type = kColNoCompression,
-      size_t _size = 0, bool _nullable = false, bool _big_endian = false)
-      : col_type(_col_type),
-        col_compression_type(_col_compression_type),
-        size(_size),
-        nullable(_nullable),
-        big_endian(_big_endian) {}
-  std::string col_type;
-  ColCompressionType col_compression_type;
-  size_t size;
-  bool nullable;
-  bool big_endian;
-};
-
-// KVPairColDeclarations is a class to hold column declaration of columns in
-// key and value.
-struct KVPairColDeclarations {
-  std::vector<ColDeclaration> *key_col_declarations;
-  std::vector<ColDeclaration> *value_col_declarations;
-  ColDeclaration *value_checksum_declaration;
-  KVPairColDeclarations(std::vector<ColDeclaration> *_key_col_declarations,
-                        std::vector<ColDeclaration> *_value_col_declarations,
-                        ColDeclaration *_value_checksum_declaration)
-      : key_col_declarations(_key_col_declarations),
-        value_col_declarations(_value_col_declarations),
-        value_checksum_declaration(_value_checksum_declaration) {}
-};
-
-// Similar to KVPairDeclarations, KVPairColBufEncoders is used to hold column
-// buffer encoders of all columns in key and value.
-struct KVPairColBufEncoders {
-  std::vector<std::unique_ptr<ColBufEncoder>> key_col_bufs;
-  std::vector<std::unique_ptr<ColBufEncoder>> value_col_bufs;
-  std::unique_ptr<ColBufEncoder> value_checksum_buf;
-
-  explicit KVPairColBufEncoders(const KVPairColDeclarations &kvp_cd) {
-    for (auto kcd : *kvp_cd.key_col_declarations) {
-      key_col_bufs.emplace_back(
-          std::move(ColBufEncoder::NewColBufEncoder(kcd)));
-    }
-    for (auto vcd : *kvp_cd.value_col_declarations) {
-      value_col_bufs.emplace_back(
-          std::move(ColBufEncoder::NewColBufEncoder(vcd)));
-    }
-    value_checksum_buf.reset(
-        ColBufEncoder::NewColBufEncoder(*kvp_cd.value_checksum_declaration));
-  }
-
-  // Helper function to call Finish()
-  void Finish() {
-    for (auto &col_buf : key_col_bufs) {
-      col_buf->Finish();
-    }
-    for (auto &col_buf : value_col_bufs) {
-      col_buf->Finish();
-    }
-    value_checksum_buf->Finish();
-  }
-};
-}  // namespace rocksdb

utilities/column_aware_encoding_exp.cc

@@ -1,176 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <cstdio>
-#include <cstdlib>
-
-#ifndef ROCKSDB_LITE
-#ifdef GFLAGS
-
-#include <inttypes.h>
-#include <vector>
-#include "rocksdb/env.h"
-#include "rocksdb/options.h"
-#include "table/block_based_table_builder.h"
-#include "table/block_based_table_reader.h"
-#include "table/format.h"
-#include "tools/sst_dump_tool_imp.h"
-#include "util/compression.h"
-#include "util/gflags_compat.h"
-#include "util/stop_watch.h"
-#include "utilities/col_buf_encoder.h"
-#include "utilities/column_aware_encoding_util.h"
-
-using GFLAGS_NAMESPACE::ParseCommandLineFlags;
-DEFINE_string(encoded_file, "", "file to store encoded data blocks");
-DEFINE_string(decoded_file, "",
-              "file to store decoded data blocks after encoding");
-DEFINE_string(format, "col", "Output Format. Can be 'row' or 'col'");
-// TODO(jhli): option `col` should be removed and replaced by general
-// column specifications.
-DEFINE_string(index_type, "col", "Index type. Can be 'primary' or 'secondary'");
-DEFINE_string(dump_file, "",
-              "Dump data blocks separated by columns in human-readable format");
-DEFINE_bool(decode, false, "Deocde blocks after they are encoded");
-DEFINE_bool(stat, false,
-            "Print column distribution statistics. Cannot decode in this mode");
-DEFINE_string(compression_type, "kNoCompression",
-              "The compression algorithm used to compress data blocks");
-
-namespace rocksdb {
-
-class ColumnAwareEncodingExp {
- public:
-  static void Run(const std::string& sst_file) {
-    bool decode = FLAGS_decode;
-    if (FLAGS_decoded_file.size() > 0) {
-      decode = true;
-    }
-    if (FLAGS_stat) {
-      decode = false;
-    }
-
-    ColumnAwareEncodingReader reader(sst_file);
-    std::vector<ColDeclaration>* key_col_declarations;
-    std::vector<ColDeclaration>* value_col_declarations;
-    ColDeclaration* value_checksum_declaration;
-    if (FLAGS_index_type == "primary") {
-      ColumnAwareEncodingReader::GetColDeclarationsPrimary(
-          &key_col_declarations, &value_col_declarations,
-          &value_checksum_declaration);
-    } else {
-      ColumnAwareEncodingReader::GetColDeclarationsSecondary(
-          &key_col_declarations, &value_col_declarations,
-          &value_checksum_declaration);
-    }
-    KVPairColDeclarations kvp_cd(key_col_declarations, value_col_declarations,
-                                 value_checksum_declaration);
-
-    if (!FLAGS_dump_file.empty()) {
-      std::vector<KVPairBlock> kv_pair_blocks;
-      reader.GetKVPairsFromDataBlocks(&kv_pair_blocks);
-      reader.DumpDataColumns(FLAGS_dump_file, kvp_cd, kv_pair_blocks);
-      return;
-    }
-    std::unordered_map<std::string, CompressionType> compressions = {
-        {"kNoCompression", CompressionType::kNoCompression},
-        {"kZlibCompression", CompressionType::kZlibCompression},
-        {"kZSTD", CompressionType::kZSTD}};
-
-    // Find Compression
-    CompressionType compression_type = compressions[FLAGS_compression_type];
-    EnvOptions env_options;
-    if (CompressionTypeSupported(compression_type)) {
-      fprintf(stdout, "[%s]\n", FLAGS_compression_type.c_str());
-      std::unique_ptr<WritableFile> encoded_out_file;
-
-      std::unique_ptr<Env> env(NewMemEnv(Env::Default()));
-      if (!FLAGS_encoded_file.empty()) {
-        env->NewWritableFile(FLAGS_encoded_file, &encoded_out_file,
-                             env_options);
-      }
-
-      std::vector<KVPairBlock> kv_pair_blocks;
-      reader.GetKVPairsFromDataBlocks(&kv_pair_blocks);
-
-      std::vector<std::string> encoded_blocks;
-      StopWatchNano sw(env.get(), true);
-      if (FLAGS_format == "col") {
-        reader.EncodeBlocks(kvp_cd, encoded_out_file.get(), compression_type,
-                            kv_pair_blocks, &encoded_blocks, FLAGS_stat);
-      } else {  // row format
-        reader.EncodeBlocksToRowFormat(encoded_out_file.get(), compression_type,
-                                       kv_pair_blocks, &encoded_blocks);
-      }
-      if (encoded_out_file != nullptr) {
-        uint64_t size = 0;
-        env->GetFileSize(FLAGS_encoded_file, &size);
-        fprintf(stdout, "File size: %" PRIu64 "\n", size);
-      }
-      uint64_t encode_time = sw.ElapsedNanosSafe(false /* reset */);
-      fprintf(stdout, "Encode time: %" PRIu64 "\n", encode_time);
-      if (decode) {
-        std::unique_ptr<WritableFile> decoded_out_file;
-        if (!FLAGS_decoded_file.empty()) {
-          env->NewWritableFile(FLAGS_decoded_file, &decoded_out_file,
-                               env_options);
-        }
-        sw.Start();
-        if (FLAGS_format == "col") {
-          reader.DecodeBlocks(kvp_cd, decoded_out_file.get(), &encoded_blocks);
-        } else {
-          reader.DecodeBlocksFromRowFormat(decoded_out_file.get(),
-                                           &encoded_blocks);
-        }
-        uint64_t decode_time = sw.ElapsedNanosSafe(true /* reset */);
-        fprintf(stdout, "Decode time: %" PRIu64 "\n", decode_time);
-      }
-    } else {
-      fprintf(stdout, "Unsupported compression type: %s.\n",
-              FLAGS_compression_type.c_str());
-    }
-    delete key_col_declarations;
-    delete value_col_declarations;
-    delete value_checksum_declaration;
-  }
-};
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  int arg_idx = ParseCommandLineFlags(&argc, &argv, true);
-  if (arg_idx >= argc) {
-    fprintf(stdout, "SST filename required.\n");
-    exit(1);
-  }
-  std::string sst_file(argv[arg_idx]);
-  if (FLAGS_format != "row" && FLAGS_format != "col") {
-    fprintf(stderr, "Format must be 'row' or 'col'\n");
-    exit(1);
-  }
-  if (FLAGS_index_type != "primary" && FLAGS_index_type != "secondary") {
-    fprintf(stderr, "Format must be 'primary' or 'secondary'\n");
-    exit(1);
-  }
-  rocksdb::ColumnAwareEncodingExp::Run(sst_file);
-  return 0;
-}
-
-#else
-int main() {
-  fprintf(stderr, "Please install gflags to run rocksdb tools\n");
-  return 1;
-}
-#endif  // GFLAGS
-#else
-int main(int /*argc*/, char** /*argv*/) {
-  fprintf(stderr, "Not supported in lite mode.\n");
-  return 1;
-}
-#endif  // ROCKSDB_LITE

utilities/column_aware_encoding_test.cc

@@ -1,254 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-#ifndef ROCKSDB_LITE
-
-#include <vector>
-#include "util/testharness.h"
-#include "util/testutil.h"
-#include "utilities/col_buf_decoder.h"
-#include "utilities/col_buf_encoder.h"
-
-namespace rocksdb {
-
-class ColumnAwareEncodingTest : public testing::Test {
- public:
-  ColumnAwareEncodingTest() {}
-
-  ~ColumnAwareEncodingTest() {}
-};
-
-class ColumnAwareEncodingTestWithSize
-    : public ColumnAwareEncodingTest,
-      public testing::WithParamInterface<size_t> {
- public:
-  ColumnAwareEncodingTestWithSize() {}
-
-  ~ColumnAwareEncodingTestWithSize() {}
-
-  static std::vector<size_t> GetValues() { return {4, 8}; }
-};
-
-INSTANTIATE_TEST_CASE_P(
-    ColumnAwareEncodingTestWithSize, ColumnAwareEncodingTestWithSize,
-    ::testing::ValuesIn(ColumnAwareEncodingTestWithSize::GetValues()));
-
-TEST_P(ColumnAwareEncodingTestWithSize, NoCompressionEncodeDecode) {
-  size_t col_size = GetParam();
-  std::unique_ptr<ColBufEncoder> col_buf_encoder(
-      new FixedLengthColBufEncoder(col_size, kColNoCompression, false, true));
-  std::string str_buf;
-  uint64_t base_val = 0x0102030405060708;
-  uint64_t val = 0;
-  memcpy(&val, &base_val, col_size);
-  const int row_count = 4;
-  for (int i = 0; i < row_count; ++i) {
-    str_buf.append(reinterpret_cast<char*>(&val), col_size);
-  }
-  const char* str_buf_ptr = str_buf.c_str();
-  for (int i = 0; i < row_count; ++i) {
-    col_buf_encoder->Append(str_buf_ptr);
-  }
-  col_buf_encoder->Finish();
-  const std::string& encoded_data = col_buf_encoder->GetData();
-  // Check correctness of encoded string length
-  ASSERT_EQ(row_count * col_size, encoded_data.size());
-
-  const char* encoded_data_ptr = encoded_data.c_str();
-  uint64_t expected_encoded_val;
-  if (col_size == 8) {
-    expected_encoded_val = port::kLittleEndian ? 0x0807060504030201 : 0x0102030405060708;
-  } else if (col_size == 4) {
-    expected_encoded_val = port::kLittleEndian ? 0x08070605 : 0x0102030400000000;
-  }
-  uint64_t encoded_val = 0;
-  for (int i = 0; i < row_count; ++i) {
-    memcpy(&encoded_val, encoded_data_ptr, col_size);
-    // Check correctness of encoded value
-    ASSERT_EQ(expected_encoded_val, encoded_val);
-    encoded_data_ptr += col_size;
-  }
-
-  std::unique_ptr<ColBufDecoder> col_buf_decoder(
-      new FixedLengthColBufDecoder(col_size, kColNoCompression, false, true));
-  encoded_data_ptr = encoded_data.c_str();
-  encoded_data_ptr += col_buf_decoder->Init(encoded_data_ptr);
-  char* decoded_data = new char[100];
-  char* decoded_data_base = decoded_data;
-  for (int i = 0; i < row_count; ++i) {
-    encoded_data_ptr +=
-        col_buf_decoder->Decode(encoded_data_ptr, &decoded_data);
-  }
-
-  // Check correctness of decoded string length
-  ASSERT_EQ(row_count * col_size, decoded_data - decoded_data_base);
-  decoded_data = decoded_data_base;
-  for (int i = 0; i < row_count; ++i) {
-    uint64_t decoded_val;
-    decoded_val = 0;
-    memcpy(&decoded_val, decoded_data, col_size);
-    // Check correctness of decoded value
-    ASSERT_EQ(val, decoded_val);
-    decoded_data += col_size;
-  }
-  delete[] decoded_data_base;
-}
-
-TEST_P(ColumnAwareEncodingTestWithSize, RleEncodeDecode) {
-  size_t col_size = GetParam();
-  std::unique_ptr<ColBufEncoder> col_buf_encoder(
-      new FixedLengthColBufEncoder(col_size, kColRle, false, true));
-  std::string str_buf;
-  uint64_t base_val = 0x0102030405060708;
-  uint64_t val = 0;
-  memcpy(&val, &base_val, col_size);
-  const int row_count = 4;
-  for (int i = 0; i < row_count; ++i) {
-    str_buf.append(reinterpret_cast<char*>(&val), col_size);
-  }
-  const char* str_buf_ptr = str_buf.c_str();
-  for (int i = 0; i < row_count; ++i) {
-    str_buf_ptr += col_buf_encoder->Append(str_buf_ptr);
-  }
-  col_buf_encoder->Finish();
-  const std::string& encoded_data = col_buf_encoder->GetData();
-  // Check correctness of encoded string length
-  ASSERT_EQ(col_size + 1, encoded_data.size());
-
-  const char* encoded_data_ptr = encoded_data.c_str();
-  uint64_t encoded_val = 0;
-  memcpy(&encoded_val, encoded_data_ptr, col_size);
-  uint64_t expected_encoded_val;
-  if (col_size == 8) {
-    expected_encoded_val = port::kLittleEndian ? 0x0807060504030201 : 0x0102030405060708;
-  } else if (col_size == 4) {
-    expected_encoded_val = port::kLittleEndian ? 0x08070605 : 0x0102030400000000;
-  }
-  // Check correctness of encoded value
-  ASSERT_EQ(expected_encoded_val, encoded_val);
-
-  std::unique_ptr<ColBufDecoder> col_buf_decoder(
-      new FixedLengthColBufDecoder(col_size, kColRle, false, true));
-  char* decoded_data = new char[100];
-  char* decoded_data_base = decoded_data;
-  encoded_data_ptr += col_buf_decoder->Init(encoded_data_ptr);
-  for (int i = 0; i < row_count; ++i) {
-    encoded_data_ptr +=
-        col_buf_decoder->Decode(encoded_data_ptr, &decoded_data);
-  }
-  // Check correctness of decoded string length
-  ASSERT_EQ(decoded_data - decoded_data_base, row_count * col_size);
-  decoded_data = decoded_data_base;
-  for (int i = 0; i < row_count; ++i) {
-    uint64_t decoded_val;
-    decoded_val = 0;
-    memcpy(&decoded_val, decoded_data, col_size);
-    // Check correctness of decoded value
-    ASSERT_EQ(val, decoded_val);
-    decoded_data += col_size;
-  }
-  delete[] decoded_data_base;
-}
-
-TEST_P(ColumnAwareEncodingTestWithSize, DeltaEncodeDecode) {
-  size_t col_size = GetParam();
-  int row_count = 4;
-  std::unique_ptr<ColBufEncoder> col_buf_encoder(
-      new FixedLengthColBufEncoder(col_size, kColDeltaVarint, false, true));
-  std::string str_buf;
-  uint64_t base_val1 = port::kLittleEndian ? 0x0102030405060708 : 0x0807060504030201;
-  uint64_t base_val2 = port::kLittleEndian ? 0x0202030405060708 : 0x0807060504030202;
-  uint64_t val1 = 0, val2 = 0;
-  memcpy(&val1, &base_val1, col_size);
-  memcpy(&val2, &base_val2, col_size);
-  const char* str_buf_ptr;
-  for (int i = 0; i < row_count / 2; ++i) {
-    str_buf = std::string(reinterpret_cast<char*>(&val1), col_size);
-    str_buf_ptr = str_buf.c_str();
-    col_buf_encoder->Append(str_buf_ptr);
-
-    str_buf = std::string(reinterpret_cast<char*>(&val2), col_size);
-    str_buf_ptr = str_buf.c_str();
-    col_buf_encoder->Append(str_buf_ptr);
-  }
-  col_buf_encoder->Finish();
-  const std::string& encoded_data = col_buf_encoder->GetData();
-  // Check encoded string length
-  int varint_len = 0;
-  if (col_size == 8) {
-    varint_len = 9;
-  } else if (col_size == 4) {
-    varint_len = port::kLittleEndian ? 5 : 9;
-  }
-  // Check encoded string length: first value is original one (val - 0), the
-  // coming three are encoded as 1, -1, 1, so they should take 1 byte in varint.
-  ASSERT_EQ(varint_len + 3 * 1, encoded_data.size());
-
-  std::unique_ptr<ColBufDecoder> col_buf_decoder(
-      new FixedLengthColBufDecoder(col_size, kColDeltaVarint, false, true));
-  char* decoded_data = new char[100];
-  char* decoded_data_base = decoded_data;
-  const char* encoded_data_ptr = encoded_data.c_str();
-  encoded_data_ptr += col_buf_decoder->Init(encoded_data_ptr);
-  for (int i = 0; i < row_count; ++i) {
-    encoded_data_ptr +=
-        col_buf_decoder->Decode(encoded_data_ptr, &decoded_data);
-  }
-
-  // Check correctness of decoded string length
-  ASSERT_EQ(row_count * col_size, decoded_data - decoded_data_base);
-  decoded_data = decoded_data_base;
-
-  // Check correctness of decoded data
-  for (int i = 0; i < row_count / 2; ++i) {
-    uint64_t decoded_val = 0;
-    memcpy(&decoded_val, decoded_data, col_size);
-    ASSERT_EQ(val1, decoded_val);
-    decoded_data += col_size;
-    memcpy(&decoded_val, decoded_data, col_size);
-    ASSERT_EQ(val2, decoded_val);
-    decoded_data += col_size;
-  }
-  delete[] decoded_data_base;
-}
-
-TEST_F(ColumnAwareEncodingTest, ChunkBufEncodeDecode) {
-  std::unique_ptr<ColBufEncoder> col_buf_encoder(
-      new VariableChunkColBufEncoder(kColDict));
-  std::string buf("12345678\377\1\0\0\0\0\0\0\0\376", 18);
-  col_buf_encoder->Append(buf.c_str());
-  col_buf_encoder->Finish();
-  const std::string& encoded_data = col_buf_encoder->GetData();
-  const char* str_ptr = encoded_data.c_str();
-
-  std::unique_ptr<ColBufDecoder> col_buf_decoder(
-      new VariableChunkColBufDecoder(kColDict));
-  str_ptr += col_buf_decoder->Init(str_ptr);
-  char* decoded_data = new char[100];
-  char* decoded_data_base = decoded_data;
-  col_buf_decoder->Decode(str_ptr, &decoded_data);
-  for (size_t i = 0; i < buf.size(); ++i) {
-    ASSERT_EQ(buf[i], decoded_data_base[i]);
-  }
-  delete[] decoded_data_base;
-}
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
-
-#else
-
-#include <cstdio>
-
-int main() {
-  fprintf(stderr,
-          "SKIPPED as column aware encoding experiment is not enabled in "
-          "ROCKSDB_LITE\n");
-}
-#endif  // ROCKSDB_LITE

utilities/column_aware_encoding_util.cc

@@ -1,491 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-#ifndef ROCKSDB_LITE
-
-#include "utilities/column_aware_encoding_util.h"
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <stddef.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <algorithm>
-#include <utility>
-#include <vector>
-#include "include/rocksdb/comparator.h"
-#include "include/rocksdb/slice.h"
-#include "rocksdb/env.h"
-#include "rocksdb/status.h"
-#include "table/block_based_table_builder.h"
-#include "table/block_based_table_factory.h"
-#include "table/format.h"
-#include "table/table_reader.h"
-#include "util/cast_util.h"
-#include "util/coding.h"
-#include "utilities/col_buf_decoder.h"
-#include "utilities/col_buf_encoder.h"
-
-#include "port/port.h"
-
-namespace rocksdb {
-
-ColumnAwareEncodingReader::ColumnAwareEncodingReader(
-    const std::string& file_path)
-    : file_name_(file_path),
-      ioptions_(options_),
-      moptions_(options_),
-      internal_comparator_(BytewiseComparator()) {
-  InitTableReader(file_name_);
-}
-
-void ColumnAwareEncodingReader::InitTableReader(const std::string& file_path) {
-  std::unique_ptr<RandomAccessFile> file;
-  uint64_t file_size;
-  options_.env->NewRandomAccessFile(file_path, &file, soptions_);
-  options_.env->GetFileSize(file_path, &file_size);
-
-  file_.reset(new RandomAccessFileReader(std::move(file), file_path));
-
-  options_.comparator = &internal_comparator_;
-  options_.table_factory = std::make_shared<BlockBasedTableFactory>();
-
-  std::unique_ptr<TableReader> table_reader;
-  options_.table_factory->NewTableReader(
-      TableReaderOptions(ioptions_, moptions_.prefix_extractor.get(), soptions_,
-                         internal_comparator_),
-      std::move(file_), file_size, &table_reader, /*enable_prefetch=*/false);
-
-  table_reader_.reset(static_cast_with_check<BlockBasedTable, TableReader>(
-      table_reader.release()));
-}
-
-void ColumnAwareEncodingReader::GetKVPairsFromDataBlocks(
-    std::vector<KVPairBlock>* kv_pair_blocks) {
-  table_reader_->GetKVPairsFromDataBlocks(kv_pair_blocks);
-}
-
-void ColumnAwareEncodingReader::DecodeBlocks(
-    const KVPairColDeclarations& kvp_col_declarations, WritableFile* out_file,
-    const std::vector<std::string>* blocks) {
-  char* decoded_content_base = new char[16384];
-  Options options;
-  ImmutableCFOptions ioptions(options);
-  for (auto& block : *blocks) {
-    KVPairColBufDecoders kvp_col_bufs(kvp_col_declarations);
-    auto& key_col_bufs = kvp_col_bufs.key_col_bufs;
-    auto& value_col_bufs = kvp_col_bufs.value_col_bufs;
-    auto& value_checksum_buf = kvp_col_bufs.value_checksum_buf;
-
-    auto& slice_final_with_bit = block;
-    uint32_t format_version = 2;
-    BlockContents contents;
-    const char* content_ptr;
-
-    CompressionType type =
-        (CompressionType)slice_final_with_bit[slice_final_with_bit.size() - 1];
-    if (type != kNoCompression) {
-      UncompressionContext uncompression_ctx(type);
-      UncompressBlockContents(uncompression_ctx, slice_final_with_bit.c_str(),
-                              slice_final_with_bit.size() - 1, &contents,
-                              format_version, ioptions);
-      content_ptr = contents.data.data();
-    } else {
-      content_ptr = slice_final_with_bit.data();
-    }
-
-    size_t num_kv_pairs;
-    const char* header_content_ptr = content_ptr;
-    num_kv_pairs = static_cast<size_t>(DecodeFixed64(header_content_ptr));
-
-    header_content_ptr += sizeof(size_t);
-    size_t num_key_columns = key_col_bufs.size();
-    size_t num_value_columns = value_col_bufs.size();
-    std::vector<const char*> key_content_ptr(num_key_columns);
-    std::vector<const char*> value_content_ptr(num_value_columns);
-    const char* checksum_content_ptr;
-
-    size_t num_columns = num_key_columns + num_value_columns;
-    const char* col_content_ptr =
-        header_content_ptr + sizeof(size_t) * num_columns;
-
-    // Read headers
-    for (size_t i = 0; i < num_key_columns; ++i) {
-      key_content_ptr[i] = col_content_ptr;
-      key_content_ptr[i] += key_col_bufs[i]->Init(key_content_ptr[i]);
-      size_t offset;
-      offset = static_cast<size_t>(DecodeFixed64(header_content_ptr));
-      header_content_ptr += sizeof(size_t);
-      col_content_ptr += offset;
-    }
-    for (size_t i = 0; i < num_value_columns; ++i) {
-      value_content_ptr[i] = col_content_ptr;
-      value_content_ptr[i] += value_col_bufs[i]->Init(value_content_ptr[i]);
-      size_t offset;
-      offset = static_cast<size_t>(DecodeFixed64(header_content_ptr));
-      header_content_ptr += sizeof(size_t);
-      col_content_ptr += offset;
-    }
-    checksum_content_ptr = col_content_ptr;
-    checksum_content_ptr += value_checksum_buf->Init(checksum_content_ptr);
-
-    // Decode block
-    char* decoded_content = decoded_content_base;
-    for (size_t j = 0; j < num_kv_pairs; ++j) {
-      for (size_t i = 0; i < num_key_columns; ++i) {
-        key_content_ptr[i] +=
-            key_col_bufs[i]->Decode(key_content_ptr[i], &decoded_content);
-      }
-      for (size_t i = 0; i < num_value_columns; ++i) {
-        value_content_ptr[i] +=
-            value_col_bufs[i]->Decode(value_content_ptr[i], &decoded_content);
-      }
-      checksum_content_ptr +=
-          value_checksum_buf->Decode(checksum_content_ptr, &decoded_content);
-    }
-
-    size_t offset = decoded_content - decoded_content_base;
-    Slice output_content(decoded_content, offset);
-
-    if (out_file != nullptr) {
-      out_file->Append(output_content);
-    }
-  }
-  delete[] decoded_content_base;
-}
-
-void ColumnAwareEncodingReader::DecodeBlocksFromRowFormat(
-    WritableFile* out_file, const std::vector<std::string>* blocks) {
-  Options options;
-  ImmutableCFOptions ioptions(options);
-  for (auto& block : *blocks) {
-    auto& slice_final_with_bit = block;
-    uint32_t format_version = 2;
-    BlockContents contents;
-    std::string decoded_content;
-
-    CompressionType type =
-        (CompressionType)slice_final_with_bit[slice_final_with_bit.size() - 1];
-    if (type != kNoCompression) {
-      UncompressionContext uncompression_ctx(type);
-      UncompressBlockContents(uncompression_ctx, slice_final_with_bit.c_str(),
-                              slice_final_with_bit.size() - 1, &contents,
-                              format_version, ioptions);
-      decoded_content = std::string(contents.data.data(), contents.data.size());
-    } else {
-      decoded_content = std::move(slice_final_with_bit);
-    }
-
-    if (out_file != nullptr) {
-      out_file->Append(decoded_content);
-    }
-  }
-}
-
-void ColumnAwareEncodingReader::DumpDataColumns(
-    const std::string& filename,
-    const KVPairColDeclarations& kvp_col_declarations,
-    const std::vector<KVPairBlock>& kv_pair_blocks) {
-  KVPairColBufEncoders kvp_col_bufs(kvp_col_declarations);
-  auto& key_col_bufs = kvp_col_bufs.key_col_bufs;
-  auto& value_col_bufs = kvp_col_bufs.value_col_bufs;
-  auto& value_checksum_buf = kvp_col_bufs.value_checksum_buf;
-
-  FILE* fp = fopen(filename.c_str(), "w");
-  size_t block_id = 1;
-  for (auto& kv_pairs : kv_pair_blocks) {
-    fprintf(fp, "---------------- Block: %-4" ROCKSDB_PRIszt " ----------------\n", block_id);
-    for (auto& kv_pair : kv_pairs) {
-      const auto& key = kv_pair.first;
-      const auto& value = kv_pair.second;
-      size_t value_offset = 0;
-
-      const char* key_ptr = key.data();
-      for (auto& buf : key_col_bufs) {
-        size_t col_size = buf->Append(key_ptr);
-        std::string tmp_buf(key_ptr, col_size);
-        Slice col(tmp_buf);
-        fprintf(fp, "%s ", col.ToString(true).c_str());
-        key_ptr += col_size;
-      }
-      fprintf(fp, "|");
-
-      const char* value_ptr = value.data();
-      for (auto& buf : value_col_bufs) {
-        size_t col_size = buf->Append(value_ptr);
-        std::string tmp_buf(value_ptr, col_size);
-        Slice col(tmp_buf);
-        fprintf(fp, " %s", col.ToString(true).c_str());
-        value_ptr += col_size;
-        value_offset += col_size;
-      }
-
-      if (value_offset < value.size()) {
-        size_t col_size = value_checksum_buf->Append(value_ptr);
-        std::string tmp_buf(value_ptr, col_size);
-        Slice col(tmp_buf);
-        fprintf(fp, "|%s", col.ToString(true).c_str());
-      } else {
-        value_checksum_buf->Append(nullptr);
-      }
-      fprintf(fp, "\n");
-    }
-    block_id++;
-  }
-  fclose(fp);
-}
-
-namespace {
-
-void CompressDataBlock(const std::string& output_content, Slice* slice_final,
-                       CompressionType* type, std::string* compressed_output) {
-  CompressionContext compression_ctx(*type);
-  uint32_t format_version = 2;  // hard-coded version
-  *slice_final = CompressBlock(output_content, compression_ctx, type,
-                               format_version, compressed_output);
-}
-
-}  // namespace
-
-void ColumnAwareEncodingReader::EncodeBlocksToRowFormat(
-    WritableFile* out_file, CompressionType compression_type,
-    const std::vector<KVPairBlock>& kv_pair_blocks,
-    std::vector<std::string>* blocks) {
-  std::string output_content;
-  for (auto& kv_pairs : kv_pair_blocks) {
-    output_content.clear();
-    std::string last_key;
-    size_t counter = 0;
-    const size_t block_restart_interval = 16;
-    for (auto& kv_pair : kv_pairs) {
-      const auto& key = kv_pair.first;
-      const auto& value = kv_pair.second;
-
-      Slice last_key_piece(last_key);
-      size_t shared = 0;
-      if (counter >= block_restart_interval) {
-        counter = 0;
-      } else {
-        const size_t min_length = std::min(last_key_piece.size(), key.size());
-        while ((shared < min_length) && last_key_piece[shared] == key[shared]) {
-          shared++;
-        }
-      }
-      const size_t non_shared = key.size() - shared;
-      output_content.append(key.c_str() + shared, non_shared);
-      output_content.append(value);
-
-      last_key.resize(shared);
-      last_key.append(key.data() + shared, non_shared);
-      counter++;
-    }
-    Slice slice_final;
-    auto type = compression_type;
-    std::string compressed_output;
-    CompressDataBlock(output_content, &slice_final, &type, &compressed_output);
-
-    if (out_file != nullptr) {
-      out_file->Append(slice_final);
-    }
-
-    // Add a bit in the end for decoding
-    std::string slice_final_with_bit(slice_final.data(), slice_final.size());
-    slice_final_with_bit.append(reinterpret_cast<char*>(&type), 1);
-    blocks->push_back(
-        std::string(slice_final_with_bit.data(), slice_final_with_bit.size()));
-  }
-}
-
-Status ColumnAwareEncodingReader::EncodeBlocks(
-    const KVPairColDeclarations& kvp_col_declarations, WritableFile* out_file,
-    CompressionType compression_type,
-    const std::vector<KVPairBlock>& kv_pair_blocks,
-    std::vector<std::string>* blocks, bool print_column_stat) {
-  std::vector<size_t> key_col_sizes(
-      kvp_col_declarations.key_col_declarations->size(), 0);
-  std::vector<size_t> value_col_sizes(
-      kvp_col_declarations.value_col_declarations->size(), 0);
-  size_t value_checksum_size = 0;
-
-  for (auto& kv_pairs : kv_pair_blocks) {
-    KVPairColBufEncoders kvp_col_bufs(kvp_col_declarations);
-    auto& key_col_bufs = kvp_col_bufs.key_col_bufs;
-    auto& value_col_bufs = kvp_col_bufs.value_col_bufs;
-    auto& value_checksum_buf = kvp_col_bufs.value_checksum_buf;
-
-    size_t num_kv_pairs = 0;
-    for (auto& kv_pair : kv_pairs) {
-      const auto& key = kv_pair.first;
-      const auto& value = kv_pair.second;
-      size_t value_offset = 0;
-      num_kv_pairs++;
-
-      const char* key_ptr = key.data();
-      for (auto& buf : key_col_bufs) {
-        size_t col_size = buf->Append(key_ptr);
-        key_ptr += col_size;
-      }
-
-      const char* value_ptr = value.data();
-      for (auto& buf : value_col_bufs) {
-        size_t col_size = buf->Append(value_ptr);
-        value_ptr += col_size;
-        value_offset += col_size;
-      }
-
-      if (value_offset < value.size()) {
-        value_checksum_buf->Append(value_ptr);
-      } else {
-        value_checksum_buf->Append(nullptr);
-      }
-    }
-
-    kvp_col_bufs.Finish();
-    // Get stats
-    // Compress and write a block
-    if (print_column_stat) {
-      for (size_t i = 0; i < key_col_bufs.size(); ++i) {
-        Slice slice_final;
-        auto type = compression_type;
-        std::string compressed_output;
-        CompressDataBlock(key_col_bufs[i]->GetData(), &slice_final, &type,
-                          &compressed_output);
-        out_file->Append(slice_final);
-        key_col_sizes[i] += slice_final.size();
-      }
-      for (size_t i = 0; i < value_col_bufs.size(); ++i) {
-        Slice slice_final;
-        auto type = compression_type;
-        std::string compressed_output;
-        CompressDataBlock(value_col_bufs[i]->GetData(), &slice_final, &type,
-                          &compressed_output);
-        out_file->Append(slice_final);
-        value_col_sizes[i] += slice_final.size();
-      }
-      Slice slice_final;
-      auto type = compression_type;
-      std::string compressed_output;
-      CompressDataBlock(value_checksum_buf->GetData(), &slice_final, &type,
-                        &compressed_output);
-      out_file->Append(slice_final);
-      value_checksum_size += slice_final.size();
-    } else {
-      std::string output_content;
-      // Write column sizes
-      PutFixed64(&output_content, num_kv_pairs);
-      for (auto& buf : key_col_bufs) {
-        size_t size = buf->GetData().size();
-        PutFixed64(&output_content, size);
-      }
-      for (auto& buf : value_col_bufs) {
-        size_t size = buf->GetData().size();
-        PutFixed64(&output_content, size);
-      }
-      // Write data
-      for (auto& buf : key_col_bufs) {
-        output_content.append(buf->GetData());
-      }
-      for (auto& buf : value_col_bufs) {
-        output_content.append(buf->GetData());
-      }
-      output_content.append(value_checksum_buf->GetData());
-
-      Slice slice_final;
-      auto type = compression_type;
-      std::string compressed_output;
-      CompressDataBlock(output_content, &slice_final, &type,
-                        &compressed_output);
-
-      if (out_file != nullptr) {
-        out_file->Append(slice_final);
-      }
-
-      // Add a bit in the end for decoding
-      std::string slice_final_with_bit(slice_final.data(),
-                                       slice_final.size() + 1);
-      slice_final_with_bit[slice_final.size()] = static_cast<char>(type);
-      blocks->push_back(std::string(slice_final_with_bit.data(),
-                                    slice_final_with_bit.size()));
-    }
-  }
-
-  if (print_column_stat) {
-    size_t total_size = 0;
-    for (size_t i = 0; i < key_col_sizes.size(); ++i)
-      total_size += key_col_sizes[i];
-    for (size_t i = 0; i < value_col_sizes.size(); ++i)
-      total_size += value_col_sizes[i];
-    total_size += value_checksum_size;
-
-    for (size_t i = 0; i < key_col_sizes.size(); ++i)
-      printf("Key col %" ROCKSDB_PRIszt " size: %" ROCKSDB_PRIszt " percentage %lf%%\n", i, key_col_sizes[i],
-             100.0 * key_col_sizes[i] / total_size);
-    for (size_t i = 0; i < value_col_sizes.size(); ++i)
-      printf("Value col %" ROCKSDB_PRIszt " size: %" ROCKSDB_PRIszt " percentage %lf%%\n", i,
-             value_col_sizes[i], 100.0 * value_col_sizes[i] / total_size);
-    printf("Value checksum size: %" ROCKSDB_PRIszt " percentage %lf%%\n", value_checksum_size,
-           100.0 * value_checksum_size / total_size);
-  }
-  return Status::OK();
-}
-
-void ColumnAwareEncodingReader::GetColDeclarationsPrimary(
-    std::vector<ColDeclaration>** key_col_declarations,
-    std::vector<ColDeclaration>** value_col_declarations,
-    ColDeclaration** value_checksum_declaration) {
-  *key_col_declarations = new std::vector<ColDeclaration>{
-      ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 4, false,
-                     true),
-      ColDeclaration("FixedLength", ColCompressionType::kColRleDeltaVarint, 8,
-                     false, true),
-      ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 8,
-                     false, true),
-      ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 8,
-                     false, true),
-      ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 8)};
-
-  *value_col_declarations = new std::vector<ColDeclaration>{
-      ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 4),
-      ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 4),
-      ColDeclaration("FixedLength", ColCompressionType::kColRle, 1),
-      ColDeclaration("VariableLength"),
-      ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 4),
-      ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 8)};
-  *value_checksum_declaration = new ColDeclaration(
-      "LongFixedLength", ColCompressionType::kColNoCompression, 9,
-      true /* nullable */);
-}
-
-void ColumnAwareEncodingReader::GetColDeclarationsSecondary(
-    std::vector<ColDeclaration>** key_col_declarations,
-    std::vector<ColDeclaration>** value_col_declarations,
-    ColDeclaration** value_checksum_declaration) {
-  *key_col_declarations = new std::vector<ColDeclaration>{
-      ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 4, false,
-                     true),
-      ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 8,
-                     false, true),
-      ColDeclaration("FixedLength", ColCompressionType::kColRleDeltaVarint, 8,
-                     false, true),
-      ColDeclaration("FixedLength", ColCompressionType::kColRle, 1),
-      ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 4,
-                     false, true),
-      ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 8,
-                     false, true),
-      ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 8, false,
-                     true),
-      ColDeclaration("VariableChunk", ColCompressionType::kColNoCompression),
-      ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 8)};
-  *value_col_declarations = new std::vector<ColDeclaration>();
-  *value_checksum_declaration = new ColDeclaration(
-      "LongFixedLength", ColCompressionType::kColNoCompression, 9,
-      true /* nullable */);
-}
-
-}  // namespace rocksdb
-
-#endif  // ROCKSDB_LITE

utilities/column_aware_encoding_util.h

@@ -1,81 +0,0 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-#pragma once
-#ifndef ROCKSDB_LITE
-
-#include <string>
-#include <vector>
-#include "db/dbformat.h"
-#include "include/rocksdb/env.h"
-#include "include/rocksdb/listener.h"
-#include "include/rocksdb/options.h"
-#include "include/rocksdb/status.h"
-#include "options/cf_options.h"
-#include "table/block_based_table_reader.h"
-
-namespace rocksdb {
-
-struct ColDeclaration;
-struct KVPairColDeclarations;
-
-class ColumnAwareEncodingReader {
- public:
-  explicit ColumnAwareEncodingReader(const std::string& file_name);
-
-  void GetKVPairsFromDataBlocks(std::vector<KVPairBlock>* kv_pair_blocks);
-
-  void EncodeBlocksToRowFormat(WritableFile* out_file,
-                               CompressionType compression_type,
-                               const std::vector<KVPairBlock>& kv_pair_blocks,
-                               std::vector<std::string>* blocks);
-
-  void DecodeBlocksFromRowFormat(WritableFile* out_file,
-                                 const std::vector<std::string>* blocks);
-
-  void DumpDataColumns(const std::string& filename,
-                       const KVPairColDeclarations& kvp_col_declarations,
-                       const std::vector<KVPairBlock>& kv_pair_blocks);
-
-  Status EncodeBlocks(const KVPairColDeclarations& kvp_col_declarations,
-                      WritableFile* out_file, CompressionType compression_type,
-                      const std::vector<KVPairBlock>& kv_pair_blocks,
-                      std::vector<std::string>* blocks, bool print_column_stat);
-
-  void DecodeBlocks(const KVPairColDeclarations& kvp_col_declarations,
-                    WritableFile* out_file,
-                    const std::vector<std::string>* blocks);
-
-  static void GetColDeclarationsPrimary(
-      std::vector<ColDeclaration>** key_col_declarations,
-      std::vector<ColDeclaration>** value_col_declarations,
-      ColDeclaration** value_checksum_declaration);
-
-  static void GetColDeclarationsSecondary(
-      std::vector<ColDeclaration>** key_col_declarations,
-      std::vector<ColDeclaration>** value_col_declarations,
-      ColDeclaration** value_checksum_declaration);
-
- private:
-  // Init the TableReader for the sst file
-  void InitTableReader(const std::string& file_path);
-
-  std::string file_name_;
-  EnvOptions soptions_;
-
-  Options options_;
-
-  Status init_result_;
-  std::unique_ptr<BlockBasedTable> table_reader_;
-  std::unique_ptr<RandomAccessFileReader> file_;
-
-  const ImmutableCFOptions ioptions_;
-  const MutableCFOptions moptions_;
-  InternalKeyComparator internal_comparator_;
-  std::unique_ptr<TableProperties> table_properties_;
-};
-
-}  // namespace rocksdb
-
-#endif  // ROCKSDB_LITE

utilities/date_tiered/date_tiered_db_impl.cc

@@ -1,399 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#ifndef ROCKSDB_LITE
-
-#include "utilities/date_tiered/date_tiered_db_impl.h"
-
-#include <limits>
-
-#include "db/db_impl.h"
-#include "db/db_iter.h"
-#include "db/write_batch_internal.h"
-#include "monitoring/instrumented_mutex.h"
-#include "options/options_helper.h"
-#include "rocksdb/convenience.h"
-#include "rocksdb/env.h"
-#include "rocksdb/iterator.h"
-#include "rocksdb/utilities/date_tiered_db.h"
-#include "table/merging_iterator.h"
-#include "util/coding.h"
-#include "util/filename.h"
-#include "util/string_util.h"
-
-namespace rocksdb {
-
-// Open the db inside DateTieredDBImpl because options needs pointer to its ttl
-DateTieredDBImpl::DateTieredDBImpl(
-    DB* db, Options options,
-    const std::vector<ColumnFamilyDescriptor>& descriptors,
-    const std::vector<ColumnFamilyHandle*>& handles, int64_t ttl,
-    int64_t column_family_interval)
-    : db_(db),
-      cf_options_(ColumnFamilyOptions(options)),
-      ioptions_(ImmutableCFOptions(options)),
-      moptions_(MutableCFOptions(options)),
-      icomp_(cf_options_.comparator),
-      ttl_(ttl),
-      column_family_interval_(column_family_interval),
-      mutex_(options.statistics.get(), db->GetEnv(), DB_MUTEX_WAIT_MICROS,
-             options.use_adaptive_mutex) {
-  latest_timebound_ = std::numeric_limits<int64_t>::min();
-  for (size_t i = 0; i < handles.size(); ++i) {
-    const auto& name = descriptors[i].name;
-    int64_t timestamp = 0;
-    try {
-      timestamp = ParseUint64(name);
-    } catch (const std::invalid_argument&) {
-      // Bypass unrelated column family, e.g. default
-      db_->DestroyColumnFamilyHandle(handles[i]);
-      continue;
-    }
-    if (timestamp > latest_timebound_) {
-      latest_timebound_ = timestamp;
-    }
-    handle_map_.insert(std::make_pair(timestamp, handles[i]));
-  }
-}
-
-DateTieredDBImpl::~DateTieredDBImpl() {
-  for (auto handle : handle_map_) {
-    db_->DestroyColumnFamilyHandle(handle.second);
-  }
-  delete db_;
-  db_ = nullptr;
-}
-
-Status DateTieredDB::Open(const Options& options, const std::string& dbname,
-                          DateTieredDB** dbptr, int64_t ttl,
-                          int64_t column_family_interval, bool read_only) {
-  DBOptions db_options(options);
-  ColumnFamilyOptions cf_options(options);
-  std::vector<ColumnFamilyDescriptor> descriptors;
-  std::vector<ColumnFamilyHandle*> handles;
-  DB* db;
-  Status s;
-
-  // Get column families
-  std::vector<std::string> column_family_names;
-  s = DB::ListColumnFamilies(db_options, dbname, &column_family_names);
-  if (!s.ok()) {
-    // No column family found. Use default
-    s = DB::Open(options, dbname, &db);
-    if (!s.ok()) {
-      return s;
-    }
-  } else {
-    for (auto name : column_family_names) {
-      descriptors.emplace_back(ColumnFamilyDescriptor(name, cf_options));
-    }
-
-    // Open database
-    if (read_only) {
-      s = DB::OpenForReadOnly(db_options, dbname, descriptors, &handles, &db);
-    } else {
-      s = DB::Open(db_options, dbname, descriptors, &handles, &db);
-    }
-  }
-
-  if (s.ok()) {
-    *dbptr = new DateTieredDBImpl(db, options, descriptors, handles, ttl,
-                                  column_family_interval);
-  }
-  return s;
-}
-
-// Checks if the string is stale or not according to TTl provided
-bool DateTieredDBImpl::IsStale(int64_t keytime, int64_t ttl, Env* env) {
-  if (ttl <= 0) {
-    // Data is fresh if TTL is non-positive
-    return false;
-  }
-  int64_t curtime;
-  if (!env->GetCurrentTime(&curtime).ok()) {
-    // Treat the data as fresh if could not get current time
-    return false;
-  }
-  return curtime >= keytime + ttl;
-}
-
-// Drop column family when all data in that column family is expired
-// TODO(jhli): Can be made a background job
-Status DateTieredDBImpl::DropObsoleteColumnFamilies() {
-  int64_t curtime;
-  Status s;
-  s = db_->GetEnv()->GetCurrentTime(&curtime);
-  if (!s.ok()) {
-    return s;
-  }
-  {
-    InstrumentedMutexLock l(&mutex_);
-    auto iter = handle_map_.begin();
-    while (iter != handle_map_.end()) {
-      if (iter->first <= curtime - ttl_) {
-        s = db_->DropColumnFamily(iter->second);
-        if (!s.ok()) {
-          return s;
-        }
-        delete iter->second;
-        iter = handle_map_.erase(iter);
-      } else {
-        break;
-      }
-    }
-  }
-  return Status::OK();
-}
-
-// Get timestamp from user key
-Status DateTieredDBImpl::GetTimestamp(const Slice& key, int64_t* result) {
-  if (key.size() < kTSLength) {
-    return Status::Corruption("Bad timestamp in key");
-  }
-  const char* pos = key.data() + key.size() - 8;
-  int64_t timestamp = 0;
-  if (port::kLittleEndian) {
-    int bytes_to_fill = 8;
-    for (int i = 0; i < bytes_to_fill; ++i) {
-      timestamp |= (static_cast<uint64_t>(static_cast<unsigned char>(pos[i]))
-                    << ((bytes_to_fill - i - 1) << 3));
-    }
-  } else {
-    memcpy(&timestamp, pos, sizeof(timestamp));
-  }
-  *result = timestamp;
-  return Status::OK();
-}
-
-Status DateTieredDBImpl::CreateColumnFamily(
-    ColumnFamilyHandle** column_family) {
-  int64_t curtime;
-  Status s;
-  mutex_.AssertHeld();
-  s = db_->GetEnv()->GetCurrentTime(&curtime);
-  if (!s.ok()) {
-    return s;
-  }
-  int64_t new_timebound;
-  if (handle_map_.empty()) {
-    new_timebound = curtime + column_family_interval_;
-  } else {
-    new_timebound =
-        latest_timebound_ +
-        ((curtime - latest_timebound_) / column_family_interval_ + 1) *
-            column_family_interval_;
-  }
-  std::string cf_name = ToString(new_timebound);
-  latest_timebound_ = new_timebound;
-  s = db_->CreateColumnFamily(cf_options_, cf_name, column_family);
-  if (s.ok()) {
-    handle_map_.insert(std::make_pair(new_timebound, *column_family));
-  }
-  return s;
-}
-
-Status DateTieredDBImpl::FindColumnFamily(int64_t keytime,
-                                          ColumnFamilyHandle** column_family,
-                                          bool create_if_missing) {
-  *column_family = nullptr;
-  {
-    InstrumentedMutexLock l(&mutex_);
-    auto iter = handle_map_.upper_bound(keytime);
-    if (iter == handle_map_.end()) {
-      if (!create_if_missing) {
-        return Status::NotFound();
-      } else {
-        return CreateColumnFamily(column_family);
-      }
-    }
-    // Move to previous element to get the appropriate time window
-    *column_family = iter->second;
-  }
-  return Status::OK();
-}
-
-Status DateTieredDBImpl::Put(const WriteOptions& options, const Slice& key,
-                             const Slice& val) {
-  int64_t timestamp = 0;
-  Status s;
-  s = GetTimestamp(key, &timestamp);
-  if (!s.ok()) {
-    return s;
-  }
-  DropObsoleteColumnFamilies();
-
-  // Prune request to obsolete data
-  if (IsStale(timestamp, ttl_, db_->GetEnv())) {
-    return Status::InvalidArgument();
-  }
-
-  // Decide column family (i.e. the time window) to put into
-  ColumnFamilyHandle* column_family;
-  s = FindColumnFamily(timestamp, &column_family, true /*create_if_missing*/);
-  if (!s.ok()) {
-    return s;
-  }
-
-  // Efficiently put with WriteBatch
-  WriteBatch batch;
-  batch.Put(column_family, key, val);
-  return Write(options, &batch);
-}
-
-Status DateTieredDBImpl::Get(const ReadOptions& options, const Slice& key,
-                             std::string* value) {
-  int64_t timestamp = 0;
-  Status s;
-  s = GetTimestamp(key, &timestamp);
-  if (!s.ok()) {
-    return s;
-  }
-  // Prune request to obsolete data
-  if (IsStale(timestamp, ttl_, db_->GetEnv())) {
-    return Status::NotFound();
-  }
-
-  // Decide column family to get from
-  ColumnFamilyHandle* column_family;
-  s = FindColumnFamily(timestamp, &column_family, false /*create_if_missing*/);
-  if (!s.ok()) {
-    return s;
-  }
-  if (column_family == nullptr) {
-    // Cannot find column family
-    return Status::NotFound();
-  }
-
-  // Get value with key
-  return db_->Get(options, column_family, key, value);
-}
-
-bool DateTieredDBImpl::KeyMayExist(const ReadOptions& options, const Slice& key,
-                                   std::string* value, bool* value_found) {
-  int64_t timestamp = 0;
-  Status s;
-  s = GetTimestamp(key, &timestamp);
-  if (!s.ok()) {
-    // Cannot get current time
-    return false;
-  }
-  // Decide column family to get from
-  ColumnFamilyHandle* column_family;
-  s = FindColumnFamily(timestamp, &column_family, false /*create_if_missing*/);
-  if (!s.ok() || column_family == nullptr) {
-    // Cannot find column family
-    return false;
-  }
-  if (IsStale(timestamp, ttl_, db_->GetEnv())) {
-    return false;
-  }
-  return db_->KeyMayExist(options, column_family, key, value, value_found);
-}
-
-Status DateTieredDBImpl::Delete(const WriteOptions& options, const Slice& key) {
-  int64_t timestamp = 0;
-  Status s;
-  s = GetTimestamp(key, &timestamp);
-  if (!s.ok()) {
-    return s;
-  }
-  DropObsoleteColumnFamilies();
-  // Prune request to obsolete data
-  if (IsStale(timestamp, ttl_, db_->GetEnv())) {
-    return Status::NotFound();
-  }
-
-  // Decide column family to get from
-  ColumnFamilyHandle* column_family;
-  s = FindColumnFamily(timestamp, &column_family, false /*create_if_missing*/);
-  if (!s.ok()) {
-    return s;
-  }
-  if (column_family == nullptr) {
-    // Cannot find column family
-    return Status::NotFound();
-  }
-
-  // Get value with key
-  return db_->Delete(options, column_family, key);
-}
-
-Status DateTieredDBImpl::Merge(const WriteOptions& options, const Slice& key,
-                               const Slice& value) {
-  // Decide column family to get from
-  int64_t timestamp = 0;
-  Status s;
-  s = GetTimestamp(key, &timestamp);
-  if (!s.ok()) {
-    // Cannot get current time
-    return s;
-  }
-  ColumnFamilyHandle* column_family;
-  s = FindColumnFamily(timestamp, &column_family, true /*create_if_missing*/);
-  if (!s.ok()) {
-    return s;
-  }
-  WriteBatch batch;
-  batch.Merge(column_family, key, value);
-  return Write(options, &batch);
-}
-
-Status DateTieredDBImpl::Write(const WriteOptions& opts, WriteBatch* updates) {
-  class Handler : public WriteBatch::Handler {
-   public:
-    explicit Handler() {}
-    WriteBatch updates_ttl;
-    Status batch_rewrite_status;
-    virtual Status PutCF(uint32_t column_family_id, const Slice& key,
-                         const Slice& value) override {
-      WriteBatchInternal::Put(&updates_ttl, column_family_id, key, value);
-      return Status::OK();
-    }
-    virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
-                           const Slice& value) override {
-      WriteBatchInternal::Merge(&updates_ttl, column_family_id, key, value);
-      return Status::OK();
-    }
-    virtual Status DeleteCF(uint32_t column_family_id,
-                            const Slice& key) override {
-      WriteBatchInternal::Delete(&updates_ttl, column_family_id, key);
-      return Status::OK();
-    }
-    virtual void LogData(const Slice& blob) override {
-      updates_ttl.PutLogData(blob);
-    }
-  };
-  Handler handler;
-  updates->Iterate(&handler);
-  if (!handler.batch_rewrite_status.ok()) {
-    return handler.batch_rewrite_status;
-  } else {
-    return db_->Write(opts, &(handler.updates_ttl));
-  }
-}
-
-Iterator* DateTieredDBImpl::NewIterator(const ReadOptions& opts) {
-  if (handle_map_.empty()) {
-    return NewEmptyIterator();
-  }
-
-  DBImpl* db_impl = reinterpret_cast<DBImpl*>(db_);
-
-  auto db_iter = NewArenaWrappedDbIterator(
-      db_impl->GetEnv(), opts, ioptions_, moptions_, kMaxSequenceNumber,
-      cf_options_.max_sequential_skip_in_iterations, 0,
-      nullptr /*read_callback*/);
-
-  auto arena = db_iter->GetArena();
-  MergeIteratorBuilder builder(&icomp_, arena);
-  for (auto& item : handle_map_) {
-    auto handle = item.second;
-    builder.AddIterator(db_impl->NewInternalIterator(
-        arena, db_iter->GetRangeDelAggregator(), kMaxSequenceNumber, handle));
-  }
-  auto internal_iter = builder.Finish();
-  db_iter->SetIterUnderDBIter(internal_iter);
-  return db_iter;
-}
-}  // namespace rocksdb
-#endif  // ROCKSDB_LITE

utilities/date_tiered/date_tiered_db_impl.h

@@ -1,93 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-#ifndef ROCKSDB_LITE
-
-#include <map>
-#include <string>
-#include <vector>
-
-#include "monitoring/instrumented_mutex.h"
-#include "options/cf_options.h"
-#include "rocksdb/db.h"
-#include "rocksdb/utilities/date_tiered_db.h"
-
-namespace rocksdb {
-
-// Implementation of DateTieredDB.
-class DateTieredDBImpl : public DateTieredDB {
- public:
-  DateTieredDBImpl(DB* db, Options options,
-                   const std::vector<ColumnFamilyDescriptor>& descriptors,
-                   const std::vector<ColumnFamilyHandle*>& handles, int64_t ttl,
-                   int64_t column_family_interval);
-
-  virtual ~DateTieredDBImpl();
-
-  Status Put(const WriteOptions& options, const Slice& key,
-             const Slice& val) override;
-
-  Status Get(const ReadOptions& options, const Slice& key,
-             std::string* value) override;
-
-  Status Delete(const WriteOptions& options, const Slice& key) override;
-
-  bool KeyMayExist(const ReadOptions& options, const Slice& key,
-                   std::string* value, bool* value_found = nullptr) override;
-
-  Status Merge(const WriteOptions& options, const Slice& key,
-               const Slice& value) override;
-
-  Iterator* NewIterator(const ReadOptions& opts) override;
-
-  Status DropObsoleteColumnFamilies() override;
-
-  // Extract timestamp from key.
-  static Status GetTimestamp(const Slice& key, int64_t* result);
-
- private:
-  // Base database object
-  DB* db_;
-
-  const ColumnFamilyOptions cf_options_;
-
-  const ImmutableCFOptions ioptions_;
-
-  const MutableCFOptions moptions_;
-
-  const InternalKeyComparator icomp_;
-
-  // Storing all column family handles for time series data.
-  std::vector<ColumnFamilyHandle*> handles_;
-
-  // Manages a mapping from a column family's maximum timestamp to its handle.
-  std::map<int64_t, ColumnFamilyHandle*> handle_map_;
-
-  // A time-to-live value to indicate when the data should be removed.
-  int64_t ttl_;
-
-  // An variable to indicate the time range of a column family.
-  int64_t column_family_interval_;
-
-  // Indicate largest maximum timestamp of a column family.
-  int64_t latest_timebound_;
-
-  // Mutex to protect handle_map_ operations.
-  InstrumentedMutex mutex_;
-
-  // Internal method to execute Put and Merge in batch.
-  Status Write(const WriteOptions& opts, WriteBatch* updates);
-
-  Status CreateColumnFamily(ColumnFamilyHandle** column_family);
-
-  Status FindColumnFamily(int64_t keytime, ColumnFamilyHandle** column_family,
-                          bool create_if_missing);
-
-  static bool IsStale(int64_t keytime, int64_t ttl, Env* env);
-};
-
-}  // namespace rocksdb
-#endif  // ROCKSDB_LITE

utilities/date_tiered/date_tiered_test.cc

@@ -1,469 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef ROCKSDB_LITE
-
-#ifndef OS_WIN
-#include <unistd.h>
-#endif
-
-#include <map>
-#include <memory>
-
-#include "rocksdb/compaction_filter.h"
-#include "rocksdb/utilities/date_tiered_db.h"
-#include "port/port.h"
-#include "util/logging.h"
-#include "util/string_util.h"
-#include "util/testharness.h"
-
-namespace rocksdb {
-
-namespace {
-
-typedef std::map<std::string, std::string> KVMap;
-}
-
-class SpecialTimeEnv : public EnvWrapper {
- public:
-  explicit SpecialTimeEnv(Env* base) : EnvWrapper(base) {
-    base->GetCurrentTime(&current_time_);
-  }
-
-  void Sleep(int64_t sleep_time) { current_time_ += sleep_time; }
-  virtual Status GetCurrentTime(int64_t* current_time) override {
-    *current_time = current_time_;
-    return Status::OK();
-  }
-
- private:
-  int64_t current_time_ = 0;
-};
-
-class DateTieredTest : public testing::Test {
- public:
-  DateTieredTest() {
-    env_.reset(new SpecialTimeEnv(Env::Default()));
-    dbname_ = test::PerThreadDBPath("date_tiered");
-    options_.create_if_missing = true;
-    options_.env = env_.get();
-    date_tiered_db_.reset(nullptr);
-    DestroyDB(dbname_, Options());
-  }
-
-  ~DateTieredTest() {
-    CloseDateTieredDB();
-    DestroyDB(dbname_, Options());
-  }
-
-  void OpenDateTieredDB(int64_t ttl, int64_t column_family_interval,
-                        bool read_only = false) {
-    ASSERT_TRUE(date_tiered_db_.get() == nullptr);
-    DateTieredDB* date_tiered_db = nullptr;
-    ASSERT_OK(DateTieredDB::Open(options_, dbname_, &date_tiered_db, ttl,
-                                 column_family_interval, read_only));
-    date_tiered_db_.reset(date_tiered_db);
-  }
-
-  void CloseDateTieredDB() { date_tiered_db_.reset(nullptr); }
-
-  Status AppendTimestamp(std::string* key) {
-    char ts[8];
-    int bytes_to_fill = 8;
-    int64_t timestamp_value = 0;
-    Status s = env_->GetCurrentTime(&timestamp_value);
-    if (!s.ok()) {
-      return s;
-    }
-    if (port::kLittleEndian) {
-      for (int i = 0; i < bytes_to_fill; ++i) {
-        ts[i] = (timestamp_value >> ((bytes_to_fill - i - 1) << 3)) & 0xFF;
-      }
-    } else {
-      memcpy(ts, static_cast<void*>(&timestamp_value), bytes_to_fill);
-    }
-    key->append(ts, 8);
-    return Status::OK();
-  }
-
-  // Populates and returns a kv-map
-  void MakeKVMap(int64_t num_entries, KVMap* kvmap) {
-    kvmap->clear();
-    int digits = 1;
-    for (int64_t dummy = num_entries; dummy /= 10; ++digits) {
-    }
-    int digits_in_i = 1;
-    for (int64_t i = 0; i < num_entries; i++) {
-      std::string key = "key";
-      std::string value = "value";
-      if (i % 10 == 0) {
-        digits_in_i++;
-      }
-      for (int j = digits_in_i; j < digits; j++) {
-        key.append("0");
-        value.append("0");
-      }
-      AppendNumberTo(&key, i);
-      AppendNumberTo(&value, i);
-      ASSERT_OK(AppendTimestamp(&key));
-      (*kvmap)[key] = value;
-    }
-    // check all insertions done
-    ASSERT_EQ(num_entries, static_cast<int64_t>(kvmap->size()));
-  }
-
-  size_t GetColumnFamilyCount() {
-    DBOptions db_options(options_);
-    std::vector<std::string> cf;
-    DB::ListColumnFamilies(db_options, dbname_, &cf);
-    return cf.size();
-  }
-
-  void Sleep(int64_t sleep_time) { env_->Sleep(sleep_time); }
-
-  static const int64_t kSampleSize_ = 100;
-  std::string dbname_;
-  std::unique_ptr<DateTieredDB> date_tiered_db_;
-  std::unique_ptr<SpecialTimeEnv> env_;
-  KVMap kvmap_;
-
- private:
-  Options options_;
-  KVMap::iterator kv_it_;
-  const std::string kNewValue_ = "new_value";
-  std::unique_ptr<CompactionFilter> test_comp_filter_;
-};
-
-// Puts a set of values and checks its presence using Get during ttl
-TEST_F(DateTieredTest, KeyLifeCycle) {
-  WriteOptions wopts;
-  ReadOptions ropts;
-
-  // T=0, open the database and insert data
-  OpenDateTieredDB(2, 2);
-  ASSERT_TRUE(date_tiered_db_.get() != nullptr);
-
-  // Create key value pairs to insert
-  KVMap map_insert;
-  MakeKVMap(kSampleSize_, &map_insert);
-
-  // Put data in database
-  for (auto& kv : map_insert) {
-    ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
-  }
-
-  Sleep(1);
-  // T=1, keys should still reside in database
-  for (auto& kv : map_insert) {
-    std::string value;
-    ASSERT_OK(date_tiered_db_->Get(ropts, kv.first, &value));
-    ASSERT_EQ(value, kv.second);
-  }
-
-  Sleep(1);
-  // T=2, keys should not be retrieved
-  for (auto& kv : map_insert) {
-    std::string value;
-    auto s = date_tiered_db_->Get(ropts, kv.first, &value);
-    ASSERT_TRUE(s.IsNotFound());
-  }
-
-  CloseDateTieredDB();
-}
-
-TEST_F(DateTieredTest, DeleteTest) {
-  WriteOptions wopts;
-  ReadOptions ropts;
-
-  // T=0, open the database and insert data
-  OpenDateTieredDB(2, 2);
-  ASSERT_TRUE(date_tiered_db_.get() != nullptr);
-
-  // Create key value pairs to insert
-  KVMap map_insert;
-  MakeKVMap(kSampleSize_, &map_insert);
-
-  // Put data in database
-  for (auto& kv : map_insert) {
-    ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
-  }
-
-  Sleep(1);
-  // Delete keys when they are not obsolete
-  for (auto& kv : map_insert) {
-    ASSERT_OK(date_tiered_db_->Delete(wopts, kv.first));
-  }
-
-  // Key should not be found
-  for (auto& kv : map_insert) {
-    std::string value;
-    auto s = date_tiered_db_->Get(ropts, kv.first, &value);
-    ASSERT_TRUE(s.IsNotFound());
-  }
-}
-
-TEST_F(DateTieredTest, KeyMayExistTest) {
-  WriteOptions wopts;
-  ReadOptions ropts;
-
-  // T=0, open the database and insert data
-  OpenDateTieredDB(2, 2);
-  ASSERT_TRUE(date_tiered_db_.get() != nullptr);
-
-  // Create key value pairs to insert
-  KVMap map_insert;
-  MakeKVMap(kSampleSize_, &map_insert);
-
-  // Put data in database
-  for (auto& kv : map_insert) {
-    ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
-  }
-
-  Sleep(1);
-  // T=1, keys should still reside in database
-  for (auto& kv : map_insert) {
-    std::string value;
-    ASSERT_TRUE(date_tiered_db_->KeyMayExist(ropts, kv.first, &value));
-    ASSERT_EQ(value, kv.second);
-  }
-}
-
-// Database open and close should not affect
-TEST_F(DateTieredTest, MultiOpen) {
-  WriteOptions wopts;
-  ReadOptions ropts;
-
-  // T=0, open the database and insert data
-  OpenDateTieredDB(4, 4);
-  ASSERT_TRUE(date_tiered_db_.get() != nullptr);
-
-  // Create key value pairs to insert
-  KVMap map_insert;
-  MakeKVMap(kSampleSize_, &map_insert);
-
-  // Put data in database
-  for (auto& kv : map_insert) {
-    ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
-  }
-  CloseDateTieredDB();
-
-  Sleep(1);
-  OpenDateTieredDB(2, 2);
-  // T=1, keys should still reside in database
-  for (auto& kv : map_insert) {
-    std::string value;
-    ASSERT_OK(date_tiered_db_->Get(ropts, kv.first, &value));
-    ASSERT_EQ(value, kv.second);
-  }
-
-  Sleep(1);
-  // T=2, keys should not be retrieved
-  for (auto& kv : map_insert) {
-    std::string value;
-    auto s = date_tiered_db_->Get(ropts, kv.first, &value);
-    ASSERT_TRUE(s.IsNotFound());
-  }
-
-  CloseDateTieredDB();
-}
-
-// If the key in Put() is obsolete, the data should not be written into database
-TEST_F(DateTieredTest, InsertObsoleteDate) {
-  WriteOptions wopts;
-  ReadOptions ropts;
-
-  // T=0, open the database and insert data
-  OpenDateTieredDB(2, 2);
-  ASSERT_TRUE(date_tiered_db_.get() != nullptr);
-
-  // Create key value pairs to insert
-  KVMap map_insert;
-  MakeKVMap(kSampleSize_, &map_insert);
-
-  Sleep(2);
-  // T=2, keys put into database are already obsolete
-  // Put data in database. Operations should not return OK
-  for (auto& kv : map_insert) {
-    auto s = date_tiered_db_->Put(wopts, kv.first, kv.second);
-    ASSERT_TRUE(s.IsInvalidArgument());
-  }
-
-  // Data should not be found in database
-  for (auto& kv : map_insert) {
-    std::string value;
-    auto s = date_tiered_db_->Get(ropts, kv.first, &value);
-    ASSERT_TRUE(s.IsNotFound());
-  }
-
-  CloseDateTieredDB();
-}
-
-// Resets the timestamp of a set of kvs by updating them and checks that they
-// are not deleted according to the old timestamp
-TEST_F(DateTieredTest, ColumnFamilyCounts) {
-  WriteOptions wopts;
-  ReadOptions ropts;
-
-  // T=0, open the database and insert data
-  OpenDateTieredDB(4, 2);
-  ASSERT_TRUE(date_tiered_db_.get() != nullptr);
-  // Only default column family
-  ASSERT_EQ(1, GetColumnFamilyCount());
-
-  // Create key value pairs to insert
-  KVMap map_insert;
-  MakeKVMap(kSampleSize_, &map_insert);
-  for (auto& kv : map_insert) {
-    ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
-  }
-  // A time series column family is created
-  ASSERT_EQ(2, GetColumnFamilyCount());
-
-  Sleep(2);
-  KVMap map_insert2;
-  MakeKVMap(kSampleSize_, &map_insert2);
-  for (auto& kv : map_insert2) {
-    ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
-  }
-  // Another time series column family is created
-  ASSERT_EQ(3, GetColumnFamilyCount());
-
-  Sleep(4);
-
-  // Data should not be found in database
-  for (auto& kv : map_insert) {
-    std::string value;
-    auto s = date_tiered_db_->Get(ropts, kv.first, &value);
-    ASSERT_TRUE(s.IsNotFound());
-  }
-
-  // Explicitly drop obsolete column families
-  date_tiered_db_->DropObsoleteColumnFamilies();
-
-  // The first column family is deleted from database
-  ASSERT_EQ(2, GetColumnFamilyCount());
-
-  CloseDateTieredDB();
-}
-
-// Puts a set of values and checks its presence using iterator during ttl
-TEST_F(DateTieredTest, IteratorLifeCycle) {
-  WriteOptions wopts;
-  ReadOptions ropts;
-
-  // T=0, open the database and insert data
-  OpenDateTieredDB(2, 2);
-  ASSERT_TRUE(date_tiered_db_.get() != nullptr);
-
-  // Create key value pairs to insert
-  KVMap map_insert;
-  MakeKVMap(kSampleSize_, &map_insert);
-  Iterator* dbiter;
-
-  // Put data in database
-  for (auto& kv : map_insert) {
-    ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
-  }
-
-  Sleep(1);
-  ASSERT_EQ(2, GetColumnFamilyCount());
-  // T=1, keys should still reside in database
-  dbiter = date_tiered_db_->NewIterator(ropts);
-  dbiter->SeekToFirst();
-  for (auto& kv : map_insert) {
-    ASSERT_TRUE(dbiter->Valid());
-    ASSERT_EQ(0, dbiter->value().compare(kv.second));
-    dbiter->Next();
-  }
-  delete dbiter;
-
-  Sleep(4);
-  // T=5, keys should not be retrieved
-  for (auto& kv : map_insert) {
-    std::string value;
-    auto s = date_tiered_db_->Get(ropts, kv.first, &value);
-    ASSERT_TRUE(s.IsNotFound());
-  }
-
-  // Explicitly drop obsolete column families
-  date_tiered_db_->DropObsoleteColumnFamilies();
-
-  // Only default column family
-  ASSERT_EQ(1, GetColumnFamilyCount());
-
-  // Empty iterator
-  dbiter = date_tiered_db_->NewIterator(ropts);
-  dbiter->Seek(map_insert.begin()->first);
-  ASSERT_FALSE(dbiter->Valid());
-  delete dbiter;
-
-  CloseDateTieredDB();
-}
-
-// Iterator should be able to merge data from multiple column families
-TEST_F(DateTieredTest, IteratorMerge) {
-  WriteOptions wopts;
-  ReadOptions ropts;
-
-  // T=0, open the database and insert data
-  OpenDateTieredDB(4, 2);
-  ASSERT_TRUE(date_tiered_db_.get() != nullptr);
-
-  Iterator* dbiter;
-
-  // Put data in database
-  KVMap map_insert1;
-  MakeKVMap(kSampleSize_, &map_insert1);
-  for (auto& kv : map_insert1) {
-    ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
-  }
-  ASSERT_EQ(2, GetColumnFamilyCount());
-
-  Sleep(2);
-  // Put more data
-  KVMap map_insert2;
-  MakeKVMap(kSampleSize_, &map_insert2);
-  for (auto& kv : map_insert2) {
-    ASSERT_OK(date_tiered_db_->Put(wopts, kv.first, kv.second));
-  }
-  // Multiple column families for time series data
-  ASSERT_EQ(3, GetColumnFamilyCount());
-
-  // Iterator should be able to merge data from different column families
-  dbiter = date_tiered_db_->NewIterator(ropts);
-  dbiter->SeekToFirst();
-  KVMap::iterator iter1 = map_insert1.begin();
-  KVMap::iterator iter2 = map_insert2.begin();
-  for (; iter1 != map_insert1.end() && iter2 != map_insert2.end();
-       iter1++, iter2++) {
-    ASSERT_TRUE(dbiter->Valid());
-    ASSERT_EQ(0, dbiter->value().compare(iter1->second));
-    dbiter->Next();
-
-    ASSERT_TRUE(dbiter->Valid());
-    ASSERT_EQ(0, dbiter->value().compare(iter2->second));
-    dbiter->Next();
-  }
-  delete dbiter;
-
-  CloseDateTieredDB();
-}
-
-}  //  namespace rocksdb
-
-// A black-box test for the DateTieredDB around rocksdb
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
-
-#else
-#include <stdio.h>
-
-int main(int /*argc*/, char** /*argv*/) {
-  fprintf(stderr, "SKIPPED as DateTieredDB is not supported in ROCKSDB_LITE\n");
-  return 0;
-}
-
-#endif  // !ROCKSDB_LITE

utilities/document/document_db_test.cc

@@ -1,338 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#ifndef ROCKSDB_LITE
-
-#include <algorithm>
-
-#include "rocksdb/utilities/json_document.h"
-#include "rocksdb/utilities/document_db.h"
-
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace rocksdb {
-
-class DocumentDBTest : public testing::Test {
- public:
-  DocumentDBTest() {
-    dbname_ = test::PerThreadDBPath("document_db_test");
-    DestroyDB(dbname_, Options());
-  }
-  ~DocumentDBTest() {
-    delete db_;
-    DestroyDB(dbname_, Options());
-  }
-
-  void AssertCursorIDs(Cursor* cursor, std::vector<int64_t> expected) {
-    std::vector<int64_t> got;
-    while (cursor->Valid()) {
-      ASSERT_TRUE(cursor->Valid());
-      ASSERT_TRUE(cursor->document().Contains("_id"));
-      got.push_back(cursor->document()["_id"].GetInt64());
-      cursor->Next();
-    }
-    std::sort(expected.begin(), expected.end());
-    std::sort(got.begin(), got.end());
-    ASSERT_TRUE(got == expected);
-  }
-
-  // converts ' to ", so that we don't have to escape " all over the place
-  std::string ConvertQuotes(const std::string& input) {
-    std::string output;
-    for (auto x : input) {
-      if (x == '\'') {
-        output.push_back('\"');
-      } else {
-        output.push_back(x);
-      }
-    }
-    return output;
-  }
-
-  void CreateIndexes(std::vector<DocumentDB::IndexDescriptor> indexes) {
-    for (auto i : indexes) {
-      ASSERT_OK(db_->CreateIndex(WriteOptions(), i));
-    }
-  }
-
-  JSONDocument* Parse(const std::string& doc) {
-    return JSONDocument::ParseJSON(ConvertQuotes(doc).c_str());
-  }
-
-  std::string dbname_;
-  DocumentDB* db_;
-};
-
-TEST_F(DocumentDBTest, SimpleQueryTest) {
-  DocumentDBOptions options;
-  DocumentDB::IndexDescriptor index;
-  index.description = Parse("{\"name\": 1}");
-  index.name = "name_index";
-
-  ASSERT_OK(DocumentDB::Open(options, dbname_, {}, &db_));
-  CreateIndexes({index});
-  delete db_;
-  db_ = nullptr;
-  // now there is index present
-  ASSERT_OK(DocumentDB::Open(options, dbname_, {index}, &db_));
-  assert(db_ != nullptr);
-  delete index.description;
-
-  std::vector<std::string> json_objects = {
-      "{\"_id\': 1, \"name\": \"One\"}",   "{\"_id\": 2, \"name\": \"Two\"}",
-      "{\"_id\": 3, \"name\": \"Three\"}", "{\"_id\": 4, \"name\": \"Four\"}"};
-
-  for (auto& json : json_objects) {
-    std::unique_ptr<JSONDocument> document(Parse(json));
-    ASSERT_TRUE(document.get() != nullptr);
-    ASSERT_OK(db_->Insert(WriteOptions(), *document));
-  }
-
-  // inserting a document with existing primary key should return failure
-  {
-    std::unique_ptr<JSONDocument> document(Parse(json_objects[0]));
-    ASSERT_TRUE(document.get() != nullptr);
-    Status s = db_->Insert(WriteOptions(), *document);
-    ASSERT_TRUE(s.IsInvalidArgument());
-  }
-
-  // find equal to "Two"
-  {
-    std::unique_ptr<JSONDocument> query(
-        Parse("[{'$filter': {'name': 'Two', '$index': 'name_index'}}]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-    AssertCursorIDs(cursor.get(), {2});
-  }
-
-  // find less than "Three"
-  {
-    std::unique_ptr<JSONDocument> query(Parse(
-        "[{'$filter': {'name': {'$lt': 'Three'}, '$index': "
-        "'name_index'}}]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-
-    AssertCursorIDs(cursor.get(), {1, 4});
-  }
-
-  // find less than "Three" without index
-  {
-    std::unique_ptr<JSONDocument> query(
-        Parse("[{'$filter': {'name': {'$lt': 'Three'} }}]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-    AssertCursorIDs(cursor.get(), {1, 4});
-  }
-
-  // remove less or equal to "Three"
-  {
-    std::unique_ptr<JSONDocument> query(
-        Parse("{'name': {'$lte': 'Three'}, '$index': 'name_index'}"));
-    ASSERT_OK(db_->Remove(ReadOptions(), WriteOptions(), *query));
-  }
-
-  // find all -- only "Two" left, everything else should be deleted
-  {
-    std::unique_ptr<JSONDocument> query(Parse("[]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-    AssertCursorIDs(cursor.get(), {2});
-  }
-}
-
-TEST_F(DocumentDBTest, ComplexQueryTest) {
-  DocumentDBOptions options;
-  DocumentDB::IndexDescriptor priority_index;
-  priority_index.description = Parse("{'priority': 1}");
-  priority_index.name = "priority";
-  DocumentDB::IndexDescriptor job_name_index;
-  job_name_index.description = Parse("{'job_name': 1}");
-  job_name_index.name = "job_name";
-  DocumentDB::IndexDescriptor progress_index;
-  progress_index.description = Parse("{'progress': 1}");
-  progress_index.name = "progress";
-
-  ASSERT_OK(DocumentDB::Open(options, dbname_, {}, &db_));
-  CreateIndexes({priority_index, progress_index});
-  delete priority_index.description;
-  delete progress_index.description;
-
-  std::vector<std::string> json_objects = {
-      "{'_id': 1, 'job_name': 'play', 'priority': 10, 'progress': 14.2}",
-      "{'_id': 2, 'job_name': 'white', 'priority': 2, 'progress': 45.1}",
-      "{'_id': 3, 'job_name': 'straw', 'priority': 5, 'progress': 83.2}",
-      "{'_id': 4, 'job_name': 'temporary', 'priority': 3, 'progress': 14.9}",
-      "{'_id': 5, 'job_name': 'white', 'priority': 4, 'progress': 44.2}",
-      "{'_id': 6, 'job_name': 'tea', 'priority': 1, 'progress': 12.4}",
-      "{'_id': 7, 'job_name': 'delete', 'priority': 2, 'progress': 77.54}",
-      "{'_id': 8, 'job_name': 'rock', 'priority': 3, 'progress': 93.24}",
-      "{'_id': 9, 'job_name': 'steady', 'priority': 3, 'progress': 9.1}",
-      "{'_id': 10, 'job_name': 'white', 'priority': 1, 'progress': 61.4}",
-      "{'_id': 11, 'job_name': 'who', 'priority': 4, 'progress': 39.41}",
-      "{'_id': 12, 'job_name': 'who', 'priority': -1, 'progress': 39.42}",
-      "{'_id': 13, 'job_name': 'who', 'priority': -2, 'progress': 39.42}", };
-
-  // add index on the fly!
-  CreateIndexes({job_name_index});
-  delete job_name_index.description;
-
-  for (auto& json : json_objects) {
-    std::unique_ptr<JSONDocument> document(Parse(json));
-    ASSERT_TRUE(document != nullptr);
-    ASSERT_OK(db_->Insert(WriteOptions(), *document));
-  }
-
-  // 2 < priority < 4 AND progress > 10.0, index priority
-  {
-    std::unique_ptr<JSONDocument> query(Parse(
-        "[{'$filter': {'priority': {'$lt': 4, '$gt': 2}, 'progress': {'$gt': "
-        "10.0}, '$index': 'priority'}}]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-    AssertCursorIDs(cursor.get(), {4, 8});
-  }
-
-  // -1 <= priority <= 1, index priority
-  {
-    std::unique_ptr<JSONDocument> query(Parse(
-        "[{'$filter': {'priority': {'$lte': 1, '$gte': -1},"
-        " '$index': 'priority'}}]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-    AssertCursorIDs(cursor.get(), {6, 10, 12});
-  }
-
-  // 2 < priority < 4 AND progress > 10.0, index progress
-  {
-    std::unique_ptr<JSONDocument> query(Parse(
-        "[{'$filter': {'priority': {'$lt': 4, '$gt': 2}, 'progress': {'$gt': "
-        "10.0}, '$index': 'progress'}}]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-    AssertCursorIDs(cursor.get(), {4, 8});
-  }
-
-  // job_name == 'white' AND priority >= 2, index job_name
-  {
-    std::unique_ptr<JSONDocument> query(Parse(
-        "[{'$filter': {'job_name': 'white', 'priority': {'$gte': "
-        "2}, '$index': 'job_name'}}]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-    AssertCursorIDs(cursor.get(), {2, 5});
-  }
-
-  // 35.0 <= progress < 65.5, index progress
-  {
-    std::unique_ptr<JSONDocument> query(Parse(
-        "[{'$filter': {'progress': {'$gt': 5.0, '$gte': 35.0, '$lt': 65.5}, "
-        "'$index': 'progress'}}]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-    AssertCursorIDs(cursor.get(), {2, 5, 10, 11, 12, 13});
-  }
-
-  // 2 < priority <= 4, index priority
-  {
-    std::unique_ptr<JSONDocument> query(Parse(
-        "[{'$filter': {'priority': {'$gt': 2, '$lt': 8, '$lte': 4}, "
-        "'$index': 'priority'}}]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-    AssertCursorIDs(cursor.get(), {4, 5, 8, 9, 11});
-  }
-
-  // Delete all whose progress is bigger than 50%
-  {
-    std::unique_ptr<JSONDocument> query(
-        Parse("{'progress': {'$gt': 50.0}, '$index': 'progress'}"));
-    ASSERT_OK(db_->Remove(ReadOptions(), WriteOptions(), *query));
-  }
-
-  // 2 < priority < 6, index priority
-  {
-    std::unique_ptr<JSONDocument> query(Parse(
-        "[{'$filter': {'priority': {'$gt': 2, '$lt': 6}, "
-        "'$index': 'priority'}}]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-    AssertCursorIDs(cursor.get(), {4, 5, 9, 11});
-  }
-
-  // update set priority to 10 where job_name is 'white'
-  {
-    std::unique_ptr<JSONDocument> query(Parse("{'job_name': 'white'}"));
-    std::unique_ptr<JSONDocument> update(Parse("{'$set': {'priority': 10}}"));
-    ASSERT_OK(db_->Update(ReadOptions(), WriteOptions(), *query, *update));
-  }
-
-  // update twice: set priority to 15 where job_name is 'white'
-  {
-    std::unique_ptr<JSONDocument> query(Parse("{'job_name': 'white'}"));
-    std::unique_ptr<JSONDocument> update(Parse("{'$set': {'priority': 10},"
-                                               "'$set': {'priority': 15}}"));
-    ASSERT_OK(db_->Update(ReadOptions(), WriteOptions(), *query, *update));
-  }
-
-  // update twice: set priority to 15 and
-  // progress to 40 where job_name is 'white'
-  {
-    std::unique_ptr<JSONDocument> query(Parse("{'job_name': 'white'}"));
-    std::unique_ptr<JSONDocument> update(
-        Parse("{'$set': {'priority': 10, 'progress': 35},"
-              "'$set': {'priority': 15, 'progress': 40}}"));
-    ASSERT_OK(db_->Update(ReadOptions(), WriteOptions(), *query, *update));
-  }
-
-  // priority < 0
-  {
-    std::unique_ptr<JSONDocument> query(
-        Parse("[{'$filter': {'priority': {'$lt': 0}, '$index': 'priority'}}]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-    ASSERT_OK(cursor->status());
-    AssertCursorIDs(cursor.get(), {12, 13});
-  }
-
-  // -2 < priority < 0
-  {
-    std::unique_ptr<JSONDocument> query(
-        Parse("[{'$filter': {'priority': {'$gt': -2, '$lt': 0},"
-        " '$index': 'priority'}}]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-    ASSERT_OK(cursor->status());
-    AssertCursorIDs(cursor.get(), {12});
-  }
-
-  // -2 <= priority < 0
-  {
-    std::unique_ptr<JSONDocument> query(
-        Parse("[{'$filter': {'priority': {'$gte': -2, '$lt': 0},"
-        " '$index': 'priority'}}]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-    ASSERT_OK(cursor->status());
-    AssertCursorIDs(cursor.get(), {12, 13});
-  }
-
-  // 4 < priority
-  {
-    std::unique_ptr<JSONDocument> query(
-        Parse("[{'$filter': {'priority': {'$gt': 4}, '$index': 'priority'}}]"));
-    std::unique_ptr<Cursor> cursor(db_->Query(ReadOptions(), *query));
-    ASSERT_OK(cursor->status());
-    AssertCursorIDs(cursor.get(), {1, 2, 5});
-  }
-
-  Status s = db_->DropIndex("doesnt-exist");
-  ASSERT_TRUE(!s.ok());
-  ASSERT_OK(db_->DropIndex("priority"));
-}
-
-}  //  namespace rocksdb
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
-
-#else
-#include <stdio.h>
-
-int main(int /*argc*/, char** /*argv*/) {
-  fprintf(stderr, "SKIPPED as DocumentDB is not supported in ROCKSDB_LITE\n");
-  return 0;
-}
-
-#endif  // !ROCKSDB_LITE

utilities/document/json_document.cc

@@ -1,610 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-#ifndef ROCKSDB_LITE
-
-#include "rocksdb/utilities/json_document.h"
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <assert.h>
-#include <inttypes.h>
-#include <string.h>
-
-#include <functional>
-#include <limits>
-#include <map>
-#include <memory>
-#include <string>
-#include <vector>
-
-
-#include "third-party/fbson/FbsonDocument.h"
-#include "third-party/fbson/FbsonJsonParser.h"
-#include "third-party/fbson/FbsonUtil.h"
-#include "util/coding.h"
-
-using std::placeholders::_1;
-
-namespace {
-
-size_t ObjectNumElem(const fbson::ObjectVal& objectVal) {
-  size_t size = 0;
-  for (auto keyValuePair : objectVal) {
-    (void)keyValuePair;
-    ++size;
-  }
-  return size;
-}
-
-template <typename Func>
-void InitJSONDocument(std::unique_ptr<char[]>* data,
-                      fbson::FbsonValue** value,
-                      Func f) {
-  // TODO(stash): maybe add function to FbsonDocument to avoid creating array?
-  fbson::FbsonWriter writer;
-  bool res __attribute__((__unused__)) = writer.writeStartArray();
-  assert(res);
-  uint32_t bytesWritten __attribute__((__unused__));
-  bytesWritten = f(writer);
-  assert(bytesWritten != 0);
-  res = writer.writeEndArray();
-  assert(res);
-  char* buf = new char[writer.getOutput()->getSize()];
-  memcpy(buf, writer.getOutput()->getBuffer(), writer.getOutput()->getSize());
-
-  *value = ((fbson::FbsonDocument *)buf)->getValue();
-  assert((*value)->isArray());
-  assert(((fbson::ArrayVal*)*value)->numElem() == 1);
-  *value = ((fbson::ArrayVal*)*value)->get(0);
-  data->reset(buf);
-}
-
-void InitString(std::unique_ptr<char[]>* data,
-                fbson::FbsonValue** value,
-                const std::string& s) {
-  InitJSONDocument(data, value, std::bind(
-      [](fbson::FbsonWriter& writer, const std::string& str) -> uint32_t {
-        bool res __attribute__((__unused__)) = writer.writeStartString();
-        assert(res);
-        auto bytesWritten = writer.writeString(str.c_str(),
-                            static_cast<uint32_t>(str.length()));
-        res = writer.writeEndString();
-        assert(res);
-        // If the string is empty, then bytesWritten == 0, and assert in
-        // InitJsonDocument will fail.
-        return bytesWritten + static_cast<uint32_t>(str.empty());
-      },
-  _1, s));
-}
-
-bool IsNumeric(fbson::FbsonValue* value) {
-  return value->isInt8() || value->isInt16() ||
-         value->isInt32() ||  value->isInt64();
-}
-
-int64_t GetInt64ValFromFbsonNumericType(fbson::FbsonValue* value) {
-  switch (value->type()) {
-    case fbson::FbsonType::T_Int8:
-      return reinterpret_cast<fbson::Int8Val*>(value)->val();
-    case fbson::FbsonType::T_Int16:
-      return reinterpret_cast<fbson::Int16Val*>(value)->val();
-    case fbson::FbsonType::T_Int32:
-      return reinterpret_cast<fbson::Int32Val*>(value)->val();
-    case fbson::FbsonType::T_Int64:
-      return reinterpret_cast<fbson::Int64Val*>(value)->val();
-    default:
-      assert(false);
-  }
-  return 0;
-}
-
-bool IsComparable(fbson::FbsonValue* left, fbson::FbsonValue* right) {
-  if (left->type() == right->type()) {
-    return true;
-  }
-  if (IsNumeric(left) && IsNumeric(right)) {
-    return true;
-  }
-  return false;
-}
-
-void CreateArray(std::unique_ptr<char[]>* data, fbson::FbsonValue** value) {
-  fbson::FbsonWriter writer;
-  bool res __attribute__((__unused__)) = writer.writeStartArray();
-  assert(res);
-  res = writer.writeEndArray();
-  assert(res);
-  data->reset(new char[writer.getOutput()->getSize()]);
-  memcpy(data->get(),
-         writer.getOutput()->getBuffer(),
-         writer.getOutput()->getSize());
-  *value = reinterpret_cast<fbson::FbsonDocument*>(data->get())->getValue();
-}
-
-void CreateObject(std::unique_ptr<char[]>* data, fbson::FbsonValue** value) {
-  fbson::FbsonWriter writer;
-  bool res __attribute__((__unused__)) = writer.writeStartObject();
-  assert(res);
-  res = writer.writeEndObject();
-  assert(res);
-  data->reset(new char[writer.getOutput()->getSize()]);
-  memcpy(data->get(),
-         writer.getOutput()->getBuffer(),
-         writer.getOutput()->getSize());
-  *value = reinterpret_cast<fbson::FbsonDocument*>(data->get())->getValue();
-}
-
-}  // namespace
-
-namespace rocksdb {
-
-
-// TODO(stash): find smth easier
-JSONDocument::JSONDocument() {
-  InitJSONDocument(&data_,
-                   &value_,
-                   std::bind(&fbson::FbsonWriter::writeNull, _1));
-}
-
-JSONDocument::JSONDocument(bool b) {
-  InitJSONDocument(&data_,
-                   &value_,
-                   std::bind(&fbson::FbsonWriter::writeBool, _1, b));
-}
-
-JSONDocument::JSONDocument(double d) {
-  InitJSONDocument(&data_,
-                   &value_,
-                   std::bind(&fbson::FbsonWriter::writeDouble, _1, d));
-}
-
-JSONDocument::JSONDocument(int8_t i) {
-  InitJSONDocument(&data_,
-                   &value_,
-                   std::bind(&fbson::FbsonWriter::writeInt8, _1, i));
-}
-
-JSONDocument::JSONDocument(int16_t i) {
-  InitJSONDocument(&data_,
-                   &value_,
-                   std::bind(&fbson::FbsonWriter::writeInt16, _1, i));
-}
-
-JSONDocument::JSONDocument(int32_t i) {
-  InitJSONDocument(&data_,
-                   &value_,
-                   std::bind(&fbson::FbsonWriter::writeInt32, _1, i));
-}
-
-JSONDocument::JSONDocument(int64_t i) {
-  InitJSONDocument(&data_,
-                   &value_,
-                   std::bind(&fbson::FbsonWriter::writeInt64, _1, i));
-}
-
-JSONDocument::JSONDocument(const std::string& s) {
-  InitString(&data_, &value_, s);
-}
-
-JSONDocument::JSONDocument(const char* s) : JSONDocument(std::string(s)) {
-}
-
-void JSONDocument::InitFromValue(const fbson::FbsonValue* val) {
-  data_.reset(new char[val->numPackedBytes()]);
-  memcpy(data_.get(), val, val->numPackedBytes());
-  value_ = reinterpret_cast<fbson::FbsonValue*>(data_.get());
-}
-
-// Private constructor
-JSONDocument::JSONDocument(fbson::FbsonValue* val, bool makeCopy) {
-  if (makeCopy) {
-    InitFromValue(val);
-  } else {
-    value_ = val;
-  }
-}
-
-JSONDocument::JSONDocument(Type _type) {
-  // TODO(icanadi) make all of this better by using templates
-  switch (_type) {
-    case kNull:
-      InitJSONDocument(&data_, &value_,
-                       std::bind(&fbson::FbsonWriter::writeNull, _1));
-      break;
-    case kObject:
-      CreateObject(&data_, &value_);
-      break;
-    case kBool:
-      InitJSONDocument(&data_, &value_,
-                       std::bind(&fbson::FbsonWriter::writeBool, _1, false));
-      break;
-    case kDouble:
-      InitJSONDocument(&data_, &value_,
-                       std::bind(&fbson::FbsonWriter::writeDouble, _1, 0.));
-      break;
-    case kArray:
-      CreateArray(&data_, &value_);
-      break;
-    case kInt64:
-      InitJSONDocument(&data_, &value_,
-                       std::bind(&fbson::FbsonWriter::writeInt64, _1, 0));
-      break;
-    case kString:
-      InitString(&data_, &value_, "");
-      break;
-    default:
-      assert(false);
-  }
-}
-
-JSONDocument::JSONDocument(const JSONDocument& jsonDocument) {
-  if (jsonDocument.IsOwner()) {
-    InitFromValue(jsonDocument.value_);
-  } else {
-    value_ = jsonDocument.value_;
-  }
-}
-
-JSONDocument::JSONDocument(JSONDocument&& jsonDocument) {
-  value_ = jsonDocument.value_;
-  data_.swap(jsonDocument.data_);
-}
-
-JSONDocument& JSONDocument::operator=(JSONDocument jsonDocument) {
-  value_ = jsonDocument.value_;
-  data_.swap(jsonDocument.data_);
-  return *this;
-}
-
-JSONDocument::Type JSONDocument::type() const {
-  switch (value_->type()) {
-    case fbson::FbsonType::T_Null:
-      return JSONDocument::kNull;
-
-    case fbson::FbsonType::T_True:
-    case fbson::FbsonType::T_False:
-      return JSONDocument::kBool;
-
-    case fbson::FbsonType::T_Int8:
-    case fbson::FbsonType::T_Int16:
-    case fbson::FbsonType::T_Int32:
-    case fbson::FbsonType::T_Int64:
-      return JSONDocument::kInt64;
-
-    case fbson::FbsonType::T_Double:
-      return JSONDocument::kDouble;
-
-    case fbson::FbsonType::T_String:
-      return JSONDocument::kString;
-
-    case fbson::FbsonType::T_Object:
-      return JSONDocument::kObject;
-
-    case fbson::FbsonType::T_Array:
-      return JSONDocument::kArray;
-
-    case fbson::FbsonType::T_Binary:
-    default:
-      assert(false);
-  }
-  return JSONDocument::kNull;
-}
-
-bool JSONDocument::Contains(const std::string& key) const {
-  assert(IsObject());
-  auto objectVal = reinterpret_cast<fbson::ObjectVal*>(value_);
-  return objectVal->find(key.c_str()) != nullptr;
-}
-
-JSONDocument JSONDocument::operator[](const std::string& key) const {
-  assert(IsObject());
-  auto objectVal = reinterpret_cast<fbson::ObjectVal*>(value_);
-  auto foundValue = objectVal->find(key.c_str());
-  assert(foundValue != nullptr);
-  // No need to save paths in const objects
-  JSONDocument ans(foundValue, false);
-  return ans;
-}
-
-size_t JSONDocument::Count() const {
-  assert(IsObject() || IsArray());
-  if (IsObject()) {
-    // TODO(stash): add to fbson?
-    const fbson::ObjectVal& objectVal =
-          *reinterpret_cast<fbson::ObjectVal*>(value_);
-    return ObjectNumElem(objectVal);
-  } else if (IsArray()) {
-    auto arrayVal = reinterpret_cast<fbson::ArrayVal*>(value_);
-    return arrayVal->numElem();
-  }
-  assert(false);
-  return 0;
-}
-
-JSONDocument JSONDocument::operator[](size_t i) const {
-  assert(IsArray());
-  auto arrayVal = reinterpret_cast<fbson::ArrayVal*>(value_);
-  auto foundValue = arrayVal->get(static_cast<int>(i));
-  JSONDocument ans(foundValue, false);
-  return ans;
-}
-
-bool JSONDocument::IsNull() const {
-  return value_->isNull();
-}
-
-bool JSONDocument::IsArray() const {
-  return value_->isArray();
-}
-
-bool JSONDocument::IsBool() const {
-  return value_->isTrue() || value_->isFalse();
-}
-
-bool JSONDocument::IsDouble() const {
-  return value_->isDouble();
-}
-
-bool JSONDocument::IsInt64() const {
-  return value_->isInt8() || value_->isInt16() ||
-         value_->isInt32() || value_->isInt64();
-}
-
-bool JSONDocument::IsObject() const {
-  return value_->isObject();
-}
-
-bool JSONDocument::IsString() const {
-  return value_->isString();
-}
-
-bool JSONDocument::GetBool() const {
-  assert(IsBool());
-  return value_->isTrue();
-}
-
-double JSONDocument::GetDouble() const {
-  assert(IsDouble());
-  return ((fbson::DoubleVal*)value_)->val();
-}
-
-int64_t JSONDocument::GetInt64() const {
-  assert(IsInt64());
-  return GetInt64ValFromFbsonNumericType(value_);
-}
-
-std::string JSONDocument::GetString() const {
-  assert(IsString());
-  fbson::StringVal* stringVal = (fbson::StringVal*)value_;
-  return std::string(stringVal->getBlob(), stringVal->getBlobLen());
-}
-
-namespace {
-
-// FbsonValue can be int8, int16, int32, int64
-bool CompareNumeric(fbson::FbsonValue* left, fbson::FbsonValue* right) {
-  assert(IsNumeric(left) && IsNumeric(right));
-  return GetInt64ValFromFbsonNumericType(left) ==
-         GetInt64ValFromFbsonNumericType(right);
-}
-
-bool CompareSimpleTypes(fbson::FbsonValue* left, fbson::FbsonValue* right) {
-  if (IsNumeric(left)) {
-    return CompareNumeric(left, right);
-  }
-  if (left->numPackedBytes() != right->numPackedBytes()) {
-    return false;
-  }
-  return memcmp(left, right, left->numPackedBytes()) == 0;
-}
-
-bool CompareFbsonValue(fbson::FbsonValue* left, fbson::FbsonValue* right) {
-  if (!IsComparable(left, right)) {
-    return false;
-  }
-
-  switch (left->type()) {
-    case fbson::FbsonType::T_True:
-    case fbson::FbsonType::T_False:
-    case fbson::FbsonType::T_Null:
-      return true;
-    case fbson::FbsonType::T_Int8:
-    case fbson::FbsonType::T_Int16:
-    case fbson::FbsonType::T_Int32:
-    case fbson::FbsonType::T_Int64:
-      return CompareNumeric(left, right);
-    case fbson::FbsonType::T_String:
-    case fbson::FbsonType::T_Double:
-      return CompareSimpleTypes(left, right);
-    case fbson::FbsonType::T_Object:
-    {
-      auto leftObject = reinterpret_cast<fbson::ObjectVal*>(left);
-      auto rightObject = reinterpret_cast<fbson::ObjectVal*>(right);
-      if (ObjectNumElem(*leftObject) != ObjectNumElem(*rightObject)) {
-        return false;
-      }
-      for (auto && keyValue : *leftObject) {
-        std::string str(keyValue.getKeyStr(), keyValue.klen());
-        if (rightObject->find(str.c_str()) == nullptr) {
-          return false;
-        }
-        if (!CompareFbsonValue(keyValue.value(),
-                               rightObject->find(str.c_str()))) {
-          return false;
-        }
-      }
-      return true;
-    }
-    case fbson::FbsonType::T_Array:
-    {
-      auto leftArr = reinterpret_cast<fbson::ArrayVal*>(left);
-      auto rightArr = reinterpret_cast<fbson::ArrayVal*>(right);
-      if (leftArr->numElem() != rightArr->numElem()) {
-        return false;
-      }
-      for (int i = 0; i < static_cast<int>(leftArr->numElem()); ++i) {
-        if (!CompareFbsonValue(leftArr->get(i), rightArr->get(i))) {
-          return false;
-        }
-      }
-      return true;
-    }
-    default:
-      assert(false);
-  }
-  return false;
-}
-
-}  // namespace
-
-bool JSONDocument::operator==(const JSONDocument& rhs) const {
-  return CompareFbsonValue(value_, rhs.value_);
-}
-
-bool JSONDocument::operator!=(const JSONDocument& rhs) const {
-  return !(*this == rhs);
-}
-
-JSONDocument JSONDocument::Copy() const {
-  return JSONDocument(value_, true);
-}
-
-bool JSONDocument::IsOwner() const {
-  return data_.get() != nullptr;
-}
-
-std::string JSONDocument::DebugString() const {
-  fbson::FbsonToJson fbsonToJson;
-  return fbsonToJson.json(value_);
-}
-
-JSONDocument::ItemsIteratorGenerator JSONDocument::Items() const {
-  assert(IsObject());
-  return ItemsIteratorGenerator(*(reinterpret_cast<fbson::ObjectVal*>(value_)));
-}
-
-// TODO(icanadi) (perf) allocate objects with arena
-JSONDocument* JSONDocument::ParseJSON(const char* json) {
-  fbson::FbsonJsonParser parser;
-  if (!parser.parse(json)) {
-    return nullptr;
-  }
-
-  auto fbsonVal = fbson::FbsonDocument::createValue(
-                    parser.getWriter().getOutput()->getBuffer(),
-              static_cast<uint32_t>(parser.getWriter().getOutput()->getSize()));
-
-  if (fbsonVal == nullptr) {
-    return nullptr;
-  }
-
-  return new JSONDocument(fbsonVal, true);
-}
-
-void JSONDocument::Serialize(std::string* dst) const {
-  // first byte is reserved for header
-  // currently, header is only version number. that will help us provide
-  // backwards compatility. we might also store more information here if
-  // necessary
-  dst->push_back(kSerializationFormatVersion);
-  dst->push_back(FBSON_VER);
-  dst->append(reinterpret_cast<char*>(value_), value_->numPackedBytes());
-}
-
-const char JSONDocument::kSerializationFormatVersion = 2;
-
-JSONDocument* JSONDocument::Deserialize(const Slice& src) {
-  Slice input(src);
-  if (src.size() == 0) {
-    return nullptr;
-  }
-  char header = input[0];
-  if (header == 1) {
-    assert(false);
-  }
-  input.remove_prefix(1);
-  auto value = fbson::FbsonDocument::createValue(input.data(),
-                static_cast<uint32_t>(input.size()));
-  if (value == nullptr) {
-    return nullptr;
-  }
-
-  return new JSONDocument(value, true);
-}
-
-class JSONDocument::const_item_iterator::Impl {
- public:
-  typedef fbson::ObjectVal::const_iterator It;
-
-  explicit Impl(It it) : it_(it) {}
-
-  const char* getKeyStr() const {
-    return it_->getKeyStr();
-  }
-
-  uint8_t klen() const {
-    return it_->klen();
-  }
-
-  It& operator++() {
-    return ++it_;
-  }
-
-  bool operator!=(const Impl& other) {
-    return it_ != other.it_;
-  }
-
-  fbson::FbsonValue* value() const {
-    return it_->value();
-  }
-
- private:
-  It it_;
-};
-
-JSONDocument::const_item_iterator::const_item_iterator(Impl* impl)
-: it_(impl) {}
-
-JSONDocument::const_item_iterator::const_item_iterator(const_item_iterator&& a)
-: it_(std::move(a.it_)) {}
-
-JSONDocument::const_item_iterator&
-  JSONDocument::const_item_iterator::operator++() {
-  ++(*it_);
-  return *this;
-}
-
-bool JSONDocument::const_item_iterator::operator!=(
-                                  const const_item_iterator& other) {
-  return *it_ != *(other.it_);
-}
-
-JSONDocument::const_item_iterator::~const_item_iterator() {
-}
-
-JSONDocument::const_item_iterator::value_type
-  JSONDocument::const_item_iterator::operator*() {
-  return JSONDocument::const_item_iterator::value_type(std::string(it_->getKeyStr(), it_->klen()),
-    JSONDocument(it_->value(), false));
-}
-
-JSONDocument::ItemsIteratorGenerator::ItemsIteratorGenerator(
-                                      const fbson::ObjectVal& object)
-  : object_(object) {}
-
-JSONDocument::const_item_iterator
-      JSONDocument::ItemsIteratorGenerator::begin() const {
-  return const_item_iterator(new const_item_iterator::Impl(object_.begin()));
-}
-
-JSONDocument::const_item_iterator
-      JSONDocument::ItemsIteratorGenerator::end() const {
-  return const_item_iterator(new const_item_iterator::Impl(object_.end()));
-}
-
-}  // namespace rocksdb
-#endif  // ROCKSDB_LITE

utilities/document/json_document_builder.cc

@@ -1,120 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#ifndef ROCKSDB_LITE
-#include <assert.h>
-#include <limits>
-#include <stdint.h>
-#include "rocksdb/utilities/json_document.h"
-#include "third-party/fbson/FbsonWriter.h"
-
-namespace rocksdb {
-JSONDocumentBuilder::JSONDocumentBuilder()
-: writer_(new fbson::FbsonWriter()) {
-}
-
-JSONDocumentBuilder::JSONDocumentBuilder(fbson::FbsonOutStream* out)
-: writer_(new fbson::FbsonWriter(*out)) {
-}
-
-void JSONDocumentBuilder::Reset() {
-  writer_->reset();
-}
-
-bool JSONDocumentBuilder::WriteStartArray() {
-  return writer_->writeStartArray();
-}
-
-bool JSONDocumentBuilder::WriteEndArray() {
-  return writer_->writeEndArray();
-}
-
-bool JSONDocumentBuilder::WriteStartObject() {
-  return writer_->writeStartObject();
-}
-
-bool JSONDocumentBuilder::WriteEndObject() {
-  return writer_->writeEndObject();
-}
-
-bool JSONDocumentBuilder::WriteKeyValue(const std::string& key,
-                                        const JSONDocument& value) {
-  assert(key.size() <= std::numeric_limits<uint8_t>::max());
-  size_t bytesWritten = writer_->writeKey(key.c_str(),
-    static_cast<uint8_t>(key.size()));
-  if (bytesWritten == 0) {
-    return false;
-  }
-  return WriteJSONDocument(value);
-}
-
-bool JSONDocumentBuilder::WriteJSONDocument(const JSONDocument& value) {
-  switch (value.type()) {
-    case JSONDocument::kNull:
-      return writer_->writeNull() != 0;
-    case JSONDocument::kInt64:
-      return writer_->writeInt64(value.GetInt64());
-    case JSONDocument::kDouble:
-      return writer_->writeDouble(value.GetDouble());
-    case JSONDocument::kBool:
-      return writer_->writeBool(value.GetBool());
-    case JSONDocument::kString:
-    {
-      bool res = writer_->writeStartString();
-      if (!res) {
-        return false;
-      }
-      const std::string& str = value.GetString();
-      res = writer_->writeString(str.c_str(),
-                  static_cast<uint32_t>(str.size()));
-      if (!res) {
-        return false;
-      }
-      return writer_->writeEndString();
-    }
-    case JSONDocument::kArray:
-    {
-      bool res = WriteStartArray();
-      if (!res) {
-        return false;
-      }
-      for (size_t i = 0; i < value.Count(); ++i) {
-        res = WriteJSONDocument(value[i]);
-        if (!res) {
-          return false;
-        }
-      }
-      return WriteEndArray();
-    }
-    case JSONDocument::kObject:
-    {
-      bool res = WriteStartObject();
-      if (!res) {
-        return false;
-      }
-      for (auto keyValue : value.Items()) {
-        WriteKeyValue(keyValue.first, keyValue.second);
-      }
-      return WriteEndObject();
-    }
-    default:
-      assert(false);
-  }
-  return false;
-}
-
-JSONDocument JSONDocumentBuilder::GetJSONDocument() {
-  fbson::FbsonValue* value =
-      fbson::FbsonDocument::createValue(writer_->getOutput()->getBuffer(),
-                       static_cast<uint32_t>(writer_->getOutput()->getSize()));
-  return JSONDocument(value, true);
-}
-
-JSONDocumentBuilder::~JSONDocumentBuilder() {
-}
-
-}  // namespace rocksdb
-
-#endif  // ROCKSDB_LITE

utilities/document/json_document_test.cc

@@ -1,343 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#ifndef ROCKSDB_LITE
-
-#include <map>
-#include <set>
-#include <string>
-
-#include "rocksdb/utilities/json_document.h"
-
-#include "util/testutil.h"
-#include "util/testharness.h"
-
-namespace rocksdb {
-namespace {
-void AssertField(const JSONDocument& json, const std::string& field) {
-  ASSERT_TRUE(json.Contains(field));
-  ASSERT_TRUE(json[field].IsNull());
-}
-
-void AssertField(const JSONDocument& json, const std::string& field,
-                 const std::string& expected) {
-  ASSERT_TRUE(json.Contains(field));
-  ASSERT_TRUE(json[field].IsString());
-  ASSERT_EQ(expected, json[field].GetString());
-}
-
-void AssertField(const JSONDocument& json, const std::string& field,
-                 int64_t expected) {
-  ASSERT_TRUE(json.Contains(field));
-  ASSERT_TRUE(json[field].IsInt64());
-  ASSERT_EQ(expected, json[field].GetInt64());
-}
-
-void AssertField(const JSONDocument& json, const std::string& field,
-                 bool expected) {
-  ASSERT_TRUE(json.Contains(field));
-  ASSERT_TRUE(json[field].IsBool());
-  ASSERT_EQ(expected, json[field].GetBool());
-}
-
-void AssertField(const JSONDocument& json, const std::string& field,
-                 double expected) {
-  ASSERT_TRUE(json.Contains(field));
-  ASSERT_TRUE(json[field].IsDouble());
-  ASSERT_DOUBLE_EQ(expected, json[field].GetDouble());
-}
-}  // namespace
-
-class JSONDocumentTest : public testing::Test {
- public:
-  JSONDocumentTest()
-  : rnd_(101)
-  {}
-
-  void AssertSampleJSON(const JSONDocument& json) {
-    AssertField(json, "title", std::string("json"));
-    AssertField(json, "type", std::string("object"));
-    // properties
-    ASSERT_TRUE(json.Contains("properties"));
-    ASSERT_TRUE(json["properties"].Contains("flags"));
-    ASSERT_TRUE(json["properties"]["flags"].IsArray());
-    ASSERT_EQ(3u, json["properties"]["flags"].Count());
-    ASSERT_TRUE(json["properties"]["flags"][0].IsInt64());
-    ASSERT_EQ(10, json["properties"]["flags"][0].GetInt64());
-    ASSERT_TRUE(json["properties"]["flags"][1].IsString());
-    ASSERT_EQ("parse", json["properties"]["flags"][1].GetString());
-    ASSERT_TRUE(json["properties"]["flags"][2].IsObject());
-    AssertField(json["properties"]["flags"][2], "tag", std::string("no"));
-    AssertField(json["properties"]["flags"][2], std::string("status"));
-    AssertField(json["properties"], "age", 110.5e-4);
-    AssertField(json["properties"], "depth", static_cast<int64_t>(-10));
-    // test iteration
-    std::set<std::string> expected({"flags", "age", "depth"});
-    for (auto item : json["properties"].Items()) {
-      auto iter = expected.find(item.first);
-      ASSERT_TRUE(iter != expected.end());
-      expected.erase(iter);
-    }
-    ASSERT_EQ(0U, expected.size());
-    ASSERT_TRUE(json.Contains("latlong"));
-    ASSERT_TRUE(json["latlong"].IsArray());
-    ASSERT_EQ(2u, json["latlong"].Count());
-    ASSERT_TRUE(json["latlong"][0].IsDouble());
-    ASSERT_EQ(53.25, json["latlong"][0].GetDouble());
-    ASSERT_TRUE(json["latlong"][1].IsDouble());
-    ASSERT_EQ(43.75, json["latlong"][1].GetDouble());
-    AssertField(json, "enabled", true);
-  }
-
-  const std::string kSampleJSON =
-      "{ \"title\" : \"json\", \"type\" : \"object\", \"properties\" : { "
-      "\"flags\": [10, \"parse\", {\"tag\": \"no\", \"status\": null}], "
-      "\"age\": 110.5e-4, \"depth\": -10 }, \"latlong\": [53.25, 43.75], "
-      "\"enabled\": true }";
-
-  const std::string kSampleJSONDifferent =
-      "{ \"title\" : \"json\", \"type\" : \"object\", \"properties\" : { "
-      "\"flags\": [10, \"parse\", {\"tag\": \"no\", \"status\": 2}], "
-      "\"age\": 110.5e-4, \"depth\": -10 }, \"latlong\": [53.25, 43.75], "
-      "\"enabled\": true }";
-
-  Random rnd_;
-};
-
-TEST_F(JSONDocumentTest, MakeNullTest) {
-  JSONDocument x;
-  ASSERT_TRUE(x.IsNull());
-  ASSERT_TRUE(x.IsOwner());
-  ASSERT_TRUE(!x.IsBool());
-}
-
-TEST_F(JSONDocumentTest, MakeBoolTest) {
-  {
-    JSONDocument x(true);
-    ASSERT_TRUE(x.IsOwner());
-    ASSERT_TRUE(x.IsBool());
-    ASSERT_TRUE(!x.IsInt64());
-    ASSERT_EQ(x.GetBool(), true);
-  }
-
-  {
-    JSONDocument x(false);
-    ASSERT_TRUE(x.IsOwner());
-    ASSERT_TRUE(x.IsBool());
-    ASSERT_TRUE(!x.IsInt64());
-    ASSERT_EQ(x.GetBool(), false);
-  }
-}
-
-TEST_F(JSONDocumentTest, MakeInt64Test) {
-  JSONDocument x(static_cast<int64_t>(16));
-  ASSERT_TRUE(x.IsInt64());
-  ASSERT_TRUE(x.IsInt64());
-  ASSERT_TRUE(!x.IsBool());
-  ASSERT_TRUE(x.IsOwner());
-  ASSERT_EQ(x.GetInt64(), 16);
-}
-
-TEST_F(JSONDocumentTest, MakeStringTest) {
-  JSONDocument x("string");
-  ASSERT_TRUE(x.IsOwner());
-  ASSERT_TRUE(x.IsString());
-  ASSERT_TRUE(!x.IsBool());
-  ASSERT_EQ(x.GetString(), "string");
-}
-
-TEST_F(JSONDocumentTest, MakeDoubleTest) {
-  JSONDocument x(5.6);
-  ASSERT_TRUE(x.IsOwner());
-  ASSERT_TRUE(x.IsDouble());
-  ASSERT_TRUE(!x.IsBool());
-  ASSERT_EQ(x.GetDouble(), 5.6);
-}
-
-TEST_F(JSONDocumentTest, MakeByTypeTest) {
-  {
-    JSONDocument x(JSONDocument::kNull);
-    ASSERT_TRUE(x.IsNull());
-  }
-  {
-    JSONDocument x(JSONDocument::kBool);
-    ASSERT_TRUE(x.IsBool());
-  }
-  {
-    JSONDocument x(JSONDocument::kString);
-    ASSERT_TRUE(x.IsString());
-  }
-  {
-    JSONDocument x(JSONDocument::kInt64);
-    ASSERT_TRUE(x.IsInt64());
-  }
-  {
-    JSONDocument x(JSONDocument::kDouble);
-    ASSERT_TRUE(x.IsDouble());
-  }
-  {
-    JSONDocument x(JSONDocument::kObject);
-    ASSERT_TRUE(x.IsObject());
-  }
-  {
-    JSONDocument x(JSONDocument::kArray);
-    ASSERT_TRUE(x.IsArray());
-  }
-}
-
-TEST_F(JSONDocumentTest, Parsing) {
-  std::unique_ptr<JSONDocument> parsed_json(
-          JSONDocument::ParseJSON(kSampleJSON.c_str()));
-  ASSERT_TRUE(parsed_json->IsOwner());
-  ASSERT_TRUE(parsed_json != nullptr);
-  AssertSampleJSON(*parsed_json);
-
-  // test deep copying
-  JSONDocument copied_json_document(*parsed_json);
-  AssertSampleJSON(copied_json_document);
-  ASSERT_TRUE(copied_json_document == *parsed_json);
-
-  std::unique_ptr<JSONDocument> parsed_different_sample(
-      JSONDocument::ParseJSON(kSampleJSONDifferent.c_str()));
-  ASSERT_TRUE(parsed_different_sample != nullptr);
-  ASSERT_TRUE(!(*parsed_different_sample == copied_json_document));
-
-  // parse error
-  const std::string kFaultyJSON =
-      kSampleJSON.substr(0, kSampleJSON.size() - 10);
-  ASSERT_TRUE(JSONDocument::ParseJSON(kFaultyJSON.c_str()) == nullptr);
-}
-
-TEST_F(JSONDocumentTest, Serialization) {
-  std::unique_ptr<JSONDocument> parsed_json(
-            JSONDocument::ParseJSON(kSampleJSON.c_str()));
-  ASSERT_TRUE(parsed_json != nullptr);
-  ASSERT_TRUE(parsed_json->IsOwner());
-  std::string serialized;
-  parsed_json->Serialize(&serialized);
-
-  std::unique_ptr<JSONDocument> deserialized_json(
-            JSONDocument::Deserialize(Slice(serialized)));
-  ASSERT_TRUE(deserialized_json != nullptr);
-  AssertSampleJSON(*deserialized_json);
-
-  // deserialization failure
-  ASSERT_TRUE(JSONDocument::Deserialize(
-                  Slice(serialized.data(), serialized.size() - 10)) == nullptr);
-}
-
-TEST_F(JSONDocumentTest, OperatorEqualsTest) {
-  // kNull
-  ASSERT_TRUE(JSONDocument() == JSONDocument());
-
-  // kBool
-  ASSERT_TRUE(JSONDocument(false) != JSONDocument());
-  ASSERT_TRUE(JSONDocument(false) == JSONDocument(false));
-  ASSERT_TRUE(JSONDocument(true) == JSONDocument(true));
-  ASSERT_TRUE(JSONDocument(false) != JSONDocument(true));
-
-  // kString
-  ASSERT_TRUE(JSONDocument("test") != JSONDocument());
-  ASSERT_TRUE(JSONDocument("test") == JSONDocument("test"));
-
-  // kInt64
-  ASSERT_TRUE(JSONDocument(static_cast<int64_t>(15)) != JSONDocument());
-  ASSERT_TRUE(JSONDocument(static_cast<int64_t>(15)) !=
-              JSONDocument(static_cast<int64_t>(14)));
-  ASSERT_TRUE(JSONDocument(static_cast<int64_t>(15)) ==
-              JSONDocument(static_cast<int64_t>(15)));
-
-  std::unique_ptr<JSONDocument> arrayWithInt8Doc(
-      JSONDocument::ParseJSON("[8]"));
-  ASSERT_TRUE(arrayWithInt8Doc != nullptr);
-  ASSERT_TRUE(arrayWithInt8Doc->IsArray());
-  ASSERT_TRUE((*arrayWithInt8Doc)[0].IsInt64());
-  ASSERT_TRUE((*arrayWithInt8Doc)[0] == JSONDocument(static_cast<int64_t>(8)));
-
-  std::unique_ptr<JSONDocument> arrayWithInt16Doc(
-      JSONDocument::ParseJSON("[512]"));
-  ASSERT_TRUE(arrayWithInt16Doc != nullptr);
-  ASSERT_TRUE(arrayWithInt16Doc->IsArray());
-  ASSERT_TRUE((*arrayWithInt16Doc)[0].IsInt64());
-  ASSERT_TRUE((*arrayWithInt16Doc)[0] ==
-              JSONDocument(static_cast<int64_t>(512)));
-
-  std::unique_ptr<JSONDocument> arrayWithInt32Doc(
-      JSONDocument::ParseJSON("[1000000]"));
-  ASSERT_TRUE(arrayWithInt32Doc != nullptr);
-  ASSERT_TRUE(arrayWithInt32Doc->IsArray());
-  ASSERT_TRUE((*arrayWithInt32Doc)[0].IsInt64());
-  ASSERT_TRUE((*arrayWithInt32Doc)[0] ==
-               JSONDocument(static_cast<int64_t>(1000000)));
-
-  // kDouble
-  ASSERT_TRUE(JSONDocument(15.) != JSONDocument());
-  ASSERT_TRUE(JSONDocument(15.) != JSONDocument(14.));
-  ASSERT_TRUE(JSONDocument(15.) == JSONDocument(15.));
-}
-
-TEST_F(JSONDocumentTest, JSONDocumentBuilderTest) {
-  std::unique_ptr<JSONDocument> parsedArray(
-      JSONDocument::ParseJSON("[1, [123, \"a\", \"b\"], {\"b\":\"c\"}]"));
-  ASSERT_TRUE(parsedArray != nullptr);
-
-  JSONDocumentBuilder builder;
-  ASSERT_TRUE(builder.WriteStartArray());
-  ASSERT_TRUE(builder.WriteJSONDocument(1));
-
-  ASSERT_TRUE(builder.WriteStartArray());
-    ASSERT_TRUE(builder.WriteJSONDocument(123));
-    ASSERT_TRUE(builder.WriteJSONDocument("a"));
-    ASSERT_TRUE(builder.WriteJSONDocument("b"));
-  ASSERT_TRUE(builder.WriteEndArray());
-
-  ASSERT_TRUE(builder.WriteStartObject());
-    ASSERT_TRUE(builder.WriteKeyValue("b", "c"));
-  ASSERT_TRUE(builder.WriteEndObject());
-
-  ASSERT_TRUE(builder.WriteEndArray());
-
-  ASSERT_TRUE(*parsedArray == builder.GetJSONDocument());
-}
-
-TEST_F(JSONDocumentTest, OwnershipTest) {
-  std::unique_ptr<JSONDocument> parsed(
-          JSONDocument::ParseJSON(kSampleJSON.c_str()));
-  ASSERT_TRUE(parsed != nullptr);
-  ASSERT_TRUE(parsed->IsOwner());
-
-  // Copy constructor from owner -> owner
-  JSONDocument copy_constructor(*parsed);
-  ASSERT_TRUE(copy_constructor.IsOwner());
-
-  // Copy constructor from non-owner -> non-owner
-  JSONDocument non_owner((*parsed)["properties"]);
-  ASSERT_TRUE(!non_owner.IsOwner());
-
-  // Move constructor from owner -> owner
-  JSONDocument moved_from_owner(std::move(copy_constructor));
-  ASSERT_TRUE(moved_from_owner.IsOwner());
-
-  // Move constructor from non-owner -> non-owner
-  JSONDocument moved_from_non_owner(std::move(non_owner));
-  ASSERT_TRUE(!moved_from_non_owner.IsOwner());
-}
-
-}  //  namespace rocksdb
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
-
-#else
-#include <stdio.h>
-
-int main(int /*argc*/, char** /*argv*/) {
-  fprintf(stderr, "SKIPPED as JSONDocument is not supported in ROCKSDB_LITE\n");
-  return 0;
-}
-
-#endif  // !ROCKSDB_LITE

utilities/geodb/geodb_impl.cc

@@ -1,478 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-#ifndef ROCKSDB_LITE
-
-#include "utilities/geodb/geodb_impl.h"
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <limits>
-#include <map>
-#include <string>
-#include <vector>
-#include "util/coding.h"
-#include "util/filename.h"
-#include "util/string_util.h"
-
-//
-// There are two types of keys. The first type of key-values
-// maps a geo location to the set of object ids and their values.
-// Table 1
-//   key     : p + : + $quadkey + : + $id +
-//             : + $latitude + : + $longitude
-//   value  :  value of the object
-// This table can be used to find all objects that reside near
-// a specified geolocation.
-//
-// Table 2
-//   key  : 'k' + : + $id
-//   value:  $quadkey
-
-namespace rocksdb {
-
-const double GeoDBImpl::PI = 3.141592653589793;
-const double GeoDBImpl::EarthRadius = 6378137;
-const double GeoDBImpl::MinLatitude = -85.05112878;
-const double GeoDBImpl::MaxLatitude = 85.05112878;
-const double GeoDBImpl::MinLongitude = -180;
-const double GeoDBImpl::MaxLongitude = 180;
-
-GeoDBImpl::GeoDBImpl(DB* db, const GeoDBOptions& options) :
-  GeoDB(db, options), db_(db), options_(options) {
-}
-
-GeoDBImpl::~GeoDBImpl() {
-}
-
-Status GeoDBImpl::Insert(const GeoObject& obj) {
-  WriteBatch batch;
-
-  // It is possible that this id is already associated with
-  // with a different position. We first have to remove that
-  // association before we can insert the new one.
-
-  // remove existing object, if it exists
-  GeoObject old;
-  Status status = GetById(obj.id, &old);
-  if (status.ok()) {
-    assert(obj.id.compare(old.id) == 0);
-    std::string quadkey = PositionToQuad(old.position, Detail);
-    std::string key1 = MakeKey1(old.position, old.id, quadkey);
-    std::string key2 = MakeKey2(old.id);
-    batch.Delete(Slice(key1));
-    batch.Delete(Slice(key2));
-  } else if (status.IsNotFound()) {
-    // What if another thread is trying to insert the same ID concurrently?
-  } else {
-    return status;
-  }
-
-  // insert new object
-  std::string quadkey = PositionToQuad(obj.position, Detail);
-  std::string key1 = MakeKey1(obj.position, obj.id, quadkey);
-  std::string key2 = MakeKey2(obj.id);
-  batch.Put(Slice(key1), Slice(obj.value));
-  batch.Put(Slice(key2), Slice(quadkey));
-  return db_->Write(woptions_, &batch);
-}
-
-Status GeoDBImpl::GetByPosition(const GeoPosition& pos,
-                                const Slice& id,
-                                std::string* value) {
-  std::string quadkey = PositionToQuad(pos, Detail);
-  std::string key1 = MakeKey1(pos, id, quadkey);
-  return db_->Get(roptions_, Slice(key1), value);
-}
-
-Status GeoDBImpl::GetById(const Slice& id, GeoObject* object) {
-  Status status;
-  std::string quadkey;
-
-  // create an iterator so that we can get a consistent picture
-  // of the database.
-  Iterator* iter = db_->NewIterator(roptions_);
-
-  // create key for table2
-  std::string kt = MakeKey2(id);
-  Slice key2(kt);
-
-  iter->Seek(key2);
-  if (iter->Valid() && iter->status().ok()) {
-    if (iter->key().compare(key2) == 0) {
-      quadkey = iter->value().ToString();
-    }
-  }
-  if (quadkey.size() == 0) {
-    delete iter;
-    return Status::NotFound(key2);
-  }
-
-  //
-  // Seek to the quadkey + id prefix
-  //
-  std::string prefix = MakeKey1Prefix(quadkey, id);
-  iter->Seek(Slice(prefix));
-  assert(iter->Valid());
-  if (!iter->Valid() || !iter->status().ok()) {
-    delete iter;
-    return Status::NotFound();
-  }
-
-  // split the key into p + quadkey + id + lat + lon
-  Slice key = iter->key();
-  std::vector<std::string> parts = StringSplit(key.ToString(), ':');
-  assert(parts.size() == 5);
-  assert(parts[0] == "p");
-  assert(parts[1] == quadkey);
-  assert(parts[2] == id);
-
-  // fill up output parameters
-  object->position.latitude = atof(parts[3].c_str());
-  object->position.longitude = atof(parts[4].c_str());
-  object->id = id.ToString();  // this is redundant
-  object->value = iter->value().ToString();
-  delete iter;
-  return Status::OK();
-}
-
-
-Status GeoDBImpl::Remove(const Slice& id) {
-  // Read the object from the database
-  GeoObject obj;
-  Status status = GetById(id, &obj);
-  if (!status.ok()) {
-    return status;
-  }
-
-  // remove the object by atomically deleting it from both tables
-  std::string quadkey = PositionToQuad(obj.position, Detail);
-  std::string key1 = MakeKey1(obj.position, obj.id, quadkey);
-  std::string key2 = MakeKey2(obj.id);
-  WriteBatch batch;
-  batch.Delete(Slice(key1));
-  batch.Delete(Slice(key2));
-  return db_->Write(woptions_, &batch);
-}
-
-class GeoIteratorImpl : public GeoIterator {
- private:
-  std::vector<GeoObject> values_;
-  std::vector<GeoObject>::iterator iter_;
- public:
-  explicit GeoIteratorImpl(std::vector<GeoObject> values)
-    : values_(std::move(values)) {
-    iter_ = values_.begin();
-  }
-  virtual void Next() override;
-  virtual bool Valid() const override;
-  virtual const GeoObject& geo_object() override;
-  virtual Status status() const override;
-};
-
-class GeoErrorIterator : public GeoIterator {
- private:
-  Status status_;
- public:
-  explicit GeoErrorIterator(Status s) : status_(s) {}
-  virtual void Next() override {};
-  virtual bool Valid() const override { return false; }
-  virtual const GeoObject& geo_object() override {
-    GeoObject* g = new GeoObject();
-    return *g;
-  }
-  virtual Status status() const override { return status_; }
-};
-
-void GeoIteratorImpl::Next() {
-  assert(Valid());
-  iter_++;
-}
-
-bool GeoIteratorImpl::Valid() const {
-  return iter_ != values_.end();
-}
-
-const GeoObject& GeoIteratorImpl::geo_object() {
-  assert(Valid());
-  return *iter_;
-}
-
-Status GeoIteratorImpl::status() const {
-  return Status::OK();
-}
-
-GeoIterator* GeoDBImpl::SearchRadial(const GeoPosition& pos,
-  double radius,
-  int number_of_values) {
-  std::vector<GeoObject> values;
-
-  // Gather all bounding quadkeys
-  std::vector<std::string> qids;
-  Status s = searchQuadIds(pos, radius, &qids);
-  if (!s.ok()) {
-    return new GeoErrorIterator(s);
-  }
-
-  // create an iterator
-  Iterator* iter = db_->NewIterator(ReadOptions());
-
-  // Process each prospective quadkey
-  for (const std::string& qid : qids) {
-    // The user is interested in only these many objects.
-    if (number_of_values == 0) {
-      break;
-    }
-
-    // convert quadkey to db key prefix
-    std::string dbkey = MakeQuadKeyPrefix(qid);
-
-    for (iter->Seek(dbkey);
-         number_of_values > 0 && iter->Valid() && iter->status().ok();
-         iter->Next()) {
-      // split the key into p + quadkey + id + lat + lon
-      Slice key = iter->key();
-      std::vector<std::string> parts = StringSplit(key.ToString(), ':');
-      assert(parts.size() == 5);
-      assert(parts[0] == "p");
-      std::string* quadkey = &parts[1];
-
-      // If the key we are looking for is a prefix of the key
-      // we found from the database, then this is one of the keys
-      // we are looking for.
-      auto res = std::mismatch(qid.begin(), qid.end(), quadkey->begin());
-      if (res.first == qid.end()) {
-        GeoPosition obj_pos(atof(parts[3].c_str()), atof(parts[4].c_str()));
-        GeoObject obj(obj_pos, parts[2], iter->value().ToString());
-        values.push_back(obj);
-        number_of_values--;
-      } else {
-        break;
-      }
-    }
-  }
-  delete iter;
-  return new GeoIteratorImpl(std::move(values));
-}
-
-std::string GeoDBImpl::MakeKey1(const GeoPosition& pos, Slice id,
-                                std::string quadkey) {
-  std::string lat = rocksdb::ToString(pos.latitude);
-  std::string lon = rocksdb::ToString(pos.longitude);
-  std::string key = "p:";
-  key.reserve(5 + quadkey.size() + id.size() + lat.size() + lon.size());
-  key.append(quadkey);
-  key.append(":");
-  key.append(id.ToString());
-  key.append(":");
-  key.append(lat);
-  key.append(":");
-  key.append(lon);
-  return key;
-}
-
-std::string GeoDBImpl::MakeKey2(Slice id) {
-  std::string key = "k:";
-  key.append(id.ToString());
-  return key;
-}
-
-std::string GeoDBImpl::MakeKey1Prefix(std::string quadkey,
-                                      Slice id) {
-  std::string key = "p:";
-  key.reserve(4 + quadkey.size() + id.size());
-  key.append(quadkey);
-  key.append(":");
-  key.append(id.ToString());
-  key.append(":");
-  return key;
-}
-
-std::string GeoDBImpl::MakeQuadKeyPrefix(std::string quadkey) {
-  std::string key = "p:";
-  key.append(quadkey);
-  return key;
-}
-
-// convert degrees to radians
-double GeoDBImpl::radians(double x) {
-  return (x * PI) / 180;
-}
-
-// convert radians to degrees
-double GeoDBImpl::degrees(double x) {
-  return (x * 180) / PI;
-}
-
-// convert a gps location to quad coordinate
-std::string GeoDBImpl::PositionToQuad(const GeoPosition& pos,
-                                      int levelOfDetail) {
-  Pixel p = PositionToPixel(pos, levelOfDetail);
-  Tile tile = PixelToTile(p);
-  return TileToQuadKey(tile, levelOfDetail);
-}
-
-GeoPosition GeoDBImpl::displaceLatLon(double lat, double lon,
-                                      double deltay, double deltax) {
-  double dLat = deltay / EarthRadius;
-  double dLon = deltax / (EarthRadius * cos(radians(lat)));
-  return GeoPosition(lat + degrees(dLat),
-                     lon + degrees(dLon));
-}
-
-//
-// Return the distance between two positions on the earth
-//
-double GeoDBImpl::distance(double lat1, double lon1,
-                           double lat2, double lon2) {
-  double lon = radians(lon2 - lon1);
-  double lat = radians(lat2 - lat1);
-
-  double a = (sin(lat / 2) * sin(lat / 2)) +
-              cos(radians(lat1)) * cos(radians(lat2)) *
-              (sin(lon / 2) * sin(lon / 2));
-  double angle = 2 * atan2(sqrt(a), sqrt(1 - a));
-  return angle * EarthRadius;
-}
-
-//
-// Returns all the quadkeys inside the search range
-//
-Status GeoDBImpl::searchQuadIds(const GeoPosition& position,
-                                double radius,
-                                std::vector<std::string>* quadKeys) {
-  // get the outline of the search square
-  GeoPosition topLeftPos = boundingTopLeft(position, radius);
-  GeoPosition bottomRightPos = boundingBottomRight(position, radius);
-
-  Pixel topLeft =  PositionToPixel(topLeftPos, Detail);
-  Pixel bottomRight =  PositionToPixel(bottomRightPos, Detail);
-
-  // how many level of details to look for
-  int numberOfTilesAtMaxDepth = static_cast<int>(std::floor((bottomRight.x - topLeft.x) / 256));
-  int zoomLevelsToRise = static_cast<int>(std::floor(std::log(numberOfTilesAtMaxDepth) / std::log(2)));
-  zoomLevelsToRise++;
-  int levels = std::max(0, Detail - zoomLevelsToRise);
-
-  quadKeys->push_back(PositionToQuad(GeoPosition(topLeftPos.latitude,
-                                                 topLeftPos.longitude),
-                                     levels));
-  quadKeys->push_back(PositionToQuad(GeoPosition(topLeftPos.latitude,
-                                                 bottomRightPos.longitude),
-                                     levels));
-  quadKeys->push_back(PositionToQuad(GeoPosition(bottomRightPos.latitude,
-                                                 topLeftPos.longitude),
-                                     levels));
-  quadKeys->push_back(PositionToQuad(GeoPosition(bottomRightPos.latitude,
-                                                 bottomRightPos.longitude),
-                                     levels));
-  return Status::OK();
-}
-
-// Determines the ground resolution (in meters per pixel) at a specified
-// latitude and level of detail.
-// Latitude (in degrees) at which to measure the ground resolution.
-// Level of detail, from 1 (lowest detail) to 23 (highest detail).
-// Returns the ground resolution, in meters per pixel.
-double GeoDBImpl::GroundResolution(double latitude, int levelOfDetail) {
-  latitude = clip(latitude, MinLatitude, MaxLatitude);
-  return cos(latitude * PI / 180) * 2 * PI * EarthRadius /
-         MapSize(levelOfDetail);
-}
-
-// Converts a point from latitude/longitude WGS-84 coordinates (in degrees)
-// into pixel XY coordinates at a specified level of detail.
-GeoDBImpl::Pixel GeoDBImpl::PositionToPixel(const GeoPosition& pos,
-                                            int levelOfDetail) {
-  double latitude = clip(pos.latitude, MinLatitude, MaxLatitude);
-  double x = (pos.longitude + 180) / 360;
-  double sinLatitude = sin(latitude * PI / 180);
-  double y = 0.5 - std::log((1 + sinLatitude) / (1 - sinLatitude)) / (4 * PI);
-  double mapSize = MapSize(levelOfDetail);
-  double X = std::floor(clip(x * mapSize + 0.5, 0, mapSize - 1));
-  double Y = std::floor(clip(y * mapSize + 0.5, 0, mapSize - 1));
-  return Pixel((unsigned int)X, (unsigned int)Y);
-}
-
-GeoPosition GeoDBImpl::PixelToPosition(const Pixel& pixel, int levelOfDetail) {
-  double mapSize = MapSize(levelOfDetail);
-  double x = (clip(pixel.x, 0, mapSize - 1) / mapSize) - 0.5;
-  double y = 0.5 - (clip(pixel.y, 0, mapSize - 1) / mapSize);
-  double latitude = 90 - 360 * atan(exp(-y * 2 * PI)) / PI;
-  double longitude = 360 * x;
-  return GeoPosition(latitude, longitude);
-}
-
-// Converts a Pixel to a Tile
-GeoDBImpl::Tile GeoDBImpl::PixelToTile(const Pixel& pixel) {
-  unsigned int tileX = static_cast<unsigned int>(std::floor(pixel.x / 256));
-  unsigned int tileY = static_cast<unsigned int>(std::floor(pixel.y / 256));
-  return Tile(tileX, tileY);
-}
-
-GeoDBImpl::Pixel GeoDBImpl::TileToPixel(const Tile& tile) {
-  unsigned int pixelX = tile.x * 256;
-  unsigned int pixelY = tile.y * 256;
-  return Pixel(pixelX, pixelY);
-}
-
-// Convert a Tile to a quadkey
-std::string GeoDBImpl::TileToQuadKey(const Tile& tile, int levelOfDetail) {
-  std::stringstream quadKey;
-  for (int i = levelOfDetail; i > 0; i--) {
-    char digit = '0';
-    int mask = 1 << (i - 1);
-    if ((tile.x & mask) != 0) {
-      digit++;
-    }
-    if ((tile.y & mask) != 0) {
-      digit++;
-      digit++;
-    }
-    quadKey << digit;
-  }
-  return quadKey.str();
-}
-
-//
-// Convert a quadkey to a tile and its level of detail
-//
-void GeoDBImpl::QuadKeyToTile(std::string quadkey, Tile* tile,
-                              int* levelOfDetail) {
-  tile->x = tile->y = 0;
-  *levelOfDetail = static_cast<int>(quadkey.size());
-  const char* key = reinterpret_cast<const char*>(quadkey.c_str());
-  for (int i = *levelOfDetail; i > 0; i--) {
-    int mask = 1 << (i - 1);
-    switch (key[*levelOfDetail - i]) {
-      case '0':
-        break;
-
-      case '1':
-        tile->x |= mask;
-        break;
-
-      case '2':
-        tile->y |= mask;
-        break;
-
-      case '3':
-        tile->x |= mask;
-        tile->y |= mask;
-        break;
-
-      default:
-        std::stringstream msg;
-        msg << quadkey;
-        msg << " Invalid QuadKey.";
-        throw std::runtime_error(msg.str());
-    }
-  }
-}
-}  // namespace rocksdb
-
-#endif  // ROCKSDB_LITE

utilities/geodb/geodb_impl.h

@@ -1,185 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-
-#ifndef ROCKSDB_LITE
-
-#pragma once
-#include <algorithm>
-#include <cmath>
-#include <string>
-#include <sstream>
-#include <stdexcept>
-#include <vector>
-
-#include "rocksdb/utilities/geo_db.h"
-#include "rocksdb/utilities/stackable_db.h"
-#include "rocksdb/env.h"
-#include "rocksdb/status.h"
-
-namespace rocksdb {
-
-// A specific implementation of GeoDB
-
-class GeoDBImpl : public GeoDB {
- public:
-  GeoDBImpl(DB* db, const GeoDBOptions& options);
-  ~GeoDBImpl();
-
-  // Associate the GPS location with the identified by 'id'. The value
-  // is a blob that is associated with this object.
-  virtual Status Insert(const GeoObject& object) override;
-
-  // Retrieve the value of the object located at the specified GPS
-  // location and is identified by the 'id'.
-  virtual Status GetByPosition(const GeoPosition& pos, const Slice& id,
-                               std::string* value) override;
-
-  // Retrieve the value of the object identified by the 'id'. This method
-  // could be potentially slower than GetByPosition
-  virtual Status GetById(const Slice& id, GeoObject* object) override;
-
-  // Delete the specified object
-  virtual Status Remove(const Slice& id) override;
-
-  // Returns a list of all items within a circular radius from the
-  // specified gps location
-  virtual GeoIterator* SearchRadial(const GeoPosition& pos, double radius,
-                                    int number_of_values) override;
-
- private:
-  DB* db_;
-  const GeoDBOptions options_;
-  const WriteOptions woptions_;
-  const ReadOptions roptions_;
-
-  // MSVC requires the definition for this static const to be in .CC file
-  // The value of PI
-  static const double PI;
-
-  // convert degrees to radians
-  static double radians(double x);
-
-  // convert radians to degrees
-  static double degrees(double x);
-
-  // A pixel class that captures X and Y coordinates
-  class Pixel {
-   public:
-    unsigned int x;
-    unsigned int y;
-    Pixel(unsigned int a, unsigned int b) :
-     x(a), y(b) {
-    }
-  };
-
-  // A Tile in the geoid
-  class Tile {
-   public:
-    unsigned int x;
-    unsigned int y;
-    Tile(unsigned int a, unsigned int b) :
-     x(a), y(b) {
-    }
-  };
-
-  // convert a gps location to quad coordinate
-  static std::string PositionToQuad(const GeoPosition& pos, int levelOfDetail);
-
-  // arbitrary constant use for WGS84 via
-  // http://en.wikipedia.org/wiki/World_Geodetic_System
-  // http://mathforum.org/library/drmath/view/51832.html
-  // http://msdn.microsoft.com/en-us/library/bb259689.aspx
-  // http://www.tuicool.com/articles/NBrE73
-  //
-  const int Detail = 23;
-  // MSVC requires the definition for this static const to be in .CC file
-  static const double EarthRadius;
-  static const double MinLatitude;
-  static const double MaxLatitude;
-  static const double MinLongitude;
-  static const double MaxLongitude;
-
-  // clips a number to the specified minimum and maximum values.
-  static double clip(double n, double minValue, double maxValue) {
-    return fmin(fmax(n, minValue), maxValue);
-  }
-
-  // Determines the map width and height (in pixels) at a specified level
-  // of detail, from 1 (lowest detail) to 23 (highest detail).
-  // Returns the map width and height in pixels.
-  static unsigned int MapSize(int levelOfDetail) {
-    return (unsigned int)(256 << levelOfDetail);
-  }
-
-  // Determines the ground resolution (in meters per pixel) at a specified
-  // latitude and level of detail.
-  // Latitude (in degrees) at which to measure the ground resolution.
-  // Level of detail, from 1 (lowest detail) to 23 (highest detail).
-  // Returns the ground resolution, in meters per pixel.
-  static double GroundResolution(double latitude, int levelOfDetail);
-
-  // Converts a point from latitude/longitude WGS-84 coordinates (in degrees)
-  // into pixel XY coordinates at a specified level of detail.
-  static Pixel PositionToPixel(const GeoPosition& pos, int levelOfDetail);
-
-  static GeoPosition PixelToPosition(const Pixel& pixel, int levelOfDetail);
-
-  // Converts a Pixel to a Tile
-  static Tile PixelToTile(const Pixel& pixel);
-
-  static Pixel TileToPixel(const Tile& tile);
-
-  // Convert a Tile to a quadkey
-  static std::string TileToQuadKey(const Tile& tile, int levelOfDetail);
-
-  // Convert a quadkey to a tile and its level of detail
-  static void QuadKeyToTile(std::string quadkey, Tile* tile,
-                            int *levelOfDetail);
-
-  // Return the distance between two positions on the earth
-  static double distance(double lat1, double lon1,
-                         double lat2, double lon2);
-  static GeoPosition displaceLatLon(double lat, double lon,
-                                    double deltay, double deltax);
-
-  //
-  // Returns the top left position after applying the delta to
-  // the specified position
-  //
-  static GeoPosition boundingTopLeft(const GeoPosition& in, double radius) {
-    return displaceLatLon(in.latitude, in.longitude, -radius, -radius);
-  }
-
-  //
-  // Returns the bottom right position after applying the delta to
-  // the specified position
-  static GeoPosition boundingBottomRight(const GeoPosition& in,
-                                         double radius) {
-    return displaceLatLon(in.latitude, in.longitude, radius, radius);
-  }
-
-  //
-  // Get all quadkeys within a radius of a specified position
-  //
-  Status searchQuadIds(const GeoPosition& position,
-                       double radius,
-                       std::vector<std::string>* quadKeys);
-
-  //
-  // Create keys for accessing rocksdb table(s)
-  //
-  static std::string MakeKey1(const GeoPosition& pos,
-                              Slice id,
-                              std::string quadkey);
-  static std::string MakeKey2(Slice id);
-  static std::string MakeKey1Prefix(std::string quadkey,
-                                    Slice id);
-  static std::string MakeQuadKeyPrefix(std::string quadkey);
-};
-
-}  // namespace rocksdb
-
-#endif  // ROCKSDB_LITE

utilities/geodb/geodb_test.cc

@@ -1,201 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-#ifndef ROCKSDB_LITE
-#include "utilities/geodb/geodb_impl.h"
-
-#include <cctype>
-#include "util/testharness.h"
-
-namespace rocksdb {
-
-class GeoDBTest : public testing::Test {
- public:
-  static const std::string kDefaultDbName;
-  static Options options;
-  DB* db;
-  GeoDB* geodb;
-
-  GeoDBTest() {
-    GeoDBOptions geodb_options;
-    EXPECT_OK(DestroyDB(kDefaultDbName, options));
-    options.create_if_missing = true;
-    Status status = DB::Open(options, kDefaultDbName, &db);
-    geodb =  new GeoDBImpl(db, geodb_options);
-  }
-
-  ~GeoDBTest() {
-    delete geodb;
-  }
-
-  GeoDB* getdb() {
-    return geodb;
-  }
-};
-
-const std::string GeoDBTest::kDefaultDbName =
-    test::PerThreadDBPath("geodb_test");
-Options GeoDBTest::options = Options();
-
-// Insert, Get and Remove
-TEST_F(GeoDBTest, SimpleTest) {
-  GeoPosition pos1(100, 101);
-  std::string id1("id1");
-  std::string value1("value1");
-
-  // insert first object into database
-  GeoObject obj1(pos1, id1, value1);
-  Status status = getdb()->Insert(obj1);
-  ASSERT_TRUE(status.ok());
-
-  // insert second object into database
-  GeoPosition pos2(200, 201);
-  std::string id2("id2");
-  std::string value2 = "value2";
-  GeoObject obj2(pos2, id2, value2);
-  status = getdb()->Insert(obj2);
-  ASSERT_TRUE(status.ok());
-
-  // retrieve first object using position
-  std::string value;
-  status = getdb()->GetByPosition(pos1, Slice(id1), &value);
-  ASSERT_TRUE(status.ok());
-  ASSERT_EQ(value, value1);
-
-  // retrieve first object using id
-  GeoObject obj;
-  status = getdb()->GetById(Slice(id1), &obj);
-  ASSERT_TRUE(status.ok());
-  ASSERT_EQ(obj.position.latitude, 100);
-  ASSERT_EQ(obj.position.longitude, 101);
-  ASSERT_EQ(obj.id.compare(id1), 0);
-  ASSERT_EQ(obj.value, value1);
-
-  // delete first object
-  status = getdb()->Remove(Slice(id1));
-  ASSERT_TRUE(status.ok());
-  status = getdb()->GetByPosition(pos1, Slice(id1), &value);
-  ASSERT_TRUE(status.IsNotFound());
-  status = getdb()->GetById(id1, &obj);
-  ASSERT_TRUE(status.IsNotFound());
-
-  // check that we can still find second object
-  status = getdb()->GetByPosition(pos2, id2, &value);
-  ASSERT_TRUE(status.ok());
-  ASSERT_EQ(value, value2);
-  status = getdb()->GetById(id2, &obj);
-  ASSERT_TRUE(status.ok());
-}
-
-// Search.
-// Verify distances via http://www.stevemorse.org/nearest/distance.php
-TEST_F(GeoDBTest, Search) {
-  GeoPosition pos1(45, 45);
-  std::string id1("mid1");
-  std::string value1 = "midvalue1";
-
-  // insert object at 45 degree latitude
-  GeoObject obj1(pos1, id1, value1);
-  Status status = getdb()->Insert(obj1);
-  ASSERT_TRUE(status.ok());
-
-  // search all objects centered at 46 degree latitude with
-  // a radius of 200 kilometers. We should find the one object that
-  // we inserted earlier.
-  GeoIterator* iter1 = getdb()->SearchRadial(GeoPosition(46, 46), 200000);
-  ASSERT_TRUE(status.ok());
-  ASSERT_EQ(iter1->geo_object().value, "midvalue1");
-  uint32_t size = 0;
-  while (iter1->Valid()) {
-    GeoObject obj;
-    status = getdb()->GetById(Slice(id1), &obj);
-    ASSERT_TRUE(status.ok());
-    ASSERT_EQ(iter1->geo_object().position.latitude, pos1.latitude);
-    ASSERT_EQ(iter1->geo_object().position.longitude, pos1.longitude);
-    ASSERT_EQ(iter1->geo_object().id.compare(id1), 0);
-    ASSERT_EQ(iter1->geo_object().value, value1);
-
-    size++;
-    iter1->Next();
-    ASSERT_TRUE(!iter1->Valid());
-  }
-  ASSERT_EQ(size, 1U);
-  delete iter1;
-
-  // search all objects centered at 46 degree latitude with
-  // a radius of 2 kilometers. There should be none.
-  GeoIterator* iter2 = getdb()->SearchRadial(GeoPosition(46, 46), 2);
-  ASSERT_TRUE(status.ok());
-  ASSERT_FALSE(iter2->Valid());
-  delete iter2;
-}
-
-TEST_F(GeoDBTest, DifferentPosInSameQuadkey) {
-  // insert obj1 into database
-  GeoPosition pos1(40.00001, 116.00001);
-  std::string id1("12");
-  std::string value1("value1");
-
-  GeoObject obj1(pos1, id1, value1);
-  Status status = getdb()->Insert(obj1);
-  ASSERT_TRUE(status.ok());
-
-  // insert obj2 into database
-  GeoPosition pos2(40.00002, 116.00002);
-  std::string id2("123");
-  std::string value2 = "value2";
-
-  GeoObject obj2(pos2, id2, value2);
-  status = getdb()->Insert(obj2);
-  ASSERT_TRUE(status.ok());
-
-  // get obj1's quadkey
-  ReadOptions opt;
-  PinnableSlice quadkey1;
-  status = getdb()->Get(opt, getdb()->DefaultColumnFamily(), "k:" + id1, &quadkey1);
-  ASSERT_TRUE(status.ok());
-
-  // get obj2's quadkey
-  PinnableSlice quadkey2;
-  status = getdb()->Get(opt, getdb()->DefaultColumnFamily(), "k:" + id2, &quadkey2);
-  ASSERT_TRUE(status.ok());
-
-  // obj1 and obj2 have the same quadkey
-  ASSERT_EQ(quadkey1, quadkey2);
-
-  // get obj1 by id, and check value
-  GeoObject obj;
-  status = getdb()->GetById(Slice(id1), &obj);
-  ASSERT_TRUE(status.ok());
-  ASSERT_EQ(obj.position.latitude, pos1.latitude);
-  ASSERT_EQ(obj.position.longitude, pos1.longitude);
-  ASSERT_EQ(obj.id.compare(id1), 0);
-  ASSERT_EQ(obj.value, value1);
-
-  // get obj2 by id, and check value
-  status = getdb()->GetById(Slice(id2), &obj);
-  ASSERT_TRUE(status.ok());
-  ASSERT_EQ(obj.position.latitude, pos2.latitude);
-  ASSERT_EQ(obj.position.longitude, pos2.longitude);
-  ASSERT_EQ(obj.id.compare(id2), 0);
-  ASSERT_EQ(obj.value, value2);
-}
-
-}  // namespace rocksdb
-
-int main(int argc, char* argv[]) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
-#else
-
-#include <stdio.h>
-
-int main() {
-  fprintf(stderr, "SKIPPED\n");
-  return 0;
-}
-
-#endif  // !ROCKSDB_LITE

utilities/redis/README

@@ -1,14 +0,0 @@
-This folder defines a REDIS-style interface for Rocksdb.
-Right now it is written as a simple tag-on in the rocksdb::RedisLists class.
-It implements Redis Lists, and supports only the "non-blocking operations".
-
-Internally, the set of lists are stored in a rocksdb database, mapping keys to
-values. Each "value" is the list itself, storing a sequence of "elements".
-Each element is stored as a 32-bit-integer, followed by a sequence of bytes.
-The 32-bit-integer represents the length of the element (that is, the number
-of bytes that follow). And then that many bytes follow.
-
-
-NOTE: This README file may be old. See the actual redis_lists.cc file for
-definitive details on the implementation. There should be a header at the top
-of that file, explaining a bit of the implementation details.

utilities/redis/redis_list_exception.h

@@ -1,22 +0,0 @@
-/**
- * A simple structure for exceptions in RedisLists.
- *
- * @author Deon Nicholas (dnicholas@fb.com)
- * Copyright 2013 Facebook
- */
-
-#pragma once
-#ifndef ROCKSDB_LITE
-#include <exception>
-
-namespace rocksdb {
-
-class RedisListException: public std::exception {
- public:
-  const char* what() const throw() override {
-    return "Invalid operation or corrupt data in Redis List.";
-  }
-};
-
-} // namespace rocksdb
-#endif

utilities/redis/redis_list_iterator.h

@@ -1,309 +0,0 @@
-// Copyright 2013 Facebook
-/**
- * RedisListIterator:
- * An abstraction over the "list" concept (e.g.: for redis lists).
- * Provides functionality to read, traverse, edit, and write these lists.
- *
- * Upon construction, the RedisListIterator is given a block of list data.
- * Internally, it stores a pointer to the data and a pointer to current item.
- * It also stores a "result" list that will be mutated over time.
- *
- * Traversal and mutation are done by "forward iteration".
- * The Push() and Skip() methods will advance the iterator to the next item.
- * However, Push() will also "write the current item to the result".
- * Skip() will simply move to next item, causing current item to be dropped.
- *
- * Upon completion, the result (accessible by WriteResult()) will be saved.
- * All "skipped" items will be gone; all "pushed" items will remain.
- *
- * @throws Any of the operations may throw a RedisListException if an invalid
- *          operation is performed or if the data is found to be corrupt.
- *
- * @notes By default, if WriteResult() is called part-way through iteration,
- *        it will automatically advance the iterator to the end, and Keep()
- *        all items that haven't been traversed yet. This may be subject
- *        to review.
- *
- * @notes Can access the "current" item via GetCurrent(), and other
- *        list-specific information such as Length().
- *
- * @notes The internal representation is due to change at any time. Presently,
- *        the list is represented as follows:
- *          - 32-bit integer header: the number of items in the list
- *          - For each item:
- *              - 32-bit int (n): the number of bytes representing this item
- *              - n bytes of data: the actual data.
- *
- * @author Deon Nicholas (dnicholas@fb.com)
- */
-
-#pragma once
-#ifndef ROCKSDB_LITE
-
-#include <string>
-
-#include "redis_list_exception.h"
-#include "rocksdb/slice.h"
-#include "util/coding.h"
-
-namespace rocksdb {
-
-/// An abstraction over the "list" concept.
-/// All operations may throw a RedisListException
-class RedisListIterator {
- public:
-  /// Construct a redis-list-iterator based on data.
-  /// If the data is non-empty, it must formatted according to @notes above.
-  ///
-  /// If the data is valid, we can assume the following invariant(s):
-  ///  a) length_, num_bytes_ are set correctly.
-  ///  b) cur_byte_ always refers to the start of the current element,
-  ///       just before the bytes that specify element length.
-  ///  c) cur_elem_ is always the index of the current element.
-  ///  d) cur_elem_length_ is always the number of bytes in current element,
-  ///       excluding the 4-byte header itself.
-  ///  e) result_ will always contain data_[0..cur_byte_) and a header
-  ///  f) Whenever corrupt data is encountered or an invalid operation is
-  ///      attempted, a RedisListException will immediately be thrown.
-  explicit RedisListIterator(const std::string& list_data)
-      : data_(list_data.data()),
-        num_bytes_(static_cast<uint32_t>(list_data.size())),
-        cur_byte_(0),
-        cur_elem_(0),
-        cur_elem_length_(0),
-        length_(0),
-        result_() {
-    // Initialize the result_ (reserve enough space for header)
-    InitializeResult();
-
-    // Parse the data only if it is not empty.
-    if (num_bytes_ == 0) {
-      return;
-    }
-
-    // If non-empty, but less than 4 bytes, data must be corrupt
-    if (num_bytes_ < sizeof(length_)) {
-      ThrowError("Corrupt header.");    // Will break control flow
-    }
-
-    // Good. The first bytes specify the number of elements
-    length_ = DecodeFixed32(data_);
-    cur_byte_ = sizeof(length_);
-
-    // If we have at least one element, point to that element.
-    // Also, read the first integer of the element (specifying the size),
-    //   if possible.
-    if (length_ > 0) {
-      if (cur_byte_ + sizeof(cur_elem_length_) <= num_bytes_) {
-        cur_elem_length_ = DecodeFixed32(data_+cur_byte_);
-      } else {
-        ThrowError("Corrupt data for first element.");
-      }
-    }
-
-    // At this point, we are fully set-up.
-    // The invariants described in the header should now be true.
-  }
-
-  /// Reserve some space for the result_.
-  /// Equivalent to result_.reserve(bytes).
-  void Reserve(int bytes) {
-    result_.reserve(bytes);
-  }
-
-  /// Go to next element in data file.
-  /// Also writes the current element to result_.
-  RedisListIterator& Push() {
-    WriteCurrentElement();
-    MoveNext();
-    return *this;
-  }
-
-  /// Go to next element in data file.
-  /// Drops/skips the current element. It will not be written to result_.
-  RedisListIterator& Skip() {
-    MoveNext();
-    --length_;          // One less item
-    --cur_elem_;        // We moved one forward, but index did not change
-    return *this;
-  }
-
-  /// Insert elem into the result_ (just BEFORE the current element / byte)
-  /// Note: if Done() (i.e.: iterator points to end), this will append elem.
-  void InsertElement(const Slice& elem) {
-    // Ensure we are in a valid state
-    CheckErrors();
-
-    const int kOrigSize = static_cast<int>(result_.size());
-    result_.resize(kOrigSize + SizeOf(elem));
-    EncodeFixed32(result_.data() + kOrigSize,
-                  static_cast<uint32_t>(elem.size()));
-    memcpy(result_.data() + kOrigSize + sizeof(uint32_t), elem.data(),
-           elem.size());
-    ++length_;
-    ++cur_elem_;
-  }
-
-  /// Access the current element, and save the result into *curElem
-  void GetCurrent(Slice* curElem) {
-    // Ensure we are in a valid state
-    CheckErrors();
-
-    // Ensure that we are not past the last element.
-    if (Done()) {
-      ThrowError("Invalid dereferencing.");
-    }
-
-    // Dereference the element
-    *curElem = Slice(data_+cur_byte_+sizeof(cur_elem_length_),
-                     cur_elem_length_);
-  }
-
-  // Number of elements
-  int Length() const {
-    return length_;
-  }
-
-  // Number of bytes in the final representation (i.e: WriteResult().size())
-  int Size() const {
-    // result_ holds the currently written data
-    // data_[cur_byte..num_bytes-1] is the remainder of the data
-    return static_cast<int>(result_.size() + (num_bytes_ - cur_byte_));
-  }
-
-  // Reached the end?
-  bool Done() const {
-    return cur_byte_ >= num_bytes_ || cur_elem_ >= length_;
-  }
-
-  /// Returns a string representing the final, edited, data.
-  /// Assumes that all bytes of data_ in the range [0,cur_byte_) have been read
-  ///  and that result_ contains this data.
-  /// The rest of the data must still be written.
-  /// So, this method ADVANCES THE ITERATOR TO THE END before writing.
-  Slice WriteResult() {
-    CheckErrors();
-
-    // The header should currently be filled with dummy data (0's)
-    // Correctly update the header.
-    // Note, this is safe since result_ is a vector (guaranteed contiguous)
-    EncodeFixed32(&result_[0],length_);
-
-    // Append the remainder of the data to the result.
-    result_.insert(result_.end(),data_+cur_byte_, data_ +num_bytes_);
-
-    // Seek to end of file
-    cur_byte_ = num_bytes_;
-    cur_elem_ = length_;
-    cur_elem_length_ = 0;
-
-    // Return the result
-    return Slice(result_.data(),result_.size());
-  }
-
- public: // Static public functions
-
-  /// An upper-bound on the amount of bytes needed to store this element.
-  /// This is used to hide representation information from the client.
-  /// E.G. This can be used to compute the bytes we want to Reserve().
-  static uint32_t SizeOf(const Slice& elem) {
-    // [Integer Length . Data]
-    return static_cast<uint32_t>(sizeof(uint32_t) + elem.size());
-  }
-
- private: // Private functions
-
-  /// Initializes the result_ string.
-  /// It will fill the first few bytes with 0's so that there is
-  ///  enough space for header information when we need to write later.
-  /// Currently, "header information" means: the length (number of elements)
-  /// Assumes that result_ is empty to begin with
-  void InitializeResult() {
-    assert(result_.empty());            // Should always be true.
-    result_.resize(sizeof(uint32_t),0); // Put a block of 0's as the header
-  }
-
-  /// Go to the next element (used in Push() and Skip())
-  void MoveNext() {
-    CheckErrors();
-
-    // Check to make sure we are not already in a finished state
-    if (Done()) {
-      ThrowError("Attempting to iterate past end of list.");
-    }
-
-    // Move forward one element.
-    cur_byte_ += sizeof(cur_elem_length_) + cur_elem_length_;
-    ++cur_elem_;
-
-    // If we are at the end, finish
-    if (Done()) {
-      cur_elem_length_ = 0;
-      return;
-    }
-
-    // Otherwise, we should be able to read the new element's length
-    if (cur_byte_ + sizeof(cur_elem_length_) > num_bytes_) {
-      ThrowError("Corrupt element data.");
-    }
-
-    // Set the new element's length
-    cur_elem_length_ = DecodeFixed32(data_+cur_byte_);
-
-    return;
-  }
-
-  /// Append the current element (pointed to by cur_byte_) to result_
-  /// Assumes result_ has already been reserved appropriately.
-  void WriteCurrentElement() {
-    // First verify that the iterator is still valid.
-    CheckErrors();
-    if (Done()) {
-      ThrowError("Attempting to write invalid element.");
-    }
-
-    // Append the cur element.
-    result_.insert(result_.end(),
-                   data_+cur_byte_,
-                   data_+cur_byte_+ sizeof(uint32_t) + cur_elem_length_);
-  }
-
-  /// Will ThrowError() if necessary.
-  /// Checks for common/ubiquitous errors that can arise after most operations.
-  /// This method should be called before any reading operation.
-  /// If this function succeeds, then we are guaranteed to be in a valid state.
-  /// Other member functions should check for errors and ThrowError() also
-  ///  if an error occurs that is specific to it even while in a valid state.
-  void CheckErrors() {
-    // Check if any crazy thing has happened recently
-    if ((cur_elem_ > length_) ||                              // Bad index
-        (cur_byte_ > num_bytes_) ||                           // No more bytes
-        (cur_byte_ + cur_elem_length_ > num_bytes_) ||        // Item too large
-        (cur_byte_ == num_bytes_ && cur_elem_ != length_) ||  // Too many items
-        (cur_elem_ == length_ && cur_byte_ != num_bytes_)) {  // Too many bytes
-      ThrowError("Corrupt data.");
-    }
-  }
-
-  /// Will throw an exception based on the passed-in message.
-  /// This function is guaranteed to STOP THE CONTROL-FLOW.
-  /// (i.e.: you do not have to call "return" after calling ThrowError)
-  void ThrowError(const char* const /*msg*/ = nullptr) {
-    // TODO: For now we ignore the msg parameter. This can be expanded later.
-    throw RedisListException();
-  }
-
- private:
-  const char* const data_;      // A pointer to the data (the first byte)
-  const uint32_t num_bytes_;    // The number of bytes in this list
-
-  uint32_t cur_byte_;           // The current byte being read
-  uint32_t cur_elem_;           // The current element being read
-  uint32_t cur_elem_length_;    // The number of bytes in current element
-
-  uint32_t length_;             // The number of elements in this list
-  std::vector<char> result_;    // The output data
-};
-
-} // namespace rocksdb
-#endif  // ROCKSDB_LITE

utilities/redis/redis_lists.cc

@@ -1,552 +0,0 @@
-// Copyright 2013 Facebook
-/**
- * A (persistent) Redis API built using the rocksdb backend.
- * Implements Redis Lists as described on: http://redis.io/commands#list
- *
- * @throws All functions may throw a RedisListException on error/corruption.
- *
- * @notes Internally, the set of lists is stored in a rocksdb database,
- *        mapping keys to values. Each "value" is the list itself, storing
- *        some kind of internal representation of the data. All the
- *        representation details are handled by the RedisListIterator class.
- *        The present file should be oblivious to the representation details,
- *        handling only the client (Redis) API, and the calls to rocksdb.
- *
- * @TODO  Presently, all operations take at least O(NV) time where
- *        N is the number of elements in the list, and V is the average
- *        number of bytes per value in the list. So maybe, with merge operator
- *        we can improve this to an optimal O(V) amortized time, since we
- *        wouldn't have to read and re-write the entire list.
- *
- * @author Deon Nicholas (dnicholas@fb.com)
- */
-
-#ifndef ROCKSDB_LITE
-#include "redis_lists.h"
-
-#include <iostream>
-#include <memory>
-#include <cmath>
-
-#include "rocksdb/slice.h"
-#include "util/coding.h"
-
-namespace rocksdb
-{
-
-/// Constructors
-
-RedisLists::RedisLists(const std::string& db_path,
-                       Options options, bool destructive)
-    : put_option_(),
-      get_option_() {
-
-  // Store the name of the database
-  db_name_ = db_path;
-
-  // If destructive, destroy the DB before re-opening it.
-  if (destructive) {
-    DestroyDB(db_name_, Options());
-  }
-
-  // Now open and deal with the db
-  DB* db;
-  Status s = DB::Open(options, db_name_, &db);
-  if (!s.ok()) {
-    std::cerr << "ERROR " << s.ToString() << std::endl;
-    assert(false);
-  }
-
-  db_ = std::unique_ptr<DB>(db);
-}
-
-
-/// Accessors
-
-// Number of elements in the list associated with key
-//   : throws RedisListException
-int RedisLists::Length(const std::string& key) {
-  // Extract the string data representing the list.
-  std::string data;
-  db_->Get(get_option_, key, &data);
-
-  // Return the length
-  RedisListIterator it(data);
-  return it.Length();
-}
-
-// Get the element at the specified index in the (list: key)
-// Returns <empty> ("") on out-of-bounds
-//   : throws RedisListException
-bool RedisLists::Index(const std::string& key, int32_t index,
-                       std::string* result) {
-  // Extract the string data representing the list.
-  std::string data;
-  db_->Get(get_option_, key, &data);
-
-  // Handle REDIS negative indices (from the end); fast iff Length() takes O(1)
-  if (index < 0) {
-    index = Length(key) - (-index);  //replace (-i) with (N-i).
-  }
-
-  // Iterate through the list until the desired index is found.
-  int curIndex = 0;
-  RedisListIterator it(data);
-  while(curIndex < index && !it.Done()) {
-    ++curIndex;
-    it.Skip();
-  }
-
-  // If we actually found the index
-  if (curIndex == index && !it.Done()) {
-    Slice elem;
-    it.GetCurrent(&elem);
-    if (result != nullptr) {
-      *result = elem.ToString();
-    }
-
-    return true;
-  } else {
-    return false;
-  }
-}
-
-// Return a truncated version of the list.
-// First, negative values for first/last are interpreted as "end of list".
-// So, if first == -1, then it is re-set to index: (Length(key) - 1)
-// Then, return exactly those indices i such that first <= i <= last.
-//   : throws RedisListException
-std::vector<std::string> RedisLists::Range(const std::string& key,
-                                           int32_t first, int32_t last) {
-  // Extract the string data representing the list.
-  std::string data;
-  db_->Get(get_option_, key, &data);
-
-  // Handle negative bounds (-1 means last element, etc.)
-  int listLen = Length(key);
-  if (first < 0) {
-    first = listLen - (-first);           // Replace (-x) with (N-x)
-  }
-  if (last < 0) {
-    last = listLen - (-last);
-  }
-
-  // Verify bounds (and truncate the range so that it is valid)
-  first = std::max(first, 0);
-  last = std::min(last, listLen-1);
-  int len = std::max(last-first+1, 0);
-
-  // Initialize the resulting list
-  std::vector<std::string> result(len);
-
-  // Traverse the list and update the vector
-  int curIdx = 0;
-  Slice elem;
-  for (RedisListIterator it(data); !it.Done() && curIdx<=last; it.Skip()) {
-    if (first <= curIdx && curIdx <= last) {
-      it.GetCurrent(&elem);
-      result[curIdx-first].assign(elem.data(),elem.size());
-    }
-
-    ++curIdx;
-  }
-
-  // Return the result. Might be empty
-  return result;
-}
-
-// Print the (list: key) out to stdout. For debugging mostly. Public for now.
-void RedisLists::Print(const std::string& key) {
-  // Extract the string data representing the list.
-  std::string data;
-  db_->Get(get_option_, key, &data);
-
-  // Iterate through the list and print the items
-  Slice elem;
-  for (RedisListIterator it(data); !it.Done(); it.Skip()) {
-    it.GetCurrent(&elem);
-    std::cout << "ITEM " << elem.ToString() << std::endl;
-  }
-
-  //Now print the byte data
-  RedisListIterator it(data);
-  std::cout << "==Printing data==" << std::endl;
-  std::cout << data.size() << std::endl;
-  std::cout << it.Size() << " " << it.Length() << std::endl;
-  Slice result = it.WriteResult();
-  std::cout << result.data() << std::endl;
-  if (true) {
-    std::cout << "size: " << result.size() << std::endl;
-    const char* val = result.data();
-    for(int i=0; i<(int)result.size(); ++i) {
-      std::cout << (int)val[i] << " " << (val[i]>=32?val[i]:' ') << std::endl;
-    }
-    std::cout << std::endl;
-  }
-}
-
-/// Insert/Update Functions
-/// Note: The "real" insert function is private. See below.
-
-// InsertBefore and InsertAfter are simply wrappers around the Insert function.
-int RedisLists::InsertBefore(const std::string& key, const std::string& pivot,
-                             const std::string& value) {
-  return Insert(key, pivot, value, false);
-}
-
-int RedisLists::InsertAfter(const std::string& key, const std::string& pivot,
-                            const std::string& value) {
-  return Insert(key, pivot, value, true);
-}
-
-// Prepend value onto beginning of (list: key)
-//   : throws RedisListException
-int RedisLists::PushLeft(const std::string& key, const std::string& value) {
-  // Get the original list data
-  std::string data;
-  db_->Get(get_option_, key, &data);
-
-  // Construct the result
-  RedisListIterator it(data);
-  it.Reserve(it.Size() + it.SizeOf(value));
-  it.InsertElement(value);
-
-  // Push the data back to the db and return the length
-  db_->Put(put_option_, key, it.WriteResult());
-  return it.Length();
-}
-
-// Append value onto end of (list: key)
-// TODO: Make this O(1) time. Might require MergeOperator.
-//   : throws RedisListException
-int RedisLists::PushRight(const std::string& key, const std::string& value) {
-  // Get the original list data
-  std::string data;
-  db_->Get(get_option_, key, &data);
-
-  // Create an iterator to the data and seek to the end.
-  RedisListIterator it(data);
-  it.Reserve(it.Size() + it.SizeOf(value));
-  while (!it.Done()) {
-    it.Push();    // Write each element as we go
-  }
-
-  // Insert the new element at the current position (the end)
-  it.InsertElement(value);
-
-  // Push it back to the db, and return length
-  db_->Put(put_option_, key, it.WriteResult());
-  return it.Length();
-}
-
-// Set (list: key)[idx] = val. Return true on success, false on fail.
-//   : throws RedisListException
-bool RedisLists::Set(const std::string& key, int32_t index,
-                     const std::string& value) {
-  // Get the original list data
-  std::string data;
-  db_->Get(get_option_, key, &data);
-
-  // Handle negative index for REDIS (meaning -index from end of list)
-  if (index < 0) {
-    index = Length(key) - (-index);
-  }
-
-  // Iterate through the list until we find the element we want
-  int curIndex = 0;
-  RedisListIterator it(data);
-  it.Reserve(it.Size() + it.SizeOf(value));  // Over-estimate is fine
-  while(curIndex < index && !it.Done()) {
-    it.Push();
-    ++curIndex;
-  }
-
-  // If not found, return false (this occurs when index was invalid)
-  if (it.Done() || curIndex != index) {
-    return false;
-  }
-
-  // Write the new element value, and drop the previous element value
-  it.InsertElement(value);
-  it.Skip();
-
-  // Write the data to the database
-  // Check status, since it needs to return true/false guarantee
-  Status s = db_->Put(put_option_, key, it.WriteResult());
-
-  // Success
-  return s.ok();
-}
-
-/// Delete / Remove / Pop functions
-
-// Trim (list: key) so that it will only contain the indices from start..stop
-//  Invalid indices will not generate an error, just empty,
-//  or the portion of the list that fits in this interval
-//   : throws RedisListException
-bool RedisLists::Trim(const std::string& key, int32_t start, int32_t stop) {
-  // Get the original list data
-  std::string data;
-  db_->Get(get_option_, key, &data);
-
-  // Handle negative indices in REDIS
-  int listLen = Length(key);
-  if (start < 0) {
-    start = listLen - (-start);
-  }
-  if (stop < 0) {
-    stop = listLen - (-stop);
-  }
-
-  // Truncate bounds to only fit in the list
-  start = std::max(start, 0);
-  stop = std::min(stop, listLen-1);
-
-  // Construct an iterator for the list. Drop all undesired elements.
-  int curIndex = 0;
-  RedisListIterator it(data);
-  it.Reserve(it.Size());          // Over-estimate
-  while(!it.Done()) {
-    // If not within the range, just skip the item (drop it).
-    // Otherwise, continue as usual.
-    if (start <= curIndex && curIndex <= stop) {
-      it.Push();
-    } else {
-      it.Skip();
-    }
-
-    // Increment the current index
-    ++curIndex;
-  }
-
-  // Write the (possibly empty) result to the database
-  Status s = db_->Put(put_option_, key, it.WriteResult());
-
-  // Return true as long as the write succeeded
-  return s.ok();
-}
-
-// Return and remove the first element in the list (or "" if empty)
-//   : throws RedisListException
-bool RedisLists::PopLeft(const std::string& key, std::string* result) {
-  // Get the original list data
-  std::string data;
-  db_->Get(get_option_, key, &data);
-
-  // Point to first element in the list (if it exists), and get its value/size
-  RedisListIterator it(data);
-  if (it.Length() > 0) {            // Proceed only if list is non-empty
-    Slice elem;
-    it.GetCurrent(&elem);           // Store the value of the first element
-    it.Reserve(it.Size() - it.SizeOf(elem));
-    it.Skip();                      // DROP the first item and move to next
-
-    // Update the db
-    db_->Put(put_option_, key, it.WriteResult());
-
-    // Return the value
-    if (result != nullptr) {
-      *result = elem.ToString();
-    }
-    return true;
-  } else {
-    return false;
-  }
-}
-
-// Remove and return the last element in the list (or "" if empty)
-// TODO: Make this O(1). Might require MergeOperator.
-//   : throws RedisListException
-bool RedisLists::PopRight(const std::string& key, std::string* result) {
-  // Extract the original list data
-  std::string data;
-  db_->Get(get_option_, key, &data);
-
-  // Construct an iterator to the data and move to last element
-  RedisListIterator it(data);
-  it.Reserve(it.Size());
-  int len = it.Length();
-  int curIndex = 0;
-  while(curIndex < (len-1) && !it.Done()) {
-    it.Push();
-    ++curIndex;
-  }
-
-  // Extract and drop/skip the last element
-  if (curIndex == len-1) {
-    assert(!it.Done());         // Sanity check. Should not have ended here.
-
-    // Extract and pop the element
-    Slice elem;
-    it.GetCurrent(&elem);       // Save value of element.
-    it.Skip();                  // Skip the element
-
-    // Write the result to the database
-    db_->Put(put_option_, key, it.WriteResult());
-
-    // Return the value
-    if (result != nullptr) {
-      *result = elem.ToString();
-    }
-    return true;
-  } else {
-    // Must have been an empty list
-    assert(it.Done() && len==0 && curIndex == 0);
-    return false;
-  }
-}
-
-// Remove the (first or last) "num" occurrences of value in (list: key)
-//   : throws RedisListException
-int RedisLists::Remove(const std::string& key, int32_t num,
-                       const std::string& value) {
-  // Negative num ==> RemoveLast; Positive num ==> Remove First
-  if (num < 0) {
-    return RemoveLast(key, -num, value);
-  } else if (num > 0) {
-    return RemoveFirst(key, num, value);
-  } else {
-    return RemoveFirst(key, Length(key), value);
-  }
-}
-
-// Remove the first "num" occurrences of value in (list: key).
-//   : throws RedisListException
-int RedisLists::RemoveFirst(const std::string& key, int32_t num,
-                            const std::string& value) {
-  // Ensure that the number is positive
-  assert(num >= 0);
-
-  // Extract the original list data
-  std::string data;
-  db_->Get(get_option_, key, &data);
-
-  // Traverse the list, appending all but the desired occurrences of value
-  int numSkipped = 0;         // Keep track of the number of times value is seen
-  Slice elem;
-  RedisListIterator it(data);
-  it.Reserve(it.Size());
-  while (!it.Done()) {
-    it.GetCurrent(&elem);
-
-    if (elem == value && numSkipped < num) {
-      // Drop this item if desired
-      it.Skip();
-      ++numSkipped;
-    } else {
-      // Otherwise keep the item and proceed as normal
-      it.Push();
-    }
-  }
-
-  // Put the result back to the database
-  db_->Put(put_option_, key, it.WriteResult());
-
-  // Return the number of elements removed
-  return numSkipped;
-}
-
-
-// Remove the last "num" occurrences of value in (list: key).
-// TODO: I traverse the list 2x. Make faster. Might require MergeOperator.
-//   : throws RedisListException
-int RedisLists::RemoveLast(const std::string& key, int32_t num,
-                           const std::string& value) {
-  // Ensure that the number is positive
-  assert(num >= 0);
-
-  // Extract the original list data
-  std::string data;
-  db_->Get(get_option_, key, &data);
-
-  // Temporary variable to hold the "current element" in the blocks below
-  Slice elem;
-
-  // Count the total number of occurrences of value
-  int totalOccs = 0;
-  for (RedisListIterator it(data); !it.Done(); it.Skip()) {
-    it.GetCurrent(&elem);
-    if (elem == value) {
-      ++totalOccs;
-    }
-  }
-
-  // Construct an iterator to the data. Reserve enough space for the result.
-  RedisListIterator it(data);
-  int bytesRemoved = std::min(num,totalOccs)*it.SizeOf(value);
-  it.Reserve(it.Size() - bytesRemoved);
-
-  // Traverse the list, appending all but the desired occurrences of value.
-  // Note: "Drop the last k occurrences" is equivalent to
-  //  "keep only the first n-k occurrences", where n is total occurrences.
-  int numKept = 0;          // Keep track of the number of times value is kept
-  while(!it.Done()) {
-    it.GetCurrent(&elem);
-
-    // If we are within the deletion range and equal to value, drop it.
-    // Otherwise, append/keep/push it.
-    if (elem == value) {
-      if (numKept < totalOccs - num) {
-        it.Push();
-        ++numKept;
-      } else {
-        it.Skip();
-      }
-    } else {
-      // Always append the others
-      it.Push();
-    }
-  }
-
-  // Put the result back to the database
-  db_->Put(put_option_, key, it.WriteResult());
-
-  // Return the number of elements removed
-  return totalOccs - numKept;
-}
-
-/// Private functions
-
-// Insert element value into (list: key), right before/after
-//  the first occurrence of pivot
-//   : throws RedisListException
-int RedisLists::Insert(const std::string& key, const std::string& pivot,
-                       const std::string& value, bool insert_after) {
-  // Get the original list data
-  std::string data;
-  db_->Get(get_option_, key, &data);
-
-  // Construct an iterator to the data and reserve enough space for result.
-  RedisListIterator it(data);
-  it.Reserve(it.Size() + it.SizeOf(value));
-
-  // Iterate through the list until we find the element we want
-  Slice elem;
-  bool found = false;
-  while(!it.Done() && !found) {
-    it.GetCurrent(&elem);
-
-    // When we find the element, insert the element and mark found
-    if (elem == pivot) {                // Found it!
-      found = true;
-      if (insert_after == true) {       // Skip one more, if inserting after it
-        it.Push();
-      }
-      it.InsertElement(value);
-    } else {
-      it.Push();
-    }
-
-  }
-
-  // Put the data (string) into the database
-  if (found) {
-    db_->Put(put_option_, key, it.WriteResult());
-  }
-
-  // Returns the new (possibly unchanged) length of the list
-  return it.Length();
-}
-
-}  // namespace rocksdb
-#endif  // ROCKSDB_LITE

utilities/redis/redis_lists.h

@@ -1,108 +0,0 @@
-/**
- * A (persistent) Redis API built using the rocksdb backend.
- * Implements Redis Lists as described on: http://redis.io/commands#list
- *
- * @throws All functions may throw a RedisListException
- *
- * @author Deon Nicholas (dnicholas@fb.com)
- * Copyright 2013 Facebook
- */
-
-#ifndef ROCKSDB_LITE
-#pragma once
-
-#include <string>
-#include "rocksdb/db.h"
-#include "redis_list_iterator.h"
-#include "redis_list_exception.h"
-
-namespace rocksdb {
-
-/// The Redis functionality (see http://redis.io/commands#list)
-/// All functions may THROW a RedisListException
-class RedisLists {
- public: // Constructors / Destructors
-  /// Construct a new RedisLists database, with name/path of db.
-  /// Will clear the database on open iff destructive is true (default false).
-  /// Otherwise, it will restore saved changes.
-  /// May throw RedisListException
-  RedisLists(const std::string& db_path,
-             Options options, bool destructive = false);
-
- public:  // Accessors
-  /// The number of items in (list: key)
-  int Length(const std::string& key);
-
-  /// Search the list for the (index)'th item (0-based) in (list:key)
-  /// A negative index indicates: "from end-of-list"
-  /// If index is within range: return true, and return the value in *result.
-  /// If (index < -length OR index>=length), then index is out of range:
-  ///   return false (and *result is left unchanged)
-  /// May throw RedisListException
-  bool Index(const std::string& key, int32_t index,
-             std::string* result);
-
-  /// Return (list: key)[first..last] (inclusive)
-  /// May throw RedisListException
-  std::vector<std::string> Range(const std::string& key,
-                                 int32_t first, int32_t last);
-
-  /// Prints the entire (list: key), for debugging.
-  void Print(const std::string& key);
-
- public: // Insert/Update
-  /// Insert value before/after pivot in (list: key). Return the length.
-  /// May throw RedisListException
-  int InsertBefore(const std::string& key, const std::string& pivot,
-                   const std::string& value);
-  int InsertAfter(const std::string& key, const std::string& pivot,
-                  const std::string& value);
-
-  /// Push / Insert value at beginning/end of the list. Return the length.
-  /// May throw RedisListException
-  int PushLeft(const std::string& key, const std::string& value);
-  int PushRight(const std::string& key, const std::string& value);
-
-  /// Set (list: key)[idx] = val. Return true on success, false on fail
-  /// May throw RedisListException
-  bool Set(const std::string& key, int32_t index, const std::string& value);
-
- public: // Delete / Remove / Pop / Trim
-  /// Trim (list: key) so that it will only contain the indices from start..stop
-  /// Returns true on success
-  /// May throw RedisListException
-  bool Trim(const std::string& key, int32_t start, int32_t stop);
-
-  /// If list is empty, return false and leave *result unchanged.
-  /// Else, remove the first/last elem, store it in *result, and return true
-  bool PopLeft(const std::string& key, std::string* result);  // First
-  bool PopRight(const std::string& key, std::string* result); // Last
-
-  /// Remove the first (or last) num occurrences of value from the list (key)
-  /// Return the number of elements removed.
-  /// May throw RedisListException
-  int Remove(const std::string& key, int32_t num,
-             const std::string& value);
-  int RemoveFirst(const std::string& key, int32_t num,
-                  const std::string& value);
-  int RemoveLast(const std::string& key, int32_t num,
-                 const std::string& value);
-
- private: // Private Functions
-  /// Calls InsertBefore or InsertAfter
-  int Insert(const std::string& key, const std::string& pivot,
-             const std::string& value, bool insert_after);
- private:
-  std::string db_name_;       // The actual database name/path
-  WriteOptions put_option_;
-  ReadOptions get_option_;
-
-  /// The backend rocksdb database.
-  /// Map : key --> list
-  ///       where a list is a sequence of elements
-  ///       and an element is a 4-byte integer (n), followed by n bytes of data
-  std::unique_ptr<DB> db_;
-};
-
-} // namespace rocksdb
-#endif  // ROCKSDB_LITE

utilities/redis/redis_lists_test.cc

@@ -1,894 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-/**
- * A test harness for the Redis API built on rocksdb.
- *
- * USAGE: Build with: "make redis_test" (in rocksdb directory).
- *        Run unit tests with: "./redis_test"
- *        Manual/Interactive user testing: "./redis_test -m"
- *        Manual user testing + restart database: "./redis_test -m -d"
- *
- * TODO:  Add LARGE random test cases to verify efficiency and scalability
- *
- * @author Deon Nicholas (dnicholas@fb.com)
- */
-
-#ifndef ROCKSDB_LITE
-
-#include <iostream>
-#include <cctype>
-
-#include "redis_lists.h"
-#include "util/testharness.h"
-#include "util/random.h"
-
-using namespace rocksdb;
-
-namespace rocksdb {
-
-class RedisListsTest : public testing::Test {
- public:
-  static const std::string kDefaultDbName;
-  static Options options;
-
-  RedisListsTest() {
-    options.create_if_missing = true;
-  }
-};
-
-const std::string RedisListsTest::kDefaultDbName =
-    test::PerThreadDBPath("redis_lists_test");
-Options RedisListsTest::options = Options();
-
-// operator== and operator<< are defined below for vectors (lists)
-// Needed for ASSERT_EQ
-
-namespace {
-void AssertListEq(const std::vector<std::string>& result,
-                  const std::vector<std::string>& expected_result) {
-  ASSERT_EQ(result.size(), expected_result.size());
-  for (size_t i = 0; i < result.size(); ++i) {
-    ASSERT_EQ(result[i], expected_result[i]);
-  }
-}
-}  // namespace
-
-// PushRight, Length, Index, Range
-TEST_F(RedisListsTest, SimpleTest) {
-  RedisLists redis(kDefaultDbName, options, true);   // Destructive
-
-  std::string tempv;  // Used below for all Index(), PopRight(), PopLeft()
-
-  // Simple PushRight (should return the new length each time)
-  ASSERT_EQ(redis.PushRight("k1", "v1"), 1);
-  ASSERT_EQ(redis.PushRight("k1", "v2"), 2);
-  ASSERT_EQ(redis.PushRight("k1", "v3"), 3);
-
-  // Check Length and Index() functions
-  ASSERT_EQ(redis.Length("k1"), 3);        // Check length
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "v1");   // Check valid indices
-  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
-  ASSERT_EQ(tempv, "v2");
-  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
-  ASSERT_EQ(tempv, "v3");
-
-  // Check range function and vectors
-  std::vector<std::string> result = redis.Range("k1", 0, 2);   // Get the list
-  std::vector<std::string> expected_result(3);
-  expected_result[0] = "v1";
-  expected_result[1] = "v2";
-  expected_result[2] = "v3";
-  AssertListEq(result, expected_result);
-}
-
-// PushLeft, Length, Index, Range
-TEST_F(RedisListsTest, SimpleTest2) {
-  RedisLists redis(kDefaultDbName, options, true);   // Destructive
-
-  std::string tempv;  // Used below for all Index(), PopRight(), PopLeft()
-
-  // Simple PushRight
-  ASSERT_EQ(redis.PushLeft("k1", "v3"), 1);
-  ASSERT_EQ(redis.PushLeft("k1", "v2"), 2);
-  ASSERT_EQ(redis.PushLeft("k1", "v1"), 3);
-
-  // Check Length and Index() functions
-  ASSERT_EQ(redis.Length("k1"), 3);        // Check length
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "v1");   // Check valid indices
-  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
-  ASSERT_EQ(tempv, "v2");
-  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
-  ASSERT_EQ(tempv, "v3");
-
-  // Check range function and vectors
-  std::vector<std::string> result = redis.Range("k1", 0, 2);   // Get the list
-  std::vector<std::string> expected_result(3);
-  expected_result[0] = "v1";
-  expected_result[1] = "v2";
-  expected_result[2] = "v3";
-  AssertListEq(result, expected_result);
-}
-
-// Exhaustive test of the Index() function
-TEST_F(RedisListsTest, IndexTest) {
-  RedisLists redis(kDefaultDbName, options, true);   // Destructive
-
-  std::string tempv;  // Used below for all Index(), PopRight(), PopLeft()
-
-  // Empty Index check (return empty and should not crash or edit tempv)
-  tempv = "yo";
-  ASSERT_TRUE(!redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "yo");
-  ASSERT_TRUE(!redis.Index("fda", 3, &tempv));
-  ASSERT_EQ(tempv, "yo");
-  ASSERT_TRUE(!redis.Index("random", -12391, &tempv));
-  ASSERT_EQ(tempv, "yo");
-
-  // Simple Pushes (will yield: [v6, v4, v4, v1, v2, v3]
-  redis.PushRight("k1", "v1");
-  redis.PushRight("k1", "v2");
-  redis.PushRight("k1", "v3");
-  redis.PushLeft("k1", "v4");
-  redis.PushLeft("k1", "v4");
-  redis.PushLeft("k1", "v6");
-
-  // Simple, non-negative indices
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "v6");
-  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
-  ASSERT_EQ(tempv, "v4");
-  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
-  ASSERT_EQ(tempv, "v4");
-  ASSERT_TRUE(redis.Index("k1", 3, &tempv));
-  ASSERT_EQ(tempv, "v1");
-  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
-  ASSERT_EQ(tempv, "v2");
-  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
-  ASSERT_EQ(tempv, "v3");
-
-  // Negative indices
-  ASSERT_TRUE(redis.Index("k1", -6, &tempv));
-  ASSERT_EQ(tempv, "v6");
-  ASSERT_TRUE(redis.Index("k1", -5, &tempv));
-  ASSERT_EQ(tempv, "v4");
-  ASSERT_TRUE(redis.Index("k1", -4, &tempv));
-  ASSERT_EQ(tempv, "v4");
-  ASSERT_TRUE(redis.Index("k1", -3, &tempv));
-  ASSERT_EQ(tempv, "v1");
-  ASSERT_TRUE(redis.Index("k1", -2, &tempv));
-  ASSERT_EQ(tempv, "v2");
-  ASSERT_TRUE(redis.Index("k1", -1, &tempv));
-  ASSERT_EQ(tempv, "v3");
-
-  // Out of bounds (return empty, no crash)
-  ASSERT_TRUE(!redis.Index("k1", 6, &tempv));
-  ASSERT_TRUE(!redis.Index("k1", 123219, &tempv));
-  ASSERT_TRUE(!redis.Index("k1", -7, &tempv));
-  ASSERT_TRUE(!redis.Index("k1", -129, &tempv));
-}
-
-
-// Exhaustive test of the Range() function
-TEST_F(RedisListsTest, RangeTest) {
-  RedisLists redis(kDefaultDbName, options, true);   // Destructive
-
-  std::string tempv;  // Used below for all Index(), PopRight(), PopLeft()
-
-  // Simple Pushes (will yield: [v6, v4, v4, v1, v2, v3])
-  redis.PushRight("k1", "v1");
-  redis.PushRight("k1", "v2");
-  redis.PushRight("k1", "v3");
-  redis.PushLeft("k1", "v4");
-  redis.PushLeft("k1", "v4");
-  redis.PushLeft("k1", "v6");
-
-  // Sanity check (check the length;  make sure it's 6)
-  ASSERT_EQ(redis.Length("k1"), 6);
-
-  // Simple range
-  std::vector<std::string> res = redis.Range("k1", 1, 4);
-  ASSERT_EQ((int)res.size(), 4);
-  ASSERT_EQ(res[0], "v4");
-  ASSERT_EQ(res[1], "v4");
-  ASSERT_EQ(res[2], "v1");
-  ASSERT_EQ(res[3], "v2");
-
-  // Negative indices (i.e.: measured from the end)
-  res = redis.Range("k1", 2, -1);
-  ASSERT_EQ((int)res.size(), 4);
-  ASSERT_EQ(res[0], "v4");
-  ASSERT_EQ(res[1], "v1");
-  ASSERT_EQ(res[2], "v2");
-  ASSERT_EQ(res[3], "v3");
-
-  res = redis.Range("k1", -6, -4);
-  ASSERT_EQ((int)res.size(), 3);
-  ASSERT_EQ(res[0], "v6");
-  ASSERT_EQ(res[1], "v4");
-  ASSERT_EQ(res[2], "v4");
-
-  res = redis.Range("k1", -1, 5);
-  ASSERT_EQ((int)res.size(), 1);
-  ASSERT_EQ(res[0], "v3");
-
-  // Partial / Broken indices
-  res = redis.Range("k1", -3, 1000000);
-  ASSERT_EQ((int)res.size(), 3);
-  ASSERT_EQ(res[0], "v1");
-  ASSERT_EQ(res[1], "v2");
-  ASSERT_EQ(res[2], "v3");
-
-  res = redis.Range("k1", -1000000, 1);
-  ASSERT_EQ((int)res.size(), 2);
-  ASSERT_EQ(res[0], "v6");
-  ASSERT_EQ(res[1], "v4");
-
-  // Invalid indices
-  res = redis.Range("k1", 7, 9);
-  ASSERT_EQ((int)res.size(), 0);
-
-  res = redis.Range("k1", -8, -7);
-  ASSERT_EQ((int)res.size(), 0);
-
-  res = redis.Range("k1", 3, 2);
-  ASSERT_EQ((int)res.size(), 0);
-
-  res = redis.Range("k1", 5, -2);
-  ASSERT_EQ((int)res.size(), 0);
-
-  // Range matches Index
-  res = redis.Range("k1", -6, -4);
-  ASSERT_TRUE(redis.Index("k1", -6, &tempv));
-  ASSERT_EQ(tempv, res[0]);
-  ASSERT_TRUE(redis.Index("k1", -5, &tempv));
-  ASSERT_EQ(tempv, res[1]);
-  ASSERT_TRUE(redis.Index("k1", -4, &tempv));
-  ASSERT_EQ(tempv, res[2]);
-
-  // Last check
-  res = redis.Range("k1", 0, -6);
-  ASSERT_EQ((int)res.size(), 1);
-  ASSERT_EQ(res[0], "v6");
-}
-
-// Exhaustive test for InsertBefore(), and InsertAfter()
-TEST_F(RedisListsTest, InsertTest) {
-  RedisLists redis(kDefaultDbName, options, true);
-
-  std::string tempv;  // Used below for all Index(), PopRight(), PopLeft()
-
-  // Insert on empty list (return 0, and do not crash)
-  ASSERT_EQ(redis.InsertBefore("k1", "non-exist", "a"), 0);
-  ASSERT_EQ(redis.InsertAfter("k1", "other-non-exist", "c"), 0);
-  ASSERT_EQ(redis.Length("k1"), 0);
-
-  // Push some preliminary stuff [g, f, e, d, c, b, a]
-  redis.PushLeft("k1", "a");
-  redis.PushLeft("k1", "b");
-  redis.PushLeft("k1", "c");
-  redis.PushLeft("k1", "d");
-  redis.PushLeft("k1", "e");
-  redis.PushLeft("k1", "f");
-  redis.PushLeft("k1", "g");
-  ASSERT_EQ(redis.Length("k1"), 7);
-
-  // Test InsertBefore
-  int newLength = redis.InsertBefore("k1", "e", "hello");
-  ASSERT_EQ(newLength, 8);
-  ASSERT_EQ(redis.Length("k1"), newLength);
-  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
-  ASSERT_EQ(tempv, "f");
-  ASSERT_TRUE(redis.Index("k1", 3, &tempv));
-  ASSERT_EQ(tempv, "e");
-  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
-  ASSERT_EQ(tempv, "hello");
-
-  // Test InsertAfter
-  newLength =  redis.InsertAfter("k1", "c", "bye");
-  ASSERT_EQ(newLength, 9);
-  ASSERT_EQ(redis.Length("k1"), newLength);
-  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
-  ASSERT_EQ(tempv, "bye");
-
-  // Test bad value on InsertBefore
-  newLength = redis.InsertBefore("k1", "yo", "x");
-  ASSERT_EQ(newLength, 9);
-  ASSERT_EQ(redis.Length("k1"), newLength);
-
-  // Test bad value on InsertAfter
-  newLength = redis.InsertAfter("k1", "xxxx", "y");
-  ASSERT_EQ(newLength, 9);
-  ASSERT_EQ(redis.Length("k1"), newLength);
-
-  // Test InsertBefore beginning
-  newLength = redis.InsertBefore("k1", "g", "begggggggggggggggg");
-  ASSERT_EQ(newLength, 10);
-  ASSERT_EQ(redis.Length("k1"), newLength);
-
-  // Test InsertAfter end
-  newLength = redis.InsertAfter("k1", "a", "enddd");
-  ASSERT_EQ(newLength, 11);
-  ASSERT_EQ(redis.Length("k1"), newLength);
-
-  // Make sure nothing weird happened.
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "begggggggggggggggg");
-  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
-  ASSERT_EQ(tempv, "g");
-  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
-  ASSERT_EQ(tempv, "f");
-  ASSERT_TRUE(redis.Index("k1", 3, &tempv));
-  ASSERT_EQ(tempv, "hello");
-  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
-  ASSERT_EQ(tempv, "e");
-  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
-  ASSERT_EQ(tempv, "d");
-  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
-  ASSERT_EQ(tempv, "c");
-  ASSERT_TRUE(redis.Index("k1", 7, &tempv));
-  ASSERT_EQ(tempv, "bye");
-  ASSERT_TRUE(redis.Index("k1", 8, &tempv));
-  ASSERT_EQ(tempv, "b");
-  ASSERT_TRUE(redis.Index("k1", 9, &tempv));
-  ASSERT_EQ(tempv, "a");
-  ASSERT_TRUE(redis.Index("k1", 10, &tempv));
-  ASSERT_EQ(tempv, "enddd");
-}
-
-// Exhaustive test of Set function
-TEST_F(RedisListsTest, SetTest) {
-  RedisLists redis(kDefaultDbName, options, true);
-
-  std::string tempv;  // Used below for all Index(), PopRight(), PopLeft()
-
-  // Set on empty list (return false, and do not crash)
-  ASSERT_EQ(redis.Set("k1", 7, "a"), false);
-  ASSERT_EQ(redis.Set("k1", 0, "a"), false);
-  ASSERT_EQ(redis.Set("k1", -49, "cx"), false);
-  ASSERT_EQ(redis.Length("k1"), 0);
-
-  // Push some preliminary stuff [g, f, e, d, c, b, a]
-  redis.PushLeft("k1", "a");
-  redis.PushLeft("k1", "b");
-  redis.PushLeft("k1", "c");
-  redis.PushLeft("k1", "d");
-  redis.PushLeft("k1", "e");
-  redis.PushLeft("k1", "f");
-  redis.PushLeft("k1", "g");
-  ASSERT_EQ(redis.Length("k1"), 7);
-
-  // Test Regular Set
-  ASSERT_TRUE(redis.Set("k1", 0, "0"));
-  ASSERT_TRUE(redis.Set("k1", 3, "3"));
-  ASSERT_TRUE(redis.Set("k1", 6, "6"));
-  ASSERT_TRUE(redis.Set("k1", 2, "2"));
-  ASSERT_TRUE(redis.Set("k1", 5, "5"));
-  ASSERT_TRUE(redis.Set("k1", 1, "1"));
-  ASSERT_TRUE(redis.Set("k1", 4, "4"));
-
-  ASSERT_EQ(redis.Length("k1"), 7); // Size should not change
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "0");
-  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
-  ASSERT_EQ(tempv, "1");
-  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
-  ASSERT_EQ(tempv, "2");
-  ASSERT_TRUE(redis.Index("k1", 3, &tempv));
-  ASSERT_EQ(tempv, "3");
-  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
-  ASSERT_EQ(tempv, "4");
-  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
-  ASSERT_EQ(tempv, "5");
-  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
-  ASSERT_EQ(tempv, "6");
-
-  // Set with negative indices
-  ASSERT_TRUE(redis.Set("k1", -7, "a"));
-  ASSERT_TRUE(redis.Set("k1", -4, "d"));
-  ASSERT_TRUE(redis.Set("k1", -1, "g"));
-  ASSERT_TRUE(redis.Set("k1", -5, "c"));
-  ASSERT_TRUE(redis.Set("k1", -2, "f"));
-  ASSERT_TRUE(redis.Set("k1", -6, "b"));
-  ASSERT_TRUE(redis.Set("k1", -3, "e"));
-
-  ASSERT_EQ(redis.Length("k1"), 7); // Size should not change
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "a");
-  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
-  ASSERT_EQ(tempv, "b");
-  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
-  ASSERT_EQ(tempv, "c");
-  ASSERT_TRUE(redis.Index("k1", 3, &tempv));
-  ASSERT_EQ(tempv, "d");
-  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
-  ASSERT_EQ(tempv, "e");
-  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
-  ASSERT_EQ(tempv, "f");
-  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
-  ASSERT_EQ(tempv, "g");
-
-  // Bad indices (just out-of-bounds / off-by-one check)
-  ASSERT_EQ(redis.Set("k1", -8, "off-by-one in negative index"), false);
-  ASSERT_EQ(redis.Set("k1", 7, "off-by-one-error in positive index"), false);
-  ASSERT_EQ(redis.Set("k1", 43892, "big random index should fail"), false);
-  ASSERT_EQ(redis.Set("k1", -21391, "large negative index should fail"), false);
-
-  // One last check (to make sure nothing weird happened)
-  ASSERT_EQ(redis.Length("k1"), 7); // Size should not change
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "a");
-  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
-  ASSERT_EQ(tempv, "b");
-  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
-  ASSERT_EQ(tempv, "c");
-  ASSERT_TRUE(redis.Index("k1", 3, &tempv));
-  ASSERT_EQ(tempv, "d");
-  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
-  ASSERT_EQ(tempv, "e");
-  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
-  ASSERT_EQ(tempv, "f");
-  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
-  ASSERT_EQ(tempv, "g");
-}
-
-// Testing Insert, Push, and Set, in a mixed environment
-TEST_F(RedisListsTest, InsertPushSetTest) {
-  RedisLists redis(kDefaultDbName, options, true);   // Destructive
-
-  std::string tempv;  // Used below for all Index(), PopRight(), PopLeft()
-
-  // A series of pushes and insertions
-  // Will result in [newbegin, z, a, aftera, x, newend]
-  // Also, check the return value sometimes (should return length)
-  int lengthCheck;
-  lengthCheck = redis.PushLeft("k1", "a");
-  ASSERT_EQ(lengthCheck, 1);
-  redis.PushLeft("k1", "z");
-  redis.PushRight("k1", "x");
-  lengthCheck = redis.InsertAfter("k1", "a", "aftera");
-  ASSERT_EQ(lengthCheck , 4);
-  redis.InsertBefore("k1", "z", "newbegin");  // InsertBefore beginning of list
-  redis.InsertAfter("k1", "x", "newend");     // InsertAfter end of list
-
-  // Check
-  std::vector<std::string> res = redis.Range("k1", 0, -1); // Get the list
-  ASSERT_EQ((int)res.size(), 6);
-  ASSERT_EQ(res[0], "newbegin");
-  ASSERT_EQ(res[5], "newend");
-  ASSERT_EQ(res[3], "aftera");
-
-  // Testing duplicate values/pivots (multiple occurrences of 'a')
-  ASSERT_TRUE(redis.Set("k1", 0, "a"));     // [a, z, a, aftera, x, newend]
-  redis.InsertAfter("k1", "a", "happy");    // [a, happy, z, a, aftera, ...]
-  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
-  ASSERT_EQ(tempv, "happy");
-  redis.InsertBefore("k1", "a", "sad");     // [sad, a, happy, z, a, aftera, ...]
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "sad");
-  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
-  ASSERT_EQ(tempv, "happy");
-  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
-  ASSERT_EQ(tempv, "aftera");
-  redis.InsertAfter("k1", "a", "zz");         // [sad, a, zz, happy, z, a, aftera, ...]
-  ASSERT_TRUE(redis.Index("k1", 2, &tempv));
-  ASSERT_EQ(tempv, "zz");
-  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
-  ASSERT_EQ(tempv, "aftera");
-  ASSERT_TRUE(redis.Set("k1", 1, "nota"));    // [sad, nota, zz, happy, z, a, ...]
-  redis.InsertBefore("k1", "a", "ba");        // [sad, nota, zz, happy, z, ba, a, ...]
-  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
-  ASSERT_EQ(tempv, "z");
-  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
-  ASSERT_EQ(tempv, "ba");
-  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
-  ASSERT_EQ(tempv, "a");
-
-  // We currently have: [sad, nota, zz, happy, z, ba, a, aftera, x, newend]
-  // redis.Print("k1");   // manually check
-
-  // Test Inserting before/after non-existent values
-  lengthCheck = redis.Length("k1"); // Ensure that the length doesn't change
-  ASSERT_EQ(lengthCheck, 10);
-  ASSERT_EQ(redis.InsertBefore("k1", "non-exist", "randval"), lengthCheck);
-  ASSERT_EQ(redis.InsertAfter("k1", "nothing", "a"), lengthCheck);
-  ASSERT_EQ(redis.InsertAfter("randKey", "randVal", "ranValue"), 0); // Empty
-  ASSERT_EQ(redis.Length("k1"), lengthCheck); // The length should not change
-
-  // Simply Test the Set() function
-  redis.Set("k1", 5, "ba2");
-  redis.InsertBefore("k1", "ba2", "beforeba2");
-  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
-  ASSERT_EQ(tempv, "z");
-  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
-  ASSERT_EQ(tempv, "beforeba2");
-  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
-  ASSERT_EQ(tempv, "ba2");
-  ASSERT_TRUE(redis.Index("k1", 7, &tempv));
-  ASSERT_EQ(tempv, "a");
-
-  // We have: [sad, nota, zz, happy, z, beforeba2, ba2, a, aftera, x, newend]
-
-  // Set() with negative indices
-  redis.Set("k1", -1, "endprank");
-  ASSERT_TRUE(!redis.Index("k1", 11, &tempv));
-  ASSERT_TRUE(redis.Index("k1", 10, &tempv));
-  ASSERT_EQ(tempv, "endprank"); // Ensure Set worked correctly
-  redis.Set("k1", -11, "t");
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "t");
-
-  // Test out of bounds Set
-  ASSERT_EQ(redis.Set("k1", -12, "ssd"), false);
-  ASSERT_EQ(redis.Set("k1", 11, "sasd"), false);
-  ASSERT_EQ(redis.Set("k1", 1200, "big"), false);
-}
-
-// Testing Trim, Pop
-TEST_F(RedisListsTest, TrimPopTest) {
-  RedisLists redis(kDefaultDbName, options, true);   // Destructive
-
-  std::string tempv;  // Used below for all Index(), PopRight(), PopLeft()
-
-  // A series of pushes and insertions
-  // Will result in [newbegin, z, a, aftera, x, newend]
-  redis.PushLeft("k1", "a");
-  redis.PushLeft("k1", "z");
-  redis.PushRight("k1", "x");
-  redis.InsertBefore("k1", "z", "newbegin");    // InsertBefore start of list
-  redis.InsertAfter("k1", "x", "newend");       // InsertAfter end of list
-  redis.InsertAfter("k1", "a", "aftera");
-
-  // Simple PopLeft/Right test
-  ASSERT_TRUE(redis.PopLeft("k1", &tempv));
-  ASSERT_EQ(tempv, "newbegin");
-  ASSERT_EQ(redis.Length("k1"), 5);
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "z");
-  ASSERT_TRUE(redis.PopRight("k1", &tempv));
-  ASSERT_EQ(tempv, "newend");
-  ASSERT_EQ(redis.Length("k1"), 4);
-  ASSERT_TRUE(redis.Index("k1", -1, &tempv));
-  ASSERT_EQ(tempv, "x");
-
-  // Now have: [z, a, aftera, x]
-
-  // Test Trim
-  ASSERT_TRUE(redis.Trim("k1", 0, -1));       // [z, a, aftera, x] (do nothing)
-  ASSERT_EQ(redis.Length("k1"), 4);
-  ASSERT_TRUE(redis.Trim("k1", 0, 2));                     // [z, a, aftera]
-  ASSERT_EQ(redis.Length("k1"), 3);
-  ASSERT_TRUE(redis.Index("k1", -1, &tempv));
-  ASSERT_EQ(tempv, "aftera");
-  ASSERT_TRUE(redis.Trim("k1", 1, 1));                     // [a]
-  ASSERT_EQ(redis.Length("k1"), 1);
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "a");
-
-  // Test out of bounds (empty) trim
-  ASSERT_TRUE(redis.Trim("k1", 1, 0));
-  ASSERT_EQ(redis.Length("k1"), 0);
-
-  // Popping with empty list (return empty without error)
-  ASSERT_TRUE(!redis.PopLeft("k1", &tempv));
-  ASSERT_TRUE(!redis.PopRight("k1", &tempv));
-  ASSERT_TRUE(redis.Trim("k1", 0, 5));
-
-  // Exhaustive Trim test (negative and invalid indices)
-  // Will start in [newbegin, z, a, aftera, x, newend]
-  redis.PushLeft("k1", "a");
-  redis.PushLeft("k1", "z");
-  redis.PushRight("k1", "x");
-  redis.InsertBefore("k1", "z", "newbegin");    // InsertBefore start of list
-  redis.InsertAfter("k1", "x", "newend");       // InsertAfter end of list
-  redis.InsertAfter("k1", "a", "aftera");
-  ASSERT_TRUE(redis.Trim("k1", -6, -1));                     // Should do nothing
-  ASSERT_EQ(redis.Length("k1"), 6);
-  ASSERT_TRUE(redis.Trim("k1", 1, -2));
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "z");
-  ASSERT_TRUE(redis.Index("k1", 3, &tempv));
-  ASSERT_EQ(tempv, "x");
-  ASSERT_EQ(redis.Length("k1"), 4);
-  ASSERT_TRUE(redis.Trim("k1", -3, -2));
-  ASSERT_EQ(redis.Length("k1"), 2);
-}
-
-// Testing Remove, RemoveFirst, RemoveLast
-TEST_F(RedisListsTest, RemoveTest) {
-  RedisLists redis(kDefaultDbName, options, true);   // Destructive
-
-  std::string tempv;  // Used below for all Index(), PopRight(), PopLeft()
-
-  // A series of pushes and insertions
-  // Will result in [newbegin, z, a, aftera, x, newend, a, a]
-  redis.PushLeft("k1", "a");
-  redis.PushLeft("k1", "z");
-  redis.PushRight("k1", "x");
-  redis.InsertBefore("k1", "z", "newbegin");    // InsertBefore start of list
-  redis.InsertAfter("k1", "x", "newend");       // InsertAfter end of list
-  redis.InsertAfter("k1", "a", "aftera");
-  redis.PushRight("k1", "a");
-  redis.PushRight("k1", "a");
-
-  // Verify
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "newbegin");
-  ASSERT_TRUE(redis.Index("k1", -1, &tempv));
-  ASSERT_EQ(tempv, "a");
-
-  // Check RemoveFirst (Remove the first two 'a')
-  // Results in [newbegin, z, aftera, x, newend, a]
-  int numRemoved = redis.Remove("k1", 2, "a");
-  ASSERT_EQ(numRemoved, 2);
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "newbegin");
-  ASSERT_TRUE(redis.Index("k1", 1, &tempv));
-  ASSERT_EQ(tempv, "z");
-  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
-  ASSERT_EQ(tempv, "newend");
-  ASSERT_TRUE(redis.Index("k1", 5, &tempv));
-  ASSERT_EQ(tempv, "a");
-  ASSERT_EQ(redis.Length("k1"), 6);
-
-  // Repopulate some stuff
-  // Results in: [x, x, x, x, x, newbegin, z, x, aftera, x, newend, a, x]
-  redis.PushLeft("k1", "x");
-  redis.PushLeft("k1", "x");
-  redis.PushLeft("k1", "x");
-  redis.PushLeft("k1", "x");
-  redis.PushLeft("k1", "x");
-  redis.PushRight("k1", "x");
-  redis.InsertAfter("k1", "z", "x");
-
-  // Test removal from end
-  numRemoved = redis.Remove("k1", -2, "x");
-  ASSERT_EQ(numRemoved, 2);
-  ASSERT_TRUE(redis.Index("k1", 8, &tempv));
-  ASSERT_EQ(tempv, "aftera");
-  ASSERT_TRUE(redis.Index("k1", 9, &tempv));
-  ASSERT_EQ(tempv, "newend");
-  ASSERT_TRUE(redis.Index("k1", 10, &tempv));
-  ASSERT_EQ(tempv, "a");
-  ASSERT_TRUE(!redis.Index("k1", 11, &tempv));
-  numRemoved = redis.Remove("k1", -2, "x");
-  ASSERT_EQ(numRemoved, 2);
-  ASSERT_TRUE(redis.Index("k1", 4, &tempv));
-  ASSERT_EQ(tempv, "newbegin");
-  ASSERT_TRUE(redis.Index("k1", 6, &tempv));
-  ASSERT_EQ(tempv, "aftera");
-
-  // We now have: [x, x, x, x, newbegin, z, aftera, newend, a]
-  ASSERT_EQ(redis.Length("k1"), 9);
-  ASSERT_TRUE(redis.Index("k1", -1, &tempv));
-  ASSERT_EQ(tempv, "a");
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "x");
-
-  // Test over-shooting (removing more than there exists)
-  numRemoved = redis.Remove("k1", -9000, "x");
-  ASSERT_EQ(numRemoved , 4);    // Only really removed 4
-  ASSERT_EQ(redis.Length("k1"), 5);
-  ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-  ASSERT_EQ(tempv, "newbegin");
-  numRemoved = redis.Remove("k1", 1, "x");
-  ASSERT_EQ(numRemoved, 0);
-
-  // Try removing ALL!
-  numRemoved = redis.Remove("k1", 0, "newbegin");   // REMOVE 0 will remove all!
-  ASSERT_EQ(numRemoved, 1);
-
-  // Removal from an empty-list
-  ASSERT_TRUE(redis.Trim("k1", 1, 0));
-  numRemoved = redis.Remove("k1", 1, "z");
-  ASSERT_EQ(numRemoved, 0);
-}
-
-
-// Test Multiple keys and Persistence
-TEST_F(RedisListsTest, PersistenceMultiKeyTest) {
-  std::string tempv;  // Used below for all Index(), PopRight(), PopLeft()
-
-  // Block one: populate a single key in the database
-  {
-    RedisLists redis(kDefaultDbName, options, true);   // Destructive
-
-    // A series of pushes and insertions
-    // Will result in [newbegin, z, a, aftera, x, newend, a, a]
-    redis.PushLeft("k1", "a");
-    redis.PushLeft("k1", "z");
-    redis.PushRight("k1", "x");
-    redis.InsertBefore("k1", "z", "newbegin");    // InsertBefore start of list
-    redis.InsertAfter("k1", "x", "newend");       // InsertAfter end of list
-    redis.InsertAfter("k1", "a", "aftera");
-    redis.PushRight("k1", "a");
-    redis.PushRight("k1", "a");
-
-    ASSERT_TRUE(redis.Index("k1", 3, &tempv));
-    ASSERT_EQ(tempv, "aftera");
-  }
-
-  // Block two: make sure changes were saved and add some other key
-  {
-    RedisLists redis(kDefaultDbName, options, false); // Persistent, non-destructive
-
-    // Check
-    ASSERT_EQ(redis.Length("k1"), 8);
-    ASSERT_TRUE(redis.Index("k1", 3, &tempv));
-    ASSERT_EQ(tempv, "aftera");
-
-    redis.PushRight("k2", "randomkey");
-    redis.PushLeft("k2", "sas");
-
-    redis.PopLeft("k1", &tempv);
-  }
-
-  // Block three: Verify the changes from block 2
-  {
-    RedisLists redis(kDefaultDbName, options, false); // Persistent, non-destructive
-
-    // Check
-    ASSERT_EQ(redis.Length("k1"), 7);
-    ASSERT_EQ(redis.Length("k2"), 2);
-    ASSERT_TRUE(redis.Index("k1", 0, &tempv));
-    ASSERT_EQ(tempv, "z");
-    ASSERT_TRUE(redis.Index("k2", -2, &tempv));
-    ASSERT_EQ(tempv, "sas");
-  }
-}
-
-/// THE manual REDIS TEST begins here
-/// THIS WILL ONLY OCCUR IF YOU RUN: ./redis_test -m
-
-namespace {
-void MakeUpper(std::string* const s) {
-  int len = static_cast<int>(s->length());
-  for (int i = 0; i < len; ++i) {
-    (*s)[i] = static_cast<char>(toupper((*s)[i]));  // C-version defined in <ctype.h>
-  }
-}
-
-/// Allows the user to enter in REDIS commands into the command-line.
-/// This is useful for manual / interacticve testing / debugging.
-///  Use destructive=true to clean the database before use.
-///  Use destructive=false to remember the previous state (i.e.: persistent)
-/// Should be called from main function.
-int manual_redis_test(bool destructive){
-  RedisLists redis(RedisListsTest::kDefaultDbName,
-                   RedisListsTest::options,
-                   destructive);
-
-  // TODO: Right now, please use spaces to separate each word.
-  //  In actual redis, you can use quotes to specify compound values
-  //  Example: RPUSH mylist "this is a compound value"
-
-  std::string command;
-  while(true) {
-    std::cin >> command;
-    MakeUpper(&command);
-
-    if (command == "LINSERT") {
-      std::string k, t, p, v;
-      std::cin >> k >> t >> p >> v;
-      MakeUpper(&t);
-      if (t=="BEFORE") {
-        std::cout << redis.InsertBefore(k, p, v) << std::endl;
-      } else if (t=="AFTER") {
-        std::cout << redis.InsertAfter(k, p, v) << std::endl;
-      }
-    } else if (command == "LPUSH") {
-      std::string k, v;
-      std::cin >> k >> v;
-      redis.PushLeft(k, v);
-    } else if (command == "RPUSH") {
-      std::string k, v;
-      std::cin >> k >> v;
-      redis.PushRight(k, v);
-    } else if (command == "LPOP") {
-      std::string k;
-      std::cin >> k;
-      std::string res;
-      redis.PopLeft(k, &res);
-      std::cout << res << std::endl;
-    } else if (command == "RPOP") {
-      std::string k;
-      std::cin >> k;
-      std::string res;
-      redis.PopRight(k, &res);
-      std::cout << res << std::endl;
-    } else if (command == "LREM") {
-      std::string k;
-      int amt;
-      std::string v;
-
-      std::cin >> k >> amt >> v;
-      std::cout << redis.Remove(k, amt, v) << std::endl;
-    } else if (command == "LLEN") {
-      std::string k;
-      std::cin >> k;
-      std::cout << redis.Length(k) << std::endl;
-    } else if (command == "LRANGE") {
-      std::string k;
-      int i, j;
-      std::cin >> k >> i >> j;
-      std::vector<std::string> res = redis.Range(k, i, j);
-      for (auto it = res.begin(); it != res.end(); ++it) {
-        std::cout << " " << (*it);
-      }
-      std::cout << std::endl;
-    } else if (command == "LTRIM") {
-      std::string k;
-      int i, j;
-      std::cin >> k >> i >> j;
-      redis.Trim(k, i, j);
-    } else if (command == "LSET") {
-      std::string k;
-      int idx;
-      std::string v;
-      std::cin >> k >> idx >> v;
-      redis.Set(k, idx, v);
-    } else if (command == "LINDEX") {
-      std::string k;
-      int idx;
-      std::cin >> k >> idx;
-      std::string res;
-      redis.Index(k, idx, &res);
-      std::cout << res << std::endl;
-    } else if (command == "PRINT") {      // Added by Deon
-      std::string k;
-      std::cin >> k;
-      redis.Print(k);
-    } else if (command == "QUIT") {
-      return 0;
-    } else {
-      std::cout << "unknown command: " << command << std::endl;
-    }
-  }
-}
-}  // namespace
-
-} // namespace rocksdb
-
-
-// USAGE: "./redis_test" for default (unit tests)
-//        "./redis_test -m" for manual testing (redis command api)
-//        "./redis_test -m -d" for destructive manual test (erase db before use)
-
-
-namespace {
-// Check for "want" argument in the argument list
-bool found_arg(int argc, char* argv[], const char* want){
-  for(int i=1; i<argc; ++i){
-    if (strcmp(argv[i], want) == 0) {
-      return true;
-    }
-  }
-  return false;
-}
-}  // namespace
-
-// Will run unit tests.
-// However, if -m is specified, it will do user manual/interactive testing
-// -m -d is manual and destructive (will clear the database before use)
-int main(int argc, char* argv[]) {
-  ::testing::InitGoogleTest(&argc, argv);
-  if (found_arg(argc, argv, "-m")) {
-    bool destructive = found_arg(argc, argv, "-d");
-    return rocksdb::manual_redis_test(destructive);
-  } else {
-    return RUN_ALL_TESTS();
-  }
-}
-
-#else
-#include <stdio.h>
-
-int main(int /*argc*/, char** /*argv*/) {
-  fprintf(stderr, "SKIPPED as redis is not supported in ROCKSDB_LITE\n");
-  return 0;
-}
-
-#endif  // !ROCKSDB_LITE

utilities/spatialdb/spatial_db.cc

@@ -1,919 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#ifndef ROCKSDB_LITE
-
-#include "rocksdb/utilities/spatial_db.h"
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <algorithm>
-#include <condition_variable>
-#include <inttypes.h>
-#include <string>
-#include <vector>
-#include <mutex>
-#include <thread>
-#include <set>
-#include <unordered_set>
-
-#include "rocksdb/cache.h"
-#include "rocksdb/options.h"
-#include "rocksdb/memtablerep.h"
-#include "rocksdb/slice_transform.h"
-#include "rocksdb/statistics.h"
-#include "rocksdb/table.h"
-#include "rocksdb/db.h"
-#include "rocksdb/utilities/stackable_db.h"
-#include "util/coding.h"
-#include "utilities/spatialdb/utils.h"
-#include "port/port.h"
-
-namespace rocksdb {
-namespace spatial {
-
-// Column families are used to store element's data and spatial indexes. We use
-// [default] column family to store the element data. This is the format of
-// [default] column family:
-// * id (fixed 64 big endian) -> blob (length prefixed slice) feature_set
-// (serialized)
-// We have one additional column family for each spatial index. The name of the
-// column family is [spatial$<spatial_index_name>]. The format is:
-// * quad_key (fixed 64 bit big endian) id (fixed 64 bit big endian) -> ""
-// We store information about indexes in [metadata] column family. Format is:
-// * spatial$<spatial_index_name> -> bbox (4 double encodings) tile_bits
-// (varint32)
-
-namespace {
-const std::string kMetadataColumnFamilyName("metadata");
-inline std::string GetSpatialIndexColumnFamilyName(
-    const std::string& spatial_index_name) {
-  return "spatial$" + spatial_index_name;
-}
-inline bool GetSpatialIndexName(const std::string& column_family_name,
-                                Slice* dst) {
-  *dst = Slice(column_family_name);
-  if (dst->starts_with("spatial$")) {
-    dst->remove_prefix(8);  // strlen("spatial$")
-    return true;
-  }
-  return false;
-}
-
-}  // namespace
-
-void Variant::Init(const Variant& v, Data& d) {
-  switch (v.type_) {
-    case kNull:
-      break;
-    case kBool:
-      d.b = v.data_.b;
-      break;
-    case kInt:
-      d.i = v.data_.i;
-      break;
-    case kDouble:
-      d.d = v.data_.d;
-      break;
-    case kString:
-      new (d.s) std::string(*GetStringPtr(v.data_));
-      break;
-    default:
-      assert(false);
-  }
-}
-
-Variant& Variant::operator=(const Variant& v) {
-  // Construct first a temp so exception from a string ctor
-  // does not change this object
-  Data tmp;
-  Init(v, tmp);
-
-  Type thisType = type_;
-  // Boils down to copying bits so safe
-  std::swap(tmp, data_);
-  type_ = v.type_;
-
-  Destroy(thisType, tmp);
-
-  return *this;
-}
-
-Variant& Variant::operator=(Variant&& rhs) {
-  Destroy(type_, data_);
-  if (rhs.type_ == kString) {
-    new (data_.s) std::string(std::move(*GetStringPtr(rhs.data_)));
-  } else {
-    data_ = rhs.data_;
-  }
-  type_ = rhs.type_;
-  rhs.type_ = kNull;
-  return *this;
-}
-
-bool Variant::operator==(const Variant& rhs) const {
-  if (type_ != rhs.type_) {
-    return false;
-  }
-
-  switch (type_) {
-    case kNull:
-      return true;
-    case kBool:
-      return data_.b == rhs.data_.b;
-    case kInt:
-      return data_.i == rhs.data_.i;
-    case kDouble:
-      return data_.d == rhs.data_.d;
-    case kString:
-      return *GetStringPtr(data_) == *GetStringPtr(rhs.data_);
-    default:
-      assert(false);
-  }
-  // it will never reach here, but otherwise the compiler complains
-  return false;
-}
-
-FeatureSet* FeatureSet::Set(const std::string& key, const Variant& value) {
-  map_.insert({key, value});
-  return this;
-}
-
-bool FeatureSet::Contains(const std::string& key) const {
-  return map_.find(key) != map_.end();
-}
-
-const Variant& FeatureSet::Get(const std::string& key) const {
-  auto itr = map_.find(key);
-  assert(itr != map_.end());
-  return itr->second;
-}
-
-FeatureSet::iterator FeatureSet::Find(const std::string& key) const {
-  return iterator(map_.find(key));
-}
-
-void FeatureSet::Clear() { map_.clear(); }
-
-void FeatureSet::Serialize(std::string* output) const {
-  for (const auto& iter : map_) {
-    PutLengthPrefixedSlice(output, iter.first);
-    output->push_back(static_cast<char>(iter.second.type()));
-    switch (iter.second.type()) {
-      case Variant::kNull:
-        break;
-      case Variant::kBool:
-        output->push_back(static_cast<char>(iter.second.get_bool()));
-        break;
-      case Variant::kInt:
-        PutVarint64(output, iter.second.get_int());
-        break;
-      case Variant::kDouble: {
-        PutDouble(output, iter.second.get_double());
-        break;
-      }
-      case Variant::kString:
-        PutLengthPrefixedSlice(output, iter.second.get_string());
-        break;
-      default:
-        assert(false);
-    }
-  }
-}
-
-bool FeatureSet::Deserialize(const Slice& input) {
-  assert(map_.empty());
-  Slice s(input);
-  while (s.size()) {
-    Slice key;
-    if (!GetLengthPrefixedSlice(&s, &key) || s.size() == 0) {
-      return false;
-    }
-    char type = s[0];
-    s.remove_prefix(1);
-    switch (type) {
-      case Variant::kNull: {
-        map_.insert({key.ToString(), Variant()});
-        break;
-      }
-      case Variant::kBool: {
-        if (s.size() == 0) {
-          return false;
-        }
-        map_.insert({key.ToString(), Variant(static_cast<bool>(s[0]))});
-        s.remove_prefix(1);
-        break;
-      }
-      case Variant::kInt: {
-        uint64_t v;
-        if (!GetVarint64(&s, &v)) {
-          return false;
-        }
-        map_.insert({key.ToString(), Variant(v)});
-        break;
-      }
-      case Variant::kDouble: {
-        double d;
-        if (!GetDouble(&s, &d)) {
-          return false;
-        }
-        map_.insert({key.ToString(), Variant(d)});
-        break;
-      }
-      case Variant::kString: {
-        Slice str;
-        if (!GetLengthPrefixedSlice(&s, &str)) {
-          return false;
-        }
-        map_.insert({key.ToString(), str.ToString()});
-        break;
-      }
-      default:
-        return false;
-    }
-  }
-  return true;
-}
-
-std::string FeatureSet::DebugString() const {
-  std::string out = "{";
-  bool comma = false;
-  for (const auto& iter : map_) {
-    if (comma) {
-      out.append(", ");
-    } else {
-      comma = true;
-    }
-    out.append("\"" + iter.first + "\": ");
-    switch (iter.second.type()) {
-      case Variant::kNull:
-        out.append("null");
-        break;
-      case Variant::kBool:
-        if (iter.second.get_bool()) {
-          out.append("true");
-        } else {
-          out.append("false");
-        }
-        break;
-      case Variant::kInt: {
-        char buf[32];
-        snprintf(buf, sizeof(buf), "%" PRIu64, iter.second.get_int());
-        out.append(buf);
-        break;
-      }
-      case Variant::kDouble: {
-        char buf[32];
-        snprintf(buf, sizeof(buf), "%lf", iter.second.get_double());
-        out.append(buf);
-        break;
-      }
-      case Variant::kString:
-        out.append("\"" + iter.second.get_string() + "\"");
-        break;
-      default:
-        assert(false);
-    }
-  }
-  return out + "}";
-}
-
-class ValueGetter {
- public:
-  ValueGetter() {}
-  virtual ~ValueGetter() {}
-
-  virtual bool Get(uint64_t id) = 0;
-  virtual const Slice value() const = 0;
-
-  virtual Status status() const = 0;
-};
-
-class ValueGetterFromDB : public ValueGetter {
- public:
-  ValueGetterFromDB(DB* db, ColumnFamilyHandle* cf) : db_(db), cf_(cf) {}
-
-  virtual bool Get(uint64_t id) override {
-    std::string encoded_id;
-    PutFixed64BigEndian(&encoded_id, id);
-    status_ = db_->Get(ReadOptions(), cf_, encoded_id, &value_);
-    if (status_.IsNotFound()) {
-      status_ = Status::Corruption("Index inconsistency");
-      return false;
-    }
-
-    return true;
-  }
-
-  virtual const Slice value() const override { return value_; }
-
-  virtual Status status() const override { return status_; }
-
- private:
-  std::string value_;
-  DB* db_;
-  ColumnFamilyHandle* cf_;
-  Status status_;
-};
-
-class ValueGetterFromIterator : public ValueGetter {
- public:
-  explicit ValueGetterFromIterator(Iterator* iterator) : iterator_(iterator) {}
-
-  virtual bool Get(uint64_t id) override {
-    std::string encoded_id;
-    PutFixed64BigEndian(&encoded_id, id);
-    iterator_->Seek(encoded_id);
-
-    if (!iterator_->Valid() || iterator_->key() != Slice(encoded_id)) {
-      status_ = Status::Corruption("Index inconsistency");
-      return false;
-    }
-
-    return true;
-  }
-
-  virtual const Slice value() const override { return iterator_->value(); }
-
-  virtual Status status() const override { return status_; }
-
- private:
-  std::unique_ptr<Iterator> iterator_;
-  Status status_;
-};
-
-class SpatialIndexCursor : public Cursor {
- public:
-  // tile_box is inclusive
-  SpatialIndexCursor(Iterator* spatial_iterator, ValueGetter* value_getter,
-                     const BoundingBox<uint64_t>& tile_bbox, uint32_t tile_bits)
-      : value_getter_(value_getter), valid_(true) {
-    // calculate quad keys we'll need to query
-    std::vector<uint64_t> quad_keys;
-    quad_keys.reserve(static_cast<size_t>((tile_bbox.max_x - tile_bbox.min_x + 1) *
-                      (tile_bbox.max_y - tile_bbox.min_y + 1)));
-    for (uint64_t x = tile_bbox.min_x; x <= tile_bbox.max_x; ++x) {
-      for (uint64_t y = tile_bbox.min_y; y <= tile_bbox.max_y; ++y) {
-        quad_keys.push_back(GetQuadKeyFromTile(x, y, tile_bits));
-      }
-    }
-    std::sort(quad_keys.begin(), quad_keys.end());
-
-    // load primary key ids for all quad keys
-    for (auto quad_key : quad_keys) {
-      std::string encoded_quad_key;
-      PutFixed64BigEndian(&encoded_quad_key, quad_key);
-      Slice slice_quad_key(encoded_quad_key);
-
-      // If CheckQuadKey is true, there is no need to reseek, since
-      // spatial_iterator is already pointing at the correct quad key. This is
-      // an optimization.
-      if (!CheckQuadKey(spatial_iterator, slice_quad_key)) {
-        spatial_iterator->Seek(slice_quad_key);
-      }
-
-      while (CheckQuadKey(spatial_iterator, slice_quad_key)) {
-        // extract ID from spatial_iterator
-        uint64_t id;
-        bool ok = GetFixed64BigEndian(
-            Slice(spatial_iterator->key().data() + sizeof(uint64_t),
-                  sizeof(uint64_t)),
-            &id);
-        if (!ok) {
-          valid_ = false;
-          status_ = Status::Corruption("Spatial index corruption");
-          break;
-        }
-        primary_key_ids_.insert(id);
-        spatial_iterator->Next();
-      }
-    }
-
-    if (!spatial_iterator->status().ok()) {
-      status_ = spatial_iterator->status();
-      valid_ = false;
-    }
-    delete spatial_iterator;
-
-    valid_ = valid_ && !primary_key_ids_.empty();
-
-    if (valid_) {
-      primary_keys_iterator_ = primary_key_ids_.begin();
-      ExtractData();
-    }
-  }
-
-  virtual bool Valid() const override { return valid_; }
-
-  virtual void Next() override {
-    assert(valid_);
-
-    ++primary_keys_iterator_;
-    if (primary_keys_iterator_ == primary_key_ids_.end()) {
-      valid_ = false;
-      return;
-    }
-
-    ExtractData();
-  }
-
-  virtual const Slice blob() override { return current_blob_; }
-  virtual const FeatureSet& feature_set() override {
-    return current_feature_set_;
-  }
-
-  virtual Status status() const override {
-    if (!status_.ok()) {
-      return status_;
-    }
-    return value_getter_->status();
-  }
-
- private:
-  // * returns true if spatial iterator is on the current quad key and all is
-  // well
-  // * returns false if spatial iterator is not on current, or iterator is
-  // invalid or corruption
-  bool CheckQuadKey(Iterator* spatial_iterator, const Slice& quad_key) {
-    if (!spatial_iterator->Valid()) {
-      return false;
-    }
-    if (spatial_iterator->key().size() != 2 * sizeof(uint64_t)) {
-      status_ = Status::Corruption("Invalid spatial index key");
-      valid_ = false;
-      return false;
-    }
-    Slice spatial_iterator_quad_key(spatial_iterator->key().data(),
-                                    sizeof(uint64_t));
-    if (spatial_iterator_quad_key != quad_key) {
-      // caller needs to reseek
-      return false;
-    }
-    // if we come to here, we have found the quad key
-    return true;
-  }
-
-  void ExtractData() {
-    assert(valid_);
-    valid_ = value_getter_->Get(*primary_keys_iterator_);
-
-    if (valid_) {
-      Slice data = value_getter_->value();
-      current_feature_set_.Clear();
-      if (!GetLengthPrefixedSlice(&data, &current_blob_) ||
-          !current_feature_set_.Deserialize(data)) {
-        status_ = Status::Corruption("Primary key column family corruption");
-        valid_ = false;
-      }
-    }
-
-  }
-
-  std::unique_ptr<ValueGetter> value_getter_;
-  bool valid_;
-  Status status_;
-
-  FeatureSet current_feature_set_;
-  Slice current_blob_;
-
-  // This is loaded from spatial iterator.
-  std::unordered_set<uint64_t> primary_key_ids_;
-  std::unordered_set<uint64_t>::iterator primary_keys_iterator_;
-};
-
-class ErrorCursor : public Cursor {
- public:
-  explicit ErrorCursor(Status s) : s_(s) { assert(!s.ok()); }
-  virtual Status status() const override { return s_; }
-  virtual bool Valid() const override { return false; }
-  virtual void Next() override { assert(false); }
-
-  virtual const Slice blob() override {
-    assert(false);
-    return Slice();
-  }
-  virtual const FeatureSet& feature_set() override {
-    assert(false);
-    // compiler complains otherwise
-    return trash_;
-  }
-
- private:
-  Status s_;
-  FeatureSet trash_;
-};
-
-class SpatialDBImpl : public SpatialDB {
- public:
-  // * db -- base DB that needs to be forwarded to StackableDB
-  // * data_column_family -- column family used to store the data
-  // * spatial_indexes -- a list of spatial indexes together with column
-  // families that correspond to those spatial indexes
-  // * next_id -- next ID in auto-incrementing ID. This is usually
-  // `max_id_currenty_in_db + 1`
-  SpatialDBImpl(
-      DB* db, ColumnFamilyHandle* data_column_family,
-      const std::vector<std::pair<SpatialIndexOptions, ColumnFamilyHandle*>>&
-          spatial_indexes,
-      uint64_t next_id, bool read_only)
-      : SpatialDB(db),
-        data_column_family_(data_column_family),
-        next_id_(next_id),
-        read_only_(read_only) {
-    for (const auto& index : spatial_indexes) {
-      name_to_index_.insert(
-          {index.first.name, IndexColumnFamily(index.first, index.second)});
-    }
-  }
-
-  ~SpatialDBImpl() {
-    for (auto& iter : name_to_index_) {
-      delete iter.second.column_family;
-    }
-    delete data_column_family_;
-  }
-
-  virtual Status Insert(
-      const WriteOptions& write_options, const BoundingBox<double>& bbox,
-      const Slice& blob, const FeatureSet& feature_set,
-      const std::vector<std::string>& spatial_indexes) override {
-    WriteBatch batch;
-
-    if (spatial_indexes.size() == 0) {
-      return Status::InvalidArgument("Spatial indexes can't be empty");
-    }
-
-    const size_t kWriteOutEveryBytes = 1024 * 1024;  // 1MB
-    uint64_t id = next_id_.fetch_add(1);
-
-    for (const auto& si : spatial_indexes) {
-      auto itr = name_to_index_.find(si);
-      if (itr == name_to_index_.end()) {
-        return Status::InvalidArgument("Can't find index " + si);
-      }
-      const auto& spatial_index = itr->second.index;
-      if (!spatial_index.bbox.Intersects(bbox)) {
-        continue;
-      }
-      BoundingBox<uint64_t> tile_bbox = GetTileBoundingBox(spatial_index, bbox);
-
-      for (uint64_t x = tile_bbox.min_x; x <= tile_bbox.max_x; ++x) {
-        for (uint64_t y = tile_bbox.min_y; y <= tile_bbox.max_y; ++y) {
-          // see above for format
-          std::string key;
-          PutFixed64BigEndian(
-              &key, GetQuadKeyFromTile(x, y, spatial_index.tile_bits));
-          PutFixed64BigEndian(&key, id);
-          batch.Put(itr->second.column_family, key, Slice());
-          if (batch.GetDataSize() >= kWriteOutEveryBytes) {
-            Status s = Write(write_options, &batch);
-            batch.Clear();
-            if (!s.ok()) {
-              return s;
-            }
-          }
-        }
-      }
-    }
-
-    // see above for format
-    std::string data_key;
-    PutFixed64BigEndian(&data_key, id);
-    std::string data_value;
-    PutLengthPrefixedSlice(&data_value, blob);
-    feature_set.Serialize(&data_value);
-    batch.Put(data_column_family_, data_key, data_value);
-
-    return Write(write_options, &batch);
-  }
-
-  virtual Status Compact(int num_threads) override {
-    std::vector<ColumnFamilyHandle*> column_families;
-    column_families.push_back(data_column_family_);
-
-    for (auto& iter : name_to_index_) {
-      column_families.push_back(iter.second.column_family);
-    }
-
-    std::mutex state_mutex;
-    std::condition_variable cv;
-    Status s;
-    int threads_running = 0;
-
-    std::vector<port::Thread> threads;
-
-    for (auto cfh : column_families) {
-      threads.emplace_back([&, cfh] {
-          {
-            std::unique_lock<std::mutex> lk(state_mutex);
-            cv.wait(lk, [&] { return threads_running < num_threads; });
-            threads_running++;
-          }
-
-          Status t = Flush(FlushOptions(), cfh);
-          if (t.ok()) {
-            t = CompactRange(CompactRangeOptions(), cfh, nullptr, nullptr);
-          }
-
-          {
-            std::unique_lock<std::mutex> lk(state_mutex);
-            threads_running--;
-            if (s.ok() && !t.ok()) {
-              s = t;
-            }
-            cv.notify_one();
-          }
-      });
-    }
-
-    for (auto& t : threads) {
-      t.join();
-    }
-
-    return s;
-  }
-
-  virtual Cursor* Query(const ReadOptions& read_options,
-                        const BoundingBox<double>& bbox,
-                        const std::string& spatial_index) override {
-    auto itr = name_to_index_.find(spatial_index);
-    if (itr == name_to_index_.end()) {
-      return new ErrorCursor(Status::InvalidArgument(
-          "Spatial index " + spatial_index + " not found"));
-    }
-    const auto& si = itr->second.index;
-    Iterator* spatial_iterator;
-    ValueGetter* value_getter;
-
-    if (read_only_) {
-      spatial_iterator = NewIterator(read_options, itr->second.column_family);
-      value_getter = new ValueGetterFromDB(this, data_column_family_);
-    } else {
-      std::vector<Iterator*> iterators;
-      Status s = NewIterators(read_options,
-                              {data_column_family_, itr->second.column_family},
-                              &iterators);
-      if (!s.ok()) {
-        return new ErrorCursor(s);
-      }
-
-      spatial_iterator = iterators[1];
-      value_getter = new ValueGetterFromIterator(iterators[0]);
-    }
-    return new SpatialIndexCursor(spatial_iterator, value_getter,
-                                  GetTileBoundingBox(si, bbox), si.tile_bits);
-  }
-
- private:
-  ColumnFamilyHandle* data_column_family_;
-  struct IndexColumnFamily {
-    SpatialIndexOptions index;
-    ColumnFamilyHandle* column_family;
-    IndexColumnFamily(const SpatialIndexOptions& _index,
-                      ColumnFamilyHandle* _cf)
-        : index(_index), column_family(_cf) {}
-  };
-  // constant after construction!
-  std::unordered_map<std::string, IndexColumnFamily> name_to_index_;
-
-  std::atomic<uint64_t> next_id_;
-  bool read_only_;
-};
-
-namespace {
-DBOptions GetDBOptionsFromSpatialDBOptions(const SpatialDBOptions& options) {
-  DBOptions db_options;
-  db_options.max_open_files = 50000;
-  db_options.max_background_compactions = 3 * options.num_threads / 4;
-  db_options.max_background_flushes =
-      options.num_threads - db_options.max_background_compactions;
-  db_options.env->SetBackgroundThreads(db_options.max_background_compactions,
-                                       Env::LOW);
-  db_options.env->SetBackgroundThreads(db_options.max_background_flushes,
-                                       Env::HIGH);
-  db_options.statistics = CreateDBStatistics();
-  if (options.bulk_load) {
-    db_options.stats_dump_period_sec = 600;
-  } else {
-    db_options.stats_dump_period_sec = 1800;  // 30min
-  }
-  return db_options;
-}
-
-ColumnFamilyOptions GetColumnFamilyOptions(const SpatialDBOptions& /*options*/,
-                                           std::shared_ptr<Cache> block_cache) {
-  ColumnFamilyOptions column_family_options;
-  column_family_options.write_buffer_size = 128 * 1024 * 1024;  // 128MB
-  column_family_options.max_write_buffer_number = 4;
-  column_family_options.max_bytes_for_level_base = 256 * 1024 * 1024;  // 256MB
-  column_family_options.target_file_size_base = 64 * 1024 * 1024;      // 64MB
-  column_family_options.level0_file_num_compaction_trigger = 2;
-  column_family_options.level0_slowdown_writes_trigger = 16;
-  column_family_options.level0_stop_writes_trigger = 32;
-  // only compress levels >= 2
-  column_family_options.compression_per_level.resize(
-      column_family_options.num_levels);
-  for (int i = 0; i < column_family_options.num_levels; ++i) {
-    if (i < 2) {
-      column_family_options.compression_per_level[i] = kNoCompression;
-    } else {
-      column_family_options.compression_per_level[i] = kLZ4Compression;
-    }
-  }
-  BlockBasedTableOptions table_options;
-  table_options.block_cache = block_cache;
-  column_family_options.table_factory.reset(
-      NewBlockBasedTableFactory(table_options));
-  return column_family_options;
-}
-
-ColumnFamilyOptions OptimizeOptionsForDataColumnFamily(
-    ColumnFamilyOptions options, std::shared_ptr<Cache> block_cache) {
-  options.prefix_extractor.reset(NewNoopTransform());
-  BlockBasedTableOptions block_based_options;
-  block_based_options.index_type = BlockBasedTableOptions::kHashSearch;
-  block_based_options.block_cache = block_cache;
-  options.table_factory.reset(NewBlockBasedTableFactory(block_based_options));
-  return options;
-}
-
-}  // namespace
-
-class MetadataStorage {
- public:
-  MetadataStorage(DB* db, ColumnFamilyHandle* cf) : db_(db), cf_(cf) {}
-  ~MetadataStorage() {}
-
-  // format: <min_x double> <min_y double> <max_x double> <max_y double>
-  // <tile_bits varint32>
-  Status AddIndex(const SpatialIndexOptions& index) {
-    std::string encoded_index;
-    PutDouble(&encoded_index, index.bbox.min_x);
-    PutDouble(&encoded_index, index.bbox.min_y);
-    PutDouble(&encoded_index, index.bbox.max_x);
-    PutDouble(&encoded_index, index.bbox.max_y);
-    PutVarint32(&encoded_index, index.tile_bits);
-    return db_->Put(WriteOptions(), cf_,
-                    GetSpatialIndexColumnFamilyName(index.name), encoded_index);
-  }
-
-  Status GetIndex(const std::string& name, SpatialIndexOptions* dst) {
-    std::string value;
-    Status s = db_->Get(ReadOptions(), cf_,
-                        GetSpatialIndexColumnFamilyName(name), &value);
-    if (!s.ok()) {
-      return s;
-    }
-    dst->name = name;
-    Slice encoded_index(value);
-    bool ok = GetDouble(&encoded_index, &(dst->bbox.min_x));
-    ok = ok && GetDouble(&encoded_index, &(dst->bbox.min_y));
-    ok = ok && GetDouble(&encoded_index, &(dst->bbox.max_x));
-    ok = ok && GetDouble(&encoded_index, &(dst->bbox.max_y));
-    ok = ok && GetVarint32(&encoded_index, &(dst->tile_bits));
-    return ok ? Status::OK() : Status::Corruption("Index encoding corrupted");
-  }
-
- private:
-  DB* db_;
-  ColumnFamilyHandle* cf_;
-};
-
-Status SpatialDB::Create(
-    const SpatialDBOptions& options, const std::string& name,
-    const std::vector<SpatialIndexOptions>& spatial_indexes) {
-  DBOptions db_options = GetDBOptionsFromSpatialDBOptions(options);
-  db_options.create_if_missing = true;
-  db_options.create_missing_column_families = true;
-  db_options.error_if_exists = true;
-
-  auto block_cache = NewLRUCache(static_cast<size_t>(options.cache_size));
-  ColumnFamilyOptions column_family_options =
-      GetColumnFamilyOptions(options, block_cache);
-
-  std::vector<ColumnFamilyDescriptor> column_families;
-  column_families.push_back(ColumnFamilyDescriptor(
-      kDefaultColumnFamilyName,
-      OptimizeOptionsForDataColumnFamily(column_family_options, block_cache)));
-  column_families.push_back(
-      ColumnFamilyDescriptor(kMetadataColumnFamilyName, column_family_options));
-
-  for (const auto& index : spatial_indexes) {
-    column_families.emplace_back(GetSpatialIndexColumnFamilyName(index.name),
-                                 column_family_options);
-  }
-
-  std::vector<ColumnFamilyHandle*> handles;
-  DB* base_db;
-  Status s = DB::Open(db_options, name, column_families, &handles, &base_db);
-  if (!s.ok()) {
-    return s;
-  }
-  MetadataStorage metadata(base_db, handles[1]);
-  for (const auto& index : spatial_indexes) {
-    s = metadata.AddIndex(index);
-    if (!s.ok()) {
-      break;
-    }
-  }
-
-  for (auto h : handles) {
-    delete h;
-  }
-  delete base_db;
-
-  return s;
-}
-
-Status SpatialDB::Open(const SpatialDBOptions& options, const std::string& name,
-                       SpatialDB** db, bool read_only) {
-  DBOptions db_options = GetDBOptionsFromSpatialDBOptions(options);
-  auto block_cache = NewLRUCache(static_cast<size_t>(options.cache_size));
-  ColumnFamilyOptions column_family_options =
-      GetColumnFamilyOptions(options, block_cache);
-
-  Status s;
-  std::vector<std::string> existing_column_families;
-  std::vector<std::string> spatial_indexes;
-  s = DB::ListColumnFamilies(db_options, name, &existing_column_families);
-  if (!s.ok()) {
-    return s;
-  }
-  for (const auto& cf_name : existing_column_families) {
-    Slice spatial_index;
-    if (GetSpatialIndexName(cf_name, &spatial_index)) {
-      spatial_indexes.emplace_back(spatial_index.data(), spatial_index.size());
-    }
-  }
-
-  std::vector<ColumnFamilyDescriptor> column_families;
-  column_families.push_back(ColumnFamilyDescriptor(
-      kDefaultColumnFamilyName,
-      OptimizeOptionsForDataColumnFamily(column_family_options, block_cache)));
-  column_families.push_back(
-      ColumnFamilyDescriptor(kMetadataColumnFamilyName, column_family_options));
-
-  for (const auto& index : spatial_indexes) {
-    column_families.emplace_back(GetSpatialIndexColumnFamilyName(index),
-                                 column_family_options);
-  }
-  std::vector<ColumnFamilyHandle*> handles;
-  DB* base_db;
-  if (read_only) {
-    s = DB::OpenForReadOnly(db_options, name, column_families, &handles,
-                            &base_db);
-  } else {
-    s = DB::Open(db_options, name, column_families, &handles, &base_db);
-  }
-  if (!s.ok()) {
-    return s;
-  }
-
-  MetadataStorage metadata(base_db, handles[1]);
-
-  std::vector<std::pair<SpatialIndexOptions, ColumnFamilyHandle*>> index_cf;
-  assert(handles.size() == spatial_indexes.size() + 2);
-  for (size_t i = 0; i < spatial_indexes.size(); ++i) {
-    SpatialIndexOptions index_options;
-    s = metadata.GetIndex(spatial_indexes[i], &index_options);
-    if (!s.ok()) {
-      break;
-    }
-    index_cf.emplace_back(index_options, handles[i + 2]);
-  }
-  uint64_t next_id = 1;
-  if (s.ok()) {
-    // find next_id
-    Iterator* iter = base_db->NewIterator(ReadOptions(), handles[0]);
-    iter->SeekToLast();
-    if (iter->Valid()) {
-      uint64_t last_id = 0;
-      if (!GetFixed64BigEndian(iter->key(), &last_id)) {
-        s = Status::Corruption("Invalid key in data column family");
-      } else {
-        next_id = last_id + 1;
-      }
-    }
-    delete iter;
-  }
-  if (!s.ok()) {
-    for (auto h : handles) {
-      delete h;
-    }
-    delete base_db;
-    return s;
-  }
-
-  // I don't need metadata column family any more, so delete it
-  delete handles[1];
-  *db = new SpatialDBImpl(base_db, handles[0], index_cf, next_id, read_only);
-  return Status::OK();
-}
-
-}  // namespace spatial
-}  // namespace rocksdb
-#endif  // ROCKSDB_LITE

utilities/spatialdb/spatial_db_test.cc

@@ -1,307 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#ifndef ROCKSDB_LITE
-
-#include <vector>
-#include <string>
-#include <set>
-
-#include "rocksdb/utilities/spatial_db.h"
-#include "util/compression.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-#include "util/random.h"
-
-namespace rocksdb {
-namespace spatial {
-
-class SpatialDBTest : public testing::Test {
- public:
-  SpatialDBTest() {
-    dbname_ = test::PerThreadDBPath("spatial_db_test");
-    DestroyDB(dbname_, Options());
-  }
-
-  void AssertCursorResults(BoundingBox<double> bbox, const std::string& index,
-                           const std::vector<std::string>& blobs) {
-    Cursor* c = db_->Query(ReadOptions(), bbox, index);
-    ASSERT_OK(c->status());
-    std::multiset<std::string> b;
-    for (auto x : blobs) {
-      b.insert(x);
-    }
-
-    while (c->Valid()) {
-      auto itr = b.find(c->blob().ToString());
-      ASSERT_TRUE(itr != b.end());
-      b.erase(itr);
-      c->Next();
-    }
-    ASSERT_EQ(b.size(), 0U);
-    ASSERT_OK(c->status());
-    delete c;
-  }
-
-  std::string dbname_;
-  SpatialDB* db_;
-};
-
-TEST_F(SpatialDBTest, FeatureSetSerializeTest) {
-  if (!LZ4_Supported()) {
-    return;
-  }
-  FeatureSet fs;
-
-  fs.Set("a", std::string("b"));
-  fs.Set("x", static_cast<uint64_t>(3));
-  fs.Set("y", false);
-  fs.Set("n", Variant());  // null
-  fs.Set("m", 3.25);
-
-  ASSERT_TRUE(fs.Find("w") == fs.end());
-  ASSERT_TRUE(fs.Find("x") != fs.end());
-  ASSERT_TRUE((*fs.Find("x")).second == Variant(static_cast<uint64_t>(3)));
-  ASSERT_TRUE((*fs.Find("y")).second != Variant(true));
-  std::set<std::string> keys({"a", "x", "y", "n", "m"});
-  for (const auto& x : fs) {
-    ASSERT_TRUE(keys.find(x.first) != keys.end());
-    keys.erase(x.first);
-  }
-  ASSERT_EQ(keys.size(), 0U);
-
-  std::string serialized;
-  fs.Serialize(&serialized);
-
-  FeatureSet deserialized;
-  ASSERT_TRUE(deserialized.Deserialize(serialized));
-
-  ASSERT_TRUE(deserialized.Contains("a"));
-  ASSERT_EQ(deserialized.Get("a").type(), Variant::kString);
-  ASSERT_EQ(deserialized.Get("a").get_string(), "b");
-  ASSERT_TRUE(deserialized.Contains("x"));
-  ASSERT_EQ(deserialized.Get("x").type(), Variant::kInt);
-  ASSERT_EQ(deserialized.Get("x").get_int(), static_cast<uint64_t>(3));
-  ASSERT_TRUE(deserialized.Contains("y"));
-  ASSERT_EQ(deserialized.Get("y").type(), Variant::kBool);
-  ASSERT_EQ(deserialized.Get("y").get_bool(), false);
-  ASSERT_TRUE(deserialized.Contains("n"));
-  ASSERT_EQ(deserialized.Get("n").type(), Variant::kNull);
-  ASSERT_TRUE(deserialized.Contains("m"));
-  ASSERT_EQ(deserialized.Get("m").type(), Variant::kDouble);
-  ASSERT_EQ(deserialized.Get("m").get_double(), 3.25);
-
-  // corrupted serialization
-  serialized = serialized.substr(0, serialized.size() - 1);
-  deserialized.Clear();
-  ASSERT_TRUE(!deserialized.Deserialize(serialized));
-}
-
-TEST_F(SpatialDBTest, TestNextID) {
-  if (!LZ4_Supported()) {
-    return;
-  }
-  ASSERT_OK(SpatialDB::Create(
-      SpatialDBOptions(), dbname_,
-      {SpatialIndexOptions("simple", BoundingBox<double>(0, 0, 100, 100), 2)}));
-
-  ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_));
-  ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(5, 5, 10, 10),
-                        "one", FeatureSet(), {"simple"}));
-  ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(10, 10, 15, 15),
-                        "two", FeatureSet(), {"simple"}));
-  delete db_;
-  db_ = nullptr;
-
-  ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_));
-  assert(db_ != nullptr);
-  ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(55, 55, 65, 65),
-                        "three", FeatureSet(), {"simple"}));
-  delete db_;
-
-  ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_));
-  AssertCursorResults(BoundingBox<double>(0, 0, 100, 100), "simple",
-                      {"one", "two", "three"});
-  delete db_;
-}
-
-TEST_F(SpatialDBTest, FeatureSetTest) {
-  if (!LZ4_Supported()) {
-    return;
-  }
-  ASSERT_OK(SpatialDB::Create(
-      SpatialDBOptions(), dbname_,
-      {SpatialIndexOptions("simple", BoundingBox<double>(0, 0, 100, 100), 2)}));
-  ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_));
-
-  FeatureSet fs;
-  fs.Set("a", std::string("b"));
-  fs.Set("c", std::string("d"));
-
-  ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(5, 5, 10, 10),
-                        "one", fs, {"simple"}));
-
-  Cursor* c =
-      db_->Query(ReadOptions(), BoundingBox<double>(5, 5, 10, 10), "simple");
-
-  ASSERT_TRUE(c->Valid());
-  ASSERT_EQ(c->blob().compare("one"), 0);
-  FeatureSet returned = c->feature_set();
-  ASSERT_TRUE(returned.Contains("a"));
-  ASSERT_TRUE(!returned.Contains("b"));
-  ASSERT_TRUE(returned.Contains("c"));
-  ASSERT_EQ(returned.Get("a").type(), Variant::kString);
-  ASSERT_EQ(returned.Get("a").get_string(), "b");
-  ASSERT_EQ(returned.Get("c").type(), Variant::kString);
-  ASSERT_EQ(returned.Get("c").get_string(), "d");
-
-  c->Next();
-  ASSERT_TRUE(!c->Valid());
-
-  delete c;
-  delete db_;
-}
-
-TEST_F(SpatialDBTest, SimpleTest) {
-  if (!LZ4_Supported()) {
-    return;
-  }
-  // iter 0 -- not read only
-  // iter 1 -- read only
-  for (int iter = 0; iter < 2; ++iter) {
-    DestroyDB(dbname_, Options());
-    ASSERT_OK(SpatialDB::Create(
-        SpatialDBOptions(), dbname_,
-        {SpatialIndexOptions("index", BoundingBox<double>(0, 0, 128, 128),
-                             3)}));
-    ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_));
-    assert(db_ != nullptr);
-
-    ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(33, 17, 63, 79),
-                          "one", FeatureSet(), {"index"}));
-    ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(65, 65, 111, 111),
-                          "two", FeatureSet(), {"index"}));
-    ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(1, 49, 127, 63),
-                          "three", FeatureSet(), {"index"}));
-    ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(20, 100, 21, 101),
-                          "four", FeatureSet(), {"index"}));
-    ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(81, 33, 127, 63),
-                          "five", FeatureSet(), {"index"}));
-    ASSERT_OK(db_->Insert(WriteOptions(), BoundingBox<double>(1, 65, 47, 95),
-                          "six", FeatureSet(), {"index"}));
-
-    if (iter == 1) {
-      delete db_;
-      db_ = nullptr;
-      ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_, true));
-    }
-
-    AssertCursorResults(BoundingBox<double>(33, 17, 47, 31), "index", {"one"});
-    AssertCursorResults(BoundingBox<double>(17, 33, 79, 63), "index",
-                        {"one", "three"});
-    AssertCursorResults(BoundingBox<double>(17, 81, 63, 111), "index",
-                        {"four", "six"});
-    AssertCursorResults(BoundingBox<double>(85, 86, 85, 86), "index", {"two"});
-    AssertCursorResults(BoundingBox<double>(33, 1, 127, 111), "index",
-                        {"one", "two", "three", "five", "six"});
-    // even though the bounding box doesn't intersect, we got "four" back
-    // because
-    // it's in the same tile
-    AssertCursorResults(BoundingBox<double>(18, 98, 19, 99), "index", {"four"});
-    AssertCursorResults(BoundingBox<double>(130, 130, 131, 131), "index", {});
-    AssertCursorResults(BoundingBox<double>(81, 17, 127, 31), "index", {});
-    AssertCursorResults(BoundingBox<double>(90, 50, 91, 51), "index",
-                        {"three", "five"});
-
-    delete db_;
-    db_ = nullptr;
-  }
-}
-
-namespace {
-std::string RandomStr(Random* rnd) {
-  std::string r;
-  for (int k = 0; k < 10; ++k) {
-    r.push_back(static_cast<char>(rnd->Uniform(26)) + 'a');
-  }
-  return r;
-}
-
-BoundingBox<int> RandomBoundingBox(int limit, Random* rnd, int max_size) {
-  BoundingBox<int> r;
-  r.min_x = rnd->Uniform(limit - 1);
-  r.min_y = rnd->Uniform(limit - 1);
-  r.max_x = r.min_x + rnd->Uniform(std::min(limit - 1 - r.min_x, max_size)) + 1;
-  r.max_y = r.min_y + rnd->Uniform(std::min(limit - 1 - r.min_y, max_size)) + 1;
-  return r;
-}
-
-BoundingBox<double> ScaleBB(BoundingBox<int> b, double step) {
-  return BoundingBox<double>(b.min_x * step + 1, b.min_y * step + 1,
-                             (b.max_x + 1) * step - 1,
-                             (b.max_y + 1) * step - 1);
-}
-
-}  // namespace
-
-TEST_F(SpatialDBTest, RandomizedTest) {
-  if (!LZ4_Supported()) {
-    return;
-  }
-  Random rnd(301);
-  std::vector<std::pair<std::string, BoundingBox<int>>> elements;
-
-  BoundingBox<double> spatial_index_bounds(0, 0, (1LL << 32), (1LL << 32));
-  ASSERT_OK(SpatialDB::Create(
-      SpatialDBOptions(), dbname_,
-      {SpatialIndexOptions("index", spatial_index_bounds, 7)}));
-  ASSERT_OK(SpatialDB::Open(SpatialDBOptions(), dbname_, &db_));
-  double step = (1LL << 32) / (1 << 7);
-
-  for (int i = 0; i < 1000; ++i) {
-    std::string blob = RandomStr(&rnd);
-    BoundingBox<int> bbox = RandomBoundingBox(128, &rnd, 10);
-    ASSERT_OK(db_->Insert(WriteOptions(), ScaleBB(bbox, step), blob,
-                          FeatureSet(), {"index"}));
-    elements.push_back(make_pair(blob, bbox));
-  }
-
-  // parallel
-  db_->Compact(2);
-  // serial
-  db_->Compact(1);
-
-  for (int i = 0; i < 1000; ++i) {
-    BoundingBox<int> int_bbox = RandomBoundingBox(128, &rnd, 10);
-    BoundingBox<double> double_bbox = ScaleBB(int_bbox, step);
-    std::vector<std::string> blobs;
-    for (auto e : elements) {
-      if (e.second.Intersects(int_bbox)) {
-        blobs.push_back(e.first);
-      }
-    }
-    AssertCursorResults(double_bbox, "index", blobs);
-  }
-
-  delete db_;
-}
-
-}  // namespace spatial
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
-
-#else
-#include <stdio.h>
-
-int main(int /*argc*/, char** /*argv*/) {
-  fprintf(stderr, "SKIPPED as SpatialDB is not supported in ROCKSDB_LITE\n");
-  return 0;
-}
-
-#endif  // !ROCKSDB_LITE

utilities/spatialdb/utils.h

@@ -1,95 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-#include <string>
-#include <algorithm>
-
-#include "rocksdb/utilities/spatial_db.h"
-
-namespace rocksdb {
-namespace spatial {
-
-// indexing idea from http://msdn.microsoft.com/en-us/library/bb259689.aspx
-inline uint64_t GetTileFromCoord(double x, double start, double end,
-                                 uint32_t tile_bits) {
-  if (x < start) {
-    return 0;
-  }
-  uint64_t tiles = 1ull << tile_bits;
-  uint64_t r = static_cast<uint64_t>(((x - start) / (end - start)) * tiles);
-  return std::min(r, tiles - 1);
-}
-
-inline uint64_t GetQuadKeyFromTile(uint64_t tile_x, uint64_t tile_y,
-                                   uint32_t tile_bits) {
-  uint64_t quad_key = 0;
-  for (uint32_t i = 0; i < tile_bits; ++i) {
-    uint64_t mask = (1ull << i);
-    quad_key |= (tile_x & mask) << i;
-    quad_key |= (tile_y & mask) << (i + 1);
-  }
-  return quad_key;
-}
-
-inline BoundingBox<uint64_t> GetTileBoundingBox(
-    const SpatialIndexOptions& spatial_index, BoundingBox<double> bbox) {
-  return BoundingBox<uint64_t>(
-      GetTileFromCoord(bbox.min_x, spatial_index.bbox.min_x,
-                       spatial_index.bbox.max_x, spatial_index.tile_bits),
-      GetTileFromCoord(bbox.min_y, spatial_index.bbox.min_y,
-                       spatial_index.bbox.max_y, spatial_index.tile_bits),
-      GetTileFromCoord(bbox.max_x, spatial_index.bbox.min_x,
-                       spatial_index.bbox.max_x, spatial_index.tile_bits),
-      GetTileFromCoord(bbox.max_y, spatial_index.bbox.min_y,
-                       spatial_index.bbox.max_y, spatial_index.tile_bits));
-}
-
-// big endian can be compared using memcpy
-inline void PutFixed64BigEndian(std::string* dst, uint64_t value) {
-  char buf[sizeof(value)];
-  buf[0] = (value >> 56) & 0xff;
-  buf[1] = (value >> 48) & 0xff;
-  buf[2] = (value >> 40) & 0xff;
-  buf[3] = (value >> 32) & 0xff;
-  buf[4] = (value >> 24) & 0xff;
-  buf[5] = (value >> 16) & 0xff;
-  buf[6] = (value >> 8) & 0xff;
-  buf[7] = value & 0xff;
-  dst->append(buf, sizeof(buf));
-}
-
-// big endian can be compared using memcpy
-inline bool GetFixed64BigEndian(const Slice& input, uint64_t* value) {
-  if (input.size() < sizeof(uint64_t)) {
-    return false;
-  }
-  auto ptr = input.data();
-  *value = (static_cast<uint64_t>(static_cast<unsigned char>(ptr[0])) << 56) |
-           (static_cast<uint64_t>(static_cast<unsigned char>(ptr[1])) << 48) |
-           (static_cast<uint64_t>(static_cast<unsigned char>(ptr[2])) << 40) |
-           (static_cast<uint64_t>(static_cast<unsigned char>(ptr[3])) << 32) |
-           (static_cast<uint64_t>(static_cast<unsigned char>(ptr[4])) << 24) |
-           (static_cast<uint64_t>(static_cast<unsigned char>(ptr[5])) << 16) |
-           (static_cast<uint64_t>(static_cast<unsigned char>(ptr[6])) << 8) |
-           static_cast<uint64_t>(static_cast<unsigned char>(ptr[7]));
-  return true;
-}
-
-inline void PutDouble(std::string* dst, double d) {
-  dst->append(reinterpret_cast<char*>(&d), sizeof(double));
-}
-
-inline bool GetDouble(Slice* input, double* d) {
-  if (input->size() < sizeof(double)) {
-    return false;
-  }
-  memcpy(d, input->data(), sizeof(double));
-  input->remove_prefix(sizeof(double));
-  return true;
-}
-
-}  // namespace spatial
-}  // namespace rocksdb