2012-07-03 07:45:59 +02:00
|
|
|
/**
|
|
|
|
* Tests for Thrift server for leveldb
|
|
|
|
* @author Dhruba Borthakur (dhruba@gmail.com)
|
|
|
|
* Copyright 2012 Facebook
|
|
|
|
*/
|
|
|
|
#include <protocol/TBinaryProtocol.h>
|
|
|
|
#include <transport/TSocket.h>
|
|
|
|
#include <transport/TBufferTransports.h>
|
2012-08-02 01:17:43 +02:00
|
|
|
#include <util/testharness.h>
|
2012-07-03 07:45:59 +02:00
|
|
|
#include <DB.h>
|
2012-08-02 01:17:43 +02:00
|
|
|
#include <AssocService.h>
|
2012-07-03 07:45:59 +02:00
|
|
|
#include <leveldb_types.h>
|
2012-07-11 23:08:46 +02:00
|
|
|
#include "server_options.h"
|
2012-07-03 07:45:59 +02:00
|
|
|
|
|
|
|
using namespace apache::thrift;
|
|
|
|
using namespace apache::thrift::protocol;
|
|
|
|
using namespace apache::thrift::transport;
|
|
|
|
using boost::shared_ptr;
|
|
|
|
using namespace Tleveldb;
|
|
|
|
|
2012-07-11 23:08:46 +02:00
|
|
|
extern "C" void startServer(int argc, char**argv);
|
2012-07-03 07:45:59 +02:00
|
|
|
extern "C" void stopServer(int port);
|
2012-07-11 23:08:46 +02:00
|
|
|
extern ServerOptions server_options;
|
2012-07-03 07:45:59 +02:00
|
|
|
|
|
|
|
static DBHandle dbhandle;
|
|
|
|
static DBClient* dbclient;
|
2012-08-02 01:17:43 +02:00
|
|
|
static AssocServiceClient* aclient;
|
|
|
|
static const Text dbname = "test-dhruba";
|
2012-07-11 23:08:46 +02:00
|
|
|
static int ARGC;
|
|
|
|
static char** ARGV;
|
|
|
|
|
|
|
|
static void cleanupDir(std::string dir) {
|
|
|
|
// remove old data, if any
|
|
|
|
char* cleanup = new char[100];
|
|
|
|
snprintf(cleanup, 100, "rm -rf %s", dir.c_str());
|
|
|
|
system(cleanup);
|
|
|
|
}
|
2012-07-03 07:45:59 +02:00
|
|
|
|
|
|
|
static void createDatabase() {
|
|
|
|
DBOptions options;
|
|
|
|
options.create_if_missing = true; // create
|
|
|
|
options.error_if_exists = false; // overwrite
|
|
|
|
options.write_buffer_size = (4<<20); // 4 MB
|
|
|
|
options.max_open_files = 1000;
|
|
|
|
options.block_size = 4096;
|
|
|
|
options.block_restart_interval = 16;
|
|
|
|
options.compression = kSnappyCompression;
|
|
|
|
|
2012-07-11 23:08:46 +02:00
|
|
|
cleanupDir(server_options.getDataDirectory(dbname));
|
|
|
|
|
2012-07-03 07:45:59 +02:00
|
|
|
// create the database
|
|
|
|
dbclient->Open(dbhandle, dbname, options);
|
|
|
|
}
|
|
|
|
|
2012-08-02 01:17:43 +02:00
|
|
|
static void initialize(int port) {
|
|
|
|
boost::shared_ptr<TSocket> socket1(new TSocket("localhost", port));
|
|
|
|
boost::shared_ptr<TTransport> transport1(new TBufferedTransport(socket1));
|
|
|
|
boost::shared_ptr<TProtocol> protocol1(new TBinaryProtocol(transport1));
|
2012-07-03 07:45:59 +02:00
|
|
|
|
|
|
|
// open database
|
2012-08-02 01:17:43 +02:00
|
|
|
dbclient = new DBClient(protocol1);
|
|
|
|
transport1->open();
|
|
|
|
|
|
|
|
boost::shared_ptr<TSocket> socket2(new TSocket("localhost", port+1));
|
|
|
|
boost::shared_ptr<TTransport> transport2(new TBufferedTransport(socket2));
|
|
|
|
boost::shared_ptr<TProtocol> protocol2(new TBinaryProtocol(transport2));
|
|
|
|
aclient = new AssocServiceClient(protocol2);
|
|
|
|
transport2->open();
|
|
|
|
|
2012-07-03 07:45:59 +02:00
|
|
|
createDatabase();
|
|
|
|
printf("Database created.\n");
|
2012-08-02 01:17:43 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Run base leveldb thrift server get/put/iter/scan tests
|
|
|
|
//
|
|
|
|
static void testClient() {
|
|
|
|
WriteOptions writeOptions;
|
|
|
|
printf("Running base leveldb operations .................\n");
|
2012-07-03 07:45:59 +02:00
|
|
|
|
|
|
|
// insert record into leveldb
|
|
|
|
Slice key;
|
|
|
|
key.data = "Key1";
|
|
|
|
key.size = 4;
|
|
|
|
Slice value;
|
|
|
|
value.data = "value1";
|
|
|
|
value.size = 6;
|
|
|
|
kv keyvalue;
|
|
|
|
keyvalue.key = key;
|
|
|
|
keyvalue.value = value;
|
|
|
|
int ret = dbclient->Put(dbhandle, keyvalue, writeOptions);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(ret == Code::kOk);
|
2012-07-03 07:45:59 +02:00
|
|
|
printf("Put Key1 suceeded.\n");
|
|
|
|
|
|
|
|
//read it back
|
|
|
|
ReadOptions readOptions;
|
|
|
|
ResultItem rValue;
|
|
|
|
dbclient->Get(rValue, dbhandle, key, readOptions);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(rValue.status == Code::kOk);
|
|
|
|
ASSERT_TRUE(value.data.compare(rValue.value.data) == 0);
|
|
|
|
ASSERT_TRUE(value.size == rValue.value.size);
|
2012-07-03 07:45:59 +02:00
|
|
|
printf("Get suceeded.\n");
|
|
|
|
|
|
|
|
// get a snapshot
|
|
|
|
ResultSnapshot rsnap;
|
|
|
|
dbclient->GetSnapshot(rsnap, dbhandle);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(rsnap.status == Code::kOk);
|
|
|
|
ASSERT_TRUE(rsnap.snapshot.snapshotid > 0);
|
2012-07-03 07:45:59 +02:00
|
|
|
printf("Snapshot suceeded.\n");
|
|
|
|
|
|
|
|
// insert a new record into leveldb
|
|
|
|
Slice key2;
|
|
|
|
key2.data = "Key2";
|
|
|
|
key2.size = 4;
|
|
|
|
Slice value2;
|
|
|
|
value2.data = "value2";
|
|
|
|
value2.size = 6;
|
|
|
|
keyvalue.key = key2;
|
|
|
|
keyvalue.value = value2;
|
|
|
|
ret = dbclient->Put(dbhandle, keyvalue, writeOptions);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(ret == Code::kOk);
|
2012-07-03 07:45:59 +02:00
|
|
|
printf("Put Key2 suceeded.\n");
|
|
|
|
|
|
|
|
// verify that a get done with a previous snapshot does not find Key2
|
|
|
|
readOptions.snapshot = rsnap.snapshot;
|
|
|
|
dbclient->Get(rValue, dbhandle, key2, readOptions);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(rValue.status == Code::kNotFound);
|
2012-07-03 07:45:59 +02:00
|
|
|
printf("Get with snapshot succeeded.\n");
|
|
|
|
|
|
|
|
// release snapshot
|
|
|
|
ret = dbclient->ReleaseSnapshot(dbhandle, rsnap.snapshot);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(ret == Code::kOk);
|
2012-07-03 07:45:59 +02:00
|
|
|
printf("Snapshot released.\n");
|
|
|
|
|
|
|
|
// if we try to re-release the same snapshot, it should fail
|
|
|
|
ret = dbclient->ReleaseSnapshot(dbhandle, rsnap.snapshot);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(ret == Code::kNotFound);
|
2012-07-03 07:45:59 +02:00
|
|
|
|
|
|
|
// compact whole database
|
|
|
|
Slice range;
|
|
|
|
range.size = 0;
|
|
|
|
ret = dbclient->CompactRange(dbhandle, range, range);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(ret == Code::kOk);
|
2012-07-03 07:45:59 +02:00
|
|
|
printf("Compaction trigger suceeded.\n");
|
|
|
|
|
|
|
|
// create a new iterator to scan all keys from the start
|
|
|
|
Slice target;
|
|
|
|
ResultIterator ri;
|
|
|
|
readOptions.snapshot.snapshotid = 0;
|
|
|
|
dbclient->NewIterator(ri, dbhandle, readOptions,
|
|
|
|
IteratorType::seekToFirst, target);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(ri.status == Code::kOk);
|
2012-07-03 07:45:59 +02:00
|
|
|
int foundPairs = 0;
|
|
|
|
while (true) {
|
|
|
|
ResultPair pair;
|
|
|
|
dbclient->GetNext(pair, dbhandle, ri.iterator);
|
|
|
|
if (pair.status == Code::kOk) {
|
|
|
|
foundPairs++;
|
|
|
|
} else {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(foundPairs == 2);
|
2012-07-03 07:45:59 +02:00
|
|
|
ret = dbclient->DeleteIterator(dbhandle, ri.iterator);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(ret == Code::kOk);
|
2012-07-03 07:45:59 +02:00
|
|
|
printf("Iterator scan-all forward passes.\n");
|
|
|
|
|
|
|
|
// create a new iterator, position at end and scan backwards
|
|
|
|
readOptions.snapshot.snapshotid = 0;
|
|
|
|
dbclient->NewIterator(ri, dbhandle, readOptions,
|
|
|
|
IteratorType::seekToLast, target);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(ri.status == Code::kOk);
|
2012-07-03 07:45:59 +02:00
|
|
|
foundPairs = 0;
|
|
|
|
while (true) {
|
|
|
|
ResultPair pair;
|
|
|
|
dbclient->GetPrev(pair, dbhandle, ri.iterator);
|
|
|
|
if (pair.status == Code::kOk) {
|
|
|
|
foundPairs++;
|
|
|
|
} else {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(foundPairs == 2);
|
2012-07-03 07:45:59 +02:00
|
|
|
ret = dbclient->DeleteIterator(dbhandle, ri.iterator);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(ret == Code::kOk);
|
2012-07-03 07:45:59 +02:00
|
|
|
printf("Iterator scan-all backward passes.\n");
|
|
|
|
|
|
|
|
// create a new iterator, position at middle
|
|
|
|
readOptions.snapshot.snapshotid = 0;
|
|
|
|
target = key;
|
|
|
|
dbclient->NewIterator(ri, dbhandle, readOptions,
|
|
|
|
IteratorType::seekToKey, target);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(ri.status == Code::kOk);
|
2012-07-03 07:45:59 +02:00
|
|
|
foundPairs = 0;
|
|
|
|
while (true) {
|
|
|
|
ResultPair pair;
|
|
|
|
dbclient->GetPrev(pair, dbhandle, ri.iterator);
|
|
|
|
if (pair.status == Code::kOk) {
|
|
|
|
foundPairs++;
|
|
|
|
} else {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(foundPairs == 1);
|
2012-07-03 07:45:59 +02:00
|
|
|
ret = dbclient->DeleteIterator(dbhandle, ri.iterator);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_TRUE(ret == Code::kOk);
|
2012-07-03 07:45:59 +02:00
|
|
|
printf("Iterator scan-selective backward passes.\n");
|
|
|
|
}
|
|
|
|
|
2012-08-02 01:17:43 +02:00
|
|
|
//
|
|
|
|
// Run assoc tests
|
|
|
|
//
|
|
|
|
static void testAssocs() {
|
|
|
|
WriteOptions writeOptions;
|
|
|
|
printf("Running assoc leveldb operations ................\n");
|
2012-07-03 07:45:59 +02:00
|
|
|
|
2012-08-02 01:17:43 +02:00
|
|
|
// insert record into leveldb
|
|
|
|
int64_t assocType = 100;
|
|
|
|
int64_t id1 = 1;
|
|
|
|
int64_t id2 = 2;
|
|
|
|
int64_t id1Type = 101;
|
|
|
|
int64_t id2Type = 102;
|
|
|
|
int64_t ts =3333;
|
|
|
|
AssocVisibility vis = AssocVisibility::VISIBLE;
|
|
|
|
bool update_count = true;
|
|
|
|
int64_t dataVersion = 5;
|
|
|
|
const Text data = "data......";
|
|
|
|
const Text wormhole_comments = "wormhole...";
|
|
|
|
int64_t count = aclient->taoAssocPut(dbname, assocType,
|
|
|
|
id1, id2, id1Type, id2Type,
|
|
|
|
ts, vis, update_count,
|
|
|
|
dataVersion, data, wormhole_comments);
|
|
|
|
ASSERT_GE(count, 0);
|
|
|
|
printf("Put AssocPut suceeded.\n");
|
|
|
|
|
|
|
|
// verify assoc counts.
|
|
|
|
int64_t cnt = aclient->taoAssocCount(dbname, assocType, id1);
|
|
|
|
ASSERT_EQ(cnt, 1);
|
|
|
|
printf("AssocCount suceeded.\n");
|
|
|
|
|
|
|
|
// verify that we can read back what we inserted earlier
|
|
|
|
std::vector<int64_t> id2list(1);
|
|
|
|
id2list[0] = id2;
|
|
|
|
std::vector<TaoAssocGetResult> readback(1);
|
|
|
|
aclient->taoAssocGet(readback, dbname,
|
|
|
|
assocType, id1, id2list);
|
|
|
|
printf("AssocGet suceeded.\n");
|
2012-08-02 02:28:35 +02:00
|
|
|
printf("size = %lu\n", readback.size());
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_EQ(1, readback.size());
|
|
|
|
ASSERT_EQ(id1Type, readback[0].id1Type);
|
2012-08-02 02:28:35 +02:00
|
|
|
printf("XXX %ld %ld\n", id1Type, readback[0].id1Type);
|
2012-08-02 01:17:43 +02:00
|
|
|
ASSERT_EQ(id2Type, readback[0].id2Type);
|
|
|
|
ASSERT_EQ(ts, readback[0].time);
|
|
|
|
ASSERT_EQ(dataVersion, readback[0].dataVersion);
|
|
|
|
ASSERT_EQ(readback[0].data.compare(wormhole_comments), 0);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// close all resources
|
|
|
|
//
|
|
|
|
static void close() {
|
|
|
|
// close database
|
|
|
|
dbclient->Close(dbhandle, dbname);
|
|
|
|
// transport->close();
|
2012-07-03 07:45:59 +02:00
|
|
|
}
|
|
|
|
|
2012-08-02 01:17:43 +02:00
|
|
|
|
2012-07-03 07:45:59 +02:00
|
|
|
int main(int argc, char **argv) {
|
|
|
|
|
2012-07-11 23:08:46 +02:00
|
|
|
ARGC = argc;
|
|
|
|
ARGV = argv;
|
2012-07-03 07:45:59 +02:00
|
|
|
|
|
|
|
// create a server
|
2012-08-02 01:17:43 +02:00
|
|
|
startServer(argc, argv);
|
2012-07-03 07:45:59 +02:00
|
|
|
printf("Server thread created.\n");
|
|
|
|
|
|
|
|
// give some time to the server to initialize itself
|
2012-07-11 23:08:46 +02:00
|
|
|
while (server_options.getPort() == 0) {
|
|
|
|
sleep(1);
|
|
|
|
}
|
2012-07-03 07:45:59 +02:00
|
|
|
|
|
|
|
// test client
|
2012-08-02 01:17:43 +02:00
|
|
|
initialize(server_options.getPort());
|
|
|
|
|
|
|
|
// run all tests
|
|
|
|
testClient();
|
|
|
|
testAssocs();
|
|
|
|
|
|
|
|
// done all tests
|
|
|
|
close();
|
2012-07-03 07:45:59 +02:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|