andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix Windows build issues (#1253)

Browse Source
This commit is contained in:
Dmitri Smirnov 2016-08-02 17:15:18 -07:00 committed by Siying Dong
parent 2306167d30
commit e4609a749b
9 changed files with 45 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@ -265,6 +265,9 @@ set(SOURCES
utilities/merge_operators/max.cc utilities/merge_operators/max.cc
utilities/merge_operators/uint64add.cc utilities/merge_operators/uint64add.cc
utilities/options/options_util.cc utilities/options/options_util.cc
utilities/persistent_cache/block_cache_tier.cc
utilities/persistent_cache/block_cache_tier_file.cc
utilities/persistent_cache/block_cache_tier_metadata.cc
utilities/persistent_cache/persistent_cache_tier.cc utilities/persistent_cache/persistent_cache_tier.cc
utilities/persistent_cache/volatile_tier_impl.cc utilities/persistent_cache/volatile_tier_impl.cc
utilities/redis/redis_lists.cc utilities/redis/redis_lists.cc

2
utilities/col_buf_decoder.cc
View File

@ -229,7 +229,7 @@ size_t VariableChunkColBufDecoder::Decode(const char* src, char** dest) {
} }
memcpy(*dest, reinterpret_cast<char*>(&chunk_buf), 8); memcpy(*dest, reinterpret_cast<char*>(&chunk_buf), 8);
*dest += 8; *dest += 8;
uint8_t mask = 0xFF - 8 + chunk_size; uint8_t mask = ((0xFF - 8) + chunk_size) & 0xFF;
memcpy(*dest, reinterpret_cast<char*>(&mask), 1); memcpy(*dest, reinterpret_cast<char*>(&mask), 1);
*dest += 1; *dest += 1;
} }

6
utilities/column_aware_encoding_exp.cc
View File

@ -106,10 +106,10 @@ class ColumnAwareEncodingExp {
if (encoded_out_file != nullptr) { if (encoded_out_file != nullptr) {
uint64_t size = 0; uint64_t size = 0;
env->GetFileSize(FLAGS_encoded_file, &size); env->GetFileSize(FLAGS_encoded_file, &size);
fprintf(stdout, "File size: %lu\n", size); fprintf(stdout, "File size: %llu\n", size);
} }
uint64_t encode_time = sw.ElapsedNanosSafe(false /* reset */); uint64_t encode_time = sw.ElapsedNanosSafe(false /* reset */);
fprintf(stdout, "Encode time:%lu\n", encode_time); fprintf(stdout, "Encode time:%llu\n", encode_time);
if (decode) { if (decode) {
unique_ptr<WritableFile> decoded_out_file; unique_ptr<WritableFile> decoded_out_file;
if (!FLAGS_decoded_file.empty()) { if (!FLAGS_decoded_file.empty()) {
@ -124,7 +124,7 @@ class ColumnAwareEncodingExp {
&encoded_blocks); &encoded_blocks);
} }
uint64_t decode_time = sw.ElapsedNanosSafe(true /* reset */); uint64_t decode_time = sw.ElapsedNanosSafe(true /* reset */);
fprintf(stdout, "Decode time:%lu\n", decode_time); fprintf(stdout, "Decode time:%llu\n", decode_time);
} }
} else { } else {
fprintf(stdout, "Unsupported compression type: %s.\n", fprintf(stdout, "Unsupported compression type: %s.\n",

10
utilities/column_aware_encoding_util.cc
View File

@ -29,6 +29,8 @@
#include "utilities/col_buf_decoder.h" #include "utilities/col_buf_decoder.h"
#include "utilities/col_buf_encoder.h" #include "utilities/col_buf_encoder.h"
#include "port/port.h"
namespace rocksdb { namespace rocksdb {
ColumnAwareEncodingReader::ColumnAwareEncodingReader( ColumnAwareEncodingReader::ColumnAwareEncodingReader(
@ -195,7 +197,7 @@ void ColumnAwareEncodingReader::DumpDataColumns(
FILE* fp = fopen(filename.c_str(), "w"); FILE* fp = fopen(filename.c_str(), "w");
size_t block_id = 1; size_t block_id = 1;
for (auto& kv_pairs : kv_pair_blocks) { for (auto& kv_pairs : kv_pair_blocks) {
fprintf(fp, "---------------- Block: %-4lu ----------------\n", block_id); fprintf(fp, "---------------- Block: %-4" ROCKSDB_PRIszt " ----------------\n", block_id);
for (auto& kv_pair : kv_pairs) { for (auto& kv_pair : kv_pairs) {
const auto& key = kv_pair.first; const auto& key = kv_pair.first;
const auto& value = kv_pair.second; const auto& value = kv_pair.second;
@ -421,12 +423,12 @@ Status ColumnAwareEncodingReader::EncodeBlocks(
total_size += value_checksum_size; total_size += value_checksum_size;
for (size_t i = 0; i < key_col_sizes.size(); ++i) for (size_t i = 0; i < key_col_sizes.size(); ++i)
printf("Key col %lu size: %lu percentage %lf%%\n", i, key_col_sizes[i], printf("Key col %" ROCKSDB_PRIszt " size: %" ROCKSDB_PRIszt " percentage %lf%%\n", i, key_col_sizes[i],
100.0 * key_col_sizes[i] / total_size); 100.0 * key_col_sizes[i] / total_size);
for (size_t i = 0; i < value_col_sizes.size(); ++i) for (size_t i = 0; i < value_col_sizes.size(); ++i)
printf("Value col %lu size: %lu percentage %lf%%\n", i, printf("Value col %" ROCKSDB_PRIszt " size: %" ROCKSDB_PRIszt " percentage %lf%%\n", i,
value_col_sizes[i], 100.0 * value_col_sizes[i] / total_size); value_col_sizes[i], 100.0 * value_col_sizes[i] / total_size);
printf("Value checksum size: %lu percentage %lf%%\n", value_checksum_size, printf("Value checksum size: %" ROCKSDB_PRIszt " percentage %lf%%\n", value_checksum_size,
100.0 * value_checksum_size / total_size); 100.0 * value_checksum_size / total_size);
} }
return Status::OK(); return Status::OK();

9
utilities/persistent_cache/block_cache_tier.h
View File

@ -4,7 +4,10 @@
// of patent rights can be found in the PATENTS file in the same directory. // of patent rights can be found in the PATENTS file in the same directory.
#pragma once #pragma once
#ifndef OS_WIN
#include <unistd.h> #include <unistd.h>
#endif // ! OS_WIN
#include <list> #include <list>
#include <memory> #include <memory>
#include <set> #include <set>
@ -22,7 +25,7 @@
#include "utilities/persistent_cache/persistent_cache_util.h" #include "utilities/persistent_cache/persistent_cache_util.h"
#include "db/skiplist.h" #include "db/skiplist.h"
#include "port/port_posix.h" #include "port/port.h"
#include "util/arena.h" #include "util/arena.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/crc32c.h" #include "util/crc32c.h"
@ -65,7 +68,11 @@ class BlockCacheTier : public PersistentCacheTier {
void TEST_Flush() override { void TEST_Flush() override {
while (insert_ops_.Size()) { while (insert_ops_.Size()) {
#ifdef OS_WIN
Sleep(1000);
#else
/* sleep override */ sleep(1); /* sleep override */ sleep(1);
#endif
} }
} }

7
utilities/persistent_cache/block_cache_tier_file.cc
View File

@ -6,7 +6,9 @@
#include "utilities/persistent_cache/block_cache_tier_file.h" #include "utilities/persistent_cache/block_cache_tier_file.h"
#ifndef OS_WIN
#include <unistd.h> #include <unistd.h>
#endif
#include <memory> #include <memory>
#include <vector> #include <vector>
@ -353,7 +355,7 @@ bool WriteableCacheFile::ExpandBuffer(const size_t size) {
return false; return false;
} }
size_ += buf->Free(); size_ += static_cast<uint32_t>(buf->Free());
free += buf->Free(); free += buf->Free();
bufs_.push_back(buf); bufs_.push_back(buf);
} }
@ -550,7 +552,8 @@ void ThreadedWriter::ThreadMain() {
while (!cache_->Reserve(io.buf_->Used())) { while (!cache_->Reserve(io.buf_->Used())) {
// We can fail to reserve space if every file in the system // We can fail to reserve space if every file in the system
// is being currently accessed // is being currently accessed
/* sleep override */ sleep(1); /* sleep override */
Env::Default()->SleepForMicroseconds(1000000);
} }
DispatchIO(io); DispatchIO(io);

4
utilities/persistent_cache/block_cache_tier_file.h
View File

@ -19,7 +19,7 @@
#include "utilities/persistent_cache/persistent_cache_tier.h" #include "utilities/persistent_cache/persistent_cache_tier.h"
#include "utilities/persistent_cache/persistent_cache_util.h" #include "utilities/persistent_cache/persistent_cache_util.h"
#include "port/port_posix.h" #include "port/port.h"
#include "util/crc32c.h" #include "util/crc32c.h"
#include "util/mutexlock.h" #include "util/mutexlock.h"
@ -200,7 +200,7 @@ class WriteableCacheFile : public RandomAccessCacheFile {
} }
// append data to end of file // append data to end of file
bool Append(const Slice&, const Slice&, LBA*) override; bool Append(const Slice&, const Slice&, LBA* const) override;
// End-of-file // End-of-file
bool Eof() const { return eof_; } bool Eof() const { return eof_; }

32
utilities/persistent_cache/persistent_cache_test.cc
View File

@ -78,7 +78,7 @@ std::unique_ptr<PersistentCacheTier> NewBlockCache(
Env* env, const std::string& path, Env* env, const std::string& path,
const uint64_t max_size = std::numeric_limits<uint64_t>::max(), const uint64_t max_size = std::numeric_limits<uint64_t>::max(),
const bool enable_direct_writes = false) { const bool enable_direct_writes = false) {
const uint32_t max_file_size = 12 * 1024 * 1024 * kStressFactor; const uint32_t max_file_size = static_cast<uint32_t>(12 * 1024 * 1024 * kStressFactor);
auto log = std::make_shared<ConsoleLogger>(); auto log = std::make_shared<ConsoleLogger>();
PersistentCacheConfig opt(env, path, max_size, log); PersistentCacheConfig opt(env, path, max_size, log);
opt.cache_file_size = max_file_size; opt.cache_file_size = max_file_size;
@ -95,7 +95,7 @@ std::unique_ptr<PersistentTieredCache> NewTieredCache(
Env* env, const std::string& path, const uint64_t max_volatile_cache_size, Env* env, const std::string& path, const uint64_t max_volatile_cache_size,
const uint64_t max_block_cache_size = const uint64_t max_block_cache_size =
std::numeric_limits<uint64_t>::max()) { std::numeric_limits<uint64_t>::max()) {
const uint32_t max_file_size = 12 * 1024 * 1024 * kStressFactor; const uint32_t max_file_size = static_cast<uint32_t>(12 * 1024 * 1024 * kStressFactor);
auto log = std::make_shared<ConsoleLogger>(); auto log = std::make_shared<ConsoleLogger>();
auto opt = PersistentCacheConfig(env, path, max_block_cache_size, log); auto opt = PersistentCacheConfig(env, path, max_block_cache_size, log);
opt.cache_file_size = max_file_size; opt.cache_file_size = max_file_size;
@ -122,7 +122,7 @@ TEST_F(PersistentCacheTierTest, VolatileCacheInsert) {
for (auto max_keys : for (auto max_keys :
{10 * 1024 * kStressFactor, 1 * 1024 * 1024 * kStressFactor}) { {10 * 1024 * kStressFactor, 1 * 1024 * 1024 * kStressFactor}) {
cache_ = std::make_shared<VolatileCacheTier>(); cache_ = std::make_shared<VolatileCacheTier>();
RunInsertTest(nthreads, max_keys); RunInsertTest(nthreads, static_cast<size_t>(max_keys));
} }
} }
} }
@ -131,8 +131,8 @@ TEST_F(PersistentCacheTierTest, VolatileCacheInsertWithEviction) {
for (auto nthreads : {1, 5}) { for (auto nthreads : {1, 5}) {
for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) { for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) {
cache_ = std::make_shared<VolatileCacheTier>( cache_ = std::make_shared<VolatileCacheTier>(
/*compressed=*/true, /*size=*/1 * 1024 * 1024 * kStressFactor); /*compressed=*/true, /*size=*/static_cast<size_t>(1 * 1024 * 1024 * kStressFactor));
RunInsertTestWithEviction(nthreads, max_keys); RunInsertTestWithEviction(nthreads, static_cast<size_t>(max_keys));
} }
} }
} }
@ -146,7 +146,7 @@ TEST_F(PersistentCacheTierTest, BlockCacheInsert) {
cache_ = NewBlockCache(Env::Default(), path_, cache_ = NewBlockCache(Env::Default(), path_,
/*size=*/std::numeric_limits<uint64_t>::max(), /*size=*/std::numeric_limits<uint64_t>::max(),
direct_writes); direct_writes);
RunInsertTest(nthreads, max_keys); RunInsertTest(nthreads, static_cast<size_t>(max_keys));
} }
} }
} }
@ -156,8 +156,8 @@ TEST_F(PersistentCacheTierTest, BlockCacheInsertWithEviction) {
for (auto nthreads : {1, 5}) { for (auto nthreads : {1, 5}) {
for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) { for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) {
cache_ = NewBlockCache(Env::Default(), path_, cache_ = NewBlockCache(Env::Default(), path_,
/*max_size=*/200 * 1024 * 1024 * kStressFactor); /*max_size=*/static_cast<size_t>(200 * 1024 * 1024 * kStressFactor));
RunInsertTestWithEviction(nthreads, max_keys); RunInsertTestWithEviction(nthreads, static_cast<size_t>(max_keys));
} }
} }
} }
@ -168,8 +168,8 @@ TEST_F(PersistentCacheTierTest, TieredCacheInsert) {
for (auto max_keys : for (auto max_keys :
{10 * 1024 * kStressFactor, 1 * 1024 * 1024 * kStressFactor}) { {10 * 1024 * kStressFactor, 1 * 1024 * 1024 * kStressFactor}) {
cache_ = NewTieredCache(Env::Default(), path_, cache_ = NewTieredCache(Env::Default(), path_,
/*memory_size=*/1 * 1024 * 1024 * kStressFactor); /*memory_size=*/static_cast<size_t>(1 * 1024 * 1024 * kStressFactor));
RunInsertTest(nthreads, max_keys); RunInsertTest(nthreads, static_cast<size_t>(max_keys));
} }
} }
} }
@ -179,9 +179,9 @@ TEST_F(PersistentCacheTierTest, TieredCacheInsertWithEviction) {
for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) { for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) {
cache_ = NewTieredCache( cache_ = NewTieredCache(
Env::Default(), path_, Env::Default(), path_,
/*memory_size=*/1 * 1024 * 1024 * kStressFactor, /*memory_size=*/static_cast<size_t>(1 * 1024 * 1024 * kStressFactor),
/*block_cache_size*/ 200 * 1024 * 1024 * kStressFactor); /*block_cache_size*/ static_cast<size_t>(200 * 1024 * 1024 * kStressFactor));
RunInsertTestWithEviction(nthreads, max_keys); RunInsertTestWithEviction(nthreads, static_cast<size_t>(max_keys));
} }
} }
} }
@ -198,7 +198,7 @@ std::shared_ptr<PersistentCacheTier> MakeBlockCache(const std::string& dbname) {
std::shared_ptr<PersistentCacheTier> MakeTieredCache( std::shared_ptr<PersistentCacheTier> MakeTieredCache(
const std::string& dbname) { const std::string& dbname) {
const auto memory_size = 1 * 1024 * 1024 * kStressFactor; const auto memory_size = 1 * 1024 * 1024 * kStressFactor;
return NewTieredCache(Env::Default(), dbname, memory_size); return NewTieredCache(Env::Default(), dbname, static_cast<size_t>(memory_size));
} }
#ifdef OS_LINUX #ifdef OS_LINUX
@ -233,12 +233,12 @@ void PersistentCacheDBTest::RunTest(
} }
// number of insertion interations // number of insertion interations
int num_iter = 100 * 1024 * kStressFactor; int num_iter = static_cast<int>(100 * 1024 * kStressFactor);
for (int iter = 0; iter < 5; iter++) { for (int iter = 0; iter < 5; iter++) {
Options options; Options options;
options.write_buffer_size = options.write_buffer_size =
64 * 1024 * kStressFactor; // small write buffer static_cast<size_t>(64 * 1024 * kStressFactor); // small write buffer
options.statistics = rocksdb::CreateDBStatistics(); options.statistics = rocksdb::CreateDBStatistics();
options = CurrentOptions(options); options = CurrentOptions(options);

2
utilities/persistent_cache/persistent_cache_tier.h
View File

@ -200,7 +200,7 @@ struct PersistentCacheConfig {
// file size in order to avoid dead lock. // file size in order to avoid dead lock.
size_t write_buffer_count() const { size_t write_buffer_count() const {
assert(write_buffer_size); assert(write_buffer_size);
return (writer_qdepth + 1.2) * cache_file_size / write_buffer_size; return (writer_qdepth + writer_qdepth/5) * cache_file_size / write_buffer_size;
} }
// writer-dispatch-size // writer-dispatch-size