From e4609a749be4978d5ae37bc7bae21f96e13dc291 Mon Sep 17 00:00:00 2001 From: Dmitri Smirnov Date: Tue, 2 Aug 2016 17:15:18 -0700 Subject: [PATCH] Fix Windows build issues (#1253) --- CMakeLists.txt | 3 ++ utilities/col_buf_decoder.cc | 2 +- utilities/column_aware_encoding_exp.cc | 6 ++-- utilities/column_aware_encoding_util.cc | 10 +++--- utilities/persistent_cache/block_cache_tier.h | 9 +++++- .../persistent_cache/block_cache_tier_file.cc | 7 ++-- .../persistent_cache/block_cache_tier_file.h | 4 +-- .../persistent_cache/persistent_cache_test.cc | 32 +++++++++---------- .../persistent_cache/persistent_cache_tier.h | 2 +- 9 files changed, 45 insertions(+), 30 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7630aa878..312b07026 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -265,6 +265,9 @@ set(SOURCES utilities/merge_operators/max.cc utilities/merge_operators/uint64add.cc utilities/options/options_util.cc + utilities/persistent_cache/block_cache_tier.cc + utilities/persistent_cache/block_cache_tier_file.cc + utilities/persistent_cache/block_cache_tier_metadata.cc utilities/persistent_cache/persistent_cache_tier.cc utilities/persistent_cache/volatile_tier_impl.cc utilities/redis/redis_lists.cc diff --git a/utilities/col_buf_decoder.cc b/utilities/col_buf_decoder.cc index e38dfef3f..ad11c0339 100644 --- a/utilities/col_buf_decoder.cc +++ b/utilities/col_buf_decoder.cc @@ -229,7 +229,7 @@ size_t VariableChunkColBufDecoder::Decode(const char* src, char** dest) { } memcpy(*dest, reinterpret_cast(&chunk_buf), 8); *dest += 8; - uint8_t mask = 0xFF - 8 + chunk_size; + uint8_t mask = ((0xFF - 8) + chunk_size) & 0xFF; memcpy(*dest, reinterpret_cast(&mask), 1); *dest += 1; } diff --git a/utilities/column_aware_encoding_exp.cc b/utilities/column_aware_encoding_exp.cc index c1cc1a906..1393bb7a1 100644 --- a/utilities/column_aware_encoding_exp.cc +++ b/utilities/column_aware_encoding_exp.cc @@ -106,10 +106,10 @@ class ColumnAwareEncodingExp { if (encoded_out_file != nullptr) { uint64_t size = 0; env->GetFileSize(FLAGS_encoded_file, &size); - fprintf(stdout, "File size: %lu\n", size); + fprintf(stdout, "File size: %llu\n", size); } uint64_t encode_time = sw.ElapsedNanosSafe(false /* reset */); - fprintf(stdout, "Encode time:%lu\n", encode_time); + fprintf(stdout, "Encode time:%llu\n", encode_time); if (decode) { unique_ptr decoded_out_file; if (!FLAGS_decoded_file.empty()) { @@ -124,7 +124,7 @@ class ColumnAwareEncodingExp { &encoded_blocks); } uint64_t decode_time = sw.ElapsedNanosSafe(true /* reset */); - fprintf(stdout, "Decode time:%lu\n", decode_time); + fprintf(stdout, "Decode time:%llu\n", decode_time); } } else { fprintf(stdout, "Unsupported compression type: %s.\n", diff --git a/utilities/column_aware_encoding_util.cc b/utilities/column_aware_encoding_util.cc index 43f9dfc25..43341d672 100644 --- a/utilities/column_aware_encoding_util.cc +++ b/utilities/column_aware_encoding_util.cc @@ -29,6 +29,8 @@ #include "utilities/col_buf_decoder.h" #include "utilities/col_buf_encoder.h" +#include "port/port.h" + namespace rocksdb { ColumnAwareEncodingReader::ColumnAwareEncodingReader( @@ -195,7 +197,7 @@ void ColumnAwareEncodingReader::DumpDataColumns( FILE* fp = fopen(filename.c_str(), "w"); size_t block_id = 1; for (auto& kv_pairs : kv_pair_blocks) { - fprintf(fp, "---------------- Block: %-4lu ----------------\n", block_id); + fprintf(fp, "---------------- Block: %-4" ROCKSDB_PRIszt " ----------------\n", block_id); for (auto& kv_pair : kv_pairs) { const auto& key = kv_pair.first; const auto& value = kv_pair.second; @@ -421,12 +423,12 @@ Status ColumnAwareEncodingReader::EncodeBlocks( total_size += value_checksum_size; for (size_t i = 0; i < key_col_sizes.size(); ++i) - printf("Key col %lu size: %lu percentage %lf%%\n", i, key_col_sizes[i], + printf("Key col %" ROCKSDB_PRIszt " size: %" ROCKSDB_PRIszt " percentage %lf%%\n", i, key_col_sizes[i], 100.0 * key_col_sizes[i] / total_size); for (size_t i = 0; i < value_col_sizes.size(); ++i) - printf("Value col %lu size: %lu percentage %lf%%\n", i, + printf("Value col %" ROCKSDB_PRIszt " size: %" ROCKSDB_PRIszt " percentage %lf%%\n", i, value_col_sizes[i], 100.0 * value_col_sizes[i] / total_size); - printf("Value checksum size: %lu percentage %lf%%\n", value_checksum_size, + printf("Value checksum size: %" ROCKSDB_PRIszt " percentage %lf%%\n", value_checksum_size, 100.0 * value_checksum_size / total_size); } return Status::OK(); diff --git a/utilities/persistent_cache/block_cache_tier.h b/utilities/persistent_cache/block_cache_tier.h index ad7f9132f..d0767abd6 100644 --- a/utilities/persistent_cache/block_cache_tier.h +++ b/utilities/persistent_cache/block_cache_tier.h @@ -4,7 +4,10 @@ // of patent rights can be found in the PATENTS file in the same directory. #pragma once +#ifndef OS_WIN #include +#endif // ! OS_WIN + #include #include #include @@ -22,7 +25,7 @@ #include "utilities/persistent_cache/persistent_cache_util.h" #include "db/skiplist.h" -#include "port/port_posix.h" +#include "port/port.h" #include "util/arena.h" #include "util/coding.h" #include "util/crc32c.h" @@ -65,7 +68,11 @@ class BlockCacheTier : public PersistentCacheTier { void TEST_Flush() override { while (insert_ops_.Size()) { +#ifdef OS_WIN + Sleep(1000); +#else /* sleep override */ sleep(1); +#endif } } diff --git a/utilities/persistent_cache/block_cache_tier_file.cc b/utilities/persistent_cache/block_cache_tier_file.cc index 7e9c1e632..0ea0345ad 100644 --- a/utilities/persistent_cache/block_cache_tier_file.cc +++ b/utilities/persistent_cache/block_cache_tier_file.cc @@ -6,7 +6,9 @@ #include "utilities/persistent_cache/block_cache_tier_file.h" +#ifndef OS_WIN #include +#endif #include #include @@ -353,7 +355,7 @@ bool WriteableCacheFile::ExpandBuffer(const size_t size) { return false; } - size_ += buf->Free(); + size_ += static_cast(buf->Free()); free += buf->Free(); bufs_.push_back(buf); } @@ -550,7 +552,8 @@ void ThreadedWriter::ThreadMain() { while (!cache_->Reserve(io.buf_->Used())) { // We can fail to reserve space if every file in the system // is being currently accessed - /* sleep override */ sleep(1); + /* sleep override */ + Env::Default()->SleepForMicroseconds(1000000); } DispatchIO(io); diff --git a/utilities/persistent_cache/block_cache_tier_file.h b/utilities/persistent_cache/block_cache_tier_file.h index 155e2886f..c2311ffcb 100644 --- a/utilities/persistent_cache/block_cache_tier_file.h +++ b/utilities/persistent_cache/block_cache_tier_file.h @@ -19,7 +19,7 @@ #include "utilities/persistent_cache/persistent_cache_tier.h" #include "utilities/persistent_cache/persistent_cache_util.h" -#include "port/port_posix.h" +#include "port/port.h" #include "util/crc32c.h" #include "util/mutexlock.h" @@ -200,7 +200,7 @@ class WriteableCacheFile : public RandomAccessCacheFile { } // append data to end of file - bool Append(const Slice&, const Slice&, LBA*) override; + bool Append(const Slice&, const Slice&, LBA* const) override; // End-of-file bool Eof() const { return eof_; } diff --git a/utilities/persistent_cache/persistent_cache_test.cc b/utilities/persistent_cache/persistent_cache_test.cc index 9d24dfbde..1c484bf7a 100644 --- a/utilities/persistent_cache/persistent_cache_test.cc +++ b/utilities/persistent_cache/persistent_cache_test.cc @@ -78,7 +78,7 @@ std::unique_ptr NewBlockCache( Env* env, const std::string& path, const uint64_t max_size = std::numeric_limits::max(), const bool enable_direct_writes = false) { - const uint32_t max_file_size = 12 * 1024 * 1024 * kStressFactor; + const uint32_t max_file_size = static_cast(12 * 1024 * 1024 * kStressFactor); auto log = std::make_shared(); PersistentCacheConfig opt(env, path, max_size, log); opt.cache_file_size = max_file_size; @@ -95,7 +95,7 @@ std::unique_ptr NewTieredCache( Env* env, const std::string& path, const uint64_t max_volatile_cache_size, const uint64_t max_block_cache_size = std::numeric_limits::max()) { - const uint32_t max_file_size = 12 * 1024 * 1024 * kStressFactor; + const uint32_t max_file_size = static_cast(12 * 1024 * 1024 * kStressFactor); auto log = std::make_shared(); auto opt = PersistentCacheConfig(env, path, max_block_cache_size, log); opt.cache_file_size = max_file_size; @@ -122,7 +122,7 @@ TEST_F(PersistentCacheTierTest, VolatileCacheInsert) { for (auto max_keys : {10 * 1024 * kStressFactor, 1 * 1024 * 1024 * kStressFactor}) { cache_ = std::make_shared(); - RunInsertTest(nthreads, max_keys); + RunInsertTest(nthreads, static_cast(max_keys)); } } } @@ -131,8 +131,8 @@ TEST_F(PersistentCacheTierTest, VolatileCacheInsertWithEviction) { for (auto nthreads : {1, 5}) { for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) { cache_ = std::make_shared( - /*compressed=*/true, /*size=*/1 * 1024 * 1024 * kStressFactor); - RunInsertTestWithEviction(nthreads, max_keys); + /*compressed=*/true, /*size=*/static_cast(1 * 1024 * 1024 * kStressFactor)); + RunInsertTestWithEviction(nthreads, static_cast(max_keys)); } } } @@ -146,7 +146,7 @@ TEST_F(PersistentCacheTierTest, BlockCacheInsert) { cache_ = NewBlockCache(Env::Default(), path_, /*size=*/std::numeric_limits::max(), direct_writes); - RunInsertTest(nthreads, max_keys); + RunInsertTest(nthreads, static_cast(max_keys)); } } } @@ -156,8 +156,8 @@ TEST_F(PersistentCacheTierTest, BlockCacheInsertWithEviction) { for (auto nthreads : {1, 5}) { for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) { cache_ = NewBlockCache(Env::Default(), path_, - /*max_size=*/200 * 1024 * 1024 * kStressFactor); - RunInsertTestWithEviction(nthreads, max_keys); + /*max_size=*/static_cast(200 * 1024 * 1024 * kStressFactor)); + RunInsertTestWithEviction(nthreads, static_cast(max_keys)); } } } @@ -168,8 +168,8 @@ TEST_F(PersistentCacheTierTest, TieredCacheInsert) { for (auto max_keys : {10 * 1024 * kStressFactor, 1 * 1024 * 1024 * kStressFactor}) { cache_ = NewTieredCache(Env::Default(), path_, - /*memory_size=*/1 * 1024 * 1024 * kStressFactor); - RunInsertTest(nthreads, max_keys); + /*memory_size=*/static_cast(1 * 1024 * 1024 * kStressFactor)); + RunInsertTest(nthreads, static_cast(max_keys)); } } } @@ -179,9 +179,9 @@ TEST_F(PersistentCacheTierTest, TieredCacheInsertWithEviction) { for (auto max_keys : {1 * 1024 * 1024 * kStressFactor}) { cache_ = NewTieredCache( Env::Default(), path_, - /*memory_size=*/1 * 1024 * 1024 * kStressFactor, - /*block_cache_size*/ 200 * 1024 * 1024 * kStressFactor); - RunInsertTestWithEviction(nthreads, max_keys); + /*memory_size=*/static_cast(1 * 1024 * 1024 * kStressFactor), + /*block_cache_size*/ static_cast(200 * 1024 * 1024 * kStressFactor)); + RunInsertTestWithEviction(nthreads, static_cast(max_keys)); } } } @@ -198,7 +198,7 @@ std::shared_ptr MakeBlockCache(const std::string& dbname) { std::shared_ptr MakeTieredCache( const std::string& dbname) { const auto memory_size = 1 * 1024 * 1024 * kStressFactor; - return NewTieredCache(Env::Default(), dbname, memory_size); + return NewTieredCache(Env::Default(), dbname, static_cast(memory_size)); } #ifdef OS_LINUX @@ -233,12 +233,12 @@ void PersistentCacheDBTest::RunTest( } // number of insertion interations - int num_iter = 100 * 1024 * kStressFactor; + int num_iter = static_cast(100 * 1024 * kStressFactor); for (int iter = 0; iter < 5; iter++) { Options options; options.write_buffer_size = - 64 * 1024 * kStressFactor; // small write buffer + static_cast(64 * 1024 * kStressFactor); // small write buffer options.statistics = rocksdb::CreateDBStatistics(); options = CurrentOptions(options); diff --git a/utilities/persistent_cache/persistent_cache_tier.h b/utilities/persistent_cache/persistent_cache_tier.h index 590e5d57f..de088feb0 100644 --- a/utilities/persistent_cache/persistent_cache_tier.h +++ b/utilities/persistent_cache/persistent_cache_tier.h @@ -200,7 +200,7 @@ struct PersistentCacheConfig { // file size in order to avoid dead lock. size_t write_buffer_count() const { assert(write_buffer_size); - return (writer_qdepth + 1.2) * cache_file_size / write_buffer_size; + return (writer_qdepth + writer_qdepth/5) * cache_file_size / write_buffer_size; } // writer-dispatch-size