Merge branch 'master' into performance

Conflicts:
	db/table_cache.cc
This commit is contained in:
kailiu 2014-01-02 16:34:59 -08:00
commit e72aa37cc5
18 changed files with 169 additions and 29 deletions

47
.clang-format Normal file
View File

@ -0,0 +1,47 @@
# Complete list of style options can be found at:
# http://clang.llvm.org/docs/ClangFormatStyleOptions.html
---
BasedOnStyle: Google
AccessModifierOffset: -1
ConstructorInitializerIndentWidth: 4
AlignEscapedNewlinesLeft: true
AlignTrailingComments: true
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakTemplateDeclarations: true
AlwaysBreakBeforeMultilineStrings: true
BreakBeforeBinaryOperators: false
BreakConstructorInitializersBeforeComma: false
BinPackParameters: false
ColumnLimit: 80
ConstructorInitializerAllOnOneLineOrOnePerLine: true
DerivePointerBinding: true
ExperimentalAutoDetectBinPacking: true
IndentCaseLabels: false
MaxEmptyLinesToKeep: 1
NamespaceIndentation: None
ObjCSpaceBeforeProtocolList: false
PenaltyBreakBeforeFirstCallParameter: 10
PenaltyBreakComment: 60
PenaltyBreakString: 1000
PenaltyBreakFirstLessLess: 20
PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 200
PointerBindsToType: true
SpacesBeforeTrailingComments: 2
Cpp11BracedListStyle: true
Standard: Cpp11
IndentWidth: 2
TabWidth: 8
UseTab: Never
BreakBeforeBraces: Attach
IndentFunctionDeclarationAfterType: false
SpacesInParentheses: false
SpacesInAngles: false
SpaceInEmptyParentheses: false
SpacesInCStyleCastParentheses: false
SpaceAfterControlStatementKeyword: true
SpaceBeforeAssignmentOperators: true
ContinuationIndentWidth: 4
...

View File

@ -44,5 +44,7 @@ libraries. You are on your own.
`make clean; make` will compile librocksdb.a (RocskDB static library) and all `make clean; make` will compile librocksdb.a (RocskDB static library) and all
the unit tests. You can run all unit tests with `make check`. the unit tests. You can run all unit tests with `make check`.
For shared library builds, exec `make librocksdb.so` instead.
If you followed the above steps and your compile or unit tests fail, If you followed the above steps and your compile or unit tests fail,
please submit an issue: (https://github.com/facebook/rocksdb/issues) please submit an issue: (https://github.com/facebook/rocksdb/issues)

View File

@ -45,8 +45,8 @@ VALGRIND_VER := $(join $(VALGRIND_VER),valgrind)
VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full
TESTS = \ TESTS = \
autovector_test \
db_test \ db_test \
autovector_test \
table_properties_collector_test \ table_properties_collector_test \
arena_test \ arena_test \
auto_roll_logger_test \ auto_roll_logger_test \

View File

@ -1,3 +1,3 @@
* Detailed instructions on how to compile using fbcode and jemalloc * Detailed instructions on how to compile using fbcode and jemalloc
* Latest release is 2.5.fb * Latest release is 2.7.fb

View File

@ -198,7 +198,7 @@ EOF
} }
EOF EOF
if [ "$?" = 0 ]; then if [ "$?" = 0 ]; then
COMMON_FLAGS="$PLATFORM_LDFLAGS -DROCKSDB_FALLOCATE_PRESENT" COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT"
fi fi
# Test whether Snappy library is installed # Test whether Snappy library is installed

View File

@ -8,7 +8,7 @@
# #
# create git version file # create git version file
VFILE=$ROCKSDB_ROOT/util/build_version.cc.tmp VFILE=$PWD/util/build_version.cc.tmp
trap "rm $VFILE" EXIT trap "rm $VFILE" EXIT
# check to see if git is in the path # check to see if git is in the path
@ -36,7 +36,7 @@ echo "const char* rocksdb_build_git_datetime = \"rocksdb_build_git_datetime:$(da
echo "const char* rocksdb_build_compile_date = __DATE__;" >> ${VFILE} echo "const char* rocksdb_build_compile_date = __DATE__;" >> ${VFILE}
echo "const char* rocksdb_build_compile_time = __TIME__;" >> ${VFILE} echo "const char* rocksdb_build_compile_time = __TIME__;" >> ${VFILE}
OUTFILE=$ROCKSDB_ROOT/util/build_version.cc OUTFILE=$PWD/util/build_version.cc
if [ ! -e $OUTFILE ] || ! cmp -s $VFILE $OUTFILE; then if [ ! -e $OUTFILE ] || ! cmp -s $VFILE $OUTFILE; then
cp $VFILE $OUTFILE cp $VFILE $OUTFILE
fi fi

View File

@ -54,7 +54,7 @@ RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic" CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic"
CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC" CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE" CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE"
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT" CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT"
CFLAGS+=" -DSNAPPY -DGFLAGS -DZLIB -DBZIP2" CFLAGS+=" -DSNAPPY -DGFLAGS -DZLIB -DBZIP2"
EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a" EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"

View File

@ -61,7 +61,7 @@ RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic" CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic"
CFLAGS+=" -nostdlib $LIBGCC_INCLUDE $GLIBC_INCLUDE" CFLAGS+=" -nostdlib $LIBGCC_INCLUDE $GLIBC_INCLUDE"
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT" CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT"
CFLAGS+=" -DSNAPPY -DGFLAGS -DZLIB -DBZIP2" CFLAGS+=" -DSNAPPY -DGFLAGS -DZLIB -DBZIP2"
EXEC_LDFLAGS="-Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so" EXEC_LDFLAGS="-Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"

61
build_tools/make_new_version.sh Executable file
View File

@ -0,0 +1,61 @@
#!/bin/bash
# Copyright (c) 2013, Facebook, Inc. All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
set -e
# Print out the colored progress info so that it can be brainlessly
# distinguished by users.
function title() {
echo -e "\033[1;32m$*\033[0m"
}
usage="Create new rocksdb version and prepare it for the release process\n"
usage+="USAGE: ./make_new_version.sh <version>"
# -- Pre-check
if [[ $# < 1 ]]; then
echo -e $usage
exit 1
fi
ROCKSDB_VERSION=$1
GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
if [ $GIT_BRANCH != "master" ]; then
echo "Error: Current branch is '$GIT_BRANCH', Please switch to master branch."
fi
# --Step 1: cutting new tag
title "Adding new tag for this release ..."
git tag -a "$ROCKSDB_VERSION.fb" -m "Rocksdb $ROCKSDB_VERSION"
# Setting up the proxy for remote repo access
export http_proxy=http://172.31.255.99:8080
export https_proxy="$http_proxy";
title "Pushing new tag to remote repo ..."
proxycmd.sh git push origin --tags
# --Step 2: Update README.fb
title "Updating the latest version info in README.fb ..."
sed -i "s/Latest release is [0-9]\+.[0-9]\+.fb/Latest release is $ROCKSDB_VERSION.fb/" README.fb
git commit README.fb -m "update the latest version in README.fb to $ROCKSDB_VERSION"
proxycmd.sh git push
# --Step 3: Prepare this repo for 3rd release
title "Cleaning up repo ..."
make clean
git clean -fxd
title "Generating the build info ..."
# Comment out the call of `build_detection_version` so that the SHA number and build date of this
# release will remain constant. Otherwise everytime we run "make" util/build_version.cc will be
# overridden.
sed -i 's/^\$PWD\/build_tools\/build_detect_version$//' build_tools/build_detect_platform
# Generate util/build_version.cc
build_tools/build_detect_version
title "Done!"

View File

@ -22,20 +22,34 @@ namespace rocksdb {
Status DBImpl::DisableFileDeletions() { Status DBImpl::DisableFileDeletions() {
MutexLock l(&mutex_); MutexLock l(&mutex_);
disable_delete_obsolete_files_ = true; ++disable_delete_obsolete_files_;
if (disable_delete_obsolete_files_ == 1) {
// if not, it has already been disabled, so don't log anything
Log(options_.info_log, "File Deletions Disabled"); Log(options_.info_log, "File Deletions Disabled");
}
return Status::OK(); return Status::OK();
} }
Status DBImpl::EnableFileDeletions() { Status DBImpl::EnableFileDeletions(bool force) {
DeletionState deletion_state; DeletionState deletion_state;
bool should_purge_files = false;
{ {
MutexLock l(&mutex_); MutexLock l(&mutex_);
disable_delete_obsolete_files_ = false; if (force) {
// if force, we need to enable file deletions right away
disable_delete_obsolete_files_ = 0;
} else if (disable_delete_obsolete_files_ > 0) {
--disable_delete_obsolete_files_;
}
if (disable_delete_obsolete_files_ == 0) {
Log(options_.info_log, "File Deletions Enabled"); Log(options_.info_log, "File Deletions Enabled");
should_purge_files = true;
FindObsoleteFiles(deletion_state, true); FindObsoleteFiles(deletion_state, true);
} }
}
if (should_purge_files) {
PurgeObsoleteFiles(deletion_state); PurgeObsoleteFiles(deletion_state);
}
LogFlush(options_.info_log); LogFlush(options_.info_log);
return Status::OK(); return Status::OK();
} }

View File

@ -247,7 +247,7 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname)
bg_logstats_scheduled_(false), bg_logstats_scheduled_(false),
manual_compaction_(nullptr), manual_compaction_(nullptr),
logger_(nullptr), logger_(nullptr),
disable_delete_obsolete_files_(false), disable_delete_obsolete_files_(0),
delete_obsolete_files_last_run_(options.env->NowMicros()), delete_obsolete_files_last_run_(options.env->NowMicros()),
purge_wal_files_last_run_(0), purge_wal_files_last_run_(0),
last_stats_dump_time_microsec_(0), last_stats_dump_time_microsec_(0),
@ -509,7 +509,7 @@ void DBImpl::FindObsoleteFiles(DeletionState& deletion_state,
mutex_.AssertHeld(); mutex_.AssertHeld();
// if deletion is disabled, do nothing // if deletion is disabled, do nothing
if (disable_delete_obsolete_files_) { if (disable_delete_obsolete_files_ > 0) {
return; return;
} }
@ -1262,7 +1262,7 @@ Status DBImpl::FlushMemTableToOutputFile(bool* madeProgress,
MaybeScheduleLogDBDeployStats(); MaybeScheduleLogDBDeployStats();
if (!disable_delete_obsolete_files_) { if (disable_delete_obsolete_files_ == 0) {
// add to deletion state // add to deletion state
deletion_state.log_delete_files.insert( deletion_state.log_delete_files.insert(
deletion_state.log_delete_files.end(), deletion_state.log_delete_files.end(),

View File

@ -73,7 +73,7 @@ class DBImpl : public DB {
virtual const Options& GetOptions() const; virtual const Options& GetOptions() const;
virtual Status Flush(const FlushOptions& options); virtual Status Flush(const FlushOptions& options);
virtual Status DisableFileDeletions(); virtual Status DisableFileDeletions();
virtual Status EnableFileDeletions(); virtual Status EnableFileDeletions(bool force);
// All the returned filenames start with "/" // All the returned filenames start with "/"
virtual Status GetLiveFiles(std::vector<std::string>&, virtual Status GetLiveFiles(std::vector<std::string>&,
uint64_t* manifest_file_size, uint64_t* manifest_file_size,
@ -417,7 +417,12 @@ class DBImpl : public DB {
int64_t volatile last_log_ts; int64_t volatile last_log_ts;
// shall we disable deletion of obsolete files // shall we disable deletion of obsolete files
bool disable_delete_obsolete_files_; // if 0 the deletion is enabled.
// if non-zero, files will not be getting deleted
// This enables two different threads to call
// EnableFileDeletions() and DisableFileDeletions()
// without any synchronization
int disable_delete_obsolete_files_;
// last time when DeleteObsoleteFiles was invoked // last time when DeleteObsoleteFiles was invoked
uint64_t delete_obsolete_files_last_run_; uint64_t delete_obsolete_files_last_run_;

View File

@ -55,7 +55,7 @@ public:
virtual Status DisableFileDeletions() { virtual Status DisableFileDeletions() {
return Status::NotSupported("Not supported operation in read only mode."); return Status::NotSupported("Not supported operation in read only mode.");
} }
virtual Status EnableFileDeletions() { virtual Status EnableFileDeletions(bool force) {
return Status::NotSupported("Not supported operation in read only mode."); return Status::NotSupported("Not supported operation in read only mode.");
} }
virtual Status GetLiveFiles(std::vector<std::string>&, virtual Status GetLiveFiles(std::vector<std::string>&,

View File

@ -4381,7 +4381,7 @@ class ModelDB: public DB {
virtual Status DisableFileDeletions() { virtual Status DisableFileDeletions() {
return Status::OK(); return Status::OK();
} }
virtual Status EnableFileDeletions() { virtual Status EnableFileDeletions(bool force) {
return Status::OK(); return Status::OK();
} }
virtual Status GetLiveFiles(std::vector<std::string>&, uint64_t* size, virtual Status GetLiveFiles(std::vector<std::string>&, uint64_t* size,

View File

@ -29,6 +29,11 @@ static void UnrefEntry(void* arg1, void* arg2) {
cache->Release(h); cache->Release(h);
} }
static Slice GetSliceForFileNumber(uint64_t* file_number) {
return Slice(reinterpret_cast<const char*>(file_number),
sizeof(*file_number));
}
TableCache::TableCache(const std::string& dbname, TableCache::TableCache(const std::string& dbname,
const Options* options, const Options* options,
const EnvOptions& storage_options, const EnvOptions& storage_options,
@ -50,8 +55,7 @@ Status TableCache::FindTable(const EnvOptions& toptions,
Cache::Handle** handle, bool* table_io, Cache::Handle** handle, bool* table_io,
const bool no_io) { const bool no_io) {
Status s; Status s;
Slice key(reinterpret_cast<const char*>(&file_number), sizeof(file_number)); Slice key = GetSliceForFileNumber(&file_number);
*handle = cache_->Lookup(key); *handle = cache_->Lookup(key);
if (*handle == nullptr) { if (*handle == nullptr) {
if (no_io) { // Dont do IO and return a not-found status if (no_io) { // Dont do IO and return a not-found status
@ -164,8 +168,7 @@ bool TableCache::PrefixMayMatch(const ReadOptions& options,
} }
void TableCache::Evict(uint64_t file_number) { void TableCache::Evict(uint64_t file_number) {
Slice key(reinterpret_cast<const char*>(&file_number), sizeof(file_number)); cache_->Erase(GetSliceForFileNumber(&file_number));
cache_->Erase(key);
} }
} // namespace rocksdb } // namespace rocksdb

View File

@ -247,7 +247,15 @@ class DB {
virtual Status DisableFileDeletions() = 0; virtual Status DisableFileDeletions() = 0;
// Allow compactions to delete obselete files. // Allow compactions to delete obselete files.
virtual Status EnableFileDeletions() = 0; // If force == true, the call to EnableFileDeletions() will guarantee that
// file deletions are enabled after the call, even if DisableFileDeletions()
// was called multiple times before.
// If force == false, EnableFileDeletions will only enable file deletion
// after it's been called at least as many times as DisableFileDeletions(),
// enabling the two methods to be called by two threads concurrently without
// synchronization -- i.e., file deletions will be enabled only after both
// threads call EnableFileDeletions()
virtual Status EnableFileDeletions(bool force = true) = 0;
// GetLiveFiles followed by GetSortedWalFiles can generate a lossless backup // GetLiveFiles followed by GetSortedWalFiles can generate a lossless backup

View File

@ -123,8 +123,8 @@ class StackableDB : public DB {
return db_->DisableFileDeletions(); return db_->DisableFileDeletions();
} }
virtual Status EnableFileDeletions() override { virtual Status EnableFileDeletions(bool force) override {
return db_->EnableFileDeletions(); return db_->EnableFileDeletions(force);
} }
virtual Status GetLiveFiles(std::vector<std::string>& vec, uint64_t* mfs, virtual Status GetLiveFiles(std::vector<std::string>& vec, uint64_t* mfs,

View File

@ -48,7 +48,7 @@ class DummyDB : public StackableDB {
return options_; return options_;
} }
virtual Status EnableFileDeletions() override { virtual Status EnableFileDeletions(bool force) override {
ASSERT_TRUE(!deletions_enabled_); ASSERT_TRUE(!deletions_enabled_);
deletions_enabled_ = true; deletions_enabled_ = true;
return Status::OK(); return Status::OK();