Compare commits
21 Commits
Author | SHA1 | Date | |
---|---|---|---|
|
ab33322619 | ||
|
ab33b1f711 | ||
|
6f2b441d58 | ||
|
75b968cc97 | ||
|
495c075f75 | ||
|
429f02b8a8 | ||
|
a3c162ca09 | ||
|
d77b69ac21 | ||
|
d7515f81d5 | ||
|
810d2bcc62 | ||
|
c7c2e1968f | ||
|
a11863cb76 | ||
|
98a2fd2f2d | ||
|
149ef19517 | ||
|
8893171698 | ||
|
82e5c842b6 | ||
|
be55065086 | ||
|
159a99f6c3 | ||
|
b573aa8535 | ||
|
3e32a88fe7 | ||
|
56b9e4aae3 |
@ -1,5 +1,10 @@
|
||||
# Rocksdb Change Log
|
||||
|
||||
## 3.10.2 (5/1/2015)
|
||||
* Make sure RocksDB is compiled with fallocate support
|
||||
* Fix possible hang in DB::Write()
|
||||
* Fix a possibility of SIGSEGV in CompactRange()
|
||||
|
||||
## 3.10.0 (3/24/2015)
|
||||
### New Features
|
||||
* GetThreadStatus() is now able to report detailed thread status, including:
|
||||
|
4
Makefile
4
Makefile
@ -138,10 +138,6 @@ install:
|
||||
WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \
|
||||
-Wno-unused-parameter
|
||||
|
||||
ifndef DISABLE_WARNING_AS_ERROR
|
||||
WARNING_FLAGS += -Werror
|
||||
endif
|
||||
|
||||
CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
|
||||
CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers
|
||||
|
||||
|
@ -50,12 +50,7 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
|
||||
FBCODE_BUILD="true"
|
||||
# If we're compiling with TSAN we need pic build
|
||||
PIC_BUILD=$COMPILE_WITH_TSAN
|
||||
if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
|
||||
source "$PWD/build_tools/fbcode_config.sh"
|
||||
else
|
||||
# we need this to build with MySQL. Don't use for other purposes.
|
||||
source "$PWD/build_tools/fbcode_config4.8.1.sh"
|
||||
fi
|
||||
source "$PWD/build_tools/fbcode_config.sh"
|
||||
fi
|
||||
|
||||
# Delete existing output, if it exists
|
||||
@ -177,6 +172,7 @@ else
|
||||
# Test whether fallocate is available
|
||||
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
||||
#include <fcntl.h>
|
||||
#include <linux/falloc.h>
|
||||
int main() {
|
||||
int fd = open("/dev/null", 0);
|
||||
fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
|
||||
|
19
build_tools/dependencies.sh
Normal file
19
build_tools/dependencies.sh
Normal file
@ -0,0 +1,19 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
||||
GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
|
||||
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
|
||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
|
||||
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
|
||||
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
|
||||
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
|
||||
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
|
||||
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
|
||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
|
||||
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
|
||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
|
||||
NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
|
||||
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
|
||||
TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
|
||||
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
|
||||
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
|
||||
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
|
||||
LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
|
@ -6,136 +6,143 @@
|
||||
# Environment variables that change the behavior of this script:
|
||||
# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
|
||||
|
||||
|
||||
BASEDIR=`dirname $BASH_SOURCE`
|
||||
source "$BASEDIR/dependencies.sh"
|
||||
|
||||
CFLAGS=""
|
||||
|
||||
# location of libgcc
|
||||
LIBGCC_BASE="/mnt/gvfs/third-party2/libgcc/0473c80518a10d6efcbe24c5eeca3fb4ec9b519c/4.9.x/gcc-4.9-glibc-2.20/e1a7e4e"
|
||||
LIBGCC_INCLUDE="$LIBGCC_BASE/include"
|
||||
LIBGCC_LIBS=" -L $LIBGCC_BASE/libs"
|
||||
# libgcc
|
||||
LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
|
||||
LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
|
||||
|
||||
# location of glibc
|
||||
GLIBC_REV=7397bed99280af5d9543439cdb7d018af7542720
|
||||
GLIBC_INCLUDE="/mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.20/gcc-4.9-glibc-2.20/99df8fc/include"
|
||||
GLIBC_LIBS=" -L /mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.20/gcc-4.9-glibc-2.20/99df8fc/lib"
|
||||
|
||||
# snappy and zlib depend are bundled with MongoDB so we wan't to pick up the bundled headers when
|
||||
# building for it and disable block compressors supported by RocksDB but not used by MongoDB.
|
||||
|
||||
if [[ -n $ROCKSDB_FOR_MONGO ]]; then
|
||||
|
||||
MONGO_SRC="$ROCKSDB_FOR_MONGO/src/third_party"
|
||||
SNAPPY_INCLUDE=" -I $MONGO_SRC/snappy-1.1.2"
|
||||
CFLAGS+=" -DSNAPPY"
|
||||
ZLIB_INCLUDE=" -I $MONGO_SRC/zlib-1.2.8"
|
||||
CFLAGS+=" -DZLIB"
|
||||
|
||||
else
|
||||
|
||||
SNAPPY_INCLUDE=" -I /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/include/"
|
||||
# glibc
|
||||
GLIBC_INCLUDE="$GLIBC_BASE/include"
|
||||
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
|
||||
|
||||
# snappy
|
||||
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/lib/libsnappy.a"
|
||||
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
|
||||
else
|
||||
SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/lib/libsnappy_pic.a"
|
||||
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
|
||||
fi
|
||||
|
||||
CFLAGS+=" -DSNAPPY"
|
||||
|
||||
if test -z $PIC_BUILD; then
|
||||
# location of zlib headers and libraries
|
||||
ZLIB_INCLUDE=" -I /mnt/gvfs/third-party2/zlib/feb983d9667f4cf5e9da07ce75abc824764b67a1/1.2.8/gcc-4.9-glibc-2.20/4230243/include/"
|
||||
ZLIB_LIBS=" /mnt/gvfs/third-party2/zlib/feb983d9667f4cf5e9da07ce75abc824764b67a1/1.2.8/gcc-4.9-glibc-2.20/4230243/lib/libz.a"
|
||||
ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
|
||||
ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
|
||||
CFLAGS+=" -DZLIB"
|
||||
|
||||
# location of bzip headers and libraries
|
||||
BZIP_INCLUDE=" -I /mnt/gvfs/third-party2/bzip2/af004cceebb2dfd173ca29933ea5915e727aad2f/1.0.6/gcc-4.9-glibc-2.20/4230243/include/"
|
||||
BZIP_LIBS=" /mnt/gvfs/third-party2/bzip2/af004cceebb2dfd173ca29933ea5915e727aad2f/1.0.6/gcc-4.9-glibc-2.20/4230243/lib/libbz2.a"
|
||||
BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
|
||||
BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
|
||||
CFLAGS+=" -DBZIP2"
|
||||
|
||||
LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/79d2943e2dd7208a3e0b06cf95e9f85f05fe9e1b/r124/gcc-4.9-glibc-2.20/4230243/include/"
|
||||
LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/79d2943e2dd7208a3e0b06cf95e9f85f05fe9e1b/r124/gcc-4.9-glibc-2.20/4230243/lib/liblz4.a"
|
||||
LZ4_INCLUDE=" -I $LZ4_BASE/include/"
|
||||
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
|
||||
CFLAGS+=" -DLZ4"
|
||||
fi
|
||||
|
||||
ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
|
||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
|
||||
CFLAGS+=" -DZSTD"
|
||||
fi
|
||||
|
||||
# location of gflags headers and libraries
|
||||
GFLAGS_INCLUDE=" -I /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/include/"
|
||||
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/lib/libgflags.a"
|
||||
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
|
||||
else
|
||||
GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/lib/libgflags_pic.a"
|
||||
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
|
||||
fi
|
||||
CFLAGS+=" -DGFLAGS=google"
|
||||
CFLAGS+=" -DGFLAGS=gflags"
|
||||
|
||||
# location of jemalloc
|
||||
JEMALLOC_INCLUDE=" -I /mnt/gvfs/third-party2/jemalloc/bcd68e5e419efa4e61b9486d6854564d6d75a0b5/3.6.0/gcc-4.9-glibc-2.20/2aafc78/include/"
|
||||
JEMALLOC_LIB=" -Wl,--whole-archive /mnt/gvfs/third-party2/jemalloc/bcd68e5e419efa4e61b9486d6854564d6d75a0b5/3.6.0/gcc-4.9-glibc-2.20/2aafc78/lib/libjemalloc.a"
|
||||
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
|
||||
JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
|
||||
|
||||
if test -z $PIC_BUILD; then
|
||||
# location of numa
|
||||
NUMA_INCLUDE=" -I /mnt/gvfs/third-party2/numa/bbefc39ecbf31d0ca184168eb613ef8d397790ee/2.0.8/gcc-4.9-glibc-2.20/4230243/include/"
|
||||
NUMA_LIB=" /mnt/gvfs/third-party2/numa/bbefc39ecbf31d0ca184168eb613ef8d397790ee/2.0.8/gcc-4.9-glibc-2.20/4230243/lib/libnuma.a"
|
||||
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
|
||||
NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
|
||||
CFLAGS+=" -DNUMA"
|
||||
|
||||
# location of libunwind
|
||||
LIBUNWIND="/mnt/gvfs/third-party2/libunwind/1de3b75e0afedfe5585b231bbb340ec7a1542335/1.1/gcc-4.9-glibc-2.20/34235e8/lib/libunwind.a"
|
||||
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
|
||||
fi
|
||||
|
||||
# use Intel SSE support for checksum calculations
|
||||
export USE_SSE=1
|
||||
# location of TBB
|
||||
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
TBB_LIBS="$TBB_BASE/lib/libtbb.a"
|
||||
else
|
||||
TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
|
||||
fi
|
||||
CFLAGS+=" -DTBB"
|
||||
|
||||
BINUTILS="/mnt/gvfs/third-party2/binutils/0b6ad0c88ddd903333a48ae8bff134efac468e4a/2.25/centos6-native/da39a3e/bin"
|
||||
# use Intel SSE support for checksum calculations
|
||||
export USE_SSE=" -msse -msse4.2 "
|
||||
|
||||
BINUTILS="$BINUTILS_BASE/bin"
|
||||
AR="$BINUTILS/ar"
|
||||
|
||||
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
|
||||
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
|
||||
|
||||
GCC_BASE="/mnt/gvfs/third-party2/gcc/1c67a0b88f64d4d9ced0382d141c76aaa7d62fba/4.9.x/centos6-native/1317bc4"
|
||||
STDLIBS="-L $GCC_BASE/lib64"
|
||||
|
||||
CLANG_BASE="/mnt/gvfs/third-party2/clang/d81444dd214df3d2466734de45bb264a0486acc3/dev"
|
||||
CLANG_BIN="$CLANG_BASE/centos6-native/af4b1a0/bin"
|
||||
CLANG_BIN="$CLANG_BASE/bin"
|
||||
CLANG_LIB="$CLANG_BASE/lib"
|
||||
CLANG_SRC="$CLANG_BASE/../../src"
|
||||
|
||||
CLANG_ANALYZER="$CLANG_BIN/clang++"
|
||||
CLANG_SCAN_BUILD="$CLANG_BASE/src/clang/tools/scan-build/scan-build"
|
||||
CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
|
||||
|
||||
if [ -z "$USE_CLANG" ]; then
|
||||
# gcc
|
||||
CC="$GCC_BASE/bin/gcc"
|
||||
CXX="$GCC_BASE/bin/g++"
|
||||
|
||||
|
||||
CFLAGS+=" -B$BINUTILS/gold"
|
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
||||
JEMALLOC=1
|
||||
else
|
||||
# clang
|
||||
CLANG_INCLUDE="$CLANG_BASE/gcc-4.9-glibc-2.20/74c386f/lib/clang/dev/include/"
|
||||
# clang
|
||||
CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
|
||||
CC="$CLANG_BIN/clang"
|
||||
CXX="$CLANG_BIN/clang++"
|
||||
|
||||
KERNEL_HEADERS_INCLUDE="/mnt/gvfs/third-party2/kernel-headers/ffd14f660a43c4b92717986b1bba66722ef089d0/3.2.18_70_fbk11_00129_gc8882d0/gcc-4.9-glibc-2.20/da39a3e/include"
|
||||
KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
|
||||
|
||||
CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x "
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x/x86_64-facebook-linux "
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
|
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
||||
CFLAGS+=" -isystem $CLANG_INCLUDE"
|
||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
|
||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
|
||||
CFLAGS+=" -Wno-expansion-to-defined "
|
||||
CXXFLAGS="-nostdinc++"
|
||||
fi
|
||||
|
||||
CFLAGS+=" $DEPS_INCLUDE"
|
||||
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_FALLOCATE_PRESENT"
|
||||
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
|
||||
CXXFLAGS+=" $CFLAGS"
|
||||
|
||||
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS $NUMA_LIB"
|
||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.9-glibc-2.20/lib/ld.so"
|
||||
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $LIBUNWIND"
|
||||
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
|
||||
EXEC_LDFLAGS+=" -B$BINUTILS/gold"
|
||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
|
||||
EXEC_LDFLAGS+=" $LIBUNWIND"
|
||||
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
|
||||
# required by libtbb
|
||||
EXEC_LDFLAGS+=" -ldl"
|
||||
|
||||
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
|
||||
|
||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS"
|
||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
|
||||
|
||||
VALGRIND_VER="/mnt/gvfs/third-party2/valgrind/6c45ef049cbf11c2df593addb712cd891049e737/3.10.0/gcc-4.9-glibc-2.20/4230243/bin/"
|
||||
VALGRIND_VER="$VALGRIND_BASE/bin/"
|
||||
|
||||
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD
|
||||
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD
|
||||
|
@ -416,6 +416,7 @@ void DBImpl::MaybeDumpStats() {
|
||||
// period in rare cases.
|
||||
last_stats_dump_time_microsec_ = now_micros;
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
bool tmp1 = false;
|
||||
bool tmp2 = false;
|
||||
DBPropertyType cf_property_type =
|
||||
@ -436,6 +437,7 @@ void DBImpl::MaybeDumpStats() {
|
||||
db_options_.info_log, "------- DUMPING STATS -------");
|
||||
Log(InfoLogLevel::INFO_LEVEL,
|
||||
db_options_.info_log, "%s", stats.c_str());
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
PrintStatistics();
|
||||
}
|
||||
@ -1241,7 +1243,8 @@ Status DBImpl::CompactRange(ColumnFamilyHandle* column_family,
|
||||
{
|
||||
InstrumentedMutexLock l(&mutex_);
|
||||
Version* base = cfd->current();
|
||||
for (int level = 1; level < cfd->NumberLevels(); level++) {
|
||||
for (int level = 1; level < base->storage_info()->num_non_empty_levels();
|
||||
level++) {
|
||||
if (base->storage_info()->OverlapInLevel(level, begin, end)) {
|
||||
max_level_with_files = level;
|
||||
}
|
||||
@ -2242,7 +2245,7 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress, JobContext* job_context,
|
||||
assert(c->num_input_files(0) == 1);
|
||||
FileMetaData* f = c->input(0, 0);
|
||||
c->edit()->DeleteFile(c->level(), f->fd.GetNumber());
|
||||
c->edit()->AddFile(c->level() + 1, f->fd.GetNumber(), f->fd.GetPathId(),
|
||||
c->edit()->AddFile(c->output_level(), f->fd.GetNumber(), f->fd.GetPathId(),
|
||||
f->fd.GetFileSize(), f->smallest, f->largest,
|
||||
f->smallest_seqno, f->largest_seqno);
|
||||
status = versions_->LogAndApply(c->column_family_data(),
|
||||
@ -3128,6 +3131,11 @@ Status DBImpl::Write(const WriteOptions& write_options, WriteBatch* my_batch) {
|
||||
|
||||
if (UNLIKELY(status.ok()) &&
|
||||
(write_controller_.IsStopped() || write_controller_.GetDelay() > 0)) {
|
||||
// If writer is stopped, we need to get it going,
|
||||
// so schedule flushes/compactions
|
||||
if (context.schedule_bg_work_) {
|
||||
MaybeScheduleFlushOrCompaction();
|
||||
}
|
||||
status = DelayWrite(expiration_time);
|
||||
}
|
||||
|
||||
|
@ -11952,6 +11952,68 @@ TEST_F(DBTest, FilterCompactionTimeTest) {
|
||||
delete itr;
|
||||
}
|
||||
|
||||
TEST_F(DBTest, EmptyCompactedDB) {
|
||||
Options options;
|
||||
options.max_open_files = -1;
|
||||
options = CurrentOptions(options);
|
||||
Close();
|
||||
ASSERT_OK(ReadOnlyReopen(options));
|
||||
Status s = Put("new", "value");
|
||||
ASSERT_TRUE(s.IsNotSupported());
|
||||
Close();
|
||||
}
|
||||
|
||||
// Github issue #596
|
||||
TEST_F(DBTest, HugeNumberOfLevels) {
|
||||
Options options = CurrentOptions();
|
||||
options.write_buffer_size = 2 * 1024 * 1024; // 2MB
|
||||
options.max_bytes_for_level_base = 2 * 1024 * 1024; // 2MB
|
||||
options.num_levels = 12;
|
||||
options.max_background_compactions = 10;
|
||||
options.max_bytes_for_level_multiplier = 2;
|
||||
options.level_compaction_dynamic_level_bytes = true;
|
||||
DestroyAndReopen(options);
|
||||
|
||||
Random rnd(301);
|
||||
for (int i = 0; i < 300000; ++i) {
|
||||
ASSERT_OK(Put(Key(i), RandomString(&rnd, 1024)));
|
||||
}
|
||||
|
||||
ASSERT_OK(db_->CompactRange(nullptr, nullptr));
|
||||
}
|
||||
|
||||
// Github issue #595
|
||||
// Large write batch with column families
|
||||
TEST_F(DBTest, LargeBatchWithColumnFamilies) {
|
||||
Options options;
|
||||
options.env = env_;
|
||||
options = CurrentOptions(options);
|
||||
options.write_buffer_size = 100000; // Small write buffer
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
int64_t j = 0;
|
||||
for (int i = 0; i < 5; i++) {
|
||||
for (int pass = 1; pass <= 3; pass++) {
|
||||
WriteBatch batch;
|
||||
size_t write_size = 1024 * 1024 * (5 + i);
|
||||
fprintf(stderr, "prepare: %ld MB, pass:%d\n", (write_size / 1024 / 1024),
|
||||
pass);
|
||||
for (;;) {
|
||||
std::string data(3000, j++ % 127 + 20);
|
||||
data += std::to_string(j);
|
||||
batch.Put(handles_[0], Slice(data), Slice(data));
|
||||
if (batch.GetDataSize() > write_size) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
fprintf(stderr, "write: %ld MB\n", (batch.GetDataSize() / 1024 / 1024));
|
||||
ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
|
||||
fprintf(stderr, "done\n");
|
||||
}
|
||||
}
|
||||
// make sure we can re-open it.
|
||||
ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options));
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
@ -20,6 +20,9 @@ FileIndexer::FileIndexer(const Comparator* ucmp)
|
||||
size_t FileIndexer::NumLevelIndex() const { return next_level_index_.size(); }
|
||||
|
||||
size_t FileIndexer::LevelIndexSize(size_t level) const {
|
||||
if (level >= next_level_index_.size()) {
|
||||
return 0;
|
||||
}
|
||||
return next_level_index_[level].num_index;
|
||||
}
|
||||
|
||||
|
@ -11,6 +11,7 @@
|
||||
#include "db/file_indexer.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/version_edit.h"
|
||||
#include "port/stack_trace.h"
|
||||
#include "rocksdb/comparator.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
@ -343,6 +344,7 @@ TEST_F(FileIndexerTest, mixed) {
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
rocksdb::port::InstallStackTraceHandler();
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
|
@ -366,7 +366,7 @@ class InternalStats {
|
||||
|
||||
void IncBytesMoved(int level, uint64_t amount) {}
|
||||
|
||||
void RecordLevelNSlowdown(int level, uint64_t micros, bool soft) {}
|
||||
void RecordLevelNSlowdown(int level, bool soft) {}
|
||||
|
||||
void AddCFStats(InternalCFStatsType type, uint64_t value) {}
|
||||
|
||||
|
@ -81,4 +81,4 @@ class ManagedIterator : public Iterator {
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
#endif // ROCKSDB_LITE
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
@ -132,8 +132,6 @@ DEFINE_int64(seed, 0,
|
||||
"Seed base for random number generators. "
|
||||
"When 0 it is deterministic.");
|
||||
|
||||
static rocksdb::Env* FLAGS_env = rocksdb::Env::Default();
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
namespace {
|
||||
|
@ -1184,6 +1184,10 @@ bool Version::Unref() {
|
||||
bool VersionStorageInfo::OverlapInLevel(int level,
|
||||
const Slice* smallest_user_key,
|
||||
const Slice* largest_user_key) {
|
||||
if (level >= num_non_empty_levels_) {
|
||||
// empty level, no overlap
|
||||
return false;
|
||||
}
|
||||
return SomeFileOverlapsRange(*internal_comparator_, (level > 0),
|
||||
level_files_brief_[level], smallest_user_key,
|
||||
largest_user_key);
|
||||
@ -1227,6 +1231,11 @@ int VersionStorageInfo::PickLevelForMemTableOutput(
|
||||
void VersionStorageInfo::GetOverlappingInputs(
|
||||
int level, const InternalKey* begin, const InternalKey* end,
|
||||
std::vector<FileMetaData*>* inputs, int hint_index, int* file_index) {
|
||||
if (level >= num_non_empty_levels_) {
|
||||
// this level is empty, no overlapping inputs
|
||||
return;
|
||||
}
|
||||
|
||||
inputs->clear();
|
||||
Slice user_begin, user_end;
|
||||
if (begin != nullptr) {
|
||||
@ -1571,12 +1580,12 @@ void VersionStorageInfo::CalculateBaseBytes(const ImmutableCFOptions& ioptions,
|
||||
}
|
||||
|
||||
// Calculate base level and its size.
|
||||
int base_level_size;
|
||||
uint64_t base_level_size;
|
||||
if (cur_level_size <= base_bytes_min) {
|
||||
// Case 1. If we make target size of last level to be max_level_size,
|
||||
// target size of the first non-empty level would be smaller than
|
||||
// base_bytes_min. We set it be base_bytes_min.
|
||||
base_level_size = static_cast<int>(base_bytes_min + 1);
|
||||
base_level_size = base_bytes_min + 1U;
|
||||
base_level_ = first_non_empty_level;
|
||||
Warn(ioptions.info_log,
|
||||
"More existing levels in DB than needed. "
|
||||
@ -1592,16 +1601,17 @@ void VersionStorageInfo::CalculateBaseBytes(const ImmutableCFOptions& ioptions,
|
||||
if (cur_level_size > base_bytes_max) {
|
||||
// Even L1 will be too large
|
||||
assert(base_level_ == 1);
|
||||
base_level_size = static_cast<int>(base_bytes_max);
|
||||
base_level_size = base_bytes_max;
|
||||
} else {
|
||||
base_level_size = static_cast<int>(cur_level_size);
|
||||
base_level_size = cur_level_size;
|
||||
}
|
||||
}
|
||||
|
||||
int level_size = base_level_size;
|
||||
uint64_t level_size = base_level_size;
|
||||
for (int i = base_level_; i < num_levels_; i++) {
|
||||
if (i > base_level_) {
|
||||
level_size = level_size * options.max_bytes_for_level_multiplier;
|
||||
level_size = MultiplyCheckOverflow(
|
||||
level_size, options.max_bytes_for_level_multiplier);
|
||||
}
|
||||
level_max_bytes_[i] = level_size;
|
||||
}
|
||||
|
@ -215,6 +215,25 @@ TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicLotsOfData) {
|
||||
ASSERT_EQ(0, logger_->log_count);
|
||||
}
|
||||
|
||||
TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicLargeLevel) {
|
||||
uint64_t kOneGB = 1000U * 1000U * 1000U;
|
||||
ioptions_.level_compaction_dynamic_level_bytes = true;
|
||||
mutable_cf_options_.max_bytes_for_level_base = 10U * kOneGB;
|
||||
mutable_cf_options_.max_bytes_for_level_multiplier = 10;
|
||||
Add(0, 1U, "1", "2", 50U);
|
||||
Add(3, 4U, "1", "2", 32U * kOneGB);
|
||||
Add(4, 5U, "1", "2", 500U * kOneGB);
|
||||
Add(5, 6U, "1", "2", 3000U * kOneGB);
|
||||
|
||||
vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
|
||||
ASSERT_EQ(vstorage_.MaxBytesForLevel(5), 3000U * kOneGB);
|
||||
ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 300U * kOneGB);
|
||||
ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 30U * kOneGB);
|
||||
ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 3U * kOneGB);
|
||||
ASSERT_EQ(vstorage_.base_level(), 2);
|
||||
ASSERT_EQ(0, logger_->log_count);
|
||||
}
|
||||
|
||||
class FindLevelFileTest : public testing::Test {
|
||||
public:
|
||||
LevelFilesBrief file_level_;
|
||||
|
@ -6,7 +6,7 @@
|
||||
|
||||
#define ROCKSDB_MAJOR 3
|
||||
#define ROCKSDB_MINOR 10
|
||||
#define ROCKSDB_PATCH 0
|
||||
#define ROCKSDB_PATCH 2
|
||||
|
||||
// Do not use these. We made the mistake of declaring macros starting with
|
||||
// double underscore. Now we have to live with our choice. We'll deprecate these
|
||||
|
@ -6,6 +6,7 @@
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
#include <set>
|
||||
#include <memory>
|
||||
#include <atomic>
|
||||
|
@ -190,16 +190,16 @@ class autovector {
|
||||
|
||||
bool empty() const { return size() == 0; }
|
||||
|
||||
// will not check boundry
|
||||
const_reference operator[](size_type n) const {
|
||||
assert(n < size());
|
||||
return n < kSize ? values_[n] : vect_[n - kSize];
|
||||
}
|
||||
|
||||
reference operator[](size_type n) {
|
||||
assert(n < size());
|
||||
return n < kSize ? values_[n] : vect_[n - kSize];
|
||||
}
|
||||
|
||||
// will check boundry
|
||||
const_reference at(size_type n) const {
|
||||
assert(n < size());
|
||||
return (*this)[n];
|
||||
|
@ -611,6 +611,8 @@ DBOptions* DBOptions::IncreaseParallelism(int total_threads) {
|
||||
return this;
|
||||
}
|
||||
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
ReadOptions::ReadOptions()
|
||||
: verify_checksums(true),
|
||||
fill_cache(true),
|
||||
@ -637,6 +639,4 @@ ReadOptions::ReadOptions(bool cksum, bool cache)
|
||||
reinterpret_cast<ReadOptions*>(this));
|
||||
}
|
||||
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
} // namespace rocksdb
|
||||
|
@ -5,6 +5,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <condition_variable>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
|
@ -10,6 +10,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
@ -20,10 +20,11 @@ void ThreadStatusUtil::TEST_SetStateDelay(
|
||||
states_delay[state].store(micro, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
void ThreadStatusUtil::TEST_StateDelay(
|
||||
const ThreadStatus::StateType state) {
|
||||
Env::Default()->SleepForMicroseconds(
|
||||
states_delay[state].load(std::memory_order_relaxed));
|
||||
void ThreadStatusUtil::TEST_StateDelay(const ThreadStatus::StateType state) {
|
||||
auto delay = states_delay[state].load(std::memory_order_relaxed);
|
||||
if (delay > 0) {
|
||||
Env::Default()->SleepForMicroseconds(delay);
|
||||
}
|
||||
}
|
||||
|
||||
#endif // !NDEBUG
|
||||
|
@ -3,13 +3,13 @@
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#ifdef XFUNC
|
||||
#include <string>
|
||||
#include "db/db_impl.h"
|
||||
#include "db/managed_iterator.h"
|
||||
#include "rocksdb/options.h"
|
||||
#include "util/xfunc.h"
|
||||
|
||||
#ifdef XFUNC
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
|
@ -17,13 +17,15 @@ namespace rocksdb {
|
||||
* with XFUNC only being set for debug builds.
|
||||
*/
|
||||
#if defined(ROCKSDB_XFTEST_FORCE)
|
||||
#ifndef ROCKSDB_LITE
|
||||
#if (ROCKSDB_XFTEST_FORCE == 1)
|
||||
#define XFUNC
|
||||
#endif
|
||||
#endif // ROCKSDB_XFTEST_FORCE == 1
|
||||
#elif defined(NDEBUG)
|
||||
#else
|
||||
#define XFUNC
|
||||
#endif
|
||||
#endif // defined(ROCKSDB_XFTEST_FORCE)
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
#ifndef XFUNC
|
||||
#define XFUNC_TEST(condition, location, lfname, fname, ...)
|
||||
|
@ -107,6 +107,9 @@ Status CompactedDBImpl::Init(const Options& options) {
|
||||
version_ = cfd_->GetSuperVersion()->current;
|
||||
user_comparator_ = cfd_->user_comparator();
|
||||
auto* vstorage = version_->storage_info();
|
||||
if (vstorage->num_non_empty_levels() == 0) {
|
||||
return Status::NotSupported("no file exists");
|
||||
}
|
||||
const LevelFilesBrief& l0 = vstorage->LevelFilesBrief(0);
|
||||
// L0 should not have files
|
||||
if (l0.num_files > 1) {
|
||||
|
Loading…
Reference in New Issue
Block a user