Compare commits
24 Commits
Author | SHA1 | Date | |
---|---|---|---|
|
1ad2267284 | ||
|
6d353bc011 | ||
|
622456584d | ||
|
4887a3f680 | ||
|
bcf926b664 | ||
|
e0ff529d5d | ||
|
bcaf392114 | ||
|
b8c78da6ed | ||
|
5dac6ba2dc | ||
|
61d6cab888 | ||
|
26a2fb1cea | ||
|
41eb08a960 | ||
|
e7708343f3 | ||
|
dbffb828fa | ||
|
2ed9b9206b | ||
|
4860bfdeab | ||
|
74bcf18772 | ||
|
18a4f6a1de | ||
|
f300da40d0 | ||
|
b8f656767a | ||
|
fab086daa1 | ||
|
f9a3440b76 | ||
|
3bbf0e1e4f | ||
|
0be02649d8 |
@ -1,5 +1,7 @@
|
||||
# Rocksdb Change Log
|
||||
|
||||
## 4.4.1 (2/18/2016)
|
||||
|
||||
## 4.4.0 (1/14/2016)
|
||||
### Public API Changes
|
||||
* Change names in CompactionPri and add a new one.
|
||||
|
4
Makefile
4
Makefile
@ -187,10 +187,6 @@ default: all
|
||||
WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \
|
||||
-Wno-unused-parameter
|
||||
|
||||
ifndef DISABLE_WARNING_AS_ERROR
|
||||
WARNING_FLAGS += -Werror
|
||||
endif
|
||||
|
||||
CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
|
||||
CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers
|
||||
|
||||
|
@ -52,12 +52,7 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
|
||||
FBCODE_BUILD="true"
|
||||
# If we're compiling with TSAN we need pic build
|
||||
PIC_BUILD=$COMPILE_WITH_TSAN
|
||||
if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
|
||||
source "$PWD/build_tools/fbcode_config.sh"
|
||||
else
|
||||
# we need this to build with MySQL. Don't use for other purposes.
|
||||
source "$PWD/build_tools/fbcode_config4.8.1.sh"
|
||||
fi
|
||||
source "$PWD/build_tools/fbcode_config.sh"
|
||||
fi
|
||||
|
||||
# Delete existing output, if it exists
|
||||
|
19
build_tools/dependencies.sh
Normal file
19
build_tools/dependencies.sh
Normal file
@ -0,0 +1,19 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
||||
GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
|
||||
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
|
||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
|
||||
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
|
||||
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
|
||||
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
|
||||
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
|
||||
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
|
||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
|
||||
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
|
||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
|
||||
NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
|
||||
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
|
||||
TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
|
||||
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
|
||||
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
|
||||
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
|
||||
LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
|
16
build_tools/dependencies_4.8.1.sh
Normal file
16
build_tools/dependencies_4.8.1.sh
Normal file
@ -0,0 +1,16 @@
|
||||
GCC_BASE=/mnt/gvfs/third-party2/gcc/ebc96bc2fb751b5a0300b8d91a95bdf24ac1d88b/4.8.1/centos6-native/cc6c9dc
|
||||
CLANG_BASE=/mnt/vol/engshare/fbcode/third-party2/clang/3.7.1/centos6-native/9d9ecb9/
|
||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/ea2fd1278810d3af2ea52218d2767e09d786dbd0/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
|
||||
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/f5484f168c0e4d19823d41df052c5870c6e575a4/2.17/gcc-4.8.1-glibc-2.17/99df8fc
|
||||
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/cbf6f1f209e5bd160bdc5d971744e039f36b1566/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/6d39cb54708049f527e713ad19f2aadb9d3667e8/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/2ddd45f0853bfc8bb1c27f0f447236a1a26c338a/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
LZ4_BASE=/mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9c833012dea9ae6de1df2cbdcb7aebc57ba2cb39/0.5.0/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/c7275a4ceae0aca0929e56964a31dafc53c1ee96/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/378c8f95cf28c833e121ca73432101e451c3e62a/4.0.3/gcc-4.8.1-glibc-2.17/8d31e51
|
||||
NUMA_BASE=/mnt/gvfs/third-party2/numa/ae54a5ed22cdabb1c6446dce4e8ffae5b4446d73/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/303048f72efc92ae079e62dfc84823401aecfd94/trunk/gcc-4.8.1-glibc-2.17/675d945
|
||||
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/1a48835975c66d30e47770ec419758ed3b9ba010/3.10.62-62_fbk17_03959_ge29cc63/gcc-4.8.1-glibc-2.17/da39a3e
|
||||
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/a5b8152b2a15ce8a98808cf954fbccec825a97bc/2.25/centos6-native/da39a3e
|
||||
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/af85c56f424cd5edfc2c97588299b44ecdec96bb/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a
|
@ -6,87 +6,101 @@
|
||||
# Environment variables that change the behavior of this script:
|
||||
# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
|
||||
|
||||
|
||||
BASEDIR=`dirname $BASH_SOURCE`
|
||||
source "$BASEDIR/dependencies.sh"
|
||||
|
||||
CFLAGS=""
|
||||
|
||||
# location of libgcc
|
||||
LIBGCC_BASE="/mnt/gvfs/third-party2/libgcc/0473c80518a10d6efcbe24c5eeca3fb4ec9b519c/4.9.x/gcc-4.9-glibc-2.20/e1a7e4e"
|
||||
LIBGCC_INCLUDE="$LIBGCC_BASE/include"
|
||||
LIBGCC_LIBS=" -L $LIBGCC_BASE/libs"
|
||||
# libgcc
|
||||
LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
|
||||
LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
|
||||
|
||||
# location of glibc
|
||||
GLIBC_REV=7397bed99280af5d9543439cdb7d018af7542720
|
||||
GLIBC_INCLUDE="/mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.20/gcc-4.9-glibc-2.20/99df8fc/include"
|
||||
GLIBC_LIBS=" -L /mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.20/gcc-4.9-glibc-2.20/99df8fc/lib"
|
||||
|
||||
SNAPPY_INCLUDE=" -I /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/include/"
|
||||
# glibc
|
||||
GLIBC_INCLUDE="$GLIBC_BASE/include"
|
||||
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
|
||||
|
||||
# snappy
|
||||
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/lib/libsnappy.a"
|
||||
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
|
||||
else
|
||||
SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/lib/libsnappy_pic.a"
|
||||
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
|
||||
fi
|
||||
|
||||
CFLAGS+=" -DSNAPPY"
|
||||
|
||||
if test -z $PIC_BUILD; then
|
||||
# location of zlib headers and libraries
|
||||
ZLIB_INCLUDE=" -I /mnt/gvfs/third-party2/zlib/feb983d9667f4cf5e9da07ce75abc824764b67a1/1.2.8/gcc-4.9-glibc-2.20/4230243/include/"
|
||||
ZLIB_LIBS=" /mnt/gvfs/third-party2/zlib/feb983d9667f4cf5e9da07ce75abc824764b67a1/1.2.8/gcc-4.9-glibc-2.20/4230243/lib/libz.a"
|
||||
ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
|
||||
ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
|
||||
CFLAGS+=" -DZLIB"
|
||||
|
||||
# location of bzip headers and libraries
|
||||
BZIP_INCLUDE=" -I /mnt/gvfs/third-party2/bzip2/af004cceebb2dfd173ca29933ea5915e727aad2f/1.0.6/gcc-4.9-glibc-2.20/4230243/include/"
|
||||
BZIP_LIBS=" /mnt/gvfs/third-party2/bzip2/af004cceebb2dfd173ca29933ea5915e727aad2f/1.0.6/gcc-4.9-glibc-2.20/4230243/lib/libbz2.a"
|
||||
BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
|
||||
BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
|
||||
CFLAGS+=" -DBZIP2"
|
||||
|
||||
LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.9-glibc-2.20/e9936bf/include/"
|
||||
LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.9-glibc-2.20/e9936bf/lib/liblz4.a"
|
||||
LZ4_INCLUDE=" -I $LZ4_BASE/include/"
|
||||
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
|
||||
CFLAGS+=" -DLZ4"
|
||||
|
||||
ZSTD_REV=810b81b4705def5243e998b54701f3c504e4009e
|
||||
ZSTD_INCLUDE=" -I /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.4.2/gcc-4.8.1-glibc-2.17/c3f970a/include"
|
||||
ZSTD_LIBS=" /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.4.2/gcc-4.8.1-glibc-2.17/c3f970a/lib/libzstd.a"
|
||||
CFLAGS+=" -DZSTD"
|
||||
fi
|
||||
|
||||
ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
|
||||
else
|
||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
|
||||
fi
|
||||
CFLAGS+=" -DZSTD"
|
||||
|
||||
# location of gflags headers and libraries
|
||||
GFLAGS_INCLUDE=" -I /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/include/"
|
||||
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/lib/libgflags.a"
|
||||
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
|
||||
else
|
||||
GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/lib/libgflags_pic.a"
|
||||
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
|
||||
fi
|
||||
CFLAGS+=" -DGFLAGS=google"
|
||||
CFLAGS+=" -DGFLAGS=gflags"
|
||||
|
||||
# location of jemalloc
|
||||
JEMALLOC_INCLUDE=" -I /mnt/gvfs/third-party2/jemalloc/bcd68e5e419efa4e61b9486d6854564d6d75a0b5/3.6.0/gcc-4.9-glibc-2.20/2aafc78/include/"
|
||||
JEMALLOC_LIB=" /mnt/gvfs/third-party2/jemalloc/bcd68e5e419efa4e61b9486d6854564d6d75a0b5/3.6.0/gcc-4.9-glibc-2.20/2aafc78/lib/libjemalloc.a"
|
||||
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
|
||||
JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
|
||||
|
||||
if test -z $PIC_BUILD; then
|
||||
# location of numa
|
||||
NUMA_INCLUDE=" -I /mnt/gvfs/third-party2/numa/bbefc39ecbf31d0ca184168eb613ef8d397790ee/2.0.8/gcc-4.9-glibc-2.20/4230243/include/"
|
||||
NUMA_LIB=" /mnt/gvfs/third-party2/numa/bbefc39ecbf31d0ca184168eb613ef8d397790ee/2.0.8/gcc-4.9-glibc-2.20/4230243/lib/libnuma.a"
|
||||
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
|
||||
NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
|
||||
CFLAGS+=" -DNUMA"
|
||||
|
||||
# location of libunwind
|
||||
LIBUNWIND="/mnt/gvfs/third-party2/libunwind/1de3b75e0afedfe5585b231bbb340ec7a1542335/1.1/gcc-4.9-glibc-2.20/34235e8/lib/libunwind.a"
|
||||
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
|
||||
fi
|
||||
|
||||
# location of TBB
|
||||
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
TBB_LIBS="$TBB_BASE/lib/libtbb.a"
|
||||
else
|
||||
TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
|
||||
fi
|
||||
CFLAGS+=" -DTBB"
|
||||
|
||||
# use Intel SSE support for checksum calculations
|
||||
export USE_SSE=1
|
||||
|
||||
BINUTILS="/mnt/gvfs/third-party2/binutils/0b6ad0c88ddd903333a48ae8bff134efac468e4a/2.25/centos6-native/da39a3e/bin"
|
||||
BINUTILS="$BINUTILS_BASE/bin"
|
||||
AR="$BINUTILS/ar"
|
||||
|
||||
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
|
||||
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
|
||||
|
||||
GCC_BASE="/mnt/gvfs/third-party2/gcc/1c67a0b88f64d4d9ced0382d141c76aaa7d62fba/4.9.x/centos6-native/1317bc4"
|
||||
STDLIBS="-L $GCC_BASE/lib64"
|
||||
|
||||
CLANG_BASE="/mnt/gvfs/third-party2/clang/d81444dd214df3d2466734de45bb264a0486acc3/dev"
|
||||
CLANG_BIN="$CLANG_BASE/centos6-native/af4b1a0/bin"
|
||||
CLANG_BIN="$CLANG_BASE/bin"
|
||||
CLANG_LIB="$CLANG_BASE/lib"
|
||||
CLANG_SRC="$CLANG_BASE/../../src"
|
||||
|
||||
CLANG_ANALYZER="$CLANG_BIN/clang++"
|
||||
CLANG_SCAN_BUILD="$CLANG_BASE/src/clang/tools/scan-build/scan-build"
|
||||
CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
|
||||
|
||||
if [ -z "$USE_CLANG" ]; then
|
||||
# gcc
|
||||
@ -94,40 +108,45 @@ if [ -z "$USE_CLANG" ]; then
|
||||
CXX="$GCC_BASE/bin/g++"
|
||||
|
||||
CFLAGS+=" -B$BINUTILS/gold"
|
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
||||
JEMALLOC=1
|
||||
else
|
||||
# clang
|
||||
CLANG_INCLUDE="$CLANG_BASE/gcc-4.9-glibc-2.20/74c386f/lib/clang/dev/include/"
|
||||
CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
|
||||
CC="$CLANG_BIN/clang"
|
||||
CXX="$CLANG_BIN/clang++"
|
||||
|
||||
KERNEL_HEADERS_INCLUDE="/mnt/gvfs/third-party2/kernel-headers/ffd14f660a43c4b92717986b1bba66722ef089d0/3.2.18_70_fbk11_00129_gc8882d0/gcc-4.9-glibc-2.20/da39a3e/include"
|
||||
KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
|
||||
|
||||
CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x "
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x/x86_64-facebook-linux "
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
|
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
||||
CFLAGS+=" -isystem $CLANG_INCLUDE"
|
||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
|
||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
|
||||
CFLAGS+=" -Wno-expansion-to-defined "
|
||||
CXXFLAGS="-nostdinc++"
|
||||
fi
|
||||
|
||||
CFLAGS+=" $DEPS_INCLUDE"
|
||||
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE"
|
||||
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
|
||||
CXXFLAGS+=" $CFLAGS"
|
||||
|
||||
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB"
|
||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.9-glibc-2.20/lib/ld.so"
|
||||
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
|
||||
EXEC_LDFLAGS+=" -B$BINUTILS/gold"
|
||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
|
||||
EXEC_LDFLAGS+=" $LIBUNWIND"
|
||||
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.9-glibc-2.20/lib"
|
||||
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
|
||||
# required by libtbb
|
||||
EXEC_LDFLAGS+=" -ldl"
|
||||
|
||||
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
|
||||
|
||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
|
||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
|
||||
|
||||
VALGRIND_VER="/mnt/gvfs/third-party2/valgrind/6c45ef049cbf11c2df593addb712cd891049e737/3.10.0/gcc-4.9-glibc-2.20/4230243/bin/"
|
||||
VALGRIND_VER="$VALGRIND_BASE/bin/"
|
||||
|
||||
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD
|
||||
|
@ -4,67 +4,58 @@
|
||||
# fbcode settings. It uses the latest g++ compiler and also
|
||||
# uses jemalloc
|
||||
|
||||
BASEDIR=`dirname $BASH_SOURCE`
|
||||
source "$BASEDIR/dependencies_4.8.1.sh"
|
||||
|
||||
# location of libgcc
|
||||
LIBGCC_BASE="/mnt/gvfs/third-party2/libgcc/d00277f4559e261ed0a81f30f23c0ce5564e359e/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc"
|
||||
LIBGCC_INCLUDE="$LIBGCC_BASE/include"
|
||||
LIBGCC_LIBS=" -L $LIBGCC_BASE/libs"
|
||||
|
||||
# location of glibc
|
||||
GLIBC_REV=0600c95b31226b5e535614c590677d87c62d8016
|
||||
GLIBC_INCLUDE="/mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.17/gcc-4.8.1-glibc-2.17/99df8fc/include"
|
||||
GLIBC_LIBS=" -L /mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.17/gcc-4.8.1-glibc-2.17/99df8fc/lib"
|
||||
GLIBC_INCLUDE="$GLIBC_BASE/include"
|
||||
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
|
||||
|
||||
# location of snappy headers and libraries
|
||||
SNAPPY_REV=cbf6f1f209e5bd160bdc5d971744e039f36b1566
|
||||
SNAPPY_INCLUDE=" -I /mnt/gvfs/third-party2/snappy/$SNAPPY_REV/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a/include"
|
||||
SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/$SNAPPY_REV/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a/lib/libsnappy.a"
|
||||
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include"
|
||||
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
|
||||
|
||||
# location of zlib headers and libraries
|
||||
ZLIB_REV=6d39cb54708049f527e713ad19f2aadb9d3667e8
|
||||
ZLIB_INCLUDE=" -I /mnt/gvfs/third-party2/zlib/$ZLIB_REV/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a/include"
|
||||
ZLIB_LIBS=" /mnt/gvfs/third-party2/zlib/$ZLIB_REV/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a/lib/libz.a"
|
||||
ZLIB_INCLUDE=" -I $ZLIB_BASE/include"
|
||||
ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
|
||||
|
||||
# location of bzip headers and libraries
|
||||
BZIP_REV=d6c789bfc2ec4c51a63d66df2878926b8158cde8
|
||||
BZIP_INCLUDE=" -I /mnt/gvfs/third-party2/bzip2/$BZIP_REV/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a/include/"
|
||||
BZIP_LIBS=" /mnt/gvfs/third-party2/bzip2/$BZIP_REV/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a/lib/libbz2.a"
|
||||
BZIP2_INCLUDE=" -I $BZIP2_BASE/include/"
|
||||
BZIP2_LIBS=" $BZIP2_BASE/lib/libbz2.a"
|
||||
|
||||
LZ4_REV=6858fac689e0f92e584224d91bdb0e39f6c8320d
|
||||
LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/$LZ4_REV/r131/gcc-4.8.1-glibc-2.17/c3f970a/include"
|
||||
LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/$LZ4_REV/r131/gcc-4.8.1-glibc-2.17/c3f970a/lib/liblz4.a"
|
||||
LZ4_INCLUDE=" -I $LZ4_BASE/include"
|
||||
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
|
||||
|
||||
ZSTD_REV=810b81b4705def5243e998b54701f3c504e4009e
|
||||
ZSTD_INCLUDE=" -I /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.4.2/gcc-4.8.1-glibc-2.17/c3f970a/include"
|
||||
ZSTD_LIBS=" /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.4.2/gcc-4.8.1-glibc-2.17/c3f970a/lib/libzstd.a"
|
||||
ZSTD_INCLUDE=" -I $ZSTD_BASE/include"
|
||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
|
||||
|
||||
# location of gflags headers and libraries
|
||||
GFLAGS_REV=c7275a4ceae0aca0929e56964a31dafc53c1ee96
|
||||
GFLAGS_INCLUDE=" -I /mnt/gvfs/third-party2/gflags/$GFLAGS_REV/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a/include/"
|
||||
GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/$GFLAGS_REV/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a/lib/libgflags.a"
|
||||
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
|
||||
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
|
||||
|
||||
# location of jemalloc
|
||||
JEMALLOC_REV=c370265e58c4b6602e798df23335a1e9913dae52
|
||||
JEMALLOC_INCLUDE=" -I /mnt/gvfs/third-party2/jemalloc/$JEMALLOC_REV/4.0.3/gcc-4.8.1-glibc-2.17/8d31e51/include"
|
||||
JEMALLOC_LIB="/mnt/gvfs/third-party2/jemalloc/$JEMALLOC_REV/4.0.3/gcc-4.8.1-glibc-2.17/8d31e51/lib/libjemalloc.a"
|
||||
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include"
|
||||
JEMALLOC_LIB="$JEMALLOC_BASE/lib/libjemalloc.a"
|
||||
|
||||
# location of numa
|
||||
NUMA_REV=ae54a5ed22cdabb1c6446dce4e8ffae5b4446d73
|
||||
NUMA_INCLUDE=" -I /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/include/"
|
||||
NUMA_LIB=" /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/lib/libnuma.a"
|
||||
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
|
||||
NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
|
||||
|
||||
# location of libunwind
|
||||
LIBUNWIND_REV=121f1a75c4414683aea8c70b761bfaf187f7c1a3
|
||||
LIBUNWIND="/mnt/gvfs/third-party2/libunwind/$LIBUNWIND_REV/trunk/gcc-4.8.1-glibc-2.17/675d945/lib/libunwind.a"
|
||||
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
|
||||
|
||||
# use Intel SSE support for checksum calculations
|
||||
export USE_SSE=1
|
||||
|
||||
BINUTILS="/mnt/gvfs/third-party2/binutils/75670d0d8ef4891fd1ec2a7513ef01cd002c823b/2.25/centos6-native/da39a3e/bin"
|
||||
BINUTILS="$BINUTILS_BASE/bin"
|
||||
AR="$BINUTILS/ar"
|
||||
|
||||
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
|
||||
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP2_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
|
||||
|
||||
GCC_BASE="/mnt/gvfs/third-party2/gcc/c0064002d2609ab649603f769f0bd110bbe48029/4.8.1/centos6-native/cc6c9dc"
|
||||
STDLIBS="-L $GCC_BASE/lib64"
|
||||
|
||||
if [ -z "$USE_CLANG" ]; then
|
||||
@ -77,12 +68,13 @@ if [ -z "$USE_CLANG" ]; then
|
||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
||||
else
|
||||
# clang
|
||||
CLANG_BASE="/mnt/gvfs/third-party2/clang/ab054e9a490a8fd4537c0b6ec56e5c91c0f81c91/3.7"
|
||||
CLANG_INCLUDE="$CLANG_BASE/gcc-4.8.1-glibc-2.17/ee9b060/lib/clang/3.7/include"
|
||||
CC="$CLANG_BASE/centos6-native/b2feaee/bin/clang"
|
||||
CXX="$CLANG_BASE/centos6-native/b2feaee/bin/clang++"
|
||||
CLANG_BIN="$CLANG_BASE/bin"
|
||||
CLANG_LIB="$CLANG_BASE/lib"
|
||||
CLANG_INCLUDE="$CLANG_LIB/clang/*/include"
|
||||
CC="$CLANG_BIN/clang"
|
||||
CXX="$CLANG_BIN/clang++"
|
||||
|
||||
KERNEL_HEADERS_INCLUDE="/mnt/gvfs/third-party2/kernel-headers/1a48835975c66d30e47770ec419758ed3b9ba010/3.10.62-62_fbk17_03959_ge29cc63/gcc-4.8.1-glibc-2.17/da39a3e/include/"
|
||||
KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include/"
|
||||
|
||||
CFLAGS="-B$BINUTILS/gold -nostdinc -nostdlib"
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1 "
|
||||
@ -100,16 +92,15 @@ CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PR
|
||||
CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA"
|
||||
CXXFLAGS+=" $CFLAGS"
|
||||
|
||||
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB"
|
||||
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB"
|
||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"
|
||||
EXEC_LDFLAGS+=" $LIBUNWIND"
|
||||
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib"
|
||||
|
||||
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
|
||||
|
||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
|
||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
|
||||
|
||||
VALGRIND_REV=af85c56f424cd5edfc2c97588299b44ecdec96bb
|
||||
VALGRIND_VER="/mnt/gvfs/third-party2/valgrind/$VALGRIND_REV/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a/bin/"
|
||||
VALGRIND_VER="$VALGRIND_BASE/bin/"
|
||||
|
||||
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE
|
||||
|
127
build_tools/update_dependencies.sh
Executable file
127
build_tools/update_dependencies.sh
Executable file
@ -0,0 +1,127 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Update dependencies.sh file with the latest avaliable versions
|
||||
|
||||
BASEDIR=$(dirname $0)
|
||||
OUTPUT=""
|
||||
|
||||
function log_variable()
|
||||
{
|
||||
echo "$1=${!1}" >> "$OUTPUT"
|
||||
}
|
||||
|
||||
|
||||
TP2_LATEST="/mnt/vol/engshare/fbcode/third-party2"
|
||||
## $1 => lib name
|
||||
## $2 => lib version (if not provided, will try to pick latest)
|
||||
## $3 => platform (if not provided, will try to pick latest gcc)
|
||||
##
|
||||
## get_lib_base will set a variable named ${LIB_NAME}_BASE to the lib location
|
||||
function get_lib_base()
|
||||
{
|
||||
local lib_name=$1
|
||||
local lib_version=$2
|
||||
local lib_platform=$3
|
||||
|
||||
local result="$TP2_LATEST/$lib_name/"
|
||||
|
||||
# Lib Version
|
||||
if [ -z "$lib_version" ] || [ "$lib_version" = "LATEST" ]; then
|
||||
# version is not provided, use latest
|
||||
result=`ls -dr1v $result/*/ | head -n1`
|
||||
else
|
||||
result="$result/$lib_version/"
|
||||
fi
|
||||
|
||||
# Lib Platform
|
||||
if [ -z "$lib_platform" ]; then
|
||||
# platform is not provided, use latest gcc
|
||||
result=`ls -dr1v $result/gcc-*[^fb]/ | head -n1`
|
||||
else
|
||||
result="$result/$lib_platform/"
|
||||
fi
|
||||
|
||||
result=`ls -1d $result/*/ | head -n1`
|
||||
|
||||
# lib_name => LIB_NAME_BASE
|
||||
local __res_var=${lib_name^^}"_BASE"
|
||||
__res_var=`echo $__res_var | tr - _`
|
||||
# LIB_NAME_BASE=$result
|
||||
eval $__res_var=`readlink -f $result`
|
||||
|
||||
log_variable $__res_var
|
||||
}
|
||||
|
||||
###########################################################
|
||||
# 4.9.x dependencies #
|
||||
###########################################################
|
||||
|
||||
OUTPUT="$BASEDIR/dependencies.sh"
|
||||
|
||||
rm -f "$OUTPUT"
|
||||
touch "$OUTPUT"
|
||||
|
||||
echo "Writing dependencies to $OUTPUT"
|
||||
|
||||
# Compilers locations
|
||||
GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.9.x/centos6-native/*/`
|
||||
CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
|
||||
|
||||
log_variable GCC_BASE
|
||||
log_variable CLANG_BASE
|
||||
|
||||
# Libraries locations
|
||||
get_lib_base libgcc 4.9.x
|
||||
get_lib_base glibc 2.20
|
||||
get_lib_base snappy LATEST
|
||||
get_lib_base zlib LATEST
|
||||
get_lib_base bzip2 LATEST
|
||||
get_lib_base lz4 LATEST
|
||||
get_lib_base zstd LATEST
|
||||
get_lib_base gflags LATEST
|
||||
get_lib_base jemalloc LATEST
|
||||
get_lib_base numa LATEST
|
||||
get_lib_base libunwind LATEST
|
||||
|
||||
get_lib_base kernel-headers LATEST
|
||||
get_lib_base binutils LATEST centos6-native
|
||||
get_lib_base valgrind LATEST
|
||||
|
||||
git diff $OUTPUT
|
||||
|
||||
###########################################################
|
||||
# 4.8.1 dependencies #
|
||||
###########################################################
|
||||
|
||||
OUTPUT="$BASEDIR/dependencies_4.8.1.sh"
|
||||
|
||||
rm -f "$OUTPUT"
|
||||
touch "$OUTPUT"
|
||||
|
||||
echo "Writing 4.8.1 dependencies to $OUTPUT"
|
||||
|
||||
# Compilers locations
|
||||
GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.8.1/centos6-native/*/`
|
||||
CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
|
||||
|
||||
log_variable GCC_BASE
|
||||
log_variable CLANG_BASE
|
||||
|
||||
# Libraries locations
|
||||
get_lib_base libgcc 4.8.1 gcc-4.8.1-glibc-2.17
|
||||
get_lib_base glibc 2.17 gcc-4.8.1-glibc-2.17
|
||||
get_lib_base snappy LATEST gcc-4.8.1-glibc-2.17
|
||||
get_lib_base zlib LATEST gcc-4.8.1-glibc-2.17
|
||||
get_lib_base bzip2 LATEST gcc-4.8.1-glibc-2.17
|
||||
get_lib_base lz4 LATEST gcc-4.8.1-glibc-2.17
|
||||
get_lib_base zstd LATEST gcc-4.8.1-glibc-2.17
|
||||
get_lib_base gflags LATEST gcc-4.8.1-glibc-2.17
|
||||
get_lib_base jemalloc LATEST gcc-4.8.1-glibc-2.17
|
||||
get_lib_base numa LATEST gcc-4.8.1-glibc-2.17
|
||||
get_lib_base libunwind LATEST gcc-4.8.1-glibc-2.17
|
||||
|
||||
get_lib_base kernel-headers LATEST gcc-4.8.1-glibc-2.17
|
||||
get_lib_base binutils LATEST centos6-native
|
||||
get_lib_base valgrind 3.8.1 gcc-4.8.1-glibc-2.17
|
||||
|
||||
git diff $OUTPUT
|
@ -135,6 +135,10 @@ Status CheckConcurrentWritesSupported(const ColumnFamilyOptions& cf_options) {
|
||||
"Delete filtering (filter_deletes) is not compatible with concurrent "
|
||||
"memtable writes (allow_concurrent_memtable_writes)");
|
||||
}
|
||||
if (!cf_options.memtable_factory->IsInsertConcurrentlySupported()) {
|
||||
return Status::InvalidArgument(
|
||||
"Memtable doesn't concurrent writes (allow_concurrent_memtable_write)");
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
@ -107,6 +107,7 @@ TEST_F(CompactFilesTest, L0ConflictsFiles) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
|
||||
delete db;
|
||||
}
|
||||
|
||||
|
@ -4102,7 +4102,9 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
|
||||
|
||||
if (write_thread_.CompleteParallelWorker(&w)) {
|
||||
// we're responsible for early exit
|
||||
auto last_sequence = w.parallel_group->last_writer->sequence;
|
||||
auto last_sequence =
|
||||
w.parallel_group->last_writer->sequence +
|
||||
WriteBatchInternal::Count(w.parallel_group->last_writer->batch) - 1;
|
||||
SetTickerCount(stats_, SEQUENCE_NUMBER, last_sequence);
|
||||
versions_->SetLastSequence(last_sequence);
|
||||
write_thread_.EarlyExitParallelGroup(&w);
|
||||
@ -4401,7 +4403,9 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
|
||||
this, true /*dont_filter_deletes*/,
|
||||
true /*concurrent_memtable_writes*/);
|
||||
|
||||
assert(last_writer->sequence == last_sequence);
|
||||
assert(last_writer->sequence +
|
||||
WriteBatchInternal::Count(last_writer->batch) - 1 ==
|
||||
last_sequence);
|
||||
// CompleteParallelWorker returns true if this thread should
|
||||
// handle exit, false means somebody else did
|
||||
exit_completed_early = !write_thread_.CompleteParallelWorker(&w);
|
||||
|
102
db/db_test.cc
102
db/db_test.cc
@ -6511,6 +6511,28 @@ TEST_F(DBTest, TableOptionsSanitizeTest) {
|
||||
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
|
||||
ASSERT_OK(TryReopen(options));
|
||||
}
|
||||
|
||||
TEST_F(DBTest, ConcurrentMemtableNotSupported) {
|
||||
Options options = CurrentOptions();
|
||||
options.allow_concurrent_memtable_write = true;
|
||||
options.soft_pending_compaction_bytes_limit = 0;
|
||||
options.hard_pending_compaction_bytes_limit = 100;
|
||||
options.create_if_missing = true;
|
||||
|
||||
DestroyDB(dbname_, options);
|
||||
options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true, 4));
|
||||
ASSERT_NOK(TryReopen(options));
|
||||
|
||||
options.memtable_factory.reset(new SkipListFactory);
|
||||
ASSERT_OK(TryReopen(options));
|
||||
|
||||
ColumnFamilyOptions cf_options(options);
|
||||
cf_options.memtable_factory.reset(
|
||||
NewHashLinkListRepFactory(4, 0, 3, true, 4));
|
||||
ColumnFamilyHandle* handle;
|
||||
ASSERT_NOK(db_->CreateColumnFamily(cf_options, "name", &handle));
|
||||
}
|
||||
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
TEST_F(DBTest, SanitizeNumThreads) {
|
||||
@ -11057,6 +11079,86 @@ TEST_F(DBTest, WalFilterTestWithChangeBatchExtraKeys) {
|
||||
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
class SliceTransformLimitedDomain : public SliceTransform {
|
||||
const char* Name() const override { return "SliceTransformLimitedDomain"; }
|
||||
|
||||
Slice Transform(const Slice& src) const override {
|
||||
return Slice(src.data(), 5);
|
||||
}
|
||||
|
||||
bool InDomain(const Slice& src) const override {
|
||||
// prefix will be x????
|
||||
return src.size() >= 5 && src[0] == 'x';
|
||||
}
|
||||
|
||||
bool InRange(const Slice& dst) const override {
|
||||
// prefix will be x????
|
||||
return dst.size() == 5 && dst[0] == 'x';
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(DBTest, PrefixExtractorFullFilter) {
|
||||
BlockBasedTableOptions bbto;
|
||||
// Full Filter Block
|
||||
bbto.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
|
||||
bbto.whole_key_filtering = false;
|
||||
|
||||
Options options = CurrentOptions();
|
||||
options.prefix_extractor = std::make_shared<SliceTransformLimitedDomain>();
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
||||
|
||||
DestroyAndReopen(options);
|
||||
|
||||
ASSERT_OK(Put("x1111_AAAA", "val1"));
|
||||
ASSERT_OK(Put("x1112_AAAA", "val2"));
|
||||
ASSERT_OK(Put("x1113_AAAA", "val3"));
|
||||
ASSERT_OK(Put("x1114_AAAA", "val4"));
|
||||
// Not in domain, wont be added to filter
|
||||
ASSERT_OK(Put("zzzzz_AAAA", "val5"));
|
||||
|
||||
ASSERT_OK(Flush());
|
||||
|
||||
ASSERT_EQ(Get("x1111_AAAA"), "val1");
|
||||
ASSERT_EQ(Get("x1112_AAAA"), "val2");
|
||||
ASSERT_EQ(Get("x1113_AAAA"), "val3");
|
||||
ASSERT_EQ(Get("x1114_AAAA"), "val4");
|
||||
// Was not added to filter but rocksdb will try to read it from the filter
|
||||
ASSERT_EQ(Get("zzzzz_AAAA"), "val5");
|
||||
}
|
||||
|
||||
TEST_F(DBTest, PrefixExtractorBlockFilter) {
|
||||
BlockBasedTableOptions bbto;
|
||||
// Block Filter Block
|
||||
bbto.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, true));
|
||||
|
||||
Options options = CurrentOptions();
|
||||
options.prefix_extractor = std::make_shared<SliceTransformLimitedDomain>();
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
||||
|
||||
DestroyAndReopen(options);
|
||||
|
||||
ASSERT_OK(Put("x1113_AAAA", "val3"));
|
||||
ASSERT_OK(Put("x1114_AAAA", "val4"));
|
||||
// Not in domain, wont be added to filter
|
||||
ASSERT_OK(Put("zzzzz_AAAA", "val1"));
|
||||
ASSERT_OK(Put("zzzzz_AAAB", "val2"));
|
||||
ASSERT_OK(Put("zzzzz_AAAC", "val3"));
|
||||
ASSERT_OK(Put("zzzzz_AAAD", "val4"));
|
||||
|
||||
ASSERT_OK(Flush());
|
||||
|
||||
std::vector<std::string> iter_res;
|
||||
auto iter = db_->NewIterator(ReadOptions());
|
||||
// Seek to a key that was not in Domain
|
||||
for (iter->Seek("zzzzz_AAAA"); iter->Valid(); iter->Next()) {
|
||||
iter_res.emplace_back(iter->value().ToString());
|
||||
}
|
||||
|
||||
std::vector<std::string> expected_res = {"val1", "val2", "val3", "val4"};
|
||||
ASSERT_EQ(iter_res, expected_res);
|
||||
delete iter;
|
||||
}
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
class BloomStatsTestWithParam
|
||||
: public DBTest,
|
||||
|
@ -89,7 +89,8 @@ bool DBTestBase::ChangeOptions(int skip_mask) {
|
||||
option_config_ == kUniversalCompaction ||
|
||||
option_config_ == kUniversalCompactionMultiLevel ||
|
||||
option_config_ == kUniversalSubcompactions ||
|
||||
option_config_ == kFIFOCompaction) {
|
||||
option_config_ == kFIFOCompaction ||
|
||||
option_config_ == kConcurrentSkipList) {
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
@ -356,6 +357,11 @@ Options DBTestBase::CurrentOptions(
|
||||
options.max_subcompactions = 4;
|
||||
break;
|
||||
}
|
||||
case kConcurrentSkipList: {
|
||||
options.allow_concurrent_memtable_write = true;
|
||||
options.enable_write_thread_adaptive_yield = true;
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
break;
|
||||
|
@ -525,9 +525,10 @@ class DBTestBase : public testing::Test {
|
||||
kOptimizeFiltersForHits = 27,
|
||||
kRowCache = 28,
|
||||
kRecycleLogFiles = 29,
|
||||
kLevelSubcompactions = 30,
|
||||
kUniversalSubcompactions = 31,
|
||||
kEnd = 30
|
||||
kConcurrentSkipList = 30,
|
||||
kEnd = 31,
|
||||
kLevelSubcompactions = 31,
|
||||
kUniversalSubcompactions = 32,
|
||||
};
|
||||
int option_config_;
|
||||
|
||||
|
@ -14,6 +14,11 @@
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
static uint64_t TestGetTickerCount(const Options& options,
|
||||
Tickers ticker_type) {
|
||||
return options.statistics->getTickerCount(ticker_type);
|
||||
}
|
||||
|
||||
static std::string CompressibleString(Random* rnd, int len) {
|
||||
std::string r;
|
||||
test::CompressibleString(rnd, 0.8, len, &r);
|
||||
@ -154,6 +159,63 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) {
|
||||
Options options;
|
||||
options = CurrentOptions(options);
|
||||
options.compaction_style = kCompactionStyleUniversal;
|
||||
options.compaction_options_universal.size_ratio = 5;
|
||||
options.num_levels = num_levels_;
|
||||
options.write_buffer_size = 105 << 10; // 105KB
|
||||
options.arena_block_size = 4 << 10;
|
||||
options.target_file_size_base = 32 << 10; // 32KB
|
||||
// trigger compaction if there are >= 4 files
|
||||
options.level0_file_num_compaction_trigger = 4;
|
||||
BlockBasedTableOptions bbto;
|
||||
bbto.cache_index_and_filter_blocks = true;
|
||||
bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
|
||||
bbto.whole_key_filtering = true;
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
||||
options.optimize_filters_for_hits = true;
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
options.memtable_factory.reset(new SpecialSkipListFactory(3));
|
||||
|
||||
DestroyAndReopen(options);
|
||||
|
||||
// block compaction from happening
|
||||
env_->SetBackgroundThreads(1, Env::LOW);
|
||||
test::SleepingBackgroundTask sleeping_task_low;
|
||||
env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
|
||||
Env::Priority::LOW);
|
||||
|
||||
Put("", "");
|
||||
for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) {
|
||||
Put(Key(num * 10), "val");
|
||||
Put(Key(30 + num * 10), "val");
|
||||
Put(Key(60 + num * 10), "val");
|
||||
|
||||
dbfull()->TEST_WaitForFlushMemTable();
|
||||
}
|
||||
|
||||
// Query set of non existing keys
|
||||
for (int i = 5; i < 90; i += 10) {
|
||||
ASSERT_EQ(Get(Key(i)), "NOT_FOUND");
|
||||
}
|
||||
|
||||
// Make sure bloom filter is used at least once.
|
||||
ASSERT_GT(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
|
||||
auto prev_counter = TestGetTickerCount(options, BLOOM_FILTER_USEFUL);
|
||||
|
||||
// Unblock compaction and wait it for happening.
|
||||
sleeping_task_low.WakeUp();
|
||||
dbfull()->TEST_WaitForCompact();
|
||||
|
||||
// The same queries will not trigger bloom filter
|
||||
for (int i = 5; i < 90; i += 10) {
|
||||
ASSERT_EQ(Get(Key(i)), "NOT_FOUND");
|
||||
}
|
||||
ASSERT_EQ(prev_counter, TestGetTickerCount(options, BLOOM_FILTER_USEFUL));
|
||||
}
|
||||
|
||||
// TODO(kailiu) The tests on UniversalCompaction has some issues:
|
||||
// 1. A lot of magic numbers ("11" or "12").
|
||||
// 2. Made assumption on the memtable flush conditions, which may change from
|
||||
|
@ -147,8 +147,9 @@ class InlineSkipList {
|
||||
// values are ok.
|
||||
std::atomic<int> max_height_; // Height of the entire list
|
||||
|
||||
// Used for optimizing sequential insert patterns. Tricky. prev_[i] for
|
||||
// i up to max_height_ - 1 (inclusive) is the predecessor of prev_[0].
|
||||
// Used for optimizing sequential insert patterns. Tricky. prev_height_
|
||||
// of zero means prev_ is undefined. Otherwise: prev_[i] for i up
|
||||
// to max_height_ - 1 (inclusive) is the predecessor of prev_[0], and
|
||||
// prev_height_ is the height of prev_[0]. prev_[0] can only be equal
|
||||
// to head when max_height_ and prev_height_ are both 1.
|
||||
Node** prev_;
|
||||
@ -510,11 +511,10 @@ InlineSkipList<Comparator>::AllocateNode(size_t key_size, int height) {
|
||||
|
||||
template <class Comparator>
|
||||
void InlineSkipList<Comparator>::Insert(const char* key) {
|
||||
// InsertConcurrently can't maintain the prev_ invariants when it needs
|
||||
// to increase max_height_. In that case it sets prev_height_ to zero,
|
||||
// letting us know that we should ignore it. A relaxed load suffices
|
||||
// here because write thread synchronization separates Insert calls
|
||||
// from InsertConcurrently calls.
|
||||
// InsertConcurrently often can't maintain the prev_ invariants, so
|
||||
// it just sets prev_height_ to zero, letting us know that we should
|
||||
// ignore it. A relaxed load suffices here because write thread
|
||||
// synchronization separates Insert calls from InsertConcurrently calls.
|
||||
auto prev_height = prev_height_.load(std::memory_order_relaxed);
|
||||
|
||||
// fast path for sequential insertion
|
||||
@ -595,15 +595,24 @@ void InlineSkipList<Comparator>::InsertConcurrently(const char* key) {
|
||||
int height = x->UnstashHeight();
|
||||
assert(height >= 1 && height <= kMaxHeight_);
|
||||
|
||||
// We don't have a lock-free algorithm for updating prev_, but we do have
|
||||
// the option of invalidating the entire sequential-insertion cache.
|
||||
// prev_'s invariant is that prev_[i] (i > 0) is the predecessor of
|
||||
// prev_[0] at that level. We're only going to violate that if height
|
||||
// > 1 and key lands after prev_[height - 1] but before prev_[0].
|
||||
// Comparisons are pretty expensive, so an easier version is to just
|
||||
// clear the cache if height > 1. We only write to prev_height_ if the
|
||||
// nobody else has, to avoid invalidating the root of the skip list in
|
||||
// all of the other CPU caches.
|
||||
if (height > 1 && prev_height_.load(std::memory_order_relaxed) != 0) {
|
||||
prev_height_.store(0, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
int max_height = max_height_.load(std::memory_order_relaxed);
|
||||
while (height > max_height) {
|
||||
if (max_height_.compare_exchange_strong(max_height, height)) {
|
||||
// successfully updated it
|
||||
max_height = height;
|
||||
|
||||
// we dont have a lock-free algorithm for fixing up prev_, so just
|
||||
// mark it invalid
|
||||
prev_height_.store(0, std::memory_order_relaxed);
|
||||
break;
|
||||
}
|
||||
// else retry, possibly exiting the loop because somebody else
|
||||
|
@ -963,7 +963,7 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k,
|
||||
bool Version::IsFilterSkipped(int level) {
|
||||
// Reaching the bottom level implies misses at all upper levels, so we'll
|
||||
// skip checking the filters when we predict a hit.
|
||||
return cfd_->ioptions()->optimize_filters_for_hits &&
|
||||
return cfd_->ioptions()->optimize_filters_for_hits && level > 0 &&
|
||||
level == storage_info_.num_non_empty_levels() - 1;
|
||||
}
|
||||
|
||||
|
@ -188,10 +188,6 @@ class MemTableRep {
|
||||
// Default: true
|
||||
virtual bool IsSnapshotSupported() const { return true; }
|
||||
|
||||
// Return true if the current MemTableRep supports concurrent inserts
|
||||
// Default: false
|
||||
virtual bool IsInsertConcurrentlySupported() const { return false; }
|
||||
|
||||
protected:
|
||||
// When *key is an internal key concatenated with the value, returns the
|
||||
// user key.
|
||||
@ -210,6 +206,10 @@ class MemTableRepFactory {
|
||||
const SliceTransform*,
|
||||
Logger* logger) = 0;
|
||||
virtual const char* Name() const = 0;
|
||||
|
||||
// Return true if the current MemTableRep supports concurrent inserts
|
||||
// Default: false
|
||||
virtual bool IsInsertConcurrentlySupported() const { return false; }
|
||||
};
|
||||
|
||||
// This uses a skip list to store keys. It is the default.
|
||||
@ -229,6 +229,8 @@ class SkipListFactory : public MemTableRepFactory {
|
||||
Logger* logger) override;
|
||||
virtual const char* Name() const override { return "SkipListFactory"; }
|
||||
|
||||
bool IsInsertConcurrentlySupported() const override { return true; }
|
||||
|
||||
private:
|
||||
const size_t lookahead_;
|
||||
};
|
||||
|
@ -6,7 +6,7 @@
|
||||
|
||||
#define ROCKSDB_MAJOR 4
|
||||
#define ROCKSDB_MINOR 4
|
||||
#define ROCKSDB_PATCH 0
|
||||
#define ROCKSDB_PATCH 1
|
||||
|
||||
// Do not use these. We made the mistake of declaring macros starting with
|
||||
// double underscore. Now we have to live with our choice. We'll deprecate these
|
||||
|
@ -1120,8 +1120,11 @@ bool BlockBasedTable::PrefixMayMatch(const Slice& internal_key) {
|
||||
}
|
||||
|
||||
assert(rep_->ioptions.prefix_extractor != nullptr);
|
||||
auto prefix = rep_->ioptions.prefix_extractor->Transform(
|
||||
ExtractUserKey(internal_key));
|
||||
auto user_key = ExtractUserKey(internal_key);
|
||||
if (!rep_->ioptions.prefix_extractor->InDomain(user_key)) {
|
||||
return true;
|
||||
}
|
||||
auto prefix = rep_->ioptions.prefix_extractor->Transform(user_key);
|
||||
InternalKey internal_key_prefix(prefix, kMaxSequenceNumber, kTypeValue);
|
||||
auto internal_prefix = internal_key_prefix.Encode();
|
||||
|
||||
@ -1207,6 +1210,7 @@ bool BlockBasedTable::FullFilterKeyMayMatch(FilterBlockReader* filter,
|
||||
return false;
|
||||
}
|
||||
if (rep_->ioptions.prefix_extractor &&
|
||||
rep_->ioptions.prefix_extractor->InDomain(user_key) &&
|
||||
!filter->PrefixMayMatch(
|
||||
rep_->ioptions.prefix_extractor->Transform(user_key))) {
|
||||
return false;
|
||||
|
@ -24,6 +24,7 @@ default_params = {
|
||||
"disable_data_sync": 0,
|
||||
"disable_wal": 0,
|
||||
"filter_deletes": lambda: random.randint(0, 1),
|
||||
"allow_concurrent_memtable_write": 0,
|
||||
"iterpercent": 10,
|
||||
"max_background_compactions": 20,
|
||||
"max_bytes_for_level_base": 10485760,
|
||||
|
@ -25,8 +25,6 @@ public:
|
||||
transform_(transform), lookahead_(lookahead) {
|
||||
}
|
||||
|
||||
virtual bool IsInsertConcurrentlySupported() const override { return true; }
|
||||
|
||||
virtual KeyHandle Allocate(const size_t len, char** buf) override {
|
||||
*buf = skip_list_.AllocateKey(len);
|
||||
return static_cast<KeyHandle>(*buf);
|
||||
|
@ -6,6 +6,7 @@
|
||||
|
||||
#include <assert.h>
|
||||
#include <condition_variable>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
|
@ -104,7 +104,6 @@ PIMAGE_TLS_CALLBACK p_thread_callback_on_exit = wintlscleanup::WinOnThreadExit;
|
||||
|
||||
void ThreadLocalPtr::InitSingletons() {
|
||||
ThreadLocalPtr::StaticMeta::InitSingletons();
|
||||
ThreadLocalPtr::Instance();
|
||||
}
|
||||
|
||||
ThreadLocalPtr::StaticMeta* ThreadLocalPtr::Instance() {
|
||||
@ -113,30 +112,46 @@ ThreadLocalPtr::StaticMeta* ThreadLocalPtr::Instance() {
|
||||
// when the function is first call. As a result, we can properly
|
||||
// control their construction order by properly preparing their
|
||||
// first function call.
|
||||
static ThreadLocalPtr::StaticMeta inst;
|
||||
return &inst;
|
||||
//
|
||||
// Note that here we decide to make "inst" a static pointer w/o deleting
|
||||
// it at the end instead of a static variable. This is to avoid the following
|
||||
// destruction order desester happens when a child thread using ThreadLocalPtr
|
||||
// dies AFTER the main thread dies: When a child thread happens to use
|
||||
// ThreadLocalPtr, it will try to delete its thread-local data on its
|
||||
// OnThreadExit when the child thread dies. However, OnThreadExit depends
|
||||
// on the following variable. As a result, if the main thread dies before any
|
||||
// child thread happen to use ThreadLocalPtr dies, then the destruction of
|
||||
// the following variable will go first, then OnThreadExit, therefore causing
|
||||
// invalid access.
|
||||
//
|
||||
// The above problem can be solved by using thread_local to store tls_ instead
|
||||
// of using __thread. The major difference between thread_local and __thread
|
||||
// is that thread_local supports dynamic construction and destruction of
|
||||
// non-primitive typed variables. As a result, we can guarantee the
|
||||
// desturction order even when the main thread dies before any child threads.
|
||||
// However, thread_local requires gcc 4.8 and is not supported in all the
|
||||
// compilers that accepts -std=c++11 (e.g., the default clang on Mac), while
|
||||
// the current RocksDB still accept gcc 4.7.
|
||||
static ThreadLocalPtr::StaticMeta* inst = new ThreadLocalPtr::StaticMeta();
|
||||
return inst;
|
||||
}
|
||||
|
||||
void ThreadLocalPtr::StaticMeta::InitSingletons() { Mutex(); }
|
||||
|
||||
port::Mutex* ThreadLocalPtr::StaticMeta::Mutex() {
|
||||
// Here we prefer function static variable instead of global
|
||||
// static variable as function static variable is initialized
|
||||
// when the function is first call. As a result, we can properly
|
||||
// control their construction order by properly preparing their
|
||||
// first function call.
|
||||
static port::Mutex mutex;
|
||||
return &mutex;
|
||||
}
|
||||
port::Mutex* ThreadLocalPtr::StaticMeta::Mutex() { return &Instance()->mutex_; }
|
||||
|
||||
void ThreadLocalPtr::StaticMeta::OnThreadExit(void* ptr) {
|
||||
auto* tls = static_cast<ThreadData*>(ptr);
|
||||
assert(tls != nullptr);
|
||||
|
||||
auto* inst = Instance();
|
||||
// Use the cached StaticMeta::Instance() instead of directly calling
|
||||
// the variable inside StaticMeta::Instance() might already go out of
|
||||
// scope here in case this OnThreadExit is called after the main thread
|
||||
// dies.
|
||||
auto* inst = tls->inst;
|
||||
pthread_setspecific(inst->pthread_key_, nullptr);
|
||||
|
||||
MutexLock l(Mutex());
|
||||
MutexLock l(inst->MemberMutex());
|
||||
inst->RemoveThreadData(tls);
|
||||
// Unref stored pointers of current thread from all instances
|
||||
uint32_t id = 0;
|
||||
@ -154,7 +169,7 @@ void ThreadLocalPtr::StaticMeta::OnThreadExit(void* ptr) {
|
||||
delete tls;
|
||||
}
|
||||
|
||||
ThreadLocalPtr::StaticMeta::StaticMeta() : next_instance_id_(0) {
|
||||
ThreadLocalPtr::StaticMeta::StaticMeta() : next_instance_id_(0), head_(this) {
|
||||
if (pthread_key_create(&pthread_key_, &OnThreadExit) != 0) {
|
||||
abort();
|
||||
}
|
||||
@ -221,7 +236,7 @@ ThreadLocalPtr::ThreadData* ThreadLocalPtr::StaticMeta::GetThreadLocal() {
|
||||
|
||||
if (UNLIKELY(tls_ == nullptr)) {
|
||||
auto* inst = Instance();
|
||||
tls_ = new ThreadData();
|
||||
tls_ = new ThreadData(inst);
|
||||
{
|
||||
// Register it in the global chain, needs to be done before thread exit
|
||||
// handler registration
|
||||
|
@ -10,6 +10,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
@ -79,6 +80,8 @@ class ThreadLocalPtr {
|
||||
std::atomic<void*> ptr;
|
||||
};
|
||||
|
||||
class StaticMeta;
|
||||
|
||||
// This is the structure that is declared as "thread_local" storage.
|
||||
// The vector keep list of atomic pointer for all instances for "current"
|
||||
// thread. The vector is indexed by an Id that is unique in process and
|
||||
@ -95,10 +98,11 @@ class ThreadLocalPtr {
|
||||
// | thread 3 | void* | void* | void* | <- ThreadData
|
||||
// ---------------------------------------------------
|
||||
struct ThreadData {
|
||||
ThreadData() : entries() {}
|
||||
explicit ThreadData(StaticMeta* _inst) : entries(), inst(_inst) {}
|
||||
std::vector<Entry> entries;
|
||||
ThreadData* next;
|
||||
ThreadData* prev;
|
||||
StaticMeta* inst;
|
||||
};
|
||||
|
||||
class StaticMeta {
|
||||
@ -139,6 +143,31 @@ class ThreadLocalPtr {
|
||||
// initialized will be no-op.
|
||||
static void InitSingletons();
|
||||
|
||||
// protect inst, next_instance_id_, free_instance_ids_, head_,
|
||||
// ThreadData.entries
|
||||
//
|
||||
// Note that here we prefer function static variable instead of the usual
|
||||
// global static variable. The reason is that c++ destruction order of
|
||||
// static variables in the reverse order of their construction order.
|
||||
// However, C++ does not guarantee any construction order when global
|
||||
// static variables are defined in different files, while the function
|
||||
// static variables are initialized when their function are first called.
|
||||
// As a result, the construction order of the function static variables
|
||||
// can be controlled by properly invoke their first function calls in
|
||||
// the right order.
|
||||
//
|
||||
// For instance, the following function contains a function static
|
||||
// variable. We place a dummy function call of this inside
|
||||
// Env::Default() to ensure the construction order of the construction
|
||||
// order.
|
||||
static port::Mutex* Mutex();
|
||||
|
||||
// Returns the member mutex of the current StaticMeta. In general,
|
||||
// Mutex() should be used instead of this one. However, in case where
|
||||
// the static variable inside Instance() goes out of scope, MemberMutex()
|
||||
// should be used. One example is OnThreadExit() function.
|
||||
port::Mutex* MemberMutex() { return &mutex_; }
|
||||
|
||||
private:
|
||||
// Get UnrefHandler for id with acquiring mutex
|
||||
// REQUIRES: mutex locked
|
||||
@ -169,24 +198,9 @@ class ThreadLocalPtr {
|
||||
|
||||
std::unordered_map<uint32_t, UnrefHandler> handler_map_;
|
||||
|
||||
// protect inst, next_instance_id_, free_instance_ids_, head_,
|
||||
// ThreadData.entries
|
||||
//
|
||||
// Note that here we prefer function static variable instead of the usual
|
||||
// global static variable. The reason is that c++ destruction order of
|
||||
// static variables in the reverse order of their construction order.
|
||||
// However, C++ does not guarantee any construction order when global
|
||||
// static variables are defined in different files, while the function
|
||||
// static variables are initialized when their function are first called.
|
||||
// As a result, the construction order of the function static variables
|
||||
// can be controlled by properly invoke their first function calls in
|
||||
// the right order.
|
||||
//
|
||||
// For instance, the following function contains a function static
|
||||
// variable. We place a dummy function call of this inside
|
||||
// Env::Default() to ensure the construction order of the construction
|
||||
// order.
|
||||
static port::Mutex* Mutex();
|
||||
// The private mutex. Developers should always use Mutex() instead of
|
||||
// using this variable directly.
|
||||
port::Mutex mutex_;
|
||||
#if ROCKSDB_SUPPORT_THREAD_LOCAL
|
||||
// Thread local storage
|
||||
static __thread ThreadData* tls_;
|
||||
|
@ -3,14 +3,17 @@
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include <pthread.h>
|
||||
#include <atomic>
|
||||
#include <string>
|
||||
|
||||
#include "rocksdb/env.h"
|
||||
#include "port/port.h"
|
||||
#include "util/autovector.h"
|
||||
#include "util/thread_local.h"
|
||||
#include "util/sync_point.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
#include "util/thread_local.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
@ -467,6 +470,37 @@ TEST_F(ThreadLocalTest, CompareAndSwap) {
|
||||
ASSERT_EQ(tls.Get(), reinterpret_cast<void*>(3));
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
void* AccessThreadLocal(void* arg) {
|
||||
TEST_SYNC_POINT("AccessThreadLocal:Start");
|
||||
ThreadLocalPtr tlp;
|
||||
tlp.Reset(new std::string("hello RocksDB"));
|
||||
TEST_SYNC_POINT("AccessThreadLocal:End");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// The following test is disabled as it requires manual steps to run it
|
||||
// correctly.
|
||||
//
|
||||
// Currently we have no way to acess SyncPoint w/o ASAN error when the
|
||||
// child thread dies after the main thread dies. So if you manually enable
|
||||
// this test and only see an ASAN error on SyncPoint, it means you pass the
|
||||
// test.
|
||||
TEST_F(ThreadLocalTest, DISABLED_MainThreadDiesFirst) {
|
||||
rocksdb::SyncPoint::GetInstance()->LoadDependency(
|
||||
{{"AccessThreadLocal:Start", "MainThreadDiesFirst:End"},
|
||||
{"PosixEnv::~PosixEnv():End", "AccessThreadLocal:End"}});
|
||||
|
||||
// Triggers the initialization of singletons.
|
||||
Env::Default();
|
||||
pthread_t t;
|
||||
pthread_create(&t, nullptr, &AccessThreadLocal, nullptr);
|
||||
TEST_SYNC_POINT("MainThreadDiesFirst:End");
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
@ -18,20 +18,19 @@ namespace rocksdb {
|
||||
|
||||
class TransactionDBMutexImpl : public TransactionDBMutex {
|
||||
public:
|
||||
TransactionDBMutexImpl() : lock_(mutex_, std::defer_lock) {}
|
||||
TransactionDBMutexImpl() {}
|
||||
~TransactionDBMutexImpl() {}
|
||||
|
||||
Status Lock() override;
|
||||
|
||||
Status TryLockFor(int64_t timeout_time) override;
|
||||
|
||||
void UnLock() override { lock_.unlock(); }
|
||||
void UnLock() override { mutex_.unlock(); }
|
||||
|
||||
friend class TransactionDBCondVarImpl;
|
||||
|
||||
private:
|
||||
std::mutex mutex_; // Do not acquire mutex_ directly. Use lock_.
|
||||
std::unique_lock<std::mutex> lock_;
|
||||
std::mutex mutex_;
|
||||
};
|
||||
|
||||
class TransactionDBCondVarImpl : public TransactionDBCondVar {
|
||||
@ -63,7 +62,7 @@ TransactionDBMutexFactoryImpl::AllocateCondVar() {
|
||||
}
|
||||
|
||||
Status TransactionDBMutexImpl::Lock() {
|
||||
lock_.lock();
|
||||
mutex_.lock();
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
@ -71,7 +70,7 @@ Status TransactionDBMutexImpl::TryLockFor(int64_t timeout_time) {
|
||||
bool locked = true;
|
||||
|
||||
if (timeout_time == 0) {
|
||||
locked = lock_.try_lock();
|
||||
locked = mutex_.try_lock();
|
||||
} else {
|
||||
// Previously, this code used a std::timed_mutex. However, this was changed
|
||||
// due to known bugs in gcc versions < 4.9.
|
||||
@ -80,7 +79,7 @@ Status TransactionDBMutexImpl::TryLockFor(int64_t timeout_time) {
|
||||
// Since this mutex isn't held for long and only a single mutex is ever
|
||||
// held at a time, it is reasonable to ignore the lock timeout_time here
|
||||
// and only check it when waiting on the condition_variable.
|
||||
lock_.lock();
|
||||
mutex_.lock();
|
||||
}
|
||||
|
||||
if (!locked) {
|
||||
@ -95,30 +94,40 @@ Status TransactionDBCondVarImpl::Wait(
|
||||
std::shared_ptr<TransactionDBMutex> mutex) {
|
||||
auto mutex_impl = reinterpret_cast<TransactionDBMutexImpl*>(mutex.get());
|
||||
|
||||
cv_.wait(mutex_impl->lock_);
|
||||
std::unique_lock<std::mutex> lock(mutex_impl->mutex_, std::adopt_lock);
|
||||
cv_.wait(lock);
|
||||
|
||||
// Make sure unique_lock doesn't unlock mutex when it destructs
|
||||
lock.release();
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status TransactionDBCondVarImpl::WaitFor(
|
||||
std::shared_ptr<TransactionDBMutex> mutex, int64_t timeout_time) {
|
||||
Status s;
|
||||
|
||||
auto mutex_impl = reinterpret_cast<TransactionDBMutexImpl*>(mutex.get());
|
||||
std::unique_lock<std::mutex> lock(mutex_impl->mutex_, std::adopt_lock);
|
||||
|
||||
if (timeout_time < 0) {
|
||||
// If timeout is negative, do not use a timeout
|
||||
cv_.wait(mutex_impl->lock_);
|
||||
cv_.wait(lock);
|
||||
} else {
|
||||
auto duration = std::chrono::microseconds(timeout_time);
|
||||
auto cv_status = cv_.wait_for(mutex_impl->lock_, duration);
|
||||
auto cv_status = cv_.wait_for(lock, duration);
|
||||
|
||||
// Check if the wait stopped due to timing out.
|
||||
if (cv_status == std::cv_status::timeout) {
|
||||
return Status::TimedOut(Status::SubCode::kMutexTimeout);
|
||||
s = Status::TimedOut(Status::SubCode::kMutexTimeout);
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure unique_lock doesn't unlock mutex when it destructs
|
||||
lock.release();
|
||||
|
||||
// CV was signaled, or we spuriously woke up (but didn't time out)
|
||||
return Status::OK();
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
Loading…
Reference in New Issue
Block a user