2020-08-13 04:28:28 +02:00
|
|
|
#!/bin/sh
|
|
|
|
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
|
|
|
#
|
|
|
|
# Set environment variables so that we can compile rocksdb using
|
|
|
|
# fbcode settings. It uses the latest g++ and clang compilers and also
|
|
|
|
# uses jemalloc
|
|
|
|
# Environment variables that change the behavior of this script:
|
|
|
|
# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
|
|
|
|
|
|
|
|
|
|
|
|
BASEDIR=`dirname $BASH_SOURCE`
|
|
|
|
source "$BASEDIR/dependencies_platform009.sh"
|
|
|
|
|
|
|
|
CFLAGS=""
|
|
|
|
|
|
|
|
# libgcc
|
|
|
|
LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/9.3.0"
|
|
|
|
LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
|
|
|
|
|
|
|
|
# glibc
|
|
|
|
GLIBC_INCLUDE="$GLIBC_BASE/include"
|
|
|
|
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
|
|
|
|
|
|
|
|
if test -z $PIC_BUILD; then
|
2022-02-14 17:53:29 +01:00
|
|
|
MAYBE_PIC=
|
2020-08-13 04:28:28 +02:00
|
|
|
else
|
2022-02-14 17:53:29 +01:00
|
|
|
MAYBE_PIC=_pic
|
2020-08-13 04:28:28 +02:00
|
|
|
fi
|
2022-02-14 17:53:29 +01:00
|
|
|
|
2022-04-18 21:47:16 +02:00
|
|
|
if ! test $ROCKSDB_DISABLE_SNAPPY; then
|
|
|
|
# snappy
|
|
|
|
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
|
|
|
|
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy${MAYBE_PIC}.a"
|
|
|
|
CFLAGS+=" -DSNAPPY"
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_ZLIB; then
|
|
|
|
# location of zlib headers and libraries
|
|
|
|
ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
|
|
|
|
ZLIB_LIBS=" $ZLIB_BASE/lib/libz${MAYBE_PIC}.a"
|
|
|
|
CFLAGS+=" -DZLIB"
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_BZIP; then
|
|
|
|
# location of bzip headers and libraries
|
|
|
|
BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
|
|
|
|
BZIP_LIBS=" $BZIP2_BASE/lib/libbz2${MAYBE_PIC}.a"
|
|
|
|
CFLAGS+=" -DBZIP2"
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_LZ4; then
|
|
|
|
LZ4_INCLUDE=" -I $LZ4_BASE/include/"
|
|
|
|
LZ4_LIBS=" $LZ4_BASE/lib/liblz4${MAYBE_PIC}.a"
|
|
|
|
CFLAGS+=" -DLZ4"
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! test $ROCKSDB_DISABLE_ZSTD; then
|
|
|
|
ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
|
|
|
|
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd${MAYBE_PIC}.a"
|
|
|
|
CFLAGS+=" -DZSTD"
|
|
|
|
fi
|
2020-08-13 04:28:28 +02:00
|
|
|
|
|
|
|
# location of gflags headers and libraries
|
|
|
|
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
|
2022-02-14 17:53:29 +01:00
|
|
|
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags${MAYBE_PIC}.a"
|
2020-08-13 04:28:28 +02:00
|
|
|
CFLAGS+=" -DGFLAGS=gflags"
|
|
|
|
|
2021-09-24 19:22:22 +02:00
|
|
|
BENCHMARK_INCLUDE=" -I $BENCHMARK_BASE/include/"
|
2022-02-14 17:53:29 +01:00
|
|
|
BENCHMARK_LIBS=" $BENCHMARK_BASE/lib/libbenchmark${MAYBE_PIC}.a"
|
2021-09-24 19:22:22 +02:00
|
|
|
|
Meta-internal folly integration with F14FastMap (#9546)
Summary:
Especially after updating to C++17, I don't see a compelling case for
*requiring* any folly components in RocksDB. I was able to purge the existing
hard dependencies, and it can be quite difficult to strip out non-trivial components
from folly for use in RocksDB. (The prospect of doing that on F14 has changed
my mind on the best approach here.)
But this change creates an optional integration where we can plug in
components from folly at compile time, starting here with F14FastMap to replace
std::unordered_map when possible (probably no public APIs for example). I have
replaced the biggest CPU users of std::unordered_map with compile-time
pluggable UnorderedMap which will use F14FastMap when USE_FOLLY is set.
USE_FOLLY is always set in the Meta-internal buck build, and a simulation of
that is in the Makefile for public CI testing. A full folly build is not needed, but
checking out the full folly repo is much simpler for getting the dependency,
and anything else we might want to optionally integrate in the future.
Some picky details:
* I don't think the distributed mutex stuff is actually used, so it was easy to remove.
* I implemented an alternative to `folly::constexpr_log2` (which is much easier
in C++17 than C++11) so that I could pull out the hard dependencies on
`ConstexprMath.h`
* I had to add noexcept move constructors/operators to some types to make
F14's complainUnlessNothrowMoveAndDestroy check happy, and I added a
macro to make that easier in some common cases.
* Updated Meta-internal buck build to use folly F14Map (always)
No updates to HISTORY.md nor INSTALL.md as this is not (yet?) considered a
production integration for open source users.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9546
Test Plan:
CircleCI tests updated so that a couple of them use folly.
Most internal unit & stress/crash tests updated to use Meta-internal latest folly.
(Note: they should probably use buck but they currently use Makefile.)
Example performance improvement: when filter partitions are pinned in cache,
they are tracked by PartitionedFilterBlockReader::filter_map_ and we can build
a test that exercises that heavily. Build DB with
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench -benchmarks=fillrandom -num=10000000 -disable_wal=1 -write_buffer_size=30000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters
```
and test with (simultaneous runs with & without folly, ~20 times each to see
convergence)
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench_folly -readonly -use_existing_db -benchmarks=readrandom -num=10000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters -duration=40 -pin_l0_filter_and_index_blocks_in_cache
```
Average ops/s no folly: 26229.2
Average ops/s with folly: 26853.3 (+2.4%)
Reviewed By: ajkr
Differential Revision: D34181736
Pulled By: pdillinger
fbshipit-source-id: ffa6ad5104c2880321d8a1aa7187e00ab0d02e94
2022-04-13 16:34:01 +02:00
|
|
|
BOOST_INCLUDE=" -I $BOOST_BASE/include/"
|
|
|
|
|
2020-08-13 04:28:28 +02:00
|
|
|
# location of jemalloc
|
|
|
|
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
|
2022-02-14 17:53:29 +01:00
|
|
|
JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc${MAYBE_PIC}.a"
|
2020-08-13 04:28:28 +02:00
|
|
|
|
2022-02-14 17:53:29 +01:00
|
|
|
# location of numa
|
|
|
|
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
|
|
|
|
NUMA_LIB=" $NUMA_BASE/lib/libnuma${MAYBE_PIC}.a"
|
|
|
|
CFLAGS+=" -DNUMA"
|
2020-08-13 04:28:28 +02:00
|
|
|
|
2022-02-14 17:53:29 +01:00
|
|
|
# location of libunwind
|
|
|
|
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind${MAYBE_PIC}.a"
|
2020-08-13 04:28:28 +02:00
|
|
|
|
|
|
|
# location of TBB
|
|
|
|
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
|
2022-02-14 17:53:29 +01:00
|
|
|
TBB_LIBS="$TBB_BASE/lib/libtbb${MAYBE_PIC}.a"
|
2020-08-13 04:28:28 +02:00
|
|
|
CFLAGS+=" -DTBB"
|
|
|
|
|
|
|
|
# location of LIBURING
|
|
|
|
LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/"
|
2022-02-14 17:53:29 +01:00
|
|
|
LIBURING_LIBS="$LIBURING_BASE/lib/liburing${MAYBE_PIC}.a"
|
2020-08-13 04:28:28 +02:00
|
|
|
CFLAGS+=" -DLIBURING"
|
|
|
|
|
|
|
|
test "$USE_SSE" || USE_SSE=1
|
|
|
|
export USE_SSE
|
|
|
|
test "$PORTABLE" || PORTABLE=1
|
|
|
|
export PORTABLE
|
|
|
|
|
|
|
|
BINUTILS="$BINUTILS_BASE/bin"
|
|
|
|
AR="$BINUTILS/ar"
|
2021-09-01 19:05:57 +02:00
|
|
|
AS="$BINUTILS/as"
|
2020-08-13 04:28:28 +02:00
|
|
|
|
Meta-internal folly integration with F14FastMap (#9546)
Summary:
Especially after updating to C++17, I don't see a compelling case for
*requiring* any folly components in RocksDB. I was able to purge the existing
hard dependencies, and it can be quite difficult to strip out non-trivial components
from folly for use in RocksDB. (The prospect of doing that on F14 has changed
my mind on the best approach here.)
But this change creates an optional integration where we can plug in
components from folly at compile time, starting here with F14FastMap to replace
std::unordered_map when possible (probably no public APIs for example). I have
replaced the biggest CPU users of std::unordered_map with compile-time
pluggable UnorderedMap which will use F14FastMap when USE_FOLLY is set.
USE_FOLLY is always set in the Meta-internal buck build, and a simulation of
that is in the Makefile for public CI testing. A full folly build is not needed, but
checking out the full folly repo is much simpler for getting the dependency,
and anything else we might want to optionally integrate in the future.
Some picky details:
* I don't think the distributed mutex stuff is actually used, so it was easy to remove.
* I implemented an alternative to `folly::constexpr_log2` (which is much easier
in C++17 than C++11) so that I could pull out the hard dependencies on
`ConstexprMath.h`
* I had to add noexcept move constructors/operators to some types to make
F14's complainUnlessNothrowMoveAndDestroy check happy, and I added a
macro to make that easier in some common cases.
* Updated Meta-internal buck build to use folly F14Map (always)
No updates to HISTORY.md nor INSTALL.md as this is not (yet?) considered a
production integration for open source users.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9546
Test Plan:
CircleCI tests updated so that a couple of them use folly.
Most internal unit & stress/crash tests updated to use Meta-internal latest folly.
(Note: they should probably use buck but they currently use Makefile.)
Example performance improvement: when filter partitions are pinned in cache,
they are tracked by PartitionedFilterBlockReader::filter_map_ and we can build
a test that exercises that heavily. Build DB with
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench -benchmarks=fillrandom -num=10000000 -disable_wal=1 -write_buffer_size=30000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters
```
and test with (simultaneous runs with & without folly, ~20 times each to see
convergence)
```
TEST_TMPDIR=/dev/shm/rocksdb ./db_bench_folly -readonly -use_existing_db -benchmarks=readrandom -num=10000000 -bloom_bits=16 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -partition_index_and_filters -duration=40 -pin_l0_filter_and_index_blocks_in_cache
```
Average ops/s no folly: 26229.2
Average ops/s with folly: 26853.3 (+2.4%)
Reviewed By: ajkr
Differential Revision: D34181736
Pulled By: pdillinger
fbshipit-source-id: ffa6ad5104c2880321d8a1aa7187e00ab0d02e94
2022-04-13 16:34:01 +02:00
|
|
|
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE $BENCHMARK_INCLUDE $BOOST_INCLUDE"
|
2020-08-13 04:28:28 +02:00
|
|
|
|
|
|
|
STDLIBS="-L $GCC_BASE/lib64"
|
|
|
|
|
|
|
|
CLANG_BIN="$CLANG_BASE/bin"
|
|
|
|
CLANG_LIB="$CLANG_BASE/lib"
|
|
|
|
CLANG_SRC="$CLANG_BASE/../../src"
|
|
|
|
|
|
|
|
CLANG_ANALYZER="$CLANG_BIN/clang++"
|
|
|
|
CLANG_SCAN_BUILD="$CLANG_SRC/llvm/clang/tools/scan-build/bin/scan-build"
|
|
|
|
|
|
|
|
if [ -z "$USE_CLANG" ]; then
|
|
|
|
# gcc
|
|
|
|
CC="$GCC_BASE/bin/gcc"
|
|
|
|
CXX="$GCC_BASE/bin/g++"
|
|
|
|
AR="$GCC_BASE/bin/gcc-ar"
|
|
|
|
|
2021-06-14 17:44:28 +02:00
|
|
|
CFLAGS+=" -B$BINUTILS"
|
2020-08-13 04:28:28 +02:00
|
|
|
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
|
|
|
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
|
|
|
JEMALLOC=1
|
|
|
|
else
|
|
|
|
# clang
|
|
|
|
CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
|
|
|
|
CC="$CLANG_BIN/clang"
|
|
|
|
CXX="$CLANG_BIN/clang++"
|
|
|
|
AR="$CLANG_BIN/llvm-ar"
|
|
|
|
|
|
|
|
KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
|
|
|
|
|
2021-06-14 17:44:28 +02:00
|
|
|
CFLAGS+=" -B$BINUTILS -nostdinc -nostdlib"
|
2020-08-13 04:28:28 +02:00
|
|
|
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/9.x "
|
|
|
|
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/9.x/x86_64-facebook-linux "
|
|
|
|
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
|
|
|
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
|
|
|
CFLAGS+=" -isystem $CLANG_INCLUDE"
|
|
|
|
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
|
|
|
|
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
|
|
|
|
CFLAGS+=" -Wno-expansion-to-defined "
|
|
|
|
CXXFLAGS="-nostdinc++"
|
|
|
|
fi
|
|
|
|
|
|
|
|
CFLAGS+=" $DEPS_INCLUDE"
|
2022-05-19 00:25:19 +02:00
|
|
|
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DHAVE_SSE42 -DROCKSDB_IOURING_PRESENT"
|
2020-08-13 04:28:28 +02:00
|
|
|
CXXFLAGS+=" $CFLAGS"
|
|
|
|
|
2021-09-24 19:22:22 +02:00
|
|
|
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS $BENCHMARK_LIBS"
|
2020-08-13 04:28:28 +02:00
|
|
|
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform009/lib/ld.so"
|
|
|
|
EXEC_LDFLAGS+=" $LIBUNWIND"
|
|
|
|
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform009/lib"
|
2021-06-14 21:15:07 +02:00
|
|
|
EXEC_LDFLAGS+=" -Wl,-rpath=$GCC_BASE/lib64"
|
2020-08-13 04:28:28 +02:00
|
|
|
# required by libtbb
|
|
|
|
EXEC_LDFLAGS+=" -ldl"
|
|
|
|
|
|
|
|
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
|
2021-09-01 23:32:56 +02:00
|
|
|
PLATFORM_LDFLAGS+=" -B$BINUTILS"
|
2020-08-13 04:28:28 +02:00
|
|
|
|
2021-09-24 19:22:22 +02:00
|
|
|
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS $LIBURING_LIBS $BENCHMARK_LIBS"
|
2020-08-13 04:28:28 +02:00
|
|
|
|
|
|
|
VALGRIND_VER="$VALGRIND_BASE/bin/"
|
|
|
|
|
|
|
|
# lua not supported because it's on track for deprecation, I think
|
|
|
|
LUA_PATH=
|
|
|
|
LUA_LIB=
|
|
|
|
|
2021-09-01 19:05:57 +02:00
|
|
|
export CC CXX AR AS CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
|