From 9a126ba3b3cb696e05d5b842ae8fe457d3b19f14 Mon Sep 17 00:00:00 2001 From: Lei Jin Date: Tue, 28 Jan 2014 16:03:55 -0800 Subject: [PATCH 1/5] only corrupt private file checksum in backupable_db_test Summary: if it happens (randomly) to corrupt shared file in the test, then the checksum will be inconsistent between meta files from different backup. BackupEngine will then detect this issue and fail. But in reality, this does not happen since the checksum is checked on every backup. So here, only corrupt checksum of private file to let BackupEngine to construct properly (but fail during restore). Test Plan: run test with valgrind Reviewers: igor Reviewed By: igor CC: leveldb Differential Revision: https://reviews.facebook.net/D15531 --- utilities/backupable/backupable_db_test.cc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/utilities/backupable/backupable_db_test.cc b/utilities/backupable/backupable_db_test.cc index c5909f8e7..89326bfe7 100644 --- a/utilities/backupable/backupable_db_test.cc +++ b/utilities/backupable/backupable_db_test.cc @@ -250,17 +250,15 @@ class FileManager : public EnvWrapper { return s; } - std::vector positions; - auto pos = metadata.find(" crc32 "); + auto pos = metadata.find("private"); + if (pos == std::string::npos) { + return Status::Corruption("private file is expected"); + } + pos = metadata.find(" crc32 ", pos + 6); if (pos == std::string::npos) { return Status::Corruption("checksum not found"); } - do { - positions.push_back(pos); - pos = metadata.find(" crc32 ", pos + 6); - } while (pos != std::string::npos); - pos = positions[rnd_.Next() % positions.size()]; if (metadata.size() < pos + 7) { return Status::Corruption("bad CRC32 checksum value"); } From 9fe60d50ff8b90c5c12bcc22ec38a067eb232e26 Mon Sep 17 00:00:00 2001 From: kailiu Date: Tue, 28 Jan 2014 13:06:09 -0800 Subject: [PATCH 2/5] Add history log and revise script Summary: * Add a change log for rocksdb releases. * Remove the hacky parts of make_new_version.sh, which are either no longer useful or will be done in our dedicated 3rd-party release tool. Test Plan: N/A Reviewers: igor, haobo, sdong, dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D15543 --- HISTORY.md | 39 +++++++++++++++++++++++++++++++++ build_tools/make_new_version.sh | 21 ------------------ 2 files changed, 39 insertions(+), 21 deletions(-) create mode 100644 HISTORY.md diff --git a/HISTORY.md b/HISTORY.md new file mode 100644 index 000000000..912599bc9 --- /dev/null +++ b/HISTORY.md @@ -0,0 +1,39 @@ +# Rocksdb Change Log + +## 2.7.0 (01/28/2014) + +### Public API changes + +* Renamed `StackableDB::GetRawDB()` to `StackableDB::GetBaseDB()`. +* Renamed `WriteBatch::Data()` `const std::string& Data() const`. +* Renamed class `TableStats` to `TableProperties`. +* Deleted class `PrefixHashRepFactory`. Please use `NewHashSkipListRepFactory()` instead. +* Supported multi-threaded `EnableFileDeletions()` and `DisableFileDeletions()`. +* Added `DB::GetOptions()`. +* Added `DB::GetDbIdentity()`. + +### New Features + +* Added [BackupableDB](https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F) +* Implemented [TailingIterator](https://github.com/facebook/rocksdb/wiki/Tailing-Iterator), a special type of iterator that + doesn't create a snapshot (can be used to read newly inserted data) + and is optimized for doing sequential reads. +* Added property block for table, which allows (1) a table to store + its metadata and (2) end user to collect and store properties they + are interested in. +* Enabled caching index and filter block in block cache (turned off by default). +* Supported error report when doing manual compaction. +* Supported additional Linux platform flavors and Mac OS. +* Put with `SliceParts` - Variant of `Put()` that gathers output like `writev(2)` +* Bug fixes and code refactor for compatibility with upcoming Column + Family feature. + +### Performance Improvements + +* Huge benchmark performance improvements by multiple efforts. For example, increase in readonly QPS from about 530k in 2.6 release to 1.1 million in 2.7 [1] +* Speeding up a way RocksDB deleted obsolete files - no longer listing the whole directory under a lock -- decrease in p99 +* Use raw pointer instead of shared pointer for statistics: [5b825d](https://github.com/facebook/rocksdb/commit/5b825d6964e26ec3b4bb6faa708ebb1787f1d7bd) -- huge increase in performance -- shared pointers are slow +* Optimized locking for `Get()` -- [1fdb3f](https://github.com/facebook/rocksdb/commit/1fdb3f7dc60e96394e3e5b69a46ede5d67fb976c) -- 1.5x QPS increase for some workloads +* Cache speedup - [e8d40c3](https://github.com/facebook/rocksdb/commit/e8d40c31b3cca0c3e1ae9abe9b9003b1288026a9) +* Implemented autovector, which allocates first N elements on stack. Most of vectors in RocksDB are small. Also, we never want to allocate heap objects while holding a mutex. -- [c01676e4](https://github.com/facebook/rocksdb/commit/c01676e46d3be08c3c140361ef1f5884f47d3b3c) +* Lots of efforts to move malloc, memcpy and IO outside of locks diff --git a/build_tools/make_new_version.sh b/build_tools/make_new_version.sh index ca8a21262..a1ee82720 100755 --- a/build_tools/make_new_version.sh +++ b/build_tools/make_new_version.sh @@ -27,7 +27,6 @@ if [ $GIT_BRANCH != "master" ]; then echo "Error: Current branch is '$GIT_BRANCH', Please switch to master branch." fi -# --Step 1: cutting new tag title "Adding new tag for this release ..." git tag -a "$ROCKSDB_VERSION.fb" -m "Rocksdb $ROCKSDB_VERSION" @@ -38,24 +37,4 @@ export https_proxy="$http_proxy"; title "Pushing new tag to remote repo ..." proxycmd.sh git push origin --tags -# --Step 2: Update README.fb -title "Updating the latest version info in README.fb ..." -sed -i "s/Latest release is [0-9]\+.[0-9]\+.fb/Latest release is $ROCKSDB_VERSION.fb/" README.fb -git commit README.fb -m "update the latest version in README.fb to $ROCKSDB_VERSION" -proxycmd.sh git push - -# --Step 3: Prepare this repo for 3rd release -title "Cleaning up repo ..." -make clean -git clean -fxd - -title "Generating the build info ..." -# Comment out the call of `build_detection_version` so that the SHA number and build date of this -# release will remain constant. Otherwise everytime we run "make" util/build_version.cc will be -# overridden. -sed -i 's/^\$PWD\/build_tools\/build_detect_version$//' build_tools/build_detect_platform - -# Generate util/build_version.cc -build_tools/build_detect_version - title "Done!" From e57f0cc1a1c19537f221c74b0171880edb3d48e3 Mon Sep 17 00:00:00 2001 From: Igor Canadi Date: Wed, 29 Jan 2014 08:16:43 -0800 Subject: [PATCH 3/5] add include to version_set.h --- db/version_set.h | 1 + 1 file changed, 1 insertion(+) diff --git a/db/version_set.h b/db/version_set.h index 376b259b9..92a9ed3cd 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "db/dbformat.h" #include "db/version_edit.h" #include "port/port.h" From 9a597dc69614a138e8ad44b8aea262fa08b20a67 Mon Sep 17 00:00:00 2001 From: Igor Canadi Date: Wed, 29 Jan 2014 08:41:43 -0800 Subject: [PATCH 4/5] Installation instructions for CentOS --- INSTALL.md | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index ab0460341..a63b9b15c 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -17,15 +17,42 @@ libraries. You are on your own. ## Supported platforms -* **Linux** +* **Linux - Ubuntu** * Upgrade your gcc to version at least 4.7 to get C++11 support. - * Install gflags. First, try: `sudo apt-get install libgflags-dev`. + * Install gflags. First, try: `sudo apt-get install libgflags-dev` If this doesn't work and you're using Ubuntu, here's a nice tutorial: (http://askubuntu.com/questions/312173/installing-gflags-12-04) * Install snappy. This is usually as easy as: `sudo apt-get install libsnappy-dev`. * Install zlib. Try: `sudo apt-get install zlib1g-dev`. * Install bzip2: `sudo apt-get install libbz2-dev`. +* **Linux - CentOS** + * Upgrade your gcc to version at least 4.7 to get C++11 support: + `yum install gcc47-c++` + * Install gflags: + + wget https://gflags.googlecode.com/files/gflags-2.0-no-svn-files.tar.gz + tar -xzvf gflags-2.0-no-svn-files.tar.gz + cd gflags-2.0 + ./configure && make && sudo make install + + * Install snappy: + + wget https://snappy.googlecode.com/files/snappy-1.1.1.tar.gz + tar -xzvf snappy-1.1.1.tar.gz + cd snappy-1.1.1 + ./configure && make && sudo make install + + * Install zlib: + + sudo yum install zlib + sudo yum install zlib-devel + + * Install bzip2: + + sudo yum install bzip2 + sudo yum install bzip2-devel + * **OS X**: * Install latest C++ compiler that supports C++ 11: * Update XCode: run `xcode-select --install` (or install it from XCode App's settting). From c9eef784b7c9565e34400ae6aca1f6b154d022d1 Mon Sep 17 00:00:00 2001 From: kailiu Date: Wed, 29 Jan 2014 10:04:31 -0800 Subject: [PATCH 5/5] Improve make_new_version.sh --- build_tools/make_new_version.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/build_tools/make_new_version.sh b/build_tools/make_new_version.sh index a1ee82720..165ed0c02 100755 --- a/build_tools/make_new_version.sh +++ b/build_tools/make_new_version.sh @@ -5,6 +5,11 @@ # of patent rights can be found in the PATENTS file in the same directory. set -e +if [ -z "$GIT" ] +then + GIT="git" +fi + # Print out the colored progress info so that it can be brainlessly # distinguished by users. function title() { @@ -28,13 +33,12 @@ if [ $GIT_BRANCH != "master" ]; then fi title "Adding new tag for this release ..." -git tag -a "$ROCKSDB_VERSION.fb" -m "Rocksdb $ROCKSDB_VERSION" +$TAG="$ROCKSDB_VERSION.fb" +$GIT tag -a "$TAG" -m "Rocksdb $ROCKSDB_VERSION" # Setting up the proxy for remote repo access -export http_proxy=http://172.31.255.99:8080 -export https_proxy="$http_proxy"; - title "Pushing new tag to remote repo ..." -proxycmd.sh git push origin --tags +$GIT push origin --tags -title "Done!" +title "Tag $TAG is pushed to github; if you want to delete it, please run" +title "git tags -d $TAG && git push origin :refs/tags/$TAG"