AIX and Solaris Sparc Support

Summary:
Replacement of #2147

The change was squashed due to a lot of conflicts.
Closes https://github.com/facebook/rocksdb/pull/2194

Differential Revision: D4929799

Pulled By: siying

fbshipit-source-id: 5cd49c254737a1d5ac13f3c035f128e86524c581

CMakeLists.txt

@@ -115,6 +115,15 @@ if(WIN32)
       set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
     endif()
   endif()
+
+  option(WITH_MD_LIBRARY "build with MD" ON)
+  if(MSVC)
+    if(WITH_MD_LIBRARY)
+      set(RUNTIME_LIBRARY "MD")
+    else()
+      set(RUNTIME_LIBRARY "MT")
+    endif()
+  endif()
 else()
   option(WITH_SSE42 "build with SSE4.2" ON)
   if(WITH_SSE42)
@@ -200,11 +209,11 @@ endif()
 if(MSVC)
   if((${OPTIMIZE_DEBUG} EQUAL 1))
     message(STATUS "Debug optimization is enabled")
-    set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /MDd")
+    set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /${RUNTIME_LIBRARY}d")
   else()
-    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /MDd")
+    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /${RUNTIME_LIBRARY}d")
   endif()
-  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /MD")
+  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /${RUNTIME_LIBRARY}")
 
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")

INSTALL.md

@@ -106,3 +106,30 @@ makes use of SSE4, add 'USE_SSE=1' before your make commands, like this: `USE_SS
 * **Windows**:
   * For building with MS Visual Studio 13 you will need Update 4 installed.
   * Read and follow the instructions at CMakeLists.txt
+
+* **AIX 6.1**
+    * Install AIX Toolbox rpms with gcc
+    * Use these environment variables:
+  
+             export PORTABLE=1
+             export CC=gcc
+             export AR="ar -X64"
+             export EXTRA_ARFLAGS=-X64
+             export EXTRA_CFLAGS=-maix64
+             export EXTRA_CXXFLAGS=-maix64
+             export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"
+             export LIBPATH=/opt/freeware/lib
+             export JAVA_HOME=/usr/java8_64
+             export PATH=/opt/freeware/bin:$PATH
+  
+* **Solaris Sparc**
+    * Install GCC 4.8.2 and higher.
+    * Use these environment variables:
+
+             export CC=gcc
+             export EXTRA_CFLAGS=-m64
+             export EXTRA_CXXFLAGS=-m64
+             export EXTRA_LDFLAGS=-m64
+             export PORTABLE=1
+             export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"
+

Makefile

@@ -14,7 +14,8 @@ CFLAGS += ${EXTRA_CFLAGS}
 CXXFLAGS += ${EXTRA_CXXFLAGS}
 LDFLAGS += $(EXTRA_LDFLAGS)
 MACHINE ?= $(shell uname -m)
-ARFLAGS = rs
+ARFLAGS = ${EXTRA_ARFLAGS} rs
+STRIPFLAGS = -S -x
 
 # Transform parallel LOG output into something more readable.
 perl_command = perl -n \
@@ -154,7 +155,9 @@ missing_make_config_paths := $(shell				\
 $(foreach path, $(missing_make_config_paths), \
 	$(warning Warning: $(path) dont exist))
 
-ifneq ($(PLATFORM), IOS)
+ifeq ($(PLATFORM), OS_AIX)
+# no debug info
+else ifneq ($(PLATFORM), IOS)
 CFLAGS += -g
 CXXFLAGS += -g
 else
@@ -162,6 +165,11 @@ else
 OPT += -DNDEBUG
 endif
 
+ifeq ($(PLATFORM), OS_AIX)
+ARFLAGS = -X64 rs
+STRIPFLAGS = -X64 -x
+endif
+
 ifeq ($(PLATFORM), OS_SOLARIS)
 	PLATFORM_CXXFLAGS += -D _GLIBCXX_USE_C99
 endif
@@ -194,6 +202,11 @@ ifdef COMPILE_WITH_TSAN
 	LUA_PATH =
 endif
 
+# AIX doesn't work with -pg
+ifeq ($(PLATFORM), OS_AIX)
+	PROFILING_FLAGS =
+endif
+
 # USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc.
 ifdef COMPILE_WITH_UBSAN
 	DISABLE_JEMALLOC=1
@@ -215,8 +228,14 @@ endif
 export GTEST_THROW_ON_FAILURE=1
 export GTEST_HAS_EXCEPTIONS=1
 GTEST_DIR = ./third-party/gtest-1.7.0/fused-src
-PLATFORM_CCFLAGS += -isystem $(GTEST_DIR)
+# AIX: pre-defined system headers are surrounded by an extern "C" block
-PLATFORM_CXXFLAGS += -isystem $(GTEST_DIR)
+ifeq ($(PLATFORM), OS_AIX)
+	PLATFORM_CCFLAGS += -I$(GTEST_DIR)
+	PLATFORM_CXXFLAGS += -I$(GTEST_DIR)
+else
+	PLATFORM_CCFLAGS += -isystem $(GTEST_DIR)
+	PLATFORM_CXXFLAGS += -isystem $(GTEST_DIR)
+endif
 
 # This (the first rule) must depend on "all".
 default: all
@@ -571,7 +590,8 @@ ifneq (,$(filter check parallel_check,$(MAKECMDGOALS)),)
 # and create a randomly-named rocksdb.XXXX directory therein.
 # We'll use that directory in the "make check" rules.
 ifeq ($(TMPD),)
-TMPD := $(shell f=/dev/shm; test -k $$f || f=/tmp;			\
+TMPDIR := $(shell echo $${TMPDIR:-/tmp})
+TMPD := $(shell f=/dev/shm; test -k $$f || f=$(TMPDIR);     \
   perl -le 'use File::Temp "tempdir";'					\
     -e 'print tempdir("'$$f'/rocksdb.XXXX", CLEANUP => 0)')
 endif
@@ -726,10 +746,12 @@ check: all
 	      echo "===== Running $$t"; ./$$t || exit 1; done;          \
 	fi
 	rm -rf $(TMPD)
+ifneq ($(PLATFORM), OS_AIX)
 ifeq ($(filter -DROCKSDB_LITE,$(OPT)),)
 	python tools/ldb_test.py
 	sh tools/rocksdb_dump_test.sh
 endif
+endif
 
 # TODO add ldb_tests
 check_some: $(SUBSET)
@@ -1402,14 +1424,18 @@ ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PA
 ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar
 SHA256_CMD = sha256sum
 
-ZLIB_VER = 1.2.11
+ZLIB_VER ?= 1.2.11
-ZLIB_SHA256 = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
+ZLIB_SHA256 ?= c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
-BZIP2_VER = 1.0.6
+ZLIB_DOWNLOAD_BASE ?= http://zlib.net
-BZIP2_SHA256 = a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
+BZIP2_VER ?= 1.0.6
-SNAPPY_VER = 1.1.4
+BZIP2_SHA256 ?= a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
-SNAPPY_SHA256 = 134bfe122fd25599bb807bb8130e7ba6d9bdb851e0b16efcb83ac4f5d0b70057
+BZIP2_DOWNLOAD_BASE ?= http://www.bzip.org
-LZ4_VER = 1.7.5
+SNAPPY_VER ?= 1.1.4
-LZ4_SHA256 = 0190cacd63022ccb86f44fa5041dc6c3804407ad61550ca21c382827319e7e7e
+SNAPPY_SHA256 ?= 134bfe122fd25599bb807bb8130e7ba6d9bdb851e0b16efcb83ac4f5d0b70057
+SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/releases/download
+LZ4_VER ?= 1.7.5
+LZ4_SHA256 ?= 0190cacd63022ccb86f44fa5041dc6c3804407ad61550ca21c382827319e7e7e
+LZ4_DOWNLOAD_BASE ?= https://github.com/lz4/lz4/archive
 
 ifeq ($(PLATFORM), OS_MACOSX)
 	ROCKSDBJNILIB = librocksdbjni-osx.jnilib
@@ -1432,47 +1458,53 @@ ifeq ($(PLATFORM), OS_SOLARIS)
 	JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/solaris
 	SHA256_CMD = digest -a sha256
 endif
+ifeq ($(PLATFORM), OS_AIX)
+	JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/aix
+	ROCKSDBJNILIB = librocksdbjni-aix.so
+	EXTRACT_SOURCES = gunzip < TAR_GZ | tar xvf -
+	SNAPPY_MAKE_TARGET = libsnappy.la
+endif
 
 libz.a:
 	-rm -rf zlib-$(ZLIB_VER)
-	curl -O -L http://zlib.net/zlib-$(ZLIB_VER).tar.gz
+	curl -O -L ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz
 	ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \
 		echo zlib-$(ZLIB_VER).tar.gz checksum mismatch, expected=\"$(ZLIB_SHA256)\" actual=\"$$ZLIB_SHA256_ACTUAL\"; \
 		exit 1; \
 	fi
 	tar xvzf zlib-$(ZLIB_VER).tar.gz
-	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC' ./configure --static && make
+	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${EXTRA_CFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --static && make
 	cp zlib-$(ZLIB_VER)/libz.a .
 
 libbz2.a:
 	-rm -rf bzip2-$(BZIP2_VER)
-	curl -O -L http://www.bzip.org/$(BZIP2_VER)/bzip2-$(BZIP2_VER).tar.gz
+	curl -O -L ${BZIP2_DOWNLOAD_BASE}/$(BZIP2_VER)/bzip2-$(BZIP2_VER).tar.gz
 	BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \
 		echo bzip2-$(BZIP2_VER).tar.gz checksum mismatch, expected=\"$(BZIP2_SHA256)\" actual=\"$$BZIP2_SHA256_ACTUAL\"; \
 		exit 1; \
 	fi
 	tar xvzf bzip2-$(BZIP2_VER).tar.gz
-	cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64'
+	cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${EXTRA_CFLAGS}' AR='ar ${EXTRA_ARFLAGS}'
 	cp bzip2-$(BZIP2_VER)/libbz2.a .
 
 libsnappy.a:
 	-rm -rf snappy-$(SNAPPY_VER)
-	curl -O -L https://github.com/google/snappy/releases/download/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz
+	curl -O -L ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz
 	SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \
 		echo snappy-$(SNAPPY_VER).tar.gz checksum mismatch, expected=\"$(SNAPPY_SHA256)\" actual=\"$$SNAPPY_SHA256_ACTUAL\"; \
 		exit 1; \
 	fi
 	tar xvzf snappy-$(SNAPPY_VER).tar.gz
-	cd snappy-$(SNAPPY_VER) && ./configure --with-pic --enable-static
+	cd snappy-$(SNAPPY_VER) && CFLAGS='${EXTRA_CFLAGS}' CXXFLAGS='${EXTRA_CXXFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --with-pic --enable-static --disable-shared
-	cd snappy-$(SNAPPY_VER) && make
+	cd snappy-$(SNAPPY_VER) && make ${SNAPPY_MAKE_TARGET}
 	cp snappy-$(SNAPPY_VER)/.libs/libsnappy.a .
 
 liblz4.a:
 	-rm -rf lz4-$(LZ4_VER)
-	curl -O -L https://github.com/lz4/lz4/archive/v$(LZ4_VER).tar.gz
+	curl -O -L ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz
 	mv v$(LZ4_VER).tar.gz lz4-$(LZ4_VER).tar.gz
 	LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \
@@ -1480,7 +1512,7 @@ liblz4.a:
 		exit 1; \
 	fi
 	tar xvzf lz4-$(LZ4_VER).tar.gz
-	cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC -O2' all
+	cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
 	cp lz4-$(LZ4_VER)/lib/liblz4.a .
 
 # A version of each $(LIBOBJECTS) compiled with -fPIC and a fixed set of static compression libraries
@@ -1504,7 +1536,7 @@ rocksdbjavastatic: $(java_static_libobjects)
 	  -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \
 	  $(java_static_libobjects) $(COVERAGEFLAGS) \
 	  $(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS)
-	cd java/target;strip -S -x $(ROCKSDBJNILIB)
+	cd java/target;strip $(STRIPFLAGS) $(ROCKSDBJNILIB)
 	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
 	cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
 	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class

build_tools/build_detect_platform

@@ -128,10 +128,17 @@ case "$TARGET_OS" in
         ;;
     SunOS)
         PLATFORM=OS_SOLARIS
-        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS"
+        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS -m64"
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -static-libstdc++ -static-libgcc -m64"
         # PORT_FILES=port/sunos/sunos_specific.cc
         ;;
+    AIX)
+        PLATFORM=OS_AIX
+        CC=gcc
+        COMMON_FLAGS="$COMMON_FLAGS -maix64 -pthread -fno-builtin-memcmp -D_REENTRANT -DOS_AIX -D__STDC_FORMAT_MACROS"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread -lpthread -lrt -maix64 -static-libstdc++ -static-libgcc"
+        # PORT_FILES=port/aix/aix_specific.cc
+        ;;
     FreeBSD)
         PLATFORM=OS_FREEBSD
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
@@ -382,6 +389,18 @@ EOF
     if [ "$?" = 0 ]; then
         PROFILING_FLAGS=-pg
     fi
+
+    # Test whether sync_file_range is supported for compatibility with an old glibc
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <fcntl.h>
+      int main() {
+        int fd = open("/dev/null", 0);
+        sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
+      }
+EOF
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT"
+    fi
 fi
 
 # TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning.
@@ -427,7 +446,7 @@ elif test -z "$PORTABLE"; then
     COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
   elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
     COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
-  else
+  elif [ "$TARGET_OS" != AIX ] && [ "$TARGET_OS" != SunOS ]; then
     COMMON_FLAGS="$COMMON_FLAGS -march=native "
   fi
 fi

db/cuckoo_table_db_test.cc

@@ -320,8 +320,14 @@ TEST_F(CuckooTableDBTest, AdaptiveTable) {
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {
+  if (rocksdb::port::kLittleEndian) {
     ::testing::InitGoogleTest(&argc, argv);
     return RUN_ALL_TESTS();
+  }
+  else {
+    fprintf(stderr, "SKIPPED as Cuckoo table doesn't support Big Endian\n");
+    return 0;
+  }
 }
 
 #else

db/db_sst_test.cc

@@ -297,7 +297,7 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
   std::vector<uint64_t> penalties;
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "DeleteScheduler::BackgroundEmptyTrash:Wait",
-      [&](void* arg) { penalties.push_back(*(static_cast<int*>(arg))); });
+      [&](void* arg) { penalties.push_back(*(static_cast<uint64_t*>(arg))); });
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "InstrumentedCondVar::TimedWaitInternal", [&](void* arg) {
         // Turn timed wait into a simulated sleep

db/db_test2.cc

@@ -1516,6 +1516,7 @@ class MockPersistentCache : public PersistentCache {
   const size_t max_size_ = 10 * 1024;  // 10KiB
 };
 
+#ifndef OS_SOLARIS // GetUniqueIdFromFile is not implemented
 TEST_F(DBTest2, PersistentCache) {
   int num_iter = 80;
 
@@ -1579,6 +1580,7 @@ TEST_F(DBTest2, PersistentCache) {
     }
   }
 }
+#endif // !OS_SOLARIS
 
 namespace {
 void CountSyncPoint() {
@@ -1702,6 +1704,7 @@ TEST_F(DBTest2, ReadAmpBitmap) {
             options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES));
 }
 
+#ifndef OS_SOLARIS // GetUniqueIdFromFile is not implemented
 TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
   if (dbname_.find("dev/shm") != std::string::npos) {
     // /dev/shm dont support getting a unique file id, this mean that
@@ -1786,6 +1789,7 @@ TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
   ASSERT_EQ(total_useful_bytes_iter1 + total_useful_bytes_iter2,
             total_loaded_bytes_iter1 + total_loaded_bytes_iter2);
 }
+#endif // !OS_SOLARIS
 
 #ifndef ROCKSDB_LITE
 TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {

db/external_sst_file_ingestion_job.cc

@@ -7,7 +7,9 @@
 
 #include "db/external_sst_file_ingestion_job.h"
 
+#ifndef __STDC_FORMAT_MACROS
 #define __STDC_FORMAT_MACROS
+#endif
 
 #include <inttypes.h>
 #include <algorithm>

db/internal_stats.h

@@ -49,6 +49,7 @@ struct DBPropertyInfo {
 extern const DBPropertyInfo* GetPropertyInfo(const Slice& property);
 
 #ifndef ROCKSDB_LITE
+#undef SCORE
 enum class LevelStatType {
   INVALID = 0,
   NUM_FILES,

db/merge_test.cc

@@ -141,7 +141,9 @@ class Counters {
   // mapped to a levedb Put
   bool set(const std::string& key, uint64_t value) {
     // just treat the internal rep of int64 as the string
-    Slice slice((char *)&value, sizeof(value));
+    char buf[sizeof(value)];
+    EncodeFixed64(buf, value);
+    Slice slice(buf, sizeof(value));
     auto s = db_->Put(put_option_, key, slice);
 
     if (s.ok()) {
@@ -375,7 +377,9 @@ void testSingleBatchSuccessiveMerge(DB* db, size_t max_num_merges,
 
   Slice key("BatchSuccessiveMerge");
   uint64_t merge_value = 1;
-  Slice merge_value_slice((char *)&merge_value, sizeof(merge_value));
+  char buf[sizeof(merge_value)];
+  EncodeFixed64(buf, merge_value);
+  Slice merge_value_slice(buf, sizeof(merge_value));
 
   // Create the batch
   WriteBatch batch;

db/perf_context_test.cc

@@ -633,18 +633,33 @@ TEST_F(PerfContextTest, MergeOperatorTime) {
   SetPerfLevel(kEnableTime);
   perf_context.Reset();
   ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
+#ifdef OS_SOLARIS
+  for (int i = 0; i < 100; i++) {
+    ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
+  }
+#endif
   EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
 
   ASSERT_OK(db->Flush(FlushOptions()));
 
   perf_context.Reset();
   ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
+#ifdef OS_SOLARIS
+  for (int i = 0; i < 100; i++) {
+    ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
+  }
+#endif
   EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
 
   ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr));
 
   perf_context.Reset();
   ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
+#ifdef OS_SOLARIS
+  for (int i = 0; i < 100; i++) {
+    ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
+  }
+#endif
   EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
 
   delete db;

db/prefix_test.cc

@@ -32,6 +32,7 @@ int main() {
 #include "util/string_util.h"
 #include "util/testharness.h"
 #include "utilities/merge_operators.h"
+#include "util/coding.h"
 
 using GFLAGS::ParseCommandLineFlags;
 
@@ -65,12 +66,16 @@ struct TestKey {
 };
 
 // return a slice backed by test_key
-inline Slice TestKeyToSlice(const TestKey& test_key) {
+inline Slice TestKeyToSlice(std::string &s, const TestKey& test_key) {
-  return Slice((const char*)&test_key, sizeof(test_key));
+  s.clear();
+  PutFixed64(&s, test_key.prefix);
+  PutFixed64(&s, test_key.sorted);
+  return Slice(s.c_str(), s.size());
 }
 
-inline const TestKey* SliceToTestKey(const Slice& slice) {
+inline const TestKey SliceToTestKey(const Slice& slice) {
-  return (const TestKey*)slice.data();
+  return TestKey(DecodeFixed64(slice.data()),
+    DecodeFixed64(slice.data() + 8));
 }
 
 class TestKeyComparator : public Comparator {
@@ -79,8 +84,10 @@ class TestKeyComparator : public Comparator {
   // Compare needs to be aware of the possibility of a and/or b is
   // prefix only
   virtual int Compare(const Slice& a, const Slice& b) const override {
-    const TestKey* key_a = SliceToTestKey(a);
+    const TestKey kkey_a = SliceToTestKey(a);
-    const TestKey* key_b = SliceToTestKey(b);
+    const TestKey kkey_b = SliceToTestKey(b);
+    const TestKey *key_a = &kkey_a;
+    const TestKey *key_b = &kkey_b;
     if (key_a->prefix != key_b->prefix) {
       if (key_a->prefix < key_b->prefix) return -1;
       if (key_a->prefix > key_b->prefix) return 1;
@@ -111,7 +118,8 @@ class TestKeyComparator : public Comparator {
   }
 
   bool operator()(const TestKey& a, const TestKey& b) const {
-    return Compare(TestKeyToSlice(a), TestKeyToSlice(b)) < 0;
+    std::string sa, sb;
+    return Compare(TestKeyToSlice(sa, a), TestKeyToSlice(sb, b)) < 0;
   }
 
   virtual const char* Name() const override {
@@ -128,30 +136,35 @@ namespace {
 void PutKey(DB* db, WriteOptions write_options, uint64_t prefix,
             uint64_t suffix, const Slice& value) {
   TestKey test_key(prefix, suffix);
-  Slice key = TestKeyToSlice(test_key);
+  std::string s;
+  Slice key = TestKeyToSlice(s, test_key);
   ASSERT_OK(db->Put(write_options, key, value));
 }
 
 void PutKey(DB* db, WriteOptions write_options, const TestKey& test_key,
             const Slice& value) {
-  Slice key = TestKeyToSlice(test_key);
+  std::string s;
+  Slice key = TestKeyToSlice(s, test_key);
   ASSERT_OK(db->Put(write_options, key, value));
 }
 
 void MergeKey(DB* db, WriteOptions write_options, const TestKey& test_key,
               const Slice& value) {
-  Slice key = TestKeyToSlice(test_key);
+  std::string s;
+  Slice key = TestKeyToSlice(s, test_key);
   ASSERT_OK(db->Merge(write_options, key, value));
 }
 
 void DeleteKey(DB* db, WriteOptions write_options, const TestKey& test_key) {
-  Slice key = TestKeyToSlice(test_key);
+  std::string s;
+  Slice key = TestKeyToSlice(s, test_key);
   ASSERT_OK(db->Delete(write_options, key));
 }
 
 void SeekIterator(Iterator* iter, uint64_t prefix, uint64_t suffix) {
   TestKey test_key(prefix, suffix);
-  Slice key = TestKeyToSlice(test_key);
+  std::string s;
+  Slice key = TestKeyToSlice(s, test_key);
   iter->Seek(key);
 }
 
@@ -160,7 +173,8 @@ const std::string kNotFoundResult = "NOT_FOUND";
 std::string Get(DB* db, const ReadOptions& read_options, uint64_t prefix,
                 uint64_t suffix) {
   TestKey test_key(prefix, suffix);
-  Slice key = TestKeyToSlice(test_key);
+  std::string s2;
+  Slice key = TestKeyToSlice(s2, test_key);
 
   std::string result;
   Status s = db->Get(read_options, key, &result);
@@ -527,7 +541,9 @@ TEST_F(PrefixTest, PrefixValid) {
       PutKey(db.get(), write_options, 12345, 9, v19);
       PutKey(db.get(), write_options, 12346, 8, v16);
       db->Flush(FlushOptions());
-      db->Delete(write_options, TestKeyToSlice(TestKey(12346, 8)));
+      TestKey test_key(12346, 8);
+      std::string s;
+      db->Delete(write_options, TestKeyToSlice(s, test_key));
       db->Flush(FlushOptions());
       read_options.prefix_same_as_start = true;
       std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
@@ -583,7 +599,8 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
        for (uint64_t sorted = 0; sorted < FLAGS_items_per_prefix; sorted++) {
         TestKey test_key(prefix, sorted);
 
-        Slice key = TestKeyToSlice(test_key);
+        std::string s;
+        Slice key = TestKeyToSlice(s, test_key);
         std::string value(FLAGS_value_size, 0);
 
         perf_context.Reset();
@@ -605,7 +622,8 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
 
     for (auto prefix : prefixes) {
       TestKey test_key(prefix, FLAGS_items_per_prefix / 2);
-      Slice key = TestKeyToSlice(test_key);
+      std::string s;
+      Slice key = TestKeyToSlice(s, test_key);
       std::string value = "v" + ToString(0);
 
       perf_context.Reset();
@@ -639,7 +657,8 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
          prefix < FLAGS_total_prefixes + 10000;
          prefix++) {
       TestKey test_key(prefix, 0);
-      Slice key = TestKeyToSlice(test_key);
+      std::string s;
+      Slice key = TestKeyToSlice(s, test_key);
 
       perf_context.Reset();
       StopWatchNano timer(Env::Default(), true);
@@ -725,8 +744,8 @@ TEST_F(PrefixTest, PrefixSeekModePrev) {
       ASSERT_TRUE(iter->Valid());
       if (FLAGS_enable_print) {
         std::cout << "round " << prefix
-                  << " iter: " << SliceToTestKey(iter->key())->prefix
+                  << " iter: " << SliceToTestKey(iter->key()).prefix
-                  << SliceToTestKey(iter->key())->sorted
+                  << SliceToTestKey(iter->key()).sorted
                   << " | map: " << it->first.prefix << it->first.sorted << " | "
                   << iter->value().ToString() << " " << it->second << std::endl;
       }
@@ -747,16 +766,16 @@ TEST_F(PrefixTest, PrefixSeekModePrev) {
           }
         }
         if (!iter->Valid() ||
-            SliceToTestKey(iter->key())->prefix != stored_prefix) {
+            SliceToTestKey(iter->key()).prefix != stored_prefix) {
           break;
         }
-        stored_prefix = SliceToTestKey(iter->key())->prefix;
+        stored_prefix = SliceToTestKey(iter->key()).prefix;
         ASSERT_TRUE(iter->Valid());
         ASSERT_NE(it, whole_map.end());
         ASSERT_EQ(iter->value(), it->second);
         if (FLAGS_enable_print) {
-          std::cout << "iter: " << SliceToTestKey(iter->key())->prefix
+          std::cout << "iter: " << SliceToTestKey(iter->key()).prefix
-                    << SliceToTestKey(iter->key())->sorted
+                    << SliceToTestKey(iter->key()).sorted
                     << " | map: " << it->first.prefix << it->first.sorted
                     << " | " << iter->value().ToString() << " " << it->second
                     << std::endl;
@@ -810,7 +829,8 @@ TEST_F(PrefixTest, PrefixSeekModePrev3) {
   options.write_buffer_size = 1024 * 1024;
   std::string v14("v14");
   TestKey upper_bound_key = TestKey(1, 5);
-  Slice upper_bound = TestKeyToSlice(upper_bound_key);
+  std::string s;
+  Slice upper_bound = TestKeyToSlice(s, upper_bound_key);
 
   {
     DestroyDB(kDbName, Options());

env/env_chroot.cc

@@ -24,11 +24,18 @@ class ChrootEnv : public EnvWrapper {
  public:
   ChrootEnv(Env* base_env, const std::string& chroot_dir)
       : EnvWrapper(base_env) {
+#if defined(OS_AIX)
+    char resolvedName[PATH_MAX];
+    char* real_chroot_dir = realpath(chroot_dir.c_str(), resolvedName);
+#else
     char* real_chroot_dir = realpath(chroot_dir.c_str(), nullptr);
+#endif
     // chroot_dir must exist so realpath() returns non-nullptr.
     assert(real_chroot_dir != nullptr);
     chroot_dir_ = real_chroot_dir;
+#if !defined(OS_AIX)
     free(real_chroot_dir);
+#endif
   }
 
   virtual Status NewSequentialFile(const std::string& fname,
@@ -258,7 +265,12 @@ class ChrootEnv : public EnvWrapper {
     }
     std::pair<Status, std::string> res;
     res.second = chroot_dir_ + path;
+#if defined(OS_AIX)
+    char resolvedName[PATH_MAX];
+    char* normalized_path = realpath(res.second.c_str(), resolvedName);
+#else
     char* normalized_path = realpath(res.second.c_str(), nullptr);
+#endif
     if (normalized_path == nullptr) {
       res.first = Status::NotFound(res.second, strerror(errno));
     } else if (strlen(normalized_path) < chroot_dir_.size() ||
@@ -269,7 +281,9 @@ class ChrootEnv : public EnvWrapper {
     } else {
       res.first = Status::OK();
     }
+#if !defined(OS_AIX)
     free(normalized_path);
+#endif
     return res;
   }
 

env/env_posix.cc

@@ -159,7 +159,7 @@ class PosixEnv : public Env {
 #ifdef ROCKSDB_LITE
       return Status::IOError(fname, "Direct I/O not supported in RocksDB lite");
 #endif  // !ROCKSDB_LITE
-#if !defined(OS_MACOSX) && !defined(OS_OPENBSD)
+#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
       flags |= O_DIRECT;
 #endif
     }
@@ -206,7 +206,7 @@ class PosixEnv : public Env {
 #ifdef ROCKSDB_LITE
       return Status::IOError(fname, "Direct I/O not supported in RocksDB lite");
 #endif  // !ROCKSDB_LITE
-#if !defined(OS_MACOSX) && !defined(OS_OPENBSD)
+#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
       flags |= O_DIRECT;
       TEST_SYNC_POINT_CALLBACK("NewRandomAccessFile:O_DIRECT", &flags);
 #endif
@@ -271,7 +271,7 @@ class PosixEnv : public Env {
       return Status::IOError(fname, "Direct I/O not supported in RocksDB lite");
 #endif  // ROCKSDB_LITE
       flags |= O_WRONLY;
-#if !defined(OS_MACOSX) && !defined(OS_OPENBSD)
+#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
       flags |= O_DIRECT;
 #endif
       TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags);
@@ -312,6 +312,14 @@ class PosixEnv : public Env {
         s = IOError(fname, errno);
         return s;
       }
+#elif defined(OS_SOLARIS)
+      if (directio(fd, DIRECTIO_ON) == -1) {
+        if (errno != ENOTTY) { // ZFS filesystems don't support DIRECTIO_ON
+          close(fd);
+          s = IOError(fname, errno);
+          return s;
+        }
+      }
 #endif
       result->reset(new PosixWritableFile(fname, fd, options));
     } else {
@@ -338,7 +346,7 @@ class PosixEnv : public Env {
       return Status::IOError(fname, "Direct I/O not supported in RocksDB lite");
 #endif  // !ROCKSDB_LITE
       flags |= O_WRONLY;
-#if !defined(OS_MACOSX) && !defined(OS_OPENBSD)
+#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
       flags |= O_DIRECT;
 #endif
       TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags);
@@ -385,6 +393,14 @@ class PosixEnv : public Env {
         s = IOError(fname, errno);
         return s;
       }
+#elif defined(OS_SOLARIS)
+      if (directio(fd, DIRECTIO_ON) == -1) {
+        if (errno != ENOTTY) { // ZFS filesystems don't support DIRECTIO_ON
+          close(fd);
+          s = IOError(fname, errno);
+          return s;
+        }
+      }
 #endif
       result->reset(new PosixWritableFile(fname, fd, options));
     } else {
@@ -668,10 +684,12 @@ class PosixEnv : public Env {
   }
 
   virtual uint64_t NowNanos() override {
-#if defined(OS_LINUX) || defined(OS_FREEBSD)
+#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_AIX)
     struct timespec ts;
     clock_gettime(CLOCK_MONOTONIC, &ts);
     return static_cast<uint64_t>(ts.tv_sec) * 1000000000 + ts.tv_nsec;
+#elif defined(OS_SOLARIS)
+    return gethrtime();
 #elif defined(__MACH__)
     clock_serv_t cclock;
     mach_timespec_t ts;

env/env_test.cc

@@ -958,7 +958,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
     // Create file.
     {
       unique_ptr<WritableFile> wfile;
-#if !defined(OS_MACOSX) && !defined(OS_WIN)
+#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX)
       if (soptions.use_direct_writes) {
         soptions.use_direct_writes = false;
       }
@@ -974,7 +974,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
       unique_ptr<RandomAccessFile> file;
       auto scratch = NewAligned(kSectorSize, 0);
       Slice result;
-#if !defined(OS_MACOSX) && !defined(OS_WIN)
+#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX)
       if (soptions.use_direct_reads) {
         soptions.use_direct_reads = false;
       }
@@ -991,7 +991,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
       unique_ptr<SequentialFile> file;
       auto scratch = NewAligned(kSectorSize, 0);
       Slice result;
-#if !defined(OS_MACOSX) && !defined(OS_WIN)
+#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX)
       if (soptions.use_direct_reads) {
         soptions.use_direct_reads = false;
       }
@@ -1136,7 +1136,7 @@ TEST_P(EnvPosixTestWithParam, Preallocation) {
     unique_ptr<WritableFile> srcfile;
     EnvOptions soptions;
     soptions.use_direct_reads = soptions.use_direct_writes = direct_io_;
-#if !defined(OS_MACOSX) && !defined(OS_WIN)
+#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX)
     if (soptions.use_direct_writes) {
       rocksdb::SyncPoint::GetInstance()->SetCallBack(
           "NewWritableFile:O_DIRECT", [&](void* arg) {
@@ -1198,7 +1198,7 @@ TEST_P(EnvPosixTestWithParam, ConsistentChildrenAttributes) {
       oss << test::TmpDir(env_) << "/testfile_" << i;
       const std::string path = oss.str();
       unique_ptr<WritableFile> file;
-#if !defined(OS_MACOSX) && !defined(OS_WIN)
+#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX)
       if (soptions.use_direct_writes) {
         rocksdb::SyncPoint::GetInstance()->SetCallBack(
             "NewWritableFile:O_DIRECT", [&](void* arg) {

env/io_posix.cc

@@ -267,7 +267,7 @@ size_t PosixHelper::GetUniqueIdFromFile(int fd, char* id, size_t max_size) {
 }
 #endif
 
-#if defined(OS_MACOSX)
+#if defined(OS_MACOSX) || defined(OS_AIX)
 size_t PosixHelper::GetUniqueIdFromFile(int fd, char* id, size_t max_size) {
   if (max_size < kMaxVarint64Length * 3) {
     return 0;
@@ -336,7 +336,7 @@ Status PosixRandomAccessFile::Read(uint64_t offset, size_t n, Slice* result,
   return s;
 }
 
-#if defined(OS_LINUX) || defined(OS_MACOSX)
+#if defined(OS_LINUX) || defined(OS_MACOSX) || defined(OS_AIX)
 size_t PosixRandomAccessFile::GetUniqueId(char* id, size_t max_size) const {
   return PosixHelper::GetUniqueIdFromFile(fd_, id, max_size);
 }
@@ -848,7 +848,7 @@ Status PosixWritableFile::Allocate(uint64_t offset, uint64_t len) {
 }
 #endif
 
-#ifdef OS_LINUX
+#ifdef ROCKSDB_RANGESYNC_PRESENT
 Status PosixWritableFile::RangeSync(uint64_t offset, uint64_t nbytes) {
   assert(offset <= std::numeric_limits<off_t>::max());
   assert(nbytes <= std::numeric_limits<off_t>::max());
@@ -859,7 +859,9 @@ Status PosixWritableFile::RangeSync(uint64_t offset, uint64_t nbytes) {
     return IOError(filename_, errno);
   }
 }
+#endif
 
+#ifdef OS_LINUX
 size_t PosixWritableFile::GetUniqueId(char* id, size_t max_size) const {
   return PosixHelper::GetUniqueIdFromFile(fd_, id, max_size);
 }
@@ -961,9 +963,11 @@ Status PosixRandomRWFile::Close() {
 PosixDirectory::~PosixDirectory() { close(fd_); }
 
 Status PosixDirectory::Fsync() {
+#ifndef OS_AIX
   if (fsync(fd_) == -1) {
     return IOError("directory", errno);
   }
+#endif
   return Status::OK();
 }
 }  // namespace rocksdb

env/io_posix.h

@@ -15,7 +15,7 @@
 
 // For non linux platform, the following macros are used only as place
 // holder.
-#if !(defined OS_LINUX) && !(defined CYGWIN)
+#if !(defined OS_LINUX) && !(defined CYGWIN) && !(defined OS_AIX)
 #define POSIX_FADV_NORMAL 0     /* [MC1] no further special treatment */
 #define POSIX_FADV_RANDOM 1     /* [MC1] expect random page refs */
 #define POSIX_FADV_SEQUENTIAL 2 /* [MC1] expect sequential page refs */
@@ -79,7 +79,7 @@ class PosixRandomAccessFile : public RandomAccessFile {
 
   virtual Status Read(uint64_t offset, size_t n, Slice* result,
                       char* scratch) const override;
-#if defined(OS_LINUX) || defined(OS_MACOSX)
+#if defined(OS_LINUX) || defined(OS_MACOSX) || defined(OS_AIX)
   virtual size_t GetUniqueId(char* id, size_t max_size) const override;
 #endif
   virtual void Hint(AccessPattern pattern) override;
@@ -126,8 +126,10 @@ class PosixWritableFile : public WritableFile {
 #ifdef ROCKSDB_FALLOCATE_PRESENT
   virtual Status Allocate(uint64_t offset, uint64_t len) override;
 #endif
-#ifdef OS_LINUX
+#ifdef ROCKSDB_RANGESYNC_PRESENT
   virtual Status RangeSync(uint64_t offset, uint64_t nbytes) override;
+#endif
+#ifdef OS_LINUX
   virtual size_t GetUniqueId(char* id, size_t max_size) const override;
 #endif
 };

java/CMakeLists.txt

@@ -1,25 +1,31 @@
 cmake_minimum_required(VERSION 2.6)
 
 set(JNI_NATIVE_SOURCES
-        rocksjni/backupenginejni.cc
         rocksjni/backupablejni.cc
+        rocksjni/backupenginejni.cc
         rocksjni/checkpoint.cc
+        rocksjni/clock_cache.cc
         rocksjni/columnfamilyhandle.cc
         rocksjni/compaction_filter.cc
+        rocksjni/compaction_options_fifo.cc
+        rocksjni/compaction_options_universal.cc
         rocksjni/comparator.cc
         rocksjni/comparatorjnicallback.cc
+        rocksjni/compression_options.cc
         rocksjni/env.cc
         rocksjni/env_options.cc
         rocksjni/external_sst_file_info.cc
         rocksjni/filter.cc
         rocksjni/iterator.cc
         rocksjni/loggerjnicallback.cc
+        rocksjni/lru_cache.cc
         rocksjni/memtablejni.cc
         rocksjni/merge_operator.cc
         rocksjni/options.cc
         rocksjni/ratelimiterjni.cc
         rocksjni/remove_emptyvalue_compactionfilterjni.cc
         rocksjni/restorejni.cc
+        rocksjni/rocksdb_exception_test.cc
         rocksjni/rocksjni.cc
         rocksjni/slice.cc
         rocksjni/snapshot.cc
@@ -29,38 +35,50 @@ set(JNI_NATIVE_SOURCES
         rocksjni/transaction_log.cc
         rocksjni/ttl.cc
         rocksjni/write_batch.cc
-        rocksjni/writebatchhandlerjnicallback.cc
-        rocksjni/write_batch_with_index.cc
         rocksjni/write_batch_test.cc
+        rocksjni/write_batch_with_index.cc
+        rocksjni/writebatchhandlerjnicallback.cc
 )
 
 set(NATIVE_JAVA_CLASSES
         org.rocksdb.AbstractCompactionFilter
         org.rocksdb.AbstractComparator
+        org.rocksdb.AbstractImmutableNativeReference
+        org.rocksdb.AbstractNativeReference
+        org.rocksdb.AbstractRocksIterator
         org.rocksdb.AbstractSlice
-        org.rocksdb.BackupEngine
+        org.rocksdb.AbstractWriteBatch
         org.rocksdb.BackupableDB
         org.rocksdb.BackupableDBOptions
+        org.rocksdb.BackupEngine
+        org.rocksdb.BackupEngineTest
         org.rocksdb.BlockBasedTableConfig
         org.rocksdb.BloomFilter
+        org.rocksdb.Cache
         org.rocksdb.Checkpoint
+        org.rocksdb.ClockCache
         org.rocksdb.ColumnFamilyHandle
         org.rocksdb.ColumnFamilyOptions
+        org.rocksdb.CompactionOptionsFIFO
+        org.rocksdb.CompactionOptionsUniversal
         org.rocksdb.Comparator
         org.rocksdb.ComparatorOptions
+        org.rocksdb.CompressionOptions
         org.rocksdb.DBOptions
         org.rocksdb.DirectComparator
         org.rocksdb.DirectSlice
         org.rocksdb.Env
         org.rocksdb.EnvOptions
         org.rocksdb.ExternalSstFileInfo
-        org.rocksdb.FlushOptions
         org.rocksdb.Filter
-        org.rocksdb.GenericRateLimiterConfig
+        org.rocksdb.FlushOptions
         org.rocksdb.HashLinkedListMemTableConfig
         org.rocksdb.HashSkipListMemTableConfig
         org.rocksdb.Logger
+        org.rocksdb.LRUCache
+        org.rocksdb.MemTableConfig
         org.rocksdb.MergeOperator
+        org.rocksdb.NativeLibraryLoader
         org.rocksdb.Options
         org.rocksdb.PlainTableConfig
         org.rocksdb.RateLimiter
@@ -69,25 +87,31 @@ set(NATIVE_JAVA_CLASSES
         org.rocksdb.RestoreBackupableDB
         org.rocksdb.RestoreOptions
         org.rocksdb.RocksDB
+        org.rocksdb.RocksDBExceptionTest
         org.rocksdb.RocksEnv
         org.rocksdb.RocksIterator
+        org.rocksdb.RocksIteratorInterface
         org.rocksdb.RocksMemEnv
+        org.rocksdb.RocksMutableObject
+        org.rocksdb.RocksObject
         org.rocksdb.SkipListMemTableConfig
         org.rocksdb.Slice
+        org.rocksdb.Snapshot
+        org.rocksdb.SnapshotTest
         org.rocksdb.SstFileWriter
         org.rocksdb.Statistics
+        org.rocksdb.StringAppendOperator
+        org.rocksdb.TableFormatConfig
         org.rocksdb.TransactionLogIterator
         org.rocksdb.TtlDB
         org.rocksdb.VectorMemTableConfig
-        org.rocksdb.Snapshot
+        org.rocksdb.WBWIRocksIterator
-        org.rocksdb.StringAppendOperator
         org.rocksdb.WriteBatch
         org.rocksdb.WriteBatch.Handler
-        org.rocksdb.WriteOptions
-        org.rocksdb.WriteBatchWithIndex
-        org.rocksdb.WBWIRocksIterator
         org.rocksdb.WriteBatchTest
         org.rocksdb.WriteBatchTestInternalHelper
+        org.rocksdb.WriteBatchWithIndex
+        org.rocksdb.WriteOptions
 )
 
 include_directories($ENV{JAVA_HOME}/include)
@@ -111,9 +135,17 @@ if(NOT EXISTS ${JAVA_TEST_LIBDIR})
   file(MAKE_DIRECTORY mkdir ${JAVA_TEST_LIBDIR})
 endif()
 
+if (DEFINED CUSTOM_REPO_URL)
+  set(SEARCH_REPO_URL ${CUSTOM_REPO_URL}/)
+  set(CENTRAL_REPO_URL ${CUSTOM_REPO_URL}/)
+else ()
+  set(SEARCH_REPO_URL "http://search.maven.org/remotecontent?filepath=")
+  set(CENTRAL_REPO_URL "http://central.maven.org/maven2/")
+endif()
+
 if(NOT EXISTS ${JAVA_JUNIT_JAR})
   message("Downloading ${JAVA_JUNIT_JAR}")
-  file(DOWNLOAD http://search.maven.org/remotecontent?filepath=junit/junit/4.12/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
+  file(DOWNLOAD ${SEARCH_REPO_URL}junit/junit/4.12/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
   list(GET downloadStatus 0 error_code)
   if(NOT error_code EQUAL 0)
     message(FATAL_ERROR "Failed downloading ${JAVA_JUNIT_JAR}")
@@ -122,7 +154,7 @@ if(NOT EXISTS ${JAVA_JUNIT_JAR})
 endif()
 if(NOT EXISTS ${JAVA_HAMCR_JAR})
   message("Downloading ${JAVA_HAMCR_JAR}")
-  file(DOWNLOAD http://search.maven.org/remotecontent?filepath=org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
+  file(DOWNLOAD ${SEARCH_REPO_URL}org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
   list(GET downloadStatus 0 error_code)
   if(NOT error_code EQUAL 0)
     message(FATAL_ERROR "Failed downloading ${JAVA_HAMCR_JAR}")
@@ -131,7 +163,7 @@ if(NOT EXISTS ${JAVA_HAMCR_JAR})
 endif()
 if(NOT EXISTS ${JAVA_MOCKITO_JAR})
   message("Downloading ${JAVA_MOCKITO_JAR}")
-  file(DOWNLOAD http://search.maven.org/remotecontent?filepath=org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
+  file(DOWNLOAD ${SEARCH_REPO_URL}org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
   list(GET downloadStatus 0 error_code)
   if(NOT error_code EQUAL 0)
     message(FATAL_ERROR "Failed downloading ${JAVA_MOCKITO_JAR}")
@@ -140,7 +172,7 @@ if(NOT EXISTS ${JAVA_MOCKITO_JAR})
 endif()
 if(NOT EXISTS ${JAVA_CGLIB_JAR})
   message("Downloading ${JAVA_CGLIB_JAR}")
-  file(DOWNLOAD http://search.maven.org/remotecontent?filepath=cglib/cglib/2.2.2/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
+  file(DOWNLOAD ${SEARCH_REPO_URL}cglib/cglib/2.2.2/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
   list(GET downloadStatus 0 error_code)
   if(NOT error_code EQUAL 0)
     message(FATAL_ERROR "Failed downloading ${JAVA_CGLIB_JAR}")
@@ -149,7 +181,7 @@ if(NOT EXISTS ${JAVA_CGLIB_JAR})
 endif()
 if(NOT EXISTS ${JAVA_ASSERTJ_JAR})
   message("Downloading ${JAVA_ASSERTJ_JAR}")
-  file(DOWNLOAD http://central.maven.org/maven2/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
+  file(DOWNLOAD ${CENTRAL_REPO_URL}org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
   list(GET downloadStatus 0 error_code)
   if(NOT error_code EQUAL 0)
     message(FATAL_ERROR "Failed downloading ${JAVA_ASSERTJ_JAR}")

java/Makefile

@@ -157,6 +157,9 @@ ifneq ($(DEBUG_LEVEL),0)
 	JAVA_ARGS = -ea -Xcheck:jni
 endif
 
+SEARCH_REPO_URL?=http://search.maven.org/remotecontent?filepath=
+CENTRAL_REPO_URL?=http://central.maven.org/maven2/
+
 clean:
 	$(AM_V_at)rm -rf include/*
 	$(AM_V_at)rm -rf test-libs/
@@ -198,11 +201,11 @@ column_family_sample: java
 
 resolve_test_deps:
 	test -d "$(JAVA_TEST_LIBDIR)" || mkdir -p "$(JAVA_TEST_LIBDIR)"
-	test -s "$(JAVA_JUNIT_JAR)" || cp $(MVN_LOCAL)/junit/junit/4.12/junit-4.12.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_JUNIT_JAR) http://search.maven.org/remotecontent?filepath=junit/junit/4.12/junit-4.12.jar
+	test -s "$(JAVA_JUNIT_JAR)" || cp $(MVN_LOCAL)/junit/junit/4.12/junit-4.12.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_JUNIT_JAR) $(SEARCH_REPO_URL)junit/junit/4.12/junit-4.12.jar
-	test -s "$(JAVA_HAMCR_JAR)" || cp $(MVN_LOCAL)/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_HAMCR_JAR) http://search.maven.org/remotecontent?filepath=org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar
+	test -s "$(JAVA_HAMCR_JAR)" || cp $(MVN_LOCAL)/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_HAMCR_JAR) $(SEARCH_REPO_URL)org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar
-	test -s "$(JAVA_MOCKITO_JAR)" || cp $(MVN_LOCAL)/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_MOCKITO_JAR)" http://search.maven.org/remotecontent?filepath=org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar
+	test -s "$(JAVA_MOCKITO_JAR)" || cp $(MVN_LOCAL)/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_MOCKITO_JAR)" $(SEARCH_REPO_URL)org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar
-	test -s "$(JAVA_CGLIB_JAR)" || cp $(MVN_LOCAL)/cglib/cglib/2.2.2/cglib-2.2.2.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_CGLIB_JAR)" http://search.maven.org/remotecontent?filepath=cglib/cglib/2.2.2/cglib-2.2.2.jar
+	test -s "$(JAVA_CGLIB_JAR)" || cp $(MVN_LOCAL)/cglib/cglib/2.2.2/cglib-2.2.2.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_CGLIB_JAR)" $(SEARCH_REPO_URL)cglib/cglib/2.2.2/cglib-2.2.2.jar
-	test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_ASSERTJ_JAR)" http://central.maven.org/maven2/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar
+	test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_ASSERTJ_JAR)" $(CENTRAL_REPO_URL)org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar
 
 java_test: java resolve_test_deps
 	$(AM_V_GEN)mkdir -p $(TEST_CLASSES)

java/rocksjni/write_batch.cc

@@ -21,7 +21,6 @@
 #include "rocksjni/writebatchhandlerjnicallback.h"
 #include "table/scoped_arena_iterator.h"
 #include "util/logging.h"
-#include "util/testharness.h"
 
 /*
  * Class:     org_rocksdb_WriteBatch

java/src/main/java/org/rocksdb/util/Environment.java

@@ -16,10 +16,13 @@ public class Environment {
     return (OS.contains("mac"));
   }
 
+  public static boolean isAix() {
+    return OS.contains("aix");
+  }
+  
   public static boolean isUnix() {
-    return (OS.contains("nix") ||
+    return OS.contains("nix") ||
-        OS.contains("nux") ||
+        OS.contains("nux");
-        OS.contains("aix"));
   }
 
   public static boolean isSolaris() {
@@ -27,6 +30,9 @@ public class Environment {
   }
 
   public static boolean is64Bit() {
+    if (ARCH.indexOf("sparcv9") >= 0) {
+      return true;
+    }
     return (ARCH.indexOf("64") > 0);
   }
 
@@ -48,6 +54,8 @@ public class Environment {
       }
     } else if (isMac()) {
       return String.format("%sjni-osx", name);
+    } else if (isAix() && is64Bit()) {
+      return String.format("%sjni-aix64", name);
     } else if (isSolaris()) {
       final String arch = is64Bit() ? "64" : "32";
       return String.format("%sjni-solaris%s", name, arch);
@@ -63,7 +71,7 @@ public class Environment {
   }
 
   private static String appendLibOsSuffix(final String libraryFileName, final boolean shared) {
-    if (isUnix() || isSolaris()) {
+    if (isUnix() || isAix() || isSolaris()) {
       return libraryFileName + ".so";
     } else if (isMac()) {
       return libraryFileName + (shared ? ".dylib" : ".jnilib");

java/src/test/java/org/rocksdb/RocksDBTest.java

@@ -53,7 +53,7 @@ public class RocksDBTest {
       } catch (final RocksDBException e) {
         assertThat(e.getStatus().getCode()).isEqualTo(Status.Code.IOError);
         assertThat(e.getStatus().getSubCode()).isEqualTo(Status.SubCode.None);
-        assertThat(e.getStatus().getState()).startsWith("lock ");
+        assertThat(e.getStatus().getState()).contains("lock ");
       }
     }
   }

java/src/test/java/org/rocksdb/util/EnvironmentTest.java

@@ -1,4 +1,4 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.
@@ -70,15 +70,16 @@ public class EnvironmentTest {
         isEqualTo("librocksdbjni-linux32.so");
     assertThat(Environment.getSharedLibraryFileName("rocksdb")).
         isEqualTo("librocksdbjni.so");
+  }
+
+  @Test(expected = UnsupportedOperationException.class)
+  public void aix32() {
     // AIX
     setEnvironmentClassFields("aix", "32");
     assertThat(Environment.isWindows()).isFalse();
     assertThat(Environment.getJniLibraryExtension()).
         isEqualTo(".so");
-    assertThat(Environment.getJniLibraryFileName("rocksdb")).
+    Environment.getJniLibraryFileName("rocksdb");
-        isEqualTo("librocksdbjni-linux32.so");
-    assertThat(Environment.getSharedLibraryFileName("rocksdb")).
-        isEqualTo("librocksdbjni.so");
   }
 
   @Test
@@ -106,7 +107,7 @@ public class EnvironmentTest {
     assertThat(Environment.getJniLibraryExtension()).
         isEqualTo(".so");
     assertThat(Environment.getJniLibraryFileName("rocksdb")).
-        isEqualTo("librocksdbjni-linux64.so");
+        isEqualTo("librocksdbjni-aix64.so");
     assertThat(Environment.getSharedLibraryFileName("rocksdb")).
         isEqualTo("librocksdbjni.so");
   }
@@ -129,12 +130,6 @@ public class EnvironmentTest {
       isEqualTo("librocksdbjni.dll");
   }
 
-  @Test(expected = UnsupportedOperationException.class)
-  public void win32(){
-    setEnvironmentClassFields("win", "32");
-    Environment.getJniLibraryFileName("rocksdb");
-  }
-
   private void setEnvironmentClassFields(String osName,
       String osArch) {
     setEnvironmentClassField(OS_FIELD_NAME, osName);

memtable/memtablerep_bench.cc

@@ -7,7 +7,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
+#ifndef __STDC_FORMAT_MACROS
 #define __STDC_FORMAT_MACROS
+#endif
 
 #ifndef GFLAGS
 #include <cstdio>

options/options_helper.cc

@@ -320,10 +320,10 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
       *reinterpret_cast<uint32_t*>(opt_address) = ParseUint32(value);
       break;
     case OptionType::kUInt64T:
-      *reinterpret_cast<uint64_t*>(opt_address) = ParseUint64(value);
+      PutUnaligned(reinterpret_cast<uint64_t*>(opt_address), ParseUint64(value));
       break;
     case OptionType::kSizeT:
-      *reinterpret_cast<size_t*>(opt_address) = ParseSizeT(value);
+      PutUnaligned(reinterpret_cast<size_t*>(opt_address), ParseSizeT(value));
       break;
     case OptionType::kString:
       *reinterpret_cast<std::string*>(opt_address) = value;
@@ -404,10 +404,18 @@ bool SerializeSingleOptionHelper(const char* opt_address,
       *value = ToString(*(reinterpret_cast<const uint32_t*>(opt_address)));
       break;
     case OptionType::kUInt64T:
-      *value = ToString(*(reinterpret_cast<const uint64_t*>(opt_address)));
+      {
+        uint64_t v;
+        GetUnaligned(reinterpret_cast<const uint64_t*>(opt_address), &v);
+        *value = ToString(v);
+      }
       break;
     case OptionType::kSizeT:
-      *value = ToString(*(reinterpret_cast<const size_t*>(opt_address)));
+      {
+        size_t v;
+        GetUnaligned(reinterpret_cast<const size_t*>(opt_address), &v);
+        *value = ToString(v);
+      }
       break;
     case OptionType::kDouble:
       *value = ToString(*(reinterpret_cast<const double*>(opt_address)));

options/options_parser.cc

@@ -528,11 +528,19 @@ bool AreEqualOptions(
       return (*reinterpret_cast<const uint32_t*>(offset1) ==
               *reinterpret_cast<const uint32_t*>(offset2));
     case OptionType::kUInt64T:
-      return (*reinterpret_cast<const uint64_t*>(offset1) ==
+      {
-              *reinterpret_cast<const uint64_t*>(offset2));
+        uint64_t v1, v2;
+        GetUnaligned(reinterpret_cast<const uint64_t*>(offset1), &v1);
+        GetUnaligned(reinterpret_cast<const uint64_t*>(offset2), &v2);
+        return (v1 == v2);
+      }
     case OptionType::kSizeT:
-      return (*reinterpret_cast<const size_t*>(offset1) ==
+      {
-              *reinterpret_cast<const size_t*>(offset2));
+        size_t v1, v2;
+        GetUnaligned(reinterpret_cast<const size_t*>(offset1), &v1);
+        GetUnaligned(reinterpret_cast<const size_t*>(offset2), &v2);
+        return (v1 == v2);
+      }
     case OptionType::kString:
       return (*reinterpret_cast<const std::string*>(offset1) ==
               *reinterpret_cast<const std::string*>(offset2));

port/port_posix.h

@@ -35,6 +35,12 @@
   #else
     #define PLATFORM_IS_LITTLE_ENDIAN false
   #endif
+  #include <alloca.h>
+#elif defined(OS_AIX)
+  #include <sys/types.h>
+  #include <arpa/nameser_compat.h>
+  #define PLATFORM_IS_LITTLE_ENDIAN (BYTE_ORDER == LITTLE_ENDIAN)
+  #include <alloca.h>
 #elif defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || \
     defined(OS_DRAGONFLYBSD) || defined(OS_ANDROID)
   #include <sys/endian.h>
@@ -56,7 +62,7 @@
 
 #if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\
     defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\
-    defined(OS_ANDROID) || defined(CYGWIN)
+    defined(OS_ANDROID) || defined(CYGWIN) || defined(OS_AIX)
 // Use fread/fwrite/fflush on platforms without _unlocked variants
 #define fread_unlocked fread
 #define fwrite_unlocked fwrite

port/stack_trace.cc

@@ -6,7 +6,7 @@
 #include "port/stack_trace.h"
 
 #if defined(ROCKSDB_LITE) || !(defined(ROCKSDB_BACKTRACE) || defined(OS_MACOSX)) || \
-    defined(CYGWIN) || defined(OS_FREEBSD)
+    defined(CYGWIN) || defined(OS_FREEBSD) || defined(OS_SOLARIS)
 
 // noop
 

table/cuckoo_table_builder_test.cc

@@ -66,8 +66,8 @@ class CuckooBuilderTest : public testing::Test {
     ASSERT_EQ(expected_unused_bucket.substr(0,
           props->fixed_key_len), unused_key);
 
-    uint32_t value_len_found =
+    uint64_t value_len_found =
-      *reinterpret_cast<const uint32_t*>(props->user_collected_properties[
+      *reinterpret_cast<const uint64_t*>(props->user_collected_properties[
                 CuckooTablePropertyNames::kValueLength].data());
     ASSERT_EQ(values.empty() ? 0 : values[0].size(), value_len_found);
     ASSERT_EQ(props->raw_value_size, values.size()*value_len_found);

table/cuckoo_table_reader_test.cc

@@ -544,9 +544,15 @@ TEST_F(CuckooReaderTest, TestReadPerformance) {
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {
+  if (rocksdb::port::kLittleEndian) {
     ::testing::InitGoogleTest(&argc, argv);
     ParseCommandLineFlags(&argc, &argv, true);
     return RUN_ALL_TESTS();
+  }
+  else {
+    fprintf(stderr, "SKIPPED as Cuckoo table doesn't support Big Endian\n");
+    return 0;
+  }
 }
 
 #endif  // GFLAGS.

table/partitioned_filter_block_test.cc

@@ -56,7 +56,7 @@ class PartitionedFilterBlockTest : public testing::Test {
     int num_keys = sizeof(keys) / sizeof(*keys);
     uint64_t max_key_size = 0;
     for (int i = 1; i < num_keys; i++) {
-      max_key_size = std::max(max_key_size, keys[i].size());
+      max_key_size = std::max(max_key_size, static_cast<uint64_t>(keys[i].size()));
     }
     uint64_t max_index_size = num_keys * (max_key_size + 8 /*handle*/);
     return max_index_size;

table/plain_table_index.cc

@@ -44,7 +44,7 @@ Status PlainTableIndex::InitFromRawData(Slice data) {
 PlainTableIndex::IndexSearchResult PlainTableIndex::GetOffset(
     uint32_t prefix_hash, uint32_t* bucket_value) const {
   int bucket = GetBucketIdFromHash(prefix_hash, index_size_);
-  *bucket_value = index_[bucket];
+  GetUnaligned(index_ + bucket, bucket_value);
   if ((*bucket_value & kSubIndexMask) == kSubIndexMask) {
     *bucket_value ^= kSubIndexMask;
     return kSubindex;
@@ -175,15 +175,15 @@ Slice PlainTableIndexBuilder::FillIndexes(
     switch (num_keys_for_bucket) {
       case 0:
         // No key for bucket
-        index[i] = PlainTableIndex::kMaxFileSize;
+        PutUnaligned(index + i, (uint32_t)PlainTableIndex::kMaxFileSize);
         break;
       case 1:
         // point directly to the file offset
-        index[i] = hash_to_offsets[i]->offset;
+        PutUnaligned(index + i, hash_to_offsets[i]->offset);
         break;
       default:
         // point to second level indexes.
-        index[i] = sub_index_offset | PlainTableIndex::kSubIndexMask;
+        PutUnaligned(index + i, sub_index_offset | PlainTableIndex::kSubIndexMask);
         char* prev_ptr = &sub_index[sub_index_offset];
         char* cur_ptr = EncodeVarint32(prev_ptr, num_keys_for_bucket);
         sub_index_offset += static_cast<uint32_t>(cur_ptr - prev_ptr);

tools/auto_sanity_test.sh

@@ -1,4 +1,4 @@
-TMP_DIR="/tmp/rocksdb-sanity-test"
+TMP_DIR="${TMPDIR:-/tmp}/rocksdb-sanity-test"
 
 if [ "$#" -lt 2 ]; then
   echo "usage: ./auto_sanity_test.sh [new_commit] [old_commit]"

tools/rocksdb_dump_test.sh

@@ -1,4 +1,4 @@
-TESTDIR=`mktemp -d /tmp/rocksdb-dump-test.XXXXX`
+TESTDIR=`mktemp -d ${TMPDIR:-/tmp}/rocksdb-dump-test.XXXXX`
 DUMPFILE="tools/sample-dump.dmp"
 
 # Verify that the sample dump file is undumpable and then redumpable.

tools/run_flash_bench.sh

@@ -102,7 +102,7 @@ if [[ $skip_low_pri_tests == 1 ]]; then
   echo "Skipping some non-critical tests because SKIP_LOW_PRI_TESTS is set."
 fi
 
-output_dir="/tmp/output"
+output_dir="${TMPDIR:-/tmp}/output"
 
 ARGS="\
 OUTPUT_DIR=$output_dir \

tools/run_leveldb.sh

@@ -71,7 +71,7 @@ db_dir=${DATA_DIR:-"/tmp/rocksdb/"}
 do_setup=${DO_SETUP:-1}
 save_setup=${SAVE_SETUP:-0}
 
-output_dir="/tmp/output"
+output_dir="${TMPDIR:-/tmp}/output"
 
 ARGS="\
 OUTPUT_DIR=$output_dir \

util/bloom_test.cc

@@ -33,7 +33,9 @@ namespace rocksdb {
 static const int kVerbose = 1;
 
 static Slice Key(int i, char* buffer) {
-  memcpy(buffer, &i, sizeof(i));
+  std::string s;
+  PutFixed32(&s, static_cast<uint32_t>(i));
+  memcpy(buffer, s.c_str(), sizeof(i));
   return Slice(buffer, sizeof(i));
 }
 

util/coding.h

@@ -21,6 +21,11 @@
 #include "rocksdb/write_batch.h"
 #include "port/port.h"
 
+// Some processors does not allow unaligned access to memory
+#if defined(__sparc)
+  #define PLATFORM_UNALIGNED_ACCESS_NOT_ALLOWED
+#endif
+
 namespace rocksdb {
 
 // The maximum length of a varint in bytes for 64-bit.
@@ -127,20 +132,20 @@ inline const char* GetVarint32Ptr(const char* p,
 
 // -- Implementation of the functions declared above
 inline void EncodeFixed32(char* buf, uint32_t value) {
-#if __BYTE_ORDER == __LITTLE_ENDIAN
+  if (port::kLittleEndian) {
     memcpy(buf, &value, sizeof(value));
-#else
+  } else {
     buf[0] = value & 0xff;
     buf[1] = (value >> 8) & 0xff;
     buf[2] = (value >> 16) & 0xff;
     buf[3] = (value >> 24) & 0xff;
-#endif
+  }
 }
 
 inline void EncodeFixed64(char* buf, uint64_t value) {
-#if __BYTE_ORDER == __LITTLE_ENDIAN
+  if (port::kLittleEndian) {
     memcpy(buf, &value, sizeof(value));
-#else
+  } else {
     buf[0] = value & 0xff;
     buf[1] = (value >> 8) & 0xff;
     buf[2] = (value >> 16) & 0xff;
@@ -149,30 +154,30 @@ inline void EncodeFixed64(char* buf, uint64_t value) {
     buf[5] = (value >> 40) & 0xff;
     buf[6] = (value >> 48) & 0xff;
     buf[7] = (value >> 56) & 0xff;
-#endif
+  }
 }
 
 // Pull the last 8 bits and cast it to a character
 inline void PutFixed32(std::string* dst, uint32_t value) {
-#if __BYTE_ORDER__ == __LITTLE_ENDIAN__
+  if (port::kLittleEndian) {
     dst->append(const_cast<const char*>(reinterpret_cast<char*>(&value)),
       sizeof(value));
-#else
+  } else {
     char buf[sizeof(value)];
     EncodeFixed32(buf, value);
     dst->append(buf, sizeof(buf));
-#endif
+  }
 }
 
 inline void PutFixed64(std::string* dst, uint64_t value) {
-#if __BYTE_ORDER__ == __LITTLE_ENDIAN__
+  if (port::kLittleEndian) {
     dst->append(const_cast<const char*>(reinterpret_cast<char*>(&value)),
       sizeof(value));
-#else
+  } else {
     char buf[sizeof(value)];
     EncodeFixed64(buf, value);
     dst->append(buf, sizeof(buf));
-#endif
+  }
 }
 
 inline void PutVarint32(std::string* dst, uint32_t v) {
@@ -346,4 +351,24 @@ inline Slice GetSliceUntil(Slice* slice, char delimiter) {
   return ret;
 }
 
+template<class T>
+inline void PutUnaligned(T *memory, const T &value) {
+#if defined(PLATFORM_UNALIGNED_ACCESS_NOT_ALLOWED)
+  char *nonAlignedMemory = reinterpret_cast<char*>(memory);
+  memcpy(nonAlignedMemory, reinterpret_cast<const char*>(&value), sizeof(T));
+#else
+  *memory = value;
+#endif
+}
+
+template<class T>
+inline void GetUnaligned(const T *memory, T *value) {
+#if defined(PLATFORM_UNALIGNED_ACCESS_NOT_ALLOWED)
+  char *nonAlignedMemory = reinterpret_cast<char*>(value);
+  memcpy(nonAlignedMemory, reinterpret_cast<const char*>(memory), sizeof(T));
+#else
+  *value = *memory;
+#endif
+}
+
 }  // namespace rocksdb

util/delete_scheduler_test.cc

@@ -91,7 +91,7 @@ TEST_F(DeleteSchedulerTest, BasicRateLimiting) {
   std::vector<uint64_t> penalties;
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "DeleteScheduler::BackgroundEmptyTrash:Wait",
-      [&](void* arg) { penalties.push_back(*(static_cast<int*>(arg))); });
+      [&](void* arg) { penalties.push_back(*(static_cast<uint64_t*>(arg))); });
 
   int num_files = 100;  // 100 files
   uint64_t file_size = 1024;  // every file is 1 kb
@@ -158,7 +158,7 @@ TEST_F(DeleteSchedulerTest, RateLimitingMultiThreaded) {
   std::vector<uint64_t> penalties;
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "DeleteScheduler::BackgroundEmptyTrash:Wait",
-      [&](void* arg) { penalties.push_back(*(static_cast<int*>(arg))); });
+      [&](void* arg) { penalties.push_back(*(static_cast<uint64_t*>(arg))); });
 
   int thread_cnt = 10;
   int num_files = 10;  // 10 files per thread

util/testharness.h

@@ -9,7 +9,11 @@
 
 #pragma once
 
+#ifdef OS_AIX
+#include "gtest/gtest.h"
+#else
 #include <gtest/gtest.h>
+#endif
 
 #include <string>
 #include "rocksdb/env.h"

util/testutil.cc

@@ -94,9 +94,13 @@ class Uint64ComparatorImpl : public Comparator {
     assert(a.size() == sizeof(uint64_t) && b.size() == sizeof(uint64_t));
     const uint64_t* left = reinterpret_cast<const uint64_t*>(a.data());
     const uint64_t* right = reinterpret_cast<const uint64_t*>(b.data());
-    if (*left == *right) {
+    uint64_t leftValue;
+    uint64_t rightValue;
+    GetUnaligned(left, &leftValue);
+    GetUnaligned(right, &rightValue);
+    if (leftValue == rightValue) {
       return 0;
-    } else if (*left < *right) {
+    } else if (leftValue < rightValue) {
       return -1;
     } else {
       return 1;

utilities/column_aware_encoding_test.cc

@@ -59,9 +59,9 @@ TEST_P(ColumnAwareEncodingTestWithSize, NoCompressionEncodeDecode) {
   const char* encoded_data_ptr = encoded_data.c_str();
   uint64_t expected_encoded_val;
   if (col_size == 8) {
-    expected_encoded_val = 0x0807060504030201;
+    expected_encoded_val = port::kLittleEndian ? 0x0807060504030201 : 0x0102030405060708;
   } else if (col_size == 4) {
-    expected_encoded_val = 0x08070605;
+    expected_encoded_val = port::kLittleEndian ? 0x08070605 : 0x0102030400000000;
   }
   uint64_t encoded_val = 0;
   for (int i = 0; i < row_count; ++i) {
@@ -122,9 +122,9 @@ TEST_P(ColumnAwareEncodingTestWithSize, RleEncodeDecode) {
   memcpy(&encoded_val, encoded_data_ptr, col_size);
   uint64_t expected_encoded_val;
   if (col_size == 8) {
-    expected_encoded_val = 0x0807060504030201;
+    expected_encoded_val = port::kLittleEndian ? 0x0807060504030201 : 0x0102030405060708;
   } else if (col_size == 4) {
-    expected_encoded_val = 0x08070605;
+    expected_encoded_val = port::kLittleEndian ? 0x08070605 : 0x0102030400000000;
   }
   // Check correctness of encoded value
   ASSERT_EQ(expected_encoded_val, encoded_val);
@@ -158,8 +158,8 @@ TEST_P(ColumnAwareEncodingTestWithSize, DeltaEncodeDecode) {
   std::unique_ptr<ColBufEncoder> col_buf_encoder(
       new FixedLengthColBufEncoder(col_size, kColDeltaVarint, false, true));
   std::string str_buf;
-  uint64_t base_val1 = 0x0102030405060708;
+  uint64_t base_val1 = port::kLittleEndian ? 0x0102030405060708 : 0x0807060504030201;
-  uint64_t base_val2 = 0x0202030405060708;
+  uint64_t base_val2 = port::kLittleEndian ? 0x0202030405060708 : 0x0807060504030202;
   uint64_t val1 = 0, val2 = 0;
   memcpy(&val1, &base_val1, col_size);
   memcpy(&val2, &base_val2, col_size);
@@ -180,7 +180,7 @@ TEST_P(ColumnAwareEncodingTestWithSize, DeltaEncodeDecode) {
   if (col_size == 8) {
     varint_len = 9;
   } else if (col_size == 4) {
-    varint_len = 5;
+    varint_len = port::kLittleEndian ? 5 : 9;
   }
   // Check encoded string length: first value is original one (val - 0), the
   // coming three are encoded as 1, -1, 1, so they should take 1 byte in varint.

utilities/persistent_cache/hash_table_test.cc

@@ -89,7 +89,7 @@ TEST_F(HashTableTest, TestInsert) {
   // verify
   for (uint64_t k = 0; k < max_keys; ++k) {
     Node val;
-    port::RWMutex* rlock;
+    port::RWMutex* rlock = nullptr;
     assert(map_.Find(Node(k), &val, &rlock));
     rlock->ReadUnlock();
     assert(val.val_ == std::string(1000, k % 255));

utilities/persistent_cache/persistent_cache_bench.cc

@@ -4,6 +4,11 @@
 //  of patent rights can be found in the PATENTS file in the same directory.
 //
 #ifndef ROCKSDB_LITE
+
+#ifndef GFLAGS
+#include <cstdio>
+int main() { fprintf(stderr, "Please install gflags to run tools\n"); }
+#else
 #include <gflags/gflags.h>
 #include <atomic>
 #include <functional>
@@ -349,6 +354,7 @@ int main(int argc, char** argv) {
 
   return 0;
 }
+#endif  // #ifndef GFLAGS
 #else
 int main(int, char**) { return 0; }
 #endif