Netty resources

.github/workflows/build.yml

@@ -0,0 +1,73 @@
+name: Check buck targets and code format
+on: [push]
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Checkout feature branch
+      uses: actions/checkout@v2
+    - name: Script
+      shell: bash
+      run: |
+            set -xe
+            REPODIR="$(pwd)"
+            export GTEST_THROW_ON_FAILURE=0
+            export GTEST_OUTPUT=\"xml:/tmp/test-results/\"
+            export SKIP_FORMAT_BUCK_CHECKS=1
+            export GTEST_COLOR=1
+            export CTEST_OUTPUT_ON_FAILURE=1
+            export CTEST_TEST_TIMEOUT=300
+            export ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib
+            export BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2
+            export SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy
+            export LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4
+            export ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd
+            export DEBIAN_FRONTEND="noninteractive"
+            export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+            export PATH=$JAVA_HOME/bin:$PATH
+            
+            # install dev deps
+            
+            sudo apt-get update -y && sudo apt-get install -y libgflags-dev ninja-build valgrind cmake libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev gnupg wget git make build-essential liblzma-dev openjdk-8-jdk curl unzip tar
+            
+            # install clang 10
+            
+            wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+            echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list
+            echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list
+            echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
+            sudo apt-get update -y && sudo apt-get install -y clang-10
+            
+            # install clang 13
+            
+            echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list
+            echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list
+            echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
+            wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+            sudo apt-get update -y && sudo apt-get install -y clang-13
+            
+            # install google/benchmark
+            
+            git clone --depth 1 --branch v1.6.1 https://github.com/google/benchmark.git ~/benchmark
+            cd ~/benchmark && mkdir build && cd build
+            cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=0
+            ninja && sudo ninja install
+            
+            # install google/gtest-parallel
+            
+            git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
+            export PATH=$HOME/gtest-parallel:$PATH
+            
+            # install google/libprotobuf-mutator
+            
+            git clone --single-branch --branch master --depth 1 https://github.com/google/libprotobuf-mutator.git ~/libprotobuf-mutator
+            cd ~/libprotobuf-mutator && mkdir build && cd build
+            cmake .. -GNinja -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang++-13 -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON
+            ninja && sudo ninja install
+            export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/OFF/:~/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/
+            export PROTOC_BIN=~/libprotobuf-mutator/build/external.protobuf/bin/protoc
+            
+            # build rocksdb java
+            
+            cd "$REPODIR"; make V=0 J=8 -j8 rocksdbjavastaticpublish

.github/workflows/sanity_check.yml

@@ -1,44 +0,0 @@
-name: Check buck targets and code format
-on: [push, pull_request]
-jobs:
-  check:
-    name: Check TARGETS file and code format
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout feature branch
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-
-    - name: Fetch from upstream
-      run: |
-        git remote add upstream https://github.com/facebook/rocksdb.git && git fetch upstream
-
-    - name: Where am I
-      run: |
-        echo git status && git status
-        echo "git remote -v" && git remote -v
-        echo git branch && git branch
-
-    - name: Setup Python
-      uses: actions/setup-python@v1
-
-    - name: Install Dependencies
-      run: python -m pip install --upgrade pip
-
-    - name: Install argparse
-      run: pip install argparse
-
-    - name: Download clang-format-diff.py
-      uses: wei/wget@v1
-      with:
-        args: https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py
-
-    - name: Check format
-      run: VERBOSE_CHECK=1 make check-format
-
-    - name: Compare buckify output
-      run: make check-buck-targets
-
-    - name: Simple source code checks
-      run: make check-sources

.gitignore

@@ -97,3 +97,15 @@ fuzz/crash-*
 
 cmake-build-*
 third-party/folly/
+
+*.gz
+
+bzip2-1.0.8/
+
+lz4-1.9.3/
+
+snappy-1.1.8/
+
+zlib-1.2.12/
+
+zstd-1.4.9/

Makefile

@@ -1999,7 +1999,7 @@ else
 	ROCKSDBJNILIB = librocksdbjni-linux$(ARCH)$(JNI_LIBC_POSTFIX).so
 endif
 endif
-ROCKSDB_JAVA_VERSION ?= $(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)
+ROCKSDB_JAVA_VERSION ?= $(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-netty
 ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-linux$(ARCH)$(JNI_LIBC_POSTFIX).jar
 ROCKSDB_JAR_ALL = rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar
 ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-javadoc.jar
@@ -2285,8 +2285,8 @@ rocksdbjavastaticpublishdocker: rocksdbjavastaticreleasedocker rocksdbjavastatic
 ROCKSDB_JAVA_RELEASE_CLASSIFIERS = javadoc sources linux64 linux32 linux64-musl linux32-musl osx win64
 
 rocksdbjavastaticpublishcentral: rocksdbjavageneratepom
-	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar
+	mvn gpg:sign-and-deploy-file -Durl=https://mvn.mchv.eu/repository/mchv -DrepositoryId=mchv -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar
-	$(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar -Dclassifier=$(classifier);)
+	$(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), mvn gpg:sign-and-deploy-file -Durl=https://mvn.mchv.eu/repository/mchv -DrepositoryId=mchvg -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar -Dclassifier=$(classifier);)
 
 rocksdbjavageneratepom:
 	cd java;cat pom.xml.template | sed 's/\$${ROCKSDB_JAVA_VERSION}/$(ROCKSDB_JAVA_VERSION)/' > pom.xml

java/Makefile

@@ -211,6 +211,15 @@ SAMPLES_MAIN_SRC = samples/src/main/java
 SAMPLES_OUTPUT = samples/target
 SAMPLES_MAIN_CLASSES = $(SAMPLES_OUTPUT)/classes
 
+JAVA_MAIN_LIBDIR = main-libs
+JAVA_NETTYC_VER = 5.0.0.Alpha2
+JAVA_NETTYC_SHA256 = 17c33fd709dbef951adb48e502b26cc2938219ddc6971038440b804af05e6f4f
+JAVA_NETTYC_JAR = netty5-common-$(JAVA_NETTYC_VER).jar
+JAVA_NETTYC_JAR_PATH = $(JAVA_MAIN_LIBDIR)/$(JAVA_NETTYC_JAR)
+JAVA_NETTYB_VER = 5.0.0.Alpha2
+JAVA_NETTYB_SHA256 = ac407d92078ba503a6445aa9df1e7608b41d5e5560eb2e951e82d643528f5e2a
+JAVA_NETTYB_JAR = netty5-buffer-$(JAVA_NETTYB_VER).jar
+JAVA_NETTYB_JAR_PATH = $(JAVA_MAIN_LIBDIR)/$(JAVA_NETTYB_JAR)
 JAVA_TEST_LIBDIR = test-libs
 JAVA_JUNIT_VER = 4.13.1
 JAVA_JUNIT_SHA256 = c30719db974d6452793fe191b3638a5777005485bae145924044530ffa5f6122
@@ -232,7 +241,8 @@ JAVA_ASSERTJ_VER = 2.9.0
 JAVA_ASSERTJ_SHA256 = 5e88ea3ecbe3c48aa1346fec76c84979fa9c8d22499f11479011691230e8babf
 JAVA_ASSERTJ_JAR = assertj-core-$(JAVA_ASSERTJ_VER).jar
 JAVA_ASSERTJ_JAR_PATH = $(JAVA_TEST_LIBDIR)/$(JAVA_ASSERTJ_JAR)
-JAVA_TESTCLASSPATH = $(JAVA_JUNIT_JAR_PATH):$(JAVA_HAMCREST_JAR_PATH):$(JAVA_MOCKITO_JAR_PATH):$(JAVA_CGLIB_JAR_PATH):$(JAVA_ASSERTJ_JAR_PATH)
+JAVA_MAINCLASSPATH = $(JAVA_NETTYC_JAR_PATH):$(JAVA_NETTYB_JAR_PATH)
+JAVA_TESTCLASSPATH = $(JAVA_MAINCLASSPATH):$(JAVA_JUNIT_JAR_PATH):$(JAVA_HAMCREST_JAR_PATH):$(JAVA_MOCKITO_JAR_PATH):$(JAVA_CGLIB_JAR_PATH):$(JAVA_ASSERTJ_JAR_PATH)
 
 MVN_LOCAL = ~/.m2/repository
 
@@ -269,13 +279,13 @@ JAVAC_MAJOR_VERSION := $(word 3,$(subst ., ,$(JAVAC_VERSION)))
 endif
 
 # Test whether the version we see meets our minimum
-MIN_JAVAC_MAJOR_VERSION := 8
+MIN_JAVAC_MAJOR_VERSION := 11
 JAVAC_VERSION_GE_MIN := $(shell [ $(JAVAC_MAJOR_VERSION) -ge $(MIN_JAVAC_MAJOR_VERSION) ] > /dev/null 2>&1 && echo true)
 
 # Set the default JAVA_ARGS to "" for DEBUG_LEVEL=0
 JAVA_ARGS ?=
 
-JAVAC_ARGS ?=
+JAVAC_ARGS ?= -release $(MIN_JAVAC_MAJOR_VERSION)
 
 # Read plugin configuration
 PLUGIN_PATH = ../plugin
@@ -322,6 +332,7 @@ clean-not-downloaded:
 	$(AM_V_at)rm -rf $(SAMPLES_OUTPUT)
 
 clean-downloaded:
+	$(AM_V_at)rm -rf $(JAVA_MAIN_LIBDIR)
 	$(AM_V_at)rm -rf $(JAVA_TEST_LIBDIR)
 
 
@@ -331,44 +342,70 @@ javadocs: java
 
 javalib: java java_test javadocs
 
-java: java-version
+java: java-version resolve_main_deps
 	$(AM_V_GEN)mkdir -p $(MAIN_CLASSES)
-	$(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -h $(NATIVE_INCLUDE) -d $(MAIN_CLASSES) $(SOURCES)
+	$(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -h $(NATIVE_INCLUDE) -cp $(JAVA_MAINCLASSPATH) -d $(MAIN_CLASSES) $(SOURCES)
 	$(AM_V_at)@cp ../HISTORY.md ./HISTORY-CPP.md
 	$(AM_V_at)@rm -f ./HISTORY-CPP.md
 
 sample: java
 	$(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES)
-	$(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBSample.java
+	$(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(JAVA_MAINCLASSPATH):$(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBSample.java
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni_not_found
-	$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBSample /tmp/rocksdbjni
+	$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(JAVA_MAINCLASSPATH):$(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBSample /tmp/rocksdbjni
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni_not_found
 
 column_family_sample: java
 	$(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES)
-	$(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBColumnFamilySample.java
+	$(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBColumnFamilySample.java
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni
-	$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBColumnFamilySample /tmp/rocksdbjni
+	$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(SAMPLES_MAIN_CLASSES) RocksDBColumnFamilySample /tmp/rocksdbjni
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni
 
 transaction_sample: java
 	$(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES)
-	$(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/TransactionSample.java
+	$(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/TransactionSample.java
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni
-	$(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) TransactionSample /tmp/rocksdbjni
+	$(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(SAMPLES_MAIN_CLASSES) TransactionSample /tmp/rocksdbjni
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni
 
 optimistic_transaction_sample: java
 	$(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES)
-	$(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/OptimisticTransactionSample.java
+	$(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/OptimisticTransactionSample.java
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni
-	$(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) OptimisticTransactionSample /tmp/rocksdbjni
+	$(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(SAMPLES_MAIN_CLASSES) OptimisticTransactionSample /tmp/rocksdbjni
 	$(AM_V_at)@rm -rf /tmp/rocksdbjni
 
 $(JAVA_TEST_LIBDIR):
 	mkdir -p "$(JAVA_TEST_LIBDIR)"
+$(JAVA_MAIN_LIBDIR):
+	mkdir -p "$(JAVA_MAIN_LIBDIR)"
+
+$(JAVA_NETTYC_JAR_PATH): $(JAVA_MAIN_LIBDIR)
+ifneq (,$(wildcard $(MVN_LOCAL)/io/netty/netty5-common/$(JAVA_NETTYC_VER)/$(JAVA_NETTYC_JAR)))
+	cp -v $(MVN_LOCAL)/io/netty/netty5-common/$(JAVA_NETTYC_VER)/$(JAVA_NETTYC_JAR) $(JAVA_MAIN_LIBDIR)
+else
+	curl --fail --insecure --output $(JAVA_NETTYC_JAR_PATH) --location $(DEPS_URL)/$(JAVA_NETTYC_JAR)
+	JAVA_NETTYC_SHA256_ACTUAL=`$(SHA256_CMD) $(JAVA_NETTYC_JAR_PATH) | cut -d ' ' -f 1`; \
+	if [ "$(JAVA_NETTYC_SHA256)" != "$$JAVA_NETTYC_SHA256_ACTUAL" ]; then \
+		echo $(JAVA_NETTYC_JAR_PATH) checksum mismatch, expected=\"$(JAVA_NETTYC_SHA256)\" actual=\"$$JAVA_NETTYC_SHA256_ACTUAL\"; \
+		exit 1; \
+	fi
+endif
+
+$(JAVA_NETTYB_JAR_PATH): $(JAVA_MAIN_LIBDIR)
+ifneq (,$(wildcard $(MVN_LOCAL)/io/netty/netty5-buffer/$(JAVA_NETTYB_VER)/$(JAVA_NETTYB_JAR)))
+	cp -v $(MVN_LOCAL)/io/netty/netty5-buffer/$(JAVA_NETTYB_VER)/$(JAVA_NETTYB_JAR) $(JAVA_MAIN_LIBDIR)
+else
+	curl --fail --insecure --output $(JAVA_NETTYB_JAR_PATH) --location $(DEPS_URL)/$(JAVA_NETTYB_JAR)
+	JAVA_NETTYB_SHA256_ACTUAL=`$(SHA256_CMD) $(JAVA_NETTYB_JAR_PATH) | cut -d ' ' -f 1`; \
+	if [ "$(JAVA_NETTYB_SHA256)" != "$$JAVA_NETTYB_SHA256_ACTUAL" ]; then \
+		echo $(JAVA_NETTYB_JAR_PATH) checksum mismatch, expected=\"$(JAVA_NETTYB_SHA256)\" actual=\"$$JAVA_NETTYB_SHA256_ACTUAL\"; \
+		exit 1; \
+	fi
+endif
 
 $(JAVA_JUNIT_JAR_PATH): $(JAVA_TEST_LIBDIR)
 ifneq (,$(wildcard $(MVN_LOCAL)/junit/junit/$(JAVA_JUNIT_VER)/$(JAVA_JUNIT_JAR)))
@@ -430,22 +467,24 @@ else
 	fi
 endif
 
+resolve_main_deps: $(JAVA_NETTYC_JAR_PATH) $(JAVA_NETTYB_JAR_PATH)
+
 resolve_test_deps: $(JAVA_JUNIT_JAR_PATH) $(JAVA_HAMCREST_JAR_PATH) $(JAVA_MOCKITO_JAR_PATH) $(JAVA_CGLIB_JAR_PATH) $(JAVA_ASSERTJ_JAR_PATH)
 
 java_test: java resolve_test_deps
 	$(AM_V_GEN)mkdir -p $(TEST_CLASSES)
-	$(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES):$(JAVA_TESTCLASSPATH) -h $(NATIVE_INCLUDE) -d $(TEST_CLASSES)\
+	$(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(JAVA_TESTCLASSPATH) -h $(NATIVE_INCLUDE) -d $(TEST_CLASSES)\
 		$(TEST_SOURCES)
 
 test: java java_test
 	$(MAKE) run_test
 
 run_test:
-	$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ALL_JAVA_TESTS)
+	$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ALL_JAVA_TESTS)
 
 run_plugin_test:
-	$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ROCKSDB_PLUGIN_JAVA_TESTS)
+	$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ROCKSDB_PLUGIN_JAVA_TESTS)
 
 db_bench: java
 	$(AM_V_GEN)mkdir -p $(BENCHMARK_MAIN_CLASSES)
-	$(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(BENCHMARK_MAIN_CLASSES) $(BENCHMARK_MAIN_SRC)/org/rocksdb/benchmark/*.java
+	$(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(JAVA_MAINCLASSPATH):$(MAIN_CLASSES) -d $(BENCHMARK_MAIN_CLASSES) $(BENCHMARK_MAIN_SRC)/org/rocksdb/benchmark/*.java

java/pom.xml.template

@@ -59,8 +59,8 @@
     </mailingLists>
 
     <properties>
-        <project.build.source>1.7</project.build.source>
+        <project.build.source>11</project.build.source>
-        <project.build.target>1.7</project.build.target>
+        <project.build.target>11</project.build.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     </properties>
 
@@ -144,6 +144,11 @@
     </build>
 
     <dependencies>
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty5-buffer</artifactId>
+            <version>5.0.0.Alpha2</version>
+        </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>

java/src/main/java/org/rocksdb/AbstractCompactionFilterFactory.java

@@ -63,13 +63,15 @@ public abstract class AbstractCompactionFilterFactory<T extends AbstractCompacti
   public abstract String name();
 
   /**
-   * We override {@link RocksCallbackObject#disposeInternal()}
+   * We override {@link RocksCallbackObject#disposeInternal(boolean)}
    * as disposing of a rocksdb::AbstractCompactionFilterFactory requires
    * a slightly different approach as it is a std::shared_ptr
    */
   @Override
-  protected void disposeInternal() {
+  protected void disposeInternal(boolean owningHandle) {
-    disposeInternal(nativeHandle_);
+    if (owningHandle) {
+      disposeInternal(nativeHandle_);
+    }
   }
 
   private native long createNewCompactionFilterFactory0();

java/src/main/java/org/rocksdb/AbstractEventListener.java

@@ -325,8 +325,10 @@ public abstract class AbstractEventListener extends RocksCallbackObject implemen
    * Deletes underlying C++ native callback object pointer
    */
   @Override
-  protected void disposeInternal() {
+  protected void disposeInternal(boolean owningHandle) {
-    disposeInternal(nativeHandle_);
+    if (owningHandle) {
+      disposeInternal(nativeHandle_);
+    }
   }
 
   private native long createNewEventListener(final long enabledEventCallbackValues);

java/src/main/java/org/rocksdb/AbstractImmutableNativeReference.java

@@ -5,8 +5,6 @@
 
 package org.rocksdb;
 
-import java.util.concurrent.atomic.AtomicBoolean;
-
 /**
  * Offers functionality for implementations of
  * {@link AbstractNativeReference} which have an immutable reference to the
@@ -20,15 +18,16 @@ public abstract class AbstractImmutableNativeReference
    * A flag indicating whether the current {@code AbstractNativeReference} is
    * responsible to free the underlying C++ object
    */
-  protected final AtomicBoolean owningHandle_;
+  protected volatile boolean owningHandle_;
 
   protected AbstractImmutableNativeReference(final boolean owningHandle) {
-    this.owningHandle_ = new AtomicBoolean(owningHandle);
+    super(owningHandle);
+    this.owningHandle_ = owningHandle;
   }
 
   @Override
   public boolean isOwningHandle() {
-    return owningHandle_.get();
+    return owningHandle_ && isAccessible();
   }
 
   /**
@@ -46,14 +45,8 @@ public abstract class AbstractImmutableNativeReference
    * </p>
    */
   protected final void disOwnNativeHandle() {
-    owningHandle_.set(false);
+    setReportLeak(false);
-  }
+    owningHandle_ = false;
-
-  @Override
-  public void close() {
-    if (owningHandle_.compareAndSet(true, false)) {
-      disposeInternal();
-    }
   }
 
   /**
@@ -61,5 +54,10 @@ public abstract class AbstractImmutableNativeReference
    * which all subclasses of {@code AbstractImmutableNativeReference} must
    * implement to release their underlying native C++ objects.
    */
-  protected abstract void disposeInternal();
+  protected abstract void disposeInternal(boolean owningHandle);
+
+  @Override
+  protected final void dispose() {
+    this.disposeInternal(this.owningHandle_);
+  }
 }

java/src/main/java/org/rocksdb/AbstractNativeReference.java

@@ -5,6 +5,16 @@
 
 package org.rocksdb;
 
+import io.netty5.buffer.api.Drop;
+import io.netty5.buffer.api.Owned;
+import io.netty5.buffer.api.internal.LeakDetection;
+import io.netty5.buffer.api.internal.LifecycleTracer;
+import io.netty5.buffer.api.internal.ResourceSupport;
+import io.netty5.util.internal.SystemPropertyUtil;
+import java.lang.ref.Cleaner;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
 /**
  * AbstractNativeReference is the base-class of all RocksDB classes that have
  * a pointer to a native C++ {@code rocksdb} object.
@@ -27,6 +37,30 @@ package org.rocksdb;
  * </p>
  */
 public abstract class AbstractNativeReference implements AutoCloseable {
+
+  static boolean leakDetectionEnabled = SystemPropertyUtil.getBoolean("io.netty5.buffer.leakDetectionEnabled", false);
+
+  private static final Cleaner cleaner = Cleaner.create();
+
+  private final AtomicInteger acquires; // Closed if negative.
+  private final LifecycleTracer tracer;
+  private final AtomicBoolean reportLeak;
+
+  protected AbstractNativeReference(boolean reportLeak) {
+    var tracer = LifecycleTracer.get();
+    var acquires = new AtomicInteger(0);
+    var reportLeakRef = new AtomicBoolean(reportLeak);
+    String name = leakDetectionEnabled ? this.getClass().getName() : "rocksdb reference";
+    cleaner.register(this, () -> {
+      if (reportLeakRef.get() && acquires.get() >= 0) {
+        LeakDetection.reportLeak(tracer, name);
+      }
+    });
+    this.tracer = tracer;
+    this.acquires = acquires;
+    this.reportLeak = reportLeakRef;
+  }
+
   /**
    * Returns true if we are responsible for freeing the underlying C++ object
    *
@@ -34,15 +68,83 @@ public abstract class AbstractNativeReference implements AutoCloseable {
    */
   protected abstract boolean isOwningHandle();
 
+  public boolean isAccessible() {
+    return acquires.get() >= 0;
+  }
+
+
   /**
-   * Frees the underlying C++ object
+   * Record the current access location for debugging purposes.
-   * <p>
+   * This information may be included if the resource throws a life-cycle related exception, or if it leaks.
-   * It is strong recommended that the developer calls this after they
+   * If this resource has already been closed, then this method has no effect.
-   * have finished using the object.</p>
+   *
-   * <p>
+   * @param hint An optional hint about this access and its context. May be {@code null}.
-   * Note, that once an instance of {@link AbstractNativeReference} has been
-   * closed, calling any of its functions will lead to undefined
-   * behavior.</p>
    */
-  @Override public abstract void close();
+  public void touch(Object hint) {
+    if (isAccessible()) {
+      tracer.touch(hint);
+    }
+  }
+
+  /**
+   * Decrement the reference count, and dispose of the resource if the last reference is closed.
+   * <p>
+   * Note, this method is not thread-safe because Resources are meant to be thread-confined.
+   * <p>
+   * Subclasses who wish to attach behaviour to the close action should override the {@link #makeInaccessible()}
+   * method instead, or make it part of their drop implementation.
+   *
+   * @throws IllegalStateException If this Resource has already been closed.
+   */
+  @Override
+  public final void close() {
+    if (acquires.get() == -1) {
+      throw attachTrace(new IllegalStateException("Double-free: Resource already closed and dropped."));
+    }
+    int acq = acquires.getAndDecrement();
+    tracer.close(acq);
+    if (acq == 0) {
+      // The 'acquires' was 0, now decremented to -1, which means we need to drop.
+      tracer.drop(0);
+      try {
+        this.dispose();
+      } finally {
+        makeInaccessible();
+      }
+    }
+  }
+
+  /**
+   * Called once
+   */
+  protected abstract void dispose();
+
+  /**
+   * Attach a trace of the life-cycle of this object as suppressed exceptions to the given throwable.
+   *
+   * @param throwable The throwable to attach a life-cycle trace to.
+   * @param <E> The concrete exception type.
+   * @return The given exception, which can then be thrown.
+   */
+  protected <E extends Throwable> E attachTrace(E throwable) {
+    return tracer.attachTrace(throwable);
+  }
+
+  /**
+   * Called when this resource needs to be considered inaccessible.
+   * This is called at the correct points, by the {@link ResourceSupport} class,
+   * when the resource is being closed or sent,
+   * and can be used to set further traps for accesses that makes accessibility checks cheap.
+   * There would normally not be any reason to call this directly from a sub-class.
+   */
+  protected void makeInaccessible() {
+  }
+
+  protected final boolean getReportLeak() {
+    return reportLeak.get();
+  }
+
+  protected final void setReportLeak(boolean reportLeak) {
+    this.reportLeak.set(reportLeak);
+  }
 }

java/src/main/java/org/rocksdb/AbstractRocksIterator.java

@@ -123,10 +123,12 @@ public abstract class AbstractRocksIterator<P extends RocksObject>
    * before freeing the native handle.</p>
    */
   @Override
-  protected void disposeInternal() {
+  protected void disposeInternal(boolean owningHandle) {
+    if (owningHandle) {
       if (parent_.isOwningHandle()) {
         disposeInternal(nativeHandle_);
       }
+    }
   }
 
   abstract boolean isValid0(long handle);

java/src/main/java/org/rocksdb/AbstractTransactionNotifier.java

@@ -47,8 +47,10 @@ public abstract class AbstractTransactionNotifier
    * Otherwise an undefined behavior will occur.
    */
   @Override
-  protected void disposeInternal() {
+  protected void disposeInternal(boolean owningHandle) {
-    disposeInternal(nativeHandle_);
+    if (owningHandle) {
+      disposeInternal(nativeHandle_);
+    }
   }
   protected final native void disposeInternal(final long handle);
 }

java/src/main/java/org/rocksdb/ColumnFamilyHandle.java

@@ -136,9 +136,11 @@ public class ColumnFamilyHandle extends RocksObject {
    * initialized before freeing the native handle.</p>
    */
   @Override
-  protected void disposeInternal() {
+  protected void disposeInternal(boolean owningHandle) {
-    if(rocksDB_.isOwningHandle()) {
+    if (owningHandle) {
-      disposeInternal(nativeHandle_);
+      if (rocksDB_.isOwningHandle()) {
+        disposeInternal(nativeHandle_);
+      }
     }
   }
 

java/src/main/java/org/rocksdb/Filter.java

@@ -27,8 +27,10 @@ public abstract class Filter extends RocksObject {
    * Otherwise an undefined behavior will occur.
    */
   @Override
-  protected void disposeInternal() {
+  protected void disposeInternal(boolean owningHandle) {
-    disposeInternal(nativeHandle_);
+    if (owningHandle) {
+      disposeInternal(nativeHandle_);
+    }
   }
 
   @Override

java/src/main/java/org/rocksdb/Logger.java

@@ -109,13 +109,15 @@ public abstract class Logger extends RocksCallbackObject {
   protected native byte infoLogLevel(long handle);
 
   /**
-   * We override {@link RocksCallbackObject#disposeInternal()}
+   * We override {@link RocksCallbackObject#disposeInternal(boolean)}
    * as disposing of a rocksdb::LoggerJniCallback requires
    * a slightly different approach as it is a std::shared_ptr
    */
   @Override
-  protected void disposeInternal() {
+  protected void disposeInternal(boolean owningHandle) {
-    disposeInternal(nativeHandle_);
+    if (owningHandle) {
+      disposeInternal(nativeHandle_);
+    }
   }
 
   private native void disposeInternal(final long handle);

java/src/main/java/org/rocksdb/NativeComparatorWrapper.java

@@ -46,13 +46,15 @@ public abstract class NativeComparatorWrapper
   }
 
   /**
-   * We override {@link RocksCallbackObject#disposeInternal()}
+   * We override {@link RocksCallbackObject#disposeInternal(boolean)}
    * as disposing of a native rocksdb::Comparator extension requires
    * a slightly different approach as it is not really a RocksCallbackObject
    */
   @Override
-  protected void disposeInternal() {
+  protected void disposeInternal(boolean owningHandle) {
-    disposeInternal(nativeHandle_);
+    if (owningHandle) {
+      disposeInternal(nativeHandle_);
+    }
   }
 
   private native void disposeInternal(final long handle);

java/src/main/java/org/rocksdb/OptimisticTransactionDB.java

@@ -109,35 +109,23 @@ public class OptimisticTransactionDB extends RocksDB
    * @throws RocksDBException if an error occurs whilst closing.
    */
   public void closeE() throws RocksDBException {
-    if (owningHandle_.compareAndSet(true, false)) {
+    this.close();
-      try {
+    RocksDBException closeEx = this.closeEx;
-        closeDatabase(nativeHandle_);
+    if (closeEx != null) {
-      } finally {
+      this.closeEx = null;
-        disposeInternal();
+      throw closeEx;
-      }
     }
   }
 
-  /**
-   * This is similar to {@link #closeE()} except that it
-   * silently ignores any errors.
-   *
-   * This will not fsync the WAL files.
-   * If syncing is required, the caller must first call {@link #syncWal()}
-   * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch
-   * with {@link WriteOptions#setSync(boolean)} set to true.
-   *
-   * See also {@link #close()}.
-   */
   @Override
-  public void close() {
+  protected void disposeInternal(boolean owningHandle) {
-    if (owningHandle_.compareAndSet(true, false)) {
+    if (owningHandle) {
       try {
         closeDatabase(nativeHandle_);
-      } catch (final RocksDBException e) {
+      } catch (RocksDBException e) {
-        // silently ignore the error report
+        closeEx = e;
       } finally {
-        disposeInternal();
+        super.disposeInternal(true);
       }
     }
   }

java/src/main/java/org/rocksdb/RocksCallbackObject.java

@@ -65,8 +65,10 @@ public abstract class RocksCallbackObject extends
    * Deletes underlying C++ native callback object pointer
    */
   @Override
-  protected void disposeInternal() {
+  protected void disposeInternal(boolean owningHandle) {
-    disposeInternal(nativeHandle_);
+    if (owningHandle) {
+      disposeInternal(nativeHandle_);
+    }
   }
 
   private native void disposeInternal(final long handle);

java/src/main/java/org/rocksdb/RocksDB.java

@@ -41,6 +41,8 @@ public class RocksDB extends RocksObject {
 
   private final List<ColumnFamilyHandle> ownedColumnFamilyHandles = new ArrayList<>();
 
+  protected RocksDBException closeEx = null;
+
   /**
    * Loads the necessary library files.
    * Calling this method twice will have no effect.
@@ -609,45 +611,28 @@ public class RocksDB extends RocksObject {
    * @throws RocksDBException if an error occurs whilst closing.
    */
   public void closeE() throws RocksDBException {
-    for (final ColumnFamilyHandle columnFamilyHandle : ownedColumnFamilyHandles) {
+    this.close();
-      columnFamilyHandle.close();
+    RocksDBException closeEx = this.closeEx;
-    }
+    if (closeEx != null) {
-    ownedColumnFamilyHandles.clear();
+      this.closeEx = null;
-
+      throw closeEx;
-    if (owningHandle_.compareAndSet(true, false)) {
-      try {
-        closeDatabase(nativeHandle_);
-      } finally {
-        disposeInternal();
-      }
     }
   }
 
-  /**
-   * This is similar to {@link #closeE()} except that it
-   * silently ignores any errors.
-   *
-   * This will not fsync the WAL files.
-   * If syncing is required, the caller must first call {@link #syncWal()}
-   * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch
-   * with {@link WriteOptions#setSync(boolean)} set to true.
-   *
-   * See also {@link #close()}.
-   */
   @Override
-  public void close() {
+  protected void disposeInternal(boolean owningHandle) {
     for (final ColumnFamilyHandle columnFamilyHandle : ownedColumnFamilyHandles) {
       columnFamilyHandle.close();
     }
     ownedColumnFamilyHandles.clear();
 
-    if (owningHandle_.compareAndSet(true, false)) {
+    if (owningHandle) {
       try {
         closeDatabase(nativeHandle_);
-      } catch (final RocksDBException e) {
+      } catch (RocksDBException e) {
-        // silently ignore the error report
+        closeEx = e;
       } finally {
-        disposeInternal();
+        super.disposeInternal(true);
       }
     }
   }

java/src/main/java/org/rocksdb/RocksMutableObject.java

@@ -5,6 +5,9 @@
 
 package org.rocksdb;
 
+import io.netty5.buffer.api.Drop;
+import io.netty5.buffer.api.Owned;
+
 /**
  * RocksMutableObject is an implementation of {@link AbstractNativeReference}
  * whose reference to the underlying native C++ object can change.
@@ -23,9 +26,11 @@ public abstract class RocksMutableObject extends AbstractNativeReference {
   private boolean owningHandle_;
 
   protected RocksMutableObject() {
+    super(false);
   }
 
   protected RocksMutableObject(final long nativeHandle) {
+    super(true);
     this.nativeHandle_ = nativeHandle;
     this.owningHandle_ = true;
   }
@@ -52,11 +57,12 @@ public abstract class RocksMutableObject extends AbstractNativeReference {
       final boolean owningNativeHandle) {
     this.nativeHandle_ = nativeHandle;
     this.owningHandle_ = owningNativeHandle;
+    this.setReportLeak(owningNativeHandle);
   }
 
   @Override
   protected synchronized boolean isOwningHandle() {
-    return this.owningHandle_;
+    return this.owningHandle_ && isAccessible();
   }
 
   /**
@@ -70,12 +76,10 @@ public abstract class RocksMutableObject extends AbstractNativeReference {
     return this.nativeHandle_;
   }
 
-  @Override
+  @Override protected synchronized void dispose() {
-  public synchronized final void close() {
     if (isOwningHandle()) {
       disposeInternal();
-      this.owningHandle_ = false;
+      nativeHandle_ = 0;
-      this.nativeHandle_ = 0;
     }
   }
 

java/src/main/java/org/rocksdb/RocksObject.java

@@ -33,8 +33,10 @@ public abstract class RocksObject extends AbstractImmutableNativeReference {
    * Deletes underlying C++ object pointer.
    */
   @Override
-  protected void disposeInternal() {
+  protected void disposeInternal(boolean owningHandle) {
-    disposeInternal(nativeHandle_);
+    if (owningHandle) {
+      disposeInternal(nativeHandle_);
+    }
   }
 
   protected abstract void disposeInternal(final long handle);

java/src/main/java/org/rocksdb/TransactionDB.java

@@ -117,35 +117,23 @@ public class TransactionDB extends RocksDB
    * @throws RocksDBException if an error occurs whilst closing.
    */
   public void closeE() throws RocksDBException {
-    if (owningHandle_.compareAndSet(true, false)) {
+    this.close();
-      try {
+    RocksDBException closeEx = this.closeEx;
-        closeDatabase(nativeHandle_);
+    if (closeEx != null) {
-      } finally {
+      this.closeEx = null;
-        disposeInternal();
+      throw closeEx;
-      }
     }
   }
 
-  /**
-   * This is similar to {@link #closeE()} except that it
-   * silently ignores any errors.
-   *
-   * This will not fsync the WAL files.
-   * If syncing is required, the caller must first call {@link #syncWal()}
-   * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch
-   * with {@link WriteOptions#setSync(boolean)} set to true.
-   *
-   * See also {@link #close()}.
-   */
   @Override
-  public void close() {
+  protected void disposeInternal(boolean owningHandle) {
-    if (owningHandle_.compareAndSet(true, false)) {
+    if (owningHandle) {
       try {
         closeDatabase(nativeHandle_);
-      } catch (final RocksDBException e) {
+      } catch (RocksDBException e) {
-        // silently ignore the error report
+        closeEx = e;
       } finally {
-        disposeInternal();
+        super.disposeInternal(true);
       }
     }
   }

java/src/main/java/org/rocksdb/TtlDB.java

@@ -155,35 +155,23 @@ public class TtlDB extends RocksDB {
    * @throws RocksDBException if an error occurs whilst closing.
    */
   public void closeE() throws RocksDBException {
-    if (owningHandle_.compareAndSet(true, false)) {
+    this.close();
-      try {
+    RocksDBException closeEx = this.closeEx;
-        closeDatabase(nativeHandle_);
+    if (closeEx != null) {
-      } finally {
+      this.closeEx = null;
-        disposeInternal();
+      throw closeEx;
-      }
     }
   }
 
-  /**
-   * <p>Close the TtlDB instance and release resource.</p>
-   *
-   *
-   * This will not fsync the WAL files.
-   * If syncing is required, the caller must first call {@link #syncWal()}
-   * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch
-   * with {@link WriteOptions#setSync(boolean)} set to true.
-   *
-   * See also {@link #close()}.
-   */
   @Override
-  public void close() {
+  protected void disposeInternal(boolean owningHandle) {
-    if (owningHandle_.compareAndSet(true, false)) {
+    if (owningHandle) {
       try {
         closeDatabase(nativeHandle_);
-      } catch (final RocksDBException e) {
+      } catch (RocksDBException e) {
-        // silently ignore the error report
+        closeEx = e;
       } finally {
-        disposeInternal();
+        super.disposeInternal(true);
       }
     }
   }

java/src/main/java/org/rocksdb/WBWIRocksIterator.java

@@ -93,10 +93,8 @@ public class WBWIRocksIterator
     }
   }
 
-  @Override
+  @Override protected void disposeInternal(boolean owningHandle) {
-  public void close() {
     entry.close();
-    super.close();
   }
 
   /**