From 80518b9d4e6f9e302deb12d4078c98d8c1c920ac Mon Sep 17 00:00:00 2001 From: Andrea Cavalli Date: Tue, 14 Jun 2022 19:07:55 +0200 Subject: [PATCH] Netty resources --- .github/workflows/build.yml | 73 +++++++++++ .github/workflows/sanity_check.yml | 44 ------- .gitignore | 12 ++ Makefile | 6 +- java/Makefile | 73 ++++++++--- java/pom.xml.template | 9 +- .../AbstractCompactionFilterFactory.java | 8 +- .../org/rocksdb/AbstractEventListener.java | 6 +- .../AbstractImmutableNativeReference.java | 26 ++-- .../org/rocksdb/AbstractNativeReference.java | 120 ++++++++++++++++-- .../org/rocksdb/AbstractRocksIterator.java | 4 +- .../rocksdb/AbstractTransactionNotifier.java | 6 +- .../java/org/rocksdb/ColumnFamilyHandle.java | 8 +- java/src/main/java/org/rocksdb/Filter.java | 6 +- java/src/main/java/org/rocksdb/Logger.java | 8 +- .../org/rocksdb/NativeComparatorWrapper.java | 8 +- .../org/rocksdb/OptimisticTransactionDB.java | 32 ++--- .../java/org/rocksdb/RocksCallbackObject.java | 6 +- java/src/main/java/org/rocksdb/RocksDB.java | 39 ++---- .../java/org/rocksdb/RocksMutableObject.java | 14 +- .../main/java/org/rocksdb/RocksObject.java | 6 +- .../main/java/org/rocksdb/TransactionDB.java | 32 ++--- java/src/main/java/org/rocksdb/TtlDB.java | 32 ++--- .../java/org/rocksdb/WBWIRocksIterator.java | 4 +- 24 files changed, 369 insertions(+), 213 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/sanity_check.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..f2ebd19f1 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,73 @@ +name: Check buck targets and code format +on: [push] +jobs: + build: + name: Build + runs-on: ubuntu-20.04 + steps: + - name: Checkout feature branch + uses: actions/checkout@v2 + - name: Script + shell: bash + run: | + set -xe + REPODIR="$(pwd)" + export GTEST_THROW_ON_FAILURE=0 + export GTEST_OUTPUT=\"xml:/tmp/test-results/\" + export SKIP_FORMAT_BUCK_CHECKS=1 + export GTEST_COLOR=1 + export CTEST_OUTPUT_ON_FAILURE=1 + export CTEST_TEST_TIMEOUT=300 + export ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib + export BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2 + export SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy + export LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4 + export ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd + export DEBIAN_FRONTEND="noninteractive" + export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + export PATH=$JAVA_HOME/bin:$PATH + + # install dev deps + + sudo apt-get update -y && sudo apt-get install -y libgflags-dev ninja-build valgrind cmake libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev gnupg wget git make build-essential liblzma-dev openjdk-8-jdk curl unzip tar + + # install clang 10 + + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list + echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list + echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable + sudo apt-get update -y && sudo apt-get install -y clang-10 + + # install clang 13 + + echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list + echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list + echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + sudo apt-get update -y && sudo apt-get install -y clang-13 + + # install google/benchmark + + git clone --depth 1 --branch v1.6.1 https://github.com/google/benchmark.git ~/benchmark + cd ~/benchmark && mkdir build && cd build + cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=0 + ninja && sudo ninja install + + # install google/gtest-parallel + + git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel + export PATH=$HOME/gtest-parallel:$PATH + + # install google/libprotobuf-mutator + + git clone --single-branch --branch master --depth 1 https://github.com/google/libprotobuf-mutator.git ~/libprotobuf-mutator + cd ~/libprotobuf-mutator && mkdir build && cd build + cmake .. -GNinja -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang++-13 -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON + ninja && sudo ninja install + export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/OFF/:~/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/ + export PROTOC_BIN=~/libprotobuf-mutator/build/external.protobuf/bin/protoc + + # build rocksdb java + + cd "$REPODIR"; make V=0 J=8 -j8 rocksdbjavastaticpublish diff --git a/.github/workflows/sanity_check.yml b/.github/workflows/sanity_check.yml deleted file mode 100644 index f16b79d4d..000000000 --- a/.github/workflows/sanity_check.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Check buck targets and code format -on: [push, pull_request] -jobs: - check: - name: Check TARGETS file and code format - runs-on: ubuntu-latest - steps: - - name: Checkout feature branch - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Fetch from upstream - run: | - git remote add upstream https://github.com/facebook/rocksdb.git && git fetch upstream - - - name: Where am I - run: | - echo git status && git status - echo "git remote -v" && git remote -v - echo git branch && git branch - - - name: Setup Python - uses: actions/setup-python@v1 - - - name: Install Dependencies - run: python -m pip install --upgrade pip - - - name: Install argparse - run: pip install argparse - - - name: Download clang-format-diff.py - uses: wei/wget@v1 - with: - args: https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py - - - name: Check format - run: VERBOSE_CHECK=1 make check-format - - - name: Compare buckify output - run: make check-buck-targets - - - name: Simple source code checks - run: make check-sources diff --git a/.gitignore b/.gitignore index 5bdb34212..4376b1754 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,15 @@ fuzz/crash-* cmake-build-* third-party/folly/ + +*.gz + +bzip2-1.0.8/ + +lz4-1.9.3/ + +snappy-1.1.8/ + +zlib-1.2.12/ + +zstd-1.4.9/ diff --git a/Makefile b/Makefile index 20c78edba..f5b3d2cf7 100644 --- a/Makefile +++ b/Makefile @@ -1999,7 +1999,7 @@ else ROCKSDBJNILIB = librocksdbjni-linux$(ARCH)$(JNI_LIBC_POSTFIX).so endif endif -ROCKSDB_JAVA_VERSION ?= $(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH) +ROCKSDB_JAVA_VERSION ?= $(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-netty ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-linux$(ARCH)$(JNI_LIBC_POSTFIX).jar ROCKSDB_JAR_ALL = rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-javadoc.jar @@ -2285,8 +2285,8 @@ rocksdbjavastaticpublishdocker: rocksdbjavastaticreleasedocker rocksdbjavastatic ROCKSDB_JAVA_RELEASE_CLASSIFIERS = javadoc sources linux64 linux32 linux64-musl linux32-musl osx win64 rocksdbjavastaticpublishcentral: rocksdbjavageneratepom - mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar - $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar -Dclassifier=$(classifier);) + mvn gpg:sign-and-deploy-file -Durl=https://mvn.mchv.eu/repository/mchv -DrepositoryId=mchv -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar + $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), mvn gpg:sign-and-deploy-file -Durl=https://mvn.mchv.eu/repository/mchv -DrepositoryId=mchvg -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar -Dclassifier=$(classifier);) rocksdbjavageneratepom: cd java;cat pom.xml.template | sed 's/\$${ROCKSDB_JAVA_VERSION}/$(ROCKSDB_JAVA_VERSION)/' > pom.xml diff --git a/java/Makefile b/java/Makefile index 9a28abadf..b76414aaf 100644 --- a/java/Makefile +++ b/java/Makefile @@ -211,6 +211,15 @@ SAMPLES_MAIN_SRC = samples/src/main/java SAMPLES_OUTPUT = samples/target SAMPLES_MAIN_CLASSES = $(SAMPLES_OUTPUT)/classes +JAVA_MAIN_LIBDIR = main-libs +JAVA_NETTYC_VER = 5.0.0.Alpha2 +JAVA_NETTYC_SHA256 = 17c33fd709dbef951adb48e502b26cc2938219ddc6971038440b804af05e6f4f +JAVA_NETTYC_JAR = netty5-common-$(JAVA_NETTYC_VER).jar +JAVA_NETTYC_JAR_PATH = $(JAVA_MAIN_LIBDIR)/$(JAVA_NETTYC_JAR) +JAVA_NETTYB_VER = 5.0.0.Alpha2 +JAVA_NETTYB_SHA256 = ac407d92078ba503a6445aa9df1e7608b41d5e5560eb2e951e82d643528f5e2a +JAVA_NETTYB_JAR = netty5-buffer-$(JAVA_NETTYB_VER).jar +JAVA_NETTYB_JAR_PATH = $(JAVA_MAIN_LIBDIR)/$(JAVA_NETTYB_JAR) JAVA_TEST_LIBDIR = test-libs JAVA_JUNIT_VER = 4.13.1 JAVA_JUNIT_SHA256 = c30719db974d6452793fe191b3638a5777005485bae145924044530ffa5f6122 @@ -232,7 +241,8 @@ JAVA_ASSERTJ_VER = 2.9.0 JAVA_ASSERTJ_SHA256 = 5e88ea3ecbe3c48aa1346fec76c84979fa9c8d22499f11479011691230e8babf JAVA_ASSERTJ_JAR = assertj-core-$(JAVA_ASSERTJ_VER).jar JAVA_ASSERTJ_JAR_PATH = $(JAVA_TEST_LIBDIR)/$(JAVA_ASSERTJ_JAR) -JAVA_TESTCLASSPATH = $(JAVA_JUNIT_JAR_PATH):$(JAVA_HAMCREST_JAR_PATH):$(JAVA_MOCKITO_JAR_PATH):$(JAVA_CGLIB_JAR_PATH):$(JAVA_ASSERTJ_JAR_PATH) +JAVA_MAINCLASSPATH = $(JAVA_NETTYC_JAR_PATH):$(JAVA_NETTYB_JAR_PATH) +JAVA_TESTCLASSPATH = $(JAVA_MAINCLASSPATH):$(JAVA_JUNIT_JAR_PATH):$(JAVA_HAMCREST_JAR_PATH):$(JAVA_MOCKITO_JAR_PATH):$(JAVA_CGLIB_JAR_PATH):$(JAVA_ASSERTJ_JAR_PATH) MVN_LOCAL = ~/.m2/repository @@ -269,13 +279,13 @@ JAVAC_MAJOR_VERSION := $(word 3,$(subst ., ,$(JAVAC_VERSION))) endif # Test whether the version we see meets our minimum -MIN_JAVAC_MAJOR_VERSION := 8 +MIN_JAVAC_MAJOR_VERSION := 11 JAVAC_VERSION_GE_MIN := $(shell [ $(JAVAC_MAJOR_VERSION) -ge $(MIN_JAVAC_MAJOR_VERSION) ] > /dev/null 2>&1 && echo true) # Set the default JAVA_ARGS to "" for DEBUG_LEVEL=0 JAVA_ARGS ?= -JAVAC_ARGS ?= +JAVAC_ARGS ?= -release $(MIN_JAVAC_MAJOR_VERSION) # Read plugin configuration PLUGIN_PATH = ../plugin @@ -322,6 +332,7 @@ clean-not-downloaded: $(AM_V_at)rm -rf $(SAMPLES_OUTPUT) clean-downloaded: + $(AM_V_at)rm -rf $(JAVA_MAIN_LIBDIR) $(AM_V_at)rm -rf $(JAVA_TEST_LIBDIR) @@ -331,44 +342,70 @@ javadocs: java javalib: java java_test javadocs -java: java-version +java: java-version resolve_main_deps $(AM_V_GEN)mkdir -p $(MAIN_CLASSES) - $(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -h $(NATIVE_INCLUDE) -d $(MAIN_CLASSES) $(SOURCES) + $(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -h $(NATIVE_INCLUDE) -cp $(JAVA_MAINCLASSPATH) -d $(MAIN_CLASSES) $(SOURCES) $(AM_V_at)@cp ../HISTORY.md ./HISTORY-CPP.md $(AM_V_at)@rm -f ./HISTORY-CPP.md sample: java $(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES) - $(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBSample.java + $(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(JAVA_MAINCLASSPATH):$(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBSample.java $(AM_V_at)@rm -rf /tmp/rocksdbjni $(AM_V_at)@rm -rf /tmp/rocksdbjni_not_found - $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBSample /tmp/rocksdbjni + $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(JAVA_MAINCLASSPATH):$(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBSample /tmp/rocksdbjni $(AM_V_at)@rm -rf /tmp/rocksdbjni $(AM_V_at)@rm -rf /tmp/rocksdbjni_not_found column_family_sample: java $(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES) - $(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBColumnFamilySample.java + $(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBColumnFamilySample.java $(AM_V_at)@rm -rf /tmp/rocksdbjni - $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBColumnFamilySample /tmp/rocksdbjni + $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(SAMPLES_MAIN_CLASSES) RocksDBColumnFamilySample /tmp/rocksdbjni $(AM_V_at)@rm -rf /tmp/rocksdbjni transaction_sample: java $(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES) - $(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/TransactionSample.java + $(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/TransactionSample.java $(AM_V_at)@rm -rf /tmp/rocksdbjni - $(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) TransactionSample /tmp/rocksdbjni + $(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(SAMPLES_MAIN_CLASSES) TransactionSample /tmp/rocksdbjni $(AM_V_at)@rm -rf /tmp/rocksdbjni optimistic_transaction_sample: java $(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES) - $(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/OptimisticTransactionSample.java + $(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/OptimisticTransactionSample.java $(AM_V_at)@rm -rf /tmp/rocksdbjni - $(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) OptimisticTransactionSample /tmp/rocksdbjni + $(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(SAMPLES_MAIN_CLASSES) OptimisticTransactionSample /tmp/rocksdbjni $(AM_V_at)@rm -rf /tmp/rocksdbjni $(JAVA_TEST_LIBDIR): mkdir -p "$(JAVA_TEST_LIBDIR)" +$(JAVA_MAIN_LIBDIR): + mkdir -p "$(JAVA_MAIN_LIBDIR)" + +$(JAVA_NETTYC_JAR_PATH): $(JAVA_MAIN_LIBDIR) +ifneq (,$(wildcard $(MVN_LOCAL)/io/netty/netty5-common/$(JAVA_NETTYC_VER)/$(JAVA_NETTYC_JAR))) + cp -v $(MVN_LOCAL)/io/netty/netty5-common/$(JAVA_NETTYC_VER)/$(JAVA_NETTYC_JAR) $(JAVA_MAIN_LIBDIR) +else + curl --fail --insecure --output $(JAVA_NETTYC_JAR_PATH) --location $(DEPS_URL)/$(JAVA_NETTYC_JAR) + JAVA_NETTYC_SHA256_ACTUAL=`$(SHA256_CMD) $(JAVA_NETTYC_JAR_PATH) | cut -d ' ' -f 1`; \ + if [ "$(JAVA_NETTYC_SHA256)" != "$$JAVA_NETTYC_SHA256_ACTUAL" ]; then \ + echo $(JAVA_NETTYC_JAR_PATH) checksum mismatch, expected=\"$(JAVA_NETTYC_SHA256)\" actual=\"$$JAVA_NETTYC_SHA256_ACTUAL\"; \ + exit 1; \ + fi +endif + +$(JAVA_NETTYB_JAR_PATH): $(JAVA_MAIN_LIBDIR) +ifneq (,$(wildcard $(MVN_LOCAL)/io/netty/netty5-buffer/$(JAVA_NETTYB_VER)/$(JAVA_NETTYB_JAR))) + cp -v $(MVN_LOCAL)/io/netty/netty5-buffer/$(JAVA_NETTYB_VER)/$(JAVA_NETTYB_JAR) $(JAVA_MAIN_LIBDIR) +else + curl --fail --insecure --output $(JAVA_NETTYB_JAR_PATH) --location $(DEPS_URL)/$(JAVA_NETTYB_JAR) + JAVA_NETTYB_SHA256_ACTUAL=`$(SHA256_CMD) $(JAVA_NETTYB_JAR_PATH) | cut -d ' ' -f 1`; \ + if [ "$(JAVA_NETTYB_SHA256)" != "$$JAVA_NETTYB_SHA256_ACTUAL" ]; then \ + echo $(JAVA_NETTYB_JAR_PATH) checksum mismatch, expected=\"$(JAVA_NETTYB_SHA256)\" actual=\"$$JAVA_NETTYB_SHA256_ACTUAL\"; \ + exit 1; \ + fi +endif $(JAVA_JUNIT_JAR_PATH): $(JAVA_TEST_LIBDIR) ifneq (,$(wildcard $(MVN_LOCAL)/junit/junit/$(JAVA_JUNIT_VER)/$(JAVA_JUNIT_JAR))) @@ -430,22 +467,24 @@ else fi endif +resolve_main_deps: $(JAVA_NETTYC_JAR_PATH) $(JAVA_NETTYB_JAR_PATH) + resolve_test_deps: $(JAVA_JUNIT_JAR_PATH) $(JAVA_HAMCREST_JAR_PATH) $(JAVA_MOCKITO_JAR_PATH) $(JAVA_CGLIB_JAR_PATH) $(JAVA_ASSERTJ_JAR_PATH) java_test: java resolve_test_deps $(AM_V_GEN)mkdir -p $(TEST_CLASSES) - $(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES):$(JAVA_TESTCLASSPATH) -h $(NATIVE_INCLUDE) -d $(TEST_CLASSES)\ + $(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(JAVA_TESTCLASSPATH) -h $(NATIVE_INCLUDE) -d $(TEST_CLASSES)\ $(TEST_SOURCES) test: java java_test $(MAKE) run_test run_test: - $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ALL_JAVA_TESTS) + $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ALL_JAVA_TESTS) run_plugin_test: - $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ROCKSDB_PLUGIN_JAVA_TESTS) + $(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ROCKSDB_PLUGIN_JAVA_TESTS) db_bench: java $(AM_V_GEN)mkdir -p $(BENCHMARK_MAIN_CLASSES) - $(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(BENCHMARK_MAIN_CLASSES) $(BENCHMARK_MAIN_SRC)/org/rocksdb/benchmark/*.java + $(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(JAVA_MAINCLASSPATH):$(MAIN_CLASSES) -d $(BENCHMARK_MAIN_CLASSES) $(BENCHMARK_MAIN_SRC)/org/rocksdb/benchmark/*.java diff --git a/java/pom.xml.template b/java/pom.xml.template index 4abff4768..fa07cd9c9 100644 --- a/java/pom.xml.template +++ b/java/pom.xml.template @@ -59,8 +59,8 @@ - 1.7 - 1.7 + 11 + 11 UTF-8 @@ -144,6 +144,11 @@ + + io.netty + netty5-buffer + 5.0.0.Alpha2 + junit junit diff --git a/java/src/main/java/org/rocksdb/AbstractCompactionFilterFactory.java b/java/src/main/java/org/rocksdb/AbstractCompactionFilterFactory.java index 380b4461d..008590e2c 100644 --- a/java/src/main/java/org/rocksdb/AbstractCompactionFilterFactory.java +++ b/java/src/main/java/org/rocksdb/AbstractCompactionFilterFactory.java @@ -63,13 +63,15 @@ public abstract class AbstractCompactionFilterFactory */ protected final void disOwnNativeHandle() { - owningHandle_.set(false); - } - - @Override - public void close() { - if (owningHandle_.compareAndSet(true, false)) { - disposeInternal(); - } + setReportLeak(false); + owningHandle_ = false; } /** @@ -61,5 +54,10 @@ public abstract class AbstractImmutableNativeReference * which all subclasses of {@code AbstractImmutableNativeReference} must * implement to release their underlying native C++ objects. */ - protected abstract void disposeInternal(); + protected abstract void disposeInternal(boolean owningHandle); + + @Override + protected final void dispose() { + this.disposeInternal(this.owningHandle_); + } } diff --git a/java/src/main/java/org/rocksdb/AbstractNativeReference.java b/java/src/main/java/org/rocksdb/AbstractNativeReference.java index 88b2963b6..36d1370bb 100644 --- a/java/src/main/java/org/rocksdb/AbstractNativeReference.java +++ b/java/src/main/java/org/rocksdb/AbstractNativeReference.java @@ -5,6 +5,16 @@ package org.rocksdb; +import io.netty5.buffer.api.Drop; +import io.netty5.buffer.api.Owned; +import io.netty5.buffer.api.internal.LeakDetection; +import io.netty5.buffer.api.internal.LifecycleTracer; +import io.netty5.buffer.api.internal.ResourceSupport; +import io.netty5.util.internal.SystemPropertyUtil; +import java.lang.ref.Cleaner; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + /** * AbstractNativeReference is the base-class of all RocksDB classes that have * a pointer to a native C++ {@code rocksdb} object. @@ -27,6 +37,30 @@ package org.rocksdb; *

*/ public abstract class AbstractNativeReference implements AutoCloseable { + + static boolean leakDetectionEnabled = SystemPropertyUtil.getBoolean("io.netty5.buffer.leakDetectionEnabled", false); + + private static final Cleaner cleaner = Cleaner.create(); + + private final AtomicInteger acquires; // Closed if negative. + private final LifecycleTracer tracer; + private final AtomicBoolean reportLeak; + + protected AbstractNativeReference(boolean reportLeak) { + var tracer = LifecycleTracer.get(); + var acquires = new AtomicInteger(0); + var reportLeakRef = new AtomicBoolean(reportLeak); + String name = leakDetectionEnabled ? this.getClass().getName() : "rocksdb reference"; + cleaner.register(this, () -> { + if (reportLeakRef.get() && acquires.get() >= 0) { + LeakDetection.reportLeak(tracer, name); + } + }); + this.tracer = tracer; + this.acquires = acquires; + this.reportLeak = reportLeakRef; + } + /** * Returns true if we are responsible for freeing the underlying C++ object * @@ -34,15 +68,83 @@ public abstract class AbstractNativeReference implements AutoCloseable { */ protected abstract boolean isOwningHandle(); + public boolean isAccessible() { + return acquires.get() >= 0; + } + + /** - * Frees the underlying C++ object - *

- * It is strong recommended that the developer calls this after they - * have finished using the object.

- *

- * Note, that once an instance of {@link AbstractNativeReference} has been - * closed, calling any of its functions will lead to undefined - * behavior.

+ * Record the current access location for debugging purposes. + * This information may be included if the resource throws a life-cycle related exception, or if it leaks. + * If this resource has already been closed, then this method has no effect. + * + * @param hint An optional hint about this access and its context. May be {@code null}. */ - @Override public abstract void close(); + public void touch(Object hint) { + if (isAccessible()) { + tracer.touch(hint); + } + } + + /** + * Decrement the reference count, and dispose of the resource if the last reference is closed. + *

+ * Note, this method is not thread-safe because Resources are meant to be thread-confined. + *

+ * Subclasses who wish to attach behaviour to the close action should override the {@link #makeInaccessible()} + * method instead, or make it part of their drop implementation. + * + * @throws IllegalStateException If this Resource has already been closed. + */ + @Override + public final void close() { + if (acquires.get() == -1) { + throw attachTrace(new IllegalStateException("Double-free: Resource already closed and dropped.")); + } + int acq = acquires.getAndDecrement(); + tracer.close(acq); + if (acq == 0) { + // The 'acquires' was 0, now decremented to -1, which means we need to drop. + tracer.drop(0); + try { + this.dispose(); + } finally { + makeInaccessible(); + } + } + } + + /** + * Called once + */ + protected abstract void dispose(); + + /** + * Attach a trace of the life-cycle of this object as suppressed exceptions to the given throwable. + * + * @param throwable The throwable to attach a life-cycle trace to. + * @param The concrete exception type. + * @return The given exception, which can then be thrown. + */ + protected E attachTrace(E throwable) { + return tracer.attachTrace(throwable); + } + + /** + * Called when this resource needs to be considered inaccessible. + * This is called at the correct points, by the {@link ResourceSupport} class, + * when the resource is being closed or sent, + * and can be used to set further traps for accesses that makes accessibility checks cheap. + * There would normally not be any reason to call this directly from a sub-class. + */ + protected void makeInaccessible() { + } + + protected final boolean getReportLeak() { + return reportLeak.get(); + } + + protected final void setReportLeak(boolean reportLeak) { + this.reportLeak.set(reportLeak); + } } diff --git a/java/src/main/java/org/rocksdb/AbstractRocksIterator.java b/java/src/main/java/org/rocksdb/AbstractRocksIterator.java index 1aade1b89..d2fbb3b06 100644 --- a/java/src/main/java/org/rocksdb/AbstractRocksIterator.java +++ b/java/src/main/java/org/rocksdb/AbstractRocksIterator.java @@ -123,10 +123,12 @@ public abstract class AbstractRocksIterator

* before freeing the native handle.

*/ @Override - protected void disposeInternal() { + protected void disposeInternal(boolean owningHandle) { + if (owningHandle) { if (parent_.isOwningHandle()) { disposeInternal(nativeHandle_); } + } } abstract boolean isValid0(long handle); diff --git a/java/src/main/java/org/rocksdb/AbstractTransactionNotifier.java b/java/src/main/java/org/rocksdb/AbstractTransactionNotifier.java index cbb49836d..d7703f1a4 100644 --- a/java/src/main/java/org/rocksdb/AbstractTransactionNotifier.java +++ b/java/src/main/java/org/rocksdb/AbstractTransactionNotifier.java @@ -47,8 +47,10 @@ public abstract class AbstractTransactionNotifier * Otherwise an undefined behavior will occur. */ @Override - protected void disposeInternal() { - disposeInternal(nativeHandle_); + protected void disposeInternal(boolean owningHandle) { + if (owningHandle) { + disposeInternal(nativeHandle_); + } } protected final native void disposeInternal(final long handle); } diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyHandle.java b/java/src/main/java/org/rocksdb/ColumnFamilyHandle.java index 1ac0a35bb..a90439e0b 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyHandle.java +++ b/java/src/main/java/org/rocksdb/ColumnFamilyHandle.java @@ -136,9 +136,11 @@ public class ColumnFamilyHandle extends RocksObject { * initialized before freeing the native handle.

*/ @Override - protected void disposeInternal() { - if(rocksDB_.isOwningHandle()) { - disposeInternal(nativeHandle_); + protected void disposeInternal(boolean owningHandle) { + if (owningHandle) { + if (rocksDB_.isOwningHandle()) { + disposeInternal(nativeHandle_); + } } } diff --git a/java/src/main/java/org/rocksdb/Filter.java b/java/src/main/java/org/rocksdb/Filter.java index 7f490cf59..44a40e746 100644 --- a/java/src/main/java/org/rocksdb/Filter.java +++ b/java/src/main/java/org/rocksdb/Filter.java @@ -27,8 +27,10 @@ public abstract class Filter extends RocksObject { * Otherwise an undefined behavior will occur. */ @Override - protected void disposeInternal() { - disposeInternal(nativeHandle_); + protected void disposeInternal(boolean owningHandle) { + if (owningHandle) { + disposeInternal(nativeHandle_); + } } @Override diff --git a/java/src/main/java/org/rocksdb/Logger.java b/java/src/main/java/org/rocksdb/Logger.java index 00a5d5674..f4c0ec804 100644 --- a/java/src/main/java/org/rocksdb/Logger.java +++ b/java/src/main/java/org/rocksdb/Logger.java @@ -109,13 +109,15 @@ public abstract class Logger extends RocksCallbackObject { protected native byte infoLogLevel(long handle); /** - * We override {@link RocksCallbackObject#disposeInternal()} + * We override {@link RocksCallbackObject#disposeInternal(boolean)} * as disposing of a rocksdb::LoggerJniCallback requires * a slightly different approach as it is a std::shared_ptr */ @Override - protected void disposeInternal() { - disposeInternal(nativeHandle_); + protected void disposeInternal(boolean owningHandle) { + if (owningHandle) { + disposeInternal(nativeHandle_); + } } private native void disposeInternal(final long handle); diff --git a/java/src/main/java/org/rocksdb/NativeComparatorWrapper.java b/java/src/main/java/org/rocksdb/NativeComparatorWrapper.java index 6acc146f7..db9780c78 100644 --- a/java/src/main/java/org/rocksdb/NativeComparatorWrapper.java +++ b/java/src/main/java/org/rocksdb/NativeComparatorWrapper.java @@ -46,13 +46,15 @@ public abstract class NativeComparatorWrapper } /** - * We override {@link RocksCallbackObject#disposeInternal()} + * We override {@link RocksCallbackObject#disposeInternal(boolean)} * as disposing of a native rocksdb::Comparator extension requires * a slightly different approach as it is not really a RocksCallbackObject */ @Override - protected void disposeInternal() { - disposeInternal(nativeHandle_); + protected void disposeInternal(boolean owningHandle) { + if (owningHandle) { + disposeInternal(nativeHandle_); + } } private native void disposeInternal(final long handle); diff --git a/java/src/main/java/org/rocksdb/OptimisticTransactionDB.java b/java/src/main/java/org/rocksdb/OptimisticTransactionDB.java index 5a2e1f3ed..5617b2da1 100644 --- a/java/src/main/java/org/rocksdb/OptimisticTransactionDB.java +++ b/java/src/main/java/org/rocksdb/OptimisticTransactionDB.java @@ -109,35 +109,23 @@ public class OptimisticTransactionDB extends RocksDB * @throws RocksDBException if an error occurs whilst closing. */ public void closeE() throws RocksDBException { - if (owningHandle_.compareAndSet(true, false)) { - try { - closeDatabase(nativeHandle_); - } finally { - disposeInternal(); - } + this.close(); + RocksDBException closeEx = this.closeEx; + if (closeEx != null) { + this.closeEx = null; + throw closeEx; } } - /** - * This is similar to {@link #closeE()} except that it - * silently ignores any errors. - * - * This will not fsync the WAL files. - * If syncing is required, the caller must first call {@link #syncWal()} - * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch - * with {@link WriteOptions#setSync(boolean)} set to true. - * - * See also {@link #close()}. - */ @Override - public void close() { - if (owningHandle_.compareAndSet(true, false)) { + protected void disposeInternal(boolean owningHandle) { + if (owningHandle) { try { closeDatabase(nativeHandle_); - } catch (final RocksDBException e) { - // silently ignore the error report + } catch (RocksDBException e) { + closeEx = e; } finally { - disposeInternal(); + super.disposeInternal(true); } } } diff --git a/java/src/main/java/org/rocksdb/RocksCallbackObject.java b/java/src/main/java/org/rocksdb/RocksCallbackObject.java index 8d7a867ee..5258c564d 100644 --- a/java/src/main/java/org/rocksdb/RocksCallbackObject.java +++ b/java/src/main/java/org/rocksdb/RocksCallbackObject.java @@ -65,8 +65,10 @@ public abstract class RocksCallbackObject extends * Deletes underlying C++ native callback object pointer */ @Override - protected void disposeInternal() { - disposeInternal(nativeHandle_); + protected void disposeInternal(boolean owningHandle) { + if (owningHandle) { + disposeInternal(nativeHandle_); + } } private native void disposeInternal(final long handle); diff --git a/java/src/main/java/org/rocksdb/RocksDB.java b/java/src/main/java/org/rocksdb/RocksDB.java index 9a8a0c384..1d12ee57f 100644 --- a/java/src/main/java/org/rocksdb/RocksDB.java +++ b/java/src/main/java/org/rocksdb/RocksDB.java @@ -41,6 +41,8 @@ public class RocksDB extends RocksObject { private final List ownedColumnFamilyHandles = new ArrayList<>(); + protected RocksDBException closeEx = null; + /** * Loads the necessary library files. * Calling this method twice will have no effect. @@ -609,45 +611,28 @@ public class RocksDB extends RocksObject { * @throws RocksDBException if an error occurs whilst closing. */ public void closeE() throws RocksDBException { - for (final ColumnFamilyHandle columnFamilyHandle : ownedColumnFamilyHandles) { - columnFamilyHandle.close(); - } - ownedColumnFamilyHandles.clear(); - - if (owningHandle_.compareAndSet(true, false)) { - try { - closeDatabase(nativeHandle_); - } finally { - disposeInternal(); - } + this.close(); + RocksDBException closeEx = this.closeEx; + if (closeEx != null) { + this.closeEx = null; + throw closeEx; } } - /** - * This is similar to {@link #closeE()} except that it - * silently ignores any errors. - * - * This will not fsync the WAL files. - * If syncing is required, the caller must first call {@link #syncWal()} - * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch - * with {@link WriteOptions#setSync(boolean)} set to true. - * - * See also {@link #close()}. - */ @Override - public void close() { + protected void disposeInternal(boolean owningHandle) { for (final ColumnFamilyHandle columnFamilyHandle : ownedColumnFamilyHandles) { columnFamilyHandle.close(); } ownedColumnFamilyHandles.clear(); - if (owningHandle_.compareAndSet(true, false)) { + if (owningHandle) { try { closeDatabase(nativeHandle_); - } catch (final RocksDBException e) { - // silently ignore the error report + } catch (RocksDBException e) { + closeEx = e; } finally { - disposeInternal(); + super.disposeInternal(true); } } } diff --git a/java/src/main/java/org/rocksdb/RocksMutableObject.java b/java/src/main/java/org/rocksdb/RocksMutableObject.java index e92289dc0..6b1757bf7 100644 --- a/java/src/main/java/org/rocksdb/RocksMutableObject.java +++ b/java/src/main/java/org/rocksdb/RocksMutableObject.java @@ -5,6 +5,9 @@ package org.rocksdb; +import io.netty5.buffer.api.Drop; +import io.netty5.buffer.api.Owned; + /** * RocksMutableObject is an implementation of {@link AbstractNativeReference} * whose reference to the underlying native C++ object can change. @@ -23,9 +26,11 @@ public abstract class RocksMutableObject extends AbstractNativeReference { private boolean owningHandle_; protected RocksMutableObject() { + super(false); } protected RocksMutableObject(final long nativeHandle) { + super(true); this.nativeHandle_ = nativeHandle; this.owningHandle_ = true; } @@ -52,11 +57,12 @@ public abstract class RocksMutableObject extends AbstractNativeReference { final boolean owningNativeHandle) { this.nativeHandle_ = nativeHandle; this.owningHandle_ = owningNativeHandle; + this.setReportLeak(owningNativeHandle); } @Override protected synchronized boolean isOwningHandle() { - return this.owningHandle_; + return this.owningHandle_ && isAccessible(); } /** @@ -70,12 +76,10 @@ public abstract class RocksMutableObject extends AbstractNativeReference { return this.nativeHandle_; } - @Override - public synchronized final void close() { + @Override protected synchronized void dispose() { if (isOwningHandle()) { disposeInternal(); - this.owningHandle_ = false; - this.nativeHandle_ = 0; + nativeHandle_ = 0; } } diff --git a/java/src/main/java/org/rocksdb/RocksObject.java b/java/src/main/java/org/rocksdb/RocksObject.java index f07e1018a..e15a5dc98 100644 --- a/java/src/main/java/org/rocksdb/RocksObject.java +++ b/java/src/main/java/org/rocksdb/RocksObject.java @@ -33,8 +33,10 @@ public abstract class RocksObject extends AbstractImmutableNativeReference { * Deletes underlying C++ object pointer. */ @Override - protected void disposeInternal() { - disposeInternal(nativeHandle_); + protected void disposeInternal(boolean owningHandle) { + if (owningHandle) { + disposeInternal(nativeHandle_); + } } protected abstract void disposeInternal(final long handle); diff --git a/java/src/main/java/org/rocksdb/TransactionDB.java b/java/src/main/java/org/rocksdb/TransactionDB.java index 86f25fe15..04805137a 100644 --- a/java/src/main/java/org/rocksdb/TransactionDB.java +++ b/java/src/main/java/org/rocksdb/TransactionDB.java @@ -117,35 +117,23 @@ public class TransactionDB extends RocksDB * @throws RocksDBException if an error occurs whilst closing. */ public void closeE() throws RocksDBException { - if (owningHandle_.compareAndSet(true, false)) { - try { - closeDatabase(nativeHandle_); - } finally { - disposeInternal(); - } + this.close(); + RocksDBException closeEx = this.closeEx; + if (closeEx != null) { + this.closeEx = null; + throw closeEx; } } - /** - * This is similar to {@link #closeE()} except that it - * silently ignores any errors. - * - * This will not fsync the WAL files. - * If syncing is required, the caller must first call {@link #syncWal()} - * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch - * with {@link WriteOptions#setSync(boolean)} set to true. - * - * See also {@link #close()}. - */ @Override - public void close() { - if (owningHandle_.compareAndSet(true, false)) { + protected void disposeInternal(boolean owningHandle) { + if (owningHandle) { try { closeDatabase(nativeHandle_); - } catch (final RocksDBException e) { - // silently ignore the error report + } catch (RocksDBException e) { + closeEx = e; } finally { - disposeInternal(); + super.disposeInternal(true); } } } diff --git a/java/src/main/java/org/rocksdb/TtlDB.java b/java/src/main/java/org/rocksdb/TtlDB.java index a7adaf4b2..3b07410ef 100644 --- a/java/src/main/java/org/rocksdb/TtlDB.java +++ b/java/src/main/java/org/rocksdb/TtlDB.java @@ -155,35 +155,23 @@ public class TtlDB extends RocksDB { * @throws RocksDBException if an error occurs whilst closing. */ public void closeE() throws RocksDBException { - if (owningHandle_.compareAndSet(true, false)) { - try { - closeDatabase(nativeHandle_); - } finally { - disposeInternal(); - } + this.close(); + RocksDBException closeEx = this.closeEx; + if (closeEx != null) { + this.closeEx = null; + throw closeEx; } } - /** - *

Close the TtlDB instance and release resource.

- * - * - * This will not fsync the WAL files. - * If syncing is required, the caller must first call {@link #syncWal()} - * or {@link #write(WriteOptions, WriteBatch)} using an empty write batch - * with {@link WriteOptions#setSync(boolean)} set to true. - * - * See also {@link #close()}. - */ @Override - public void close() { - if (owningHandle_.compareAndSet(true, false)) { + protected void disposeInternal(boolean owningHandle) { + if (owningHandle) { try { closeDatabase(nativeHandle_); - } catch (final RocksDBException e) { - // silently ignore the error report + } catch (RocksDBException e) { + closeEx = e; } finally { - disposeInternal(); + super.disposeInternal(true); } } } diff --git a/java/src/main/java/org/rocksdb/WBWIRocksIterator.java b/java/src/main/java/org/rocksdb/WBWIRocksIterator.java index ce146eb3f..2033b38e2 100644 --- a/java/src/main/java/org/rocksdb/WBWIRocksIterator.java +++ b/java/src/main/java/org/rocksdb/WBWIRocksIterator.java @@ -93,10 +93,8 @@ public class WBWIRocksIterator } } - @Override - public void close() { + @Override protected void disposeInternal(boolean owningHandle) { entry.close(); - super.close(); } /**