Netty resources

This commit is contained in:
Andrea Cavalli 2022-06-14 19:07:55 +02:00
parent 8e0f495253
commit 80518b9d4e
24 changed files with 369 additions and 213 deletions

73
.github/workflows/build.yml vendored Normal file
View File

@ -0,0 +1,73 @@
name: Check buck targets and code format
on: [push]
jobs:
build:
name: Build
runs-on: ubuntu-20.04
steps:
- name: Checkout feature branch
uses: actions/checkout@v2
- name: Script
shell: bash
run: |
set -xe
REPODIR="$(pwd)"
export GTEST_THROW_ON_FAILURE=0
export GTEST_OUTPUT=\"xml:/tmp/test-results/\"
export SKIP_FORMAT_BUCK_CHECKS=1
export GTEST_COLOR=1
export CTEST_OUTPUT_ON_FAILURE=1
export CTEST_TEST_TIMEOUT=300
export ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib
export BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2
export SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy
export LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4
export ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd
export DEBIAN_FRONTEND="noninteractive"
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
export PATH=$JAVA_HOME/bin:$PATH
# install dev deps
sudo apt-get update -y && sudo apt-get install -y libgflags-dev ninja-build valgrind cmake libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev gnupg wget git make build-essential liblzma-dev openjdk-8-jdk curl unzip tar
# install clang 10
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list
echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list
echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
sudo apt-get update -y && sudo apt-get install -y clang-10
# install clang 13
echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list
echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list
echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
sudo apt-get update -y && sudo apt-get install -y clang-13
# install google/benchmark
git clone --depth 1 --branch v1.6.1 https://github.com/google/benchmark.git ~/benchmark
cd ~/benchmark && mkdir build && cd build
cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=0
ninja && sudo ninja install
# install google/gtest-parallel
git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
export PATH=$HOME/gtest-parallel:$PATH
# install google/libprotobuf-mutator
git clone --single-branch --branch master --depth 1 https://github.com/google/libprotobuf-mutator.git ~/libprotobuf-mutator
cd ~/libprotobuf-mutator && mkdir build && cd build
cmake .. -GNinja -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang++-13 -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON
ninja && sudo ninja install
export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/OFF/:~/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/
export PROTOC_BIN=~/libprotobuf-mutator/build/external.protobuf/bin/protoc
# build rocksdb java
cd "$REPODIR"; make V=0 J=8 -j8 rocksdbjavastaticpublish

View File

@ -1,44 +0,0 @@
name: Check buck targets and code format
on: [push, pull_request]
jobs:
check:
name: Check TARGETS file and code format
runs-on: ubuntu-latest
steps:
- name: Checkout feature branch
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Fetch from upstream
run: |
git remote add upstream https://github.com/facebook/rocksdb.git && git fetch upstream
- name: Where am I
run: |
echo git status && git status
echo "git remote -v" && git remote -v
echo git branch && git branch
- name: Setup Python
uses: actions/setup-python@v1
- name: Install Dependencies
run: python -m pip install --upgrade pip
- name: Install argparse
run: pip install argparse
- name: Download clang-format-diff.py
uses: wei/wget@v1
with:
args: https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py
- name: Check format
run: VERBOSE_CHECK=1 make check-format
- name: Compare buckify output
run: make check-buck-targets
- name: Simple source code checks
run: make check-sources

12
.gitignore vendored
View File

@ -97,3 +97,15 @@ fuzz/crash-*
cmake-build-*
third-party/folly/
*.gz
bzip2-1.0.8/
lz4-1.9.3/
snappy-1.1.8/
zlib-1.2.12/
zstd-1.4.9/

View File

@ -1999,7 +1999,7 @@ else
ROCKSDBJNILIB = librocksdbjni-linux$(ARCH)$(JNI_LIBC_POSTFIX).so
endif
endif
ROCKSDB_JAVA_VERSION ?= $(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)
ROCKSDB_JAVA_VERSION ?= $(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-netty
ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-linux$(ARCH)$(JNI_LIBC_POSTFIX).jar
ROCKSDB_JAR_ALL = rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar
ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_JAVA_VERSION)-javadoc.jar
@ -2285,8 +2285,8 @@ rocksdbjavastaticpublishdocker: rocksdbjavastaticreleasedocker rocksdbjavastatic
ROCKSDB_JAVA_RELEASE_CLASSIFIERS = javadoc sources linux64 linux32 linux64-musl linux32-musl osx win64
rocksdbjavastaticpublishcentral: rocksdbjavageneratepom
mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar
$(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar -Dclassifier=$(classifier);)
mvn gpg:sign-and-deploy-file -Durl=https://mvn.mchv.eu/repository/mchv -DrepositoryId=mchv -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar
$(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), mvn gpg:sign-and-deploy-file -Durl=https://mvn.mchv.eu/repository/mchv -DrepositoryId=mchvg -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar -Dclassifier=$(classifier);)
rocksdbjavageneratepom:
cd java;cat pom.xml.template | sed 's/\$${ROCKSDB_JAVA_VERSION}/$(ROCKSDB_JAVA_VERSION)/' > pom.xml

View File

@ -211,6 +211,15 @@ SAMPLES_MAIN_SRC = samples/src/main/java
SAMPLES_OUTPUT = samples/target
SAMPLES_MAIN_CLASSES = $(SAMPLES_OUTPUT)/classes
JAVA_MAIN_LIBDIR = main-libs
JAVA_NETTYC_VER = 5.0.0.Alpha2
JAVA_NETTYC_SHA256 = 17c33fd709dbef951adb48e502b26cc2938219ddc6971038440b804af05e6f4f
JAVA_NETTYC_JAR = netty5-common-$(JAVA_NETTYC_VER).jar
JAVA_NETTYC_JAR_PATH = $(JAVA_MAIN_LIBDIR)/$(JAVA_NETTYC_JAR)
JAVA_NETTYB_VER = 5.0.0.Alpha2
JAVA_NETTYB_SHA256 = ac407d92078ba503a6445aa9df1e7608b41d5e5560eb2e951e82d643528f5e2a
JAVA_NETTYB_JAR = netty5-buffer-$(JAVA_NETTYB_VER).jar
JAVA_NETTYB_JAR_PATH = $(JAVA_MAIN_LIBDIR)/$(JAVA_NETTYB_JAR)
JAVA_TEST_LIBDIR = test-libs
JAVA_JUNIT_VER = 4.13.1
JAVA_JUNIT_SHA256 = c30719db974d6452793fe191b3638a5777005485bae145924044530ffa5f6122
@ -232,7 +241,8 @@ JAVA_ASSERTJ_VER = 2.9.0
JAVA_ASSERTJ_SHA256 = 5e88ea3ecbe3c48aa1346fec76c84979fa9c8d22499f11479011691230e8babf
JAVA_ASSERTJ_JAR = assertj-core-$(JAVA_ASSERTJ_VER).jar
JAVA_ASSERTJ_JAR_PATH = $(JAVA_TEST_LIBDIR)/$(JAVA_ASSERTJ_JAR)
JAVA_TESTCLASSPATH = $(JAVA_JUNIT_JAR_PATH):$(JAVA_HAMCREST_JAR_PATH):$(JAVA_MOCKITO_JAR_PATH):$(JAVA_CGLIB_JAR_PATH):$(JAVA_ASSERTJ_JAR_PATH)
JAVA_MAINCLASSPATH = $(JAVA_NETTYC_JAR_PATH):$(JAVA_NETTYB_JAR_PATH)
JAVA_TESTCLASSPATH = $(JAVA_MAINCLASSPATH):$(JAVA_JUNIT_JAR_PATH):$(JAVA_HAMCREST_JAR_PATH):$(JAVA_MOCKITO_JAR_PATH):$(JAVA_CGLIB_JAR_PATH):$(JAVA_ASSERTJ_JAR_PATH)
MVN_LOCAL = ~/.m2/repository
@ -269,13 +279,13 @@ JAVAC_MAJOR_VERSION := $(word 3,$(subst ., ,$(JAVAC_VERSION)))
endif
# Test whether the version we see meets our minimum
MIN_JAVAC_MAJOR_VERSION := 8
MIN_JAVAC_MAJOR_VERSION := 11
JAVAC_VERSION_GE_MIN := $(shell [ $(JAVAC_MAJOR_VERSION) -ge $(MIN_JAVAC_MAJOR_VERSION) ] > /dev/null 2>&1 && echo true)
# Set the default JAVA_ARGS to "" for DEBUG_LEVEL=0
JAVA_ARGS ?=
JAVAC_ARGS ?=
JAVAC_ARGS ?= -release $(MIN_JAVAC_MAJOR_VERSION)
# Read plugin configuration
PLUGIN_PATH = ../plugin
@ -322,6 +332,7 @@ clean-not-downloaded:
$(AM_V_at)rm -rf $(SAMPLES_OUTPUT)
clean-downloaded:
$(AM_V_at)rm -rf $(JAVA_MAIN_LIBDIR)
$(AM_V_at)rm -rf $(JAVA_TEST_LIBDIR)
@ -331,44 +342,70 @@ javadocs: java
javalib: java java_test javadocs
java: java-version
java: java-version resolve_main_deps
$(AM_V_GEN)mkdir -p $(MAIN_CLASSES)
$(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -h $(NATIVE_INCLUDE) -d $(MAIN_CLASSES) $(SOURCES)
$(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -h $(NATIVE_INCLUDE) -cp $(JAVA_MAINCLASSPATH) -d $(MAIN_CLASSES) $(SOURCES)
$(AM_V_at)@cp ../HISTORY.md ./HISTORY-CPP.md
$(AM_V_at)@rm -f ./HISTORY-CPP.md
sample: java
$(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES)
$(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBSample.java
$(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(JAVA_MAINCLASSPATH):$(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBSample.java
$(AM_V_at)@rm -rf /tmp/rocksdbjni
$(AM_V_at)@rm -rf /tmp/rocksdbjni_not_found
$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBSample /tmp/rocksdbjni
$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(JAVA_MAINCLASSPATH):$(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBSample /tmp/rocksdbjni
$(AM_V_at)@rm -rf /tmp/rocksdbjni
$(AM_V_at)@rm -rf /tmp/rocksdbjni_not_found
column_family_sample: java
$(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES)
$(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBColumnFamilySample.java
$(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBColumnFamilySample.java
$(AM_V_at)@rm -rf /tmp/rocksdbjni
$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBColumnFamilySample /tmp/rocksdbjni
$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(SAMPLES_MAIN_CLASSES) RocksDBColumnFamilySample /tmp/rocksdbjni
$(AM_V_at)@rm -rf /tmp/rocksdbjni
transaction_sample: java
$(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES)
$(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/TransactionSample.java
$(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/TransactionSample.java
$(AM_V_at)@rm -rf /tmp/rocksdbjni
$(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) TransactionSample /tmp/rocksdbjni
$(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(SAMPLES_MAIN_CLASSES) TransactionSample /tmp/rocksdbjni
$(AM_V_at)@rm -rf /tmp/rocksdbjni
optimistic_transaction_sample: java
$(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES)
$(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/OptimisticTransactionSample.java
$(AM_V_at)$(JAVAC_CMD) -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/OptimisticTransactionSample.java
$(AM_V_at)@rm -rf /tmp/rocksdbjni
$(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) OptimisticTransactionSample /tmp/rocksdbjni
$(JAVA_CMD) -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(SAMPLES_MAIN_CLASSES) OptimisticTransactionSample /tmp/rocksdbjni
$(AM_V_at)@rm -rf /tmp/rocksdbjni
$(JAVA_TEST_LIBDIR):
mkdir -p "$(JAVA_TEST_LIBDIR)"
$(JAVA_MAIN_LIBDIR):
mkdir -p "$(JAVA_MAIN_LIBDIR)"
$(JAVA_NETTYC_JAR_PATH): $(JAVA_MAIN_LIBDIR)
ifneq (,$(wildcard $(MVN_LOCAL)/io/netty/netty5-common/$(JAVA_NETTYC_VER)/$(JAVA_NETTYC_JAR)))
cp -v $(MVN_LOCAL)/io/netty/netty5-common/$(JAVA_NETTYC_VER)/$(JAVA_NETTYC_JAR) $(JAVA_MAIN_LIBDIR)
else
curl --fail --insecure --output $(JAVA_NETTYC_JAR_PATH) --location $(DEPS_URL)/$(JAVA_NETTYC_JAR)
JAVA_NETTYC_SHA256_ACTUAL=`$(SHA256_CMD) $(JAVA_NETTYC_JAR_PATH) | cut -d ' ' -f 1`; \
if [ "$(JAVA_NETTYC_SHA256)" != "$$JAVA_NETTYC_SHA256_ACTUAL" ]; then \
echo $(JAVA_NETTYC_JAR_PATH) checksum mismatch, expected=\"$(JAVA_NETTYC_SHA256)\" actual=\"$$JAVA_NETTYC_SHA256_ACTUAL\"; \
exit 1; \
fi
endif
$(JAVA_NETTYB_JAR_PATH): $(JAVA_MAIN_LIBDIR)
ifneq (,$(wildcard $(MVN_LOCAL)/io/netty/netty5-buffer/$(JAVA_NETTYB_VER)/$(JAVA_NETTYB_JAR)))
cp -v $(MVN_LOCAL)/io/netty/netty5-buffer/$(JAVA_NETTYB_VER)/$(JAVA_NETTYB_JAR) $(JAVA_MAIN_LIBDIR)
else
curl --fail --insecure --output $(JAVA_NETTYB_JAR_PATH) --location $(DEPS_URL)/$(JAVA_NETTYB_JAR)
JAVA_NETTYB_SHA256_ACTUAL=`$(SHA256_CMD) $(JAVA_NETTYB_JAR_PATH) | cut -d ' ' -f 1`; \
if [ "$(JAVA_NETTYB_SHA256)" != "$$JAVA_NETTYB_SHA256_ACTUAL" ]; then \
echo $(JAVA_NETTYB_JAR_PATH) checksum mismatch, expected=\"$(JAVA_NETTYB_SHA256)\" actual=\"$$JAVA_NETTYB_SHA256_ACTUAL\"; \
exit 1; \
fi
endif
$(JAVA_JUNIT_JAR_PATH): $(JAVA_TEST_LIBDIR)
ifneq (,$(wildcard $(MVN_LOCAL)/junit/junit/$(JAVA_JUNIT_VER)/$(JAVA_JUNIT_JAR)))
@ -430,22 +467,24 @@ else
fi
endif
resolve_main_deps: $(JAVA_NETTYC_JAR_PATH) $(JAVA_NETTYB_JAR_PATH)
resolve_test_deps: $(JAVA_JUNIT_JAR_PATH) $(JAVA_HAMCREST_JAR_PATH) $(JAVA_MOCKITO_JAR_PATH) $(JAVA_CGLIB_JAR_PATH) $(JAVA_ASSERTJ_JAR_PATH)
java_test: java resolve_test_deps
$(AM_V_GEN)mkdir -p $(TEST_CLASSES)
$(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES):$(JAVA_TESTCLASSPATH) -h $(NATIVE_INCLUDE) -d $(TEST_CLASSES)\
$(AM_V_at) $(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(JAVA_TESTCLASSPATH) -h $(NATIVE_INCLUDE) -d $(TEST_CLASSES)\
$(TEST_SOURCES)
test: java java_test
$(MAKE) run_test
run_test:
$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ALL_JAVA_TESTS)
$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ALL_JAVA_TESTS)
run_plugin_test:
$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ROCKSDB_PLUGIN_JAVA_TESTS)
$(JAVA_CMD) $(JAVA_ARGS) -Djava.library.path=target -cp "$(MAIN_CLASSES):$(JAVA_MAINCLASSPATH):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(ROCKSDB_PLUGIN_JAVA_TESTS)
db_bench: java
$(AM_V_GEN)mkdir -p $(BENCHMARK_MAIN_CLASSES)
$(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(MAIN_CLASSES) -d $(BENCHMARK_MAIN_CLASSES) $(BENCHMARK_MAIN_SRC)/org/rocksdb/benchmark/*.java
$(AM_V_at)$(JAVAC_CMD) $(JAVAC_ARGS) -cp $(JAVA_MAINCLASSPATH):$(MAIN_CLASSES) -d $(BENCHMARK_MAIN_CLASSES) $(BENCHMARK_MAIN_SRC)/org/rocksdb/benchmark/*.java

View File

@ -59,8 +59,8 @@
</mailingLists>
<properties>
<project.build.source>1.7</project.build.source>
<project.build.target>1.7</project.build.target>
<project.build.source>11</project.build.source>
<project.build.target>11</project.build.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
@ -144,6 +144,11 @@
</build>
<dependencies>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty5-buffer</artifactId>
<version>5.0.0.Alpha2</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

View File

@ -63,13 +63,15 @@ public abstract class AbstractCompactionFilterFactory<T extends AbstractCompacti
public abstract String name();
/**
* We override {@link RocksCallbackObject#disposeInternal()}
* We override {@link RocksCallbackObject#disposeInternal(boolean)}
* as disposing of a rocksdb::AbstractCompactionFilterFactory requires
* a slightly different approach as it is a std::shared_ptr
*/
@Override
protected void disposeInternal() {
disposeInternal(nativeHandle_);
protected void disposeInternal(boolean owningHandle) {
if (owningHandle) {
disposeInternal(nativeHandle_);
}
}
private native long createNewCompactionFilterFactory0();

View File

@ -325,8 +325,10 @@ public abstract class AbstractEventListener extends RocksCallbackObject implemen
* Deletes underlying C++ native callback object pointer
*/
@Override
protected void disposeInternal() {
disposeInternal(nativeHandle_);
protected void disposeInternal(boolean owningHandle) {
if (owningHandle) {
disposeInternal(nativeHandle_);
}
}
private native long createNewEventListener(final long enabledEventCallbackValues);

View File

@ -5,8 +5,6 @@
package org.rocksdb;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* Offers functionality for implementations of
* {@link AbstractNativeReference} which have an immutable reference to the
@ -20,15 +18,16 @@ public abstract class AbstractImmutableNativeReference
* A flag indicating whether the current {@code AbstractNativeReference} is
* responsible to free the underlying C++ object
*/
protected final AtomicBoolean owningHandle_;
protected volatile boolean owningHandle_;
protected AbstractImmutableNativeReference(final boolean owningHandle) {
this.owningHandle_ = new AtomicBoolean(owningHandle);
super(owningHandle);
this.owningHandle_ = owningHandle;
}
@Override
public boolean isOwningHandle() {
return owningHandle_.get();
return owningHandle_ && isAccessible();
}
/**
@ -46,14 +45,8 @@ public abstract class AbstractImmutableNativeReference
* </p>
*/
protected final void disOwnNativeHandle() {
owningHandle_.set(false);
}
@Override
public void close() {
if (owningHandle_.compareAndSet(true, false)) {
disposeInternal();
}
setReportLeak(false);
owningHandle_ = false;
}
/**
@ -61,5 +54,10 @@ public abstract class AbstractImmutableNativeReference
* which all subclasses of {@code AbstractImmutableNativeReference} must
* implement to release their underlying native C++ objects.
*/
protected abstract void disposeInternal();
protected abstract void disposeInternal(boolean owningHandle);
@Override
protected final void dispose() {
this.disposeInternal(this.owningHandle_);
}
}

View File

@ -5,6 +5,16 @@
package org.rocksdb;
import io.netty5.buffer.api.Drop;
import io.netty5.buffer.api.Owned;
import io.netty5.buffer.api.internal.LeakDetection;
import io.netty5.buffer.api.internal.LifecycleTracer;
import io.netty5.buffer.api.internal.ResourceSupport;
import io.netty5.util.internal.SystemPropertyUtil;
import java.lang.ref.Cleaner;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
/**
* AbstractNativeReference is the base-class of all RocksDB classes that have
* a pointer to a native C++ {@code rocksdb} object.
@ -27,6 +37,30 @@ package org.rocksdb;
* </p>
*/
public abstract class AbstractNativeReference implements AutoCloseable {
static boolean leakDetectionEnabled = SystemPropertyUtil.getBoolean("io.netty5.buffer.leakDetectionEnabled", false);
private static final Cleaner cleaner = Cleaner.create();
private final AtomicInteger acquires; // Closed if negative.
private final LifecycleTracer tracer;
private final AtomicBoolean reportLeak;
protected AbstractNativeReference(boolean reportLeak) {
var tracer = LifecycleTracer.get();
var acquires = new AtomicInteger(0);
var reportLeakRef = new AtomicBoolean(reportLeak);
String name = leakDetectionEnabled ? this.getClass().getName() : "rocksdb reference";
cleaner.register(this, () -> {
if (reportLeakRef.get() && acquires.get() >= 0) {
LeakDetection.reportLeak(tracer, name);
}
});
this.tracer = tracer;
this.acquires = acquires;
this.reportLeak = reportLeakRef;
}
/**
* Returns true if we are responsible for freeing the underlying C++ object
*
@ -34,15 +68,83 @@ public abstract class AbstractNativeReference implements AutoCloseable {
*/
protected abstract boolean isOwningHandle();
public boolean isAccessible() {
return acquires.get() >= 0;
}
/**
* Frees the underlying C++ object
* <p>
* It is strong recommended that the developer calls this after they
* have finished using the object.</p>
* <p>
* Note, that once an instance of {@link AbstractNativeReference} has been
* closed, calling any of its functions will lead to undefined
* behavior.</p>
* Record the current access location for debugging purposes.
* This information may be included if the resource throws a life-cycle related exception, or if it leaks.
* If this resource has already been closed, then this method has no effect.
*
* @param hint An optional hint about this access and its context. May be {@code null}.
*/
@Override public abstract void close();
public void touch(Object hint) {
if (isAccessible()) {
tracer.touch(hint);
}
}
/**
* Decrement the reference count, and dispose of the resource if the last reference is closed.
* <p>
* Note, this method is not thread-safe because Resources are meant to be thread-confined.
* <p>
* Subclasses who wish to attach behaviour to the close action should override the {@link #makeInaccessible()}
* method instead, or make it part of their drop implementation.
*
* @throws IllegalStateException If this Resource has already been closed.
*/
@Override
public final void close() {
if (acquires.get() == -1) {
throw attachTrace(new IllegalStateException("Double-free: Resource already closed and dropped."));
}
int acq = acquires.getAndDecrement();
tracer.close(acq);
if (acq == 0) {
// The 'acquires' was 0, now decremented to -1, which means we need to drop.
tracer.drop(0);
try {
this.dispose();
} finally {
makeInaccessible();
}
}
}
/**
* Called once
*/
protected abstract void dispose();
/**
* Attach a trace of the life-cycle of this object as suppressed exceptions to the given throwable.
*
* @param throwable The throwable to attach a life-cycle trace to.
* @param <E> The concrete exception type.
* @return The given exception, which can then be thrown.
*/
protected <E extends Throwable> E attachTrace(E throwable) {
return tracer.attachTrace(throwable);
}
/**
* Called when this resource needs to be considered inaccessible.
* This is called at the correct points, by the {@link ResourceSupport} class,
* when the resource is being closed or sent,
* and can be used to set further traps for accesses that makes accessibility checks cheap.
* There would normally not be any reason to call this directly from a sub-class.
*/
protected void makeInaccessible() {
}
protected final boolean getReportLeak() {
return reportLeak.get();
}
protected final void setReportLeak(boolean reportLeak) {
this.reportLeak.set(reportLeak);
}
}

View File

@ -123,10 +123,12 @@ public abstract class AbstractRocksIterator<P extends RocksObject>
* before freeing the native handle.</p>
*/
@Override
protected void disposeInternal() {
protected void disposeInternal(boolean owningHandle) {
if (owningHandle) {
if (parent_.isOwningHandle()) {
disposeInternal(nativeHandle_);
}
}
}
abstract boolean isValid0(long handle);

View File

@ -47,8 +47,10 @@ public abstract class AbstractTransactionNotifier
* Otherwise an undefined behavior will occur.
*/
@Override
protected void disposeInternal() {
disposeInternal(nativeHandle_);
protected void disposeInternal(boolean owningHandle) {
if (owningHandle) {
disposeInternal(nativeHandle_);
}
}
protected final native void disposeInternal(final long handle);
}

View File

@ -136,9 +136,11 @@ public class ColumnFamilyHandle extends RocksObject {
* initialized before freeing the native handle.</p>
*/
@Override
protected void disposeInternal() {
if(rocksDB_.isOwningHandle()) {
disposeInternal(nativeHandle_);
protected void disposeInternal(boolean owningHandle) {
if (owningHandle) {
if (rocksDB_.isOwningHandle()) {
disposeInternal(nativeHandle_);
}
}
}

View File

@ -27,8 +27,10 @@ public abstract class Filter extends RocksObject {
* Otherwise an undefined behavior will occur.
*/
@Override
protected void disposeInternal() {
disposeInternal(nativeHandle_);
protected void disposeInternal(boolean owningHandle) {
if (owningHandle) {
disposeInternal(nativeHandle_);
}
}
@Override

View File

@ -109,13 +109,15 @@ public abstract class Logger extends RocksCallbackObject {
protected native byte infoLogLevel(long handle);
/**
* We override {@link RocksCallbackObject#disposeInternal()}
* We override {@link RocksCallbackObject#disposeInternal(boolean)}
* as disposing of a rocksdb::LoggerJniCallback requires
* a slightly different approach as it is a std::shared_ptr
*/
@Override
protected void disposeInternal() {
disposeInternal(nativeHandle_);
protected void disposeInternal(boolean owningHandle) {
if (owningHandle) {
disposeInternal(nativeHandle_);
}
}
private native void disposeInternal(final long handle);

View File

@ -46,13 +46,15 @@ public abstract class NativeComparatorWrapper
}
/**
* We override {@link RocksCallbackObject#disposeInternal()}
* We override {@link RocksCallbackObject#disposeInternal(boolean)}
* as disposing of a native rocksdb::Comparator extension requires
* a slightly different approach as it is not really a RocksCallbackObject
*/
@Override
protected void disposeInternal() {
disposeInternal(nativeHandle_);
protected void disposeInternal(boolean owningHandle) {
if (owningHandle) {
disposeInternal(nativeHandle_);
}
}
private native void disposeInternal(final long handle);

View File

@ -109,35 +109,23 @@ public class OptimisticTransactionDB extends RocksDB
* @throws RocksDBException if an error occurs whilst closing.
*/
public void closeE() throws RocksDBException {
if (owningHandle_.compareAndSet(true, false)) {
try {
closeDatabase(nativeHandle_);
} finally {
disposeInternal();
}
this.close();
RocksDBException closeEx = this.closeEx;
if (closeEx != null) {
this.closeEx = null;
throw closeEx;
}
}
/**
* This is similar to {@link #closeE()} except that it
* silently ignores any errors.
*
* This will not fsync the WAL files.
* If syncing is required, the caller must first call {@link #syncWal()}
* or {@link #write(WriteOptions, WriteBatch)} using an empty write batch
* with {@link WriteOptions#setSync(boolean)} set to true.
*
* See also {@link #close()}.
*/
@Override
public void close() {
if (owningHandle_.compareAndSet(true, false)) {
protected void disposeInternal(boolean owningHandle) {
if (owningHandle) {
try {
closeDatabase(nativeHandle_);
} catch (final RocksDBException e) {
// silently ignore the error report
} catch (RocksDBException e) {
closeEx = e;
} finally {
disposeInternal();
super.disposeInternal(true);
}
}
}

View File

@ -65,8 +65,10 @@ public abstract class RocksCallbackObject extends
* Deletes underlying C++ native callback object pointer
*/
@Override
protected void disposeInternal() {
disposeInternal(nativeHandle_);
protected void disposeInternal(boolean owningHandle) {
if (owningHandle) {
disposeInternal(nativeHandle_);
}
}
private native void disposeInternal(final long handle);

View File

@ -41,6 +41,8 @@ public class RocksDB extends RocksObject {
private final List<ColumnFamilyHandle> ownedColumnFamilyHandles = new ArrayList<>();
protected RocksDBException closeEx = null;
/**
* Loads the necessary library files.
* Calling this method twice will have no effect.
@ -609,45 +611,28 @@ public class RocksDB extends RocksObject {
* @throws RocksDBException if an error occurs whilst closing.
*/
public void closeE() throws RocksDBException {
for (final ColumnFamilyHandle columnFamilyHandle : ownedColumnFamilyHandles) {
columnFamilyHandle.close();
}
ownedColumnFamilyHandles.clear();
if (owningHandle_.compareAndSet(true, false)) {
try {
closeDatabase(nativeHandle_);
} finally {
disposeInternal();
}
this.close();
RocksDBException closeEx = this.closeEx;
if (closeEx != null) {
this.closeEx = null;
throw closeEx;
}
}
/**
* This is similar to {@link #closeE()} except that it
* silently ignores any errors.
*
* This will not fsync the WAL files.
* If syncing is required, the caller must first call {@link #syncWal()}
* or {@link #write(WriteOptions, WriteBatch)} using an empty write batch
* with {@link WriteOptions#setSync(boolean)} set to true.
*
* See also {@link #close()}.
*/
@Override
public void close() {
protected void disposeInternal(boolean owningHandle) {
for (final ColumnFamilyHandle columnFamilyHandle : ownedColumnFamilyHandles) {
columnFamilyHandle.close();
}
ownedColumnFamilyHandles.clear();
if (owningHandle_.compareAndSet(true, false)) {
if (owningHandle) {
try {
closeDatabase(nativeHandle_);
} catch (final RocksDBException e) {
// silently ignore the error report
} catch (RocksDBException e) {
closeEx = e;
} finally {
disposeInternal();
super.disposeInternal(true);
}
}
}

View File

@ -5,6 +5,9 @@
package org.rocksdb;
import io.netty5.buffer.api.Drop;
import io.netty5.buffer.api.Owned;
/**
* RocksMutableObject is an implementation of {@link AbstractNativeReference}
* whose reference to the underlying native C++ object can change.
@ -23,9 +26,11 @@ public abstract class RocksMutableObject extends AbstractNativeReference {
private boolean owningHandle_;
protected RocksMutableObject() {
super(false);
}
protected RocksMutableObject(final long nativeHandle) {
super(true);
this.nativeHandle_ = nativeHandle;
this.owningHandle_ = true;
}
@ -52,11 +57,12 @@ public abstract class RocksMutableObject extends AbstractNativeReference {
final boolean owningNativeHandle) {
this.nativeHandle_ = nativeHandle;
this.owningHandle_ = owningNativeHandle;
this.setReportLeak(owningNativeHandle);
}
@Override
protected synchronized boolean isOwningHandle() {
return this.owningHandle_;
return this.owningHandle_ && isAccessible();
}
/**
@ -70,12 +76,10 @@ public abstract class RocksMutableObject extends AbstractNativeReference {
return this.nativeHandle_;
}
@Override
public synchronized final void close() {
@Override protected synchronized void dispose() {
if (isOwningHandle()) {
disposeInternal();
this.owningHandle_ = false;
this.nativeHandle_ = 0;
nativeHandle_ = 0;
}
}

View File

@ -33,8 +33,10 @@ public abstract class RocksObject extends AbstractImmutableNativeReference {
* Deletes underlying C++ object pointer.
*/
@Override
protected void disposeInternal() {
disposeInternal(nativeHandle_);
protected void disposeInternal(boolean owningHandle) {
if (owningHandle) {
disposeInternal(nativeHandle_);
}
}
protected abstract void disposeInternal(final long handle);

View File

@ -117,35 +117,23 @@ public class TransactionDB extends RocksDB
* @throws RocksDBException if an error occurs whilst closing.
*/
public void closeE() throws RocksDBException {
if (owningHandle_.compareAndSet(true, false)) {
try {
closeDatabase(nativeHandle_);
} finally {
disposeInternal();
}
this.close();
RocksDBException closeEx = this.closeEx;
if (closeEx != null) {
this.closeEx = null;
throw closeEx;
}
}
/**
* This is similar to {@link #closeE()} except that it
* silently ignores any errors.
*
* This will not fsync the WAL files.
* If syncing is required, the caller must first call {@link #syncWal()}
* or {@link #write(WriteOptions, WriteBatch)} using an empty write batch
* with {@link WriteOptions#setSync(boolean)} set to true.
*
* See also {@link #close()}.
*/
@Override
public void close() {
if (owningHandle_.compareAndSet(true, false)) {
protected void disposeInternal(boolean owningHandle) {
if (owningHandle) {
try {
closeDatabase(nativeHandle_);
} catch (final RocksDBException e) {
// silently ignore the error report
} catch (RocksDBException e) {
closeEx = e;
} finally {
disposeInternal();
super.disposeInternal(true);
}
}
}

View File

@ -155,35 +155,23 @@ public class TtlDB extends RocksDB {
* @throws RocksDBException if an error occurs whilst closing.
*/
public void closeE() throws RocksDBException {
if (owningHandle_.compareAndSet(true, false)) {
try {
closeDatabase(nativeHandle_);
} finally {
disposeInternal();
}
this.close();
RocksDBException closeEx = this.closeEx;
if (closeEx != null) {
this.closeEx = null;
throw closeEx;
}
}
/**
* <p>Close the TtlDB instance and release resource.</p>
*
*
* This will not fsync the WAL files.
* If syncing is required, the caller must first call {@link #syncWal()}
* or {@link #write(WriteOptions, WriteBatch)} using an empty write batch
* with {@link WriteOptions#setSync(boolean)} set to true.
*
* See also {@link #close()}.
*/
@Override
public void close() {
if (owningHandle_.compareAndSet(true, false)) {
protected void disposeInternal(boolean owningHandle) {
if (owningHandle) {
try {
closeDatabase(nativeHandle_);
} catch (final RocksDBException e) {
// silently ignore the error report
} catch (RocksDBException e) {
closeEx = e;
} finally {
disposeInternal();
super.disposeInternal(true);
}
}
}

View File

@ -93,10 +93,8 @@ public class WBWIRocksIterator
}
}
@Override
public void close() {
@Override protected void disposeInternal(boolean owningHandle) {
entry.close();
super.close();
}
/**