Fix RocksJava releases for macOS (#9662)

Summary:
Addresses the problems described in https://github.com/facebook/rocksdb/pull/9254#issuecomment-1054598516 and https://github.com/facebook/rocksdb/pull/9254#issuecomment-1059574837 that have blocked a RocksJava release

**NOTE** Also needs to be ported to 6.29.fb branch.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9662

Reviewed By: ajkr

Differential Revision: D34689200

Pulled By: pdillinger

fbshipit-source-id: c62fe34c54f05be5a00ee1daec8ec7454baa5eb8
This commit is contained in:
Adam Retter 2022-03-07 10:50:52 -08:00 committed by Facebook GitHub Bot
parent f20b674796
commit dab19afe56
4 changed files with 148 additions and 59 deletions

View File

@ -329,8 +329,8 @@ ifneq ($(MACHINE), arm64)
# linking with jemalloc (as it won't be arm64-compatible) and remove some other options
# set during platform detection
DISABLE_JEMALLOC=1
PLATFORM_CFLAGS := $(filter-out -march=native -DHAVE_SSE42, $(PLATFORM_CFLAGS))
PLATFORM_CXXFLAGS := $(filter-out -march=native -DHAVE_SSE42, $(PLATFORM_CXXFLAGS))
PLATFORM_CFLAGS := $(filter-out -march=native -DHAVE_SSE42 -DHAVE_AVX2, $(PLATFORM_CFLAGS))
PLATFORM_CXXFLAGS := $(filter-out -march=native -DHAVE_SSE42 -DHAVE_AVX2, $(PLATFORM_CXXFLAGS))
endif
endif
endif
@ -2121,7 +2121,9 @@ CURL_SSL_OPTS ?= --tlsv1
ifeq ($(PLATFORM), OS_MACOSX)
ifeq (,$(findstring librocksdbjni-osx,$(ROCKSDBJNILIB)))
ifeq ($(MACHINE),arm64)
ROCKSDBJNILIB = librocksdbjni-osx-aarch64.jnilib
ROCKSDBJNILIB = librocksdbjni-osx-arm64.jnilib
else ifeq ($(MACHINE),x86_64)
ROCKSDBJNILIB = librocksdbjni-osx-x86_64.jnilib
else
ROCKSDBJNILIB = librocksdbjni-osx.jnilib
endif
@ -2252,15 +2254,20 @@ endif
$(MAKE) rocksdbjavastatic_deps
$(MAKE) rocksdbjavastatic_libobjects
$(MAKE) rocksdbjavastatic_javalib
$(MAKE) rocksdbjavastatic_jar
$(MAKE) rocksdbjava_jar
rocksdbjavastaticosx: rocksdbjavastaticosx_archs
mv java/target/librocksdbjni-osx-x86_64.jnilib java/target/librocksdbjni-osx.jnilib
mv java/target/librocksdbjni-osx-arm64.jnilib java/target/librocksdbjni-osx-aarch64.jnilib
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) librocksdbjni-osx-x86_64.jnilib librocksdbjni-osx-arm64.jnilib
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1
rocksdbjavastaticosx_ub: rocksdbjavastaticosx_archs
lipo -create -output ./java/target/$(ROCKSDBJNILIB) java/target/librocksdbjni-osx-x86_64.jnilib java/target/librocksdbjni-osx-arm64.jnilib
$(MAKE) rocksdbjavastatic_jar
cd java/target; lipo -create -output librocksdbjni-osx.jnilib librocksdbjni-osx-x86_64.jnilib librocksdbjni-osx-arm64.jnilib
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) librocksdbjni-osx.jnilib
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1
rocksdbjavastaticosx_archs:
$(MAKE) rocksdbjavastaticosx_arch_x86_64
@ -2294,28 +2301,32 @@ rocksdbjavastatic_javalib:
strip $(STRIPFLAGS) $(ROCKSDBJNILIB); \
fi
rocksdbjavastatic_jar:
rocksdbjava_jar:
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
cd java/target/apidocs; $(JAR_CMD) -cf ../$(ROCKSDB_JAVADOCS_JAR) *
cd java/src/main/java; $(JAR_CMD) -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org
openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1
rocksdbjava_javadocs_jar:
cd java/target/apidocs; $(JAR_CMD) -cf ../$(ROCKSDB_JAVADOCS_JAR) *
openssl sha1 java/target/$(ROCKSDB_JAVADOCS_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAVADOCS_JAR).sha1
rocksdbjava_sources_jar:
cd java/src/main/java; $(JAR_CMD) -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org
openssl sha1 java/target/$(ROCKSDB_SOURCES_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_SOURCES_JAR).sha1
rocksdbjavastatic_deps: $(JAVA_COMPRESSIONS)
rocksdbjavastatic_libobjects: $(LIB_OBJECTS)
rocksdbjavastaticrelease: rocksdbjavastaticosx
rocksdbjavastaticrelease: rocksdbjavastaticosx rocksdbjava_javadocs_jar rocksdbjava_sources_jar
cd java/crossbuild && (vagrant destroy -f || true) && vagrant up linux32 && vagrant halt linux32 && vagrant up linux64 && vagrant halt linux64 && vagrant up linux64-musl && vagrant halt linux64-musl
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1
rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl
rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl rocksdbjava_javadocs_jar rocksdbjava_sources_jar
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class

View File

@ -18,7 +18,11 @@ public class NativeLibraryLoader {
private static final String sharedLibraryName = Environment.getSharedLibraryName("rocksdb");
private static final String jniLibraryName = Environment.getJniLibraryName("rocksdb");
private static final /* @Nullable */ String fallbackJniLibraryName =
Environment.getFallbackJniLibraryName("rocksdb");
private static final String jniLibraryFileName = Environment.getJniLibraryFileName("rocksdb");
private static final /* @Nullable */ String fallbackJniLibraryFileName =
Environment.getFallbackJniLibraryFileName("rocksdb");
private static final String tempFilePrefix = "librocksdbjni";
private static final String tempFileSuffix = Environment.getJniLibraryExtension();
@ -49,14 +53,33 @@ public class NativeLibraryLoader {
*/
public synchronized void loadLibrary(final String tmpDir) throws IOException {
try {
System.loadLibrary(sharedLibraryName);
} catch(final UnsatisfiedLinkError ule1) {
// try dynamic library
System.loadLibrary(sharedLibraryName);
return;
} catch (final UnsatisfiedLinkError ule) {
// ignore - try from static library
}
try {
// try static library
System.loadLibrary(jniLibraryName);
return;
} catch (final UnsatisfiedLinkError ule) {
// ignore - then try static library fallback or from jar
}
if (fallbackJniLibraryName != null) {
try {
System.loadLibrary(jniLibraryName);
} catch(final UnsatisfiedLinkError ule2) {
loadLibraryFromJar(tmpDir);
// try static library fallback
System.loadLibrary(fallbackJniLibraryName);
return;
} catch (final UnsatisfiedLinkError ule) {
// ignore - then try from jar
}
}
// try jar
loadLibraryFromJar(tmpDir);
}
/**
@ -83,43 +106,62 @@ public class NativeLibraryLoader {
File loadLibraryFromJarToTemp(final String tmpDir)
throws IOException {
final File temp;
if (tmpDir == null || tmpDir.isEmpty()) {
temp = File.createTempFile(tempFilePrefix, tempFileSuffix);
} else {
final File parentDir = new File(tmpDir);
if (!parentDir.exists()) {
throw new RuntimeException(
"Directory: " + parentDir.getAbsolutePath() + " does not exist!");
}
temp = new File(parentDir, jniLibraryFileName);
if (temp.exists() && !temp.delete()) {
throw new RuntimeException("File: " + temp.getAbsolutePath()
+ " already exists and cannot be removed.");
}
if (!temp.createNewFile()) {
throw new RuntimeException("File: " + temp.getAbsolutePath()
+ " could not be created.");
}
}
InputStream is = null;
try {
// attempt to look up the static library in the jar file
String libraryFileName = jniLibraryFileName;
is = getClass().getClassLoader().getResourceAsStream(libraryFileName);
if (!temp.exists()) {
throw new RuntimeException("File " + temp.getAbsolutePath() + " does not exist.");
} else {
temp.deleteOnExit();
}
// attempt to copy the library from the Jar file to the temp destination
try (final InputStream is = getClass().getClassLoader().
getResourceAsStream(jniLibraryFileName)) {
if (is == null) {
throw new RuntimeException(jniLibraryFileName + " was not found inside JAR.");
// is there a fallback we can try
if (fallbackJniLibraryFileName == null) {
throw new RuntimeException(libraryFileName + " was not found inside JAR.");
}
// attempt to look up the fallback static library in the jar file
libraryFileName = fallbackJniLibraryFileName;
is = getClass().getClassLoader().getResourceAsStream(libraryFileName);
if (is == null) {
throw new RuntimeException(libraryFileName + " was not found inside JAR.");
}
}
// create a temporary file to copy the library to
final File temp;
if (tmpDir == null || tmpDir.isEmpty()) {
temp = File.createTempFile(tempFilePrefix, tempFileSuffix);
} else {
Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING);
final File parentDir = new File(tmpDir);
if (!parentDir.exists()) {
throw new RuntimeException(
"Directory: " + parentDir.getAbsolutePath() + " does not exist!");
}
temp = new File(parentDir, libraryFileName);
if (temp.exists() && !temp.delete()) {
throw new RuntimeException(
"File: " + temp.getAbsolutePath() + " already exists and cannot be removed.");
}
if (!temp.createNewFile()) {
throw new RuntimeException("File: " + temp.getAbsolutePath() + " could not be created.");
}
}
if (!temp.exists()) {
throw new RuntimeException("File " + temp.getAbsolutePath() + " does not exist.");
} else {
temp.deleteOnExit();
}
// copy the library from the Jar file to the temp destination
Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING);
// return the temporary library file
return temp;
} finally {
if (is != null) {
is.close();
}
}
return temp;
}
/**

View File

@ -110,8 +110,14 @@ public class Environment {
return String.format("%sjni-linux%s%s", name, arch, getLibcPostfix());
}
} else if (isMac()) {
if (isAarch64()) {
return String.format("%sjni-osx-%s", name, ARCH);
if (is64Bit()) {
final String arch;
if (isAarch64()) {
arch = "arm64";
} else {
arch = "x86_64";
}
return String.format("%sjni-osx-%s", name, arch);
} else {
return String.format("%sjni-osx", name);
}
@ -131,10 +137,25 @@ public class Environment {
throw new UnsupportedOperationException(String.format("Cannot determine JNI library name for ARCH='%s' OS='%s' name='%s'", ARCH, OS, name));
}
public static /*@Nullable*/ String getFallbackJniLibraryName(final String name) {
if (isMac() && is64Bit()) {
return String.format("%sjni-osx", name);
}
return null;
}
public static String getJniLibraryFileName(final String name) {
return appendLibOsSuffix("lib" + getJniLibraryName(name), false);
}
public static /*@Nullable*/ String getFallbackJniLibraryFileName(final String name) {
final String fallbackJniLibraryName = getFallbackJniLibraryName(name);
if (fallbackJniLibraryName == null) {
return null;
}
return appendLibOsSuffix("lib" + fallbackJniLibraryName, false);
}
private static String appendLibOsSuffix(final String libraryFileName, final boolean shared) {
if (isUnix() || isAix() || isSolaris() || isFreeBSD() || isOpenBSD()) {
return libraryFileName + ".so";

View File

@ -9,7 +9,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import static org.assertj.core.api.Assertions.assertThat;
@ -37,18 +36,21 @@ public class EnvironmentTest {
isEqualTo(".jnilib");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-osx.jnilib");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.dylib");
}
@Test
public void mac64() {
setEnvironmentClassFields("mac", "x86-64");
public void mac64_x86_64() {
setEnvironmentClassFields("mac", "x86_64");
assertThat(Environment.isWindows()).isFalse();
assertThat(Environment.getJniLibraryExtension()).
isEqualTo(".jnilib");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-osx.jnilib");
assertThat(Environment.getJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-osx-x86_64.jnilib");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-osx.jnilib");
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.dylib");
}
@ -59,7 +61,9 @@ public class EnvironmentTest {
assertThat(Environment.isWindows()).isFalse();
assertThat(Environment.getJniLibraryExtension()).isEqualTo(".jnilib");
assertThat(Environment.getJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-osx-aarch64.jnilib");
.isEqualTo("librocksdbjni-osx-arm64.jnilib");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-osx.jnilib");
assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.dylib");
}
@ -73,6 +77,7 @@ public class EnvironmentTest {
isEqualTo(".so");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-linux32.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.so");
// Linux musl-libc (Alpine)
@ -103,7 +108,8 @@ public class EnvironmentTest {
assertThat(Environment.isWindows()).isFalse();
assertThat(Environment.getJniLibraryExtension()).
isEqualTo(".so");
Environment.getJniLibraryFileName("rocksdb");
assertThat(Environment.getJniLibraryFileName("rocksdb")).isEqualTo("blah");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
}
@Test
@ -115,6 +121,7 @@ public class EnvironmentTest {
isEqualTo(".so");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-linux64.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.so");
// Linux musl-libc (Alpine)
@ -124,6 +131,7 @@ public class EnvironmentTest {
isEqualTo(".so");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-linux64-musl.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.so");
// UNIX
@ -134,6 +142,7 @@ public class EnvironmentTest {
isEqualTo(".so");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-linux64.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.so");
// AIX
@ -143,6 +152,7 @@ public class EnvironmentTest {
isEqualTo(".so");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-aix64.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.so");
}
@ -161,6 +171,7 @@ public class EnvironmentTest {
isEqualTo(".dll");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-win64.dll");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.dll");
}
@ -177,6 +188,7 @@ public class EnvironmentTest {
assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-ppc64le");
assertThat(Environment.getJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-linux-ppc64le.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so");
// Linux musl-libc (Alpine)
setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, true);
@ -189,6 +201,7 @@ public class EnvironmentTest {
assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-ppc64le-musl");
assertThat(Environment.getJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-linux-ppc64le-musl.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so");
setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false);
}
@ -205,6 +218,7 @@ public class EnvironmentTest {
assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-aarch64");
assertThat(Environment.getJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-linux-aarch64.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so");
// Linux musl-libc (Alpine)
setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, true);
@ -217,6 +231,7 @@ public class EnvironmentTest {
assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-aarch64-musl");
assertThat(Environment.getJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-linux-aarch64-musl.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so");
setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false);
}