andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Small improvements to Docker build for RocksJava (#6079)

Browse Source 
Summary:
* We can reuse downloaded 3rd-party libraries
* We can isolate the build to a Docker volume. This is useful for investigating failed builds, as we can examine the volume by assigning it a name during the build.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6079

Differential Revision: D18710263

fbshipit-source-id: 93f456ba44b49e48941c43b0c4d53995ecc1f404
This commit is contained in:
Adam Retter 2019-11-26 10:52:04 -08:00 committed by Facebook Github Bot
parent 4f17d33db4
commit 7f14519577
3 changed files with 29 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
Makefile
View File

@ -1103,14 +1103,23 @@ unity_test: db/db_test.o db/db_test_util.o $(TESTHARNESS) $(TOOLLIBOBJECTS) unit
rocksdb.h rocksdb.cc: build_tools/amalgamate.py Makefile $(LIB_SOURCES) unity.cc rocksdb.h rocksdb.cc: build_tools/amalgamate.py Makefile $(LIB_SOURCES) unity.cc
build_tools/amalgamate.py -I. -i./include unity.cc -x include/rocksdb/c.h -H rocksdb.h -o rocksdb.cc build_tools/amalgamate.py -I. -i./include unity.cc -x include/rocksdb/c.h -H rocksdb.h -o rocksdb.cc
clean: clean: clean-ext-libraries-all clean-rocks
clean-not-downloaded: clean-ext-libraries-bin clean-rocks
clean-rocks:
rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(PARALLEL_TEST) $(LIBRARY) $(SHARED) rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(PARALLEL_TEST) $(LIBRARY) $(SHARED)
rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report
$(FIND) . -name "*.[oda]" -exec rm -f {} \; $(FIND) . -name "*.[oda]" -exec rm -f {} \;
$(FIND) . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \; $(FIND) . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
rm -rf bzip2* snappy* zlib* lz4* zstd*
cd java; $(MAKE) clean cd java; $(MAKE) clean
clean-ext-libraries-all:
rm -rf bzip2* snappy* zlib* lz4* zstd*
clean-ext-libraries-bin:
find . -maxdepth 1 -type d \( -name bzip2\* -or -name snappy\* -or -name zlib\* -or -name lz4\* -or -name zstd\* \) -prune -exec rm -rf {} \;
tags: tags:
ctags -R . ctags -R .
cscope -b `$(FIND) . -name '*.cc'` `$(FIND) . -name '*.h'` `$(FIND) . -name '*.c'` cscope -b `$(FIND) . -name '*.cc'` `$(FIND) . -name '*.h'` `$(FIND) . -name '*.c'`
@ -1913,23 +1922,19 @@ rocksdbjavastaticreleasedocker: rocksdbjavastatic rocksdbjavastaticdockerx86 roc
rocksdbjavastaticdockerx86: rocksdbjavastaticdockerx86:
mkdir -p java/target mkdir -p java/target
docker run --rm --name rocksdb_linux_x86-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh docker run --rm --name rocksdb_linux_x86-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
rocksdbjavastaticdockerx86_64: rocksdbjavastaticdockerx86_64:
mkdir -p java/target mkdir -p java/target
docker run --rm --name rocksdb_linux_x64-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh docker run --rm --name rocksdb_linux_x64-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
rocksdbjavastaticdockerppc64le: rocksdbjavastaticdockerppc64le:
mkdir -p java/target mkdir -p java/target
docker run --rm --name rocksdb_linux_ppc64le-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh docker run --rm --name rocksdb_linux_ppc64le-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
rocksdbjavastaticdockerarm64v8: rocksdbjavastaticdockerarm64v8:
mkdir -p java/target mkdir -p java/target
DOCKER_LINUX_ARM64V8_CONTAINER=`docker ps -aqf name=rocksdb_linux_arm64v8-be`; \ docker run --rm --name rocksdb_linux_arm64v8-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
if [ -z "$$DOCKER_LINUX_ARM64V8_CONTAINER" ]; then \
docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_arm64v8-be evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
fi
docker start -a rocksdb_linux_arm64v8-be
rocksdbjavastaticpublish: rocksdbjavastaticrelease rocksdbjavastaticpublishcentral rocksdbjavastaticpublish: rocksdbjavastaticrelease rocksdbjavastaticpublishcentral

2
java/crossbuild/build-linux-centos.sh
View File

@ -32,7 +32,7 @@ export PATH=$JAVA_HOME:/usr/local/bin:$PATH
# build rocksdb # build rocksdb
cd /rocksdb cd /rocksdb
scl enable devtoolset-2 'make jclean clean' scl enable devtoolset-2 'make clean-not-downloaded'
scl enable devtoolset-2 'PORTABLE=1 make -j8 rocksdbjavastatic' scl enable devtoolset-2 'PORTABLE=1 make -j8 rocksdbjavastatic'
cp /rocksdb/java/target/librocksdbjni-* /rocksdb-build cp /rocksdb/java/target/librocksdbjni-* /rocksdb-build
cp /rocksdb/java/target/rocksdbjni-* /rocksdb-build cp /rocksdb/java/target/rocksdbjni-* /rocksdb-build

21
java/crossbuild/docker-build-linux-centos.sh
View File

@ -4,26 +4,31 @@
set -e set -e
#set -x #set -x
rm -rf /rocksdb-local # just in-case this is run outside Docker
cp -r /rocksdb-host /rocksdb-local mkdir -p /rocksdb-local-build
cd /rocksdb-local
# Use scl devtoolset if available (i.e. CentOS <7) rm -rf /rocksdb-local-build/*
cp -r /rocksdb-host/* /rocksdb-local-build
cd /rocksdb-local-build
# Use scl devtoolset if available
if hash scl 2>/dev/null; then if hash scl 2>/dev/null; then
if scl --list | grep -q 'devtoolset-7'; then if scl --list | grep -q 'devtoolset-7'; then
scl enable devtoolset-7 'make jclean clean' # CentOS 7+
scl enable devtoolset-7 'make clean-not-downloaded'
scl enable devtoolset-7 'PORTABLE=1 make -j2 rocksdbjavastatic' scl enable devtoolset-7 'PORTABLE=1 make -j2 rocksdbjavastatic'
elif scl --list | grep -q 'devtoolset-2'; then elif scl --list | grep -q 'devtoolset-2'; then
scl enable devtoolset-2 'make jclean clean' # CentOS 5 or 6
scl enable devtoolset-2 'make clean-not-downloaded'
scl enable devtoolset-2 'PORTABLE=1 make -j2 rocksdbjavastatic' scl enable devtoolset-2 'PORTABLE=1 make -j2 rocksdbjavastatic'
else else
echo "Could not find devtoolset" echo "Could not find devtoolset"
exit 1; exit 1;
fi fi
else else
make jclean clean make clean-not-downloaded
PORTABLE=1 make -j2 rocksdbjavastatic PORTABLE=1 make -j2 rocksdbjavastatic
fi fi
cp java/target/librocksdbjni-linux*.so java/target/rocksdbjni-*-linux*.jar /rocksdb-host/java/target cp java/target/librocksdbjni-linux*.so java/target/rocksdbjni-*-linux*.jar /rocksdb-java-target