Clean up the Makefile and the build scripts

Summary: As Aaron suggested, there are quite some problems with our Makefile and scripts. So in this diff I did some cleanup for them and revise some part of the scripts/makefile to help people better understand some mysterious parts.

Test Plan:
Ran make in several modes;
Ran the updated scripts.

Reviewers: dhruba, emayanke, akushner

Differential Revision: https://reviews.facebook.net/D12285

.gitignore

@@ -18,3 +18,6 @@ leveldb_shell
 manifest_dump
 sst_dump
 util/build_version.cc
+build_tools/VALGRIND_LOGS/
+coverage/COVERAGE_REPORT
+util/build_version.cc.tmp

Makefile

@@ -14,7 +14,7 @@ OPT += -O2 -fno-omit-frame-pointer -momit-leaf-frame-pointer
 #-----------------------------------------------
 
 # detect what platform we're building on
-$(shell ./build_detect_platform build_config.mk)
+$(shell (cd build_tools/; ROCKSDB_ROOT=.. ./build_detect_platform ../build_config.mk))
 # this file is generated by the previous line to set build flags and sources
 include build_config.mk
 
@@ -31,7 +31,7 @@ MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o)
 TESTUTIL = ./util/testutil.o
 TESTHARNESS = ./util/testharness.o $(TESTUTIL)
 VALGRIND_ERROR = 2
-VALGRIND_DIR = VALGRIND_LOGS
+VALGRIND_DIR = build_tools/VALGRIND_LOGS
 VALGRIND_VER := $(join $(VALGRIND_VER),valgrind)
 VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full
 
@@ -109,7 +109,8 @@ endif  # PLATFORM_SHARED_EXT
 
 all: $(SHARED) $(LIBRARY) $(PROGRAMS)
 
-.PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests release tags valgrind_check whitebox_crash_test
+.PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests \
+	release tags valgrind_check whitebox_crash_test
 
 release:
 	$(MAKE) clean
@@ -120,7 +121,7 @@ coverage:
 	COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" $(MAKE) all check
 	(cd coverage; ./coverage_test.sh)
 	# Delete intermediate files
-	find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs rm
+	find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs --no-run-if-empty rm
 
 check: all $(PROGRAMS) $(TESTS) $(TOOLS) ldb_tests
 	for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
@@ -137,6 +138,7 @@ whitebox_crash_test: db_stress
 	python -u tools/db_crashtest2.py
 
 valgrind_check: all $(PROGRAMS) $(TESTS)
+	mkdir -p $(VALGRIND_DIR)
 	echo TESTS THAT HAVE VALGRIND ERRORS > $(VALGRIND_DIR)/valgrind_failed_tests; \
 	echo TIMES in seconds TAKEN BY TESTS ON VALGRIND > $(VALGRIND_DIR)/valgrind_tests_times; \
 	for t in $(filter-out skiplist_test,$(TESTS)); do \
@@ -152,13 +154,19 @@ valgrind_check: all $(PROGRAMS) $(TESTS)
 clean:
 	-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) $(THRIFTSERVER) build_config.mk
 	-rm -rf ios-x86/* ios-arm/*
-	-find . -name "*.[od]" | xargs rm
+	-find . -name "*.[od]" | xargs --no-run-if-empty rm
+	-find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs --no-run-if-empty rm
+tags:
+	ctags * -R
+	cscope -b `find . -name '*.cc'` `find . -name '*.h'`
 
+# ---------------------------------------------------------------------------
+# 	Unit tests and tools
+# ---------------------------------------------------------------------------
 $(LIBRARY): $(LIBOBJECTS)
 	rm -f $@
 	$(AR) -rs $@ $(LIBOBJECTS)
 
-
 db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL)
 	$(CXX) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
 
@@ -283,9 +291,9 @@ sst_dump: tools/sst_dump.o $(LIBOBJECTS)
 ldb: tools/ldb.o $(LIBOBJECTS)
 	$(CXX) tools/ldb.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
-tags:
+# ---------------------------------------------------------------------------
-	ctags `find . -name '*.cc'` `find . -name '*.h'`
+#  	Platform-specific compilation
-	cscope -b `find . -name '*.cc'` `find . -name '*.h'`
+# ---------------------------------------------------------------------------
 
 ifeq ($(PLATFORM), IOS)
 # For iOS, create universal object files to be used on both the simulator and
@@ -317,9 +325,20 @@ else
 	$(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
 endif
 
+# ---------------------------------------------------------------------------
+#  	Source files dependencies detection
+# ---------------------------------------------------------------------------
+
+# Add proper dependency support so changing a .h file forces a .cc file to
+# rebuild.
+
+# The .d file indicates .cc file's dependencies on .h files. We generate such
+# dependency by g++'s -MM option, whose output is a make dependency rule.
+# The sed command makes sure the "target" file in the generated .d file has
+# the correct path prefix.
 %.d: %.cc
-	$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -MM $< -o $*.d $(COVERAGEFLAGS)
+	$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -MM $< -o $@
-	@sed -i -e 's|.*:|$*.o:|' < $*.d
+	@sed -i -e 's|.*:|$*.o:|' $@
 
 DEPFILES = $(filter-out util/build_version.d,$(SOURCES:.cc=.d))
 
@@ -328,4 +347,3 @@ depend: $(DEPFILES)
 ifneq ($(MAKECMDGOALS),clean)
 -include $(DEPFILES)
 endif
-

VALGRIND_LOGS/README

@@ -1,4 +0,0 @@
-This directory stores the tests that failed valgrind and the times associated
-with the failed runs.
-"make valgrind_check" can be invoked to call valgrind on the rocksdb tests and
-generate files in this directory

build_detect_version

@@ -1,30 +0,0 @@
-#!/bin/sh
-#
-# Record the version of the source that we are compiling.
-# We keep a record of the git revision in util/version.cc. This source file
-# is then built as a regular source file as part of the compilation process.
-# One can run "strings executable_filename | grep _build_" to find the version of
-# the source that we used to build the executable file.
-#
-
-# create git version file
-VFILE=$(mktemp)
-trap "rm $VFILE" EXIT
-
-# check to see if git is in the path
-which git > /dev/null
-
-if [ "$?" = 0 ]; then
-  env -i git rev-parse HEAD | awk ' BEGIN {print "#include \"build_version.h\""} {print "const char * leveldb_build_git_sha = \"leveldb_build_git_sha:" $0"\";"} END {}' >  ${VFILE}
-else
-  echo "git not found"| awk ' BEGIN {print "#include \"build_version.h\""} {print "const char * leveldb_build_git_sha = \"leveldb_build_git_sha:git not found\";"} END {}' > ${VFILE}
-fi
-
-date | awk 'BEGIN {} {print "const char * leveldb_build_git_datetime = \"leveldb_build_git_datetime:"$0"\";"} END {} ' >> ${VFILE}
-echo "const char * leveldb_build_compile_date = __DATE__;" >> ${VFILE}
-echo "const char * leveldb_build_compile_time = __TIME__;" >> ${VFILE}
-
-OUTFILE=util/build_version.cc
-if [ ! -e $OUTFILE ] || ! cmp -s $VFILE $OUTFILE; then
-    cp $VFILE $OUTFILE
-fi

build_tools/build_detect_platform

@@ -31,9 +31,9 @@ fi
 # Default to fbcode gcc on internal fb machines
 if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
     if [ -z "$USE_CLANG" ]; then
-        source ./fbcode.gcc471.sh
+        source fbcode.gcc471.sh
     else
-        source ./fbcode.clang31.sh
+        source fbcode.clang31.sh
     fi
 fi
 
@@ -65,7 +65,7 @@ PLATFORM_SHARED_CFLAGS="-fPIC"
 PLATFORM_SHARED_VERSIONED=true
 
 # generic port files (working on all platform by #ifdef) go directly in /port
-GENERIC_PORT_FILES=`find port -name '*.cc' | tr "\n" " "`
+GENERIC_PORT_FILES=`cd $ROCKSDB_ROOT; find port -name '*.cc' | tr "\n" " "`
 
 # On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
 case "$TARGET_OS" in
@@ -146,8 +146,8 @@ fi
 set -f # temporarily disable globbing so that our patterns arent expanded
 PRUNE_TEST="-name *test*.cc -prune"
 PRUNE_BENCH="-name *_bench.cc -prune"
-PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
+PORTABLE_FILES=`cd $ROCKSDB_ROOT; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
-PORTABLE_CPP=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort | tr "\n" " "`
+PORTABLE_CPP=`cd $ROCKSDB_ROOT; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort | tr "\n" " "`
 set +f # re-enable globbing
 
 # The sources consist of the portable files, plus the platform-specific port

build_tools/build_detect_version

@@ -0,0 +1,42 @@
+#!/bin/sh
+#
+# Record the version of the source that we are compiling.
+# We keep a record of the git revision in util/version.cc. This source file
+# is then built as a regular source file as part of the compilation process.
+# One can run "strings executable_filename | grep _build_" to find the version of
+# the source that we used to build the executable file.
+#
+
+# create git version file
+VFILE=$ROCKSDB_ROOT/util/build_version.cc.tmp
+trap "rm $VFILE" EXIT
+
+# check to see if git is in the path
+which git > /dev/null
+
+if [ "$?" = 0 ]; then
+  env -i git rev-parse HEAD |
+  awk '
+  BEGIN {
+    print "#include \"build_version.h\"\n"
+  } 
+  { print "const char* leveldb_build_git_sha = \"leveldb_build_git_sha:" $0"\";" }
+  ' >  ${VFILE}
+else
+  echo "git not found" |
+  awk '
+    BEGIN {
+      print "#include \"build_version.h\"" 
+    } 
+    { print "const char* leveldb_build_git_sha = \"leveldb_build_git_sha:git not found\";" }
+    ' > ${VFILE}
+fi
+
+echo "const char* leveldb_build_git_datetime = \"leveldb_build_git_datetime:$(date)\";" >> ${VFILE}
+echo "const char* leveldb_build_compile_date = __DATE__;" >> ${VFILE}
+echo "const char* leveldb_build_compile_time = __TIME__;" >> ${VFILE}
+
+OUTFILE=$ROCKSDB_ROOT/util/build_version.cc
+if [ ! -e $OUTFILE ] || ! cmp -s $VFILE $OUTFILE; then
+    cp $VFILE $OUTFILE
+fi

build_tools/regression_build_test.sh

@@ -14,8 +14,8 @@ fi
 # On the production build servers, set data and stat
 # files/directories not in /tmp or else the tempdir cleaning
 # scripts will make you very unhappy.
-DATA_DIR=${DATA_DIR:-$(mktemp --tmpdir -d rocksdb_XXXX)}
+DATA_DIR=${DATA_DIR:-$(mktemp -t -d rocksdb_XXXX)}
-STAT_FILE=${STAT_FILE:-$(mktemp --tmpdir -u rocksdb_test_stats_XXXX)}
+STAT_FILE=${STAT_FILE:-$(mktemp -t -u rocksdb_test_stats_XXXX)}
 
 function cleanup {
   rm -rf $DATA_DIR

build_tools/valgrind_test.sh

@@ -2,7 +2,7 @@
 #A shell script for Jenknis to run valgrind on rocksdb tests
 #Returns 0 on success when there are no failed tests 
 
-VALGRIND_DIR=VALGRIND_LOGS
+VALGRIND_DIR=build_tools/VALGRIND_LOGS
 make clean
 make -j$(nproc) valgrind_check
 NUM_FAILED_TESTS=$((`wc -l $VALGRIND_DIR/valgrind_failed_tests | awk '{print $1}'` - 1))

coverage/coverage_test.sh

@@ -11,13 +11,13 @@ fi
 ROOT=".."
 # Fetch right version of gcov
 if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
-  source $ROOT/fbcode.gcc471.sh
+  source $ROOT/build_tools/fbcode.gcc471.sh
   GCOV=$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1/cc6c9dc/bin/gcov
 else
   GCOV=$(which gcov)
 fi
 
-COVERAGE_DIR=$(mktemp -t -d rocksdb_coverage_XXXX)
+COVERAGE_DIR="$PWD/COVERAGE_REPORT"
 mkdir -p $COVERAGE_DIR
 
 # Find all gcno files to generate the coverage report
@@ -47,16 +47,14 @@ echo -e "Generated coverage report for recently updated files: $RECENT_REPORT\n"
 # Generate the html report. If we cannot find lcov in this machine, we'll simply
 # skip this step.
 echo "Generating the html coverage report..."
-set +e
+
-LCOV=$(which lcov 2>/dev/null)
+LCOV=$(which lcov || true 2>/dev/null)
 if [ -z $LCOV ]
 then
   echo "Skip: Cannot find lcov to generate the html report."
   exit 0
 fi
 
-set -e
-
 (cd $ROOT; lcov --no-external \
      --capture  \
      --directory $PWD \

e

@@ -1,2 +0,0 @@
-ar: creating libmemenv.a
-ar: creating libleveldb.a