Make builds reproducible (#7866)
Summary: Closes https://github.com/facebook/rocksdb/issues/7035 Changed how build_version.cc was generated: - Included the GIT tag/branch in the build_version file - Changed the "Build Date" to be: - If the GIT branch is "clean" (no changes), the date of the last git commit - If the branch is not clean, the current date - Added APIs to access the "build information", rather than accessing the strings directly. The build_version.cc file is now regenerated whenever the library objects are rebuilt. Verified that the built files remain the same size across builds on a "clean build" and the same information is reported by sst_dump --version Pull Request resolved: https://github.com/facebook/rocksdb/pull/7866 Reviewed By: pdillinger Differential Revision: D26086565 Pulled By: mrambacher fbshipit-source-id: 6fcbe47f6033989d5cf26a0ccb6dfdd9dd239d7f
This commit is contained in:
mrambacher
Facebook GitHub Bot
parent
c696f27432
commit
0a9a05ae12
@ -170,23 +170,25 @@ else()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
string(TIMESTAMP TS "%Y/%m/%d %H:%M:%S" UTC)
|
||||
set(GIT_DATE_TIME "${TS}" CACHE STRING "the time we first built rocksdb")
|
||||
string(TIMESTAMP TS "%Y-%m-%d %H:%M:%S" UTC)
|
||||
set(BUILD_DATE "${TS}" CACHE STRING "the time we first built rocksdb")
|
||||
|
||||
find_package(Git)
|
||||
|
||||
if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
|
||||
if(WIN32)
|
||||
execute_process(COMMAND $ENV{COMSPEC} /C ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
|
||||
else()
|
||||
execute_process(COMMAND ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
|
||||
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_SHA COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD )
|
||||
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" RESULT_VARIABLE GIT_MOD COMMAND "${GIT_EXECUTABLE}" diff-index HEAD --quiet)
|
||||
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_DATE COMMAND "${GIT_EXECUTABLE}" log -1 --date=format:"%Y-%m-%d %T" --format="%ad")
|
||||
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_TAG RESULT_VARIABLE rv COMMAND "${GIT_EXECUTABLE}" symbolic-ref -q --short HEAD OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||
if (rv AND NOT rv EQUAL 0)
|
||||
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_TAG COMMAND "${GIT_EXECUTABLE}" describe --tags --exact-match OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||
endif()
|
||||
else()
|
||||
set(GIT_SHA 0)
|
||||
set(GIT_MOD 0)
|
||||
endif()
|
||||
|
||||
string(REGEX REPLACE "[^0-9a-f]+" "" GIT_SHA "${GIT_SHA}")
|
||||
|
||||
string(REGEX REPLACE "[^0-9a-fA-F]+" "" GIT_SHA "${GIT_SHA}")
|
||||
string(REGEX REPLACE "[^0-9: /-]+" "" GIT_DATE "${GIT_DATE}")
|
||||
|
||||
option(WITH_MD_LIBRARY "build with MD" ON)
|
||||
if(WIN32 AND MSVC)
|
||||
@ -199,9 +201,7 @@ endif()
|
||||
|
||||
set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
|
||||
configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
|
||||
add_library(build_version OBJECT ${BUILD_VERSION_CC})
|
||||
target_include_directories(build_version PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/util)
|
||||
|
||||
if(MSVC)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324")
|
||||
@ -831,8 +831,7 @@ set(SOURCES
|
||||
utilities/transactions/write_unprepared_txn_db.cc
|
||||
utilities/ttl/db_ttl_impl.cc
|
||||
utilities/write_batch_with_index/write_batch_with_index.cc
|
||||
utilities/write_batch_with_index/write_batch_with_index_internal.cc
|
||||
$<TARGET_OBJECTS:build_version>)
|
||||
utilities/write_batch_with_index/write_batch_with_index_internal.cc)
|
||||
|
||||
list(APPEND SOURCES
|
||||
utilities/transactions/lock/range/range_tree/lib/locktree/concurrent_tree.cc
|
||||
@ -923,12 +922,12 @@ else()
|
||||
set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
|
||||
endif()
|
||||
|
||||
add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
|
||||
add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES} ${BUILD_VERSION_CC})
|
||||
target_link_libraries(${ROCKSDB_STATIC_LIB} PRIVATE
|
||||
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
||||
|
||||
if(ROCKSDB_BUILD_SHARED)
|
||||
add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
|
||||
add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES} ${BUILD_VERSION_CC})
|
||||
target_link_libraries(${ROCKSDB_SHARED_LIB} PRIVATE
|
||||
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
||||
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
### Public API Change
|
||||
* Add a public API WriteBufferManager::dummy_entries_in_cache_usage() which reports the size of dummy entries stored in cache (passed to WriteBufferManager). Dummy entries are used to account for DataBlocks.
|
||||
* Add a SystemClock class that contains the time-related methods from Env. The original methods in Env may be deprecated in a future release. This class will allow easier testing, development, and expansion of time-related features.
|
||||
* Add a public API GetRocksBuildProperties and GetRocksBuildInfoAsString to get properties about the current build. These properties may include settings related to the GIT settings (branch, timestamp). This change also sets the "build date" based on the GIT properties, rather than the actual build time, thereby enabling more reproducible builds.
|
||||
## 6.16.0 (12/18/2020)
|
||||
### Behavior Changes
|
||||
* Attempting to write a merge operand without explicitly configuring `merge_operator` now fails immediately, causing the DB to enter read-only mode. Previously, failure was deferred until the `merge_operator` was needed by a user read or a background operation.
|
||||
|
||||
56
Makefile
56
Makefile
@ -479,31 +479,6 @@ CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverl
|
||||
|
||||
LDFLAGS += $(PLATFORM_LDFLAGS)
|
||||
|
||||
# If NO_UPDATE_BUILD_VERSION is set we don't update util/build_version.cc, but
|
||||
# the file needs to already exist or else the build will fail
|
||||
ifndef NO_UPDATE_BUILD_VERSION
|
||||
date := $(shell date +%F)
|
||||
ifdef FORCE_GIT_SHA
|
||||
git_sha := $(FORCE_GIT_SHA)
|
||||
else
|
||||
git_sha := $(shell git rev-parse HEAD 2>/dev/null)
|
||||
endif
|
||||
gen_build_version = sed -e s/@@GIT_SHA@@/$(git_sha)/ -e s/@@GIT_DATE_TIME@@/$(date)/ util/build_version.cc.in
|
||||
|
||||
# Record the version of the source that we are compiling.
|
||||
# We keep a record of the git revision in this file. It is then built
|
||||
# as a regular source file as part of the compilation process.
|
||||
# One can run "strings executable_filename | grep _build_" to find
|
||||
# the version of the source that we used to build the executable file.
|
||||
FORCE:
|
||||
util/build_version.cc: FORCE
|
||||
$(AM_V_GEN)rm -f $@-t
|
||||
$(AM_V_at)$(gen_build_version) > $@-t
|
||||
$(AM_V_at)if test -f $@; then \
|
||||
cmp -s $@-t $@ && rm -f $@-t || mv -f $@-t $@; \
|
||||
else mv -f $@-t $@; fi
|
||||
endif
|
||||
|
||||
OBJ_DIR?=.
|
||||
LIB_OBJECTS = $(patsubst %.cc, $(OBJ_DIR)/%.o, $(LIB_SOURCES))
|
||||
ifeq ($(HAVE_POWER8),1)
|
||||
@ -863,6 +838,37 @@ ROCKSDB_MAJOR = $(shell egrep "ROCKSDB_MAJOR.[0-9]" include/rocksdb/version.h |
|
||||
ROCKSDB_MINOR = $(shell egrep "ROCKSDB_MINOR.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
|
||||
ROCKSDB_PATCH = $(shell egrep "ROCKSDB_PATCH.[0-9]" include/rocksdb/version.h | cut -d ' ' -f 3)
|
||||
|
||||
# If NO_UPDATE_BUILD_VERSION is set we don't update util/build_version.cc, but
|
||||
# the file needs to already exist or else the build will fail
|
||||
ifndef NO_UPDATE_BUILD_VERSION
|
||||
|
||||
# By default, use the current date-time as the date. If there are no changes,
|
||||
# we will use the last commit date instead.
|
||||
build_date := $(shell date "+%Y-%m-%d %T")
|
||||
|
||||
ifdef FORCE_GIT_SHA
|
||||
git_sha := $(FORCE_GIT_SHA)
|
||||
git_mod := 1
|
||||
git_date := $(build_date)
|
||||
else
|
||||
git_sha := $(shell git rev-parse HEAD 2>/dev/null)
|
||||
git_tag := $(shell git symbolic-ref -q --short HEAD || git describe --tags --exact-match 2>/dev/null)
|
||||
git_mod := $(shell git diff-index HEAD --quiet 2>/dev/null; echo $$?)
|
||||
git_date := $(shell git log -1 --date=format:"%Y-%m-%d %T" --format="%ad" 2>/dev/null)
|
||||
endif
|
||||
gen_build_version = sed -e s/@GIT_SHA@/$(git_sha)/ -e s:@GIT_TAG@:"$(git_tag)": -e s/@GIT_MOD@/"$(git_mod)"/ -e s/@BUILD_DATE@/"$(build_date)"/ -e s/@GIT_DATE@/"$(git_date)"/ util/build_version.cc.in
|
||||
|
||||
# Record the version of the source that we are compiling.
|
||||
# We keep a record of the git revision in this file. It is then built
|
||||
# as a regular source file as part of the compilation process.
|
||||
# One can run "strings executable_filename | grep _build_" to find
|
||||
# the version of the source that we used to build the executable file.
|
||||
util/build_version.cc: $(filter-out $(OBJ_DIR)/util/build_version.o, $(LIB_OBJECTS)) util/build_version.cc.in
|
||||
$(AM_V_GEN)rm -f $@-t
|
||||
$(AM_V_at)$(gen_build_version) > $@
|
||||
endif
|
||||
CLEAN_FILES += util/build_version.cc
|
||||
|
||||
default: all
|
||||
|
||||
#-----------------------------------------------
|
||||
|
||||
@ -82,6 +82,7 @@
|
||||
#include "rocksdb/stats_history.h"
|
||||
#include "rocksdb/status.h"
|
||||
#include "rocksdb/table.h"
|
||||
#include "rocksdb/version.h"
|
||||
#include "rocksdb/write_buffer_manager.h"
|
||||
#include "table/block_based/block.h"
|
||||
#include "table/block_based/block_based_table_factory.h"
|
||||
@ -93,7 +94,6 @@
|
||||
#include "table/two_level_iterator.h"
|
||||
#include "test_util/sync_point.h"
|
||||
#include "util/autovector.h"
|
||||
#include "util/build_version.h"
|
||||
#include "util/cast_util.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/compression.h"
|
||||
@ -4207,16 +4207,17 @@ void DBImpl::EraseThreadStatusDbInfo() const {}
|
||||
//
|
||||
// A global method that can dump out the build version
|
||||
void DumpRocksDBBuildVersion(Logger* log) {
|
||||
#if !defined(IOS_CROSS_COMPILE)
|
||||
// if we compile with Xcode, we don't run build_detect_version, so we don't
|
||||
// generate util/build_version.cc
|
||||
ROCKS_LOG_HEADER(log, "RocksDB version: %d.%d.%d\n", ROCKSDB_MAJOR,
|
||||
ROCKSDB_MINOR, ROCKSDB_PATCH);
|
||||
ROCKS_LOG_HEADER(log, "Git sha %s", rocksdb_build_git_sha);
|
||||
ROCKS_LOG_HEADER(log, "Compile date %s", rocksdb_build_compile_date);
|
||||
#else
|
||||
(void)log; // ignore "-Wunused-parameter"
|
||||
#endif
|
||||
ROCKS_LOG_HEADER(log, "RocksDB version: %s\n",
|
||||
GetRocksVersionAsString().c_str());
|
||||
const auto& props = GetRocksBuildProperties();
|
||||
const auto& sha = props.find("rocksdb_build_git_sha");
|
||||
if (sha != props.end()) {
|
||||
ROCKS_LOG_HEADER(log, "Git sha %s", sha->second.c_str());
|
||||
}
|
||||
const auto date = props.find("rocksdb_build_date");
|
||||
if (date != props.end()) {
|
||||
ROCKS_LOG_HEADER(log, "Compile date %s", date->second.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
|
||||
@ -26,7 +26,6 @@
|
||||
#include "rocksdb/sst_partitioner.h"
|
||||
#include "rocksdb/types.h"
|
||||
#include "rocksdb/universal_compaction.h"
|
||||
#include "rocksdb/version.h"
|
||||
#include "rocksdb/write_buffer_manager.h"
|
||||
|
||||
#ifdef max
|
||||
|
||||
@ -4,6 +4,11 @@
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "rocksdb/rocksdb_namespace.h"
|
||||
|
||||
#define ROCKSDB_MAJOR 6
|
||||
#define ROCKSDB_MINOR 17
|
||||
#define ROCKSDB_PATCH 0
|
||||
@ -14,3 +19,23 @@
|
||||
#define __ROCKSDB_MAJOR__ ROCKSDB_MAJOR
|
||||
#define __ROCKSDB_MINOR__ ROCKSDB_MINOR
|
||||
#define __ROCKSDB_PATCH__ ROCKSDB_PATCH
|
||||
|
||||
namespace ROCKSDB_NAMESPACE {
|
||||
// Returns a set of properties indicating how/when/where this version of RocksDB
|
||||
// was created.
|
||||
const std::unordered_map<std::string, std::string>& GetRocksBuildProperties();
|
||||
|
||||
// Returns the current version of RocksDB as a string (e.g. "6.16.0").
|
||||
// If with_patch is true, the patch is included (6.16.x).
|
||||
// Otherwise, only major and minor version is included (6.16)
|
||||
std::string GetRocksVersionAsString(bool with_patch = true);
|
||||
|
||||
// Gets the set of build properties (@see GetRocksBuildProperties) into a
|
||||
// string. Properties are returned one-per-line, with the first line being:
|
||||
// "<program> from RocksDB <version>.
|
||||
// If verbose is true, the full set of properties is
|
||||
// printed. If verbose is false, only the version information (@see
|
||||
// GetRocksVersionString) is printed.
|
||||
std::string GetRocksBuildInfoAsString(const std::string& program,
|
||||
bool verbose = false);
|
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
|
||||
@ -280,8 +280,7 @@ int SSTDumpTool::Run(int argc, char const* const* argv, Options options) {
|
||||
print_help(/*to_stderr*/ false);
|
||||
return 0;
|
||||
} else if (strcmp(argv[i], "--version") == 0) {
|
||||
printf("sst_dump from RocksDB %d.%d.%d\n", ROCKSDB_MAJOR, ROCKSDB_MINOR,
|
||||
ROCKSDB_PATCH);
|
||||
printf("%s\n", GetRocksBuildInfoAsString("sst_dump").c_str());
|
||||
return 0;
|
||||
} else {
|
||||
fprintf(stderr, "Unrecognized argument '%s'\n\n", argv[i]);
|
||||
|
||||
@ -1,5 +1,71 @@
|
||||
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
||||
#include "build_version.h"
|
||||
const char* rocksdb_build_git_sha = "rocksdb_build_git_sha:@@GIT_SHA@@";
|
||||
const char* rocksdb_build_git_date = "rocksdb_build_git_date:@@GIT_DATE_TIME@@";
|
||||
const char* rocksdb_build_compile_date = __DATE__;
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "rocksdb/version.h"
|
||||
#include "util/string_util.h"
|
||||
|
||||
// The build script may replace these values with real values based
|
||||
// on whether or not GIT is available and the platform settings
|
||||
static const std::string rocksdb_build_git_sha = "rocksdb_build_git_sha:@GIT_SHA@";
|
||||
static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:@GIT_TAG@";
|
||||
#define HAS_GIT_CHANGES @GIT_MOD@
|
||||
#if HAS_GIT_CHANGES == 0
|
||||
// If HAS_GIT_CHANGES is 0, the GIT date is used.
|
||||
// Use the time the branch/tag was last modified
|
||||
static const std::string rocksdb_build_date = "rocksdb_build_date:@GIT_DATE@";
|
||||
#else
|
||||
// If HAS_GIT_CHANGES is > 0, the branch/tag has modifications.
|
||||
// Use the time the build was created.
|
||||
static const std::string rocksdb_build_date = "rocksdb_build_date:@BUILD_DATE@";
|
||||
#endif
|
||||
|
||||
namespace ROCKSDB_NAMESPACE {
|
||||
static void AddProperty(std::unordered_map<std::string, std::string> *props, const std::string& name) {
|
||||
size_t colon = name.find(":");
|
||||
if (colon != std::string::npos && colon > 0 && colon < name.length() - 1) {
|
||||
// If we found a "@:", then this property was a build-time substitution that failed. Skip it
|
||||
size_t at = name.find("@", colon);
|
||||
if (at != colon + 1) {
|
||||
// Everything before the colon is the name, after is the value
|
||||
(*props)[name.substr(0, colon)] = name.substr(colon + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static std::unordered_map<std::string, std::string>* LoadPropertiesSet() {
|
||||
auto * properties = new std::unordered_map<std::string, std::string>();
|
||||
AddProperty(properties, rocksdb_build_git_sha);
|
||||
AddProperty(properties, rocksdb_build_git_tag);
|
||||
AddProperty(properties, rocksdb_build_date);
|
||||
return properties;
|
||||
}
|
||||
|
||||
const std::unordered_map<std::string, std::string>& GetRocksBuildProperties() {
|
||||
static std::unique_ptr<std::unordered_map<std::string, std::string>> props(LoadPropertiesSet());
|
||||
return *props;
|
||||
}
|
||||
|
||||
std::string GetRocksVersionAsString(bool with_patch) {
|
||||
std::string version = ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR);
|
||||
if (with_patch) {
|
||||
return version + "." + ToString(ROCKSDB_PATCH);
|
||||
} else {
|
||||
return version;
|
||||
}
|
||||
}
|
||||
|
||||
std::string GetRocksBuildInfoAsString(const std::string& program, bool verbose) {
|
||||
std::string info = program + " (RocksDB) " + GetRocksVersionAsString(true);
|
||||
if (verbose) {
|
||||
for (const auto& it : GetRocksBuildProperties()) {
|
||||
info.append("\n ");
|
||||
info.append(it.first);
|
||||
info.append(": ");
|
||||
info.append(it.second);
|
||||
}
|
||||
}
|
||||
return info;
|
||||
}
|
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
|
||||
|
||||
@ -1,15 +0,0 @@
|
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
//
|
||||
#pragma once
|
||||
#if !defined(IOS_CROSS_COMPILE)
|
||||
// if we compile with Xcode, we don't run build_detect_version, so we don't
|
||||
// generate these variables
|
||||
// this variable tells us about the git revision
|
||||
extern const char* rocksdb_build_git_sha;
|
||||
|
||||
// Date on which the code was compiled:
|
||||
extern const char* rocksdb_build_compile_date;
|
||||
#endif
|
||||
Loading…
Reference in New Issue
Block a user