build: make it compile with @mode/win (#7406)

Summary:
While rocksdb can compile on both macOS and Linux with Buck, it couldn't be
compiled on Windows. The only way to compile it on Windows was with the CMake
build.

To keep the multi-platform complexity low, I've simply included all the Windows
bits in the TARGETS file, and added large #if blocks when not on Windows, the
same was done on the posix specific files.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7406

Test Plan:
On my devserver:
  buck test //rocksdb/...
On Windows:
  buck build mode/win //rocksdb/src:rocksdb_lib

Reviewed By: pdillinger

Differential Revision: D23874358

Pulled By: xavierd

fbshipit-source-id: 8768b5d16d7e8f44b5ca1e2483881ca4b24bffbe
This commit is contained in:
Xavier Deguillard 2020-09-23 12:54:29 -07:00 committed by Facebook GitHub Bot
parent ac1734d06b
commit 249f2b59a0
18 changed files with 123 additions and 48 deletions

30
TARGETS
View File

@ -25,13 +25,16 @@ ROCKSDB_EXTERNAL_DEPS = [
("gflags", None, "gflags"), ("gflags", None, "gflags"),
("lz4", None, "lz4"), ("lz4", None, "lz4"),
("zstd", None), ("zstd", None),
("tbb", None),
] ]
ROCKSDB_OS_DEPS = [ ROCKSDB_OS_DEPS = [
( (
"linux", "linux",
["third-party//numa:numa", "third-party//liburing:uring"], ["third-party//numa:numa", "third-party//liburing:uring", "third-party//tbb:tbb"],
),
(
"macos",
["third-party//tbb:tbb"],
), ),
] ]
@ -49,17 +52,27 @@ ROCKSDB_OS_PREPROCESSOR_FLAGS = [
"-DHAVE_SSE42", "-DHAVE_SSE42",
"-DLIBURING", "-DLIBURING",
"-DNUMA", "-DNUMA",
"-DROCKSDB_PLATFORM_POSIX",
"-DROCKSDB_LIB_IO_POSIX",
"-DTBB",
], ],
), ),
( (
"macos", "macos",
["-DOS_MACOSX"], [
"-DOS_MACOSX",
"-DROCKSDB_PLATFORM_POSIX",
"-DROCKSDB_LIB_IO_POSIX",
"-DTBB",
],
),
(
"windows",
[ "-DOS_WIN", "-DWIN32", "-D_MBCS", "-DWIN64", "-DNOMINMAX" ]
), ),
] ]
ROCKSDB_PREPROCESSOR_FLAGS = [ ROCKSDB_PREPROCESSOR_FLAGS = [
"-DROCKSDB_PLATFORM_POSIX",
"-DROCKSDB_LIB_IO_POSIX",
"-DROCKSDB_SUPPORT_THREAD_LOCAL", "-DROCKSDB_SUPPORT_THREAD_LOCAL",
# Flags to enable libs we include # Flags to enable libs we include
@ -70,7 +83,6 @@ ROCKSDB_PREPROCESSOR_FLAGS = [
"-DZSTD", "-DZSTD",
"-DZSTD_STATIC_LINKING_ONLY", "-DZSTD_STATIC_LINKING_ONLY",
"-DGFLAGS=gflags", "-DGFLAGS=gflags",
"-DTBB",
# Added missing flags from output of build_detect_platform # Added missing flags from output of build_detect_platform
"-DROCKSDB_BACKTRACE", "-DROCKSDB_BACKTRACE",
@ -246,6 +258,12 @@ cpp_library(
"options/options_parser.cc", "options/options_parser.cc",
"port/port_posix.cc", "port/port_posix.cc",
"port/stack_trace.cc", "port/stack_trace.cc",
"port/win/env_default.cc",
"port/win/env_win.cc",
"port/win/io_win.cc",
"port/win/port_win.cc",
"port/win/win_logger.cc",
"port/win/win_thread.cc",
"table/adaptive/adaptive_table_factory.cc", "table/adaptive/adaptive_table_factory.cc",
"table/block_based/binary_search_index_reader.cc", "table/block_based/binary_search_index_reader.cc",
"table/block_based/block.cc", "table/block_based/block.cc",

View File

@ -27,9 +27,9 @@ def pretty_list(lst, indent=8):
class TARGETSBuilder(object): class TARGETSBuilder(object):
def __init__(self, path): def __init__(self, path):
self.path = path self.path = path
self.targets_file = open(path, 'w') self.targets_file = open(path, 'wb')
header = targets_cfg.rocksdb_target_header_template header = targets_cfg.rocksdb_target_header_template
self.targets_file.write(header) self.targets_file.write(header.encode("utf-8"))
self.total_lib = 0 self.total_lib = 0
self.total_bin = 0 self.total_bin = 0
self.total_test = 0 self.total_test = 0
@ -52,7 +52,7 @@ class TARGETSBuilder(object):
headers_attr_prefix=headers_attr_prefix, headers_attr_prefix=headers_attr_prefix,
headers=headers, headers=headers,
deps=pretty_list(deps), deps=pretty_list(deps),
extra_external_deps=extra_external_deps)) extra_external_deps=extra_external_deps).encode("utf-8"))
self.total_lib = self.total_lib + 1 self.total_lib = self.total_lib + 1
def add_rocksdb_library(self, name, srcs, headers=None): def add_rocksdb_library(self, name, srcs, headers=None):
@ -66,18 +66,18 @@ class TARGETSBuilder(object):
name=name, name=name,
srcs=pretty_list(srcs), srcs=pretty_list(srcs),
headers_attr_prefix=headers_attr_prefix, headers_attr_prefix=headers_attr_prefix,
headers=headers)) headers=headers).encode("utf-8"))
self.total_lib = self.total_lib + 1 self.total_lib = self.total_lib + 1
def add_binary(self, name, srcs, deps=None): def add_binary(self, name, srcs, deps=None):
self.targets_file.write(targets_cfg.binary_template % ( self.targets_file.write(targets_cfg.binary_template.format(
name, name=name,
pretty_list(srcs), srcs=pretty_list(srcs),
pretty_list(deps))) deps=pretty_list(deps)).encode("utf-8"))
self.total_bin = self.total_bin + 1 self.total_bin = self.total_bin + 1
def add_c_test(self): def add_c_test(self):
self.targets_file.write(""" self.targets_file.write(b"""
if not is_opt_mode: if not is_opt_mode:
cpp_binary( cpp_binary(
name = "c_test_bin", name = "c_test_bin",
@ -119,5 +119,5 @@ if not is_opt_mode:
self.total_test = self.total_test + 1 self.total_test = self.total_test + 1
def flush_tests(self): def flush_tests(self):
self.targets_file.write(targets_cfg.unittests_template % self.tests_cfg) self.targets_file.write(targets_cfg.unittests_template.format(tests=self.tests_cfg).encode("utf-8"))
self.tests_cfg = "" self.tests_cfg = ""

View File

@ -32,13 +32,16 @@ ROCKSDB_EXTERNAL_DEPS = [
("gflags", None, "gflags"), ("gflags", None, "gflags"),
("lz4", None, "lz4"), ("lz4", None, "lz4"),
("zstd", None), ("zstd", None),
("tbb", None),
] ]
ROCKSDB_OS_DEPS = [ ROCKSDB_OS_DEPS = [
( (
"linux", "linux",
["third-party//numa:numa", "third-party//liburing:uring"], ["third-party//numa:numa", "third-party//liburing:uring", "third-party//tbb:tbb"],
),
(
"macos",
["third-party//tbb:tbb"],
), ),
] ]
@ -56,17 +59,27 @@ ROCKSDB_OS_PREPROCESSOR_FLAGS = [
"-DHAVE_SSE42", "-DHAVE_SSE42",
"-DLIBURING", "-DLIBURING",
"-DNUMA", "-DNUMA",
"-DROCKSDB_PLATFORM_POSIX",
"-DROCKSDB_LIB_IO_POSIX",
"-DTBB",
], ],
), ),
( (
"macos", "macos",
["-DOS_MACOSX"], [
"-DOS_MACOSX",
"-DROCKSDB_PLATFORM_POSIX",
"-DROCKSDB_LIB_IO_POSIX",
"-DTBB",
],
),
(
"windows",
[ "-DOS_WIN", "-DWIN32", "-D_MBCS", "-DWIN64", "-DNOMINMAX" ]
), ),
] ]
ROCKSDB_PREPROCESSOR_FLAGS = [ ROCKSDB_PREPROCESSOR_FLAGS = [
"-DROCKSDB_PLATFORM_POSIX",
"-DROCKSDB_LIB_IO_POSIX",
"-DROCKSDB_SUPPORT_THREAD_LOCAL", "-DROCKSDB_SUPPORT_THREAD_LOCAL",
# Flags to enable libs we include # Flags to enable libs we include
@ -77,7 +90,6 @@ ROCKSDB_PREPROCESSOR_FLAGS = [
"-DZSTD", "-DZSTD",
"-DZSTD_STATIC_LINKING_ONLY", "-DZSTD_STATIC_LINKING_ONLY",
"-DGFLAGS=gflags", "-DGFLAGS=gflags",
"-DTBB",
# Added missing flags from output of build_detect_platform # Added missing flags from output of build_detect_platform
"-DROCKSDB_BACKTRACE", "-DROCKSDB_BACKTRACE",
@ -154,12 +166,12 @@ cpp_library(
binary_template = """ binary_template = """
cpp_binary( cpp_binary(
name = "%s", name = "{name}",
srcs = [%s], srcs = [{srcs}],
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS, arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
compiler_flags = ROCKSDB_COMPILER_FLAGS, compiler_flags = ROCKSDB_COMPILER_FLAGS,
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS, preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
deps = [%s], deps = [{deps}],
external_deps = ROCKSDB_EXTERNAL_DEPS, external_deps = ROCKSDB_EXTERNAL_DEPS,
) )
""" """
@ -176,7 +188,7 @@ test_cfg_template = """ [
unittests_template = """ unittests_template = """
# [test_name, test_src, test_type, extra_deps, extra_compiler_flags] # [test_name, test_src, test_type, extra_deps, extra_compiler_flags]
ROCKS_TESTS = [ ROCKS_TESTS = [
%s] {tests}]
# Generate a test rule for each entry in ROCKS_TESTS # Generate a test rule for each entry in ROCKS_TESTS
# Do not build the tests in opt mode, since SyncPoint and other test code # Do not build the tests in opt mode, since SyncPoint and other test code

5
env/env_posix.cc vendored
View File

@ -6,6 +6,9 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors // found in the LICENSE file. See the AUTHORS file for names of contributors
#if !defined(OS_WIN)
#include <dirent.h> #include <dirent.h>
#ifndef ROCKSDB_NO_DYNAMIC_EXTENSION #ifndef ROCKSDB_NO_DYNAMIC_EXTENSION
#include <dlfcn.h> #include <dlfcn.h>
@ -517,3 +520,5 @@ std::unique_ptr<Env> NewCompositeEnv(std::shared_ptr<FileSystem> fs) {
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE
#endif

5
env/fs_posix.cc vendored
View File

@ -6,6 +6,9 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors // found in the LICENSE file. See the AUTHORS file for names of contributors
#if !defined(OS_WIN)
#include <dirent.h> #include <dirent.h>
#ifndef ROCKSDB_NO_DYNAMIC_EXTENSION #ifndef ROCKSDB_NO_DYNAMIC_EXTENSION
#include <dlfcn.h> #include <dlfcn.h>
@ -1060,3 +1063,5 @@ static FactoryFunc<FileSystem> posix_filesystem_reg =
#endif #endif
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE
#endif

View File

@ -7,6 +7,8 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#if !defined(OS_WIN)
#include "port/port_posix.h" #include "port/port_posix.h"
#include <assert.h> #include <assert.h>
@ -262,3 +264,5 @@ void SetCpuPriority(ThreadId id, CpuPriority priority) {
} // namespace port } // namespace port
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE
#endif

View File

@ -5,8 +5,9 @@
// //
#include "port/stack_trace.h" #include "port/stack_trace.h"
#if defined(ROCKSDB_LITE) || !(defined(ROCKSDB_BACKTRACE) || defined(OS_MACOSX)) || \ #if defined(ROCKSDB_LITE) || \
defined(CYGWIN) || defined(OS_FREEBSD) || defined(OS_SOLARIS) !(defined(ROCKSDB_BACKTRACE) || defined(OS_MACOSX)) || defined(CYGWIN) || \
defined(OS_FREEBSD) || defined(OS_SOLARIS) || defined(OS_WIN)
// noop // noop

View File

@ -7,6 +7,8 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#if defined(OS_WIN)
#include <mutex> #include <mutex>
#include <rocksdb/env.h> #include <rocksdb/env.h>
@ -39,3 +41,5 @@ Env* Env::Default() {
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE
#endif

View File

@ -7,6 +7,8 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#if defined(OS_WIN)
#include "port/win/env_win.h" #include "port/win/env_win.h"
#include "port/win/win_thread.h" #include "port/win/win_thread.h"
#include <algorithm> #include <algorithm>
@ -1549,3 +1551,5 @@ std::string Env::GenerateUniqueId() {
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE
#endif

View File

@ -311,7 +311,7 @@ public:
void StartThread(void(*function)(void* arg), void* arg) override; void StartThread(void(*function)(void* arg), void* arg) override;
void WaitForJoin(); void WaitForJoin() override;
unsigned int GetThreadPoolQueueLen(Env::Priority pri) const override; unsigned int GetThreadPoolQueueLen(Env::Priority pri) const override;

View File

@ -7,6 +7,8 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#if defined(OS_WIN)
#include "port/win/io_win.h" #include "port/win/io_win.h"
#include "monitoring/iostats_context_imp.h" #include "monitoring/iostats_context_imp.h"
@ -642,8 +644,8 @@ Status WinSequentialFile::Skip(uint64_t n) {
BOOL ret = SetFilePointerEx(hFile_, li, NULL, FILE_CURRENT); BOOL ret = SetFilePointerEx(hFile_, li, NULL, FILE_CURRENT);
if (ret == FALSE) { if (ret == FALSE) {
auto lastError = GetLastError(); auto lastError = GetLastError();
return IOErrorFromWindowsError("Skip SetFilePointerEx():" + filename_, return IOErrorFromWindowsError("Skip SetFilePointerEx():" + filename_,
lastError); lastError);
} }
return Status::OK(); return Status::OK();
} }
@ -767,8 +769,8 @@ Status WinWritableImpl::AppendImpl(const Slice& data) {
Status s; Status s;
if (data.size() > std::numeric_limits<DWORD>::max()) { if (data.size() > std::numeric_limits<DWORD>::max()) {
return Status::InvalidArgument("data is too long for a single write" + return Status::InvalidArgument("data is too long for a single write" +
file_data_->GetName()); file_data_->GetName());
} }
size_t bytes_written = 0; // out param size_t bytes_written = 0; // out param
@ -805,8 +807,8 @@ Status WinWritableImpl::AppendImpl(const Slice& data) {
// is sector aligned // is sector aligned
next_write_offset_ += bytes_written; next_write_offset_ += bytes_written;
} else { } else {
s = Status::IOError("Failed to write all bytes: " + s = Status::IOError("Failed to write all bytes: " +
file_data_->GetName()); file_data_->GetName());
} }
} }
@ -1067,3 +1069,5 @@ WinFileLock::~WinFileLock() {
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE
#endif

View File

@ -409,7 +409,7 @@ class WinRandomRWFile : private WinFileData,
virtual Status Sync() override; virtual Status Sync() override;
virtual Status Fsync() { return Sync(); } virtual Status Fsync() override { return Sync(); }
virtual Status Close() override; virtual Status Close() override;
}; };

View File

@ -7,9 +7,7 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#if !defined(OS_WIN) && !defined(WIN32) && !defined(_WIN32) #if defined(OS_WIN)
#error Windows Specific Code
#endif
#include "port/win/port_win.h" #include "port/win/port_win.h"
@ -272,3 +270,5 @@ void SetCpuPriority(ThreadId id, CpuPriority priority) {
} // namespace port } // namespace port
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE
#endif

View File

@ -7,6 +7,8 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#if defined(OS_WIN)
#ifndef ROCKSDB_JEMALLOC #ifndef ROCKSDB_JEMALLOC
# error This file can only be part of jemalloc aware build # error This file can only be part of jemalloc aware build
#endif #endif
@ -73,3 +75,5 @@ void operator delete[](void* p) {
je_free(p); je_free(p);
} }
} }
#endif

View File

@ -10,6 +10,8 @@
// Logger implementation that can be shared by all environments // Logger implementation that can be shared by all environments
// where enough posix functionality is available. // where enough posix functionality is available.
#if defined(OS_WIN)
#include "port/win/win_logger.h" #include "port/win/win_logger.h"
#include "port/win/io_win.h" #include "port/win/io_win.h"
@ -47,13 +49,11 @@ void WinLogger::DebugWriter(const char* str, int len) {
BOOL ret = WriteFile(file_, str, len, &bytesWritten, NULL); BOOL ret = WriteFile(file_, str, len, &bytesWritten, NULL);
if (ret == FALSE) { if (ret == FALSE) {
std::string errSz = GetWindowsErrSz(GetLastError()); std::string errSz = GetWindowsErrSz(GetLastError());
fprintf(stderr, errSz.c_str()); fprintf(stderr, "%s", errSz.c_str());
} }
} }
WinLogger::~WinLogger() { WinLogger::~WinLogger() { CloseInternal(); }
CloseInternal();
}
Status WinLogger::CloseImpl() { Status WinLogger::CloseImpl() {
return CloseInternal(); return CloseInternal();
@ -65,15 +65,13 @@ Status WinLogger::CloseInternal() {
BOOL ret = FlushFileBuffers(file_); BOOL ret = FlushFileBuffers(file_);
if (ret == 0) { if (ret == 0) {
auto lastError = GetLastError(); auto lastError = GetLastError();
s = IOErrorFromWindowsError("Failed to flush LOG on Close() ", s = IOErrorFromWindowsError("Failed to flush LOG on Close() ", lastError);
lastError);
} }
ret = CloseHandle(file_); ret = CloseHandle(file_);
// On error the return value is zero // On error the return value is zero
if (ret == 0 && s.ok()) { if (ret == 0 && s.ok()) {
auto lastError = GetLastError(); auto lastError = GetLastError();
s = IOErrorFromWindowsError("Failed to flush LOG on Close() ", s = IOErrorFromWindowsError("Failed to flush LOG on Close() ", lastError);
lastError);
} }
file_ = INVALID_HANDLE_VALUE; file_ = INVALID_HANDLE_VALUE;
closed_ = true; closed_ = true;
@ -163,7 +161,7 @@ void WinLogger::Logv(const char* format, va_list ap) {
&bytesWritten, NULL); &bytesWritten, NULL);
if (ret == FALSE) { if (ret == FALSE) {
std::string errSz = GetWindowsErrSz(GetLastError()); std::string errSz = GetWindowsErrSz(GetLastError());
fprintf(stderr, errSz.c_str()); fprintf(stderr, "%s", errSz.c_str());
} }
flush_pending_ = true; flush_pending_ = true;
@ -190,3 +188,5 @@ size_t WinLogger::GetLogFileSize() const { return log_size_; }
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE
#endif

View File

@ -7,6 +7,8 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#if defined(OS_WIN)
#include "port/win/win_thread.h" #include "port/win/win_thread.h"
#include <assert.h> #include <assert.h>
@ -177,3 +179,5 @@ unsigned int __stdcall WindowsThread::Data::ThreadProc(void* arg) {
} }
} // namespace port } // namespace port
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE
#endif

View File

@ -7,6 +7,8 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#if defined(OS_WIN)
#include "port/win/xpress_win.h" #include "port/win/xpress_win.h"
#include <windows.h> #include <windows.h>
@ -224,3 +226,5 @@ char* Decompress(const char* input_data, size_t input_length,
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE
#endif #endif
#endif

6
src.mk
View File

@ -129,6 +129,12 @@ LIB_SOURCES = \
options/options_helper.cc \ options/options_helper.cc \
options/options_parser.cc \ options/options_parser.cc \
port/port_posix.cc \ port/port_posix.cc \
port/win/env_default.cc \
port/win/env_win.cc \
port/win/io_win.cc \
port/win/port_win.cc \
port/win/win_logger.cc \
port/win/win_thread.cc \
port/stack_trace.cc \ port/stack_trace.cc \
table/adaptive/adaptive_table_factory.cc \ table/adaptive/adaptive_table_factory.cc \
table/block_based/binary_search_index_reader.cc \ table/block_based/binary_search_index_reader.cc \