Merge commit 'f45d80fe16f99d112d545b7cd74ce46342fe3437'

This commit is contained in:
Andrea Cavalli 2020-07-09 00:10:53 +02:00
commit 886a40118e
81 changed files with 2394 additions and 538 deletions

View File

@ -189,7 +189,7 @@ elseif (INTEL)
endif()
if (WIN32)
add_definitions(-DNTDDI_VERSION=0x06020000 -DWINVER=0x0602 -D_WIN32_WINNT=0x0602 -DNOMINMAX -DUNICODE -D_UNICODE -DWIN32_LEAN_AND_MEAN)
add_definitions(-DNTDDI_VERSION=0x06020000 -DWINVER=0x0602 -D_WIN32_WINNT=0x0602 -DPSAPI_VERSION=1 -DNOMINMAX -DUNICODE -D_UNICODE -DWIN32_LEAN_AND_MEAN)
endif()
if (CYGWIN)
add_definitions(-D_DEFAULT_SOURCE=1 -DFD_SETSIZE=4096)

View File

@ -39,9 +39,9 @@ function split_file($file, $chunks, $undo) {
$cmake_new_files = $new_files;
if ($is_generated) {
foreach ($cmake_new_files as &$file_ref) {
$file_ref = str_replace('td/generate', '${CMAKE_CURRENT_SOURCE_DIR}', $file_ref);
$file_ref = str_replace('td/generate/auto/td', '${TD_AUTO_INCLUDE_DIR}', $file_ref);
}
$cmake_cpp_name = str_replace('td/generate', '${CMAKE_CURRENT_SOURCE_DIR}', $cmake_cpp_name);
$cmake_cpp_name = str_replace('td/generate/auto/td', '${TD_AUTO_INCLUDE_DIR}', $cmake_cpp_name);
}
if ($undo) {

View File

@ -24,6 +24,7 @@
#include <vector>
static constexpr int DATA_SIZE = 8 << 10;
static constexpr int SHORT_DATA_SIZE = 64;
class SHA1Bench : public td::Benchmark {
public:
@ -48,14 +49,45 @@ class SHA1Bench : public td::Benchmark {
}
};
class AESBench : public td::Benchmark {
class AesEcbBench : public td::Benchmark {
public:
alignas(64) unsigned char data[DATA_SIZE];
td::UInt256 key;
td::UInt256 iv;
std::string get_description() const override {
return PSTRING() << "AES OpenSSL [" << (DATA_SIZE >> 10) << "KB]";
return PSTRING() << "AES ECB OpenSSL [" << (DATA_SIZE >> 10) << "KB]";
}
void start_up() override {
for (int i = 0; i < DATA_SIZE; i++) {
data[i] = 123;
}
td::Random::secure_bytes(key.raw, sizeof(key));
td::Random::secure_bytes(iv.raw, sizeof(iv));
}
void run(int n) override {
td::AesState state;
state.init(td::as_slice(key), true);
td::MutableSlice data_slice(data, DATA_SIZE);
for (int i = 0; i <= n; i++) {
size_t step = 16;
for (size_t offset = 0; offset + step <= data_slice.size(); offset += step) {
state.encrypt(data_slice.ubegin() + offset, data_slice.ubegin() + offset, static_cast<int>(step));
}
}
}
};
class AesIgeEncryptBench : public td::Benchmark {
public:
alignas(64) unsigned char data[DATA_SIZE];
td::UInt256 key;
td::UInt256 iv;
std::string get_description() const override {
return PSTRING() << "AES IGE OpenSSL encrypt [" << (DATA_SIZE >> 10) << "KB]";
}
void start_up() override {
@ -68,8 +100,125 @@ class AESBench : public td::Benchmark {
void run(int n) override {
td::MutableSlice data_slice(data, DATA_SIZE);
td::AesIgeState state;
state.init(as_slice(key), as_slice(iv), true);
for (int i = 0; i < n; i++) {
td::aes_ige_encrypt(as_slice(key), as_slice(iv), data_slice, data_slice);
state.encrypt(data_slice, data_slice);
}
}
};
class AesIgeDecryptBench : public td::Benchmark {
public:
alignas(64) unsigned char data[DATA_SIZE];
td::UInt256 key;
td::UInt256 iv;
std::string get_description() const override {
return PSTRING() << "AES IGE OpenSSL decrypt [" << (DATA_SIZE >> 10) << "KB]";
}
void start_up() override {
for (int i = 0; i < DATA_SIZE; i++) {
data[i] = 123;
}
td::Random::secure_bytes(key.raw, sizeof(key));
td::Random::secure_bytes(iv.raw, sizeof(iv));
}
void run(int n) override {
td::MutableSlice data_slice(data, DATA_SIZE);
td::AesIgeState state;
state.init(as_slice(key), as_slice(iv), false);
for (int i = 0; i < n; i++) {
state.decrypt(data_slice, data_slice);
}
}
};
class AesCtrBench : public td::Benchmark {
public:
alignas(64) unsigned char data[DATA_SIZE];
td::UInt256 key;
td::UInt128 iv;
std::string get_description() const override {
return PSTRING() << "AES CTR OpenSSL [" << (DATA_SIZE >> 10) << "KB]";
}
void start_up() override {
for (int i = 0; i < DATA_SIZE; i++) {
data[i] = 123;
}
td::Random::secure_bytes(key.raw, sizeof(key));
td::Random::secure_bytes(iv.raw, sizeof(iv));
}
void run(int n) override {
td::MutableSlice data_slice(data, DATA_SIZE);
td::AesCtrState state;
state.init(as_slice(key), as_slice(iv));
for (int i = 0; i < n; i++) {
state.encrypt(data_slice, data_slice);
}
}
};
class AesCbcBench : public td::Benchmark {
public:
alignas(64) unsigned char data[DATA_SIZE];
td::UInt256 key;
td::UInt128 iv;
std::string get_description() const override {
return PSTRING() << "AES CBC OpenSSL [" << (DATA_SIZE >> 10) << "KB]";
}
void start_up() override {
for (int i = 0; i < DATA_SIZE; i++) {
data[i] = 123;
}
td::Random::secure_bytes(as_slice(key));
td::Random::secure_bytes(as_slice(iv));
}
void run(int n) override {
td::MutableSlice data_slice(data, DATA_SIZE);
for (int i = 0; i < n; i++) {
td::aes_cbc_encrypt(as_slice(key), as_slice(iv), data_slice, data_slice);
}
}
};
template <bool use_state>
class AesIgeShortBench : public td::Benchmark {
public:
alignas(64) unsigned char data[SHORT_DATA_SIZE];
td::UInt256 key;
td::UInt256 iv;
std::string get_description() const override {
return PSTRING() << "AES IGE OpenSSL " << (use_state ? "EVP" : "C ") << "[" << SHORT_DATA_SIZE << "B]";
}
void start_up() override {
for (int i = 0; i < SHORT_DATA_SIZE; i++) {
data[i] = 123;
}
td::Random::secure_bytes(as_slice(key));
td::Random::secure_bytes(as_slice(iv));
}
void run(int n) override {
td::MutableSlice data_slice(data, SHORT_DATA_SIZE);
for (int i = 0; i < n; i++) {
if (use_state) {
td::AesIgeState ige;
ige.init(as_slice(key), as_slice(iv), false);
ige.decrypt(data_slice, data_slice);
} else {
td::aes_ige_decrypt(as_slice(key), as_slice(iv), data_slice, data_slice);
}
}
}
};
@ -112,13 +261,13 @@ BENCH(SslRand, "ssl_rand_int32") {
std::vector<td::thread> v;
std::atomic<td::uint32> sum{0};
for (int i = 0; i < 3; i++) {
v.push_back(td::thread([&] {
v.emplace_back([&sum, n] {
td::int32 res = 0;
for (int j = 0; j < n; j++) {
res ^= td::Random::secure_int32();
}
sum += res;
}));
});
}
for (auto &x : v) {
x.join();
@ -196,6 +345,15 @@ class Crc64Bench : public td::Benchmark {
};
int main() {
td::init_openssl_threads();
td::bench(AesIgeShortBench<true>());
td::bench(AesIgeShortBench<false>());
td::bench(AesIgeEncryptBench());
td::bench(AesIgeDecryptBench());
td::bench(AesEcbBench());
td::bench(AesCtrBench());
td::bench(Pbkdf2Bench());
td::bench(RandBench());
td::bench(CppRandBench());
@ -206,7 +364,6 @@ int main() {
#endif
td::bench(SslRandBufBench());
td::bench(SHA1Bench());
td::bench(AESBench());
td::bench(Crc32Bench());
td::bench(Crc64Bench());
return 0;

View File

@ -40,8 +40,8 @@ class IostreamWriteBench : public td::Benchmark {
protected:
std::string file_name_;
std::ofstream stream;
enum { buffer_size = 1 << 20 };
char buffer[buffer_size];
static constexpr std::size_t BUFFER_SIZE = 1 << 20;
char buffer[BUFFER_SIZE];
public:
std::string get_description() const override {
@ -52,7 +52,7 @@ class IostreamWriteBench : public td::Benchmark {
file_name_ = create_tmp_file();
stream.open(file_name_.c_str());
CHECK(stream.is_open());
// stream.rdbuf()->pubsetbuf(buffer, buffer_size);
// stream.rdbuf()->pubsetbuf(buffer, BUFFER_SIZE);
}
void run(int n) override {
@ -71,8 +71,8 @@ class FILEWriteBench : public td::Benchmark {
protected:
std::string file_name_;
FILE *file;
enum { buffer_size = 1 << 20 };
char buffer[buffer_size];
static constexpr std::size_t BUFFER_SIZE = 1 << 20;
char buffer[BUFFER_SIZE];
public:
std::string get_description() const override {
@ -82,7 +82,7 @@ class FILEWriteBench : public td::Benchmark {
void start_up() override {
file_name_ = create_tmp_file();
file = fopen(file_name_.c_str(), "w");
// setvbuf(file, buffer, _IOFBF, buffer_size);
// setvbuf(file, buffer, _IOFBF, BUFFER_SIZE);
}
void run(int n) override {
@ -123,8 +123,8 @@ class LogWriteBench : public td::Benchmark {
std::string file_name_;
std::ofstream stream;
std::streambuf *old_buf;
enum { buffer_size = 1 << 20 };
char buffer[buffer_size];
static constexpr std::size_t BUFFER_SIZE = 1 << 20;
char buffer[BUFFER_SIZE];
public:
std::string get_description() const override {

View File

@ -837,7 +837,7 @@ class QueueBenchmark : public td::Benchmark {
template <class QueueT>
class RingBenchmark : public td::Benchmark {
enum { QN = 504 };
static constexpr int QN = 504;
struct Thread {
int int_id;

View File

@ -515,7 +515,8 @@ function onOptionsChanged() {
pre_text.push('Note that following calls to <code>pkg</code> needs to be run as <code>root</code>.');
}
if (os_openbsd) {
pre_text.push('Note that following instruction is for OpenBSD 6.4 and default KSH shell.');
pre_text.push('Note that following instruction is for OpenBSD 6.7 and default KSH shell.');
pre_text.push('Note that building requires a lot of memory, so you may need to increase allowed per-process memory usage in /etc/login.conf or build from root.');
}
if (os_netbsd) {
pre_text.push('Note that following instruction is for NetBSD 8.0 and default SH shell.');
@ -648,7 +649,7 @@ function onOptionsChanged() {
if (target === 'JNI') {
packages += ' jdk';
}
commands.push('pkg_add ' + packages);
commands.push('pkg_add -z ' + packages);
if (!use_root) {
commands.push('exit');
}

View File

@ -117,8 +117,8 @@ static Backtrace get_backtrace() {
return res;
}
static constexpr std::size_t reserved = 16;
static constexpr std::int32_t malloc_info_magic = 0x27138373;
static constexpr std::size_t RESERVED_SIZE = 16;
static constexpr std::int32_t MALLOC_INFO_MAGIC = 0x27138373;
struct malloc_info {
std::int32_t magic;
std::int32_t size;
@ -139,9 +139,9 @@ struct HashtableNode {
std::atomic<std::size_t> size;
};
static constexpr std::size_t ht_max_size = 1000000;
static constexpr std::size_t HT_MAX_SIZE = 1000000;
static std::atomic<std::size_t> ht_size{0};
static std::array<HashtableNode, ht_max_size> ht;
static std::array<HashtableNode, HT_MAX_SIZE> ht;
std::size_t get_ht_size() {
return ht_size.load();
@ -154,9 +154,9 @@ std::int32_t get_ht_pos(const Backtrace &bt, bool force = false) {
while (true) {
auto pos_hash = ht[pos].hash.load();
if (pos_hash == 0) {
if (ht_size > ht_max_size / 2) {
if (ht_size > HT_MAX_SIZE / 2) {
if (force) {
assert(ht_size * 10 < ht_max_size * 7);
assert(ht_size * 10 < HT_MAX_SIZE * 7);
} else {
Backtrace unknown_bt{{nullptr}};
unknown_bt[0] = reinterpret_cast<void *>(1);
@ -206,8 +206,8 @@ void register_xalloc(malloc_info *info, std::int32_t diff) {
extern "C" {
static void *malloc_with_frame(std::size_t size, const Backtrace &frame) {
static_assert(reserved % alignof(std::max_align_t) == 0, "fail");
static_assert(reserved >= sizeof(malloc_info), "fail");
static_assert(RESERVED_SIZE % alignof(std::max_align_t) == 0, "fail");
static_assert(RESERVED_SIZE >= sizeof(malloc_info), "fail");
#if TD_DARWIN
static void *malloc_void = dlsym(RTLD_NEXT, "malloc");
static auto malloc_old = *reinterpret_cast<decltype(malloc) **>(&malloc_void);
@ -215,26 +215,26 @@ static void *malloc_with_frame(std::size_t size, const Backtrace &frame) {
extern decltype(malloc) __libc_malloc;
static auto malloc_old = __libc_malloc;
#endif
auto *info = static_cast<malloc_info *>(malloc_old(size + reserved));
auto *info = static_cast<malloc_info *>(malloc_old(size + RESERVED_SIZE));
auto *buf = reinterpret_cast<char *>(info);
info->magic = malloc_info_magic;
info->magic = MALLOC_INFO_MAGIC;
info->size = static_cast<std::int32_t>(size);
info->ht_pos = get_ht_pos(frame);
register_xalloc(info, +1);
void *data = buf + reserved;
void *data = buf + RESERVED_SIZE;
return data;
}
static malloc_info *get_info(void *data_void) {
char *data = static_cast<char *>(data_void);
auto *buf = data - reserved;
auto *buf = data - RESERVED_SIZE;
auto *info = reinterpret_cast<malloc_info *>(buf);
assert(info->magic == malloc_info_magic);
assert(info->magic == MALLOC_INFO_MAGIC);
return info;
}

View File

@ -82,13 +82,13 @@ class TlWriterDotNet : public TL_writer {
static std::string to_cCamelCase(const std::string &name, bool flag) {
bool next_to_upper = flag;
std::string result;
for (int i = 0; i < (int)name.size(); i++) {
for (std::size_t i = 0; i < name.size(); i++) {
if (!is_alnum(name[i])) {
next_to_upper = true;
continue;
}
if (next_to_upper) {
result += (char)to_upper(name[i]);
result += to_upper(name[i]);
next_to_upper = false;
} else {
result += name[i];
@ -98,7 +98,7 @@ class TlWriterDotNet : public TL_writer {
}
std::string gen_native_field_name(std::string name) const {
for (int i = 0; i < (int)name.size(); i++) {
for (std::size_t i = 0; i < name.size(); i++) {
if (!is_alnum(name[i])) {
name[i] = '_';
}
@ -115,7 +115,7 @@ class TlWriterDotNet : public TL_writer {
if (name == "#") {
return "int32_t";
}
for (int i = 0; i < (int)name.size(); i++) {
for (std::size_t i = 0; i < name.size(); i++) {
if (!is_alnum(name[i])) {
name[i] = '_';
}
@ -163,7 +163,7 @@ class TlWriterDotNet : public TL_writer {
if (name == "Vector") {
assert(t->arity == 1);
assert((int)tree_type->children.size() == 1);
assert(tree_type->children.size() == 1);
assert(tree_type->children[0]->get_type() == NODE_TYPE_TYPE);
const tl_tree_type *child = static_cast<const tl_tree_type *>(tree_type->children[0]);
@ -172,7 +172,7 @@ class TlWriterDotNet : public TL_writer {
assert(!is_built_in_simple_type(name) && !is_built_in_complex_type(name));
for (int i = 0; i < (int)tree_type->children.size(); i++) {
for (std::size_t i = 0; i < tree_type->children.size(); i++) {
assert(tree_type->children[i]->get_type() == NODE_TYPE_NAT_CONST);
}

View File

@ -66,7 +66,9 @@ class AuthKey {
auth_key_.clear();
}
enum : int32 { AUTH_FLAG = 1, WAS_AUTH_FLAG = 2, HAS_CREATED_AT = 4 };
static constexpr int32 AUTH_FLAG = 1;
static constexpr int32 WAS_AUTH_FLAG = 2;
static constexpr int32 HAS_CREATED_AT = 4;
template <class StorerT>
void store(StorerT &storer) const {

View File

@ -29,7 +29,7 @@ Result<size_t> Transport::read_next(BufferSlice *message, uint32 *quick_ack) {
if (r_size.is_error() || r_size.ok() != 0) {
return r_size;
}
if (http_query_.type_ != HttpQuery::Type::RESPONSE) {
if (http_query_.type_ != HttpQuery::Type::Response) {
return Status::Error("Unexpected HTTP query type");
}
if (http_query_.container_.size() != 2u) {

View File

@ -19,9 +19,8 @@
namespace td {
namespace mtproto {
ActorOwn<> create_ping_actor(std::string debug, unique_ptr<RawConnection> raw_connection,
unique_ptr<AuthData> auth_data, Promise<unique_ptr<RawConnection>> promise,
ActorShared<> parent) {
ActorOwn<> create_ping_actor(string debug, unique_ptr<RawConnection> raw_connection, unique_ptr<AuthData> auth_data,
Promise<unique_ptr<RawConnection>> promise, ActorShared<> parent) {
class PingActor : public Actor {
public:
PingActor(unique_ptr<RawConnection> raw_connection, unique_ptr<AuthData> auth_data,

View File

@ -790,7 +790,7 @@ vector<string> CallActor::get_emojis_fingerprint(const string &key, const string
vector<string> result;
result.reserve(4);
for (int i = 0; i < 4; i++) {
uint64 num = bswap64(as<uint64>(sha256_buf + 8 * i));
uint64 num = big_endian_to_host64(as<uint64>(sha256_buf + 8 * i));
result.push_back(get_emoji_fingerprint(num));
}
return result;

View File

@ -113,13 +113,13 @@ class Global : public ActorContext {
}
bool have_net_query_dispatcher() const {
return net_query_dispatcher_ != nullptr;
return net_query_dispatcher_.get() != nullptr;
}
void set_shared_config(unique_ptr<ConfigShared> shared_config);
ConfigShared &shared_config() {
CHECK(shared_config_ != nullptr);
CHECK(shared_config_.get() != nullptr);
return *shared_config_;
}

View File

@ -10907,18 +10907,17 @@ void MessagesManager::init() {
auto *list = get_dialog_list(DialogListId(folder_id));
CHECK(list != nullptr);
CHECK(list->pinned_dialogs_.empty());
if (!r_dialog_ids.empty()) {
for (auto &r_dialog_id : reversed(r_dialog_ids)) {
auto dialog_id = r_dialog_id.move_as_ok();
auto order = get_next_pinned_dialog_order();
list->pinned_dialogs_.emplace_back(order, dialog_id);
list->pinned_dialog_id_orders_.emplace(dialog_id, order);
}
std::reverse(list->pinned_dialogs_.begin(), list->pinned_dialogs_.end());
list->are_pinned_dialogs_inited_ = true;
update_list_last_pinned_dialog_date(*list);
for (auto &r_dialog_id : reversed(r_dialog_ids)) {
auto dialog_id = r_dialog_id.move_as_ok();
auto order = get_next_pinned_dialog_order();
list->pinned_dialogs_.emplace_back(order, dialog_id);
list->pinned_dialog_id_orders_.emplace(dialog_id, order);
}
std::reverse(list->pinned_dialogs_.begin(), list->pinned_dialogs_.end());
list->are_pinned_dialogs_inited_ = true;
update_list_last_pinned_dialog_date(*list);
LOG(INFO) << "Loaded pinned chats " << list->pinned_dialogs_ << " in " << folder_id;
}
}
@ -12990,8 +12989,9 @@ void MessagesManager::on_get_dialogs(FolderId folder_id, vector<tl_object_ptr<te
}
} else if (promise) {
LOG(ERROR) << "Last server dialog date didn't increased from " << folder->last_server_dialog_date_ << " to "
<< max_dialog_date << " after receiving " << dialogs.size() << " chats from " << total_count << " in "
<< folder_id << ". last_dialog_date = " << folder->folder_last_dialog_date_
<< max_dialog_date << " after receiving " << dialogs.size() << " chats " << added_dialog_ids
<< " from " << total_count << " in " << folder_id
<< ". last_dialog_date = " << folder->folder_last_dialog_date_
<< ", last_loaded_database_dialog_date = " << folder->last_loaded_database_dialog_date_;
}
}

View File

@ -3077,7 +3077,9 @@ Status NotificationManager::process_push_notification_payload(string payload, bo
dialog_id = DialogId(secret_chat_id);
}
if (!dialog_id.is_valid()) {
// TODO if (loc_key == "ENCRYPTED_MESSAGE") ?
if (loc_key == "ENCRYPTED_MESSAGE" || loc_key == "MESSAGE_MUTED") {
return Status::Error(406, "Force loading data from the server");
}
return Status::Error("Can't find dialog_id");
}

View File

@ -47,7 +47,6 @@ class RequestActor : public Actor {
}
stop();
} else {
LOG_CHECK(!promise.was_set_value) << future.empty() << " " << future.get_state();
CHECK(!future.empty());
CHECK(future.get_state() == FutureActor<T>::State::Waiting);
if (--tries_left_ == 0) {
@ -64,7 +63,7 @@ class RequestActor : public Actor {
void raw_event(const Event::Raw &event) override {
if (future_.is_error()) {
auto error = future_.move_as_error();
if (error == Status::Error<FutureActor<T>::Hangup>()) {
if (error == Status::Error<FutureActor<T>::HANGUP_ERROR_CODE>()) {
// dropping query due to lost authorization or lost promise
// td may be already closed, so we should check is auth_manager_ is empty
bool is_authorized = td->auth_manager_ && td->auth_manager_->is_authorized();

View File

@ -23,6 +23,7 @@
#include "td/utils/JsonBuilder.h"
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/OptionParser.h"
#include "td/utils/port/FileFd.h"
#include "td/utils/port/PollFlags.h"
#include "td/utils/port/signals.h"
@ -4062,6 +4063,29 @@ class CliClient final : public Actor {
fd.write("a").ignore();
fd.seek(size).ignore();
fd.truncate_to_current_position(size).ignore();
} else if (op == "mem") {
auto r_mem_stats = mem_stat();
if (r_mem_stats.is_error()) {
LOG(ERROR) << r_mem_stats.error();
} else {
auto stats = r_mem_stats.move_as_ok();
LOG(ERROR) << "RSS = " << stats.resident_size_ << ", peak RSS = " << stats.resident_size_peak_ << ", VSZ "
<< stats.virtual_size_ << ", peak VSZ = " << stats.virtual_size_peak_;
}
} else if (op == "cpu") {
uint32 inc_count = to_integer<uint32>(args);
while (inc_count-- > 0) {
cpu_counter_++;
}
auto r_cpu_stats = cpu_stat();
if (r_cpu_stats.is_error()) {
LOG(ERROR) << r_cpu_stats.error();
} else {
auto stats = r_cpu_stats.move_as_ok();
LOG(ERROR) << cpu_counter_ << ", total ticks = " << stats.total_ticks_
<< ", user ticks = " << stats.process_user_ticks_
<< ", system ticks = " << stats.process_system_ticks_;
}
} else if (op == "SetVerbosity" || op == "SV") {
Log::set_verbosity_level(to_integer<int>(args));
} else if (op[0] == 'v' && op[1] == 'v') {
@ -4247,8 +4271,11 @@ class CliClient final : public Actor {
bool disable_network_ = false;
int api_id_ = 0;
std::string api_hash_;
static std::atomic<uint64> cpu_counter_;
};
CliClient *CliClient::instance_ = nullptr;
std::atomic<uint64> CliClient::cpu_counter_;
void quit() {
CliClient::quit_instance();
@ -4261,10 +4288,6 @@ static void fail_signal(int sig) {
}
}
static void usage() {
//TODO:
}
static void on_fatal_error(const char *error) {
std::cerr << "Fatal error: " << error << std::endl;
}
@ -4307,52 +4330,72 @@ void main(int argc, char **argv) {
}
return std::string();
}(std::getenv("TD_API_HASH"));
// TODO port OptionsParser to Windows
for (int i = 1; i < argc; i++) {
if (!std::strcmp(argv[i], "--test")) {
use_test_dc = true;
} else if (!std::strncmp(argv[i], "-v", 2)) {
const char *arg = argv[i] + 2;
if (*arg == '\0' && i + 1 < argc) {
arg = argv[++i];
}
int new_verbosity = 1;
while (*arg == 'v') {
new_verbosity++;
arg++;
}
if (*arg) {
new_verbosity += to_integer<int>(Slice(arg)) - (new_verbosity == 1);
}
new_verbosity_level = VERBOSITY_NAME(FATAL) + new_verbosity;
} else if (!std::strncmp(argv[i], "-l", 2)) {
const char *arg = argv[i] + 2;
if (*arg == '\0' && i + 1 < argc) {
arg = argv[++i];
}
if (file_log.init(arg).is_ok() && file_log.init(arg).is_ok() && file_log.init(arg, 1000 << 20).is_ok()) {
log_interface = &ts_log;
}
} else if (!std::strcmp(argv[i], "-W")) {
get_chat_list = true;
} else if (!std::strcmp(argv[i], "--disable-network") || !std::strcmp(argv[i], "-n")) {
disable_network = true;
} else if (!std::strcmp(argv[i], "--api_id") || !std::strcmp(argv[i], "--api-id")) {
if (i + 1 >= argc) {
return usage();
}
api_id = to_integer<int32>(Slice(argv[++i]));
} else if (!std::strcmp(argv[i], "--api_hash") || !std::strcmp(argv[i], "--api-hash")) {
if (i + 1 >= argc) {
return usage();
}
api_hash = argv[++i];
td::OptionParser options;
options.set_description("TDLib test client");
options.add_option('\0', "test", "Use test DC", [&] {
use_test_dc = true;
return Status::OK();
});
options.add_option('v', "verbosity", "Set verbosity level", [&](Slice level) {
int new_verbosity = 1;
while (begins_with(level, "v")) {
new_verbosity++;
level.remove_prefix(1);
}
if (!level.empty()) {
new_verbosity += to_integer<int>(level) - (new_verbosity == 1);
}
new_verbosity_level = VERBOSITY_NAME(FATAL) + new_verbosity;
return Status::OK();
});
options.add_option('l', "log", "Log to file", [&](Slice file_name) {
if (file_log.init(file_name.str()).is_ok() && file_log.init(file_name.str()).is_ok() &&
file_log.init(file_name.str(), 1000 << 20).is_ok()) {
log_interface = &ts_log;
}
return Status::OK();
});
options.add_option('W', "", "Preload chat list", [&] {
get_chat_list = true;
return Status::OK();
});
options.add_option('n', "disable-network", "Disable network", [&] {
disable_network = true;
return Status::OK();
});
options.add_option('\0', "api-id", "Set Telegram API ID", [&](Slice parameter) {
api_id = to_integer<int32>(parameter);
return Status::OK();
});
options.add_option('\0', "api_id", "Set Telegram API ID", [&](Slice parameter) {
api_id = to_integer<int32>(parameter);
return Status::OK();
});
options.add_option('\0', "api-hash", "Set Telegram API hash", [&](Slice parameter) {
api_hash = parameter.str();
return Status::OK();
});
options.add_option('\0', "api_hash", "Set Telegram API hash", [&](Slice parameter) {
api_hash = parameter.str();
return Status::OK();
});
auto res = options.run(argc, argv);
if (res.is_error()) {
LOG(PLAIN) << "tg_cli: " << res.error().message();
LOG(PLAIN) << options;
return;
}
if (!res.ok().empty()) {
LOG(PLAIN) << "tg_cli: " << "Have unexpected non-option parameters";
LOG(PLAIN) << options;
return;
}
if (api_id == 0 || api_hash.empty()) {
LOG(ERROR) << "You should provide some valid api_id and api_hash";
return usage();
LOG(PLAIN) << "tg_cli: " << "You should provide some valid api_id and api_hash";
LOG(PLAIN) << options;
return;
}
SET_VERBOSITY_LEVEL(new_verbosity_level);

View File

@ -77,7 +77,7 @@ class FileDb : public FileDbInterface {
}
void load_file_data(const string &key, Promise<FileData> promise) {
promise.set_result(load_file_data_impl(actor_id(this), file_pmc(), key));
promise.set_result(load_file_data_impl(actor_id(this), file_pmc(), key, current_pmc_id_));
}
void clear_file_data(FileDbId id, const string &remote_key, const string &local_key, const string &generate_key) {
@ -194,7 +194,7 @@ class FileDb : public FileDbInterface {
}
Result<FileData> get_file_data_sync_impl(string key) override {
return load_file_data_impl(file_db_actor_.get(), file_kv_safe_->get(), key);
return load_file_data_impl(file_db_actor_.get(), file_kv_safe_->get(), key, current_pmc_id_);
}
void clear_file_data(FileDbId id, const FileData &file_data) override {
@ -247,7 +247,7 @@ class FileDb : public FileDbInterface {
std::shared_ptr<SqliteKeyValueSafe> file_kv_safe_;
static Result<FileData> load_file_data_impl(ActorId<FileDbActor> file_db_actor_id, SqliteKeyValue &pmc,
const string &key) {
const string &key, FileDbId current_pmc_id) {
//LOG(DEBUG) << "Load by key " << format::as_hex_dump<4>(Slice(key));
TRY_RESULT(id, get_id(pmc, key));
@ -256,7 +256,8 @@ class FileDb : public FileDbInterface {
int attempt_count = 0;
while (true) {
if (attempt_count > 100) {
LOG(FATAL) << "Cycle in file database?";
LOG(FATAL) << "Cycle in file database? current_pmc_id=" << current_pmc_id << " key=" << key
<< " links=" << format::as_array(ids);
}
attempt_count++;

View File

@ -51,4 +51,8 @@ class FileDbId {
}
};
inline StringBuilder &operator<<(StringBuilder &sb, const FileDbId &id) {
return sb << "FileDbId{" << id.get() << "}";
}
} // namespace td

View File

@ -65,7 +65,7 @@ class SecretChatLogEventBase : public SecretChatEvent {
// inputEncryptedFile#5a17b5e5 id:long access_hash:long = InputEncryptedFile;
// inputEncryptedFileBigUploaded#2dc173c8 id:long parts:int key_fingerprint:int = InputEncryptedFile;
struct EncryptedInputFile {
static constexpr int32 magic = 0x4328d38a;
static constexpr int32 MAGIC = 0x4328d38a;
enum Type : int32 { Empty = 0, Uploaded = 1, BigUploaded = 2, Location = 3 } type = Type::Empty;
int64 id = 0;
int64 access_hash = 0;
@ -75,7 +75,7 @@ struct EncryptedInputFile {
template <class StorerT>
void store(StorerT &storer) const {
using td::store;
store(magic, storer);
store(MAGIC, storer);
store(type, storer);
store(id, storer);
store(access_hash, storer);
@ -104,7 +104,7 @@ struct EncryptedInputFile {
parse(parts, parser);
parse(key_fingerprint, parser);
if (got_magic != magic) {
if (got_magic != MAGIC) {
parser.set_error("EncryptedInputFile magic mismatch");
return;
}
@ -156,7 +156,7 @@ inline StringBuilder &operator<<(StringBuilder &sb, const EncryptedInputFile &fi
// encryptedFile#4a70994c id:long access_hash:long size:int dc_id:int key_fingerprint:int = EncryptedFile;
struct EncryptedFileLocation {
static constexpr int32 magic = 0x473d738a;
static constexpr int32 MAGIC = 0x473d738a;
int64 id = 0;
int64 access_hash = 0;
int32 size = 0;
@ -169,7 +169,7 @@ struct EncryptedFileLocation {
template <class StorerT>
void store(StorerT &storer) const {
using td::store;
store(magic, storer);
store(MAGIC, storer);
store(id, storer);
store(access_hash, storer);
store(size, storer);
@ -189,7 +189,7 @@ struct EncryptedFileLocation {
parse(dc_id, parser);
parse(key_fingerprint, parser);
if (got_magic != magic) {
if (got_magic != MAGIC) {
parser.set_error("EncryptedFileLocation magic mismatch");
return;
}

View File

@ -332,7 +332,7 @@ Result<string> check_url(Slice url) {
}
TRY_RESULT(http_url, parse_url(url));
if (is_tg || is_ton) {
if (tolower_begins_with(url, "http://") || http_url.protocol_ == HttpUrl::Protocol::HTTPS ||
if (tolower_begins_with(url, "http://") || http_url.protocol_ == HttpUrl::Protocol::Https ||
!http_url.userinfo_.empty() || http_url.specified_port_ != 0 || http_url.is_ipv6_) {
return Status::Error(is_tg ? Slice("Wrong tg URL") : Slice("Wrong ton URL"));
}

View File

@ -327,11 +327,11 @@ void ConnectionCreator::ping_proxy(int32 proxy_id, Promise<double> promise) {
continue;
}
ping_proxy_socket_fd(r_socket_fd.move_as_ok(), r_transport_type.move_as_ok(),
PromiseCreator::lambda([actor_id = actor_id(this), token](Result<double> result) {
send_closure(actor_id, &ConnectionCreator::on_ping_main_dc_result, token,
std::move(result));
}));
ping_proxy_socket_fd(
r_socket_fd.move_as_ok(), r_transport_type.move_as_ok(), PSTRING() << info.option->get_ip_address(),
PromiseCreator::lambda([actor_id = actor_id(this), token](Result<double> result) {
send_closure(actor_id, &ConnectionCreator::on_ping_main_dc_result, token, std::move(result));
}));
}
return;
}
@ -367,14 +367,14 @@ void ConnectionCreator::ping_proxy_resolved(int32 proxy_id, IPAddress ip_address
}
auto socket_fd = r_socket_fd.move_as_ok();
auto connection_promise =
PromiseCreator::lambda([promise = std::move(promise), actor_id = actor_id(this),
transport_type = extra.transport_type](Result<ConnectionData> r_connection_data) mutable {
auto connection_promise = PromiseCreator::lambda(
[promise = std::move(promise), actor_id = actor_id(this), transport_type = extra.transport_type,
debug_str = std::move(extra.debug_str)](Result<ConnectionData> r_connection_data) mutable {
if (r_connection_data.is_error()) {
return promise.set_error(Status::Error(400, r_connection_data.error().public_message()));
}
send_closure(actor_id, &ConnectionCreator::ping_proxy_socket_fd, r_connection_data.move_as_ok().socket_fd,
std::move(transport_type), std::move(promise));
std::move(transport_type), std::move(debug_str), std::move(promise));
});
CHECK(proxy.use_proxy());
auto token = next_token();
@ -387,11 +387,11 @@ void ConnectionCreator::ping_proxy_resolved(int32 proxy_id, IPAddress ip_address
}
void ConnectionCreator::ping_proxy_socket_fd(SocketFd socket_fd, mtproto::TransportType transport_type,
Promise<double> promise) {
string debug_str, Promise<double> promise) {
auto token = next_token();
auto raw_connection = make_unique<mtproto::RawConnection>(std::move(socket_fd), std::move(transport_type), nullptr);
children_[token] = {
false, create_ping_actor("", std::move(raw_connection), nullptr,
false, create_ping_actor(std::move(debug_str), std::move(raw_connection), nullptr,
PromiseCreator::lambda([promise = std::move(promise)](
Result<unique_ptr<mtproto::RawConnection>> result) mutable {
if (result.is_error()) {

View File

@ -243,7 +243,8 @@ class ConnectionCreator : public NetQueryCallback {
void ping_proxy_resolved(int32 proxy_id, IPAddress ip_address, Promise<double> promise);
void ping_proxy_socket_fd(SocketFd socket_fd, mtproto::TransportType transport_type, Promise<double> promise);
void ping_proxy_socket_fd(SocketFd socket_fd, mtproto::TransportType transport_type, string debug_str,
Promise<double> promise);
void on_ping_main_dc_result(uint64 token, Result<double> result);
};

View File

@ -86,7 +86,7 @@ void NetStatsManager::get_network_stats(bool current, Promise<NetworkStats> prom
if (id == 0) {
} else if (id == 1) {
total = stats;
} else if (id == call_net_stats_id_) {
} else if (id == CALL_NET_STATS_ID) {
} else if (file_type != FileType::None) {
total_files = total_files + stats;
}
@ -109,7 +109,7 @@ void NetStatsManager::get_network_stats(bool current, Promise<NetworkStats> prom
entry.duration = stats.duration;
if (id == 0) {
result.entries.push_back(std::move(entry));
} else if (id == call_net_stats_id_) {
} else if (id == CALL_NET_STATS_ID) {
entry.is_call = true;
result.entries.push_back(std::move(entry));
} else if (file_type != FileType::None) {

View File

@ -119,7 +119,7 @@ class NetStatsManager : public Actor {
NetStatsInfo media_net_stats_;
std::array<NetStatsInfo, file_type_size> files_stats_;
NetStatsInfo call_net_stats_;
static constexpr int32 call_net_stats_id_{file_type_size + 2};
static constexpr int32 CALL_NET_STATS_ID{file_type_size + 2};
template <class F>
void for_each_stat(F &&f) {
@ -130,7 +130,7 @@ class NetStatsManager : public Actor {
auto file_type = static_cast<FileType>(file_type_i);
f(stat, file_type_i + 2, get_file_type_name(file_type), file_type);
}
f(call_net_stats_, call_net_stats_id_, CSlice("calls"), FileType::None);
f(call_net_stats_, CALL_NET_STATS_ID, CSlice("calls"), FileType::None);
}
void add_network_stats_impl(NetStatsInfo &info, const NetworkStatsEntry &entry);

View File

@ -62,12 +62,10 @@ class SafePromise;
template <class T = Unit>
class Promise {
public:
bool was_set_value{false};
void set_value(T &&value) {
if (!promise_) {
return;
}
was_set_value = true;
promise_->set_value(std::move(value));
promise_.reset();
}
@ -75,7 +73,6 @@ class Promise {
if (!promise_) {
return;
}
was_set_value = true;
promise_->set_error(std::move(error));
promise_.reset();
}
@ -83,7 +80,6 @@ class Promise {
if (!promise_) {
return;
}
was_set_value = true;
promise_->set_result(std::move(result));
promise_.reset();
}
@ -412,7 +408,7 @@ class FutureActor final : public Actor {
public:
enum State { Waiting, Ready };
static constexpr int Hangup = 426487;
static constexpr int HANGUP_ERROR_CODE = 426487;
FutureActor() = default;
@ -491,7 +487,7 @@ class FutureActor final : public Actor {
}
void hangup() override {
set_error(Status::Error<Hangup>());
set_error(Status::Error<HANGUP_ERROR_CODE>());
}
void start_up() override {

View File

@ -17,6 +17,7 @@ set(TDDB_SOURCE
td/db/SqliteKeyValue.cpp
td/db/SqliteKeyValueAsync.cpp
td/db/SqliteStatement.cpp
td/db/TQueue.cpp
td/db/detail/RawSqliteDb.cpp
@ -31,7 +32,6 @@ set(TDDB_SOURCE
td/db/BinlogKeyValue.h
td/db/DbKey.h
td/db/KeyValueSyncInterface.h
td/db/Pmc.h
td/db/SeqKeyValue.h
td/db/SqliteConnectionSafe.h
td/db/SqliteDb.h
@ -39,6 +39,7 @@ set(TDDB_SOURCE
td/db/SqliteKeyValueAsync.h
td/db/SqliteKeyValueSafe.h
td/db/SqliteStatement.h
td/db/TQueue.h
td/db/TsSeqKeyValue.h
td/db/detail/RawSqliteDb.h

View File

@ -34,7 +34,7 @@ namespace td {
template <class BinlogT>
class BinlogKeyValue : public KeyValueSyncInterface {
public:
static constexpr int32 magic = 0x2a280000;
static constexpr int32 MAGIC = 0x2a280000;
struct Event : public Storer {
Event() = default;
@ -236,7 +236,7 @@ class BinlogKeyValue : public KeyValueSyncInterface {
std::unordered_map<string, std::pair<string, uint64>> map_;
std::shared_ptr<BinlogT> binlog_;
RwMutex rw_mutex_;
int32 magic_ = magic;
int32 magic_ = MAGIC;
};
template <>

432
tddb/td/db/TQueue.cpp Normal file
View File

@ -0,0 +1,432 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/db/TQueue.h"
#include "td/db/binlog/Binlog.h"
#include "td/db/binlog/BinlogEvent.h"
#include "td/db/binlog/BinlogHelper.h"
#include "td/db/binlog/BinlogInterface.h"
#include "td/utils/format.h"
#include "td/utils/misc.h"
#include "td/utils/port/Clocks.h"
#include "td/utils/Random.h"
#include "td/utils/StorerBase.h"
#include "td/utils/Time.h"
#include "td/utils/tl_helpers.h"
#include "td/utils/tl_parsers.h"
#include "td/utils/tl_storers.h"
#include "td/utils/VectorQueue.h"
#include <algorithm>
#include <unordered_map>
namespace td {
using EventId = TQueue::EventId;
EventId::EventId() {
}
Result<EventId> EventId::from_int32(int32 id) {
if (!is_valid_id(id)) {
return Status::Error("Invalid ID");
}
return EventId(id);
}
bool EventId::is_valid() const {
return !empty() && is_valid_id(id_);
}
int32 EventId::value() const {
return id_;
}
Result<EventId> EventId::next() const {
return from_int32(id_ + 1);
}
Result<EventId> EventId::advance(size_t offset) const {
TRY_RESULT(new_id, narrow_cast_safe<int32>(id_ + offset));
return from_int32(new_id);
}
bool EventId::empty() const {
return id_ == 0;
}
bool EventId::operator==(const EventId &other) const {
return id_ == other.id_;
}
bool EventId::operator!=(const EventId &other) const {
return !(*this == other);
}
bool EventId::operator<(const EventId &other) const {
return id_ < other.id_;
}
StringBuilder &operator<<(StringBuilder &string_builder, const EventId id) {
return string_builder << "EventId{" << id.value() << "}";
}
EventId::EventId(int32 id) : id_(id) {
CHECK(is_valid_id(id));
}
bool EventId::is_valid_id(int32 id) {
return 0 <= id && id < MAX_ID;
}
class TQueueImpl : public TQueue {
static constexpr size_t MAX_EVENT_LEN = 65536 * 8;
static constexpr size_t MAX_QUEUE_EVENTS = 1000000;
public:
void set_callback(unique_ptr<StorageCallback> callback) override {
callback_ = std::move(callback);
}
unique_ptr<StorageCallback> extract_callback() override {
return std::move(callback_);
}
bool do_push(QueueId queue_id, RawEvent &&raw_event) override {
CHECK(raw_event.event_id.is_valid());
auto &q = queues_[queue_id];
if (q.events.empty() || q.events.back().event_id < raw_event.event_id) {
if (raw_event.logevent_id == 0 && callback_ != nullptr) {
raw_event.logevent_id = callback_->push(queue_id, raw_event);
}
q.tail_id = raw_event.event_id.next().move_as_ok();
q.events.push(std::move(raw_event));
return true;
}
return false;
}
Result<EventId> push(QueueId queue_id, string data, double expires_at, int64 extra, EventId hint_new_id) override {
auto &q = queues_[queue_id];
if (q.events.size() >= MAX_QUEUE_EVENTS) {
return Status::Error("Queue is full");
}
if (data.empty()) {
return Status::Error("Data is empty");
}
if (data.size() > MAX_EVENT_LEN) {
return Status::Error("Data is too big");
}
EventId event_id;
while (true) {
if (q.tail_id.empty()) {
if (hint_new_id.empty()) {
q.tail_id = EventId::from_int32(Random::fast(2 * MAX_QUEUE_EVENTS + 1, EventId::MAX_ID / 2)).move_as_ok();
} else {
q.tail_id = hint_new_id;
}
}
event_id = q.tail_id;
CHECK(event_id.is_valid());
if (event_id.next().is_ok()) {
break;
}
for (auto &event : q.events.as_mutable_span()) {
pop(queue_id, event, {});
}
q.tail_id = EventId();
q.events = {};
CHECK(hint_new_id.next().is_ok());
}
RawEvent raw_event;
raw_event.event_id = event_id;
raw_event.data = std::move(data);
raw_event.expires_at = expires_at;
raw_event.extra = extra;
bool is_added = do_push(queue_id, std::move(raw_event));
CHECK(is_added);
return event_id;
}
EventId get_head(QueueId queue_id) const override {
auto it = queues_.find(queue_id);
if (it == queues_.end()) {
return EventId();
}
auto &q = it->second;
if (q.events.empty()) {
return q.tail_id;
}
return q.events.front().event_id;
}
EventId get_tail(QueueId queue_id) const override {
auto it = queues_.find(queue_id);
if (it == queues_.end()) {
return EventId();
}
auto &q = it->second;
return q.tail_id;
}
void forget(QueueId queue_id, EventId event_id) override {
auto q_it = queues_.find(queue_id);
if (q_it == queues_.end()) {
return;
}
auto &q = q_it->second;
auto from_events = q.events.as_mutable_span();
auto it = std::lower_bound(from_events.begin(), from_events.end(), event_id,
[](auto &event, EventId event_id) { return event.event_id < event_id; });
if (it == from_events.end() || !(it->event_id == event_id)) {
return;
}
pop(queue_id, *it, q.tail_id);
}
Result<size_t> get(QueueId queue_id, EventId from_id, bool forget_previous, double now,
MutableSpan<Event> &result_events) override {
auto it = queues_.find(queue_id);
if (it == queues_.end()) {
result_events.truncate(0);
return 0;
}
auto &q = it->second;
// Some sanity checks
if (from_id.value() > q.tail_id.value() + 10) {
return Status::Error("Specified from_id is in the future");
}
if (from_id.value() < q.tail_id.value() - static_cast<int32>(MAX_QUEUE_EVENTS) * 2) {
return Status::Error("Specified from_id is in the past");
}
MutableSpan<RawEvent> from_events;
size_t ready_n = 0;
size_t i = 0;
while (true) {
from_events = q.events.as_mutable_span();
ready_n = 0;
size_t first_i = 0;
if (!forget_previous) {
first_i = std::lower_bound(from_events.begin(), from_events.end(), from_id,
[](auto &event, EventId event_id) { return event.event_id < event_id; }) -
from_events.begin();
}
for (i = first_i; i < from_events.size(); i++) {
auto &from = from_events[i];
try_pop(queue_id, from, forget_previous ? from_id : EventId{}, q.tail_id, now);
if (from.data.empty()) {
continue;
}
if (ready_n == result_events.size()) {
break;
}
CHECK(!(from.event_id < from_id));
auto &to = result_events[ready_n];
to.data = from.data;
to.id = from.event_id;
to.expires_at = from.expires_at;
to.extra = from.extra;
ready_n++;
}
// compactify skipped events
if ((ready_n + 1) * 2 < i + first_i) {
compactify(q.events, i);
continue;
}
break;
}
result_events.truncate(ready_n);
size_t left_n = from_events.size() - i;
return ready_n + left_n;
}
void run_gc(double now) override {
for (auto &it : queues_) {
for (auto &e : it.second.events.as_mutable_span()) {
try_pop(it.first, e, EventId(), it.second.tail_id, now);
}
}
}
private:
struct Queue {
EventId tail_id;
VectorQueue<RawEvent> events;
};
std::unordered_map<QueueId, Queue> queues_;
unique_ptr<StorageCallback> callback_;
static void compactify(VectorQueue<RawEvent> &events, size_t prefix) {
if (prefix == events.size()) {
CHECK(!events.empty());
prefix--;
}
auto processed = events.as_mutable_span().substr(0, prefix);
auto removed_n =
processed.rend() - std::remove_if(processed.rbegin(), processed.rend(), [](auto &e) { return e.data.empty(); });
events.pop_n(removed_n);
}
void try_pop(QueueId queue_id, RawEvent &event, EventId from_id, EventId tail_id, double now) {
if (event.expires_at < now || event.event_id < from_id || event.data.empty()) {
pop(queue_id, event, tail_id);
}
}
void pop(QueueId queue_id, RawEvent &event, EventId tail_id) {
if (callback_ == nullptr || event.logevent_id == 0) {
event.logevent_id = 0;
event.data = {};
return;
}
if (event.event_id.next().ok() == tail_id) {
if (!event.data.empty()) {
event.data = {};
callback_->push(queue_id, event);
}
} else {
callback_->pop(event.logevent_id);
event.logevent_id = 0;
event.data = {};
}
}
};
unique_ptr<TQueue> TQueue::create() {
return make_unique<TQueueImpl>();
}
struct TQueueLogEvent : public Storer {
int64 queue_id;
int32 event_id;
int32 expires_at;
Slice data;
int64 extra;
template <class StorerT>
void store(StorerT &&storer) const {
using td::store;
store(queue_id, storer);
store(event_id, storer);
store(expires_at, storer);
store(data, storer);
if (extra != 0) {
store(extra, storer);
}
}
template <class ParserT>
void parse(ParserT &&parser, int32 has_extra) {
using td::parse;
parse(queue_id, parser);
parse(event_id, parser);
parse(expires_at, parser);
data = parser.template fetch_string<Slice>();
if (has_extra == 0) {
extra = 0;
} else {
parse(extra, parser);
}
}
size_t size() const override {
TlStorerCalcLength storer;
store(storer);
return storer.get_length();
}
size_t store(uint8 *ptr) const override {
TlStorerUnsafe storer(ptr);
store(storer);
return static_cast<size_t>(storer.get_buf() - ptr);
}
};
template <class BinlogT>
TQueueBinlog<BinlogT>::TQueueBinlog() {
diff_ = Clocks::system() - Time::now();
}
template <class BinlogT>
uint64 TQueueBinlog<BinlogT>::push(QueueId queue_id, const RawEvent &event) {
TQueueLogEvent log_event;
log_event.queue_id = queue_id;
log_event.event_id = event.event_id.value();
log_event.expires_at = static_cast<int32>(event.expires_at + diff_ + 1);
log_event.data = event.data;
log_event.extra = event.extra;
auto magic = magic_ + (log_event.extra != 0);
if (event.logevent_id == 0) {
return binlog_->add(magic, log_event);
}
binlog_->rewrite(event.logevent_id, magic, log_event);
return event.logevent_id;
}
template <class BinlogT>
void TQueueBinlog<BinlogT>::pop(uint64 logevent_id) {
binlog_->erase(logevent_id);
}
template <class BinlogT>
Status TQueueBinlog<BinlogT>::replay(const BinlogEvent &binlog_event, TQueue &q) const {
TQueueLogEvent event;
TlParser parser(binlog_event.data_);
int32 has_extra = binlog_event.type_ - magic_;
if (has_extra != 0 && has_extra != 1) {
return Status::Error("Wrong magic");
}
event.parse(parser, has_extra);
parser.fetch_end();
TRY_STATUS(parser.get_status());
TRY_RESULT(event_id, EventId::from_int32(event.event_id));
RawEvent raw_event;
raw_event.logevent_id = binlog_event.id_;
raw_event.event_id = event_id;
raw_event.expires_at = event.expires_at - diff_;
raw_event.data = event.data.str();
raw_event.extra = event.extra;
if (!q.do_push(event.queue_id, std::move(raw_event))) {
return Status::Error("Failed to add event");
}
return Status::OK();
}
template class TQueueBinlog<BinlogInterface>;
template class TQueueBinlog<Binlog>;
uint64 TQueueMemoryStorage::push(QueueId queue_id, const RawEvent &event) {
auto logevent_id = event.logevent_id == 0 ? next_logevent_id_++ : event.logevent_id;
events_[logevent_id] = std::make_pair(queue_id, event);
return logevent_id;
}
void TQueueMemoryStorage::pop(uint64 logevent_id) {
events_.erase(logevent_id);
}
void TQueueMemoryStorage::replay(TQueue &q) const {
for (auto &e : events_) {
auto x = e.second;
x.second.logevent_id = e.first;
bool is_added = q.do_push(x.first, std::move(x.second));
CHECK(is_added);
}
}
} // namespace td

148
tddb/td/db/TQueue.h Normal file
View File

@ -0,0 +1,148 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/Span.h"
#include "td/utils/Status.h"
#include "td/utils/StringBuilder.h"
#include <map>
#include <memory>
#include <utility>
namespace td {
class TQueue {
public:
class EventId {
public:
static constexpr int32 MAX_ID = 2000000000;
EventId();
static Result<EventId> from_int32(int32 id);
bool is_valid() const;
int32 value() const;
Result<EventId> next() const;
Result<EventId> advance(size_t offset) const;
bool empty() const;
bool operator==(const EventId &other) const;
bool operator!=(const EventId &other) const;
bool operator<(const EventId &other) const;
private:
int32 id_{0};
explicit EventId(int32 id);
static bool is_valid_id(int32 id);
};
struct Event {
EventId id;
Slice data;
int64 extra{0};
double expires_at{0};
};
struct RawEvent {
uint64 logevent_id{0};
EventId event_id;
string data;
int64 extra{0};
double expires_at{0};
};
using QueueId = int64;
class StorageCallback {
public:
using QueueId = TQueue::QueueId;
using RawEvent = TQueue::RawEvent;
StorageCallback() = default;
StorageCallback(const StorageCallback &) = delete;
StorageCallback &operator=(const StorageCallback &) = delete;
StorageCallback(StorageCallback &&) = delete;
StorageCallback &operator=(StorageCallback &&) = delete;
virtual ~StorageCallback() = default;
virtual uint64 push(QueueId queue_id, const RawEvent &event) = 0;
virtual void pop(uint64 logevent_id) = 0;
};
static unique_ptr<TQueue> create();
TQueue() = default;
TQueue(const TQueue &) = delete;
TQueue &operator=(const TQueue &) = delete;
TQueue(TQueue &&) = delete;
TQueue &operator=(TQueue &&) = delete;
virtual ~TQueue() = default;
virtual void set_callback(unique_ptr<StorageCallback> callback) = 0;
virtual unique_ptr<StorageCallback> extract_callback() = 0;
virtual bool do_push(QueueId queue_id, RawEvent &&raw_event) = 0;
virtual Result<EventId> push(QueueId queue_id, string data, double expires_at, int64 extra, EventId hint_new_id) = 0;
virtual void forget(QueueId queue_id, EventId event_id) = 0;
virtual EventId get_head(QueueId queue_id) const = 0;
virtual EventId get_tail(QueueId queue_id) const = 0;
virtual Result<size_t> get(QueueId queue_id, EventId from_id, bool forget_previous, double now,
MutableSpan<Event> &result_events) = 0;
virtual void run_gc(double now) = 0;
};
StringBuilder &operator<<(StringBuilder &string_builder, const TQueue::EventId id);
struct BinlogEvent;
template <class BinlogT>
class TQueueBinlog : public TQueue::StorageCallback {
public:
TQueueBinlog();
uint64 push(QueueId queue_id, const RawEvent &event) override;
void pop(uint64 logevent_id) override;
Status replay(const BinlogEvent &binlog_event, TQueue &q) const;
void set_binlog(std::shared_ptr<BinlogT> binlog) {
binlog_ = std::move(binlog);
}
private:
std::shared_ptr<BinlogT> binlog_;
int32 magic_{2314};
double diff_{0};
};
class TQueueMemoryStorage : public TQueue::StorageCallback {
public:
uint64 push(QueueId queue_id, const RawEvent &event) override;
void pop(uint64 logevent_id) override;
void replay(TQueue &q) const;
private:
uint64 next_logevent_id_{1};
std::map<uint64, std::pair<QueueId, RawEvent>> events_;
};
} // namespace td

View File

@ -118,10 +118,10 @@ class BinlogReader {
it.advance(4, MutableSlice(buf, 4));
size_ = static_cast<size_t>(TlParser(Slice(buf, 4)).fetch_int());
if (size_ > MAX_EVENT_SIZE) {
if (size_ > BinlogEvent::MAX_SIZE) {
return Status::Error(PSLICE() << "Too big event " << tag("size", size_));
}
if (size_ < MIN_EVENT_SIZE) {
if (size_ < BinlogEvent::MIN_SIZE) {
return Status::Error(PSLICE() << "Too small event " << tag("size", size_));
}
if (size_ % 4 != 0) {

View File

@ -18,6 +18,7 @@
#include "td/utils/port/FileFd.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include "td/utils/StorerBase.h"
#include "td/utils/UInt.h"
#include <functional>
@ -68,9 +69,25 @@ class Binlog {
return fd_.empty();
}
//void add_raw_event(BufferSlice &&raw_event) {
//add_event(BinlogEvent(std::move(raw_event)));
//}
uint64 add(int32 type, const Storer &storer) {
auto logevent_id = next_id();
add_raw_event(BinlogEvent::create_raw(logevent_id, type, 0, storer), {});
return logevent_id;
}
uint64 rewrite(uint64 logevent_id, int32 type, const Storer &storer) {
auto seq_no = next_id();
add_raw_event(BinlogEvent::create_raw(logevent_id, type, BinlogEvent::Flags::Rewrite, storer), {});
return seq_no;
}
uint64 erase(uint64 logevent_id) {
auto seq_no = next_id();
add_raw_event(BinlogEvent::create_raw(logevent_id, BinlogEvent::ServiceTypes::Empty, BinlogEvent::Flags::Rewrite,
EmptyStorer()),
{});
return seq_no;
}
void add_raw_event(BufferSlice &&raw_event, BinlogDebugInfo info) {
add_event(BinlogEvent(std::move(raw_event), info));
@ -131,8 +148,6 @@ class Binlog {
bool need_sync_{false};
enum class State { Empty, Load, Reindex, Run } state_{State::Empty};
static constexpr uint32 MAX_EVENT_SIZE = 65536;
Result<FileFd> open_binlog(const string &path, int32 flags);
size_t flush_events_buffer(bool force);
void do_add_event(BinlogEvent &&event);

View File

@ -23,13 +23,13 @@ Status BinlogEvent::init(BufferSlice &&raw_event, bool check_crc) {
type_ = parser.fetch_int();
flags_ = parser.fetch_int();
extra_ = parser.fetch_long();
CHECK(size_ >= MIN_EVENT_SIZE);
auto slice_data = parser.fetch_string_raw<Slice>(size_ - MIN_EVENT_SIZE);
CHECK(size_ >= MIN_SIZE);
auto slice_data = parser.fetch_string_raw<Slice>(size_ - MIN_SIZE);
data_ = MutableSlice(const_cast<char *>(slice_data.begin()), slice_data.size());
crc32_ = static_cast<uint32>(parser.fetch_int());
if (check_crc) {
CHECK(size_ >= EVENT_TAIL_SIZE);
auto calculated_crc = crc32(raw_event.as_slice().truncate(size_ - EVENT_TAIL_SIZE));
CHECK(size_ >= TAIL_SIZE);
auto calculated_crc = crc32(raw_event.as_slice().truncate(size_ - TAIL_SIZE));
if (calculated_crc != crc32_) {
return Status::Error(PSLICE() << "crc mismatch " << tag("actual", format::as_hex(calculated_crc))
<< tag("expected", format::as_hex(crc32_)) << public_to_string());
@ -52,7 +52,7 @@ Status BinlogEvent::validate() const {
}
BufferSlice BinlogEvent::create_raw(uint64 id, int32 type, int32 flags, const Storer &storer) {
auto raw_event = BufferSlice{storer.size() + MIN_EVENT_SIZE};
auto raw_event = BufferSlice{storer.size() + MIN_SIZE};
TlStorerUnsafe tl_storer(raw_event.as_slice().ubegin());
tl_storer.store_int(narrow_cast<int32>(raw_event.size()));
@ -61,11 +61,11 @@ BufferSlice BinlogEvent::create_raw(uint64 id, int32 type, int32 flags, const St
tl_storer.store_int(flags);
tl_storer.store_long(0);
CHECK(tl_storer.get_buf() == raw_event.as_slice().ubegin() + EVENT_HEADER_SIZE);
CHECK(tl_storer.get_buf() == raw_event.as_slice().ubegin() + HEADER_SIZE);
tl_storer.store_storer(storer);
CHECK(tl_storer.get_buf() == raw_event.as_slice().uend() - EVENT_TAIL_SIZE);
tl_storer.store_int(::td::crc32(raw_event.as_slice().truncate(raw_event.size() - EVENT_TAIL_SIZE)));
CHECK(tl_storer.get_buf() == raw_event.as_slice().uend() - TAIL_SIZE);
tl_storer.store_int(::td::crc32(raw_event.as_slice().truncate(raw_event.size() - TAIL_SIZE)));
return raw_event;
}

View File

@ -32,11 +32,6 @@ inline auto EmptyStorer() {
return create_default_storer(impl);
}
static constexpr size_t MAX_EVENT_SIZE = 1 << 24;
static constexpr size_t EVENT_HEADER_SIZE = 4 + 8 + 4 + 4 + 8;
static constexpr size_t EVENT_TAIL_SIZE = 4;
static constexpr size_t MIN_EVENT_SIZE = EVENT_HEADER_SIZE + EVENT_TAIL_SIZE;
extern int32 VERBOSITY_NAME(binlog);
struct BinlogDebugInfo {
@ -46,6 +41,7 @@ struct BinlogDebugInfo {
const char *file{""};
int line{0};
};
inline StringBuilder &operator<<(StringBuilder &sb, const BinlogDebugInfo &info) {
if (info.line == 0) {
return sb;
@ -54,6 +50,11 @@ inline StringBuilder &operator<<(StringBuilder &sb, const BinlogDebugInfo &info)
}
struct BinlogEvent {
static constexpr size_t MAX_SIZE = 1 << 24;
static constexpr size_t HEADER_SIZE = 4 + 8 + 4 + 4 + 8;
static constexpr size_t TAIL_SIZE = 4;
static constexpr size_t MIN_SIZE = HEADER_SIZE + TAIL_SIZE;
int64 offset_;
uint32 size_;

View File

@ -9,38 +9,24 @@
#include "td/actor/PromiseFuture.h"
#include "td/db/binlog/BinlogEvent.h"
#include "td/db/binlog/BinlogInterface.h"
#include "td/utils/common.h"
#include "td/utils/StorerBase.h"
namespace td {
template <class BinlogT, class StorerT>
uint64 binlog_add(const BinlogT &binlog_ptr, int32 type, const StorerT &storer, Promise<> promise = Promise<>()) {
auto logevent_id = binlog_ptr->next_id();
binlog_ptr->add_raw_event(logevent_id, BinlogEvent::create_raw(logevent_id, type, 0, storer), std::move(promise));
return logevent_id;
inline uint64 binlog_add(BinlogInterface *binlog_ptr, int32 type, const Storer &storer, Promise<> promise = Promise<>()) {
return binlog_ptr->add(type, storer, std::move(promise));
}
template <class BinlogT, class StorerT>
uint64 binlog_rewrite(const BinlogT &binlog_ptr, uint64 logevent_id, int32 type, const StorerT &storer,
inline uint64 binlog_rewrite(BinlogInterface *binlog_ptr, uint64 logevent_id, int32 type, const Storer &storer,
Promise<> promise = Promise<>()) {
auto seq_no = binlog_ptr->next_id();
binlog_ptr->add_raw_event(seq_no, BinlogEvent::create_raw(logevent_id, type, BinlogEvent::Flags::Rewrite, storer),
std::move(promise));
return seq_no;
return binlog_ptr->rewrite(logevent_id, type, storer, std::move(promise));
}
#define binlog_erase(...) binlog_erase_impl({__FILE__, __LINE__}, __VA_ARGS__)
template <class BinlogT>
uint64 binlog_erase_impl(BinlogDebugInfo info, const BinlogT &binlog_ptr, uint64 logevent_id,
Promise<> promise = Promise<>()) {
auto seq_no = binlog_ptr->next_id();
binlog_ptr->add_raw_event(info, seq_no,
BinlogEvent::create_raw(logevent_id, BinlogEvent::ServiceTypes::Empty,
BinlogEvent::Flags::Rewrite, EmptyStorer()),
std::move(promise));
return seq_no;
inline uint64 binlog_erase(BinlogInterface *binlog_ptr, uint64 logevent_id, Promise<> promise = Promise<>()) {
return binlog_ptr->erase(logevent_id, std::move(promise));
}
} // namespace td

View File

@ -13,6 +13,7 @@
#include "td/utils/buffer.h"
#include "td/utils/common.h"
#include "td/utils/StorerBase.h"
namespace td {
@ -40,6 +41,29 @@ class BinlogInterface {
void lazy_sync(Promise<> promise = Promise<>()) {
add_raw_event_impl(next_id(), BufferSlice(), std::move(promise), {});
}
uint64 add(int32 type, const Storer &storer, Promise<> promise = Promise<>()) {
auto logevent_id = next_id();
add_raw_event_impl(logevent_id, BinlogEvent::create_raw(logevent_id, type, 0, storer), std::move(promise), {});
return logevent_id;
}
uint64 rewrite(uint64 logevent_id, int32 type, const Storer &storer, Promise<> promise = Promise<>()) {
auto seq_no = next_id();
add_raw_event_impl(seq_no, BinlogEvent::create_raw(logevent_id, type, BinlogEvent::Flags::Rewrite, storer),
std::move(promise), {});
return seq_no;
}
uint64 erase(uint64 logevent_id, Promise<> promise = Promise<>()) {
auto seq_no = next_id();
add_raw_event_impl(seq_no,
BinlogEvent::create_raw(logevent_id, BinlogEvent::ServiceTypes::Empty,
BinlogEvent::Flags::Rewrite, EmptyStorer()),
std::move(promise), {});
return seq_no;
}
virtual void force_sync(Promise<> promise) = 0;
virtual void force_flush() = 0;
virtual void change_key(DbKey db_key, Promise<> promise = Promise<>()) = 0;

View File

@ -164,7 +164,7 @@ void HttpConnectionBase::loop() {
}
if (state_ == State::Close) {
LOG_IF(INFO, fd_.need_flush_write()) << "Close nonempty connection";
LOG_IF(INFO, want_read && (fd_.input_buffer().size() > 0 || current_query_->type_ != HttpQuery::Type::EMPTY))
LOG_IF(INFO, want_read && (fd_.input_buffer().size() > 0 || current_query_->type_ != HttpQuery::Type::Empty))
<< "Close connection while reading request/response";
return stop();
}

View File

@ -24,8 +24,8 @@ MutableSlice HttpQuery::get_arg(Slice key) const {
return it == args_.end() ? MutableSlice() : it->second;
}
std::vector<std::pair<string, string>> HttpQuery::get_args() const {
std::vector<std::pair<string, string>> res;
td::vector<std::pair<string, string>> HttpQuery::get_args() const {
td::vector<std::pair<string, string>> res;
res.reserve(args_.size());
for (auto &it : args_) {
res.emplace_back(it.first.str(), it.second.str());
@ -48,20 +48,20 @@ int HttpQuery::get_retry_after() const {
StringBuilder &operator<<(StringBuilder &sb, const HttpQuery &q) {
switch (q.type_) {
case HttpQuery::Type::EMPTY:
case HttpQuery::Type::Empty:
sb << "EMPTY";
return sb;
case HttpQuery::Type::GET:
case HttpQuery::Type::Get:
sb << "GET";
break;
case HttpQuery::Type::POST:
case HttpQuery::Type::Post:
sb << "POST";
break;
case HttpQuery::Type::RESPONSE:
case HttpQuery::Type::Response:
sb << "RESPONSE";
break;
}
if (q.type_ == HttpQuery::Type::RESPONSE) {
if (q.type_ == HttpQuery::Type::Response) {
sb << ":" << q.code_ << ":" << q.reason_;
} else {
sb << ":" << q.url_path_;

View File

@ -19,25 +19,25 @@ namespace td {
class HttpQuery {
public:
enum class Type : int8 { EMPTY, GET, POST, RESPONSE };
enum class Type : int8 { Empty, Get, Post, Response };
std::vector<BufferSlice> container_;
Type type_ = Type::EMPTY;
td::vector<BufferSlice> container_;
Type type_ = Type::Empty;
int32 code_ = 0;
MutableSlice url_path_;
std::vector<std::pair<MutableSlice, MutableSlice>> args_;
td::vector<std::pair<MutableSlice, MutableSlice>> args_;
MutableSlice reason_;
bool keep_alive_ = true;
std::vector<std::pair<MutableSlice, MutableSlice>> headers_;
std::vector<HttpFile> files_;
td::vector<std::pair<MutableSlice, MutableSlice>> headers_;
td::vector<HttpFile> files_;
MutableSlice content_;
Slice get_header(Slice key) const;
MutableSlice get_arg(Slice key) const;
std::vector<std::pair<string, string>> get_args() const;
td::vector<std::pair<string, string>> get_args() const;
int get_retry_after() const;
};

View File

@ -663,12 +663,12 @@ Status HttpReader::parse_head(MutableSlice head) {
parser.skip(' ');
// GET POST HTTP/1.1
if (type == "GET") {
query_->type_ = HttpQuery::Type::GET;
query_->type_ = HttpQuery::Type::Get;
} else if (type == "POST") {
query_->type_ = HttpQuery::Type::POST;
query_->type_ = HttpQuery::Type::Post;
} else if (type.size() >= 4 && type.substr(0, 4) == "HTTP") {
if (type == "HTTP/1.1" || type == "HTTP/1.0") {
query_->type_ = HttpQuery::Type::RESPONSE;
query_->type_ = HttpQuery::Type::Response;
} else {
LOG(INFO) << "Unsupported HTTP version: " << type;
return Status::Error(505, "HTTP Version Not Supported");
@ -680,7 +680,7 @@ Status HttpReader::parse_head(MutableSlice head) {
query_->args_.clear();
if (query_->type_ == HttpQuery::Type::RESPONSE) {
if (query_->type_ == HttpQuery::Type::Response) {
query_->code_ = to_integer<int32>(parser.read_till(' '));
parser.skip(' ');
query_->reason_ = parser.read_till('\r');

View File

@ -74,7 +74,7 @@ Status Wget::try_init() {
TRY_STATUS(addr.init_host_port(url.host_, url.port_, prefer_ipv6_));
TRY_RESULT(fd, SocketFd::open(addr));
if (url.protocol_ == HttpUrl::Protocol::HTTP) {
if (url.protocol_ == HttpUrl::Protocol::Http) {
connection_ = create_actor<HttpOutboundConnection>("Connect", std::move(fd), SslStream{},
std::numeric_limits<std::size_t>::max(), 0, 0,
ActorOwn<HttpOutboundConnection::Callback>(actor_id(this)));

View File

@ -4,14 +4,6 @@ if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
set(CMAKE_INSTALL_LIBDIR "lib")
endif()
if (WIN32)
if (WINGETOPT_FOUND)
set(TD_HAVE_GETOPT 1)
endif()
else()
set(TD_HAVE_GETOPT 1)
endif()
if (NOT ZLIB_FOUND)
find_package(ZLIB)
endif()
@ -58,6 +50,7 @@ set(TDUTILS_SOURCE
td/utils/port/MemoryMapping.cpp
td/utils/port/path.cpp
td/utils/port/PollFlags.cpp
td/utils/port/rlimit.cpp
td/utils/port/ServerSocketFd.cpp
td/utils/port/signals.cpp
td/utils/port/sleep.cpp
@ -67,6 +60,7 @@ set(TDUTILS_SOURCE
td/utils/port/StdStreams.cpp
td/utils/port/thread_local.cpp
td/utils/port/UdpSocketFd.cpp
td/utils/port/user.cpp
td/utils/port/wstring_convert.cpp
td/utils/port/detail/Epoll.cpp
@ -102,7 +96,7 @@ set(TDUTILS_SOURCE
td/utils/misc.cpp
td/utils/MimeType.cpp
td/utils/MpmcQueue.cpp
td/utils/OptionsParser.cpp
td/utils/OptionParser.cpp
td/utils/PathView.cpp
td/utils/Random.cpp
td/utils/SharedSlice.cpp
@ -133,6 +127,7 @@ set(TDUTILS_SOURCE
td/utils/port/Poll.h
td/utils/port/PollBase.h
td/utils/port/PollFlags.h
td/utils/port/rlimit.h
td/utils/port/RwMutex.h
td/utils/port/ServerSocketFd.h
td/utils/port/signals.h
@ -144,6 +139,7 @@ set(TDUTILS_SOURCE
td/utils/port/thread.h
td/utils/port/thread_local.h
td/utils/port/UdpSocketFd.h
td/utils/port/user.h
td/utils/port/wstring_convert.h
td/utils/port/detail/Epoll.h
@ -156,6 +152,7 @@ set(TDUTILS_SOURCE
td/utils/port/detail/Poll.h
td/utils/port/detail/PollableFd.h
td/utils/port/detail/Select.h
td/utils/port/detail/skip_eintr.h
td/utils/port/detail/ThreadIdGuard.h
td/utils/port/detail/ThreadPthread.h
td/utils/port/detail/ThreadStl.h
@ -217,7 +214,7 @@ set(TDUTILS_SOURCE
td/utils/ObjectPool.h
td/utils/Observer.h
td/utils/optional.h
td/utils/OptionsParser.h
td/utils/OptionParser.h
td/utils/OrderedEventsProcessor.h
td/utils/overloaded.h
td/utils/Parser.h
@ -275,6 +272,7 @@ set(TDUTILS_TEST_SOURCE
${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcQueue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcWaiter.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/MpscLinkQueue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/OptionParser.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/OrderedEventsProcessor.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/port.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test/pq.cpp
@ -291,7 +289,7 @@ if (WIN32)
# find_library(WS2_32_LIBRARY ws2_32)
# find_library(MSWSOCK_LIBRARY Mswsock)
# target_link_libraries(tdutils PRIVATE ${WS2_32_LIBRARY} ${MSWSOCK_LIBRARY})
target_link_libraries(tdutils PRIVATE ws2_32 Mswsock Normaliz)
target_link_libraries(tdutils PRIVATE ws2_32 Mswsock Normaliz psapi)
endif()
if (NOT CMAKE_CROSSCOMPILING)
add_dependencies(tdutils tdmime_auto)
@ -319,10 +317,6 @@ if (ABSL_FOUND)
target_link_libraries(tdutils PUBLIC absl::flat_hash_map absl::flat_hash_set absl::hash)
endif()
if (WIN32 AND WINGETOPT_FOUND)
target_link_libraries(tdutils PRIVATE wingetopt)
endif()
if (ANDROID)
target_link_libraries(tdutils PRIVATE log)
endif()

View File

@ -64,7 +64,8 @@ class UdpReaderHelper {
}
private:
enum : size_t { MAX_PACKET_SIZE = 2048, RESERVED_SIZE = MAX_PACKET_SIZE * 8 };
static constexpr size_t MAX_PACKET_SIZE = 2048;
static constexpr size_t RESERVED_SIZE = MAX_PACKET_SIZE * 8;
UdpMessage message_;
BufferSlice buffer_;
};
@ -98,7 +99,7 @@ class UdpReader {
}
private:
enum : size_t { BUFFER_SIZE = 16 };
static constexpr size_t BUFFER_SIZE = 16;
std::array<UdpSocketFd::InboundMessage, BUFFER_SIZE> messages_;
std::array<UdpReaderHelper, BUFFER_SIZE> helpers_;
};

View File

@ -17,10 +17,10 @@ namespace td {
string HttpUrl::get_url() const {
string result;
switch (protocol_) {
case Protocol::HTTP:
case Protocol::Http:
result += "http://";
break;
case Protocol::HTTPS:
case Protocol::Https:
result += "https://";
break;
default:
@ -49,9 +49,9 @@ Result<HttpUrl> parse_url(Slice url, HttpUrl::Protocol default_protocol) {
if (parser.start_with("://")) {
parser.advance(3);
if (protocol_str == "http") {
protocol = HttpUrl::Protocol::HTTP;
protocol = HttpUrl::Protocol::Http;
} else if (protocol_str == "https") {
protocol = HttpUrl::Protocol::HTTPS;
protocol = HttpUrl::Protocol::Https;
} else {
return Status::Error("Unsupported URL protocol");
}
@ -99,10 +99,10 @@ Result<HttpUrl> parse_url(Slice url, HttpUrl::Protocol default_protocol) {
int specified_port = port;
if (port == 0) {
if (protocol == HttpUrl::Protocol::HTTP) {
if (protocol == HttpUrl::Protocol::Http) {
port = 80;
} else {
CHECK(protocol == HttpUrl::Protocol::HTTPS);
CHECK(protocol == HttpUrl::Protocol::Https);
port = 443;
}
}
@ -169,7 +169,7 @@ Result<HttpUrl> parse_url(Slice url, HttpUrl::Protocol default_protocol) {
}
StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url) {
sb << tag("protocol", url.protocol_ == HttpUrl::Protocol::HTTP ? "HTTP" : "HTTPS") << tag("userinfo", url.userinfo_)
sb << tag("protocol", url.protocol_ == HttpUrl::Protocol::Http ? "HTTP" : "HTTPS") << tag("userinfo", url.userinfo_)
<< tag("host", url.host_) << tag("port", url.port_) << tag("query", url.query_);
return sb;
}

View File

@ -15,7 +15,7 @@ namespace td {
class HttpUrl {
public:
enum class Protocol { HTTP, HTTPS } protocol_ = Protocol::HTTP;
enum class Protocol { Http, Https } protocol_ = Protocol::Http;
string userinfo_;
string host_;
bool is_ipv6_ = false;
@ -37,7 +37,7 @@ class HttpUrl {
};
Result<HttpUrl> parse_url(Slice url,
HttpUrl::Protocol default_protocol = HttpUrl::Protocol::HTTP) TD_WARN_UNUSED_RESULT;
HttpUrl::Protocol default_protocol = HttpUrl::Protocol::Http) TD_WARN_UNUSED_RESULT;
StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url);

View File

@ -0,0 +1,183 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/OptionParser.h"
#include "td/utils/misc.h"
#include <cstring>
#include <unordered_map>
namespace td {
void OptionParser::set_description(string description) {
description_ = std::move(description);
}
void OptionParser::add_option(Option::Type type, char short_key, Slice long_key, Slice description,
std::function<Status(Slice)> callback) {
options_.push_back(Option{type, short_key, long_key.str(), description.str(), std::move(callback)});
}
void OptionParser::add_option(char short_key, Slice long_key, Slice description,
std::function<Status(Slice)> callback) {
add_option(Option::Type::Arg, short_key, long_key, description, std::move(callback));
}
void OptionParser::add_option(char short_key, Slice long_key, Slice description, std::function<Status(void)> callback) {
// Ouch. There must be some better way
add_option(Option::Type::NoArg, short_key, long_key, description,
std::bind([](std::function<Status(void)> &func, Slice) { return func(); }, std::move(callback),
std::placeholders::_1));
}
Result<vector<char *>> OptionParser::run(int argc, char *argv[]) {
std::unordered_map<char, const Option *> short_options;
std::unordered_map<string, const Option *> long_options;
for (auto &opt : options_) {
if (opt.short_key != '\0') {
short_options[opt.short_key] = &opt;
}
if (!opt.long_key.empty()) {
long_options[opt.long_key] = &opt;
}
}
vector<char *> non_options;
for (int arg_pos = 1; arg_pos < argc; arg_pos++) {
const char *arg = argv[arg_pos];
if (arg[0] != '-' || arg[1] == '\0') {
non_options.push_back(argv[arg_pos]);
continue;
}
if (arg[1] == '-' && arg[2] == '\0') {
// "--"; after it everything is non-option
while (++arg_pos < argc) {
non_options.push_back(argv[arg_pos]);
}
break;
}
if (arg[1] == '-') {
// long option
Slice long_arg(arg + 2, std::strlen(arg + 2));
Slice param;
auto equal_pos = long_arg.find('=');
bool has_equal = equal_pos != Slice::npos;
if (has_equal) {
param = long_arg.substr(equal_pos + 1);
long_arg = long_arg.substr(0, equal_pos);
}
auto it = long_options.find(long_arg.str());
if (it == long_options.end()) {
return Status::Error(PSLICE() << "Option " << long_arg << " was unrecognized");
}
auto option = it->second;
switch (option->type) {
case Option::Type::NoArg:
if (has_equal) {
return Status::Error(PSLICE() << "Option " << long_arg << " must not have argument");
}
break;
case Option::Type::Arg:
if (!has_equal) {
if (++arg_pos == argc) {
return Status::Error(PSLICE() << "Option " << long_arg << " must have argument");
}
param = Slice(argv[arg_pos], std::strlen(argv[arg_pos]));
}
break;
default:
UNREACHABLE();
}
TRY_STATUS(option->arg_callback(param));
continue;
}
for (size_t opt_pos = 1; arg[opt_pos] != '\0'; opt_pos++) {
auto it = short_options.find(arg[opt_pos]);
if (it == short_options.end()) {
return Status::Error(PSLICE() << "Option " << arg[opt_pos] << " was unrecognized");
}
auto option = it->second;
Slice param;
switch (option->type) {
case Option::Type::NoArg:
// nothing to do
break;
case Option::Type::Arg:
if (arg[opt_pos + 1] == '\0') {
if (++arg_pos == argc) {
return Status::Error(PSLICE() << "Option " << arg[opt_pos] << " must have argument");
}
param = Slice(argv[arg_pos], std::strlen(argv[arg_pos]));
} else {
param = Slice(arg + opt_pos + 1, std::strlen(arg + opt_pos + 1));
opt_pos += param.size();
}
break;
default:
UNREACHABLE();
}
TRY_STATUS(option->arg_callback(param));
}
}
return std::move(non_options);
}
StringBuilder &operator<<(StringBuilder &sb, const OptionParser &o) {
if (!o.description_.empty()) {
sb << o.description_ << ". ";
}
sb << "Options:\n";
size_t max_length = 0;
for (auto &opt : o.options_) {
bool has_short_key = opt.short_key != '\0';
bool has_long_key = !opt.long_key.empty();
size_t length = (has_short_key ? 2 : 0) + (has_long_key ? 2 + opt.long_key.size() + 2 * has_short_key : 0);
if (opt.type != OptionParser::Option::Type::NoArg) {
length += 5;
}
if (length > max_length) {
max_length = length;
}
}
max_length++;
for (auto &opt : o.options_) {
bool has_short_key = opt.short_key != '\0';
sb << " ";
size_t length = max_length;
if (has_short_key) {
sb << '-' << opt.short_key;
length -= 2;
}
if (!opt.long_key.empty()) {
if (has_short_key) {
sb << ", ";
length -= 2;
}
sb << "--" << opt.long_key;
length -= 2 + opt.long_key.size();
}
if (opt.type != OptionParser::Option::Type::NoArg) {
sb << "<arg>";
length -= 5;
}
sb << string(length, ' ') << opt.description;
sb << '\n';
}
return sb;
}
} // namespace td

View File

@ -15,34 +15,35 @@
namespace td {
class OptionsParser {
class OptionParser {
class Option {
public:
enum class Type { NoArg, Arg, OptionalArg };
enum class Type { NoArg, Arg };
Type type;
char short_key;
std::string long_key;
std::string description;
string long_key;
string description;
std::function<Status(Slice)> arg_callback;
};
public:
void set_description(std::string description);
void add_option(Option::Type type, char short_key, Slice long_key, Slice description,
std::function<Status(Slice)> callback);
public:
void set_description(string description);
void add_option(char short_key, Slice long_key, Slice description, std::function<Status(Slice)> callback);
void add_option(char short_key, Slice long_key, Slice description, std::function<Status(void)> callback);
Result<int> run(int argc, char *argv[]) TD_WARN_UNUSED_RESULT;
// returns found non-option parameters
Result<vector<char *>> run(int argc, char *argv[]) TD_WARN_UNUSED_RESULT;
friend StringBuilder &operator<<(StringBuilder &sb, const OptionsParser &o);
friend StringBuilder &operator<<(StringBuilder &sb, const OptionParser &o);
private:
std::vector<Option> options_;
std::string description_;
vector<Option> options_;
string description_;
};
} // namespace td

View File

@ -1,130 +0,0 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/OptionsParser.h"
#if TD_HAVE_GETOPT
#include "getopt.h"
#endif
#if !TD_WINDOWS
#include <getopt.h>
#include <unistd.h>
#endif
namespace td {
void OptionsParser::set_description(std::string description) {
description_ = std::move(description);
}
void OptionsParser::add_option(Option::Type type, char short_key, Slice long_key, Slice description,
std::function<Status(Slice)> callback) {
options_.push_back(Option{type, short_key, long_key.str(), description.str(), std::move(callback)});
}
void OptionsParser::add_option(char short_key, Slice long_key, Slice description,
std::function<Status(Slice)> callback) {
add_option(Option::Type::Arg, short_key, long_key, description, std::move(callback));
}
void OptionsParser::add_option(char short_key, Slice long_key, Slice description,
std::function<Status(void)> callback) {
// Ouch. There must be some better way
add_option(Option::Type::NoArg, short_key, long_key, description,
std::bind([](std::function<Status(void)> &func, Slice) { return func(); }, std::move(callback),
std::placeholders::_1));
}
Result<int> OptionsParser::run(int argc, char *argv[]) {
#if TD_HAVE_GETOPT
char buff[1024];
StringBuilder sb(MutableSlice{buff, sizeof(buff)});
for (auto &opt : options_) {
CHECK(opt.type != Option::Type::OptionalArg);
sb << opt.short_key;
if (opt.type == Option::Type::Arg) {
sb << ":";
}
}
if (sb.is_error()) {
return Status::Error("Can't parse options");
}
CSlice short_options = sb.as_cslice();
vector<option> long_options;
for (auto &opt : options_) {
if (opt.long_key.empty()) {
continue;
}
option o;
o.flag = nullptr;
o.val = opt.short_key;
o.has_arg = opt.type == Option::Type::Arg ? required_argument : no_argument;
o.name = opt.long_key.c_str();
long_options.push_back(o);
}
long_options.push_back({nullptr, 0, nullptr, 0});
while (true) {
int opt_i = getopt_long(argc, argv, short_options.c_str(), &long_options[0], nullptr);
if (opt_i == ':') {
return Status::Error("Missing argument");
}
if (opt_i == '?') {
return Status::Error("Unrecognized option");
}
if (opt_i == -1) {
break;
}
bool found = false;
for (auto &opt : options_) {
if (opt.short_key == opt_i) {
Slice arg;
if (opt.type == Option::Type::Arg) {
arg = Slice(optarg);
}
auto status = opt.arg_callback(arg);
if (status.is_error()) {
return std::move(status);
}
found = true;
break;
}
}
if (!found) {
return Status::Error("Unknown argument");
}
}
return optind;
#else
return -1;
#endif
}
StringBuilder &operator<<(StringBuilder &sb, const OptionsParser &o) {
sb << o.description_ << "\n";
for (auto &opt : o.options_) {
sb << "-" << opt.short_key;
if (!opt.long_key.empty()) {
sb << "|--" << opt.long_key;
}
if (opt.type == OptionsParser::Option::Type::OptionalArg) {
sb << "[";
}
if (opt.type != OptionsParser::Option::Type::NoArg) {
sb << "<arg>";
}
if (opt.type == OptionsParser::Option::Type::OptionalArg) {
sb << "]";
}
sb << "\t" << opt.description;
sb << "\n";
}
return sb;
}
} // namespace td

View File

@ -160,8 +160,23 @@ uint64 Random::Xorshift128plus::operator()() {
seed_[1] = x ^ y ^ (x >> 17) ^ (y >> 26);
return seed_[1] + y;
}
int Random::Xorshift128plus::fast(int min, int max) {
return static_cast<int>((*this)() % (max - min + 1) + min);
}
void Random::Xorshift128plus::bytes(MutableSlice dest) {
int cnt = 0;
uint64 buf = 0;
for (auto &c : dest) {
if (cnt == 0) {
buf = operator()();
cnt = 8;
}
cnt--;
c = static_cast<char>(buf & 255);
buf >>= 8;
}
}
} // namespace td

View File

@ -37,6 +37,7 @@ class Random {
Xorshift128plus(uint64 seed_a, uint64 seed_b);
uint64 operator()();
int fast(int min, int max);
void bytes(MutableSlice dest);
private:
uint64 seed_[2];

View File

@ -104,7 +104,7 @@ string winerror_to_string(int code);
#endif
class Status {
enum class ErrorType : int8 { general, os };
enum class ErrorType : int8 { General, Os };
public:
Status() = default;
@ -129,7 +129,7 @@ class Status {
}
static Status Error(int err, Slice message = Slice()) TD_WARN_UNUSED_RESULT {
return Status(false, ErrorType::general, err, message);
return Status(false, ErrorType::General, err, message);
}
static Status Error(Slice message) TD_WARN_UNUSED_RESULT {
@ -138,13 +138,13 @@ class Status {
#if TD_PORT_WINDOWS
static Status WindowsError(int saved_error, Slice message) TD_WARN_UNUSED_RESULT {
return Status(false, ErrorType::os, saved_error, message);
return Status(false, ErrorType::Os, saved_error, message);
}
#endif
#if TD_PORT_POSIX
static Status PosixError(int32 saved_errno, Slice message) TD_WARN_UNUSED_RESULT {
return Status(false, ErrorType::os, saved_errno, message);
return Status(false, ErrorType::Os, saved_errno, message);
}
#endif
@ -154,7 +154,7 @@ class Status {
template <int Code>
static Status Error() {
static Status status(true, ErrorType::general, Code, "");
static Status status(true, ErrorType::General, Code, "");
return status.clone_static();
}
@ -164,10 +164,10 @@ class Status {
}
Info info = get_info();
switch (info.error_type) {
case ErrorType::general:
case ErrorType::General:
sb << "[Error";
break;
case ErrorType::os:
case ErrorType::Os:
#if TD_PORT_POSIX
sb << "[PosixError : " << strerror_safe(info.error_code);
#elif TD_PORT_WINDOWS
@ -246,9 +246,9 @@ class Status {
}
Info info = get_info();
switch (info.error_type) {
case ErrorType::general:
case ErrorType::General:
return message().str();
case ErrorType::os:
case ErrorType::Os:
#if TD_PORT_POSIX
return strerror_safe(info.error_code).str();
#elif TD_PORT_WINDOWS
@ -276,10 +276,10 @@ class Status {
CHECK(is_error());
Info info = get_info();
switch (info.error_type) {
case ErrorType::general:
case ErrorType::General:
return Error(code(), PSLICE() << prefix << message());
case ErrorType::os:
return Status(false, ErrorType::os, code(), PSLICE() << prefix << message());
case ErrorType::Os:
return Status(false, ErrorType::Os, code(), PSLICE() << prefix << message());
default:
UNREACHABLE();
return {};

View File

@ -22,14 +22,14 @@ namespace td {
StringBuilder::StringBuilder(MutableSlice slice, bool use_buffer)
: begin_ptr_(slice.begin()), current_ptr_(begin_ptr_), use_buffer_(use_buffer) {
if (slice.size() <= reserved_size) {
auto buffer_size = reserved_size + 100;
if (slice.size() <= RESERVED_SIZE) {
auto buffer_size = RESERVED_SIZE + 100;
buffer_ = std::make_unique<char[]>(buffer_size);
begin_ptr_ = buffer_.get();
current_ptr_ = begin_ptr_;
end_ptr_ = begin_ptr_ + buffer_size - reserved_size;
end_ptr_ = begin_ptr_ + buffer_size - RESERVED_SIZE;
} else {
end_ptr_ = slice.end() - reserved_size;
end_ptr_ = slice.end() - RESERVED_SIZE;
}
}
@ -39,7 +39,7 @@ StringBuilder &StringBuilder::operator<<(Slice slice) {
if (end_ptr_ < current_ptr_) {
return on_error();
}
auto available_size = static_cast<size_t>(end_ptr_ + reserved_size - 1 - current_ptr_);
auto available_size = static_cast<size_t>(end_ptr_ + RESERVED_SIZE - 1 - current_ptr_);
if (size > available_size) {
error_flag_ = true;
size = available_size;
@ -101,12 +101,12 @@ bool StringBuilder::reserve_inner(size_t size) {
}
size_t old_data_size = current_ptr_ - begin_ptr_;
if (size >= std::numeric_limits<size_t>::max() - reserved_size - old_data_size - 1) {
if (size >= std::numeric_limits<size_t>::max() - RESERVED_SIZE - old_data_size - 1) {
return false;
}
size_t need_data_size = old_data_size + size;
size_t old_buffer_size = end_ptr_ - begin_ptr_;
if (old_buffer_size >= (std::numeric_limits<size_t>::max() - reserved_size) / 2 - 2) {
if (old_buffer_size >= (std::numeric_limits<size_t>::max() - RESERVED_SIZE) / 2 - 2) {
return false;
}
size_t new_buffer_size = (old_buffer_size + 1) * 2;
@ -116,13 +116,13 @@ bool StringBuilder::reserve_inner(size_t size) {
if (new_buffer_size < 100) {
new_buffer_size = 100;
}
new_buffer_size += reserved_size;
new_buffer_size += RESERVED_SIZE;
auto new_buffer = std::make_unique<char[]>(new_buffer_size);
std::memcpy(new_buffer.get(), begin_ptr_, old_data_size);
buffer_ = std::move(new_buffer);
begin_ptr_ = buffer_.get();
current_ptr_ = begin_ptr_ + old_data_size;
end_ptr_ = begin_ptr_ + new_buffer_size - reserved_size;
end_ptr_ = begin_ptr_ + new_buffer_size - RESERVED_SIZE;
CHECK(end_ptr_ > current_ptr_);
CHECK(static_cast<size_t>(end_ptr_ - current_ptr_) >= size);
return true;
@ -193,7 +193,7 @@ StringBuilder &StringBuilder::operator<<(FixedDouble x) {
*ss << x.d;
int len = narrow_cast<int>(static_cast<std::streamoff>(ss->tellp()));
auto left = end_ptr_ + reserved_size - current_ptr_;
auto left = end_ptr_ + RESERVED_SIZE - current_ptr_;
if (unlikely(len >= left)) {
error_flag_ = true;
len = left ? narrow_cast<int>(left - 1) : 0;
@ -207,7 +207,7 @@ StringBuilder &StringBuilder::operator<<(const void *ptr) {
if (unlikely(!reserve())) {
return on_error();
}
current_ptr_ += std::snprintf(current_ptr_, reserved_size, "%p", ptr);
current_ptr_ += std::snprintf(current_ptr_, RESERVED_SIZE, "%p", ptr);
return *this;
}

View File

@ -26,7 +26,7 @@ class StringBuilder {
}
MutableCSlice as_cslice() {
if (current_ptr_ >= end_ptr_ + reserved_size) {
if (current_ptr_ >= end_ptr_ + RESERVED_SIZE) {
std::abort(); // shouldn't happen
}
*current_ptr_ = 0;
@ -104,7 +104,7 @@ class StringBuilder {
bool error_flag_ = false;
bool use_buffer_ = false;
std::unique_ptr<char[]> buffer_;
static constexpr size_t reserved_size = 30;
static constexpr size_t RESERVED_SIZE = 30;
StringBuilder &on_error() {
error_flag_ = true;
@ -115,7 +115,7 @@ class StringBuilder {
if (end_ptr_ > current_ptr_) {
return true;
}
return reserve_inner(reserved_size);
return reserve_inner(RESERVED_SIZE);
}
bool reserve(size_t size) {
if (end_ptr_ > current_ptr_ && static_cast<size_t>(end_ptr_ - current_ptr_) >= size) {

View File

@ -47,6 +47,15 @@ inline uint64 lower_bit64(uint64 x) {
return x & bits_negate64(x);
}
inline uint64 host_to_big_endian64(uint64 x) {
// NB: works only for little-endian systems
return bswap64(x);
}
inline uint64 big_endian_to_host64(uint64 x) {
// NB: works only for little-endian systems
return bswap64(x);
}
//TODO: optimize
inline int32 count_leading_zeroes_non_zero32(uint32 x) {
DCHECK(x != 0);

View File

@ -5,5 +5,4 @@
#cmakedefine01 TD_HAVE_CRC32C
#cmakedefine01 TD_HAVE_COROUTINES
#cmakedefine01 TD_HAVE_ABSL
#cmakedefine01 TD_HAVE_GETOPT
#cmakedefine01 TD_FD_DEBUG

View File

@ -8,6 +8,7 @@
#include "td/utils/as.h"
#include "td/utils/BigNum.h"
#include "td/utils/bits.h"
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/misc.h"
@ -15,6 +16,7 @@
#include "td/utils/port/thread_local.h"
#include "td/utils/Random.h"
#include "td/utils/ScopeGuard.h"
#include "td/utils/SharedSlice.h"
#include "td/utils/StackAllocator.h"
#include "td/utils/StringBuilder.h"
@ -249,6 +251,220 @@ int pq_factorize(Slice pq_str, string *p_str, string *q_str) {
return 0;
}
struct AesBlock {
uint64 hi;
uint64 lo;
uint8 *raw() {
return reinterpret_cast<uint8 *>(this);
}
const uint8 *raw() const {
return reinterpret_cast<const uint8 *>(this);
}
Slice as_slice() const {
return Slice(raw(), AES_BLOCK_SIZE);
}
AesBlock operator^(const AesBlock &b) const {
AesBlock res;
res.hi = hi ^ b.hi;
res.lo = lo ^ b.lo;
return res;
}
void operator^=(const AesBlock &b) {
hi ^= b.hi;
lo ^= b.lo;
}
void load(const uint8 *from) {
*this = as<AesBlock>(from);
}
void store(uint8 *to) {
as<AesBlock>(to) = *this;
}
AesBlock inc() const {
#if SIZE_MAX == UINT64_MAX
AesBlock res;
res.lo = host_to_big_endian64(big_endian_to_host64(lo) + 1);
if (res.lo == 0) {
res.hi = host_to_big_endian64(big_endian_to_host64(hi) + 1);
} else {
res.hi = hi;
}
return res;
#else
AesBlock res = *this;
auto ptr = res.raw();
if (++ptr[15] == 0) {
for (int i = 14; i >= 0; i--) {
if (++ptr[i] != 0) {
break;
}
}
}
return res;
#endif
}
};
static_assert(sizeof(AesBlock) == 16, "");
static_assert(sizeof(AesBlock) == AES_BLOCK_SIZE, "");
class XorBytes {
public:
static void run(const uint8 *a, const uint8 *b, uint8 *c, size_t n) {
static constexpr int BLOCK_SIZE = 16;
auto block_cnt = n / BLOCK_SIZE;
n -= block_cnt * BLOCK_SIZE;
while (block_cnt-- > 0) {
Block<BLOCK_SIZE> a_big = as<Block<BLOCK_SIZE>>(a);
Block<BLOCK_SIZE> b_big = as<Block<BLOCK_SIZE>>(b);
as<Block<BLOCK_SIZE>>(c) = a_big ^ b_big;
a += BLOCK_SIZE;
b += BLOCK_SIZE;
c += BLOCK_SIZE;
}
while (n-- > 0) {
c[n] = a[n] ^ b[n];
}
}
private:
template <size_t N>
struct alignas(N) Block {
uint8 data[N];
Block operator^(const Block &b) const & {
Block res;
for (size_t i = 0; i < N; i++) {
res.data[i] = data[i] ^ b.data[i];
}
return res;
}
};
};
struct AesCtrCounterPack {
static constexpr size_t BLOCK_COUNT = 32;
AesBlock blocks[BLOCK_COUNT];
uint8 *raw() {
return reinterpret_cast<uint8 *>(this);
}
const uint8 *raw() const {
return reinterpret_cast<const uint8 *>(this);
}
static size_t size() {
return sizeof(blocks);
}
void init(AesBlock block) {
blocks[0] = block;
for (size_t i = 1; i < BLOCK_COUNT; i++) {
blocks[i] = blocks[i - 1].inc();
}
}
};
class Evp {
public:
Evp() {
ctx_ = EVP_CIPHER_CTX_new();
LOG_IF(FATAL, ctx_ == nullptr);
}
Evp(const Evp &from) = delete;
Evp &operator=(const Evp &from) = delete;
Evp(Evp &&from) = delete;
Evp &operator=(Evp &&from) = delete;
~Evp() {
CHECK(ctx_ != nullptr);
EVP_CIPHER_CTX_free(ctx_);
}
void init_encrypt_ecb(Slice key) {
init(Type::Ecb, true, EVP_aes_256_ecb(), key);
}
void init_decrypt_ecb(Slice key) {
init(Type::Ecb, false, EVP_aes_256_ecb(), key);
}
void init_encrypt_cbc(Slice key) {
init(Type::Cbc, true, EVP_aes_256_cbc(), key);
}
void init_decrypt_cbc(Slice key) {
init(Type::Cbc, false, EVP_aes_256_cbc(), key);
}
void init_iv(Slice iv) {
int res = EVP_CipherInit_ex(ctx_, nullptr, nullptr, nullptr, iv.ubegin(), -1);
LOG_IF(FATAL, res != 1);
}
void encrypt(const uint8 *src, uint8 *dst, int size) {
// CHECK(type_ != Type::Empty && is_encrypt_);
CHECK(size % AES_BLOCK_SIZE == 0);
int len;
int res = EVP_EncryptUpdate(ctx_, dst, &len, src, size);
LOG_IF(FATAL, res != 1);
CHECK(len == size);
}
void decrypt(const uint8 *src, uint8 *dst, int size) {
// CHECK(type_ != Type::Empty && !is_encrypt_);
CHECK(size % AES_BLOCK_SIZE == 0);
int len;
int res = EVP_DecryptUpdate(ctx_, dst, &len, src, size);
LOG_IF(FATAL, res != 1);
CHECK(len == size);
}
private:
EVP_CIPHER_CTX *ctx_{nullptr};
enum class Type : int8 { Empty, Ecb, Cbc };
// Type type_{Type::Empty};
// bool is_encrypt_ = false;
void init(Type type, bool is_encrypt, const EVP_CIPHER *cipher, Slice key) {
// type_ = type;
// is_encrypt_ = is_encrypt;
int res = EVP_CipherInit_ex(ctx_, cipher, nullptr, key.ubegin(), nullptr, is_encrypt ? 1 : 0);
LOG_IF(FATAL, res != 1);
EVP_CIPHER_CTX_set_padding(ctx_, 0);
}
};
struct AesState::Impl {
Evp evp;
};
AesState::AesState() = default;
AesState::AesState(AesState &&from) = default;
AesState &AesState::operator=(AesState &&from) = default;
AesState::~AesState() = default;
void AesState::init(Slice key, bool encrypt) {
CHECK(key.size() == 32);
if (!impl_) {
impl_ = make_unique<Impl>();
}
if (encrypt) {
impl_->evp.init_encrypt_ecb(key);
} else {
impl_->evp.init_decrypt_ecb(key);
}
}
void AesState::encrypt(const uint8 *src, uint8 *dst, int size) {
CHECK(impl_);
impl_->evp.encrypt(src, dst, size);
}
void AesState::decrypt(const uint8 *src, uint8 *dst, int size) {
CHECK(impl_);
impl_->evp.decrypt(src, dst, size);
}
static void aes_ige_xcrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to, bool encrypt_flag) {
CHECK(aes_key.size() == 32);
CHECK(aes_iv.size() == 32);
@ -264,12 +480,128 @@ static void aes_ige_xcrypt(Slice aes_key, MutableSlice aes_iv, Slice from, Mutab
AES_ige_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv.ubegin(), encrypt_flag);
}
class AesIgeStateImpl {
public:
void init(Slice key, Slice iv, bool encrypt) {
CHECK(key.size() == 32);
CHECK(iv.size() == 32);
if (encrypt) {
evp_.init_encrypt_cbc(key);
} else {
evp_.init_decrypt_ecb(key);
}
encrypted_iv_.load(iv.ubegin());
plaintext_iv_.load(iv.ubegin() + AES_BLOCK_SIZE);
}
void encrypt(Slice from, MutableSlice to) {
CHECK(from.size() % AES_BLOCK_SIZE == 0);
CHECK(to.size() >= from.size());
auto len = to.size() / AES_BLOCK_SIZE;
auto in = from.ubegin();
auto out = to.ubegin();
static constexpr size_t BLOCK_COUNT = 31;
while (len != 0) {
AesBlock data[BLOCK_COUNT];
AesBlock data_xored[BLOCK_COUNT];
auto count = td::min(BLOCK_COUNT, len);
std::memcpy(data, in, AES_BLOCK_SIZE * count);
data_xored[0] = data[0];
if (count > 1) {
data_xored[1] = plaintext_iv_ ^ data[1];
for (size_t i = 2; i < count; i++) {
data_xored[i] = data[i - 2] ^ data[i];
}
}
evp_.init_iv(encrypted_iv_.as_slice());
int inlen = static_cast<int>(AES_BLOCK_SIZE * count);
evp_.encrypt(data_xored[0].raw(), data_xored[0].raw(), inlen);
data_xored[0] ^= plaintext_iv_;
for (size_t i = 1; i < count; i++) {
data_xored[i] ^= data[i - 1];
}
plaintext_iv_ = data[count - 1];
encrypted_iv_ = data_xored[count - 1];
std::memcpy(out, data_xored, AES_BLOCK_SIZE * count);
len -= count;
in += AES_BLOCK_SIZE * count;
out += AES_BLOCK_SIZE * count;
}
}
void decrypt(Slice from, MutableSlice to) {
CHECK(from.size() % AES_BLOCK_SIZE == 0);
CHECK(to.size() >= from.size());
auto len = to.size() / AES_BLOCK_SIZE;
auto in = from.ubegin();
auto out = to.ubegin();
AesBlock encrypted;
while (len) {
encrypted.load(in);
plaintext_iv_ ^= encrypted;
evp_.decrypt(plaintext_iv_.raw(), plaintext_iv_.raw(), AES_BLOCK_SIZE);
plaintext_iv_ ^= encrypted_iv_;
plaintext_iv_.store(out);
encrypted_iv_ = encrypted;
--len;
in += AES_BLOCK_SIZE;
out += AES_BLOCK_SIZE;
}
}
private:
Evp evp_;
AesBlock encrypted_iv_;
AesBlock plaintext_iv_;
};
AesIgeState::AesIgeState() = default;
AesIgeState::AesIgeState(AesIgeState &&from) = default;
AesIgeState &AesIgeState::operator=(AesIgeState &&from) = default;
AesIgeState::~AesIgeState() = default;
void AesIgeState::init(Slice key, Slice iv, bool encrypt) {
if (!impl_) {
impl_ = make_unique<AesIgeStateImpl>();
}
impl_->init(key, iv, encrypt);
}
void AesIgeState::encrypt(Slice from, MutableSlice to) {
impl_->encrypt(from, to);
}
void AesIgeState::decrypt(Slice from, MutableSlice to) {
impl_->decrypt(from, to);
}
void aes_ige_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) {
aes_ige_xcrypt(aes_key, aes_iv, from, to, true);
if (from.size() <= 128) {
return aes_ige_xcrypt(aes_key, aes_iv, from, to, true);
}
AesIgeStateImpl state;
state.init(aes_key, aes_iv, true);
state.encrypt(from, to);
}
void aes_ige_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) {
aes_ige_xcrypt(aes_key, aes_iv, from, to, false);
if (from.size() <= 128) {
return aes_ige_xcrypt(aes_key, aes_iv, from, to, false);
}
AesIgeStateImpl state;
state.init(aes_key, aes_iv, false);
state.decrypt(from, to);
}
static void aes_cbc_xcrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to, bool encrypt_flag) {
@ -313,35 +645,43 @@ class AesCtrState::Impl {
CHECK(key.size() == 32);
CHECK(iv.size() == 16);
static_assert(AES_BLOCK_SIZE == 16, "");
if (AES_set_encrypt_key(key.ubegin(), 256, &aes_key) < 0) {
LOG(FATAL) << "Failed to set encrypt key";
}
counter.as_mutable_slice().copy_from(iv);
current_pos = 0;
evp_.init_encrypt_ecb(key);
counter_.load(iv.ubegin());
fill();
}
void encrypt(Slice from, MutableSlice to) {
CHECK(to.size() >= from.size());
for (size_t i = 0; i < from.size(); i++) {
if (current_pos == 0) {
AES_encrypt(counter.as_slice().ubegin(), encrypted_counter.as_mutable_slice().ubegin(), &aes_key);
uint8 *ptr = counter.as_mutable_slice().ubegin();
for (int j = 15; j >= 0; j--) {
if (++ptr[j] != 0) {
break;
}
}
auto *src = from.ubegin();
auto *dst = to.ubegin();
auto n = from.size();
while (n != 0) {
size_t left = encrypted_counter_.raw() + AesCtrCounterPack::size() - current_;
if (left == 0) {
fill();
left = AesCtrCounterPack::size();
}
to[i] = static_cast<char>(from[i] ^ encrypted_counter[current_pos]);
current_pos = (current_pos + 1) & 15;
size_t min_n = td::min(n, left);
XorBytes::run(src, current_, dst, min_n);
src += min_n;
dst += min_n;
n -= min_n;
current_ += min_n;
}
}
private:
AES_KEY aes_key;
SecureString counter{AES_BLOCK_SIZE};
SecureString encrypted_counter{AES_BLOCK_SIZE};
uint8 current_pos;
Evp evp_;
uint8 *current_;
AesBlock counter_;
AesCtrCounterPack encrypted_counter_;
void fill() {
encrypted_counter_.init(counter_);
counter_ = encrypted_counter_.blocks[AesCtrCounterPack::BLOCK_COUNT - 1].inc();
current_ = encrypted_counter_.raw();
evp_.encrypt(current_, current_, static_cast<int>(AesCtrCounterPack::size()));
}
};
AesCtrState::AesCtrState() = default;

View File

@ -21,9 +21,50 @@ void init_crypto();
int pq_factorize(Slice pq_str, string *p_str, string *q_str);
class AesState {
public:
AesState();
AesState(const AesState &from) = delete;
AesState &operator=(const AesState &from) = delete;
AesState(AesState &&from);
AesState &operator=(AesState &&from);
~AesState();
void init(Slice key, bool encrypt);
void encrypt(const uint8 *src, uint8 *dst, int size);
void decrypt(const uint8 *src, uint8 *dst, int size);
private:
struct Impl;
unique_ptr<Impl> impl_;
};
void aes_ige_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
void aes_ige_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
class AesIgeStateImpl;
class AesIgeState {
public:
AesIgeState();
AesIgeState(const AesIgeState &from) = delete;
AesIgeState &operator=(const AesIgeState &from) = delete;
AesIgeState(AesIgeState &&from);
AesIgeState &operator=(AesIgeState &&from);
~AesIgeState();
void init(Slice key, Slice iv, bool encrypt);
void encrypt(Slice from, MutableSlice to);
void decrypt(Slice from, MutableSlice to);
private:
unique_ptr<AesIgeStateImpl> impl_;
};
void aes_cbc_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
void aes_cbc_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);

View File

@ -45,6 +45,9 @@ TD_THREAD_LOCAL const char *Logger::tag2_ = nullptr;
Logger::Logger(LogInterface &log, const LogOptions &options, int log_level, Slice file_name, int line_num,
Slice comment)
: Logger(log, options, log_level) {
if (log_level == VERBOSITY_NAME(PLAIN) && &options == &log_options) {
return;
}
if (!options_.add_info) {
return;
}

View File

@ -15,6 +15,7 @@
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/port/detail/PollableFd.h"
#include "td/utils/port/detail/skip_eintr.h"
#include "td/utils/port/PollFlags.h"
#include "td/utils/port/sleep.h"
#include "td/utils/ScopeGuard.h"

View File

@ -10,6 +10,7 @@
#include "td/utils/common.h"
#include "td/utils/logging.h"
#include "td/utils/port/detail/skip_eintr.h"
#include "td/utils/port/IPAddress.h"
#include "td/utils/port/PollFlags.h"

View File

@ -10,6 +10,7 @@
#include "td/utils/format.h"
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/port/detail/skip_eintr.h"
#include "td/utils/port/PollFlags.h"
#if TD_PORT_WINDOWS

View File

@ -6,7 +6,6 @@
//
#include "td/utils/port/Stat.h"
#include "td/utils/port/detail/PollableFd.h"
#include "td/utils/port/FileFd.h"
#if TD_PORT_POSIX
@ -15,6 +14,7 @@
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/port/Clocks.h"
#include "td/utils/port/detail/skip_eintr.h"
#include "td/utils/ScopeGuard.h"
#include <utility>
@ -38,8 +38,20 @@
#include <sys/syscall.h>
#endif
#elif TD_PORT_WINDOWS
#include "td/utils/port/thread.h"
#ifndef PSAPI_VERSION
#define PSAPI_VERSION 1
#endif
#include <psapi.h>
#endif
namespace td {
#if TD_PORT_POSIX
namespace detail {
template <class...>
@ -176,14 +188,20 @@ Status update_atime(CSlice path) {
};
return detail::update_atime(file.get_native_fd().fd());
}
#endif
Result<Stat> stat(CSlice path) {
#if TD_PORT_POSIX
struct ::stat buf;
int err = detail::skip_eintr([&] { return ::stat(path.c_str(), &buf); });
if (err < 0) {
return OS_ERROR(PSLICE() << "Stat for file \"" << path << "\" failed");
}
return detail::from_native_stat(buf);
#elif TD_PORT_WINDOWS
TRY_RESULT(fd, FileFd::open(path, FileFd::Flags::Read | FileFd::PrivateFlags::WinStat));
return fd.stat();
#endif
}
Result<MemStat> mem_stat() {
@ -260,6 +278,19 @@ Result<MemStat> mem_stat() {
s++;
}
return res;
#elif TD_WINDOWS
PROCESS_MEMORY_COUNTERS_EX counters;
if (!GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PROCESS_MEMORY_COUNTERS *>(&counters),
sizeof(counters))) {
return Status::Error("Call to GetProcessMemoryInfo failed");
}
MemStat res;
res.resident_size_ = counters.WorkingSetSize;
res.resident_size_peak_ = counters.PeakWorkingSetSize;
res.virtual_size_ = counters.PrivateUsage;
res.virtual_size_peak_ = counters.PeakPagefileUsage;
return res;
#else
return Status::Error("Not supported");
@ -276,7 +307,9 @@ Status cpu_stat_self(CpuStat &stat) {
constexpr int TMEM_SIZE = 10000;
char mem[TMEM_SIZE];
TRY_RESULT(size, fd.read(MutableSlice(mem, TMEM_SIZE - 1)));
CHECK(size < TMEM_SIZE - 1);
if (size >= TMEM_SIZE - 1) {
return Status::Error("Failed for read /proc/self/stat");
}
mem[size] = 0;
char *s = mem;
@ -285,10 +318,10 @@ Status cpu_stat_self(CpuStat &stat) {
while (pass_cnt < 15) {
if (pass_cnt == 13) {
stat.process_user_ticks = to_integer<uint64>(Slice(s, t));
stat.process_user_ticks_ = to_integer<uint64>(Slice(s, t));
}
if (pass_cnt == 14) {
stat.process_system_ticks = to_integer<uint64>(Slice(s, t));
stat.process_system_ticks_ = to_integer<uint64>(Slice(s, t));
}
while (*s && *s != ' ') {
s++;
@ -302,6 +335,7 @@ Status cpu_stat_self(CpuStat &stat) {
}
return Status::OK();
}
Status cpu_stat_total(CpuStat &stat) {
TRY_RESULT(fd, FileFd::open("/proc/stat", FileFd::Read));
SCOPE_EXIT {
@ -311,14 +345,17 @@ Status cpu_stat_total(CpuStat &stat) {
constexpr int TMEM_SIZE = 10000;
char mem[TMEM_SIZE];
TRY_RESULT(size, fd.read(MutableSlice(mem, TMEM_SIZE - 1)));
CHECK(size < TMEM_SIZE - 1);
if (size >= TMEM_SIZE - 1) {
return Status::Error("Failed for read /proc/stat");
}
mem[size] = 0;
uint64 sum = 0, cur = 0;
uint64 sum = 0;
uint64 cur = 0;
for (size_t i = 0; i < size; i++) {
int c = mem[i];
char c = mem[i];
if (c >= '0' && c <= '9') {
cur = cur * 10 + (uint64)c - '0';
cur = cur * 10 + static_cast<uint64>(c) - '0';
} else {
sum += cur;
cur = 0;
@ -328,7 +365,7 @@ Status cpu_stat_total(CpuStat &stat) {
}
}
stat.total_ticks = sum;
stat.total_ticks_ = sum;
return Status::OK();
}
#endif
@ -338,25 +375,28 @@ Result<CpuStat> cpu_stat() {
CpuStat stat;
TRY_STATUS(cpu_stat_self(stat));
TRY_STATUS(cpu_stat_total(stat));
return stat;
#elif TD_WINDOWS
CpuStat stat;
stat.total_ticks_ = static_cast<uint64>(GetTickCount64()) * 10000;
auto hardware_concurrency = thread::hardware_concurrency();
if (hardware_concurrency != 0) {
stat.total_ticks_ *= hardware_concurrency;
}
FILETIME ignored_time;
FILETIME kernel_time;
FILETIME user_time;
if (!GetProcessTimes(GetCurrentProcess(), &ignored_time, &ignored_time, &kernel_time, &user_time)) {
return Status::Error("Failed to call GetProcessTimes");
}
stat.process_system_ticks_ = kernel_time.dwLowDateTime + (static_cast<uint64>(kernel_time.dwHighDateTime) << 32);
stat.process_user_ticks_ = user_time.dwLowDateTime + (static_cast<uint64>(user_time.dwHighDateTime) << 32);
return stat;
#else
return Status::Error("Not supported");
#endif
}
} // namespace td
#endif
#if TD_PORT_WINDOWS
namespace td {
Result<Stat> stat(CSlice path) {
TRY_RESULT(fd, FileFd::open(path, FileFd::Flags::Read | FileFd::PrivateFlags::WinStat));
return fd.stat();
}
Result<CpuStat> cpu_stat() {
return Status::Error("Not supported");
}
} // namespace td
#endif

View File

@ -26,20 +26,13 @@ struct Stat {
Result<Stat> stat(CSlice path) TD_WARN_UNUSED_RESULT;
struct CpuStat {
uint64 total_ticks{0};
uint64 process_user_ticks{0};
uint64 process_system_ticks{0};
uint64 total_ticks_{0};
uint64 process_user_ticks_{0};
uint64 process_system_ticks_{0};
};
Result<CpuStat> cpu_stat() TD_WARN_UNUSED_RESULT;
#if TD_PORT_POSIX
namespace detail {
Result<Stat> fstat(int native_fd);
} // namespace detail
Status update_atime(CSlice path) TD_WARN_UNUSED_RESULT;
struct MemStat {
uint64 resident_size_ = 0;
uint64 resident_size_peak_ = 0;
@ -49,6 +42,14 @@ struct MemStat {
Result<MemStat> mem_stat() TD_WARN_UNUSED_RESULT;
#if TD_PORT_POSIX
namespace detail {
Result<Stat> fstat(int native_fd);
} // namespace detail
Status update_atime(CSlice path) TD_WARN_UNUSED_RESULT;
#endif
} // namespace td

View File

@ -10,6 +10,7 @@
#include "td/utils/format.h"
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/port/detail/skip_eintr.h"
#include "td/utils/port/PollFlags.h"
#include "td/utils/port/SocketFd.h"
#include "td/utils/VectorQueue.h"
@ -169,7 +170,8 @@ class UdpSocketFdImpl : private Iocp::Callback {
UdpMessage to_receive_;
WSAMSG receive_message_;
UdpSocketReceiveHelper receive_helper_;
enum : size_t { MAX_PACKET_SIZE = 2048, RESERVED_SIZE = MAX_PACKET_SIZE * 8 };
static constexpr size_t MAX_PACKET_SIZE = 2048;
static constexpr size_t RESERVED_SIZE = MAX_PACKET_SIZE * 8;
BufferSlice receive_buffer_;
UdpMessage to_send_;
@ -812,11 +814,11 @@ static Result<uint32> maximize_buffer(int socket_fd, int optname, uint32 max) {
}
Result<uint32> UdpSocketFd::maximize_snd_buffer(uint32 max) {
return maximize_buffer(get_native_fd().fd(), SO_SNDBUF, max == 0 ? default_udp_max_snd_buffer_size : max);
return maximize_buffer(get_native_fd().fd(), SO_SNDBUF, max == 0 ? DEFAULT_UDP_MAX_SND_BUFFER_SIZE : max);
}
Result<uint32> UdpSocketFd::maximize_rcv_buffer(uint32 max) {
return maximize_buffer(get_native_fd().fd(), SO_RCVBUF, max == 0 ? default_udp_max_rcv_buffer_size : max);
return maximize_buffer(get_native_fd().fd(), SO_RCVBUF, max == 0 ? DEFAULT_UDP_MAX_RCV_BUFFER_SIZE : max);
}
#else
Result<uint32> UdpSocketFd::maximize_snd_buffer(uint32 max) {

View File

@ -84,8 +84,8 @@ class UdpSocketFd {
#endif
private:
static constexpr uint32 default_udp_max_snd_buffer_size = (1 << 24);
static constexpr uint32 default_udp_max_rcv_buffer_size = (1 << 24);
static constexpr uint32 DEFAULT_UDP_MAX_SND_BUFFER_SIZE = (1 << 24);
static constexpr uint32 DEFAULT_UDP_MAX_RCV_BUFFER_SIZE = (1 << 24);
std::unique_ptr<detail::UdpSocketFdImpl, detail::UdpSocketFdImplDeleter> impl_;
explicit UdpSocketFd(unique_ptr<detail::UdpSocketFdImpl> impl);
};

View File

@ -13,6 +13,7 @@ char disable_linker_warning_about_empty_file_event_fd_linux_cpp TD_UNUSED;
#include "td/utils/logging.h"
#include "td/utils/misc.h"
#include "td/utils/port/detail/NativeFd.h"
#include "td/utils/port/detail/skip_eintr.h"
#include "td/utils/port/PollFlags.h"
#include "td/utils/ScopeGuard.h"
#include "td/utils/Slice.h"

View File

@ -16,9 +16,7 @@
#include "td/utils/SpinLock.h"
#include <atomic>
#include <cerrno>
#include <memory>
#include <type_traits>
namespace td {
@ -209,30 +207,6 @@ inline const NativeFd &PollableFd::native_fd() const {
return fd_info_->native_fd();
}
#if TD_PORT_POSIX
namespace detail {
template <class F>
auto skip_eintr(F &&f) {
decltype(f()) res;
static_assert(std::is_integral<decltype(res)>::value, "integral type expected");
do {
errno = 0; // just in case
res = f();
} while (res < 0 && errno == EINTR);
return res;
}
template <class F>
auto skip_eintr_cstr(F &&f) {
char *res;
do {
errno = 0; // just in case
res = f();
} while (res == nullptr && errno == EINTR);
return res;
}
} // namespace detail
#endif
template <class FdT>
bool can_read(const FdT &fd) {
return fd.get_poll_info().get_flags().can_read() || fd.get_poll_info().get_flags().has_pending_error();

View File

@ -0,0 +1,39 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include <cerrno>
#include <type_traits>
namespace td {
#if TD_PORT_POSIX
namespace detail {
template <class F>
auto skip_eintr(F &&f) {
decltype(f()) res;
static_assert(std::is_integral<decltype(res)>::value, "integral type expected");
do {
errno = 0; // just in case
res = f();
} while (res < 0 && errno == EINTR);
return res;
}
template <class F>
auto skip_eintr_cstr(F &&f) {
char *res;
do {
errno = 0; // just in case
res = f();
} while (res == nullptr && errno == EINTR);
return res;
}
} // namespace detail
#endif
} // namespace td

View File

@ -10,7 +10,7 @@
#include "td/utils/format.h"
#include "td/utils/logging.h"
#include "td/utils/port/detail/PollableFd.h"
#include "td/utils/port/detail/skip_eintr.h"
#include "td/utils/ScopeGuard.h"
#if TD_PORT_WINDOWS

View File

@ -0,0 +1,57 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/port/rlimit.h"
#include "td/utils/port/config.h"
#include "td/utils/misc.h"
#if TD_PORT_POSIX
#include <sys/resource.h>
#include <sys/time.h>
#include <sys/types.h>
#endif
namespace td {
#if TD_PORT_POSIX
static int get_resource(ResourceLimitType type) {
switch (type) {
case ResourceLimitType::NoFile:
return RLIMIT_NOFILE;
default:
UNREACHABLE();
return -1;
}
}
#endif
Status set_resource_limit(ResourceLimitType type, uint64 value) {
#if TD_PORT_POSIX
int resource = get_resource(type);
rlimit rlim;
if (getrlimit(resource, &rlim) == -1) {
return OS_ERROR("Failed to get current resource limit");
}
TRY_RESULT(new_value, narrow_cast_safe<rlim_t>(value));
if (rlim.rlim_max < new_value) {
rlim.rlim_max = new_value;
}
rlim.rlim_cur = new_value;
if (setrlimit(resource, &rlim) < 0) {
return OS_ERROR("Failed to set resource limit");
}
return Status::OK();
#elif TD_PORT_WINDOWS
return Status::OK(); // Windows has no limits
#endif
}
} // namespace td

View File

@ -6,15 +6,13 @@
//
#pragma once
#include "td/db/binlog/ConcurrentBinlog.h"
#include "td/db/BinlogKeyValue.h"
#include <memory>
#include "td/utils/common.h"
#include "td/utils/Status.h"
namespace td {
using BinlogPmcBase = BinlogKeyValue<ConcurrentBinlog>;
using BinlogPmc = std::shared_ptr<BinlogPmcBase>;
using BinlogPmcPtr = BinlogPmcBase *;
enum class ResourceLimitType { NoFile };
Status set_resource_limit(ResourceLimitType type, uint64 value);
} // namespace td

View File

@ -0,0 +1,55 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/port/user.h"
#include "td/utils/port/config.h"
#if TD_PORT_POSIX
#include <grp.h>
#include <pwd.h>
#if TD_DARWIN || TD_FREEBSD || TD_NETBSD
#include <sys/param.h>
#endif
#include <sys/types.h>
#include <unistd.h>
#endif
namespace td {
Status change_user(CSlice username, CSlice groupname) {
#if TD_PORT_POSIX
passwd *pw = getpwnam(username.c_str());
if (pw == nullptr) {
return OS_ERROR(PSTRING() << "Can't find the user '" << username << "' to switch to");
}
uid_t uid = pw->pw_uid;
gid_t gid = pw->pw_gid;
if (setgroups(1, &gid) == -1) {
return OS_ERROR("Failed to clear supplementary group list");
}
if (!groupname.empty()) {
group *g = getgrnam(groupname.c_str());
if (g == nullptr) {
return OS_ERROR("Can't find the group to switch to");
}
gid = g->gr_gid;
} else if (initgroups(username.c_str(), gid) == -1) {
return OS_ERROR("Failed to load groups of user");
}
if (setgid(gid) == -1) {
return OS_ERROR("failed to set effective group ID");
}
if (setuid(uid) == -1) {
return OS_ERROR("failed to set effective user ID");
}
return Status::OK();
#else
return Status::Error("Changing effective user is not supported");
#endif
}
} // namespace td

View File

@ -0,0 +1,16 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
namespace td {
Status change_user(CSlice username, CSlice groupname = CSlice());
}

View File

@ -187,7 +187,7 @@ class HashMapBenchmark : public td::Benchmark {
size_t threads_n = 16;
int mod_;
static constexpr size_t mul_ = 7273; //1000000000 + 7;
static constexpr size_t MUL = 7273; //1000000000 + 7;
int n_;
public:
@ -197,7 +197,7 @@ class HashMapBenchmark : public td::Benchmark {
return HashMap::get_name();
}
void start_up_n(int n) override {
n *= (int)threads_n;
n *= static_cast<int>(threads_n);
n_ = n;
hash_map = td::make_unique<HashMap>(n * 2);
}
@ -211,7 +211,7 @@ class HashMapBenchmark : public td::Benchmark {
size_t r = n * (i + 1) / threads_n;
threads.emplace_back([l, r, this] {
for (size_t i = l; i < r; i++) {
auto x = td::narrow_cast<int>((i + 1) * mul_ % n_) + 3;
auto x = td::narrow_cast<int>((i + 1) * MUL % n_) + 3;
auto y = td::narrow_cast<int>(i + 2);
hash_map->insert(x, y);
}
@ -224,7 +224,7 @@ class HashMapBenchmark : public td::Benchmark {
void tear_down() override {
for (int i = 0; i < n_; i++) {
auto x = td::narrow_cast<int>((i + 1) * mul_ % n_) + 3;
auto x = td::narrow_cast<int>((i + 1) * MUL % n_) + 3;
auto y = td::narrow_cast<int>(i + 2);
ASSERT_EQ(y, hash_map->find(x, -1));
}

View File

@ -17,28 +17,28 @@
#include <set>
#include <utility>
struct Data {
struct ListData {
td::MovableValue<td::uint64> value;
td::MovableValue<bool> in_list;
Data() = default;
Data(td::uint64 value, bool in_list) : value(value), in_list(in_list) {
ListData() = default;
ListData(td::uint64 value, bool in_list) : value(value), in_list(in_list) {
}
};
struct Node : public td::ListNode {
Node() = default;
explicit Node(Data data) : data(std::move(data)) {
explicit Node(ListData data) : data(std::move(data)) {
}
Data data;
ListData data;
};
static Data &get_data(Node &node) {
static ListData &get_data(Node &node) {
return node.data;
}
static Data &get_data(td::TsListNode<Data> &node) {
static ListData &get_data(td::TsListNode<ListData> &node) {
return node.get_data_unsafe();
}
@ -46,7 +46,7 @@ static std::unique_lock<std::mutex> lock(td::ListNode &node) {
return {};
}
static std::unique_lock<std::mutex> lock(td::TsListNode<Data> &node) {
static std::unique_lock<std::mutex> lock(td::TsListNode<ListData> &node) {
return node.lock();
}
@ -66,26 +66,30 @@ static void do_run_list_test(ListRootT &root, std::atomic<td::uint64> &id) {
nodes.emplace_back(NodeT({next_id(), false}));
};
auto pop_node = [&] {
if (nodes.size() == 0) {
if (nodes.empty()) {
return;
}
nodes.pop_back();
};
auto random_node_index = [&] {
CHECK(!nodes.empty());
return rnd.fast(0, static_cast<int>(nodes.size()) - 1);
};
auto link_node = [&] {
if (nodes.empty()) {
return;
}
auto i = rnd.fast(0, (int)nodes.size() - 1);
auto i = random_node_index();
nodes[i].remove();
get_data(nodes[i]) = Data(next_id(), true);
get_data(nodes[i]) = ListData(next_id(), true);
root.put(&nodes[i]);
};
auto unlink_node = [&] {
if (nodes.empty()) {
return;
}
auto i = rnd.fast(0, (int)nodes.size() - 1);
auto i = random_node_index();
nodes[i].remove();
get_data(nodes[i]).in_list = false;
};
@ -93,16 +97,16 @@ static void do_run_list_test(ListRootT &root, std::atomic<td::uint64> &id) {
if (nodes.empty()) {
return;
}
auto i = rnd.fast(0, (int)nodes.size() - 1);
auto j = rnd.fast(0, (int)nodes.size() - 1);
auto i = random_node_index();
auto j = random_node_index();
std::swap(nodes[i], nodes[j]);
};
auto set_node = [&] {
if (nodes.empty()) {
return;
}
auto i = rnd.fast(0, (int)nodes.size() - 1);
auto j = rnd.fast(0, (int)nodes.size() - 1);
auto i = random_node_index();
auto j = random_node_index();
nodes[i] = std::move(nodes[j]);
};
auto validate = [&] {
@ -144,19 +148,19 @@ TEST(Misc, List) {
}
TEST(Misc, TsList) {
td::TsList<Data> root;
td::TsList<ListData> root;
std::atomic<td::uint64> id{0};
for (std::size_t i = 0; i < 4; i++) {
do_run_list_test<td::TsListNode<Data>, td::TsList<Data>, td::TsListNode<Data>>(root, id);
do_run_list_test<td::TsListNode<ListData>, td::TsList<ListData>, td::TsListNode<ListData>>(root, id);
}
}
TEST(Misc, TsListConcurrent) {
td::TsList<Data> root;
td::TsList<ListData> root;
td::vector<td::thread> threads;
std::atomic<td::uint64> id{0};
for (std::size_t i = 0; i < 4; i++) {
threads.emplace_back(
[&] { do_run_list_test<td::TsListNode<Data>, td::TsList<Data>, td::TsListNode<Data>>(root, id); });
[&] { do_run_list_test<td::TsListNode<ListData>, td::TsList<ListData>, td::TsListNode<ListData>>(root, id); });
}
}

View File

@ -0,0 +1,92 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/utils/common.h"
#include "td/utils/misc.h"
#include "td/utils/OptionParser.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include "td/utils/tests.h"
TEST(OptionParser, run) {
td::OptionParser options;
options.set_description("test description");
td::string exename = "exename";
td::vector<td::string> args;
auto run_option_parser = [&](td::string command_line) {
args = td::full_split(command_line, ' ');
td::vector<char *> argv;
argv.push_back(&exename[0]);
for (auto &arg : args) {
argv.push_back(&arg[0]);
}
return options.run(static_cast<int>(argv.size()), &argv[0]);
};
td::uint64 chosen_options = 0;
td::vector<td::string> chosen_parameters;
auto test_success = [&](td::string command_line, td::uint64 expected_options,
td::vector<td::string> expected_parameters, td::vector<td::string> expected_result) {
chosen_options = 0;
chosen_parameters.clear();
auto result = run_option_parser(command_line);
ASSERT_TRUE(result.is_ok());
ASSERT_EQ(expected_options, chosen_options);
ASSERT_EQ(expected_parameters, chosen_parameters);
ASSERT_EQ(expected_result.size(), result.ok().size());
for (size_t i = 0; i < expected_result.size(); i++) {
ASSERT_STREQ(expected_result[i], td::string(result.ok()[i]));
}
};
auto test_fail = [&](td::string command_line) {
auto result = run_option_parser(command_line);
ASSERT_TRUE(result.is_error());
};
options.add_option('q', "", "", [&] {
chosen_options += 1;
return td::Status::OK();
});
options.add_option('\0', "http-port2", "", [&] {
chosen_options += 10;
return td::Status::OK();
});
options.add_option('p', "http-port", "", [&](td::Slice parameter) {
chosen_options += 100;
chosen_parameters.push_back(parameter.str());
return td::Status::OK();
});
options.add_option('v', "test", "", [&] {
chosen_options += 1000;
return td::Status::OK();
});
test_fail("-http-port2");
test_success("-", 0, {}, {"-"});
test_fail("--http-port");
test_fail("--http-port3");
test_fail("--http-por");
test_fail("--http-port2=1");
test_fail("--q");
test_fail("-qvp");
test_fail("-p");
test_fail("-u");
test_success("-q", 1, {}, {});
test_success("-vvvvvvvvvv", 10000, {}, {});
test_success("-qpv", 101, {"v"}, {});
test_success("-qp -v", 101, {"-v"}, {});
test_success("-qp --http-port2", 101, {"--http-port2"}, {});
test_success("-qp -- -v", 1101, {"--"}, {});
test_success("-qvqvpqv", 2102, {"qv"}, {});
test_success("aba --http-port2 caba --http-port2 dabacaba", 20, {}, {"aba", "caba", "dabacaba"});
test_success("das -pqwerty -- -v asd --http-port", 100, {"qwerty"}, {"das", "-v", "asd", "--http-port"});
test_success("-p option --http-port option2 --http-port=option3 --http-port=", 400,
{"option", "option2", "option3", ""}, {});
test_success("", 0, {}, {});
test_success("a", 0, {}, {"a"});
test_success("", 0, {}, {});
}

View File

@ -9,6 +9,7 @@
#include "td/utils/common.h"
#include "td/utils/crypto.h"
#include "td/utils/logging.h"
#include "td/utils/Random.h"
#include "td/utils/Slice.h"
#include "td/utils/tests.h"
#include "td/utils/UInt.h"
@ -18,6 +19,29 @@
static td::vector<td::string> strings{"", "1", "short test string", td::string(1000000, 'a')};
#if TD_HAVE_OPENSSL
#if TD_HAVE_ZLIB
TEST(Crypto, Aes) {
td::Random::Xorshift128plus rnd(123);
td::UInt256 key;
rnd.bytes(as_slice(key));
td::string plaintext(16, '\0');
td::string encrypted(16, '\0');
td::string decrypted(16, '\0');
rnd.bytes(plaintext);
td::AesState encryptor;
encryptor.init(as_slice(key), true);
td::AesState decryptor;
decryptor.init(as_slice(key), false);
encryptor.encrypt(td::as_slice(plaintext).ubegin(), td::as_slice(encrypted).ubegin(), 16);
decryptor.decrypt(td::as_slice(encrypted).ubegin(), td::as_slice(decrypted).ubegin(), 16);
CHECK(decrypted == plaintext);
CHECK(decrypted != encrypted);
CHECK(td::crc32(encrypted) == 178892237);
}
TEST(Crypto, AesCtrState) {
td::vector<td::uint32> answers1{0u, 1141589763u, 596296607u, 3673001485u, 2302125528u,
330967191u, 2047392231u, 3537459563u, 307747798u, 2149598133u};
@ -51,7 +75,7 @@ TEST(Crypto, AesCtrState) {
ASSERT_EQ(answers1[i], td::crc32(t));
state.init(as_slice(key), as_slice(iv));
state.decrypt(t, t);
ASSERT_STREQ(s, t);
ASSERT_STREQ(td::base64_encode(s), td::base64_encode(t));
for (auto &c : iv.raw) {
c = 0xFF;
@ -64,6 +88,82 @@ TEST(Crypto, AesCtrState) {
}
}
TEST(Crypto, AesIgeState) {
td::vector<td::uint32> answers1{0u, 2045698207u, 2423540300u, 525522475u, 1545267325u};
std::size_t i = 0;
for (auto length : {0, 16, 32, 256, 1024}) {
td::uint32 seed = length;
td::string s(length, '\0');
for (auto &c : s) {
seed = seed * 123457567u + 987651241u;
c = static_cast<char>((seed >> 23) & 255);
}
td::UInt256 key;
for (auto &c : key.raw) {
seed = seed * 123457567u + 987651241u;
c = (seed >> 23) & 255;
}
td::UInt256 iv;
for (auto &c : iv.raw) {
seed = seed * 123457567u + 987651241u;
c = (seed >> 23) & 255;
}
td::AesIgeState state;
state.init(as_slice(key), as_slice(iv), true);
td::string t(length, '\0');
state.encrypt(s, t);
ASSERT_EQ(answers1[i], td::crc32(t));
state.init(as_slice(key), as_slice(iv), false);
state.decrypt(t, t);
ASSERT_STREQ(td::base64_encode(s), td::base64_encode(t));
i++;
}
}
TEST(Crypto, AesCbcState) {
td::vector<td::uint32> answers1{0u, 3617355989u, 3449188102u, 186999968u, 4244808847u};
std::size_t i = 0;
for (auto length : {0, 16, 32, 256, 1024}) {
td::uint32 seed = length;
td::string s(length, '\0');
for (auto &c : s) {
seed = seed * 123457567u + 987651241u;
c = static_cast<char>((seed >> 23) & 255);
}
td::UInt256 key;
for (auto &c : key.raw) {
seed = seed * 123457567u + 987651241u;
c = (seed >> 23) & 255;
}
td::UInt128 iv;
for (auto &c : iv.raw) {
seed = seed * 123457567u + 987651241u;
c = (seed >> 23) & 255;
}
td::AesCbcState state(as_slice(key), as_slice(iv));
//state.init(as_slice(key), as_slice(iv), true);
td::string t(length, '\0');
state.encrypt(s, t);
ASSERT_EQ(answers1[i], td::crc32(t));
//state.init(as_slice(key), as_slice(iv), false);
state = td::AesCbcState(as_slice(key), as_slice(iv));
state.decrypt(t, t);
ASSERT_STREQ(td::base64_encode(s), td::base64_encode(t));
i++;
}
}
#endif
TEST(Crypto, Sha256State) {
for (auto length : {0, 1, 31, 32, 33, 9999, 10000, 10001, 999999, 1000001}) {
auto s = td::rand_string(std::numeric_limits<char>::min(), std::numeric_limits<char>::max(), length);
@ -85,8 +185,8 @@ TEST(Crypto, Sha256State) {
}
TEST(Crypto, PBKDF) {
td::vector<td::string> passwords{"", "qwerty", std::string(1000, 'a')};
td::vector<td::string> salts{"", "qwerty", std::string(1000, 'a')};
td::vector<td::string> passwords{"", "qwerty", td::string(1000, 'a')};
td::vector<td::string> salts{"", "qwerty", td::string(1000, 'a')};
td::vector<int> iteration_counts{1, 2, 1000};
td::vector<td::Slice> answers{
"984LZT0tcqQQjPWr6RL/3Xd2Ftu7J6cOggTzri0Pb60=", "lzmEEdaupDp3rO+SImq4J41NsGaL0denanJfdoCsRcU=",
@ -187,7 +287,7 @@ TEST(Crypto, crc32c_benchmark) {
public:
explicit Crc32cExtendBenchmark(size_t chunk_size) : chunk_size_(chunk_size) {
}
std::string get_description() const override {
td::string get_description() const override {
return PSTRING() << "Crc32c with chunk_size=" << chunk_size_;
}
void start_up_n(int n) override {
@ -197,7 +297,7 @@ TEST(Crypto, crc32c_benchmark) {
} else {
cnt_ = 1;
}
data_ = std::string(n, 'a');
data_ = td::string(n, 'a');
}
void run(int n) override {
td::uint32 res = 0;
@ -214,7 +314,7 @@ TEST(Crypto, crc32c_benchmark) {
private:
size_t chunk_size_;
std::string data_;
td::string data_;
int cnt_;
};
bench(Crc32cExtendBenchmark(2));

View File

@ -12,6 +12,7 @@ set(TD_TEST_SOURCE
${CMAKE_CURRENT_SOURCE_DIR}/set_with_position.cpp
${CMAKE_CURRENT_SOURCE_DIR}/string_cleaning.cpp
${CMAKE_CURRENT_SOURCE_DIR}/tdclient.cpp
${CMAKE_CURRENT_SOURCE_DIR}/tqueue.cpp
${CMAKE_CURRENT_SOURCE_DIR}/data.cpp
${CMAKE_CURRENT_SOURCE_DIR}/data.h

View File

@ -7,7 +7,11 @@
#include "td/utils/tests.h"
#include "td/utils/common.h"
#include "td/utils/crypto.h"
#include "td/utils/logging.h"
#include "td/utils/OptionParser.h"
#include "td/utils/Slice.h"
#include "td/utils/Status.h"
#include <cstring>
@ -16,18 +20,26 @@
#endif
int main(int argc, char **argv) {
// TODO port OptionsParser to Windows
td::init_openssl_threads();
td::TestsRunner &runner = td::TestsRunner::get_default();
SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR));
for (int i = 1; i < argc; i++) {
if (!std::strcmp(argv[i], "--filter")) {
CHECK(i + 1 < argc);
runner.add_substr_filter(argv[++i]);
}
if (!std::strcmp(argv[i], "--stress")) {
runner.set_stress_flag(true);
}
td::OptionParser options;
options.add_option('\0', "filter", "Run only specified tests", [&](td::Slice filter) {
runner.add_substr_filter(filter.str());
return td::Status::OK();
});
options.add_option('\0', "stress", "Run tests infinitely", [&] {
runner.set_stress_flag(true);
return td::Status::OK();
});
auto result = options.run(argc, argv);
if (result.is_error() || !result.ok().empty()) {
LOG(PLAIN) << options;
return 1;
}
#if TD_EMSCRIPTEN
emscripten_set_main_loop(
[] {

View File

@ -467,7 +467,7 @@ TEST(MessageEntities, url) {
check_url("@.", {});
check_url(
"a.b.google.com dfsknnfs gsdfgsg http://códuia.de/ dffdg,\" 12)(cpia.de/())(\" http://гришка.рф/ sdufhdf "
"http://xn--80afpi2a3c.xn--p1ai/ I have a good time.Thanks, guys!\n\n(hdfughidufhgdis)go#ogle.com гришка.рф "
"http://xn--80afpi2a3c.xn--p1ai/ I have a good time.Thanks, guys!\n\n(hdfughidufhgdis) go#ogle.com гришка.рф "
"hsighsdf gi почта.рф\n\n✪df.ws/123 "
"xn--80afpi2a3c.xn--p1ai\n\nhttp://foo.com/blah_blah\nhttp://foo.com/blah_blah/\n(Something like "
"http://foo.com/blah_blah)\nhttp://foo.com/blah_blah_(wikipedi8989a_Вася)\n(Something like "

192
test/tqueue.cpp Normal file
View File

@ -0,0 +1,192 @@
//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/db/binlog/Binlog.h"
#include "td/db/binlog/BinlogEvent.h"
#include "td/db/binlog/BinlogHelper.h"
#include "td/db/TQueue.h"
#include "td/utils/int_types.h"
#include "td/utils/misc.h"
#include "td/utils/port/path.h"
#include "td/utils/Slice.h"
#include "td/utils/Span.h"
#include "td/utils/Status.h"
#include "td/utils/tests.h"
#include "td/utils/VectorQueue.h"
#include <map>
#include <set>
TEST(TQueue, hands) {
td::TQueue::Event events[100];
auto events_span = td::MutableSpan<td::TQueue::Event>(events, 100);
auto tqueue = td::TQueue::create();
auto qid = 12;
ASSERT_EQ(true, tqueue->get_head(qid).empty());
ASSERT_EQ(true, tqueue->get_tail(qid).empty());
tqueue->push(qid, "hello", 0, 0, td::TQueue::EventId());
auto head = tqueue->get_head(qid);
auto tail = tqueue->get_tail(qid);
ASSERT_EQ(head.next().ok(), tail);
ASSERT_EQ(1u, tqueue->get(qid, head, true, 0, events_span).move_as_ok());
ASSERT_EQ(1u, tqueue->get(qid, head, true, 0, events_span).move_as_ok());
ASSERT_EQ(0u, tqueue->get(qid, tail, false, 0, events_span).move_as_ok());
ASSERT_EQ(1u, tqueue->get(qid, head, true, 0, events_span).move_as_ok());
ASSERT_EQ(0u, tqueue->get(qid, tail, true, 0, events_span).move_as_ok());
ASSERT_EQ(0u, tqueue->get(qid, head, true, 0, events_span).move_as_ok());
}
class TestTQueue {
public:
using EventId = td::TQueue::EventId;
static td::CSlice binlog_path() {
return td::CSlice("tqueue_binlog");
}
TestTQueue() {
baseline_ = td::TQueue::create();
memory_ = td::TQueue::create();
auto memory_storage = td::make_unique<td::TQueueMemoryStorage>();
memory_storage_ = memory_storage.get();
memory_->set_callback(std::move(memory_storage));
binlog_ = td::TQueue::create();
auto tqueue_binlog = td::make_unique<td::TQueueBinlog<td::Binlog>>();
td::Binlog::destroy(binlog_path()).ensure();
auto binlog = std::make_shared<td::Binlog>();
binlog->init(binlog_path().str(), [&](const td::BinlogEvent &event) { UNREACHABLE(); }).ensure();
tqueue_binlog->set_binlog(std::move(binlog));
binlog_->set_callback(std::move(tqueue_binlog));
}
void restart(td::Random::Xorshift128plus &rnd, double now) {
if (rnd.fast(0, 10) == 0) {
baseline_->run_gc(now);
}
memory_->extract_callback().release();
auto memory_storage = td::unique_ptr<td::TQueueMemoryStorage>(memory_storage_);
memory_ = td::TQueue::create();
memory_storage->replay(*memory_);
memory_->set_callback(std::move(memory_storage));
if (rnd.fast(0, 10) == 0) {
memory_->run_gc(now);
}
if (rnd.fast(0, 30) != 0) {
return;
}
LOG(INFO) << "Restart binlog";
binlog_ = td::TQueue::create();
auto tqueue_binlog = td::make_unique<td::TQueueBinlog<td::Binlog>>();
auto binlog = std::make_shared<td::Binlog>();
binlog->init(binlog_path().str(), [&](const td::BinlogEvent &event) { tqueue_binlog->replay(event, *binlog_); })
.ensure();
tqueue_binlog->set_binlog(std::move(binlog));
binlog_->set_callback(std::move(tqueue_binlog));
if (rnd.fast(0, 2) == 0) {
binlog_->run_gc(now);
}
}
EventId push(td::TQueue::QueueId queue_id, td::string data, double expires_at, EventId new_id = EventId()) {
auto a_id = baseline_->push(queue_id, data, expires_at, 0, new_id).move_as_ok();
auto b_id = memory_->push(queue_id, data, expires_at, 0, new_id).move_as_ok();
auto c_id = binlog_->push(queue_id, data, expires_at, 0, new_id).move_as_ok();
ASSERT_EQ(a_id, b_id);
ASSERT_EQ(a_id, c_id);
return a_id;
}
void check_head_tail(td::TQueue::QueueId qid, double now) {
//ASSERT_EQ(baseline_->get_head(qid), memory_->get_head(qid));
//ASSERT_EQ(baseline_->get_head(qid), binlog_->get_head(qid));
ASSERT_EQ(baseline_->get_tail(qid), memory_->get_tail(qid));
ASSERT_EQ(baseline_->get_tail(qid), binlog_->get_tail(qid));
}
void check_get(td::TQueue::QueueId qid, td::Random::Xorshift128plus &rnd, double now) {
td::TQueue::Event a[10];
td::MutableSpan<td::TQueue::Event> a_span(a, 10);
td::TQueue::Event b[10];
td::MutableSpan<td::TQueue::Event> b_span(b, 10);
td::TQueue::Event c[10];
td::MutableSpan<td::TQueue::Event> c_span(c, 10);
auto a_from = baseline_->get_head(qid);
//auto b_from = memory_->get_head(qid);
//auto c_from = binlog_->get_head(qid);
//ASSERT_EQ(a_from, b_from);
//ASSERT_EQ(a_from, c_from);
auto tmp = a_from.advance(rnd.fast(-10, 10));
if (tmp.is_ok()) {
a_from = tmp.move_as_ok();
}
baseline_->get(qid, a_from, true, now, a_span).move_as_ok();
memory_->get(qid, a_from, true, now, b_span).move_as_ok();
binlog_->get(qid, a_from, true, now, c_span).move_as_ok();
ASSERT_EQ(a_span.size(), b_span.size());
ASSERT_EQ(a_span.size(), c_span.size());
for (size_t i = 0; i < a_span.size(); i++) {
ASSERT_EQ(a_span[i].id, b_span[i].id);
ASSERT_EQ(a_span[i].id, c_span[i].id);
ASSERT_EQ(a_span[i].data, b_span[i].data);
ASSERT_EQ(a_span[i].data, c_span[i].data);
}
}
private:
td::unique_ptr<td::TQueue> baseline_;
td::unique_ptr<td::TQueue> memory_;
td::unique_ptr<td::TQueue> binlog_;
td::TQueueMemoryStorage *memory_storage_{nullptr};
};
TEST(TQueue, random) {
using EventId = td::TQueue::EventId;
td::Random::Xorshift128plus rnd(123);
auto next_queue_id = [&rnd] {
return rnd.fast(1, 10);
};
auto next_first_id = [&rnd] {
if (rnd.fast(0, 3) == 0) {
return EventId::from_int32(EventId::MAX_ID - 20).move_as_ok();
}
return EventId::from_int32(rnd.fast(1000000000, 1500000000)).move_as_ok();
};
TestTQueue q;
double now = 0;
auto push_event = [&] {
auto data = PSTRING() << rnd();
if (rnd.fast(0, 10000) == 0) {
data = td::string(1 << 19, '\0');
}
q.push(next_queue_id(), data, now + rnd.fast(-10, 10) * 10 + 5, next_first_id());
};
auto inc_now = [&] {
now += 10;
};
auto check_head_tail = [&] {
q.check_head_tail(next_queue_id(), now);
};
auto restart = [&] {
q.restart(rnd, now);
};
auto get = [&] {
q.check_get(next_queue_id(), rnd, now);
};
td::RandomSteps steps({{push_event, 100}, {check_head_tail, 10}, {get, 40}, {inc_now, 5}, {restart, 1}});
for (int i = 0; i < 100000; i++) {
steps.step(rnd);
}
}