Merge commit 'f45d80fe16f99d112d545b7cd74ce46342fe3437'
This commit is contained in:
commit
886a40118e
@ -189,7 +189,7 @@ elseif (INTEL)
|
||||
endif()
|
||||
|
||||
if (WIN32)
|
||||
add_definitions(-DNTDDI_VERSION=0x06020000 -DWINVER=0x0602 -D_WIN32_WINNT=0x0602 -DNOMINMAX -DUNICODE -D_UNICODE -DWIN32_LEAN_AND_MEAN)
|
||||
add_definitions(-DNTDDI_VERSION=0x06020000 -DWINVER=0x0602 -D_WIN32_WINNT=0x0602 -DPSAPI_VERSION=1 -DNOMINMAX -DUNICODE -D_UNICODE -DWIN32_LEAN_AND_MEAN)
|
||||
endif()
|
||||
if (CYGWIN)
|
||||
add_definitions(-D_DEFAULT_SOURCE=1 -DFD_SETSIZE=4096)
|
||||
|
@ -39,9 +39,9 @@ function split_file($file, $chunks, $undo) {
|
||||
$cmake_new_files = $new_files;
|
||||
if ($is_generated) {
|
||||
foreach ($cmake_new_files as &$file_ref) {
|
||||
$file_ref = str_replace('td/generate', '${CMAKE_CURRENT_SOURCE_DIR}', $file_ref);
|
||||
$file_ref = str_replace('td/generate/auto/td', '${TD_AUTO_INCLUDE_DIR}', $file_ref);
|
||||
}
|
||||
$cmake_cpp_name = str_replace('td/generate', '${CMAKE_CURRENT_SOURCE_DIR}', $cmake_cpp_name);
|
||||
$cmake_cpp_name = str_replace('td/generate/auto/td', '${TD_AUTO_INCLUDE_DIR}', $cmake_cpp_name);
|
||||
}
|
||||
|
||||
if ($undo) {
|
||||
|
@ -24,6 +24,7 @@
|
||||
#include <vector>
|
||||
|
||||
static constexpr int DATA_SIZE = 8 << 10;
|
||||
static constexpr int SHORT_DATA_SIZE = 64;
|
||||
|
||||
class SHA1Bench : public td::Benchmark {
|
||||
public:
|
||||
@ -48,14 +49,45 @@ class SHA1Bench : public td::Benchmark {
|
||||
}
|
||||
};
|
||||
|
||||
class AESBench : public td::Benchmark {
|
||||
class AesEcbBench : public td::Benchmark {
|
||||
public:
|
||||
alignas(64) unsigned char data[DATA_SIZE];
|
||||
td::UInt256 key;
|
||||
td::UInt256 iv;
|
||||
|
||||
std::string get_description() const override {
|
||||
return PSTRING() << "AES OpenSSL [" << (DATA_SIZE >> 10) << "KB]";
|
||||
return PSTRING() << "AES ECB OpenSSL [" << (DATA_SIZE >> 10) << "KB]";
|
||||
}
|
||||
|
||||
void start_up() override {
|
||||
for (int i = 0; i < DATA_SIZE; i++) {
|
||||
data[i] = 123;
|
||||
}
|
||||
td::Random::secure_bytes(key.raw, sizeof(key));
|
||||
td::Random::secure_bytes(iv.raw, sizeof(iv));
|
||||
}
|
||||
|
||||
void run(int n) override {
|
||||
td::AesState state;
|
||||
state.init(td::as_slice(key), true);
|
||||
td::MutableSlice data_slice(data, DATA_SIZE);
|
||||
for (int i = 0; i <= n; i++) {
|
||||
size_t step = 16;
|
||||
for (size_t offset = 0; offset + step <= data_slice.size(); offset += step) {
|
||||
state.encrypt(data_slice.ubegin() + offset, data_slice.ubegin() + offset, static_cast<int>(step));
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
class AesIgeEncryptBench : public td::Benchmark {
|
||||
public:
|
||||
alignas(64) unsigned char data[DATA_SIZE];
|
||||
td::UInt256 key;
|
||||
td::UInt256 iv;
|
||||
|
||||
std::string get_description() const override {
|
||||
return PSTRING() << "AES IGE OpenSSL encrypt [" << (DATA_SIZE >> 10) << "KB]";
|
||||
}
|
||||
|
||||
void start_up() override {
|
||||
@ -68,8 +100,125 @@ class AESBench : public td::Benchmark {
|
||||
|
||||
void run(int n) override {
|
||||
td::MutableSlice data_slice(data, DATA_SIZE);
|
||||
td::AesIgeState state;
|
||||
state.init(as_slice(key), as_slice(iv), true);
|
||||
for (int i = 0; i < n; i++) {
|
||||
td::aes_ige_encrypt(as_slice(key), as_slice(iv), data_slice, data_slice);
|
||||
state.encrypt(data_slice, data_slice);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
class AesIgeDecryptBench : public td::Benchmark {
|
||||
public:
|
||||
alignas(64) unsigned char data[DATA_SIZE];
|
||||
td::UInt256 key;
|
||||
td::UInt256 iv;
|
||||
|
||||
std::string get_description() const override {
|
||||
return PSTRING() << "AES IGE OpenSSL decrypt [" << (DATA_SIZE >> 10) << "KB]";
|
||||
}
|
||||
|
||||
void start_up() override {
|
||||
for (int i = 0; i < DATA_SIZE; i++) {
|
||||
data[i] = 123;
|
||||
}
|
||||
td::Random::secure_bytes(key.raw, sizeof(key));
|
||||
td::Random::secure_bytes(iv.raw, sizeof(iv));
|
||||
}
|
||||
|
||||
void run(int n) override {
|
||||
td::MutableSlice data_slice(data, DATA_SIZE);
|
||||
td::AesIgeState state;
|
||||
state.init(as_slice(key), as_slice(iv), false);
|
||||
for (int i = 0; i < n; i++) {
|
||||
state.decrypt(data_slice, data_slice);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
class AesCtrBench : public td::Benchmark {
|
||||
public:
|
||||
alignas(64) unsigned char data[DATA_SIZE];
|
||||
td::UInt256 key;
|
||||
td::UInt128 iv;
|
||||
|
||||
std::string get_description() const override {
|
||||
return PSTRING() << "AES CTR OpenSSL [" << (DATA_SIZE >> 10) << "KB]";
|
||||
}
|
||||
|
||||
void start_up() override {
|
||||
for (int i = 0; i < DATA_SIZE; i++) {
|
||||
data[i] = 123;
|
||||
}
|
||||
td::Random::secure_bytes(key.raw, sizeof(key));
|
||||
td::Random::secure_bytes(iv.raw, sizeof(iv));
|
||||
}
|
||||
|
||||
void run(int n) override {
|
||||
td::MutableSlice data_slice(data, DATA_SIZE);
|
||||
td::AesCtrState state;
|
||||
state.init(as_slice(key), as_slice(iv));
|
||||
for (int i = 0; i < n; i++) {
|
||||
state.encrypt(data_slice, data_slice);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
class AesCbcBench : public td::Benchmark {
|
||||
public:
|
||||
alignas(64) unsigned char data[DATA_SIZE];
|
||||
td::UInt256 key;
|
||||
td::UInt128 iv;
|
||||
|
||||
std::string get_description() const override {
|
||||
return PSTRING() << "AES CBC OpenSSL [" << (DATA_SIZE >> 10) << "KB]";
|
||||
}
|
||||
|
||||
void start_up() override {
|
||||
for (int i = 0; i < DATA_SIZE; i++) {
|
||||
data[i] = 123;
|
||||
}
|
||||
td::Random::secure_bytes(as_slice(key));
|
||||
td::Random::secure_bytes(as_slice(iv));
|
||||
}
|
||||
|
||||
void run(int n) override {
|
||||
td::MutableSlice data_slice(data, DATA_SIZE);
|
||||
for (int i = 0; i < n; i++) {
|
||||
td::aes_cbc_encrypt(as_slice(key), as_slice(iv), data_slice, data_slice);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <bool use_state>
|
||||
class AesIgeShortBench : public td::Benchmark {
|
||||
public:
|
||||
alignas(64) unsigned char data[SHORT_DATA_SIZE];
|
||||
td::UInt256 key;
|
||||
td::UInt256 iv;
|
||||
|
||||
std::string get_description() const override {
|
||||
return PSTRING() << "AES IGE OpenSSL " << (use_state ? "EVP" : "C ") << "[" << SHORT_DATA_SIZE << "B]";
|
||||
}
|
||||
|
||||
void start_up() override {
|
||||
for (int i = 0; i < SHORT_DATA_SIZE; i++) {
|
||||
data[i] = 123;
|
||||
}
|
||||
td::Random::secure_bytes(as_slice(key));
|
||||
td::Random::secure_bytes(as_slice(iv));
|
||||
}
|
||||
|
||||
void run(int n) override {
|
||||
td::MutableSlice data_slice(data, SHORT_DATA_SIZE);
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (use_state) {
|
||||
td::AesIgeState ige;
|
||||
ige.init(as_slice(key), as_slice(iv), false);
|
||||
ige.decrypt(data_slice, data_slice);
|
||||
} else {
|
||||
td::aes_ige_decrypt(as_slice(key), as_slice(iv), data_slice, data_slice);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
@ -112,13 +261,13 @@ BENCH(SslRand, "ssl_rand_int32") {
|
||||
std::vector<td::thread> v;
|
||||
std::atomic<td::uint32> sum{0};
|
||||
for (int i = 0; i < 3; i++) {
|
||||
v.push_back(td::thread([&] {
|
||||
v.emplace_back([&sum, n] {
|
||||
td::int32 res = 0;
|
||||
for (int j = 0; j < n; j++) {
|
||||
res ^= td::Random::secure_int32();
|
||||
}
|
||||
sum += res;
|
||||
}));
|
||||
});
|
||||
}
|
||||
for (auto &x : v) {
|
||||
x.join();
|
||||
@ -196,6 +345,15 @@ class Crc64Bench : public td::Benchmark {
|
||||
};
|
||||
|
||||
int main() {
|
||||
td::init_openssl_threads();
|
||||
|
||||
td::bench(AesIgeShortBench<true>());
|
||||
td::bench(AesIgeShortBench<false>());
|
||||
td::bench(AesIgeEncryptBench());
|
||||
td::bench(AesIgeDecryptBench());
|
||||
td::bench(AesEcbBench());
|
||||
td::bench(AesCtrBench());
|
||||
|
||||
td::bench(Pbkdf2Bench());
|
||||
td::bench(RandBench());
|
||||
td::bench(CppRandBench());
|
||||
@ -206,7 +364,6 @@ int main() {
|
||||
#endif
|
||||
td::bench(SslRandBufBench());
|
||||
td::bench(SHA1Bench());
|
||||
td::bench(AESBench());
|
||||
td::bench(Crc32Bench());
|
||||
td::bench(Crc64Bench());
|
||||
return 0;
|
||||
|
@ -40,8 +40,8 @@ class IostreamWriteBench : public td::Benchmark {
|
||||
protected:
|
||||
std::string file_name_;
|
||||
std::ofstream stream;
|
||||
enum { buffer_size = 1 << 20 };
|
||||
char buffer[buffer_size];
|
||||
static constexpr std::size_t BUFFER_SIZE = 1 << 20;
|
||||
char buffer[BUFFER_SIZE];
|
||||
|
||||
public:
|
||||
std::string get_description() const override {
|
||||
@ -52,7 +52,7 @@ class IostreamWriteBench : public td::Benchmark {
|
||||
file_name_ = create_tmp_file();
|
||||
stream.open(file_name_.c_str());
|
||||
CHECK(stream.is_open());
|
||||
// stream.rdbuf()->pubsetbuf(buffer, buffer_size);
|
||||
// stream.rdbuf()->pubsetbuf(buffer, BUFFER_SIZE);
|
||||
}
|
||||
|
||||
void run(int n) override {
|
||||
@ -71,8 +71,8 @@ class FILEWriteBench : public td::Benchmark {
|
||||
protected:
|
||||
std::string file_name_;
|
||||
FILE *file;
|
||||
enum { buffer_size = 1 << 20 };
|
||||
char buffer[buffer_size];
|
||||
static constexpr std::size_t BUFFER_SIZE = 1 << 20;
|
||||
char buffer[BUFFER_SIZE];
|
||||
|
||||
public:
|
||||
std::string get_description() const override {
|
||||
@ -82,7 +82,7 @@ class FILEWriteBench : public td::Benchmark {
|
||||
void start_up() override {
|
||||
file_name_ = create_tmp_file();
|
||||
file = fopen(file_name_.c_str(), "w");
|
||||
// setvbuf(file, buffer, _IOFBF, buffer_size);
|
||||
// setvbuf(file, buffer, _IOFBF, BUFFER_SIZE);
|
||||
}
|
||||
|
||||
void run(int n) override {
|
||||
@ -123,8 +123,8 @@ class LogWriteBench : public td::Benchmark {
|
||||
std::string file_name_;
|
||||
std::ofstream stream;
|
||||
std::streambuf *old_buf;
|
||||
enum { buffer_size = 1 << 20 };
|
||||
char buffer[buffer_size];
|
||||
static constexpr std::size_t BUFFER_SIZE = 1 << 20;
|
||||
char buffer[BUFFER_SIZE];
|
||||
|
||||
public:
|
||||
std::string get_description() const override {
|
||||
|
@ -837,7 +837,7 @@ class QueueBenchmark : public td::Benchmark {
|
||||
|
||||
template <class QueueT>
|
||||
class RingBenchmark : public td::Benchmark {
|
||||
enum { QN = 504 };
|
||||
static constexpr int QN = 504;
|
||||
|
||||
struct Thread {
|
||||
int int_id;
|
||||
|
@ -515,7 +515,8 @@ function onOptionsChanged() {
|
||||
pre_text.push('Note that following calls to <code>pkg</code> needs to be run as <code>root</code>.');
|
||||
}
|
||||
if (os_openbsd) {
|
||||
pre_text.push('Note that following instruction is for OpenBSD 6.4 and default KSH shell.');
|
||||
pre_text.push('Note that following instruction is for OpenBSD 6.7 and default KSH shell.');
|
||||
pre_text.push('Note that building requires a lot of memory, so you may need to increase allowed per-process memory usage in /etc/login.conf or build from root.');
|
||||
}
|
||||
if (os_netbsd) {
|
||||
pre_text.push('Note that following instruction is for NetBSD 8.0 and default SH shell.');
|
||||
@ -648,7 +649,7 @@ function onOptionsChanged() {
|
||||
if (target === 'JNI') {
|
||||
packages += ' jdk';
|
||||
}
|
||||
commands.push('pkg_add ' + packages);
|
||||
commands.push('pkg_add -z ' + packages);
|
||||
if (!use_root) {
|
||||
commands.push('exit');
|
||||
}
|
||||
|
@ -117,8 +117,8 @@ static Backtrace get_backtrace() {
|
||||
return res;
|
||||
}
|
||||
|
||||
static constexpr std::size_t reserved = 16;
|
||||
static constexpr std::int32_t malloc_info_magic = 0x27138373;
|
||||
static constexpr std::size_t RESERVED_SIZE = 16;
|
||||
static constexpr std::int32_t MALLOC_INFO_MAGIC = 0x27138373;
|
||||
struct malloc_info {
|
||||
std::int32_t magic;
|
||||
std::int32_t size;
|
||||
@ -139,9 +139,9 @@ struct HashtableNode {
|
||||
std::atomic<std::size_t> size;
|
||||
};
|
||||
|
||||
static constexpr std::size_t ht_max_size = 1000000;
|
||||
static constexpr std::size_t HT_MAX_SIZE = 1000000;
|
||||
static std::atomic<std::size_t> ht_size{0};
|
||||
static std::array<HashtableNode, ht_max_size> ht;
|
||||
static std::array<HashtableNode, HT_MAX_SIZE> ht;
|
||||
|
||||
std::size_t get_ht_size() {
|
||||
return ht_size.load();
|
||||
@ -154,9 +154,9 @@ std::int32_t get_ht_pos(const Backtrace &bt, bool force = false) {
|
||||
while (true) {
|
||||
auto pos_hash = ht[pos].hash.load();
|
||||
if (pos_hash == 0) {
|
||||
if (ht_size > ht_max_size / 2) {
|
||||
if (ht_size > HT_MAX_SIZE / 2) {
|
||||
if (force) {
|
||||
assert(ht_size * 10 < ht_max_size * 7);
|
||||
assert(ht_size * 10 < HT_MAX_SIZE * 7);
|
||||
} else {
|
||||
Backtrace unknown_bt{{nullptr}};
|
||||
unknown_bt[0] = reinterpret_cast<void *>(1);
|
||||
@ -206,8 +206,8 @@ void register_xalloc(malloc_info *info, std::int32_t diff) {
|
||||
extern "C" {
|
||||
|
||||
static void *malloc_with_frame(std::size_t size, const Backtrace &frame) {
|
||||
static_assert(reserved % alignof(std::max_align_t) == 0, "fail");
|
||||
static_assert(reserved >= sizeof(malloc_info), "fail");
|
||||
static_assert(RESERVED_SIZE % alignof(std::max_align_t) == 0, "fail");
|
||||
static_assert(RESERVED_SIZE >= sizeof(malloc_info), "fail");
|
||||
#if TD_DARWIN
|
||||
static void *malloc_void = dlsym(RTLD_NEXT, "malloc");
|
||||
static auto malloc_old = *reinterpret_cast<decltype(malloc) **>(&malloc_void);
|
||||
@ -215,26 +215,26 @@ static void *malloc_with_frame(std::size_t size, const Backtrace &frame) {
|
||||
extern decltype(malloc) __libc_malloc;
|
||||
static auto malloc_old = __libc_malloc;
|
||||
#endif
|
||||
auto *info = static_cast<malloc_info *>(malloc_old(size + reserved));
|
||||
auto *info = static_cast<malloc_info *>(malloc_old(size + RESERVED_SIZE));
|
||||
auto *buf = reinterpret_cast<char *>(info);
|
||||
|
||||
info->magic = malloc_info_magic;
|
||||
info->magic = MALLOC_INFO_MAGIC;
|
||||
info->size = static_cast<std::int32_t>(size);
|
||||
info->ht_pos = get_ht_pos(frame);
|
||||
|
||||
register_xalloc(info, +1);
|
||||
|
||||
void *data = buf + reserved;
|
||||
void *data = buf + RESERVED_SIZE;
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
static malloc_info *get_info(void *data_void) {
|
||||
char *data = static_cast<char *>(data_void);
|
||||
auto *buf = data - reserved;
|
||||
auto *buf = data - RESERVED_SIZE;
|
||||
|
||||
auto *info = reinterpret_cast<malloc_info *>(buf);
|
||||
assert(info->magic == malloc_info_magic);
|
||||
assert(info->magic == MALLOC_INFO_MAGIC);
|
||||
return info;
|
||||
}
|
||||
|
||||
|
@ -82,13 +82,13 @@ class TlWriterDotNet : public TL_writer {
|
||||
static std::string to_cCamelCase(const std::string &name, bool flag) {
|
||||
bool next_to_upper = flag;
|
||||
std::string result;
|
||||
for (int i = 0; i < (int)name.size(); i++) {
|
||||
for (std::size_t i = 0; i < name.size(); i++) {
|
||||
if (!is_alnum(name[i])) {
|
||||
next_to_upper = true;
|
||||
continue;
|
||||
}
|
||||
if (next_to_upper) {
|
||||
result += (char)to_upper(name[i]);
|
||||
result += to_upper(name[i]);
|
||||
next_to_upper = false;
|
||||
} else {
|
||||
result += name[i];
|
||||
@ -98,7 +98,7 @@ class TlWriterDotNet : public TL_writer {
|
||||
}
|
||||
|
||||
std::string gen_native_field_name(std::string name) const {
|
||||
for (int i = 0; i < (int)name.size(); i++) {
|
||||
for (std::size_t i = 0; i < name.size(); i++) {
|
||||
if (!is_alnum(name[i])) {
|
||||
name[i] = '_';
|
||||
}
|
||||
@ -115,7 +115,7 @@ class TlWriterDotNet : public TL_writer {
|
||||
if (name == "#") {
|
||||
return "int32_t";
|
||||
}
|
||||
for (int i = 0; i < (int)name.size(); i++) {
|
||||
for (std::size_t i = 0; i < name.size(); i++) {
|
||||
if (!is_alnum(name[i])) {
|
||||
name[i] = '_';
|
||||
}
|
||||
@ -163,7 +163,7 @@ class TlWriterDotNet : public TL_writer {
|
||||
|
||||
if (name == "Vector") {
|
||||
assert(t->arity == 1);
|
||||
assert((int)tree_type->children.size() == 1);
|
||||
assert(tree_type->children.size() == 1);
|
||||
assert(tree_type->children[0]->get_type() == NODE_TYPE_TYPE);
|
||||
const tl_tree_type *child = static_cast<const tl_tree_type *>(tree_type->children[0]);
|
||||
|
||||
@ -172,7 +172,7 @@ class TlWriterDotNet : public TL_writer {
|
||||
|
||||
assert(!is_built_in_simple_type(name) && !is_built_in_complex_type(name));
|
||||
|
||||
for (int i = 0; i < (int)tree_type->children.size(); i++) {
|
||||
for (std::size_t i = 0; i < tree_type->children.size(); i++) {
|
||||
assert(tree_type->children[i]->get_type() == NODE_TYPE_NAT_CONST);
|
||||
}
|
||||
|
||||
|
@ -66,7 +66,9 @@ class AuthKey {
|
||||
auth_key_.clear();
|
||||
}
|
||||
|
||||
enum : int32 { AUTH_FLAG = 1, WAS_AUTH_FLAG = 2, HAS_CREATED_AT = 4 };
|
||||
static constexpr int32 AUTH_FLAG = 1;
|
||||
static constexpr int32 WAS_AUTH_FLAG = 2;
|
||||
static constexpr int32 HAS_CREATED_AT = 4;
|
||||
|
||||
template <class StorerT>
|
||||
void store(StorerT &storer) const {
|
||||
|
@ -29,7 +29,7 @@ Result<size_t> Transport::read_next(BufferSlice *message, uint32 *quick_ack) {
|
||||
if (r_size.is_error() || r_size.ok() != 0) {
|
||||
return r_size;
|
||||
}
|
||||
if (http_query_.type_ != HttpQuery::Type::RESPONSE) {
|
||||
if (http_query_.type_ != HttpQuery::Type::Response) {
|
||||
return Status::Error("Unexpected HTTP query type");
|
||||
}
|
||||
if (http_query_.container_.size() != 2u) {
|
||||
|
@ -19,9 +19,8 @@
|
||||
namespace td {
|
||||
namespace mtproto {
|
||||
|
||||
ActorOwn<> create_ping_actor(std::string debug, unique_ptr<RawConnection> raw_connection,
|
||||
unique_ptr<AuthData> auth_data, Promise<unique_ptr<RawConnection>> promise,
|
||||
ActorShared<> parent) {
|
||||
ActorOwn<> create_ping_actor(string debug, unique_ptr<RawConnection> raw_connection, unique_ptr<AuthData> auth_data,
|
||||
Promise<unique_ptr<RawConnection>> promise, ActorShared<> parent) {
|
||||
class PingActor : public Actor {
|
||||
public:
|
||||
PingActor(unique_ptr<RawConnection> raw_connection, unique_ptr<AuthData> auth_data,
|
||||
|
@ -790,7 +790,7 @@ vector<string> CallActor::get_emojis_fingerprint(const string &key, const string
|
||||
vector<string> result;
|
||||
result.reserve(4);
|
||||
for (int i = 0; i < 4; i++) {
|
||||
uint64 num = bswap64(as<uint64>(sha256_buf + 8 * i));
|
||||
uint64 num = big_endian_to_host64(as<uint64>(sha256_buf + 8 * i));
|
||||
result.push_back(get_emoji_fingerprint(num));
|
||||
}
|
||||
return result;
|
||||
|
@ -113,13 +113,13 @@ class Global : public ActorContext {
|
||||
}
|
||||
|
||||
bool have_net_query_dispatcher() const {
|
||||
return net_query_dispatcher_ != nullptr;
|
||||
return net_query_dispatcher_.get() != nullptr;
|
||||
}
|
||||
|
||||
void set_shared_config(unique_ptr<ConfigShared> shared_config);
|
||||
|
||||
ConfigShared &shared_config() {
|
||||
CHECK(shared_config_ != nullptr);
|
||||
CHECK(shared_config_.get() != nullptr);
|
||||
return *shared_config_;
|
||||
}
|
||||
|
||||
|
@ -10907,18 +10907,17 @@ void MessagesManager::init() {
|
||||
auto *list = get_dialog_list(DialogListId(folder_id));
|
||||
CHECK(list != nullptr);
|
||||
CHECK(list->pinned_dialogs_.empty());
|
||||
if (!r_dialog_ids.empty()) {
|
||||
for (auto &r_dialog_id : reversed(r_dialog_ids)) {
|
||||
auto dialog_id = r_dialog_id.move_as_ok();
|
||||
auto order = get_next_pinned_dialog_order();
|
||||
list->pinned_dialogs_.emplace_back(order, dialog_id);
|
||||
list->pinned_dialog_id_orders_.emplace(dialog_id, order);
|
||||
}
|
||||
std::reverse(list->pinned_dialogs_.begin(), list->pinned_dialogs_.end());
|
||||
list->are_pinned_dialogs_inited_ = true;
|
||||
|
||||
update_list_last_pinned_dialog_date(*list);
|
||||
for (auto &r_dialog_id : reversed(r_dialog_ids)) {
|
||||
auto dialog_id = r_dialog_id.move_as_ok();
|
||||
auto order = get_next_pinned_dialog_order();
|
||||
list->pinned_dialogs_.emplace_back(order, dialog_id);
|
||||
list->pinned_dialog_id_orders_.emplace(dialog_id, order);
|
||||
}
|
||||
std::reverse(list->pinned_dialogs_.begin(), list->pinned_dialogs_.end());
|
||||
list->are_pinned_dialogs_inited_ = true;
|
||||
update_list_last_pinned_dialog_date(*list);
|
||||
|
||||
LOG(INFO) << "Loaded pinned chats " << list->pinned_dialogs_ << " in " << folder_id;
|
||||
}
|
||||
}
|
||||
|
||||
@ -12990,8 +12989,9 @@ void MessagesManager::on_get_dialogs(FolderId folder_id, vector<tl_object_ptr<te
|
||||
}
|
||||
} else if (promise) {
|
||||
LOG(ERROR) << "Last server dialog date didn't increased from " << folder->last_server_dialog_date_ << " to "
|
||||
<< max_dialog_date << " after receiving " << dialogs.size() << " chats from " << total_count << " in "
|
||||
<< folder_id << ". last_dialog_date = " << folder->folder_last_dialog_date_
|
||||
<< max_dialog_date << " after receiving " << dialogs.size() << " chats " << added_dialog_ids
|
||||
<< " from " << total_count << " in " << folder_id
|
||||
<< ". last_dialog_date = " << folder->folder_last_dialog_date_
|
||||
<< ", last_loaded_database_dialog_date = " << folder->last_loaded_database_dialog_date_;
|
||||
}
|
||||
}
|
||||
|
@ -3077,7 +3077,9 @@ Status NotificationManager::process_push_notification_payload(string payload, bo
|
||||
dialog_id = DialogId(secret_chat_id);
|
||||
}
|
||||
if (!dialog_id.is_valid()) {
|
||||
// TODO if (loc_key == "ENCRYPTED_MESSAGE") ?
|
||||
if (loc_key == "ENCRYPTED_MESSAGE" || loc_key == "MESSAGE_MUTED") {
|
||||
return Status::Error(406, "Force loading data from the server");
|
||||
}
|
||||
return Status::Error("Can't find dialog_id");
|
||||
}
|
||||
|
||||
|
@ -47,7 +47,6 @@ class RequestActor : public Actor {
|
||||
}
|
||||
stop();
|
||||
} else {
|
||||
LOG_CHECK(!promise.was_set_value) << future.empty() << " " << future.get_state();
|
||||
CHECK(!future.empty());
|
||||
CHECK(future.get_state() == FutureActor<T>::State::Waiting);
|
||||
if (--tries_left_ == 0) {
|
||||
@ -64,7 +63,7 @@ class RequestActor : public Actor {
|
||||
void raw_event(const Event::Raw &event) override {
|
||||
if (future_.is_error()) {
|
||||
auto error = future_.move_as_error();
|
||||
if (error == Status::Error<FutureActor<T>::Hangup>()) {
|
||||
if (error == Status::Error<FutureActor<T>::HANGUP_ERROR_CODE>()) {
|
||||
// dropping query due to lost authorization or lost promise
|
||||
// td may be already closed, so we should check is auth_manager_ is empty
|
||||
bool is_authorized = td->auth_manager_ && td->auth_manager_->is_authorized();
|
||||
|
@ -23,6 +23,7 @@
|
||||
#include "td/utils/JsonBuilder.h"
|
||||
#include "td/utils/logging.h"
|
||||
#include "td/utils/misc.h"
|
||||
#include "td/utils/OptionParser.h"
|
||||
#include "td/utils/port/FileFd.h"
|
||||
#include "td/utils/port/PollFlags.h"
|
||||
#include "td/utils/port/signals.h"
|
||||
@ -4062,6 +4063,29 @@ class CliClient final : public Actor {
|
||||
fd.write("a").ignore();
|
||||
fd.seek(size).ignore();
|
||||
fd.truncate_to_current_position(size).ignore();
|
||||
} else if (op == "mem") {
|
||||
auto r_mem_stats = mem_stat();
|
||||
if (r_mem_stats.is_error()) {
|
||||
LOG(ERROR) << r_mem_stats.error();
|
||||
} else {
|
||||
auto stats = r_mem_stats.move_as_ok();
|
||||
LOG(ERROR) << "RSS = " << stats.resident_size_ << ", peak RSS = " << stats.resident_size_peak_ << ", VSZ "
|
||||
<< stats.virtual_size_ << ", peak VSZ = " << stats.virtual_size_peak_;
|
||||
}
|
||||
} else if (op == "cpu") {
|
||||
uint32 inc_count = to_integer<uint32>(args);
|
||||
while (inc_count-- > 0) {
|
||||
cpu_counter_++;
|
||||
}
|
||||
auto r_cpu_stats = cpu_stat();
|
||||
if (r_cpu_stats.is_error()) {
|
||||
LOG(ERROR) << r_cpu_stats.error();
|
||||
} else {
|
||||
auto stats = r_cpu_stats.move_as_ok();
|
||||
LOG(ERROR) << cpu_counter_ << ", total ticks = " << stats.total_ticks_
|
||||
<< ", user ticks = " << stats.process_user_ticks_
|
||||
<< ", system ticks = " << stats.process_system_ticks_;
|
||||
}
|
||||
} else if (op == "SetVerbosity" || op == "SV") {
|
||||
Log::set_verbosity_level(to_integer<int>(args));
|
||||
} else if (op[0] == 'v' && op[1] == 'v') {
|
||||
@ -4247,8 +4271,11 @@ class CliClient final : public Actor {
|
||||
bool disable_network_ = false;
|
||||
int api_id_ = 0;
|
||||
std::string api_hash_;
|
||||
|
||||
static std::atomic<uint64> cpu_counter_;
|
||||
};
|
||||
CliClient *CliClient::instance_ = nullptr;
|
||||
std::atomic<uint64> CliClient::cpu_counter_;
|
||||
|
||||
void quit() {
|
||||
CliClient::quit_instance();
|
||||
@ -4261,10 +4288,6 @@ static void fail_signal(int sig) {
|
||||
}
|
||||
}
|
||||
|
||||
static void usage() {
|
||||
//TODO:
|
||||
}
|
||||
|
||||
static void on_fatal_error(const char *error) {
|
||||
std::cerr << "Fatal error: " << error << std::endl;
|
||||
}
|
||||
@ -4307,52 +4330,72 @@ void main(int argc, char **argv) {
|
||||
}
|
||||
return std::string();
|
||||
}(std::getenv("TD_API_HASH"));
|
||||
// TODO port OptionsParser to Windows
|
||||
for (int i = 1; i < argc; i++) {
|
||||
if (!std::strcmp(argv[i], "--test")) {
|
||||
use_test_dc = true;
|
||||
} else if (!std::strncmp(argv[i], "-v", 2)) {
|
||||
const char *arg = argv[i] + 2;
|
||||
if (*arg == '\0' && i + 1 < argc) {
|
||||
arg = argv[++i];
|
||||
}
|
||||
int new_verbosity = 1;
|
||||
while (*arg == 'v') {
|
||||
new_verbosity++;
|
||||
arg++;
|
||||
}
|
||||
if (*arg) {
|
||||
new_verbosity += to_integer<int>(Slice(arg)) - (new_verbosity == 1);
|
||||
}
|
||||
new_verbosity_level = VERBOSITY_NAME(FATAL) + new_verbosity;
|
||||
} else if (!std::strncmp(argv[i], "-l", 2)) {
|
||||
const char *arg = argv[i] + 2;
|
||||
if (*arg == '\0' && i + 1 < argc) {
|
||||
arg = argv[++i];
|
||||
}
|
||||
if (file_log.init(arg).is_ok() && file_log.init(arg).is_ok() && file_log.init(arg, 1000 << 20).is_ok()) {
|
||||
log_interface = &ts_log;
|
||||
}
|
||||
} else if (!std::strcmp(argv[i], "-W")) {
|
||||
get_chat_list = true;
|
||||
} else if (!std::strcmp(argv[i], "--disable-network") || !std::strcmp(argv[i], "-n")) {
|
||||
disable_network = true;
|
||||
} else if (!std::strcmp(argv[i], "--api_id") || !std::strcmp(argv[i], "--api-id")) {
|
||||
if (i + 1 >= argc) {
|
||||
return usage();
|
||||
}
|
||||
api_id = to_integer<int32>(Slice(argv[++i]));
|
||||
} else if (!std::strcmp(argv[i], "--api_hash") || !std::strcmp(argv[i], "--api-hash")) {
|
||||
if (i + 1 >= argc) {
|
||||
return usage();
|
||||
}
|
||||
api_hash = argv[++i];
|
||||
|
||||
td::OptionParser options;
|
||||
options.set_description("TDLib test client");
|
||||
options.add_option('\0', "test", "Use test DC", [&] {
|
||||
use_test_dc = true;
|
||||
return Status::OK();
|
||||
});
|
||||
options.add_option('v', "verbosity", "Set verbosity level", [&](Slice level) {
|
||||
int new_verbosity = 1;
|
||||
while (begins_with(level, "v")) {
|
||||
new_verbosity++;
|
||||
level.remove_prefix(1);
|
||||
}
|
||||
if (!level.empty()) {
|
||||
new_verbosity += to_integer<int>(level) - (new_verbosity == 1);
|
||||
}
|
||||
new_verbosity_level = VERBOSITY_NAME(FATAL) + new_verbosity;
|
||||
return Status::OK();
|
||||
});
|
||||
options.add_option('l', "log", "Log to file", [&](Slice file_name) {
|
||||
if (file_log.init(file_name.str()).is_ok() && file_log.init(file_name.str()).is_ok() &&
|
||||
file_log.init(file_name.str(), 1000 << 20).is_ok()) {
|
||||
log_interface = &ts_log;
|
||||
}
|
||||
return Status::OK();
|
||||
});
|
||||
options.add_option('W', "", "Preload chat list", [&] {
|
||||
get_chat_list = true;
|
||||
return Status::OK();
|
||||
});
|
||||
options.add_option('n', "disable-network", "Disable network", [&] {
|
||||
disable_network = true;
|
||||
return Status::OK();
|
||||
});
|
||||
options.add_option('\0', "api-id", "Set Telegram API ID", [&](Slice parameter) {
|
||||
api_id = to_integer<int32>(parameter);
|
||||
return Status::OK();
|
||||
});
|
||||
options.add_option('\0', "api_id", "Set Telegram API ID", [&](Slice parameter) {
|
||||
api_id = to_integer<int32>(parameter);
|
||||
return Status::OK();
|
||||
});
|
||||
options.add_option('\0', "api-hash", "Set Telegram API hash", [&](Slice parameter) {
|
||||
api_hash = parameter.str();
|
||||
return Status::OK();
|
||||
});
|
||||
options.add_option('\0', "api_hash", "Set Telegram API hash", [&](Slice parameter) {
|
||||
api_hash = parameter.str();
|
||||
return Status::OK();
|
||||
});
|
||||
auto res = options.run(argc, argv);
|
||||
if (res.is_error()) {
|
||||
LOG(PLAIN) << "tg_cli: " << res.error().message();
|
||||
LOG(PLAIN) << options;
|
||||
return;
|
||||
}
|
||||
if (!res.ok().empty()) {
|
||||
LOG(PLAIN) << "tg_cli: " << "Have unexpected non-option parameters";
|
||||
LOG(PLAIN) << options;
|
||||
return;
|
||||
}
|
||||
|
||||
if (api_id == 0 || api_hash.empty()) {
|
||||
LOG(ERROR) << "You should provide some valid api_id and api_hash";
|
||||
return usage();
|
||||
LOG(PLAIN) << "tg_cli: " << "You should provide some valid api_id and api_hash";
|
||||
LOG(PLAIN) << options;
|
||||
return;
|
||||
}
|
||||
|
||||
SET_VERBOSITY_LEVEL(new_verbosity_level);
|
||||
|
@ -77,7 +77,7 @@ class FileDb : public FileDbInterface {
|
||||
}
|
||||
|
||||
void load_file_data(const string &key, Promise<FileData> promise) {
|
||||
promise.set_result(load_file_data_impl(actor_id(this), file_pmc(), key));
|
||||
promise.set_result(load_file_data_impl(actor_id(this), file_pmc(), key, current_pmc_id_));
|
||||
}
|
||||
|
||||
void clear_file_data(FileDbId id, const string &remote_key, const string &local_key, const string &generate_key) {
|
||||
@ -194,7 +194,7 @@ class FileDb : public FileDbInterface {
|
||||
}
|
||||
|
||||
Result<FileData> get_file_data_sync_impl(string key) override {
|
||||
return load_file_data_impl(file_db_actor_.get(), file_kv_safe_->get(), key);
|
||||
return load_file_data_impl(file_db_actor_.get(), file_kv_safe_->get(), key, current_pmc_id_);
|
||||
}
|
||||
|
||||
void clear_file_data(FileDbId id, const FileData &file_data) override {
|
||||
@ -247,7 +247,7 @@ class FileDb : public FileDbInterface {
|
||||
std::shared_ptr<SqliteKeyValueSafe> file_kv_safe_;
|
||||
|
||||
static Result<FileData> load_file_data_impl(ActorId<FileDbActor> file_db_actor_id, SqliteKeyValue &pmc,
|
||||
const string &key) {
|
||||
const string &key, FileDbId current_pmc_id) {
|
||||
//LOG(DEBUG) << "Load by key " << format::as_hex_dump<4>(Slice(key));
|
||||
TRY_RESULT(id, get_id(pmc, key));
|
||||
|
||||
@ -256,7 +256,8 @@ class FileDb : public FileDbInterface {
|
||||
int attempt_count = 0;
|
||||
while (true) {
|
||||
if (attempt_count > 100) {
|
||||
LOG(FATAL) << "Cycle in file database?";
|
||||
LOG(FATAL) << "Cycle in file database? current_pmc_id=" << current_pmc_id << " key=" << key
|
||||
<< " links=" << format::as_array(ids);
|
||||
}
|
||||
attempt_count++;
|
||||
|
||||
|
@ -51,4 +51,8 @@ class FileDbId {
|
||||
}
|
||||
};
|
||||
|
||||
inline StringBuilder &operator<<(StringBuilder &sb, const FileDbId &id) {
|
||||
return sb << "FileDbId{" << id.get() << "}";
|
||||
}
|
||||
|
||||
} // namespace td
|
||||
|
@ -65,7 +65,7 @@ class SecretChatLogEventBase : public SecretChatEvent {
|
||||
// inputEncryptedFile#5a17b5e5 id:long access_hash:long = InputEncryptedFile;
|
||||
// inputEncryptedFileBigUploaded#2dc173c8 id:long parts:int key_fingerprint:int = InputEncryptedFile;
|
||||
struct EncryptedInputFile {
|
||||
static constexpr int32 magic = 0x4328d38a;
|
||||
static constexpr int32 MAGIC = 0x4328d38a;
|
||||
enum Type : int32 { Empty = 0, Uploaded = 1, BigUploaded = 2, Location = 3 } type = Type::Empty;
|
||||
int64 id = 0;
|
||||
int64 access_hash = 0;
|
||||
@ -75,7 +75,7 @@ struct EncryptedInputFile {
|
||||
template <class StorerT>
|
||||
void store(StorerT &storer) const {
|
||||
using td::store;
|
||||
store(magic, storer);
|
||||
store(MAGIC, storer);
|
||||
store(type, storer);
|
||||
store(id, storer);
|
||||
store(access_hash, storer);
|
||||
@ -104,7 +104,7 @@ struct EncryptedInputFile {
|
||||
parse(parts, parser);
|
||||
parse(key_fingerprint, parser);
|
||||
|
||||
if (got_magic != magic) {
|
||||
if (got_magic != MAGIC) {
|
||||
parser.set_error("EncryptedInputFile magic mismatch");
|
||||
return;
|
||||
}
|
||||
@ -156,7 +156,7 @@ inline StringBuilder &operator<<(StringBuilder &sb, const EncryptedInputFile &fi
|
||||
|
||||
// encryptedFile#4a70994c id:long access_hash:long size:int dc_id:int key_fingerprint:int = EncryptedFile;
|
||||
struct EncryptedFileLocation {
|
||||
static constexpr int32 magic = 0x473d738a;
|
||||
static constexpr int32 MAGIC = 0x473d738a;
|
||||
int64 id = 0;
|
||||
int64 access_hash = 0;
|
||||
int32 size = 0;
|
||||
@ -169,7 +169,7 @@ struct EncryptedFileLocation {
|
||||
template <class StorerT>
|
||||
void store(StorerT &storer) const {
|
||||
using td::store;
|
||||
store(magic, storer);
|
||||
store(MAGIC, storer);
|
||||
store(id, storer);
|
||||
store(access_hash, storer);
|
||||
store(size, storer);
|
||||
@ -189,7 +189,7 @@ struct EncryptedFileLocation {
|
||||
parse(dc_id, parser);
|
||||
parse(key_fingerprint, parser);
|
||||
|
||||
if (got_magic != magic) {
|
||||
if (got_magic != MAGIC) {
|
||||
parser.set_error("EncryptedFileLocation magic mismatch");
|
||||
return;
|
||||
}
|
||||
|
@ -332,7 +332,7 @@ Result<string> check_url(Slice url) {
|
||||
}
|
||||
TRY_RESULT(http_url, parse_url(url));
|
||||
if (is_tg || is_ton) {
|
||||
if (tolower_begins_with(url, "http://") || http_url.protocol_ == HttpUrl::Protocol::HTTPS ||
|
||||
if (tolower_begins_with(url, "http://") || http_url.protocol_ == HttpUrl::Protocol::Https ||
|
||||
!http_url.userinfo_.empty() || http_url.specified_port_ != 0 || http_url.is_ipv6_) {
|
||||
return Status::Error(is_tg ? Slice("Wrong tg URL") : Slice("Wrong ton URL"));
|
||||
}
|
||||
|
@ -327,11 +327,11 @@ void ConnectionCreator::ping_proxy(int32 proxy_id, Promise<double> promise) {
|
||||
continue;
|
||||
}
|
||||
|
||||
ping_proxy_socket_fd(r_socket_fd.move_as_ok(), r_transport_type.move_as_ok(),
|
||||
PromiseCreator::lambda([actor_id = actor_id(this), token](Result<double> result) {
|
||||
send_closure(actor_id, &ConnectionCreator::on_ping_main_dc_result, token,
|
||||
std::move(result));
|
||||
}));
|
||||
ping_proxy_socket_fd(
|
||||
r_socket_fd.move_as_ok(), r_transport_type.move_as_ok(), PSTRING() << info.option->get_ip_address(),
|
||||
PromiseCreator::lambda([actor_id = actor_id(this), token](Result<double> result) {
|
||||
send_closure(actor_id, &ConnectionCreator::on_ping_main_dc_result, token, std::move(result));
|
||||
}));
|
||||
}
|
||||
return;
|
||||
}
|
||||
@ -367,14 +367,14 @@ void ConnectionCreator::ping_proxy_resolved(int32 proxy_id, IPAddress ip_address
|
||||
}
|
||||
auto socket_fd = r_socket_fd.move_as_ok();
|
||||
|
||||
auto connection_promise =
|
||||
PromiseCreator::lambda([promise = std::move(promise), actor_id = actor_id(this),
|
||||
transport_type = extra.transport_type](Result<ConnectionData> r_connection_data) mutable {
|
||||
auto connection_promise = PromiseCreator::lambda(
|
||||
[promise = std::move(promise), actor_id = actor_id(this), transport_type = extra.transport_type,
|
||||
debug_str = std::move(extra.debug_str)](Result<ConnectionData> r_connection_data) mutable {
|
||||
if (r_connection_data.is_error()) {
|
||||
return promise.set_error(Status::Error(400, r_connection_data.error().public_message()));
|
||||
}
|
||||
send_closure(actor_id, &ConnectionCreator::ping_proxy_socket_fd, r_connection_data.move_as_ok().socket_fd,
|
||||
std::move(transport_type), std::move(promise));
|
||||
std::move(transport_type), std::move(debug_str), std::move(promise));
|
||||
});
|
||||
CHECK(proxy.use_proxy());
|
||||
auto token = next_token();
|
||||
@ -387,11 +387,11 @@ void ConnectionCreator::ping_proxy_resolved(int32 proxy_id, IPAddress ip_address
|
||||
}
|
||||
|
||||
void ConnectionCreator::ping_proxy_socket_fd(SocketFd socket_fd, mtproto::TransportType transport_type,
|
||||
Promise<double> promise) {
|
||||
string debug_str, Promise<double> promise) {
|
||||
auto token = next_token();
|
||||
auto raw_connection = make_unique<mtproto::RawConnection>(std::move(socket_fd), std::move(transport_type), nullptr);
|
||||
children_[token] = {
|
||||
false, create_ping_actor("", std::move(raw_connection), nullptr,
|
||||
false, create_ping_actor(std::move(debug_str), std::move(raw_connection), nullptr,
|
||||
PromiseCreator::lambda([promise = std::move(promise)](
|
||||
Result<unique_ptr<mtproto::RawConnection>> result) mutable {
|
||||
if (result.is_error()) {
|
||||
|
@ -243,7 +243,8 @@ class ConnectionCreator : public NetQueryCallback {
|
||||
|
||||
void ping_proxy_resolved(int32 proxy_id, IPAddress ip_address, Promise<double> promise);
|
||||
|
||||
void ping_proxy_socket_fd(SocketFd socket_fd, mtproto::TransportType transport_type, Promise<double> promise);
|
||||
void ping_proxy_socket_fd(SocketFd socket_fd, mtproto::TransportType transport_type, string debug_str,
|
||||
Promise<double> promise);
|
||||
|
||||
void on_ping_main_dc_result(uint64 token, Result<double> result);
|
||||
};
|
||||
|
@ -86,7 +86,7 @@ void NetStatsManager::get_network_stats(bool current, Promise<NetworkStats> prom
|
||||
if (id == 0) {
|
||||
} else if (id == 1) {
|
||||
total = stats;
|
||||
} else if (id == call_net_stats_id_) {
|
||||
} else if (id == CALL_NET_STATS_ID) {
|
||||
} else if (file_type != FileType::None) {
|
||||
total_files = total_files + stats;
|
||||
}
|
||||
@ -109,7 +109,7 @@ void NetStatsManager::get_network_stats(bool current, Promise<NetworkStats> prom
|
||||
entry.duration = stats.duration;
|
||||
if (id == 0) {
|
||||
result.entries.push_back(std::move(entry));
|
||||
} else if (id == call_net_stats_id_) {
|
||||
} else if (id == CALL_NET_STATS_ID) {
|
||||
entry.is_call = true;
|
||||
result.entries.push_back(std::move(entry));
|
||||
} else if (file_type != FileType::None) {
|
||||
|
@ -119,7 +119,7 @@ class NetStatsManager : public Actor {
|
||||
NetStatsInfo media_net_stats_;
|
||||
std::array<NetStatsInfo, file_type_size> files_stats_;
|
||||
NetStatsInfo call_net_stats_;
|
||||
static constexpr int32 call_net_stats_id_{file_type_size + 2};
|
||||
static constexpr int32 CALL_NET_STATS_ID{file_type_size + 2};
|
||||
|
||||
template <class F>
|
||||
void for_each_stat(F &&f) {
|
||||
@ -130,7 +130,7 @@ class NetStatsManager : public Actor {
|
||||
auto file_type = static_cast<FileType>(file_type_i);
|
||||
f(stat, file_type_i + 2, get_file_type_name(file_type), file_type);
|
||||
}
|
||||
f(call_net_stats_, call_net_stats_id_, CSlice("calls"), FileType::None);
|
||||
f(call_net_stats_, CALL_NET_STATS_ID, CSlice("calls"), FileType::None);
|
||||
}
|
||||
|
||||
void add_network_stats_impl(NetStatsInfo &info, const NetworkStatsEntry &entry);
|
||||
|
@ -62,12 +62,10 @@ class SafePromise;
|
||||
template <class T = Unit>
|
||||
class Promise {
|
||||
public:
|
||||
bool was_set_value{false};
|
||||
void set_value(T &&value) {
|
||||
if (!promise_) {
|
||||
return;
|
||||
}
|
||||
was_set_value = true;
|
||||
promise_->set_value(std::move(value));
|
||||
promise_.reset();
|
||||
}
|
||||
@ -75,7 +73,6 @@ class Promise {
|
||||
if (!promise_) {
|
||||
return;
|
||||
}
|
||||
was_set_value = true;
|
||||
promise_->set_error(std::move(error));
|
||||
promise_.reset();
|
||||
}
|
||||
@ -83,7 +80,6 @@ class Promise {
|
||||
if (!promise_) {
|
||||
return;
|
||||
}
|
||||
was_set_value = true;
|
||||
promise_->set_result(std::move(result));
|
||||
promise_.reset();
|
||||
}
|
||||
@ -412,7 +408,7 @@ class FutureActor final : public Actor {
|
||||
public:
|
||||
enum State { Waiting, Ready };
|
||||
|
||||
static constexpr int Hangup = 426487;
|
||||
static constexpr int HANGUP_ERROR_CODE = 426487;
|
||||
|
||||
FutureActor() = default;
|
||||
|
||||
@ -491,7 +487,7 @@ class FutureActor final : public Actor {
|
||||
}
|
||||
|
||||
void hangup() override {
|
||||
set_error(Status::Error<Hangup>());
|
||||
set_error(Status::Error<HANGUP_ERROR_CODE>());
|
||||
}
|
||||
|
||||
void start_up() override {
|
||||
|
@ -17,6 +17,7 @@ set(TDDB_SOURCE
|
||||
td/db/SqliteKeyValue.cpp
|
||||
td/db/SqliteKeyValueAsync.cpp
|
||||
td/db/SqliteStatement.cpp
|
||||
td/db/TQueue.cpp
|
||||
|
||||
td/db/detail/RawSqliteDb.cpp
|
||||
|
||||
@ -31,7 +32,6 @@ set(TDDB_SOURCE
|
||||
td/db/BinlogKeyValue.h
|
||||
td/db/DbKey.h
|
||||
td/db/KeyValueSyncInterface.h
|
||||
td/db/Pmc.h
|
||||
td/db/SeqKeyValue.h
|
||||
td/db/SqliteConnectionSafe.h
|
||||
td/db/SqliteDb.h
|
||||
@ -39,6 +39,7 @@ set(TDDB_SOURCE
|
||||
td/db/SqliteKeyValueAsync.h
|
||||
td/db/SqliteKeyValueSafe.h
|
||||
td/db/SqliteStatement.h
|
||||
td/db/TQueue.h
|
||||
td/db/TsSeqKeyValue.h
|
||||
|
||||
td/db/detail/RawSqliteDb.h
|
||||
|
@ -34,7 +34,7 @@ namespace td {
|
||||
template <class BinlogT>
|
||||
class BinlogKeyValue : public KeyValueSyncInterface {
|
||||
public:
|
||||
static constexpr int32 magic = 0x2a280000;
|
||||
static constexpr int32 MAGIC = 0x2a280000;
|
||||
|
||||
struct Event : public Storer {
|
||||
Event() = default;
|
||||
@ -236,7 +236,7 @@ class BinlogKeyValue : public KeyValueSyncInterface {
|
||||
std::unordered_map<string, std::pair<string, uint64>> map_;
|
||||
std::shared_ptr<BinlogT> binlog_;
|
||||
RwMutex rw_mutex_;
|
||||
int32 magic_ = magic;
|
||||
int32 magic_ = MAGIC;
|
||||
};
|
||||
|
||||
template <>
|
||||
|
432
tddb/td/db/TQueue.cpp
Normal file
432
tddb/td/db/TQueue.cpp
Normal file
@ -0,0 +1,432 @@
|
||||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#include "td/db/TQueue.h"
|
||||
|
||||
#include "td/db/binlog/Binlog.h"
|
||||
#include "td/db/binlog/BinlogEvent.h"
|
||||
#include "td/db/binlog/BinlogHelper.h"
|
||||
#include "td/db/binlog/BinlogInterface.h"
|
||||
|
||||
#include "td/utils/format.h"
|
||||
#include "td/utils/misc.h"
|
||||
#include "td/utils/port/Clocks.h"
|
||||
#include "td/utils/Random.h"
|
||||
#include "td/utils/StorerBase.h"
|
||||
#include "td/utils/Time.h"
|
||||
#include "td/utils/tl_helpers.h"
|
||||
#include "td/utils/tl_parsers.h"
|
||||
#include "td/utils/tl_storers.h"
|
||||
#include "td/utils/VectorQueue.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
|
||||
namespace td {
|
||||
|
||||
using EventId = TQueue::EventId;
|
||||
|
||||
EventId::EventId() {
|
||||
}
|
||||
|
||||
Result<EventId> EventId::from_int32(int32 id) {
|
||||
if (!is_valid_id(id)) {
|
||||
return Status::Error("Invalid ID");
|
||||
}
|
||||
return EventId(id);
|
||||
}
|
||||
|
||||
bool EventId::is_valid() const {
|
||||
return !empty() && is_valid_id(id_);
|
||||
}
|
||||
|
||||
int32 EventId::value() const {
|
||||
return id_;
|
||||
}
|
||||
|
||||
Result<EventId> EventId::next() const {
|
||||
return from_int32(id_ + 1);
|
||||
}
|
||||
|
||||
Result<EventId> EventId::advance(size_t offset) const {
|
||||
TRY_RESULT(new_id, narrow_cast_safe<int32>(id_ + offset));
|
||||
return from_int32(new_id);
|
||||
}
|
||||
|
||||
bool EventId::empty() const {
|
||||
return id_ == 0;
|
||||
}
|
||||
|
||||
bool EventId::operator==(const EventId &other) const {
|
||||
return id_ == other.id_;
|
||||
}
|
||||
|
||||
bool EventId::operator!=(const EventId &other) const {
|
||||
return !(*this == other);
|
||||
}
|
||||
|
||||
bool EventId::operator<(const EventId &other) const {
|
||||
return id_ < other.id_;
|
||||
}
|
||||
|
||||
StringBuilder &operator<<(StringBuilder &string_builder, const EventId id) {
|
||||
return string_builder << "EventId{" << id.value() << "}";
|
||||
}
|
||||
|
||||
EventId::EventId(int32 id) : id_(id) {
|
||||
CHECK(is_valid_id(id));
|
||||
}
|
||||
|
||||
bool EventId::is_valid_id(int32 id) {
|
||||
return 0 <= id && id < MAX_ID;
|
||||
}
|
||||
|
||||
class TQueueImpl : public TQueue {
|
||||
static constexpr size_t MAX_EVENT_LEN = 65536 * 8;
|
||||
static constexpr size_t MAX_QUEUE_EVENTS = 1000000;
|
||||
|
||||
public:
|
||||
void set_callback(unique_ptr<StorageCallback> callback) override {
|
||||
callback_ = std::move(callback);
|
||||
}
|
||||
unique_ptr<StorageCallback> extract_callback() override {
|
||||
return std::move(callback_);
|
||||
}
|
||||
|
||||
bool do_push(QueueId queue_id, RawEvent &&raw_event) override {
|
||||
CHECK(raw_event.event_id.is_valid());
|
||||
auto &q = queues_[queue_id];
|
||||
if (q.events.empty() || q.events.back().event_id < raw_event.event_id) {
|
||||
if (raw_event.logevent_id == 0 && callback_ != nullptr) {
|
||||
raw_event.logevent_id = callback_->push(queue_id, raw_event);
|
||||
}
|
||||
q.tail_id = raw_event.event_id.next().move_as_ok();
|
||||
q.events.push(std::move(raw_event));
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
Result<EventId> push(QueueId queue_id, string data, double expires_at, int64 extra, EventId hint_new_id) override {
|
||||
auto &q = queues_[queue_id];
|
||||
if (q.events.size() >= MAX_QUEUE_EVENTS) {
|
||||
return Status::Error("Queue is full");
|
||||
}
|
||||
if (data.empty()) {
|
||||
return Status::Error("Data is empty");
|
||||
}
|
||||
if (data.size() > MAX_EVENT_LEN) {
|
||||
return Status::Error("Data is too big");
|
||||
}
|
||||
EventId event_id;
|
||||
while (true) {
|
||||
if (q.tail_id.empty()) {
|
||||
if (hint_new_id.empty()) {
|
||||
q.tail_id = EventId::from_int32(Random::fast(2 * MAX_QUEUE_EVENTS + 1, EventId::MAX_ID / 2)).move_as_ok();
|
||||
} else {
|
||||
q.tail_id = hint_new_id;
|
||||
}
|
||||
}
|
||||
event_id = q.tail_id;
|
||||
CHECK(event_id.is_valid());
|
||||
if (event_id.next().is_ok()) {
|
||||
break;
|
||||
}
|
||||
for (auto &event : q.events.as_mutable_span()) {
|
||||
pop(queue_id, event, {});
|
||||
}
|
||||
q.tail_id = EventId();
|
||||
q.events = {};
|
||||
CHECK(hint_new_id.next().is_ok());
|
||||
}
|
||||
|
||||
RawEvent raw_event;
|
||||
raw_event.event_id = event_id;
|
||||
raw_event.data = std::move(data);
|
||||
raw_event.expires_at = expires_at;
|
||||
raw_event.extra = extra;
|
||||
bool is_added = do_push(queue_id, std::move(raw_event));
|
||||
CHECK(is_added);
|
||||
return event_id;
|
||||
}
|
||||
|
||||
EventId get_head(QueueId queue_id) const override {
|
||||
auto it = queues_.find(queue_id);
|
||||
if (it == queues_.end()) {
|
||||
return EventId();
|
||||
}
|
||||
auto &q = it->second;
|
||||
if (q.events.empty()) {
|
||||
return q.tail_id;
|
||||
}
|
||||
return q.events.front().event_id;
|
||||
}
|
||||
|
||||
EventId get_tail(QueueId queue_id) const override {
|
||||
auto it = queues_.find(queue_id);
|
||||
if (it == queues_.end()) {
|
||||
return EventId();
|
||||
}
|
||||
auto &q = it->second;
|
||||
return q.tail_id;
|
||||
}
|
||||
|
||||
void forget(QueueId queue_id, EventId event_id) override {
|
||||
auto q_it = queues_.find(queue_id);
|
||||
if (q_it == queues_.end()) {
|
||||
return;
|
||||
}
|
||||
auto &q = q_it->second;
|
||||
auto from_events = q.events.as_mutable_span();
|
||||
auto it = std::lower_bound(from_events.begin(), from_events.end(), event_id,
|
||||
[](auto &event, EventId event_id) { return event.event_id < event_id; });
|
||||
if (it == from_events.end() || !(it->event_id == event_id)) {
|
||||
return;
|
||||
}
|
||||
pop(queue_id, *it, q.tail_id);
|
||||
}
|
||||
|
||||
Result<size_t> get(QueueId queue_id, EventId from_id, bool forget_previous, double now,
|
||||
MutableSpan<Event> &result_events) override {
|
||||
auto it = queues_.find(queue_id);
|
||||
if (it == queues_.end()) {
|
||||
result_events.truncate(0);
|
||||
return 0;
|
||||
}
|
||||
auto &q = it->second;
|
||||
// Some sanity checks
|
||||
if (from_id.value() > q.tail_id.value() + 10) {
|
||||
return Status::Error("Specified from_id is in the future");
|
||||
}
|
||||
if (from_id.value() < q.tail_id.value() - static_cast<int32>(MAX_QUEUE_EVENTS) * 2) {
|
||||
return Status::Error("Specified from_id is in the past");
|
||||
}
|
||||
|
||||
MutableSpan<RawEvent> from_events;
|
||||
size_t ready_n = 0;
|
||||
size_t i = 0;
|
||||
|
||||
while (true) {
|
||||
from_events = q.events.as_mutable_span();
|
||||
ready_n = 0;
|
||||
size_t first_i = 0;
|
||||
if (!forget_previous) {
|
||||
first_i = std::lower_bound(from_events.begin(), from_events.end(), from_id,
|
||||
[](auto &event, EventId event_id) { return event.event_id < event_id; }) -
|
||||
from_events.begin();
|
||||
}
|
||||
for (i = first_i; i < from_events.size(); i++) {
|
||||
auto &from = from_events[i];
|
||||
try_pop(queue_id, from, forget_previous ? from_id : EventId{}, q.tail_id, now);
|
||||
if (from.data.empty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ready_n == result_events.size()) {
|
||||
break;
|
||||
}
|
||||
|
||||
CHECK(!(from.event_id < from_id));
|
||||
|
||||
auto &to = result_events[ready_n];
|
||||
to.data = from.data;
|
||||
to.id = from.event_id;
|
||||
to.expires_at = from.expires_at;
|
||||
to.extra = from.extra;
|
||||
ready_n++;
|
||||
}
|
||||
|
||||
// compactify skipped events
|
||||
if ((ready_n + 1) * 2 < i + first_i) {
|
||||
compactify(q.events, i);
|
||||
continue;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
result_events.truncate(ready_n);
|
||||
size_t left_n = from_events.size() - i;
|
||||
return ready_n + left_n;
|
||||
}
|
||||
|
||||
void run_gc(double now) override {
|
||||
for (auto &it : queues_) {
|
||||
for (auto &e : it.second.events.as_mutable_span()) {
|
||||
try_pop(it.first, e, EventId(), it.second.tail_id, now);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
struct Queue {
|
||||
EventId tail_id;
|
||||
VectorQueue<RawEvent> events;
|
||||
};
|
||||
|
||||
std::unordered_map<QueueId, Queue> queues_;
|
||||
unique_ptr<StorageCallback> callback_;
|
||||
|
||||
static void compactify(VectorQueue<RawEvent> &events, size_t prefix) {
|
||||
if (prefix == events.size()) {
|
||||
CHECK(!events.empty());
|
||||
prefix--;
|
||||
}
|
||||
auto processed = events.as_mutable_span().substr(0, prefix);
|
||||
auto removed_n =
|
||||
processed.rend() - std::remove_if(processed.rbegin(), processed.rend(), [](auto &e) { return e.data.empty(); });
|
||||
events.pop_n(removed_n);
|
||||
}
|
||||
|
||||
void try_pop(QueueId queue_id, RawEvent &event, EventId from_id, EventId tail_id, double now) {
|
||||
if (event.expires_at < now || event.event_id < from_id || event.data.empty()) {
|
||||
pop(queue_id, event, tail_id);
|
||||
}
|
||||
}
|
||||
|
||||
void pop(QueueId queue_id, RawEvent &event, EventId tail_id) {
|
||||
if (callback_ == nullptr || event.logevent_id == 0) {
|
||||
event.logevent_id = 0;
|
||||
event.data = {};
|
||||
return;
|
||||
}
|
||||
|
||||
if (event.event_id.next().ok() == tail_id) {
|
||||
if (!event.data.empty()) {
|
||||
event.data = {};
|
||||
callback_->push(queue_id, event);
|
||||
}
|
||||
} else {
|
||||
callback_->pop(event.logevent_id);
|
||||
event.logevent_id = 0;
|
||||
event.data = {};
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
unique_ptr<TQueue> TQueue::create() {
|
||||
return make_unique<TQueueImpl>();
|
||||
}
|
||||
|
||||
struct TQueueLogEvent : public Storer {
|
||||
int64 queue_id;
|
||||
int32 event_id;
|
||||
int32 expires_at;
|
||||
Slice data;
|
||||
int64 extra;
|
||||
|
||||
template <class StorerT>
|
||||
void store(StorerT &&storer) const {
|
||||
using td::store;
|
||||
store(queue_id, storer);
|
||||
store(event_id, storer);
|
||||
store(expires_at, storer);
|
||||
store(data, storer);
|
||||
if (extra != 0) {
|
||||
store(extra, storer);
|
||||
}
|
||||
}
|
||||
|
||||
template <class ParserT>
|
||||
void parse(ParserT &&parser, int32 has_extra) {
|
||||
using td::parse;
|
||||
parse(queue_id, parser);
|
||||
parse(event_id, parser);
|
||||
parse(expires_at, parser);
|
||||
data = parser.template fetch_string<Slice>();
|
||||
if (has_extra == 0) {
|
||||
extra = 0;
|
||||
} else {
|
||||
parse(extra, parser);
|
||||
}
|
||||
}
|
||||
|
||||
size_t size() const override {
|
||||
TlStorerCalcLength storer;
|
||||
store(storer);
|
||||
return storer.get_length();
|
||||
}
|
||||
|
||||
size_t store(uint8 *ptr) const override {
|
||||
TlStorerUnsafe storer(ptr);
|
||||
store(storer);
|
||||
return static_cast<size_t>(storer.get_buf() - ptr);
|
||||
}
|
||||
};
|
||||
|
||||
template <class BinlogT>
|
||||
TQueueBinlog<BinlogT>::TQueueBinlog() {
|
||||
diff_ = Clocks::system() - Time::now();
|
||||
}
|
||||
|
||||
template <class BinlogT>
|
||||
uint64 TQueueBinlog<BinlogT>::push(QueueId queue_id, const RawEvent &event) {
|
||||
TQueueLogEvent log_event;
|
||||
log_event.queue_id = queue_id;
|
||||
log_event.event_id = event.event_id.value();
|
||||
log_event.expires_at = static_cast<int32>(event.expires_at + diff_ + 1);
|
||||
log_event.data = event.data;
|
||||
log_event.extra = event.extra;
|
||||
auto magic = magic_ + (log_event.extra != 0);
|
||||
if (event.logevent_id == 0) {
|
||||
return binlog_->add(magic, log_event);
|
||||
}
|
||||
binlog_->rewrite(event.logevent_id, magic, log_event);
|
||||
return event.logevent_id;
|
||||
}
|
||||
|
||||
template <class BinlogT>
|
||||
void TQueueBinlog<BinlogT>::pop(uint64 logevent_id) {
|
||||
binlog_->erase(logevent_id);
|
||||
}
|
||||
|
||||
template <class BinlogT>
|
||||
Status TQueueBinlog<BinlogT>::replay(const BinlogEvent &binlog_event, TQueue &q) const {
|
||||
TQueueLogEvent event;
|
||||
TlParser parser(binlog_event.data_);
|
||||
int32 has_extra = binlog_event.type_ - magic_;
|
||||
if (has_extra != 0 && has_extra != 1) {
|
||||
return Status::Error("Wrong magic");
|
||||
}
|
||||
event.parse(parser, has_extra);
|
||||
parser.fetch_end();
|
||||
TRY_STATUS(parser.get_status());
|
||||
TRY_RESULT(event_id, EventId::from_int32(event.event_id));
|
||||
RawEvent raw_event;
|
||||
raw_event.logevent_id = binlog_event.id_;
|
||||
raw_event.event_id = event_id;
|
||||
raw_event.expires_at = event.expires_at - diff_;
|
||||
raw_event.data = event.data.str();
|
||||
raw_event.extra = event.extra;
|
||||
if (!q.do_push(event.queue_id, std::move(raw_event))) {
|
||||
return Status::Error("Failed to add event");
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
template class TQueueBinlog<BinlogInterface>;
|
||||
template class TQueueBinlog<Binlog>;
|
||||
|
||||
uint64 TQueueMemoryStorage::push(QueueId queue_id, const RawEvent &event) {
|
||||
auto logevent_id = event.logevent_id == 0 ? next_logevent_id_++ : event.logevent_id;
|
||||
events_[logevent_id] = std::make_pair(queue_id, event);
|
||||
return logevent_id;
|
||||
}
|
||||
|
||||
void TQueueMemoryStorage::pop(uint64 logevent_id) {
|
||||
events_.erase(logevent_id);
|
||||
}
|
||||
|
||||
void TQueueMemoryStorage::replay(TQueue &q) const {
|
||||
for (auto &e : events_) {
|
||||
auto x = e.second;
|
||||
x.second.logevent_id = e.first;
|
||||
bool is_added = q.do_push(x.first, std::move(x.second));
|
||||
CHECK(is_added);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace td
|
148
tddb/td/db/TQueue.h
Normal file
148
tddb/td/db/TQueue.h
Normal file
@ -0,0 +1,148 @@
|
||||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#include "td/utils/common.h"
|
||||
#include "td/utils/Slice.h"
|
||||
#include "td/utils/Span.h"
|
||||
#include "td/utils/Status.h"
|
||||
#include "td/utils/StringBuilder.h"
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
|
||||
namespace td {
|
||||
|
||||
class TQueue {
|
||||
public:
|
||||
class EventId {
|
||||
public:
|
||||
static constexpr int32 MAX_ID = 2000000000;
|
||||
|
||||
EventId();
|
||||
|
||||
static Result<EventId> from_int32(int32 id);
|
||||
|
||||
bool is_valid() const;
|
||||
|
||||
int32 value() const;
|
||||
|
||||
Result<EventId> next() const;
|
||||
|
||||
Result<EventId> advance(size_t offset) const;
|
||||
|
||||
bool empty() const;
|
||||
|
||||
bool operator==(const EventId &other) const;
|
||||
bool operator!=(const EventId &other) const;
|
||||
bool operator<(const EventId &other) const;
|
||||
|
||||
private:
|
||||
int32 id_{0};
|
||||
|
||||
explicit EventId(int32 id);
|
||||
|
||||
static bool is_valid_id(int32 id);
|
||||
};
|
||||
|
||||
struct Event {
|
||||
EventId id;
|
||||
Slice data;
|
||||
int64 extra{0};
|
||||
double expires_at{0};
|
||||
};
|
||||
|
||||
struct RawEvent {
|
||||
uint64 logevent_id{0};
|
||||
EventId event_id;
|
||||
string data;
|
||||
int64 extra{0};
|
||||
double expires_at{0};
|
||||
};
|
||||
|
||||
using QueueId = int64;
|
||||
|
||||
class StorageCallback {
|
||||
public:
|
||||
using QueueId = TQueue::QueueId;
|
||||
using RawEvent = TQueue::RawEvent;
|
||||
|
||||
StorageCallback() = default;
|
||||
StorageCallback(const StorageCallback &) = delete;
|
||||
StorageCallback &operator=(const StorageCallback &) = delete;
|
||||
StorageCallback(StorageCallback &&) = delete;
|
||||
StorageCallback &operator=(StorageCallback &&) = delete;
|
||||
virtual ~StorageCallback() = default;
|
||||
|
||||
virtual uint64 push(QueueId queue_id, const RawEvent &event) = 0;
|
||||
virtual void pop(uint64 logevent_id) = 0;
|
||||
};
|
||||
|
||||
static unique_ptr<TQueue> create();
|
||||
|
||||
TQueue() = default;
|
||||
TQueue(const TQueue &) = delete;
|
||||
TQueue &operator=(const TQueue &) = delete;
|
||||
TQueue(TQueue &&) = delete;
|
||||
TQueue &operator=(TQueue &&) = delete;
|
||||
|
||||
virtual ~TQueue() = default;
|
||||
|
||||
virtual void set_callback(unique_ptr<StorageCallback> callback) = 0;
|
||||
virtual unique_ptr<StorageCallback> extract_callback() = 0;
|
||||
|
||||
virtual bool do_push(QueueId queue_id, RawEvent &&raw_event) = 0;
|
||||
|
||||
virtual Result<EventId> push(QueueId queue_id, string data, double expires_at, int64 extra, EventId hint_new_id) = 0;
|
||||
|
||||
virtual void forget(QueueId queue_id, EventId event_id) = 0;
|
||||
|
||||
virtual EventId get_head(QueueId queue_id) const = 0;
|
||||
virtual EventId get_tail(QueueId queue_id) const = 0;
|
||||
|
||||
virtual Result<size_t> get(QueueId queue_id, EventId from_id, bool forget_previous, double now,
|
||||
MutableSpan<Event> &result_events) = 0;
|
||||
|
||||
virtual void run_gc(double now) = 0;
|
||||
};
|
||||
|
||||
StringBuilder &operator<<(StringBuilder &string_builder, const TQueue::EventId id);
|
||||
|
||||
struct BinlogEvent;
|
||||
|
||||
template <class BinlogT>
|
||||
class TQueueBinlog : public TQueue::StorageCallback {
|
||||
public:
|
||||
TQueueBinlog();
|
||||
|
||||
uint64 push(QueueId queue_id, const RawEvent &event) override;
|
||||
void pop(uint64 logevent_id) override;
|
||||
Status replay(const BinlogEvent &binlog_event, TQueue &q) const;
|
||||
|
||||
void set_binlog(std::shared_ptr<BinlogT> binlog) {
|
||||
binlog_ = std::move(binlog);
|
||||
}
|
||||
|
||||
private:
|
||||
std::shared_ptr<BinlogT> binlog_;
|
||||
int32 magic_{2314};
|
||||
double diff_{0};
|
||||
};
|
||||
|
||||
class TQueueMemoryStorage : public TQueue::StorageCallback {
|
||||
public:
|
||||
uint64 push(QueueId queue_id, const RawEvent &event) override;
|
||||
void pop(uint64 logevent_id) override;
|
||||
void replay(TQueue &q) const;
|
||||
|
||||
private:
|
||||
uint64 next_logevent_id_{1};
|
||||
std::map<uint64, std::pair<QueueId, RawEvent>> events_;
|
||||
};
|
||||
|
||||
} // namespace td
|
@ -118,10 +118,10 @@ class BinlogReader {
|
||||
it.advance(4, MutableSlice(buf, 4));
|
||||
size_ = static_cast<size_t>(TlParser(Slice(buf, 4)).fetch_int());
|
||||
|
||||
if (size_ > MAX_EVENT_SIZE) {
|
||||
if (size_ > BinlogEvent::MAX_SIZE) {
|
||||
return Status::Error(PSLICE() << "Too big event " << tag("size", size_));
|
||||
}
|
||||
if (size_ < MIN_EVENT_SIZE) {
|
||||
if (size_ < BinlogEvent::MIN_SIZE) {
|
||||
return Status::Error(PSLICE() << "Too small event " << tag("size", size_));
|
||||
}
|
||||
if (size_ % 4 != 0) {
|
||||
|
@ -18,6 +18,7 @@
|
||||
#include "td/utils/port/FileFd.h"
|
||||
#include "td/utils/Slice.h"
|
||||
#include "td/utils/Status.h"
|
||||
#include "td/utils/StorerBase.h"
|
||||
#include "td/utils/UInt.h"
|
||||
|
||||
#include <functional>
|
||||
@ -68,9 +69,25 @@ class Binlog {
|
||||
return fd_.empty();
|
||||
}
|
||||
|
||||
//void add_raw_event(BufferSlice &&raw_event) {
|
||||
//add_event(BinlogEvent(std::move(raw_event)));
|
||||
//}
|
||||
uint64 add(int32 type, const Storer &storer) {
|
||||
auto logevent_id = next_id();
|
||||
add_raw_event(BinlogEvent::create_raw(logevent_id, type, 0, storer), {});
|
||||
return logevent_id;
|
||||
}
|
||||
|
||||
uint64 rewrite(uint64 logevent_id, int32 type, const Storer &storer) {
|
||||
auto seq_no = next_id();
|
||||
add_raw_event(BinlogEvent::create_raw(logevent_id, type, BinlogEvent::Flags::Rewrite, storer), {});
|
||||
return seq_no;
|
||||
}
|
||||
|
||||
uint64 erase(uint64 logevent_id) {
|
||||
auto seq_no = next_id();
|
||||
add_raw_event(BinlogEvent::create_raw(logevent_id, BinlogEvent::ServiceTypes::Empty, BinlogEvent::Flags::Rewrite,
|
||||
EmptyStorer()),
|
||||
{});
|
||||
return seq_no;
|
||||
}
|
||||
|
||||
void add_raw_event(BufferSlice &&raw_event, BinlogDebugInfo info) {
|
||||
add_event(BinlogEvent(std::move(raw_event), info));
|
||||
@ -131,8 +148,6 @@ class Binlog {
|
||||
bool need_sync_{false};
|
||||
enum class State { Empty, Load, Reindex, Run } state_{State::Empty};
|
||||
|
||||
static constexpr uint32 MAX_EVENT_SIZE = 65536;
|
||||
|
||||
Result<FileFd> open_binlog(const string &path, int32 flags);
|
||||
size_t flush_events_buffer(bool force);
|
||||
void do_add_event(BinlogEvent &&event);
|
||||
|
@ -23,13 +23,13 @@ Status BinlogEvent::init(BufferSlice &&raw_event, bool check_crc) {
|
||||
type_ = parser.fetch_int();
|
||||
flags_ = parser.fetch_int();
|
||||
extra_ = parser.fetch_long();
|
||||
CHECK(size_ >= MIN_EVENT_SIZE);
|
||||
auto slice_data = parser.fetch_string_raw<Slice>(size_ - MIN_EVENT_SIZE);
|
||||
CHECK(size_ >= MIN_SIZE);
|
||||
auto slice_data = parser.fetch_string_raw<Slice>(size_ - MIN_SIZE);
|
||||
data_ = MutableSlice(const_cast<char *>(slice_data.begin()), slice_data.size());
|
||||
crc32_ = static_cast<uint32>(parser.fetch_int());
|
||||
if (check_crc) {
|
||||
CHECK(size_ >= EVENT_TAIL_SIZE);
|
||||
auto calculated_crc = crc32(raw_event.as_slice().truncate(size_ - EVENT_TAIL_SIZE));
|
||||
CHECK(size_ >= TAIL_SIZE);
|
||||
auto calculated_crc = crc32(raw_event.as_slice().truncate(size_ - TAIL_SIZE));
|
||||
if (calculated_crc != crc32_) {
|
||||
return Status::Error(PSLICE() << "crc mismatch " << tag("actual", format::as_hex(calculated_crc))
|
||||
<< tag("expected", format::as_hex(crc32_)) << public_to_string());
|
||||
@ -52,7 +52,7 @@ Status BinlogEvent::validate() const {
|
||||
}
|
||||
|
||||
BufferSlice BinlogEvent::create_raw(uint64 id, int32 type, int32 flags, const Storer &storer) {
|
||||
auto raw_event = BufferSlice{storer.size() + MIN_EVENT_SIZE};
|
||||
auto raw_event = BufferSlice{storer.size() + MIN_SIZE};
|
||||
|
||||
TlStorerUnsafe tl_storer(raw_event.as_slice().ubegin());
|
||||
tl_storer.store_int(narrow_cast<int32>(raw_event.size()));
|
||||
@ -61,11 +61,11 @@ BufferSlice BinlogEvent::create_raw(uint64 id, int32 type, int32 flags, const St
|
||||
tl_storer.store_int(flags);
|
||||
tl_storer.store_long(0);
|
||||
|
||||
CHECK(tl_storer.get_buf() == raw_event.as_slice().ubegin() + EVENT_HEADER_SIZE);
|
||||
CHECK(tl_storer.get_buf() == raw_event.as_slice().ubegin() + HEADER_SIZE);
|
||||
tl_storer.store_storer(storer);
|
||||
|
||||
CHECK(tl_storer.get_buf() == raw_event.as_slice().uend() - EVENT_TAIL_SIZE);
|
||||
tl_storer.store_int(::td::crc32(raw_event.as_slice().truncate(raw_event.size() - EVENT_TAIL_SIZE)));
|
||||
CHECK(tl_storer.get_buf() == raw_event.as_slice().uend() - TAIL_SIZE);
|
||||
tl_storer.store_int(::td::crc32(raw_event.as_slice().truncate(raw_event.size() - TAIL_SIZE)));
|
||||
|
||||
return raw_event;
|
||||
}
|
||||
|
@ -32,11 +32,6 @@ inline auto EmptyStorer() {
|
||||
return create_default_storer(impl);
|
||||
}
|
||||
|
||||
static constexpr size_t MAX_EVENT_SIZE = 1 << 24;
|
||||
static constexpr size_t EVENT_HEADER_SIZE = 4 + 8 + 4 + 4 + 8;
|
||||
static constexpr size_t EVENT_TAIL_SIZE = 4;
|
||||
static constexpr size_t MIN_EVENT_SIZE = EVENT_HEADER_SIZE + EVENT_TAIL_SIZE;
|
||||
|
||||
extern int32 VERBOSITY_NAME(binlog);
|
||||
|
||||
struct BinlogDebugInfo {
|
||||
@ -46,6 +41,7 @@ struct BinlogDebugInfo {
|
||||
const char *file{""};
|
||||
int line{0};
|
||||
};
|
||||
|
||||
inline StringBuilder &operator<<(StringBuilder &sb, const BinlogDebugInfo &info) {
|
||||
if (info.line == 0) {
|
||||
return sb;
|
||||
@ -54,6 +50,11 @@ inline StringBuilder &operator<<(StringBuilder &sb, const BinlogDebugInfo &info)
|
||||
}
|
||||
|
||||
struct BinlogEvent {
|
||||
static constexpr size_t MAX_SIZE = 1 << 24;
|
||||
static constexpr size_t HEADER_SIZE = 4 + 8 + 4 + 4 + 8;
|
||||
static constexpr size_t TAIL_SIZE = 4;
|
||||
static constexpr size_t MIN_SIZE = HEADER_SIZE + TAIL_SIZE;
|
||||
|
||||
int64 offset_;
|
||||
|
||||
uint32 size_;
|
||||
|
@ -9,38 +9,24 @@
|
||||
#include "td/actor/PromiseFuture.h"
|
||||
|
||||
#include "td/db/binlog/BinlogEvent.h"
|
||||
#include "td/db/binlog/BinlogInterface.h"
|
||||
|
||||
#include "td/utils/common.h"
|
||||
#include "td/utils/StorerBase.h"
|
||||
|
||||
namespace td {
|
||||
|
||||
template <class BinlogT, class StorerT>
|
||||
uint64 binlog_add(const BinlogT &binlog_ptr, int32 type, const StorerT &storer, Promise<> promise = Promise<>()) {
|
||||
auto logevent_id = binlog_ptr->next_id();
|
||||
binlog_ptr->add_raw_event(logevent_id, BinlogEvent::create_raw(logevent_id, type, 0, storer), std::move(promise));
|
||||
return logevent_id;
|
||||
inline uint64 binlog_add(BinlogInterface *binlog_ptr, int32 type, const Storer &storer, Promise<> promise = Promise<>()) {
|
||||
return binlog_ptr->add(type, storer, std::move(promise));
|
||||
}
|
||||
|
||||
template <class BinlogT, class StorerT>
|
||||
uint64 binlog_rewrite(const BinlogT &binlog_ptr, uint64 logevent_id, int32 type, const StorerT &storer,
|
||||
inline uint64 binlog_rewrite(BinlogInterface *binlog_ptr, uint64 logevent_id, int32 type, const Storer &storer,
|
||||
Promise<> promise = Promise<>()) {
|
||||
auto seq_no = binlog_ptr->next_id();
|
||||
binlog_ptr->add_raw_event(seq_no, BinlogEvent::create_raw(logevent_id, type, BinlogEvent::Flags::Rewrite, storer),
|
||||
std::move(promise));
|
||||
return seq_no;
|
||||
return binlog_ptr->rewrite(logevent_id, type, storer, std::move(promise));
|
||||
}
|
||||
|
||||
#define binlog_erase(...) binlog_erase_impl({__FILE__, __LINE__}, __VA_ARGS__)
|
||||
|
||||
template <class BinlogT>
|
||||
uint64 binlog_erase_impl(BinlogDebugInfo info, const BinlogT &binlog_ptr, uint64 logevent_id,
|
||||
Promise<> promise = Promise<>()) {
|
||||
auto seq_no = binlog_ptr->next_id();
|
||||
binlog_ptr->add_raw_event(info, seq_no,
|
||||
BinlogEvent::create_raw(logevent_id, BinlogEvent::ServiceTypes::Empty,
|
||||
BinlogEvent::Flags::Rewrite, EmptyStorer()),
|
||||
std::move(promise));
|
||||
return seq_no;
|
||||
inline uint64 binlog_erase(BinlogInterface *binlog_ptr, uint64 logevent_id, Promise<> promise = Promise<>()) {
|
||||
return binlog_ptr->erase(logevent_id, std::move(promise));
|
||||
}
|
||||
|
||||
} // namespace td
|
||||
|
@ -13,6 +13,7 @@
|
||||
|
||||
#include "td/utils/buffer.h"
|
||||
#include "td/utils/common.h"
|
||||
#include "td/utils/StorerBase.h"
|
||||
|
||||
namespace td {
|
||||
|
||||
@ -40,6 +41,29 @@ class BinlogInterface {
|
||||
void lazy_sync(Promise<> promise = Promise<>()) {
|
||||
add_raw_event_impl(next_id(), BufferSlice(), std::move(promise), {});
|
||||
}
|
||||
|
||||
uint64 add(int32 type, const Storer &storer, Promise<> promise = Promise<>()) {
|
||||
auto logevent_id = next_id();
|
||||
add_raw_event_impl(logevent_id, BinlogEvent::create_raw(logevent_id, type, 0, storer), std::move(promise), {});
|
||||
return logevent_id;
|
||||
}
|
||||
|
||||
uint64 rewrite(uint64 logevent_id, int32 type, const Storer &storer, Promise<> promise = Promise<>()) {
|
||||
auto seq_no = next_id();
|
||||
add_raw_event_impl(seq_no, BinlogEvent::create_raw(logevent_id, type, BinlogEvent::Flags::Rewrite, storer),
|
||||
std::move(promise), {});
|
||||
return seq_no;
|
||||
}
|
||||
|
||||
uint64 erase(uint64 logevent_id, Promise<> promise = Promise<>()) {
|
||||
auto seq_no = next_id();
|
||||
add_raw_event_impl(seq_no,
|
||||
BinlogEvent::create_raw(logevent_id, BinlogEvent::ServiceTypes::Empty,
|
||||
BinlogEvent::Flags::Rewrite, EmptyStorer()),
|
||||
std::move(promise), {});
|
||||
return seq_no;
|
||||
}
|
||||
|
||||
virtual void force_sync(Promise<> promise) = 0;
|
||||
virtual void force_flush() = 0;
|
||||
virtual void change_key(DbKey db_key, Promise<> promise = Promise<>()) = 0;
|
||||
|
@ -164,7 +164,7 @@ void HttpConnectionBase::loop() {
|
||||
}
|
||||
if (state_ == State::Close) {
|
||||
LOG_IF(INFO, fd_.need_flush_write()) << "Close nonempty connection";
|
||||
LOG_IF(INFO, want_read && (fd_.input_buffer().size() > 0 || current_query_->type_ != HttpQuery::Type::EMPTY))
|
||||
LOG_IF(INFO, want_read && (fd_.input_buffer().size() > 0 || current_query_->type_ != HttpQuery::Type::Empty))
|
||||
<< "Close connection while reading request/response";
|
||||
return stop();
|
||||
}
|
||||
|
@ -24,8 +24,8 @@ MutableSlice HttpQuery::get_arg(Slice key) const {
|
||||
return it == args_.end() ? MutableSlice() : it->second;
|
||||
}
|
||||
|
||||
std::vector<std::pair<string, string>> HttpQuery::get_args() const {
|
||||
std::vector<std::pair<string, string>> res;
|
||||
td::vector<std::pair<string, string>> HttpQuery::get_args() const {
|
||||
td::vector<std::pair<string, string>> res;
|
||||
res.reserve(args_.size());
|
||||
for (auto &it : args_) {
|
||||
res.emplace_back(it.first.str(), it.second.str());
|
||||
@ -48,20 +48,20 @@ int HttpQuery::get_retry_after() const {
|
||||
|
||||
StringBuilder &operator<<(StringBuilder &sb, const HttpQuery &q) {
|
||||
switch (q.type_) {
|
||||
case HttpQuery::Type::EMPTY:
|
||||
case HttpQuery::Type::Empty:
|
||||
sb << "EMPTY";
|
||||
return sb;
|
||||
case HttpQuery::Type::GET:
|
||||
case HttpQuery::Type::Get:
|
||||
sb << "GET";
|
||||
break;
|
||||
case HttpQuery::Type::POST:
|
||||
case HttpQuery::Type::Post:
|
||||
sb << "POST";
|
||||
break;
|
||||
case HttpQuery::Type::RESPONSE:
|
||||
case HttpQuery::Type::Response:
|
||||
sb << "RESPONSE";
|
||||
break;
|
||||
}
|
||||
if (q.type_ == HttpQuery::Type::RESPONSE) {
|
||||
if (q.type_ == HttpQuery::Type::Response) {
|
||||
sb << ":" << q.code_ << ":" << q.reason_;
|
||||
} else {
|
||||
sb << ":" << q.url_path_;
|
||||
|
@ -19,25 +19,25 @@ namespace td {
|
||||
|
||||
class HttpQuery {
|
||||
public:
|
||||
enum class Type : int8 { EMPTY, GET, POST, RESPONSE };
|
||||
enum class Type : int8 { Empty, Get, Post, Response };
|
||||
|
||||
std::vector<BufferSlice> container_;
|
||||
Type type_ = Type::EMPTY;
|
||||
td::vector<BufferSlice> container_;
|
||||
Type type_ = Type::Empty;
|
||||
int32 code_ = 0;
|
||||
MutableSlice url_path_;
|
||||
std::vector<std::pair<MutableSlice, MutableSlice>> args_;
|
||||
td::vector<std::pair<MutableSlice, MutableSlice>> args_;
|
||||
MutableSlice reason_;
|
||||
|
||||
bool keep_alive_ = true;
|
||||
std::vector<std::pair<MutableSlice, MutableSlice>> headers_;
|
||||
std::vector<HttpFile> files_;
|
||||
td::vector<std::pair<MutableSlice, MutableSlice>> headers_;
|
||||
td::vector<HttpFile> files_;
|
||||
MutableSlice content_;
|
||||
|
||||
Slice get_header(Slice key) const;
|
||||
|
||||
MutableSlice get_arg(Slice key) const;
|
||||
|
||||
std::vector<std::pair<string, string>> get_args() const;
|
||||
td::vector<std::pair<string, string>> get_args() const;
|
||||
|
||||
int get_retry_after() const;
|
||||
};
|
||||
|
@ -663,12 +663,12 @@ Status HttpReader::parse_head(MutableSlice head) {
|
||||
parser.skip(' ');
|
||||
// GET POST HTTP/1.1
|
||||
if (type == "GET") {
|
||||
query_->type_ = HttpQuery::Type::GET;
|
||||
query_->type_ = HttpQuery::Type::Get;
|
||||
} else if (type == "POST") {
|
||||
query_->type_ = HttpQuery::Type::POST;
|
||||
query_->type_ = HttpQuery::Type::Post;
|
||||
} else if (type.size() >= 4 && type.substr(0, 4) == "HTTP") {
|
||||
if (type == "HTTP/1.1" || type == "HTTP/1.0") {
|
||||
query_->type_ = HttpQuery::Type::RESPONSE;
|
||||
query_->type_ = HttpQuery::Type::Response;
|
||||
} else {
|
||||
LOG(INFO) << "Unsupported HTTP version: " << type;
|
||||
return Status::Error(505, "HTTP Version Not Supported");
|
||||
@ -680,7 +680,7 @@ Status HttpReader::parse_head(MutableSlice head) {
|
||||
|
||||
query_->args_.clear();
|
||||
|
||||
if (query_->type_ == HttpQuery::Type::RESPONSE) {
|
||||
if (query_->type_ == HttpQuery::Type::Response) {
|
||||
query_->code_ = to_integer<int32>(parser.read_till(' '));
|
||||
parser.skip(' ');
|
||||
query_->reason_ = parser.read_till('\r');
|
||||
|
@ -74,7 +74,7 @@ Status Wget::try_init() {
|
||||
TRY_STATUS(addr.init_host_port(url.host_, url.port_, prefer_ipv6_));
|
||||
|
||||
TRY_RESULT(fd, SocketFd::open(addr));
|
||||
if (url.protocol_ == HttpUrl::Protocol::HTTP) {
|
||||
if (url.protocol_ == HttpUrl::Protocol::Http) {
|
||||
connection_ = create_actor<HttpOutboundConnection>("Connect", std::move(fd), SslStream{},
|
||||
std::numeric_limits<std::size_t>::max(), 0, 0,
|
||||
ActorOwn<HttpOutboundConnection::Callback>(actor_id(this)));
|
||||
|
@ -4,14 +4,6 @@ if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
|
||||
set(CMAKE_INSTALL_LIBDIR "lib")
|
||||
endif()
|
||||
|
||||
if (WIN32)
|
||||
if (WINGETOPT_FOUND)
|
||||
set(TD_HAVE_GETOPT 1)
|
||||
endif()
|
||||
else()
|
||||
set(TD_HAVE_GETOPT 1)
|
||||
endif()
|
||||
|
||||
if (NOT ZLIB_FOUND)
|
||||
find_package(ZLIB)
|
||||
endif()
|
||||
@ -58,6 +50,7 @@ set(TDUTILS_SOURCE
|
||||
td/utils/port/MemoryMapping.cpp
|
||||
td/utils/port/path.cpp
|
||||
td/utils/port/PollFlags.cpp
|
||||
td/utils/port/rlimit.cpp
|
||||
td/utils/port/ServerSocketFd.cpp
|
||||
td/utils/port/signals.cpp
|
||||
td/utils/port/sleep.cpp
|
||||
@ -67,6 +60,7 @@ set(TDUTILS_SOURCE
|
||||
td/utils/port/StdStreams.cpp
|
||||
td/utils/port/thread_local.cpp
|
||||
td/utils/port/UdpSocketFd.cpp
|
||||
td/utils/port/user.cpp
|
||||
td/utils/port/wstring_convert.cpp
|
||||
|
||||
td/utils/port/detail/Epoll.cpp
|
||||
@ -102,7 +96,7 @@ set(TDUTILS_SOURCE
|
||||
td/utils/misc.cpp
|
||||
td/utils/MimeType.cpp
|
||||
td/utils/MpmcQueue.cpp
|
||||
td/utils/OptionsParser.cpp
|
||||
td/utils/OptionParser.cpp
|
||||
td/utils/PathView.cpp
|
||||
td/utils/Random.cpp
|
||||
td/utils/SharedSlice.cpp
|
||||
@ -133,6 +127,7 @@ set(TDUTILS_SOURCE
|
||||
td/utils/port/Poll.h
|
||||
td/utils/port/PollBase.h
|
||||
td/utils/port/PollFlags.h
|
||||
td/utils/port/rlimit.h
|
||||
td/utils/port/RwMutex.h
|
||||
td/utils/port/ServerSocketFd.h
|
||||
td/utils/port/signals.h
|
||||
@ -144,6 +139,7 @@ set(TDUTILS_SOURCE
|
||||
td/utils/port/thread.h
|
||||
td/utils/port/thread_local.h
|
||||
td/utils/port/UdpSocketFd.h
|
||||
td/utils/port/user.h
|
||||
td/utils/port/wstring_convert.h
|
||||
|
||||
td/utils/port/detail/Epoll.h
|
||||
@ -156,6 +152,7 @@ set(TDUTILS_SOURCE
|
||||
td/utils/port/detail/Poll.h
|
||||
td/utils/port/detail/PollableFd.h
|
||||
td/utils/port/detail/Select.h
|
||||
td/utils/port/detail/skip_eintr.h
|
||||
td/utils/port/detail/ThreadIdGuard.h
|
||||
td/utils/port/detail/ThreadPthread.h
|
||||
td/utils/port/detail/ThreadStl.h
|
||||
@ -217,7 +214,7 @@ set(TDUTILS_SOURCE
|
||||
td/utils/ObjectPool.h
|
||||
td/utils/Observer.h
|
||||
td/utils/optional.h
|
||||
td/utils/OptionsParser.h
|
||||
td/utils/OptionParser.h
|
||||
td/utils/OrderedEventsProcessor.h
|
||||
td/utils/overloaded.h
|
||||
td/utils/Parser.h
|
||||
@ -275,6 +272,7 @@ set(TDUTILS_TEST_SOURCE
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcQueue.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/test/MpmcWaiter.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/test/MpscLinkQueue.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/test/OptionParser.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/test/OrderedEventsProcessor.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/test/port.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/test/pq.cpp
|
||||
@ -291,7 +289,7 @@ if (WIN32)
|
||||
# find_library(WS2_32_LIBRARY ws2_32)
|
||||
# find_library(MSWSOCK_LIBRARY Mswsock)
|
||||
# target_link_libraries(tdutils PRIVATE ${WS2_32_LIBRARY} ${MSWSOCK_LIBRARY})
|
||||
target_link_libraries(tdutils PRIVATE ws2_32 Mswsock Normaliz)
|
||||
target_link_libraries(tdutils PRIVATE ws2_32 Mswsock Normaliz psapi)
|
||||
endif()
|
||||
if (NOT CMAKE_CROSSCOMPILING)
|
||||
add_dependencies(tdutils tdmime_auto)
|
||||
@ -319,10 +317,6 @@ if (ABSL_FOUND)
|
||||
target_link_libraries(tdutils PUBLIC absl::flat_hash_map absl::flat_hash_set absl::hash)
|
||||
endif()
|
||||
|
||||
if (WIN32 AND WINGETOPT_FOUND)
|
||||
target_link_libraries(tdutils PRIVATE wingetopt)
|
||||
endif()
|
||||
|
||||
if (ANDROID)
|
||||
target_link_libraries(tdutils PRIVATE log)
|
||||
endif()
|
||||
|
@ -64,7 +64,8 @@ class UdpReaderHelper {
|
||||
}
|
||||
|
||||
private:
|
||||
enum : size_t { MAX_PACKET_SIZE = 2048, RESERVED_SIZE = MAX_PACKET_SIZE * 8 };
|
||||
static constexpr size_t MAX_PACKET_SIZE = 2048;
|
||||
static constexpr size_t RESERVED_SIZE = MAX_PACKET_SIZE * 8;
|
||||
UdpMessage message_;
|
||||
BufferSlice buffer_;
|
||||
};
|
||||
@ -98,7 +99,7 @@ class UdpReader {
|
||||
}
|
||||
|
||||
private:
|
||||
enum : size_t { BUFFER_SIZE = 16 };
|
||||
static constexpr size_t BUFFER_SIZE = 16;
|
||||
std::array<UdpSocketFd::InboundMessage, BUFFER_SIZE> messages_;
|
||||
std::array<UdpReaderHelper, BUFFER_SIZE> helpers_;
|
||||
};
|
||||
|
@ -17,10 +17,10 @@ namespace td {
|
||||
string HttpUrl::get_url() const {
|
||||
string result;
|
||||
switch (protocol_) {
|
||||
case Protocol::HTTP:
|
||||
case Protocol::Http:
|
||||
result += "http://";
|
||||
break;
|
||||
case Protocol::HTTPS:
|
||||
case Protocol::Https:
|
||||
result += "https://";
|
||||
break;
|
||||
default:
|
||||
@ -49,9 +49,9 @@ Result<HttpUrl> parse_url(Slice url, HttpUrl::Protocol default_protocol) {
|
||||
if (parser.start_with("://")) {
|
||||
parser.advance(3);
|
||||
if (protocol_str == "http") {
|
||||
protocol = HttpUrl::Protocol::HTTP;
|
||||
protocol = HttpUrl::Protocol::Http;
|
||||
} else if (protocol_str == "https") {
|
||||
protocol = HttpUrl::Protocol::HTTPS;
|
||||
protocol = HttpUrl::Protocol::Https;
|
||||
} else {
|
||||
return Status::Error("Unsupported URL protocol");
|
||||
}
|
||||
@ -99,10 +99,10 @@ Result<HttpUrl> parse_url(Slice url, HttpUrl::Protocol default_protocol) {
|
||||
|
||||
int specified_port = port;
|
||||
if (port == 0) {
|
||||
if (protocol == HttpUrl::Protocol::HTTP) {
|
||||
if (protocol == HttpUrl::Protocol::Http) {
|
||||
port = 80;
|
||||
} else {
|
||||
CHECK(protocol == HttpUrl::Protocol::HTTPS);
|
||||
CHECK(protocol == HttpUrl::Protocol::Https);
|
||||
port = 443;
|
||||
}
|
||||
}
|
||||
@ -169,7 +169,7 @@ Result<HttpUrl> parse_url(Slice url, HttpUrl::Protocol default_protocol) {
|
||||
}
|
||||
|
||||
StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url) {
|
||||
sb << tag("protocol", url.protocol_ == HttpUrl::Protocol::HTTP ? "HTTP" : "HTTPS") << tag("userinfo", url.userinfo_)
|
||||
sb << tag("protocol", url.protocol_ == HttpUrl::Protocol::Http ? "HTTP" : "HTTPS") << tag("userinfo", url.userinfo_)
|
||||
<< tag("host", url.host_) << tag("port", url.port_) << tag("query", url.query_);
|
||||
return sb;
|
||||
}
|
||||
|
@ -15,7 +15,7 @@ namespace td {
|
||||
|
||||
class HttpUrl {
|
||||
public:
|
||||
enum class Protocol { HTTP, HTTPS } protocol_ = Protocol::HTTP;
|
||||
enum class Protocol { Http, Https } protocol_ = Protocol::Http;
|
||||
string userinfo_;
|
||||
string host_;
|
||||
bool is_ipv6_ = false;
|
||||
@ -37,7 +37,7 @@ class HttpUrl {
|
||||
};
|
||||
|
||||
Result<HttpUrl> parse_url(Slice url,
|
||||
HttpUrl::Protocol default_protocol = HttpUrl::Protocol::HTTP) TD_WARN_UNUSED_RESULT;
|
||||
HttpUrl::Protocol default_protocol = HttpUrl::Protocol::Http) TD_WARN_UNUSED_RESULT;
|
||||
|
||||
StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url);
|
||||
|
||||
|
183
tdutils/td/utils/OptionParser.cpp
Normal file
183
tdutils/td/utils/OptionParser.cpp
Normal file
@ -0,0 +1,183 @@
|
||||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#include "td/utils/OptionParser.h"
|
||||
|
||||
#include "td/utils/misc.h"
|
||||
|
||||
#include <cstring>
|
||||
#include <unordered_map>
|
||||
|
||||
namespace td {
|
||||
|
||||
void OptionParser::set_description(string description) {
|
||||
description_ = std::move(description);
|
||||
}
|
||||
|
||||
void OptionParser::add_option(Option::Type type, char short_key, Slice long_key, Slice description,
|
||||
std::function<Status(Slice)> callback) {
|
||||
options_.push_back(Option{type, short_key, long_key.str(), description.str(), std::move(callback)});
|
||||
}
|
||||
|
||||
void OptionParser::add_option(char short_key, Slice long_key, Slice description,
|
||||
std::function<Status(Slice)> callback) {
|
||||
add_option(Option::Type::Arg, short_key, long_key, description, std::move(callback));
|
||||
}
|
||||
|
||||
void OptionParser::add_option(char short_key, Slice long_key, Slice description, std::function<Status(void)> callback) {
|
||||
// Ouch. There must be some better way
|
||||
add_option(Option::Type::NoArg, short_key, long_key, description,
|
||||
std::bind([](std::function<Status(void)> &func, Slice) { return func(); }, std::move(callback),
|
||||
std::placeholders::_1));
|
||||
}
|
||||
|
||||
Result<vector<char *>> OptionParser::run(int argc, char *argv[]) {
|
||||
std::unordered_map<char, const Option *> short_options;
|
||||
std::unordered_map<string, const Option *> long_options;
|
||||
for (auto &opt : options_) {
|
||||
if (opt.short_key != '\0') {
|
||||
short_options[opt.short_key] = &opt;
|
||||
}
|
||||
if (!opt.long_key.empty()) {
|
||||
long_options[opt.long_key] = &opt;
|
||||
}
|
||||
}
|
||||
|
||||
vector<char *> non_options;
|
||||
for (int arg_pos = 1; arg_pos < argc; arg_pos++) {
|
||||
const char *arg = argv[arg_pos];
|
||||
if (arg[0] != '-' || arg[1] == '\0') {
|
||||
non_options.push_back(argv[arg_pos]);
|
||||
continue;
|
||||
}
|
||||
if (arg[1] == '-' && arg[2] == '\0') {
|
||||
// "--"; after it everything is non-option
|
||||
while (++arg_pos < argc) {
|
||||
non_options.push_back(argv[arg_pos]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (arg[1] == '-') {
|
||||
// long option
|
||||
Slice long_arg(arg + 2, std::strlen(arg + 2));
|
||||
Slice param;
|
||||
auto equal_pos = long_arg.find('=');
|
||||
bool has_equal = equal_pos != Slice::npos;
|
||||
if (has_equal) {
|
||||
param = long_arg.substr(equal_pos + 1);
|
||||
long_arg = long_arg.substr(0, equal_pos);
|
||||
}
|
||||
|
||||
auto it = long_options.find(long_arg.str());
|
||||
if (it == long_options.end()) {
|
||||
return Status::Error(PSLICE() << "Option " << long_arg << " was unrecognized");
|
||||
}
|
||||
|
||||
auto option = it->second;
|
||||
switch (option->type) {
|
||||
case Option::Type::NoArg:
|
||||
if (has_equal) {
|
||||
return Status::Error(PSLICE() << "Option " << long_arg << " must not have argument");
|
||||
}
|
||||
break;
|
||||
case Option::Type::Arg:
|
||||
if (!has_equal) {
|
||||
if (++arg_pos == argc) {
|
||||
return Status::Error(PSLICE() << "Option " << long_arg << " must have argument");
|
||||
}
|
||||
param = Slice(argv[arg_pos], std::strlen(argv[arg_pos]));
|
||||
}
|
||||
break;
|
||||
default:
|
||||
UNREACHABLE();
|
||||
}
|
||||
|
||||
TRY_STATUS(option->arg_callback(param));
|
||||
continue;
|
||||
}
|
||||
|
||||
for (size_t opt_pos = 1; arg[opt_pos] != '\0'; opt_pos++) {
|
||||
auto it = short_options.find(arg[opt_pos]);
|
||||
if (it == short_options.end()) {
|
||||
return Status::Error(PSLICE() << "Option " << arg[opt_pos] << " was unrecognized");
|
||||
}
|
||||
|
||||
auto option = it->second;
|
||||
Slice param;
|
||||
switch (option->type) {
|
||||
case Option::Type::NoArg:
|
||||
// nothing to do
|
||||
break;
|
||||
case Option::Type::Arg:
|
||||
if (arg[opt_pos + 1] == '\0') {
|
||||
if (++arg_pos == argc) {
|
||||
return Status::Error(PSLICE() << "Option " << arg[opt_pos] << " must have argument");
|
||||
}
|
||||
param = Slice(argv[arg_pos], std::strlen(argv[arg_pos]));
|
||||
} else {
|
||||
param = Slice(arg + opt_pos + 1, std::strlen(arg + opt_pos + 1));
|
||||
opt_pos += param.size();
|
||||
}
|
||||
break;
|
||||
default:
|
||||
UNREACHABLE();
|
||||
}
|
||||
|
||||
TRY_STATUS(option->arg_callback(param));
|
||||
}
|
||||
}
|
||||
|
||||
return std::move(non_options);
|
||||
}
|
||||
|
||||
StringBuilder &operator<<(StringBuilder &sb, const OptionParser &o) {
|
||||
if (!o.description_.empty()) {
|
||||
sb << o.description_ << ". ";
|
||||
}
|
||||
sb << "Options:\n";
|
||||
|
||||
size_t max_length = 0;
|
||||
for (auto &opt : o.options_) {
|
||||
bool has_short_key = opt.short_key != '\0';
|
||||
bool has_long_key = !opt.long_key.empty();
|
||||
size_t length = (has_short_key ? 2 : 0) + (has_long_key ? 2 + opt.long_key.size() + 2 * has_short_key : 0);
|
||||
if (opt.type != OptionParser::Option::Type::NoArg) {
|
||||
length += 5;
|
||||
}
|
||||
if (length > max_length) {
|
||||
max_length = length;
|
||||
}
|
||||
}
|
||||
max_length++;
|
||||
|
||||
for (auto &opt : o.options_) {
|
||||
bool has_short_key = opt.short_key != '\0';
|
||||
sb << " ";
|
||||
size_t length = max_length;
|
||||
if (has_short_key) {
|
||||
sb << '-' << opt.short_key;
|
||||
length -= 2;
|
||||
}
|
||||
if (!opt.long_key.empty()) {
|
||||
if (has_short_key) {
|
||||
sb << ", ";
|
||||
length -= 2;
|
||||
}
|
||||
sb << "--" << opt.long_key;
|
||||
length -= 2 + opt.long_key.size();
|
||||
}
|
||||
if (opt.type != OptionParser::Option::Type::NoArg) {
|
||||
sb << "<arg>";
|
||||
length -= 5;
|
||||
}
|
||||
sb << string(length, ' ') << opt.description;
|
||||
sb << '\n';
|
||||
}
|
||||
return sb;
|
||||
}
|
||||
|
||||
} // namespace td
|
@ -15,34 +15,35 @@
|
||||
|
||||
namespace td {
|
||||
|
||||
class OptionsParser {
|
||||
class OptionParser {
|
||||
class Option {
|
||||
public:
|
||||
enum class Type { NoArg, Arg, OptionalArg };
|
||||
enum class Type { NoArg, Arg };
|
||||
Type type;
|
||||
char short_key;
|
||||
std::string long_key;
|
||||
std::string description;
|
||||
string long_key;
|
||||
string description;
|
||||
std::function<Status(Slice)> arg_callback;
|
||||
};
|
||||
|
||||
public:
|
||||
void set_description(std::string description);
|
||||
|
||||
void add_option(Option::Type type, char short_key, Slice long_key, Slice description,
|
||||
std::function<Status(Slice)> callback);
|
||||
|
||||
public:
|
||||
void set_description(string description);
|
||||
|
||||
void add_option(char short_key, Slice long_key, Slice description, std::function<Status(Slice)> callback);
|
||||
|
||||
void add_option(char short_key, Slice long_key, Slice description, std::function<Status(void)> callback);
|
||||
|
||||
Result<int> run(int argc, char *argv[]) TD_WARN_UNUSED_RESULT;
|
||||
// returns found non-option parameters
|
||||
Result<vector<char *>> run(int argc, char *argv[]) TD_WARN_UNUSED_RESULT;
|
||||
|
||||
friend StringBuilder &operator<<(StringBuilder &sb, const OptionsParser &o);
|
||||
friend StringBuilder &operator<<(StringBuilder &sb, const OptionParser &o);
|
||||
|
||||
private:
|
||||
std::vector<Option> options_;
|
||||
std::string description_;
|
||||
vector<Option> options_;
|
||||
string description_;
|
||||
};
|
||||
|
||||
} // namespace td
|
@ -1,130 +0,0 @@
|
||||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#include "td/utils/OptionsParser.h"
|
||||
|
||||
#if TD_HAVE_GETOPT
|
||||
#include "getopt.h"
|
||||
#endif
|
||||
|
||||
#if !TD_WINDOWS
|
||||
#include <getopt.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace td {
|
||||
|
||||
void OptionsParser::set_description(std::string description) {
|
||||
description_ = std::move(description);
|
||||
}
|
||||
|
||||
void OptionsParser::add_option(Option::Type type, char short_key, Slice long_key, Slice description,
|
||||
std::function<Status(Slice)> callback) {
|
||||
options_.push_back(Option{type, short_key, long_key.str(), description.str(), std::move(callback)});
|
||||
}
|
||||
|
||||
void OptionsParser::add_option(char short_key, Slice long_key, Slice description,
|
||||
std::function<Status(Slice)> callback) {
|
||||
add_option(Option::Type::Arg, short_key, long_key, description, std::move(callback));
|
||||
}
|
||||
|
||||
void OptionsParser::add_option(char short_key, Slice long_key, Slice description,
|
||||
std::function<Status(void)> callback) {
|
||||
// Ouch. There must be some better way
|
||||
add_option(Option::Type::NoArg, short_key, long_key, description,
|
||||
std::bind([](std::function<Status(void)> &func, Slice) { return func(); }, std::move(callback),
|
||||
std::placeholders::_1));
|
||||
}
|
||||
|
||||
Result<int> OptionsParser::run(int argc, char *argv[]) {
|
||||
#if TD_HAVE_GETOPT
|
||||
char buff[1024];
|
||||
StringBuilder sb(MutableSlice{buff, sizeof(buff)});
|
||||
for (auto &opt : options_) {
|
||||
CHECK(opt.type != Option::Type::OptionalArg);
|
||||
sb << opt.short_key;
|
||||
if (opt.type == Option::Type::Arg) {
|
||||
sb << ":";
|
||||
}
|
||||
}
|
||||
if (sb.is_error()) {
|
||||
return Status::Error("Can't parse options");
|
||||
}
|
||||
CSlice short_options = sb.as_cslice();
|
||||
|
||||
vector<option> long_options;
|
||||
for (auto &opt : options_) {
|
||||
if (opt.long_key.empty()) {
|
||||
continue;
|
||||
}
|
||||
option o;
|
||||
o.flag = nullptr;
|
||||
o.val = opt.short_key;
|
||||
o.has_arg = opt.type == Option::Type::Arg ? required_argument : no_argument;
|
||||
o.name = opt.long_key.c_str();
|
||||
long_options.push_back(o);
|
||||
}
|
||||
long_options.push_back({nullptr, 0, nullptr, 0});
|
||||
|
||||
while (true) {
|
||||
int opt_i = getopt_long(argc, argv, short_options.c_str(), &long_options[0], nullptr);
|
||||
if (opt_i == ':') {
|
||||
return Status::Error("Missing argument");
|
||||
}
|
||||
if (opt_i == '?') {
|
||||
return Status::Error("Unrecognized option");
|
||||
}
|
||||
if (opt_i == -1) {
|
||||
break;
|
||||
}
|
||||
bool found = false;
|
||||
for (auto &opt : options_) {
|
||||
if (opt.short_key == opt_i) {
|
||||
Slice arg;
|
||||
if (opt.type == Option::Type::Arg) {
|
||||
arg = Slice(optarg);
|
||||
}
|
||||
auto status = opt.arg_callback(arg);
|
||||
if (status.is_error()) {
|
||||
return std::move(status);
|
||||
}
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found) {
|
||||
return Status::Error("Unknown argument");
|
||||
}
|
||||
}
|
||||
return optind;
|
||||
#else
|
||||
return -1;
|
||||
#endif
|
||||
}
|
||||
|
||||
StringBuilder &operator<<(StringBuilder &sb, const OptionsParser &o) {
|
||||
sb << o.description_ << "\n";
|
||||
for (auto &opt : o.options_) {
|
||||
sb << "-" << opt.short_key;
|
||||
if (!opt.long_key.empty()) {
|
||||
sb << "|--" << opt.long_key;
|
||||
}
|
||||
if (opt.type == OptionsParser::Option::Type::OptionalArg) {
|
||||
sb << "[";
|
||||
}
|
||||
if (opt.type != OptionsParser::Option::Type::NoArg) {
|
||||
sb << "<arg>";
|
||||
}
|
||||
if (opt.type == OptionsParser::Option::Type::OptionalArg) {
|
||||
sb << "]";
|
||||
}
|
||||
sb << "\t" << opt.description;
|
||||
sb << "\n";
|
||||
}
|
||||
return sb;
|
||||
}
|
||||
|
||||
} // namespace td
|
@ -160,8 +160,23 @@ uint64 Random::Xorshift128plus::operator()() {
|
||||
seed_[1] = x ^ y ^ (x >> 17) ^ (y >> 26);
|
||||
return seed_[1] + y;
|
||||
}
|
||||
|
||||
int Random::Xorshift128plus::fast(int min, int max) {
|
||||
return static_cast<int>((*this)() % (max - min + 1) + min);
|
||||
}
|
||||
|
||||
void Random::Xorshift128plus::bytes(MutableSlice dest) {
|
||||
int cnt = 0;
|
||||
uint64 buf = 0;
|
||||
for (auto &c : dest) {
|
||||
if (cnt == 0) {
|
||||
buf = operator()();
|
||||
cnt = 8;
|
||||
}
|
||||
cnt--;
|
||||
c = static_cast<char>(buf & 255);
|
||||
buf >>= 8;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace td
|
||||
|
@ -37,6 +37,7 @@ class Random {
|
||||
Xorshift128plus(uint64 seed_a, uint64 seed_b);
|
||||
uint64 operator()();
|
||||
int fast(int min, int max);
|
||||
void bytes(MutableSlice dest);
|
||||
|
||||
private:
|
||||
uint64 seed_[2];
|
||||
|
@ -104,7 +104,7 @@ string winerror_to_string(int code);
|
||||
#endif
|
||||
|
||||
class Status {
|
||||
enum class ErrorType : int8 { general, os };
|
||||
enum class ErrorType : int8 { General, Os };
|
||||
|
||||
public:
|
||||
Status() = default;
|
||||
@ -129,7 +129,7 @@ class Status {
|
||||
}
|
||||
|
||||
static Status Error(int err, Slice message = Slice()) TD_WARN_UNUSED_RESULT {
|
||||
return Status(false, ErrorType::general, err, message);
|
||||
return Status(false, ErrorType::General, err, message);
|
||||
}
|
||||
|
||||
static Status Error(Slice message) TD_WARN_UNUSED_RESULT {
|
||||
@ -138,13 +138,13 @@ class Status {
|
||||
|
||||
#if TD_PORT_WINDOWS
|
||||
static Status WindowsError(int saved_error, Slice message) TD_WARN_UNUSED_RESULT {
|
||||
return Status(false, ErrorType::os, saved_error, message);
|
||||
return Status(false, ErrorType::Os, saved_error, message);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if TD_PORT_POSIX
|
||||
static Status PosixError(int32 saved_errno, Slice message) TD_WARN_UNUSED_RESULT {
|
||||
return Status(false, ErrorType::os, saved_errno, message);
|
||||
return Status(false, ErrorType::Os, saved_errno, message);
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -154,7 +154,7 @@ class Status {
|
||||
|
||||
template <int Code>
|
||||
static Status Error() {
|
||||
static Status status(true, ErrorType::general, Code, "");
|
||||
static Status status(true, ErrorType::General, Code, "");
|
||||
return status.clone_static();
|
||||
}
|
||||
|
||||
@ -164,10 +164,10 @@ class Status {
|
||||
}
|
||||
Info info = get_info();
|
||||
switch (info.error_type) {
|
||||
case ErrorType::general:
|
||||
case ErrorType::General:
|
||||
sb << "[Error";
|
||||
break;
|
||||
case ErrorType::os:
|
||||
case ErrorType::Os:
|
||||
#if TD_PORT_POSIX
|
||||
sb << "[PosixError : " << strerror_safe(info.error_code);
|
||||
#elif TD_PORT_WINDOWS
|
||||
@ -246,9 +246,9 @@ class Status {
|
||||
}
|
||||
Info info = get_info();
|
||||
switch (info.error_type) {
|
||||
case ErrorType::general:
|
||||
case ErrorType::General:
|
||||
return message().str();
|
||||
case ErrorType::os:
|
||||
case ErrorType::Os:
|
||||
#if TD_PORT_POSIX
|
||||
return strerror_safe(info.error_code).str();
|
||||
#elif TD_PORT_WINDOWS
|
||||
@ -276,10 +276,10 @@ class Status {
|
||||
CHECK(is_error());
|
||||
Info info = get_info();
|
||||
switch (info.error_type) {
|
||||
case ErrorType::general:
|
||||
case ErrorType::General:
|
||||
return Error(code(), PSLICE() << prefix << message());
|
||||
case ErrorType::os:
|
||||
return Status(false, ErrorType::os, code(), PSLICE() << prefix << message());
|
||||
case ErrorType::Os:
|
||||
return Status(false, ErrorType::Os, code(), PSLICE() << prefix << message());
|
||||
default:
|
||||
UNREACHABLE();
|
||||
return {};
|
||||
|
@ -22,14 +22,14 @@ namespace td {
|
||||
|
||||
StringBuilder::StringBuilder(MutableSlice slice, bool use_buffer)
|
||||
: begin_ptr_(slice.begin()), current_ptr_(begin_ptr_), use_buffer_(use_buffer) {
|
||||
if (slice.size() <= reserved_size) {
|
||||
auto buffer_size = reserved_size + 100;
|
||||
if (slice.size() <= RESERVED_SIZE) {
|
||||
auto buffer_size = RESERVED_SIZE + 100;
|
||||
buffer_ = std::make_unique<char[]>(buffer_size);
|
||||
begin_ptr_ = buffer_.get();
|
||||
current_ptr_ = begin_ptr_;
|
||||
end_ptr_ = begin_ptr_ + buffer_size - reserved_size;
|
||||
end_ptr_ = begin_ptr_ + buffer_size - RESERVED_SIZE;
|
||||
} else {
|
||||
end_ptr_ = slice.end() - reserved_size;
|
||||
end_ptr_ = slice.end() - RESERVED_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
@ -39,7 +39,7 @@ StringBuilder &StringBuilder::operator<<(Slice slice) {
|
||||
if (end_ptr_ < current_ptr_) {
|
||||
return on_error();
|
||||
}
|
||||
auto available_size = static_cast<size_t>(end_ptr_ + reserved_size - 1 - current_ptr_);
|
||||
auto available_size = static_cast<size_t>(end_ptr_ + RESERVED_SIZE - 1 - current_ptr_);
|
||||
if (size > available_size) {
|
||||
error_flag_ = true;
|
||||
size = available_size;
|
||||
@ -101,12 +101,12 @@ bool StringBuilder::reserve_inner(size_t size) {
|
||||
}
|
||||
|
||||
size_t old_data_size = current_ptr_ - begin_ptr_;
|
||||
if (size >= std::numeric_limits<size_t>::max() - reserved_size - old_data_size - 1) {
|
||||
if (size >= std::numeric_limits<size_t>::max() - RESERVED_SIZE - old_data_size - 1) {
|
||||
return false;
|
||||
}
|
||||
size_t need_data_size = old_data_size + size;
|
||||
size_t old_buffer_size = end_ptr_ - begin_ptr_;
|
||||
if (old_buffer_size >= (std::numeric_limits<size_t>::max() - reserved_size) / 2 - 2) {
|
||||
if (old_buffer_size >= (std::numeric_limits<size_t>::max() - RESERVED_SIZE) / 2 - 2) {
|
||||
return false;
|
||||
}
|
||||
size_t new_buffer_size = (old_buffer_size + 1) * 2;
|
||||
@ -116,13 +116,13 @@ bool StringBuilder::reserve_inner(size_t size) {
|
||||
if (new_buffer_size < 100) {
|
||||
new_buffer_size = 100;
|
||||
}
|
||||
new_buffer_size += reserved_size;
|
||||
new_buffer_size += RESERVED_SIZE;
|
||||
auto new_buffer = std::make_unique<char[]>(new_buffer_size);
|
||||
std::memcpy(new_buffer.get(), begin_ptr_, old_data_size);
|
||||
buffer_ = std::move(new_buffer);
|
||||
begin_ptr_ = buffer_.get();
|
||||
current_ptr_ = begin_ptr_ + old_data_size;
|
||||
end_ptr_ = begin_ptr_ + new_buffer_size - reserved_size;
|
||||
end_ptr_ = begin_ptr_ + new_buffer_size - RESERVED_SIZE;
|
||||
CHECK(end_ptr_ > current_ptr_);
|
||||
CHECK(static_cast<size_t>(end_ptr_ - current_ptr_) >= size);
|
||||
return true;
|
||||
@ -193,7 +193,7 @@ StringBuilder &StringBuilder::operator<<(FixedDouble x) {
|
||||
*ss << x.d;
|
||||
|
||||
int len = narrow_cast<int>(static_cast<std::streamoff>(ss->tellp()));
|
||||
auto left = end_ptr_ + reserved_size - current_ptr_;
|
||||
auto left = end_ptr_ + RESERVED_SIZE - current_ptr_;
|
||||
if (unlikely(len >= left)) {
|
||||
error_flag_ = true;
|
||||
len = left ? narrow_cast<int>(left - 1) : 0;
|
||||
@ -207,7 +207,7 @@ StringBuilder &StringBuilder::operator<<(const void *ptr) {
|
||||
if (unlikely(!reserve())) {
|
||||
return on_error();
|
||||
}
|
||||
current_ptr_ += std::snprintf(current_ptr_, reserved_size, "%p", ptr);
|
||||
current_ptr_ += std::snprintf(current_ptr_, RESERVED_SIZE, "%p", ptr);
|
||||
return *this;
|
||||
}
|
||||
|
||||
|
@ -26,7 +26,7 @@ class StringBuilder {
|
||||
}
|
||||
|
||||
MutableCSlice as_cslice() {
|
||||
if (current_ptr_ >= end_ptr_ + reserved_size) {
|
||||
if (current_ptr_ >= end_ptr_ + RESERVED_SIZE) {
|
||||
std::abort(); // shouldn't happen
|
||||
}
|
||||
*current_ptr_ = 0;
|
||||
@ -104,7 +104,7 @@ class StringBuilder {
|
||||
bool error_flag_ = false;
|
||||
bool use_buffer_ = false;
|
||||
std::unique_ptr<char[]> buffer_;
|
||||
static constexpr size_t reserved_size = 30;
|
||||
static constexpr size_t RESERVED_SIZE = 30;
|
||||
|
||||
StringBuilder &on_error() {
|
||||
error_flag_ = true;
|
||||
@ -115,7 +115,7 @@ class StringBuilder {
|
||||
if (end_ptr_ > current_ptr_) {
|
||||
return true;
|
||||
}
|
||||
return reserve_inner(reserved_size);
|
||||
return reserve_inner(RESERVED_SIZE);
|
||||
}
|
||||
bool reserve(size_t size) {
|
||||
if (end_ptr_ > current_ptr_ && static_cast<size_t>(end_ptr_ - current_ptr_) >= size) {
|
||||
|
@ -47,6 +47,15 @@ inline uint64 lower_bit64(uint64 x) {
|
||||
return x & bits_negate64(x);
|
||||
}
|
||||
|
||||
inline uint64 host_to_big_endian64(uint64 x) {
|
||||
// NB: works only for little-endian systems
|
||||
return bswap64(x);
|
||||
}
|
||||
inline uint64 big_endian_to_host64(uint64 x) {
|
||||
// NB: works only for little-endian systems
|
||||
return bswap64(x);
|
||||
}
|
||||
|
||||
//TODO: optimize
|
||||
inline int32 count_leading_zeroes_non_zero32(uint32 x) {
|
||||
DCHECK(x != 0);
|
||||
|
@ -5,5 +5,4 @@
|
||||
#cmakedefine01 TD_HAVE_CRC32C
|
||||
#cmakedefine01 TD_HAVE_COROUTINES
|
||||
#cmakedefine01 TD_HAVE_ABSL
|
||||
#cmakedefine01 TD_HAVE_GETOPT
|
||||
#cmakedefine01 TD_FD_DEBUG
|
||||
|
@ -8,6 +8,7 @@
|
||||
|
||||
#include "td/utils/as.h"
|
||||
#include "td/utils/BigNum.h"
|
||||
#include "td/utils/bits.h"
|
||||
#include "td/utils/common.h"
|
||||
#include "td/utils/logging.h"
|
||||
#include "td/utils/misc.h"
|
||||
@ -15,6 +16,7 @@
|
||||
#include "td/utils/port/thread_local.h"
|
||||
#include "td/utils/Random.h"
|
||||
#include "td/utils/ScopeGuard.h"
|
||||
#include "td/utils/SharedSlice.h"
|
||||
#include "td/utils/StackAllocator.h"
|
||||
#include "td/utils/StringBuilder.h"
|
||||
|
||||
@ -249,6 +251,220 @@ int pq_factorize(Slice pq_str, string *p_str, string *q_str) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct AesBlock {
|
||||
uint64 hi;
|
||||
uint64 lo;
|
||||
|
||||
uint8 *raw() {
|
||||
return reinterpret_cast<uint8 *>(this);
|
||||
}
|
||||
const uint8 *raw() const {
|
||||
return reinterpret_cast<const uint8 *>(this);
|
||||
}
|
||||
Slice as_slice() const {
|
||||
return Slice(raw(), AES_BLOCK_SIZE);
|
||||
}
|
||||
|
||||
AesBlock operator^(const AesBlock &b) const {
|
||||
AesBlock res;
|
||||
res.hi = hi ^ b.hi;
|
||||
res.lo = lo ^ b.lo;
|
||||
return res;
|
||||
}
|
||||
void operator^=(const AesBlock &b) {
|
||||
hi ^= b.hi;
|
||||
lo ^= b.lo;
|
||||
}
|
||||
|
||||
void load(const uint8 *from) {
|
||||
*this = as<AesBlock>(from);
|
||||
}
|
||||
void store(uint8 *to) {
|
||||
as<AesBlock>(to) = *this;
|
||||
}
|
||||
|
||||
AesBlock inc() const {
|
||||
#if SIZE_MAX == UINT64_MAX
|
||||
AesBlock res;
|
||||
res.lo = host_to_big_endian64(big_endian_to_host64(lo) + 1);
|
||||
if (res.lo == 0) {
|
||||
res.hi = host_to_big_endian64(big_endian_to_host64(hi) + 1);
|
||||
} else {
|
||||
res.hi = hi;
|
||||
}
|
||||
return res;
|
||||
#else
|
||||
AesBlock res = *this;
|
||||
auto ptr = res.raw();
|
||||
if (++ptr[15] == 0) {
|
||||
for (int i = 14; i >= 0; i--) {
|
||||
if (++ptr[i] != 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return res;
|
||||
#endif
|
||||
}
|
||||
};
|
||||
static_assert(sizeof(AesBlock) == 16, "");
|
||||
static_assert(sizeof(AesBlock) == AES_BLOCK_SIZE, "");
|
||||
|
||||
class XorBytes {
|
||||
public:
|
||||
static void run(const uint8 *a, const uint8 *b, uint8 *c, size_t n) {
|
||||
static constexpr int BLOCK_SIZE = 16;
|
||||
auto block_cnt = n / BLOCK_SIZE;
|
||||
n -= block_cnt * BLOCK_SIZE;
|
||||
while (block_cnt-- > 0) {
|
||||
Block<BLOCK_SIZE> a_big = as<Block<BLOCK_SIZE>>(a);
|
||||
Block<BLOCK_SIZE> b_big = as<Block<BLOCK_SIZE>>(b);
|
||||
as<Block<BLOCK_SIZE>>(c) = a_big ^ b_big;
|
||||
a += BLOCK_SIZE;
|
||||
b += BLOCK_SIZE;
|
||||
c += BLOCK_SIZE;
|
||||
}
|
||||
while (n-- > 0) {
|
||||
c[n] = a[n] ^ b[n];
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
template <size_t N>
|
||||
struct alignas(N) Block {
|
||||
uint8 data[N];
|
||||
Block operator^(const Block &b) const & {
|
||||
Block res;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
res.data[i] = data[i] ^ b.data[i];
|
||||
}
|
||||
return res;
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
struct AesCtrCounterPack {
|
||||
static constexpr size_t BLOCK_COUNT = 32;
|
||||
AesBlock blocks[BLOCK_COUNT];
|
||||
uint8 *raw() {
|
||||
return reinterpret_cast<uint8 *>(this);
|
||||
}
|
||||
const uint8 *raw() const {
|
||||
return reinterpret_cast<const uint8 *>(this);
|
||||
}
|
||||
|
||||
static size_t size() {
|
||||
return sizeof(blocks);
|
||||
}
|
||||
|
||||
void init(AesBlock block) {
|
||||
blocks[0] = block;
|
||||
for (size_t i = 1; i < BLOCK_COUNT; i++) {
|
||||
blocks[i] = blocks[i - 1].inc();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
class Evp {
|
||||
public:
|
||||
Evp() {
|
||||
ctx_ = EVP_CIPHER_CTX_new();
|
||||
LOG_IF(FATAL, ctx_ == nullptr);
|
||||
}
|
||||
Evp(const Evp &from) = delete;
|
||||
Evp &operator=(const Evp &from) = delete;
|
||||
Evp(Evp &&from) = delete;
|
||||
Evp &operator=(Evp &&from) = delete;
|
||||
~Evp() {
|
||||
CHECK(ctx_ != nullptr);
|
||||
EVP_CIPHER_CTX_free(ctx_);
|
||||
}
|
||||
|
||||
void init_encrypt_ecb(Slice key) {
|
||||
init(Type::Ecb, true, EVP_aes_256_ecb(), key);
|
||||
}
|
||||
|
||||
void init_decrypt_ecb(Slice key) {
|
||||
init(Type::Ecb, false, EVP_aes_256_ecb(), key);
|
||||
}
|
||||
|
||||
void init_encrypt_cbc(Slice key) {
|
||||
init(Type::Cbc, true, EVP_aes_256_cbc(), key);
|
||||
}
|
||||
|
||||
void init_decrypt_cbc(Slice key) {
|
||||
init(Type::Cbc, false, EVP_aes_256_cbc(), key);
|
||||
}
|
||||
|
||||
void init_iv(Slice iv) {
|
||||
int res = EVP_CipherInit_ex(ctx_, nullptr, nullptr, nullptr, iv.ubegin(), -1);
|
||||
LOG_IF(FATAL, res != 1);
|
||||
}
|
||||
|
||||
void encrypt(const uint8 *src, uint8 *dst, int size) {
|
||||
// CHECK(type_ != Type::Empty && is_encrypt_);
|
||||
CHECK(size % AES_BLOCK_SIZE == 0);
|
||||
int len;
|
||||
int res = EVP_EncryptUpdate(ctx_, dst, &len, src, size);
|
||||
LOG_IF(FATAL, res != 1);
|
||||
CHECK(len == size);
|
||||
}
|
||||
|
||||
void decrypt(const uint8 *src, uint8 *dst, int size) {
|
||||
// CHECK(type_ != Type::Empty && !is_encrypt_);
|
||||
CHECK(size % AES_BLOCK_SIZE == 0);
|
||||
int len;
|
||||
int res = EVP_DecryptUpdate(ctx_, dst, &len, src, size);
|
||||
LOG_IF(FATAL, res != 1);
|
||||
CHECK(len == size);
|
||||
}
|
||||
|
||||
private:
|
||||
EVP_CIPHER_CTX *ctx_{nullptr};
|
||||
enum class Type : int8 { Empty, Ecb, Cbc };
|
||||
// Type type_{Type::Empty};
|
||||
// bool is_encrypt_ = false;
|
||||
|
||||
void init(Type type, bool is_encrypt, const EVP_CIPHER *cipher, Slice key) {
|
||||
// type_ = type;
|
||||
// is_encrypt_ = is_encrypt;
|
||||
int res = EVP_CipherInit_ex(ctx_, cipher, nullptr, key.ubegin(), nullptr, is_encrypt ? 1 : 0);
|
||||
LOG_IF(FATAL, res != 1);
|
||||
EVP_CIPHER_CTX_set_padding(ctx_, 0);
|
||||
}
|
||||
};
|
||||
|
||||
struct AesState::Impl {
|
||||
Evp evp;
|
||||
};
|
||||
|
||||
AesState::AesState() = default;
|
||||
AesState::AesState(AesState &&from) = default;
|
||||
AesState &AesState::operator=(AesState &&from) = default;
|
||||
AesState::~AesState() = default;
|
||||
|
||||
void AesState::init(Slice key, bool encrypt) {
|
||||
CHECK(key.size() == 32);
|
||||
if (!impl_) {
|
||||
impl_ = make_unique<Impl>();
|
||||
}
|
||||
if (encrypt) {
|
||||
impl_->evp.init_encrypt_ecb(key);
|
||||
} else {
|
||||
impl_->evp.init_decrypt_ecb(key);
|
||||
}
|
||||
}
|
||||
|
||||
void AesState::encrypt(const uint8 *src, uint8 *dst, int size) {
|
||||
CHECK(impl_);
|
||||
impl_->evp.encrypt(src, dst, size);
|
||||
}
|
||||
|
||||
void AesState::decrypt(const uint8 *src, uint8 *dst, int size) {
|
||||
CHECK(impl_);
|
||||
impl_->evp.decrypt(src, dst, size);
|
||||
}
|
||||
|
||||
static void aes_ige_xcrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to, bool encrypt_flag) {
|
||||
CHECK(aes_key.size() == 32);
|
||||
CHECK(aes_iv.size() == 32);
|
||||
@ -264,12 +480,128 @@ static void aes_ige_xcrypt(Slice aes_key, MutableSlice aes_iv, Slice from, Mutab
|
||||
AES_ige_encrypt(from.ubegin(), to.ubegin(), from.size(), &key, aes_iv.ubegin(), encrypt_flag);
|
||||
}
|
||||
|
||||
class AesIgeStateImpl {
|
||||
public:
|
||||
void init(Slice key, Slice iv, bool encrypt) {
|
||||
CHECK(key.size() == 32);
|
||||
CHECK(iv.size() == 32);
|
||||
if (encrypt) {
|
||||
evp_.init_encrypt_cbc(key);
|
||||
} else {
|
||||
evp_.init_decrypt_ecb(key);
|
||||
}
|
||||
|
||||
encrypted_iv_.load(iv.ubegin());
|
||||
plaintext_iv_.load(iv.ubegin() + AES_BLOCK_SIZE);
|
||||
}
|
||||
void encrypt(Slice from, MutableSlice to) {
|
||||
CHECK(from.size() % AES_BLOCK_SIZE == 0);
|
||||
CHECK(to.size() >= from.size());
|
||||
auto len = to.size() / AES_BLOCK_SIZE;
|
||||
auto in = from.ubegin();
|
||||
auto out = to.ubegin();
|
||||
|
||||
static constexpr size_t BLOCK_COUNT = 31;
|
||||
while (len != 0) {
|
||||
AesBlock data[BLOCK_COUNT];
|
||||
AesBlock data_xored[BLOCK_COUNT];
|
||||
|
||||
auto count = td::min(BLOCK_COUNT, len);
|
||||
std::memcpy(data, in, AES_BLOCK_SIZE * count);
|
||||
data_xored[0] = data[0];
|
||||
if (count > 1) {
|
||||
data_xored[1] = plaintext_iv_ ^ data[1];
|
||||
for (size_t i = 2; i < count; i++) {
|
||||
data_xored[i] = data[i - 2] ^ data[i];
|
||||
}
|
||||
}
|
||||
|
||||
evp_.init_iv(encrypted_iv_.as_slice());
|
||||
int inlen = static_cast<int>(AES_BLOCK_SIZE * count);
|
||||
evp_.encrypt(data_xored[0].raw(), data_xored[0].raw(), inlen);
|
||||
|
||||
data_xored[0] ^= plaintext_iv_;
|
||||
for (size_t i = 1; i < count; i++) {
|
||||
data_xored[i] ^= data[i - 1];
|
||||
}
|
||||
plaintext_iv_ = data[count - 1];
|
||||
encrypted_iv_ = data_xored[count - 1];
|
||||
|
||||
std::memcpy(out, data_xored, AES_BLOCK_SIZE * count);
|
||||
len -= count;
|
||||
in += AES_BLOCK_SIZE * count;
|
||||
out += AES_BLOCK_SIZE * count;
|
||||
}
|
||||
}
|
||||
|
||||
void decrypt(Slice from, MutableSlice to) {
|
||||
CHECK(from.size() % AES_BLOCK_SIZE == 0);
|
||||
CHECK(to.size() >= from.size());
|
||||
auto len = to.size() / AES_BLOCK_SIZE;
|
||||
auto in = from.ubegin();
|
||||
auto out = to.ubegin();
|
||||
|
||||
AesBlock encrypted;
|
||||
|
||||
while (len) {
|
||||
encrypted.load(in);
|
||||
|
||||
plaintext_iv_ ^= encrypted;
|
||||
evp_.decrypt(plaintext_iv_.raw(), plaintext_iv_.raw(), AES_BLOCK_SIZE);
|
||||
plaintext_iv_ ^= encrypted_iv_;
|
||||
|
||||
plaintext_iv_.store(out);
|
||||
encrypted_iv_ = encrypted;
|
||||
|
||||
--len;
|
||||
in += AES_BLOCK_SIZE;
|
||||
out += AES_BLOCK_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
Evp evp_;
|
||||
AesBlock encrypted_iv_;
|
||||
AesBlock plaintext_iv_;
|
||||
};
|
||||
|
||||
AesIgeState::AesIgeState() = default;
|
||||
AesIgeState::AesIgeState(AesIgeState &&from) = default;
|
||||
AesIgeState &AesIgeState::operator=(AesIgeState &&from) = default;
|
||||
AesIgeState::~AesIgeState() = default;
|
||||
|
||||
void AesIgeState::init(Slice key, Slice iv, bool encrypt) {
|
||||
if (!impl_) {
|
||||
impl_ = make_unique<AesIgeStateImpl>();
|
||||
}
|
||||
|
||||
impl_->init(key, iv, encrypt);
|
||||
}
|
||||
|
||||
void AesIgeState::encrypt(Slice from, MutableSlice to) {
|
||||
impl_->encrypt(from, to);
|
||||
}
|
||||
|
||||
void AesIgeState::decrypt(Slice from, MutableSlice to) {
|
||||
impl_->decrypt(from, to);
|
||||
}
|
||||
|
||||
void aes_ige_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) {
|
||||
aes_ige_xcrypt(aes_key, aes_iv, from, to, true);
|
||||
if (from.size() <= 128) {
|
||||
return aes_ige_xcrypt(aes_key, aes_iv, from, to, true);
|
||||
}
|
||||
AesIgeStateImpl state;
|
||||
state.init(aes_key, aes_iv, true);
|
||||
state.encrypt(from, to);
|
||||
}
|
||||
|
||||
void aes_ige_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to) {
|
||||
aes_ige_xcrypt(aes_key, aes_iv, from, to, false);
|
||||
if (from.size() <= 128) {
|
||||
return aes_ige_xcrypt(aes_key, aes_iv, from, to, false);
|
||||
}
|
||||
AesIgeStateImpl state;
|
||||
state.init(aes_key, aes_iv, false);
|
||||
state.decrypt(from, to);
|
||||
}
|
||||
|
||||
static void aes_cbc_xcrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to, bool encrypt_flag) {
|
||||
@ -313,35 +645,43 @@ class AesCtrState::Impl {
|
||||
CHECK(key.size() == 32);
|
||||
CHECK(iv.size() == 16);
|
||||
static_assert(AES_BLOCK_SIZE == 16, "");
|
||||
if (AES_set_encrypt_key(key.ubegin(), 256, &aes_key) < 0) {
|
||||
LOG(FATAL) << "Failed to set encrypt key";
|
||||
}
|
||||
counter.as_mutable_slice().copy_from(iv);
|
||||
current_pos = 0;
|
||||
evp_.init_encrypt_ecb(key);
|
||||
counter_.load(iv.ubegin());
|
||||
fill();
|
||||
}
|
||||
|
||||
void encrypt(Slice from, MutableSlice to) {
|
||||
CHECK(to.size() >= from.size());
|
||||
for (size_t i = 0; i < from.size(); i++) {
|
||||
if (current_pos == 0) {
|
||||
AES_encrypt(counter.as_slice().ubegin(), encrypted_counter.as_mutable_slice().ubegin(), &aes_key);
|
||||
uint8 *ptr = counter.as_mutable_slice().ubegin();
|
||||
for (int j = 15; j >= 0; j--) {
|
||||
if (++ptr[j] != 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
auto *src = from.ubegin();
|
||||
auto *dst = to.ubegin();
|
||||
auto n = from.size();
|
||||
while (n != 0) {
|
||||
size_t left = encrypted_counter_.raw() + AesCtrCounterPack::size() - current_;
|
||||
if (left == 0) {
|
||||
fill();
|
||||
left = AesCtrCounterPack::size();
|
||||
}
|
||||
to[i] = static_cast<char>(from[i] ^ encrypted_counter[current_pos]);
|
||||
current_pos = (current_pos + 1) & 15;
|
||||
size_t min_n = td::min(n, left);
|
||||
XorBytes::run(src, current_, dst, min_n);
|
||||
src += min_n;
|
||||
dst += min_n;
|
||||
n -= min_n;
|
||||
current_ += min_n;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
AES_KEY aes_key;
|
||||
SecureString counter{AES_BLOCK_SIZE};
|
||||
SecureString encrypted_counter{AES_BLOCK_SIZE};
|
||||
uint8 current_pos;
|
||||
Evp evp_;
|
||||
|
||||
uint8 *current_;
|
||||
AesBlock counter_;
|
||||
AesCtrCounterPack encrypted_counter_;
|
||||
|
||||
void fill() {
|
||||
encrypted_counter_.init(counter_);
|
||||
counter_ = encrypted_counter_.blocks[AesCtrCounterPack::BLOCK_COUNT - 1].inc();
|
||||
current_ = encrypted_counter_.raw();
|
||||
evp_.encrypt(current_, current_, static_cast<int>(AesCtrCounterPack::size()));
|
||||
}
|
||||
};
|
||||
|
||||
AesCtrState::AesCtrState() = default;
|
||||
|
@ -21,9 +21,50 @@ void init_crypto();
|
||||
|
||||
int pq_factorize(Slice pq_str, string *p_str, string *q_str);
|
||||
|
||||
class AesState {
|
||||
public:
|
||||
AesState();
|
||||
AesState(const AesState &from) = delete;
|
||||
AesState &operator=(const AesState &from) = delete;
|
||||
AesState(AesState &&from);
|
||||
AesState &operator=(AesState &&from);
|
||||
~AesState();
|
||||
|
||||
void init(Slice key, bool encrypt);
|
||||
|
||||
void encrypt(const uint8 *src, uint8 *dst, int size);
|
||||
|
||||
void decrypt(const uint8 *src, uint8 *dst, int size);
|
||||
|
||||
private:
|
||||
struct Impl;
|
||||
unique_ptr<Impl> impl_;
|
||||
};
|
||||
|
||||
void aes_ige_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
|
||||
void aes_ige_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
|
||||
|
||||
class AesIgeStateImpl;
|
||||
|
||||
class AesIgeState {
|
||||
public:
|
||||
AesIgeState();
|
||||
AesIgeState(const AesIgeState &from) = delete;
|
||||
AesIgeState &operator=(const AesIgeState &from) = delete;
|
||||
AesIgeState(AesIgeState &&from);
|
||||
AesIgeState &operator=(AesIgeState &&from);
|
||||
~AesIgeState();
|
||||
|
||||
void init(Slice key, Slice iv, bool encrypt);
|
||||
|
||||
void encrypt(Slice from, MutableSlice to);
|
||||
|
||||
void decrypt(Slice from, MutableSlice to);
|
||||
|
||||
private:
|
||||
unique_ptr<AesIgeStateImpl> impl_;
|
||||
};
|
||||
|
||||
void aes_cbc_encrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
|
||||
void aes_cbc_decrypt(Slice aes_key, MutableSlice aes_iv, Slice from, MutableSlice to);
|
||||
|
||||
|
@ -45,6 +45,9 @@ TD_THREAD_LOCAL const char *Logger::tag2_ = nullptr;
|
||||
Logger::Logger(LogInterface &log, const LogOptions &options, int log_level, Slice file_name, int line_num,
|
||||
Slice comment)
|
||||
: Logger(log, options, log_level) {
|
||||
if (log_level == VERBOSITY_NAME(PLAIN) && &options == &log_options) {
|
||||
return;
|
||||
}
|
||||
if (!options_.add_info) {
|
||||
return;
|
||||
}
|
||||
|
@ -15,6 +15,7 @@
|
||||
#include "td/utils/logging.h"
|
||||
#include "td/utils/misc.h"
|
||||
#include "td/utils/port/detail/PollableFd.h"
|
||||
#include "td/utils/port/detail/skip_eintr.h"
|
||||
#include "td/utils/port/PollFlags.h"
|
||||
#include "td/utils/port/sleep.h"
|
||||
#include "td/utils/ScopeGuard.h"
|
||||
|
@ -10,6 +10,7 @@
|
||||
|
||||
#include "td/utils/common.h"
|
||||
#include "td/utils/logging.h"
|
||||
#include "td/utils/port/detail/skip_eintr.h"
|
||||
#include "td/utils/port/IPAddress.h"
|
||||
#include "td/utils/port/PollFlags.h"
|
||||
|
||||
|
@ -10,6 +10,7 @@
|
||||
#include "td/utils/format.h"
|
||||
#include "td/utils/logging.h"
|
||||
#include "td/utils/misc.h"
|
||||
#include "td/utils/port/detail/skip_eintr.h"
|
||||
#include "td/utils/port/PollFlags.h"
|
||||
|
||||
#if TD_PORT_WINDOWS
|
||||
|
@ -6,7 +6,6 @@
|
||||
//
|
||||
#include "td/utils/port/Stat.h"
|
||||
|
||||
#include "td/utils/port/detail/PollableFd.h"
|
||||
#include "td/utils/port/FileFd.h"
|
||||
|
||||
#if TD_PORT_POSIX
|
||||
@ -15,6 +14,7 @@
|
||||
#include "td/utils/logging.h"
|
||||
#include "td/utils/misc.h"
|
||||
#include "td/utils/port/Clocks.h"
|
||||
#include "td/utils/port/detail/skip_eintr.h"
|
||||
#include "td/utils/ScopeGuard.h"
|
||||
|
||||
#include <utility>
|
||||
@ -38,8 +38,20 @@
|
||||
#include <sys/syscall.h>
|
||||
#endif
|
||||
|
||||
#elif TD_PORT_WINDOWS
|
||||
|
||||
#include "td/utils/port/thread.h"
|
||||
|
||||
#ifndef PSAPI_VERSION
|
||||
#define PSAPI_VERSION 1
|
||||
#endif
|
||||
#include <psapi.h>
|
||||
|
||||
#endif
|
||||
|
||||
namespace td {
|
||||
|
||||
#if TD_PORT_POSIX
|
||||
namespace detail {
|
||||
|
||||
template <class...>
|
||||
@ -176,14 +188,20 @@ Status update_atime(CSlice path) {
|
||||
};
|
||||
return detail::update_atime(file.get_native_fd().fd());
|
||||
}
|
||||
#endif
|
||||
|
||||
Result<Stat> stat(CSlice path) {
|
||||
#if TD_PORT_POSIX
|
||||
struct ::stat buf;
|
||||
int err = detail::skip_eintr([&] { return ::stat(path.c_str(), &buf); });
|
||||
if (err < 0) {
|
||||
return OS_ERROR(PSLICE() << "Stat for file \"" << path << "\" failed");
|
||||
}
|
||||
return detail::from_native_stat(buf);
|
||||
#elif TD_PORT_WINDOWS
|
||||
TRY_RESULT(fd, FileFd::open(path, FileFd::Flags::Read | FileFd::PrivateFlags::WinStat));
|
||||
return fd.stat();
|
||||
#endif
|
||||
}
|
||||
|
||||
Result<MemStat> mem_stat() {
|
||||
@ -260,6 +278,19 @@ Result<MemStat> mem_stat() {
|
||||
s++;
|
||||
}
|
||||
|
||||
return res;
|
||||
#elif TD_WINDOWS
|
||||
PROCESS_MEMORY_COUNTERS_EX counters;
|
||||
if (!GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PROCESS_MEMORY_COUNTERS *>(&counters),
|
||||
sizeof(counters))) {
|
||||
return Status::Error("Call to GetProcessMemoryInfo failed");
|
||||
}
|
||||
|
||||
MemStat res;
|
||||
res.resident_size_ = counters.WorkingSetSize;
|
||||
res.resident_size_peak_ = counters.PeakWorkingSetSize;
|
||||
res.virtual_size_ = counters.PrivateUsage;
|
||||
res.virtual_size_peak_ = counters.PeakPagefileUsage;
|
||||
return res;
|
||||
#else
|
||||
return Status::Error("Not supported");
|
||||
@ -276,7 +307,9 @@ Status cpu_stat_self(CpuStat &stat) {
|
||||
constexpr int TMEM_SIZE = 10000;
|
||||
char mem[TMEM_SIZE];
|
||||
TRY_RESULT(size, fd.read(MutableSlice(mem, TMEM_SIZE - 1)));
|
||||
CHECK(size < TMEM_SIZE - 1);
|
||||
if (size >= TMEM_SIZE - 1) {
|
||||
return Status::Error("Failed for read /proc/self/stat");
|
||||
}
|
||||
mem[size] = 0;
|
||||
|
||||
char *s = mem;
|
||||
@ -285,10 +318,10 @@ Status cpu_stat_self(CpuStat &stat) {
|
||||
|
||||
while (pass_cnt < 15) {
|
||||
if (pass_cnt == 13) {
|
||||
stat.process_user_ticks = to_integer<uint64>(Slice(s, t));
|
||||
stat.process_user_ticks_ = to_integer<uint64>(Slice(s, t));
|
||||
}
|
||||
if (pass_cnt == 14) {
|
||||
stat.process_system_ticks = to_integer<uint64>(Slice(s, t));
|
||||
stat.process_system_ticks_ = to_integer<uint64>(Slice(s, t));
|
||||
}
|
||||
while (*s && *s != ' ') {
|
||||
s++;
|
||||
@ -302,6 +335,7 @@ Status cpu_stat_self(CpuStat &stat) {
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status cpu_stat_total(CpuStat &stat) {
|
||||
TRY_RESULT(fd, FileFd::open("/proc/stat", FileFd::Read));
|
||||
SCOPE_EXIT {
|
||||
@ -311,14 +345,17 @@ Status cpu_stat_total(CpuStat &stat) {
|
||||
constexpr int TMEM_SIZE = 10000;
|
||||
char mem[TMEM_SIZE];
|
||||
TRY_RESULT(size, fd.read(MutableSlice(mem, TMEM_SIZE - 1)));
|
||||
CHECK(size < TMEM_SIZE - 1);
|
||||
if (size >= TMEM_SIZE - 1) {
|
||||
return Status::Error("Failed for read /proc/stat");
|
||||
}
|
||||
mem[size] = 0;
|
||||
|
||||
uint64 sum = 0, cur = 0;
|
||||
uint64 sum = 0;
|
||||
uint64 cur = 0;
|
||||
for (size_t i = 0; i < size; i++) {
|
||||
int c = mem[i];
|
||||
char c = mem[i];
|
||||
if (c >= '0' && c <= '9') {
|
||||
cur = cur * 10 + (uint64)c - '0';
|
||||
cur = cur * 10 + static_cast<uint64>(c) - '0';
|
||||
} else {
|
||||
sum += cur;
|
||||
cur = 0;
|
||||
@ -328,7 +365,7 @@ Status cpu_stat_total(CpuStat &stat) {
|
||||
}
|
||||
}
|
||||
|
||||
stat.total_ticks = sum;
|
||||
stat.total_ticks_ = sum;
|
||||
return Status::OK();
|
||||
}
|
||||
#endif
|
||||
@ -338,25 +375,28 @@ Result<CpuStat> cpu_stat() {
|
||||
CpuStat stat;
|
||||
TRY_STATUS(cpu_stat_self(stat));
|
||||
TRY_STATUS(cpu_stat_total(stat));
|
||||
return stat;
|
||||
#elif TD_WINDOWS
|
||||
CpuStat stat;
|
||||
stat.total_ticks_ = static_cast<uint64>(GetTickCount64()) * 10000;
|
||||
auto hardware_concurrency = thread::hardware_concurrency();
|
||||
if (hardware_concurrency != 0) {
|
||||
stat.total_ticks_ *= hardware_concurrency;
|
||||
}
|
||||
|
||||
FILETIME ignored_time;
|
||||
FILETIME kernel_time;
|
||||
FILETIME user_time;
|
||||
if (!GetProcessTimes(GetCurrentProcess(), &ignored_time, &ignored_time, &kernel_time, &user_time)) {
|
||||
return Status::Error("Failed to call GetProcessTimes");
|
||||
}
|
||||
stat.process_system_ticks_ = kernel_time.dwLowDateTime + (static_cast<uint64>(kernel_time.dwHighDateTime) << 32);
|
||||
stat.process_user_ticks_ = user_time.dwLowDateTime + (static_cast<uint64>(user_time.dwHighDateTime) << 32);
|
||||
|
||||
return stat;
|
||||
#else
|
||||
return Status::Error("Not supported");
|
||||
#endif
|
||||
}
|
||||
} // namespace td
|
||||
#endif
|
||||
|
||||
#if TD_PORT_WINDOWS
|
||||
namespace td {
|
||||
|
||||
Result<Stat> stat(CSlice path) {
|
||||
TRY_RESULT(fd, FileFd::open(path, FileFd::Flags::Read | FileFd::PrivateFlags::WinStat));
|
||||
return fd.stat();
|
||||
}
|
||||
|
||||
Result<CpuStat> cpu_stat() {
|
||||
return Status::Error("Not supported");
|
||||
}
|
||||
|
||||
} // namespace td
|
||||
#endif
|
||||
|
@ -26,20 +26,13 @@ struct Stat {
|
||||
Result<Stat> stat(CSlice path) TD_WARN_UNUSED_RESULT;
|
||||
|
||||
struct CpuStat {
|
||||
uint64 total_ticks{0};
|
||||
uint64 process_user_ticks{0};
|
||||
uint64 process_system_ticks{0};
|
||||
uint64 total_ticks_{0};
|
||||
uint64 process_user_ticks_{0};
|
||||
uint64 process_system_ticks_{0};
|
||||
};
|
||||
|
||||
Result<CpuStat> cpu_stat() TD_WARN_UNUSED_RESULT;
|
||||
|
||||
#if TD_PORT_POSIX
|
||||
|
||||
namespace detail {
|
||||
Result<Stat> fstat(int native_fd);
|
||||
} // namespace detail
|
||||
|
||||
Status update_atime(CSlice path) TD_WARN_UNUSED_RESULT;
|
||||
|
||||
struct MemStat {
|
||||
uint64 resident_size_ = 0;
|
||||
uint64 resident_size_peak_ = 0;
|
||||
@ -49,6 +42,14 @@ struct MemStat {
|
||||
|
||||
Result<MemStat> mem_stat() TD_WARN_UNUSED_RESULT;
|
||||
|
||||
#if TD_PORT_POSIX
|
||||
|
||||
namespace detail {
|
||||
Result<Stat> fstat(int native_fd);
|
||||
} // namespace detail
|
||||
|
||||
Status update_atime(CSlice path) TD_WARN_UNUSED_RESULT;
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace td
|
||||
|
@ -10,6 +10,7 @@
|
||||
#include "td/utils/format.h"
|
||||
#include "td/utils/logging.h"
|
||||
#include "td/utils/misc.h"
|
||||
#include "td/utils/port/detail/skip_eintr.h"
|
||||
#include "td/utils/port/PollFlags.h"
|
||||
#include "td/utils/port/SocketFd.h"
|
||||
#include "td/utils/VectorQueue.h"
|
||||
@ -169,7 +170,8 @@ class UdpSocketFdImpl : private Iocp::Callback {
|
||||
UdpMessage to_receive_;
|
||||
WSAMSG receive_message_;
|
||||
UdpSocketReceiveHelper receive_helper_;
|
||||
enum : size_t { MAX_PACKET_SIZE = 2048, RESERVED_SIZE = MAX_PACKET_SIZE * 8 };
|
||||
static constexpr size_t MAX_PACKET_SIZE = 2048;
|
||||
static constexpr size_t RESERVED_SIZE = MAX_PACKET_SIZE * 8;
|
||||
BufferSlice receive_buffer_;
|
||||
|
||||
UdpMessage to_send_;
|
||||
@ -812,11 +814,11 @@ static Result<uint32> maximize_buffer(int socket_fd, int optname, uint32 max) {
|
||||
}
|
||||
|
||||
Result<uint32> UdpSocketFd::maximize_snd_buffer(uint32 max) {
|
||||
return maximize_buffer(get_native_fd().fd(), SO_SNDBUF, max == 0 ? default_udp_max_snd_buffer_size : max);
|
||||
return maximize_buffer(get_native_fd().fd(), SO_SNDBUF, max == 0 ? DEFAULT_UDP_MAX_SND_BUFFER_SIZE : max);
|
||||
}
|
||||
|
||||
Result<uint32> UdpSocketFd::maximize_rcv_buffer(uint32 max) {
|
||||
return maximize_buffer(get_native_fd().fd(), SO_RCVBUF, max == 0 ? default_udp_max_rcv_buffer_size : max);
|
||||
return maximize_buffer(get_native_fd().fd(), SO_RCVBUF, max == 0 ? DEFAULT_UDP_MAX_RCV_BUFFER_SIZE : max);
|
||||
}
|
||||
#else
|
||||
Result<uint32> UdpSocketFd::maximize_snd_buffer(uint32 max) {
|
||||
|
@ -84,8 +84,8 @@ class UdpSocketFd {
|
||||
#endif
|
||||
|
||||
private:
|
||||
static constexpr uint32 default_udp_max_snd_buffer_size = (1 << 24);
|
||||
static constexpr uint32 default_udp_max_rcv_buffer_size = (1 << 24);
|
||||
static constexpr uint32 DEFAULT_UDP_MAX_SND_BUFFER_SIZE = (1 << 24);
|
||||
static constexpr uint32 DEFAULT_UDP_MAX_RCV_BUFFER_SIZE = (1 << 24);
|
||||
std::unique_ptr<detail::UdpSocketFdImpl, detail::UdpSocketFdImplDeleter> impl_;
|
||||
explicit UdpSocketFd(unique_ptr<detail::UdpSocketFdImpl> impl);
|
||||
};
|
||||
|
@ -13,6 +13,7 @@ char disable_linker_warning_about_empty_file_event_fd_linux_cpp TD_UNUSED;
|
||||
#include "td/utils/logging.h"
|
||||
#include "td/utils/misc.h"
|
||||
#include "td/utils/port/detail/NativeFd.h"
|
||||
#include "td/utils/port/detail/skip_eintr.h"
|
||||
#include "td/utils/port/PollFlags.h"
|
||||
#include "td/utils/ScopeGuard.h"
|
||||
#include "td/utils/Slice.h"
|
||||
|
@ -16,9 +16,7 @@
|
||||
#include "td/utils/SpinLock.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <cerrno>
|
||||
#include <memory>
|
||||
#include <type_traits>
|
||||
|
||||
namespace td {
|
||||
|
||||
@ -209,30 +207,6 @@ inline const NativeFd &PollableFd::native_fd() const {
|
||||
return fd_info_->native_fd();
|
||||
}
|
||||
|
||||
#if TD_PORT_POSIX
|
||||
namespace detail {
|
||||
template <class F>
|
||||
auto skip_eintr(F &&f) {
|
||||
decltype(f()) res;
|
||||
static_assert(std::is_integral<decltype(res)>::value, "integral type expected");
|
||||
do {
|
||||
errno = 0; // just in case
|
||||
res = f();
|
||||
} while (res < 0 && errno == EINTR);
|
||||
return res;
|
||||
}
|
||||
template <class F>
|
||||
auto skip_eintr_cstr(F &&f) {
|
||||
char *res;
|
||||
do {
|
||||
errno = 0; // just in case
|
||||
res = f();
|
||||
} while (res == nullptr && errno == EINTR);
|
||||
return res;
|
||||
}
|
||||
} // namespace detail
|
||||
#endif
|
||||
|
||||
template <class FdT>
|
||||
bool can_read(const FdT &fd) {
|
||||
return fd.get_poll_info().get_flags().can_read() || fd.get_poll_info().get_flags().has_pending_error();
|
||||
|
39
tdutils/td/utils/port/detail/skip_eintr.h
Normal file
39
tdutils/td/utils/port/detail/skip_eintr.h
Normal file
@ -0,0 +1,39 @@
|
||||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#include <cerrno>
|
||||
#include <type_traits>
|
||||
|
||||
namespace td {
|
||||
|
||||
#if TD_PORT_POSIX
|
||||
namespace detail {
|
||||
template <class F>
|
||||
auto skip_eintr(F &&f) {
|
||||
decltype(f()) res;
|
||||
static_assert(std::is_integral<decltype(res)>::value, "integral type expected");
|
||||
do {
|
||||
errno = 0; // just in case
|
||||
res = f();
|
||||
} while (res < 0 && errno == EINTR);
|
||||
return res;
|
||||
}
|
||||
|
||||
template <class F>
|
||||
auto skip_eintr_cstr(F &&f) {
|
||||
char *res;
|
||||
do {
|
||||
errno = 0; // just in case
|
||||
res = f();
|
||||
} while (res == nullptr && errno == EINTR);
|
||||
return res;
|
||||
}
|
||||
} // namespace detail
|
||||
#endif
|
||||
|
||||
} // namespace td
|
@ -10,7 +10,7 @@
|
||||
|
||||
#include "td/utils/format.h"
|
||||
#include "td/utils/logging.h"
|
||||
#include "td/utils/port/detail/PollableFd.h"
|
||||
#include "td/utils/port/detail/skip_eintr.h"
|
||||
#include "td/utils/ScopeGuard.h"
|
||||
|
||||
#if TD_PORT_WINDOWS
|
||||
|
57
tdutils/td/utils/port/rlimit.cpp
Normal file
57
tdutils/td/utils/port/rlimit.cpp
Normal file
@ -0,0 +1,57 @@
|
||||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#include "td/utils/port/rlimit.h"
|
||||
|
||||
#include "td/utils/port/config.h"
|
||||
|
||||
#include "td/utils/misc.h"
|
||||
|
||||
#if TD_PORT_POSIX
|
||||
#include <sys/resource.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#endif
|
||||
|
||||
namespace td {
|
||||
|
||||
#if TD_PORT_POSIX
|
||||
static int get_resource(ResourceLimitType type) {
|
||||
switch (type) {
|
||||
case ResourceLimitType::NoFile:
|
||||
return RLIMIT_NOFILE;
|
||||
default:
|
||||
UNREACHABLE();
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
Status set_resource_limit(ResourceLimitType type, uint64 value) {
|
||||
#if TD_PORT_POSIX
|
||||
int resource = get_resource(type);
|
||||
|
||||
rlimit rlim;
|
||||
if (getrlimit(resource, &rlim) == -1) {
|
||||
return OS_ERROR("Failed to get current resource limit");
|
||||
}
|
||||
|
||||
TRY_RESULT(new_value, narrow_cast_safe<rlim_t>(value));
|
||||
if (rlim.rlim_max < new_value) {
|
||||
rlim.rlim_max = new_value;
|
||||
}
|
||||
rlim.rlim_cur = new_value;
|
||||
|
||||
if (setrlimit(resource, &rlim) < 0) {
|
||||
return OS_ERROR("Failed to set resource limit");
|
||||
}
|
||||
return Status::OK();
|
||||
#elif TD_PORT_WINDOWS
|
||||
return Status::OK(); // Windows has no limits
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace td
|
@ -6,15 +6,13 @@
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#include "td/db/binlog/ConcurrentBinlog.h"
|
||||
#include "td/db/BinlogKeyValue.h"
|
||||
|
||||
#include <memory>
|
||||
#include "td/utils/common.h"
|
||||
#include "td/utils/Status.h"
|
||||
|
||||
namespace td {
|
||||
|
||||
using BinlogPmcBase = BinlogKeyValue<ConcurrentBinlog>;
|
||||
using BinlogPmc = std::shared_ptr<BinlogPmcBase>;
|
||||
using BinlogPmcPtr = BinlogPmcBase *;
|
||||
enum class ResourceLimitType { NoFile };
|
||||
|
||||
Status set_resource_limit(ResourceLimitType type, uint64 value);
|
||||
|
||||
} // namespace td
|
55
tdutils/td/utils/port/user.cpp
Normal file
55
tdutils/td/utils/port/user.cpp
Normal file
@ -0,0 +1,55 @@
|
||||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#include "td/utils/port/user.h"
|
||||
|
||||
#include "td/utils/port/config.h"
|
||||
|
||||
#if TD_PORT_POSIX
|
||||
#include <grp.h>
|
||||
#include <pwd.h>
|
||||
#if TD_DARWIN || TD_FREEBSD || TD_NETBSD
|
||||
#include <sys/param.h>
|
||||
#endif
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace td {
|
||||
|
||||
Status change_user(CSlice username, CSlice groupname) {
|
||||
#if TD_PORT_POSIX
|
||||
passwd *pw = getpwnam(username.c_str());
|
||||
if (pw == nullptr) {
|
||||
return OS_ERROR(PSTRING() << "Can't find the user '" << username << "' to switch to");
|
||||
}
|
||||
uid_t uid = pw->pw_uid;
|
||||
gid_t gid = pw->pw_gid;
|
||||
if (setgroups(1, &gid) == -1) {
|
||||
return OS_ERROR("Failed to clear supplementary group list");
|
||||
}
|
||||
if (!groupname.empty()) {
|
||||
group *g = getgrnam(groupname.c_str());
|
||||
if (g == nullptr) {
|
||||
return OS_ERROR("Can't find the group to switch to");
|
||||
}
|
||||
gid = g->gr_gid;
|
||||
} else if (initgroups(username.c_str(), gid) == -1) {
|
||||
return OS_ERROR("Failed to load groups of user");
|
||||
}
|
||||
if (setgid(gid) == -1) {
|
||||
return OS_ERROR("failed to set effective group ID");
|
||||
}
|
||||
if (setuid(uid) == -1) {
|
||||
return OS_ERROR("failed to set effective user ID");
|
||||
}
|
||||
return Status::OK();
|
||||
#else
|
||||
return Status::Error("Changing effective user is not supported");
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace td
|
16
tdutils/td/utils/port/user.h
Normal file
16
tdutils/td/utils/port/user.h
Normal file
@ -0,0 +1,16 @@
|
||||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#include "td/utils/Slice.h"
|
||||
#include "td/utils/Status.h"
|
||||
|
||||
namespace td {
|
||||
|
||||
Status change_user(CSlice username, CSlice groupname = CSlice());
|
||||
|
||||
}
|
@ -187,7 +187,7 @@ class HashMapBenchmark : public td::Benchmark {
|
||||
|
||||
size_t threads_n = 16;
|
||||
int mod_;
|
||||
static constexpr size_t mul_ = 7273; //1000000000 + 7;
|
||||
static constexpr size_t MUL = 7273; //1000000000 + 7;
|
||||
int n_;
|
||||
|
||||
public:
|
||||
@ -197,7 +197,7 @@ class HashMapBenchmark : public td::Benchmark {
|
||||
return HashMap::get_name();
|
||||
}
|
||||
void start_up_n(int n) override {
|
||||
n *= (int)threads_n;
|
||||
n *= static_cast<int>(threads_n);
|
||||
n_ = n;
|
||||
hash_map = td::make_unique<HashMap>(n * 2);
|
||||
}
|
||||
@ -211,7 +211,7 @@ class HashMapBenchmark : public td::Benchmark {
|
||||
size_t r = n * (i + 1) / threads_n;
|
||||
threads.emplace_back([l, r, this] {
|
||||
for (size_t i = l; i < r; i++) {
|
||||
auto x = td::narrow_cast<int>((i + 1) * mul_ % n_) + 3;
|
||||
auto x = td::narrow_cast<int>((i + 1) * MUL % n_) + 3;
|
||||
auto y = td::narrow_cast<int>(i + 2);
|
||||
hash_map->insert(x, y);
|
||||
}
|
||||
@ -224,7 +224,7 @@ class HashMapBenchmark : public td::Benchmark {
|
||||
|
||||
void tear_down() override {
|
||||
for (int i = 0; i < n_; i++) {
|
||||
auto x = td::narrow_cast<int>((i + 1) * mul_ % n_) + 3;
|
||||
auto x = td::narrow_cast<int>((i + 1) * MUL % n_) + 3;
|
||||
auto y = td::narrow_cast<int>(i + 2);
|
||||
ASSERT_EQ(y, hash_map->find(x, -1));
|
||||
}
|
||||
|
@ -17,28 +17,28 @@
|
||||
#include <set>
|
||||
#include <utility>
|
||||
|
||||
struct Data {
|
||||
struct ListData {
|
||||
td::MovableValue<td::uint64> value;
|
||||
td::MovableValue<bool> in_list;
|
||||
|
||||
Data() = default;
|
||||
Data(td::uint64 value, bool in_list) : value(value), in_list(in_list) {
|
||||
ListData() = default;
|
||||
ListData(td::uint64 value, bool in_list) : value(value), in_list(in_list) {
|
||||
}
|
||||
};
|
||||
|
||||
struct Node : public td::ListNode {
|
||||
Node() = default;
|
||||
explicit Node(Data data) : data(std::move(data)) {
|
||||
explicit Node(ListData data) : data(std::move(data)) {
|
||||
}
|
||||
|
||||
Data data;
|
||||
ListData data;
|
||||
};
|
||||
|
||||
static Data &get_data(Node &node) {
|
||||
static ListData &get_data(Node &node) {
|
||||
return node.data;
|
||||
}
|
||||
|
||||
static Data &get_data(td::TsListNode<Data> &node) {
|
||||
static ListData &get_data(td::TsListNode<ListData> &node) {
|
||||
return node.get_data_unsafe();
|
||||
}
|
||||
|
||||
@ -46,7 +46,7 @@ static std::unique_lock<std::mutex> lock(td::ListNode &node) {
|
||||
return {};
|
||||
}
|
||||
|
||||
static std::unique_lock<std::mutex> lock(td::TsListNode<Data> &node) {
|
||||
static std::unique_lock<std::mutex> lock(td::TsListNode<ListData> &node) {
|
||||
return node.lock();
|
||||
}
|
||||
|
||||
@ -66,26 +66,30 @@ static void do_run_list_test(ListRootT &root, std::atomic<td::uint64> &id) {
|
||||
nodes.emplace_back(NodeT({next_id(), false}));
|
||||
};
|
||||
auto pop_node = [&] {
|
||||
if (nodes.size() == 0) {
|
||||
if (nodes.empty()) {
|
||||
return;
|
||||
}
|
||||
nodes.pop_back();
|
||||
};
|
||||
auto random_node_index = [&] {
|
||||
CHECK(!nodes.empty());
|
||||
return rnd.fast(0, static_cast<int>(nodes.size()) - 1);
|
||||
};
|
||||
|
||||
auto link_node = [&] {
|
||||
if (nodes.empty()) {
|
||||
return;
|
||||
}
|
||||
auto i = rnd.fast(0, (int)nodes.size() - 1);
|
||||
auto i = random_node_index();
|
||||
nodes[i].remove();
|
||||
get_data(nodes[i]) = Data(next_id(), true);
|
||||
get_data(nodes[i]) = ListData(next_id(), true);
|
||||
root.put(&nodes[i]);
|
||||
};
|
||||
auto unlink_node = [&] {
|
||||
if (nodes.empty()) {
|
||||
return;
|
||||
}
|
||||
auto i = rnd.fast(0, (int)nodes.size() - 1);
|
||||
auto i = random_node_index();
|
||||
nodes[i].remove();
|
||||
get_data(nodes[i]).in_list = false;
|
||||
};
|
||||
@ -93,16 +97,16 @@ static void do_run_list_test(ListRootT &root, std::atomic<td::uint64> &id) {
|
||||
if (nodes.empty()) {
|
||||
return;
|
||||
}
|
||||
auto i = rnd.fast(0, (int)nodes.size() - 1);
|
||||
auto j = rnd.fast(0, (int)nodes.size() - 1);
|
||||
auto i = random_node_index();
|
||||
auto j = random_node_index();
|
||||
std::swap(nodes[i], nodes[j]);
|
||||
};
|
||||
auto set_node = [&] {
|
||||
if (nodes.empty()) {
|
||||
return;
|
||||
}
|
||||
auto i = rnd.fast(0, (int)nodes.size() - 1);
|
||||
auto j = rnd.fast(0, (int)nodes.size() - 1);
|
||||
auto i = random_node_index();
|
||||
auto j = random_node_index();
|
||||
nodes[i] = std::move(nodes[j]);
|
||||
};
|
||||
auto validate = [&] {
|
||||
@ -144,19 +148,19 @@ TEST(Misc, List) {
|
||||
}
|
||||
|
||||
TEST(Misc, TsList) {
|
||||
td::TsList<Data> root;
|
||||
td::TsList<ListData> root;
|
||||
std::atomic<td::uint64> id{0};
|
||||
for (std::size_t i = 0; i < 4; i++) {
|
||||
do_run_list_test<td::TsListNode<Data>, td::TsList<Data>, td::TsListNode<Data>>(root, id);
|
||||
do_run_list_test<td::TsListNode<ListData>, td::TsList<ListData>, td::TsListNode<ListData>>(root, id);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(Misc, TsListConcurrent) {
|
||||
td::TsList<Data> root;
|
||||
td::TsList<ListData> root;
|
||||
td::vector<td::thread> threads;
|
||||
std::atomic<td::uint64> id{0};
|
||||
for (std::size_t i = 0; i < 4; i++) {
|
||||
threads.emplace_back(
|
||||
[&] { do_run_list_test<td::TsListNode<Data>, td::TsList<Data>, td::TsListNode<Data>>(root, id); });
|
||||
[&] { do_run_list_test<td::TsListNode<ListData>, td::TsList<ListData>, td::TsListNode<ListData>>(root, id); });
|
||||
}
|
||||
}
|
||||
|
92
tdutils/test/OptionParser.cpp
Normal file
92
tdutils/test/OptionParser.cpp
Normal file
@ -0,0 +1,92 @@
|
||||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#include "td/utils/common.h"
|
||||
#include "td/utils/misc.h"
|
||||
#include "td/utils/OptionParser.h"
|
||||
#include "td/utils/Slice.h"
|
||||
#include "td/utils/Status.h"
|
||||
#include "td/utils/tests.h"
|
||||
|
||||
TEST(OptionParser, run) {
|
||||
td::OptionParser options;
|
||||
options.set_description("test description");
|
||||
|
||||
td::string exename = "exename";
|
||||
td::vector<td::string> args;
|
||||
auto run_option_parser = [&](td::string command_line) {
|
||||
args = td::full_split(command_line, ' ');
|
||||
td::vector<char *> argv;
|
||||
argv.push_back(&exename[0]);
|
||||
for (auto &arg : args) {
|
||||
argv.push_back(&arg[0]);
|
||||
}
|
||||
return options.run(static_cast<int>(argv.size()), &argv[0]);
|
||||
};
|
||||
|
||||
td::uint64 chosen_options = 0;
|
||||
td::vector<td::string> chosen_parameters;
|
||||
auto test_success = [&](td::string command_line, td::uint64 expected_options,
|
||||
td::vector<td::string> expected_parameters, td::vector<td::string> expected_result) {
|
||||
chosen_options = 0;
|
||||
chosen_parameters.clear();
|
||||
auto result = run_option_parser(command_line);
|
||||
ASSERT_TRUE(result.is_ok());
|
||||
ASSERT_EQ(expected_options, chosen_options);
|
||||
ASSERT_EQ(expected_parameters, chosen_parameters);
|
||||
ASSERT_EQ(expected_result.size(), result.ok().size());
|
||||
for (size_t i = 0; i < expected_result.size(); i++) {
|
||||
ASSERT_STREQ(expected_result[i], td::string(result.ok()[i]));
|
||||
}
|
||||
};
|
||||
auto test_fail = [&](td::string command_line) {
|
||||
auto result = run_option_parser(command_line);
|
||||
ASSERT_TRUE(result.is_error());
|
||||
};
|
||||
|
||||
options.add_option('q', "", "", [&] {
|
||||
chosen_options += 1;
|
||||
return td::Status::OK();
|
||||
});
|
||||
options.add_option('\0', "http-port2", "", [&] {
|
||||
chosen_options += 10;
|
||||
return td::Status::OK();
|
||||
});
|
||||
options.add_option('p', "http-port", "", [&](td::Slice parameter) {
|
||||
chosen_options += 100;
|
||||
chosen_parameters.push_back(parameter.str());
|
||||
return td::Status::OK();
|
||||
});
|
||||
options.add_option('v', "test", "", [&] {
|
||||
chosen_options += 1000;
|
||||
return td::Status::OK();
|
||||
});
|
||||
|
||||
test_fail("-http-port2");
|
||||
test_success("-", 0, {}, {"-"});
|
||||
test_fail("--http-port");
|
||||
test_fail("--http-port3");
|
||||
test_fail("--http-por");
|
||||
test_fail("--http-port2=1");
|
||||
test_fail("--q");
|
||||
test_fail("-qvp");
|
||||
test_fail("-p");
|
||||
test_fail("-u");
|
||||
test_success("-q", 1, {}, {});
|
||||
test_success("-vvvvvvvvvv", 10000, {}, {});
|
||||
test_success("-qpv", 101, {"v"}, {});
|
||||
test_success("-qp -v", 101, {"-v"}, {});
|
||||
test_success("-qp --http-port2", 101, {"--http-port2"}, {});
|
||||
test_success("-qp -- -v", 1101, {"--"}, {});
|
||||
test_success("-qvqvpqv", 2102, {"qv"}, {});
|
||||
test_success("aba --http-port2 caba --http-port2 dabacaba", 20, {}, {"aba", "caba", "dabacaba"});
|
||||
test_success("das -pqwerty -- -v asd --http-port", 100, {"qwerty"}, {"das", "-v", "asd", "--http-port"});
|
||||
test_success("-p option --http-port option2 --http-port=option3 --http-port=", 400,
|
||||
{"option", "option2", "option3", ""}, {});
|
||||
test_success("", 0, {}, {});
|
||||
test_success("a", 0, {}, {"a"});
|
||||
test_success("", 0, {}, {});
|
||||
}
|
@ -9,6 +9,7 @@
|
||||
#include "td/utils/common.h"
|
||||
#include "td/utils/crypto.h"
|
||||
#include "td/utils/logging.h"
|
||||
#include "td/utils/Random.h"
|
||||
#include "td/utils/Slice.h"
|
||||
#include "td/utils/tests.h"
|
||||
#include "td/utils/UInt.h"
|
||||
@ -18,6 +19,29 @@
|
||||
static td::vector<td::string> strings{"", "1", "short test string", td::string(1000000, 'a')};
|
||||
|
||||
#if TD_HAVE_OPENSSL
|
||||
#if TD_HAVE_ZLIB
|
||||
TEST(Crypto, Aes) {
|
||||
td::Random::Xorshift128plus rnd(123);
|
||||
td::UInt256 key;
|
||||
rnd.bytes(as_slice(key));
|
||||
td::string plaintext(16, '\0');
|
||||
td::string encrypted(16, '\0');
|
||||
td::string decrypted(16, '\0');
|
||||
rnd.bytes(plaintext);
|
||||
|
||||
td::AesState encryptor;
|
||||
encryptor.init(as_slice(key), true);
|
||||
td::AesState decryptor;
|
||||
decryptor.init(as_slice(key), false);
|
||||
|
||||
encryptor.encrypt(td::as_slice(plaintext).ubegin(), td::as_slice(encrypted).ubegin(), 16);
|
||||
decryptor.decrypt(td::as_slice(encrypted).ubegin(), td::as_slice(decrypted).ubegin(), 16);
|
||||
|
||||
CHECK(decrypted == plaintext);
|
||||
CHECK(decrypted != encrypted);
|
||||
CHECK(td::crc32(encrypted) == 178892237);
|
||||
}
|
||||
|
||||
TEST(Crypto, AesCtrState) {
|
||||
td::vector<td::uint32> answers1{0u, 1141589763u, 596296607u, 3673001485u, 2302125528u,
|
||||
330967191u, 2047392231u, 3537459563u, 307747798u, 2149598133u};
|
||||
@ -51,7 +75,7 @@ TEST(Crypto, AesCtrState) {
|
||||
ASSERT_EQ(answers1[i], td::crc32(t));
|
||||
state.init(as_slice(key), as_slice(iv));
|
||||
state.decrypt(t, t);
|
||||
ASSERT_STREQ(s, t);
|
||||
ASSERT_STREQ(td::base64_encode(s), td::base64_encode(t));
|
||||
|
||||
for (auto &c : iv.raw) {
|
||||
c = 0xFF;
|
||||
@ -64,6 +88,82 @@ TEST(Crypto, AesCtrState) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST(Crypto, AesIgeState) {
|
||||
td::vector<td::uint32> answers1{0u, 2045698207u, 2423540300u, 525522475u, 1545267325u};
|
||||
|
||||
std::size_t i = 0;
|
||||
for (auto length : {0, 16, 32, 256, 1024}) {
|
||||
td::uint32 seed = length;
|
||||
td::string s(length, '\0');
|
||||
for (auto &c : s) {
|
||||
seed = seed * 123457567u + 987651241u;
|
||||
c = static_cast<char>((seed >> 23) & 255);
|
||||
}
|
||||
|
||||
td::UInt256 key;
|
||||
for (auto &c : key.raw) {
|
||||
seed = seed * 123457567u + 987651241u;
|
||||
c = (seed >> 23) & 255;
|
||||
}
|
||||
td::UInt256 iv;
|
||||
for (auto &c : iv.raw) {
|
||||
seed = seed * 123457567u + 987651241u;
|
||||
c = (seed >> 23) & 255;
|
||||
}
|
||||
|
||||
td::AesIgeState state;
|
||||
state.init(as_slice(key), as_slice(iv), true);
|
||||
td::string t(length, '\0');
|
||||
state.encrypt(s, t);
|
||||
|
||||
ASSERT_EQ(answers1[i], td::crc32(t));
|
||||
|
||||
state.init(as_slice(key), as_slice(iv), false);
|
||||
state.decrypt(t, t);
|
||||
ASSERT_STREQ(td::base64_encode(s), td::base64_encode(t));
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
TEST(Crypto, AesCbcState) {
|
||||
td::vector<td::uint32> answers1{0u, 3617355989u, 3449188102u, 186999968u, 4244808847u};
|
||||
|
||||
std::size_t i = 0;
|
||||
for (auto length : {0, 16, 32, 256, 1024}) {
|
||||
td::uint32 seed = length;
|
||||
td::string s(length, '\0');
|
||||
for (auto &c : s) {
|
||||
seed = seed * 123457567u + 987651241u;
|
||||
c = static_cast<char>((seed >> 23) & 255);
|
||||
}
|
||||
|
||||
td::UInt256 key;
|
||||
for (auto &c : key.raw) {
|
||||
seed = seed * 123457567u + 987651241u;
|
||||
c = (seed >> 23) & 255;
|
||||
}
|
||||
td::UInt128 iv;
|
||||
for (auto &c : iv.raw) {
|
||||
seed = seed * 123457567u + 987651241u;
|
||||
c = (seed >> 23) & 255;
|
||||
}
|
||||
|
||||
td::AesCbcState state(as_slice(key), as_slice(iv));
|
||||
//state.init(as_slice(key), as_slice(iv), true);
|
||||
td::string t(length, '\0');
|
||||
state.encrypt(s, t);
|
||||
|
||||
ASSERT_EQ(answers1[i], td::crc32(t));
|
||||
|
||||
//state.init(as_slice(key), as_slice(iv), false);
|
||||
state = td::AesCbcState(as_slice(key), as_slice(iv));
|
||||
state.decrypt(t, t);
|
||||
ASSERT_STREQ(td::base64_encode(s), td::base64_encode(t));
|
||||
i++;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST(Crypto, Sha256State) {
|
||||
for (auto length : {0, 1, 31, 32, 33, 9999, 10000, 10001, 999999, 1000001}) {
|
||||
auto s = td::rand_string(std::numeric_limits<char>::min(), std::numeric_limits<char>::max(), length);
|
||||
@ -85,8 +185,8 @@ TEST(Crypto, Sha256State) {
|
||||
}
|
||||
|
||||
TEST(Crypto, PBKDF) {
|
||||
td::vector<td::string> passwords{"", "qwerty", std::string(1000, 'a')};
|
||||
td::vector<td::string> salts{"", "qwerty", std::string(1000, 'a')};
|
||||
td::vector<td::string> passwords{"", "qwerty", td::string(1000, 'a')};
|
||||
td::vector<td::string> salts{"", "qwerty", td::string(1000, 'a')};
|
||||
td::vector<int> iteration_counts{1, 2, 1000};
|
||||
td::vector<td::Slice> answers{
|
||||
"984LZT0tcqQQjPWr6RL/3Xd2Ftu7J6cOggTzri0Pb60=", "lzmEEdaupDp3rO+SImq4J41NsGaL0denanJfdoCsRcU=",
|
||||
@ -187,7 +287,7 @@ TEST(Crypto, crc32c_benchmark) {
|
||||
public:
|
||||
explicit Crc32cExtendBenchmark(size_t chunk_size) : chunk_size_(chunk_size) {
|
||||
}
|
||||
std::string get_description() const override {
|
||||
td::string get_description() const override {
|
||||
return PSTRING() << "Crc32c with chunk_size=" << chunk_size_;
|
||||
}
|
||||
void start_up_n(int n) override {
|
||||
@ -197,7 +297,7 @@ TEST(Crypto, crc32c_benchmark) {
|
||||
} else {
|
||||
cnt_ = 1;
|
||||
}
|
||||
data_ = std::string(n, 'a');
|
||||
data_ = td::string(n, 'a');
|
||||
}
|
||||
void run(int n) override {
|
||||
td::uint32 res = 0;
|
||||
@ -214,7 +314,7 @@ TEST(Crypto, crc32c_benchmark) {
|
||||
|
||||
private:
|
||||
size_t chunk_size_;
|
||||
std::string data_;
|
||||
td::string data_;
|
||||
int cnt_;
|
||||
};
|
||||
bench(Crc32cExtendBenchmark(2));
|
||||
|
@ -12,6 +12,7 @@ set(TD_TEST_SOURCE
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/set_with_position.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/string_cleaning.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tdclient.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tqueue.cpp
|
||||
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/data.cpp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/data.h
|
||||
|
@ -7,7 +7,11 @@
|
||||
#include "td/utils/tests.h"
|
||||
|
||||
#include "td/utils/common.h"
|
||||
#include "td/utils/crypto.h"
|
||||
#include "td/utils/logging.h"
|
||||
#include "td/utils/OptionParser.h"
|
||||
#include "td/utils/Slice.h"
|
||||
#include "td/utils/Status.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
@ -16,18 +20,26 @@
|
||||
#endif
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
// TODO port OptionsParser to Windows
|
||||
td::init_openssl_threads();
|
||||
|
||||
td::TestsRunner &runner = td::TestsRunner::get_default();
|
||||
SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR));
|
||||
for (int i = 1; i < argc; i++) {
|
||||
if (!std::strcmp(argv[i], "--filter")) {
|
||||
CHECK(i + 1 < argc);
|
||||
runner.add_substr_filter(argv[++i]);
|
||||
}
|
||||
if (!std::strcmp(argv[i], "--stress")) {
|
||||
runner.set_stress_flag(true);
|
||||
}
|
||||
|
||||
td::OptionParser options;
|
||||
options.add_option('\0', "filter", "Run only specified tests", [&](td::Slice filter) {
|
||||
runner.add_substr_filter(filter.str());
|
||||
return td::Status::OK();
|
||||
});
|
||||
options.add_option('\0', "stress", "Run tests infinitely", [&] {
|
||||
runner.set_stress_flag(true);
|
||||
return td::Status::OK();
|
||||
});
|
||||
auto result = options.run(argc, argv);
|
||||
if (result.is_error() || !result.ok().empty()) {
|
||||
LOG(PLAIN) << options;
|
||||
return 1;
|
||||
}
|
||||
|
||||
#if TD_EMSCRIPTEN
|
||||
emscripten_set_main_loop(
|
||||
[] {
|
||||
|
@ -467,7 +467,7 @@ TEST(MessageEntities, url) {
|
||||
check_url("@.", {});
|
||||
check_url(
|
||||
"a.b.google.com dfsknnfs gsdfgsg http://códuia.de/ dffdg,\" 12)(cpia.de/())(\" http://гришка.рф/ sdufhdf "
|
||||
"http://xn--80afpi2a3c.xn--p1ai/ I have a good time.Thanks, guys!\n\n(hdfughidufhgdis)go#ogle.com гришка.рф "
|
||||
"http://xn--80afpi2a3c.xn--p1ai/ I have a good time.Thanks, guys!\n\n(hdfughidufhgdis) go#ogle.com гришка.рф "
|
||||
"hsighsdf gi почта.рф\n\n✪df.ws/123 "
|
||||
"xn--80afpi2a3c.xn--p1ai\n\nhttp://foo.com/blah_blah\nhttp://foo.com/blah_blah/\n(Something like "
|
||||
"http://foo.com/blah_blah)\nhttp://foo.com/blah_blah_(wikipedi8989a_Вася)\n(Something like "
|
||||
|
192
test/tqueue.cpp
Normal file
192
test/tqueue.cpp
Normal file
@ -0,0 +1,192 @@
|
||||
//
|
||||
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#include "td/db/binlog/Binlog.h"
|
||||
#include "td/db/binlog/BinlogEvent.h"
|
||||
#include "td/db/binlog/BinlogHelper.h"
|
||||
#include "td/db/TQueue.h"
|
||||
|
||||
#include "td/utils/int_types.h"
|
||||
#include "td/utils/misc.h"
|
||||
#include "td/utils/port/path.h"
|
||||
#include "td/utils/Slice.h"
|
||||
#include "td/utils/Span.h"
|
||||
#include "td/utils/Status.h"
|
||||
#include "td/utils/tests.h"
|
||||
#include "td/utils/VectorQueue.h"
|
||||
|
||||
#include <map>
|
||||
#include <set>
|
||||
|
||||
TEST(TQueue, hands) {
|
||||
td::TQueue::Event events[100];
|
||||
auto events_span = td::MutableSpan<td::TQueue::Event>(events, 100);
|
||||
|
||||
auto tqueue = td::TQueue::create();
|
||||
auto qid = 12;
|
||||
ASSERT_EQ(true, tqueue->get_head(qid).empty());
|
||||
ASSERT_EQ(true, tqueue->get_tail(qid).empty());
|
||||
tqueue->push(qid, "hello", 0, 0, td::TQueue::EventId());
|
||||
auto head = tqueue->get_head(qid);
|
||||
auto tail = tqueue->get_tail(qid);
|
||||
ASSERT_EQ(head.next().ok(), tail);
|
||||
ASSERT_EQ(1u, tqueue->get(qid, head, true, 0, events_span).move_as_ok());
|
||||
ASSERT_EQ(1u, tqueue->get(qid, head, true, 0, events_span).move_as_ok());
|
||||
ASSERT_EQ(0u, tqueue->get(qid, tail, false, 0, events_span).move_as_ok());
|
||||
ASSERT_EQ(1u, tqueue->get(qid, head, true, 0, events_span).move_as_ok());
|
||||
ASSERT_EQ(0u, tqueue->get(qid, tail, true, 0, events_span).move_as_ok());
|
||||
ASSERT_EQ(0u, tqueue->get(qid, head, true, 0, events_span).move_as_ok());
|
||||
}
|
||||
|
||||
class TestTQueue {
|
||||
public:
|
||||
using EventId = td::TQueue::EventId;
|
||||
|
||||
static td::CSlice binlog_path() {
|
||||
return td::CSlice("tqueue_binlog");
|
||||
}
|
||||
|
||||
TestTQueue() {
|
||||
baseline_ = td::TQueue::create();
|
||||
|
||||
memory_ = td::TQueue::create();
|
||||
auto memory_storage = td::make_unique<td::TQueueMemoryStorage>();
|
||||
memory_storage_ = memory_storage.get();
|
||||
memory_->set_callback(std::move(memory_storage));
|
||||
|
||||
binlog_ = td::TQueue::create();
|
||||
auto tqueue_binlog = td::make_unique<td::TQueueBinlog<td::Binlog>>();
|
||||
td::Binlog::destroy(binlog_path()).ensure();
|
||||
auto binlog = std::make_shared<td::Binlog>();
|
||||
binlog->init(binlog_path().str(), [&](const td::BinlogEvent &event) { UNREACHABLE(); }).ensure();
|
||||
tqueue_binlog->set_binlog(std::move(binlog));
|
||||
binlog_->set_callback(std::move(tqueue_binlog));
|
||||
}
|
||||
|
||||
void restart(td::Random::Xorshift128plus &rnd, double now) {
|
||||
if (rnd.fast(0, 10) == 0) {
|
||||
baseline_->run_gc(now);
|
||||
}
|
||||
|
||||
memory_->extract_callback().release();
|
||||
auto memory_storage = td::unique_ptr<td::TQueueMemoryStorage>(memory_storage_);
|
||||
memory_ = td::TQueue::create();
|
||||
memory_storage->replay(*memory_);
|
||||
memory_->set_callback(std::move(memory_storage));
|
||||
if (rnd.fast(0, 10) == 0) {
|
||||
memory_->run_gc(now);
|
||||
}
|
||||
|
||||
if (rnd.fast(0, 30) != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
LOG(INFO) << "Restart binlog";
|
||||
binlog_ = td::TQueue::create();
|
||||
auto tqueue_binlog = td::make_unique<td::TQueueBinlog<td::Binlog>>();
|
||||
auto binlog = std::make_shared<td::Binlog>();
|
||||
binlog->init(binlog_path().str(), [&](const td::BinlogEvent &event) { tqueue_binlog->replay(event, *binlog_); })
|
||||
.ensure();
|
||||
tqueue_binlog->set_binlog(std::move(binlog));
|
||||
binlog_->set_callback(std::move(tqueue_binlog));
|
||||
if (rnd.fast(0, 2) == 0) {
|
||||
binlog_->run_gc(now);
|
||||
}
|
||||
}
|
||||
|
||||
EventId push(td::TQueue::QueueId queue_id, td::string data, double expires_at, EventId new_id = EventId()) {
|
||||
auto a_id = baseline_->push(queue_id, data, expires_at, 0, new_id).move_as_ok();
|
||||
auto b_id = memory_->push(queue_id, data, expires_at, 0, new_id).move_as_ok();
|
||||
auto c_id = binlog_->push(queue_id, data, expires_at, 0, new_id).move_as_ok();
|
||||
ASSERT_EQ(a_id, b_id);
|
||||
ASSERT_EQ(a_id, c_id);
|
||||
return a_id;
|
||||
}
|
||||
|
||||
void check_head_tail(td::TQueue::QueueId qid, double now) {
|
||||
//ASSERT_EQ(baseline_->get_head(qid), memory_->get_head(qid));
|
||||
//ASSERT_EQ(baseline_->get_head(qid), binlog_->get_head(qid));
|
||||
ASSERT_EQ(baseline_->get_tail(qid), memory_->get_tail(qid));
|
||||
ASSERT_EQ(baseline_->get_tail(qid), binlog_->get_tail(qid));
|
||||
}
|
||||
|
||||
void check_get(td::TQueue::QueueId qid, td::Random::Xorshift128plus &rnd, double now) {
|
||||
td::TQueue::Event a[10];
|
||||
td::MutableSpan<td::TQueue::Event> a_span(a, 10);
|
||||
td::TQueue::Event b[10];
|
||||
td::MutableSpan<td::TQueue::Event> b_span(b, 10);
|
||||
td::TQueue::Event c[10];
|
||||
td::MutableSpan<td::TQueue::Event> c_span(c, 10);
|
||||
|
||||
auto a_from = baseline_->get_head(qid);
|
||||
//auto b_from = memory_->get_head(qid);
|
||||
//auto c_from = binlog_->get_head(qid);
|
||||
//ASSERT_EQ(a_from, b_from);
|
||||
//ASSERT_EQ(a_from, c_from);
|
||||
|
||||
auto tmp = a_from.advance(rnd.fast(-10, 10));
|
||||
if (tmp.is_ok()) {
|
||||
a_from = tmp.move_as_ok();
|
||||
}
|
||||
baseline_->get(qid, a_from, true, now, a_span).move_as_ok();
|
||||
memory_->get(qid, a_from, true, now, b_span).move_as_ok();
|
||||
binlog_->get(qid, a_from, true, now, c_span).move_as_ok();
|
||||
ASSERT_EQ(a_span.size(), b_span.size());
|
||||
ASSERT_EQ(a_span.size(), c_span.size());
|
||||
for (size_t i = 0; i < a_span.size(); i++) {
|
||||
ASSERT_EQ(a_span[i].id, b_span[i].id);
|
||||
ASSERT_EQ(a_span[i].id, c_span[i].id);
|
||||
ASSERT_EQ(a_span[i].data, b_span[i].data);
|
||||
ASSERT_EQ(a_span[i].data, c_span[i].data);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
td::unique_ptr<td::TQueue> baseline_;
|
||||
td::unique_ptr<td::TQueue> memory_;
|
||||
td::unique_ptr<td::TQueue> binlog_;
|
||||
td::TQueueMemoryStorage *memory_storage_{nullptr};
|
||||
};
|
||||
|
||||
TEST(TQueue, random) {
|
||||
using EventId = td::TQueue::EventId;
|
||||
td::Random::Xorshift128plus rnd(123);
|
||||
auto next_queue_id = [&rnd] {
|
||||
return rnd.fast(1, 10);
|
||||
};
|
||||
auto next_first_id = [&rnd] {
|
||||
if (rnd.fast(0, 3) == 0) {
|
||||
return EventId::from_int32(EventId::MAX_ID - 20).move_as_ok();
|
||||
}
|
||||
return EventId::from_int32(rnd.fast(1000000000, 1500000000)).move_as_ok();
|
||||
};
|
||||
|
||||
TestTQueue q;
|
||||
double now = 0;
|
||||
auto push_event = [&] {
|
||||
auto data = PSTRING() << rnd();
|
||||
if (rnd.fast(0, 10000) == 0) {
|
||||
data = td::string(1 << 19, '\0');
|
||||
}
|
||||
q.push(next_queue_id(), data, now + rnd.fast(-10, 10) * 10 + 5, next_first_id());
|
||||
};
|
||||
auto inc_now = [&] {
|
||||
now += 10;
|
||||
};
|
||||
auto check_head_tail = [&] {
|
||||
q.check_head_tail(next_queue_id(), now);
|
||||
};
|
||||
auto restart = [&] {
|
||||
q.restart(rnd, now);
|
||||
};
|
||||
auto get = [&] {
|
||||
q.check_get(next_queue_id(), rnd, now);
|
||||
};
|
||||
td::RandomSteps steps({{push_event, 100}, {check_head_tail, 10}, {get, 40}, {inc_now, 5}, {restart, 1}});
|
||||
for (int i = 0; i < 100000; i++) {
|
||||
steps.step(rnd);
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user