From 0a1fb007d9de14ca009bff5f3c8d3107151e0d56 Mon Sep 17 00:00:00 2001 From: levlam Date: Mon, 1 Jul 2019 16:18:28 +0300 Subject: [PATCH] TlsInit fixes. GitOrigin-RevId: 798d053d68b6e0955b6e0e0c7c8d36592f76c987 --- td/mtproto/TcpTransport.cpp | 12 ++- td/mtproto/TcpTransport.h | 11 ++- td/mtproto/TlsInit.cpp | 105 +++++++++++--------------- td/mtproto/TlsInit.h | 1 + td/mtproto/TlsReaderByteFlow.cpp | 4 +- td/mtproto/TlsReaderByteFlow.h | 1 - td/telegram/net/ConnectionCreator.cpp | 32 +++++--- tdutils/td/utils/port/SocketFd.cpp | 3 + test/mtproto.cpp | 15 ++-- 9 files changed, 93 insertions(+), 91 deletions(-) diff --git a/td/mtproto/TcpTransport.cpp b/td/mtproto/TcpTransport.cpp index 709fc9bc9..505432ba6 100644 --- a/td/mtproto/TcpTransport.cpp +++ b/td/mtproto/TcpTransport.cpp @@ -220,13 +220,11 @@ void ObfuscatedTransport::write(BufferWriter &&message, bool quick_ack) { } void ObfuscatedTransport::do_write_tls(BufferWriter &&message) { - size_t size = message.size(); - - if (size > (1 << 14)) { + if (message.size() > MAX_TLS_PACKET_LENGTH) { auto buffer_slice = message.as_buffer_slice(); auto slice = buffer_slice.as_slice(); while (!slice.empty()) { - auto buf = buffer_slice.from_slice(slice.substr(0, 1 << 14)); + auto buf = buffer_slice.from_slice(slice.substr(0, MAX_TLS_PACKET_LENGTH)); slice.remove_prefix(buf.size()); BufferBuilder builder; builder.append(std::move(buf)); @@ -241,11 +239,11 @@ void ObfuscatedTransport::do_write_tls(BufferWriter &&message) { void ObfuscatedTransport::do_write_tls(BufferBuilder &&builder) { size_t size = builder.size(); - CHECK(size <= (1 << 14)); + CHECK(size <= MAX_TLS_PACKET_LENGTH); char buf[] = "\x17\x03\x03\x00\x00"; - buf[3] = (size >> 8) & 0xff; - buf[4] = size & 0xff; + buf[3] = static_cast((size >> 8) & 0xff); + buf[4] = static_cast(size & 0xff); builder.prepend(Slice(buf, 5)); if (is_first_tls_packet_) { diff --git a/td/mtproto/TcpTransport.h b/td/mtproto/TcpTransport.h index 17fc52d18..e554c6733 100644 --- a/td/mtproto/TcpTransport.h +++ b/td/mtproto/TcpTransport.h @@ -7,8 +7,8 @@ #pragma once #include "td/mtproto/IStreamTransport.h" -#include "td/mtproto/TransportType.h" #include "td/mtproto/TlsReaderByteFlow.h" +#include "td/mtproto/TransportType.h" #include "td/utils/AesCtrByteFlow.h" #include "td/utils/buffer.h" @@ -125,7 +125,7 @@ class ObfuscatedTransport : public IStreamTransport { public: ObfuscatedTransport(int16 dc_id, std::string secret) : dc_id_(dc_id), secret_(std::move(secret)), impl_(secret_.size() >= 17) { - emulate_tls_ = secret.size() >= 17 && secret[0] == '\xee'; + emulate_tls_ = secret_.size() >= 17 && secret_[0] == '\xee'; } Result read_next(BufferSlice *message, uint32 *quick_ack) override TD_WARN_UNUSED_RESULT; @@ -154,6 +154,9 @@ class ObfuscatedTransport : public IStreamTransport { res += 6; } } + if (res & 3) { + res += 4 - (res & 3); + } return res; } @@ -168,7 +171,7 @@ class ObfuscatedTransport : public IStreamTransport { private: int16 dc_id_; std::string secret_; - bool emulate_tls_; + bool emulate_tls_{false}; bool is_first_tls_packet_{true}; TransportImpl impl_; TlsReaderByteFlow tls_reader_byte_flow_; @@ -176,6 +179,8 @@ class ObfuscatedTransport : public IStreamTransport { ByteFlowSink byte_flow_sink_; ChainBufferReader *input_; + static constexpr int32 MAX_TLS_PACKET_LENGTH = 1 << 14; + // TODO: use ByteFlow? // One problem is that BufferedFd owns output_buffer_ // The other problem is that first 56 bytes must be sent unencrypted. diff --git a/td/mtproto/TlsInit.cpp b/td/mtproto/TlsInit.cpp index 31b012247..03a8d4942 100644 --- a/td/mtproto/TlsInit.cpp +++ b/td/mtproto/TlsInit.cpp @@ -8,14 +8,18 @@ #include "td/utils/as.h" #include "td/utils/crypto.h" +#include "td/utils/port/Clocks.h" #include "td/utils/Random.h" #include "td/utils/Span.h" +#include + namespace td { + void Grease::init(MutableSlice res) { Random::secure_bytes(res); for (auto &c : res) { - c = (c & 0xF0) + 0x0A; + c = static_cast((c & 0xF0) + 0x0A); } for (size_t i = 1; i < res.size(); i += 2) { if (res[i] == res[i - 1]) { @@ -110,29 +114,36 @@ class TlsHello { Op::string("\x03\x04\x03\x03\x03\x02\x03\x01\x00\x1b\x00\x03\x02\x00\x02"), Op::grease(3), Op::string("\x00\x01\x00\x00\x15")}; + res.grease_size_ = 7; return res; }(); return result; } + Span get_ops() const { return ops_; } + size_t get_grease_size() const { + return grease_size_; + } + private: std::vector ops_; + size_t grease_size_; }; class TlsHelloContext { public: - explicit TlsHelloContext(std::string domain) { - Grease::init(MutableSlice(grease_.data(), grease_.size())); - domain_ = std::move(domain); + TlsHelloContext(size_t grease_size, std::string domain) : grease_(grease_size, '\0'), domain_(std::move(domain)) { + Grease::init(grease_); } + char get_grease(size_t i) const { CHECK(i < grease_.size()); return grease_[i]; } - size_t grease_size() const { + size_t get_grease_size() const { return grease_.size(); } Slice get_domain() const { @@ -140,8 +151,7 @@ class TlsHelloContext { } private: - constexpr static size_t MAX_GREASE = 8; - std::array grease_; + std::string grease_; std::string domain_; }; @@ -174,7 +184,7 @@ class TlsHelloCalcLength { break; case Type::Grease: CHECK(context); - if (op.seed < 0 || static_cast(op.seed) >= context->grease_size()) { + if (op.seed < 0 || static_cast(op.seed) >= context->get_grease_size()) { return on_error(Status::Error("Invalid grease seed")); } size_ += 2; @@ -234,9 +244,9 @@ class TlsHelloCalcLength { class TlsHelloStore { public: - TlsHelloStore(MutableSlice dest) : data_(dest), dest_(dest) { + explicit TlsHelloStore(MutableSlice dest) : data_(dest), dest_(dest) { } - void do_op(const TlsHello::Op &op, TlsHelloContext *context) { + void do_op(const TlsHello::Op &op, const TlsHelloContext *context) { using Type = TlsHello::Op::Type; switch (op.type) { case Type::String: @@ -281,19 +291,20 @@ class TlsHelloStore { data_[begin_offset + 1] = static_cast(size & 0xff); break; } + default: + UNREACHABLE(); } } - void finish(int32 unix_time) { + void finish(Slice secret, int32 unix_time) { int zero_pad = 515 - static_cast(get_offset()); using Op = TlsHello::Op; do_op(Op::begin_scope(), nullptr); do_op(Op::zero(zero_pad), nullptr); do_op(Op::end_scope(), nullptr); - auto tmp = sha256(data_); - auto hash_dest = data_.substr(11); - hash_dest.copy_from(tmp); + auto hash_dest = data_.substr(11, 32); + hmac_sha256(secret, data_, hash_dest); int32 old = as(hash_dest.substr(28).data()); as(hash_dest.substr(28).data()) = old ^ unix_time; CHECK(dest_.empty()); @@ -303,97 +314,68 @@ class TlsHelloStore { MutableSlice data_; MutableSlice dest_; std::vector scope_offset_; - size_t get_offset() { + + size_t get_offset() const { return data_.size() - dest_.size(); } }; class TlsObfusaction { public: - static std::string generate_header(std::string domain, int32 unix_time) { + static std::string generate_header(std::string domain, Slice secret, int32 unix_time) { + CHECK(!domain.empty()); + CHECK(secret.size() == 16); + auto &hello = TlsHello::get_default(); - TlsHelloContext context(domain); + TlsHelloContext context(hello.get_grease_size(), std::move(domain)); TlsHelloCalcLength calc_length; for (auto &op : hello.get_ops()) { calc_length.do_op(op, &context); } auto length = calc_length.finish().move_as_ok(); - std::string data(length, 0); + std::string data(length, '\0'); TlsHelloStore storer(data); for (auto &op : hello.get_ops()) { storer.do_op(op, &context); } - storer.finish(0); + storer.finish(secret, unix_time); return data; } }; void TlsInit::send_hello() { - auto hello = TlsObfusaction::generate_header(username_, 0); + auto hello = + TlsObfusaction::generate_header(username_, password_, static_cast(Clocks::system())); // TODO correct time fd_.output_buffer().append(hello); state_ = State::WaitHelloResponse; } Status TlsInit::wait_hello_response() { - //[ auto it = fd_.input_buffer().clone(); - //S "\x16\x03\x03" - { - Slice first = "\x16\x03\x03"; - std::string got_first(first.size(), 0); - if (it.size() < first.size()) { - return td::Status::OK(); + for (auto first : {Slice("\x16\x03\x03"), Slice("\x14\x03\x03\x00\x01\x01\x17\x03\x03")}) { + if (it.size() < first.size() + 2) { + return Status::OK(); } + + std::string got_first(first.size(), '\0'); it.advance(first.size(), got_first); if (first != got_first) { return Status::Error("First part of response to hello is invalid"); } - } - //[ - { - if (it.size() < 2) { - return td::Status::OK(); - } uint8 tmp[2]; it.advance(2, MutableSlice(tmp, 2)); size_t skip_size = (tmp[0] << 8) + tmp[1]; if (it.size() < skip_size) { - return td::Status::OK(); + return Status::OK(); } it.advance(skip_size); } - //S "\x14\x03\x03\x00\x01\x01\x17\x03\x03" - { - Slice first = "\x14\x03\x03\x00\x01\x01\x17\x03\x03"; - std::string got_first(first.size(), 0); - if (it.size() < first.size()) { - return td::Status::OK(); - } - it.advance(first.size(), got_first); - if (first != got_first) { - return Status::Error("Second part of response to hello is invalid"); - } - } - - //[ - { - if (it.size() < 2) { - return td::Status::OK(); - } - uint8 tmp[2]; - it.advance(2, MutableSlice(tmp, 2)); - size_t skip_size = (tmp[0] << 8) + tmp[1]; - if (it.size() < skip_size) { - return td::Status::OK(); - } - it.advance(skip_size); - } fd_.input_buffer() = std::move(it); stop(); - return td::Status::OK(); + return Status::OK(); } Status TlsInit::loop_impl() { @@ -407,4 +389,5 @@ Status TlsInit::loop_impl() { } return Status::OK(); } + } // namespace td diff --git a/td/mtproto/TlsInit.h b/td/mtproto/TlsInit.h index 769a563a4..2b8323e75 100644 --- a/td/mtproto/TlsInit.h +++ b/td/mtproto/TlsInit.h @@ -8,6 +8,7 @@ #include "td/net/TransparentProxy.h" +#include "td/utils/Slice.h" #include "td/utils/Status.h" namespace td { diff --git a/td/mtproto/TlsReaderByteFlow.cpp b/td/mtproto/TlsReaderByteFlow.cpp index 26ea2cbb6..8dafe044c 100644 --- a/td/mtproto/TlsReaderByteFlow.cpp +++ b/td/mtproto/TlsReaderByteFlow.cpp @@ -21,12 +21,12 @@ void TlsReaderByteFlow::loop() { uint8 buf[5]; it.advance(5, MutableSlice(buf, 5)); if (Slice(buf, 3) != Slice("\x17\x03\x03")) { - close_input(td::Status::Error("Invalid bytes at the beginning of a packet (emulated tls)")); + close_input(Status::Error("Invalid bytes at the beginning of a packet (emulated tls)")); return; } size_t len = (buf[3] << 8) | buf[4]; if (len > (1 << 14)) { - close_input(td::Status::Error("Packet lenght is too big (emulated tls)")); + close_input(Status::Error("Packet length is too big (emulated tls)")); return; } diff --git a/td/mtproto/TlsReaderByteFlow.h b/td/mtproto/TlsReaderByteFlow.h index bb6eca45e..f85fc8a75 100644 --- a/td/mtproto/TlsReaderByteFlow.h +++ b/td/mtproto/TlsReaderByteFlow.h @@ -12,7 +12,6 @@ namespace td { class TlsReaderByteFlow final : public ByteFlowBase { public: - TlsReaderByteFlow() = default; void loop() override; }; diff --git a/td/telegram/net/ConnectionCreator.cpp b/td/telegram/net/ConnectionCreator.cpp index 54d1b0b89..356f3a42a 100644 --- a/td/telegram/net/ConnectionCreator.cpp +++ b/td/telegram/net/ConnectionCreator.cpp @@ -502,6 +502,8 @@ void ConnectionCreator::ping_proxy_resolved(int32 proxy_id, IPAddress ip_address } auto socket_fd = r_socket_fd.move_as_ok(); + bool emulate_tls = extra.transport_type.emulate_tls(); + auto socket_fd_promise = PromiseCreator::lambda([promise = std::move(promise), actor_id = actor_id(this), transport_type = std::move(extra.transport_type)](Result r_socket_fd) mutable { @@ -512,7 +514,7 @@ void ConnectionCreator::ping_proxy_resolved(int32 proxy_id, IPAddress ip_address std::move(transport_type), std::move(promise)); }); CHECK(proxy.use_proxy()); - if (proxy.use_socks5_proxy() || proxy.use_http_tcp_proxy()) { + if (proxy.use_socks5_proxy() || proxy.use_http_tcp_proxy() || emulate_tls) { class Callback : public TransparentProxy::Callback { public: explicit Callback(Promise promise) : promise_(std::move(promise)) { @@ -526,18 +528,25 @@ void ConnectionCreator::ping_proxy_resolved(int32 proxy_id, IPAddress ip_address private: Promise promise_; }; + auto callback = make_unique(std::move(socket_fd_promise)); + LOG(INFO) << "Start ping proxy: " << extra.debug_str; auto token = next_token(); if (proxy.use_socks5_proxy()) { children_[token] = { false, create_actor("PingSocks5", std::move(socket_fd), extra.mtproto_ip, proxy.proxy().user().str(), - proxy.proxy().password().str(), - make_unique(std::move(socket_fd_promise)), create_reference(token))}; - } else { + proxy.proxy().password().str(), std::move(callback), create_reference(token))}; + } else if (proxy.use_http_tcp_proxy()) { + children_[token] = {false, create_actor("PingHttpProxy", std::move(socket_fd), extra.mtproto_ip, + proxy.proxy().user().str(), proxy.proxy().password().str(), + std::move(callback), create_reference(token))}; + } else if (emulate_tls) { children_[token] = { - false, create_actor("PingHttpProxy", std::move(socket_fd), extra.mtproto_ip, - proxy.proxy().user().str(), proxy.proxy().password().str(), - make_unique(std::move(socket_fd_promise)), create_reference(token))}; + false, create_actor("PingTlsInit", std::move(socket_fd), extra.mtproto_ip, "www.google.com", + hex_decode(proxy.proxy().secret().substr(2)).move_as_ok(), std::move(callback), + create_reference(token))}; + } else { + UNREACHABLE(); } } else { socket_fd_promise.set_value(std::move(socket_fd)); @@ -1020,10 +1029,13 @@ void ConnectionCreator::client_loop(ClientInfo &client) { children_[token] = {true, create_actor("HttpProxy", std::move(socket_fd), extra.mtproto_ip, proxy.proxy().user().str(), proxy.proxy().password().str(), std::move(callback), create_reference(token))}; + } else if (emulate_tls) { + children_[token] = { + true, create_actor("TlsInit", std::move(socket_fd), extra.mtproto_ip, "www.google.com", + hex_decode(proxy.proxy().secret().substr(2)).move_as_ok(), std::move(callback), + create_reference(token))}; } else { - children_[token] = {true, create_actor("HttpProxy", std::move(socket_fd), extra.mtproto_ip, - "www.google.com" /*todo use domain*/, "", std::move(callback), - create_reference(token))}; + UNREACHABLE(); } } else { VLOG(connections) << "In client_loop: create new direct connection " << extra.debug_str; diff --git a/tdutils/td/utils/port/SocketFd.cpp b/tdutils/td/utils/port/SocketFd.cpp index b27530c50..5895bc1bd 100644 --- a/tdutils/td/utils/port/SocketFd.cpp +++ b/tdutils/td/utils/port/SocketFd.cpp @@ -86,6 +86,7 @@ class SocketFdImpl : private Iocp::Callback { } Result write(Slice data) { + // LOG(ERROR) << "Write: " << format::as_hex_dump<0>(data); output_writer_.append(data); if (is_write_waiting_) { auto lock = lock_.lock(); @@ -103,6 +104,8 @@ class SocketFdImpl : private Iocp::Callback { auto res = input_reader_.advance(td::min(slice.size(), input_reader_.size()), slice); if (res == 0) { get_poll_info().clear_flags(PollFlags::Read()); + } else { + // LOG(ERROR) << "Read: " << format::as_hex_dump<0>(Slice(slice.substr(0, res))); } return res; } diff --git a/test/mtproto.cpp b/test/mtproto.cpp index 3959a8d0d..e5eaed462 100644 --- a/test/mtproto.cpp +++ b/test/mtproto.cpp @@ -30,15 +30,12 @@ #include "td/telegram/net/Session.h" #include "td/telegram/NotificationManager.h" -#include "td/utils/as.h" #include "td/utils/base64.h" #include "td/utils/common.h" -#include "td/utils/crypto.h" #include "td/utils/logging.h" #include "td/utils/port/IPAddress.h" #include "td/utils/port/SocketFd.h" #include "td/utils/Random.h" -#include "td/utils/Span.h" #include "td/utils/Status.h" #include "td/utils/Time.h" @@ -591,15 +588,18 @@ class Mtproto_FastPing : public Test { }; RegisterTest mtproto_fastping("Mtproto_FastPing"); -TEST(Mtproto, TlsObfusaction) { - std::string s(10000, 'a'); +TEST(Mtproto, Grease) { + std::string s(10000, '0'); Grease::init(s); for (auto c : s) { - CHECK((c & 0xf) == 0xa); + CHECK((c & 0xF) == 0xA); } for (size_t i = 1; i < s.size(); i += 2) { CHECK(s[i] != s[i - 1]); } +} + +TEST(Mtproto, TlsObfusaction) { std::string domain = "www.google.com"; SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR)); ConcurrentScheduler sched; @@ -619,7 +619,8 @@ TEST(Mtproto, TlsObfusaction) { IPAddress ip_address; ip_address.init_host_port(domain, 443).ensure(); SocketFd fd = SocketFd::open(ip_address).move_as_ok(); - create_actor("TlsInit", std::move(fd), IPAddress(), domain, "", make_unique(), ActorShared<>()) + create_actor("TlsInit", std::move(fd), IPAddress(), domain, "0123456789secret", make_unique(), + ActorShared<>()) .release(); }