From 8de0bcd0a6dbf7cfc482f031c774a333fca0e7d1 Mon Sep 17 00:00:00 2001 From: levlam Date: Fri, 27 Jul 2018 03:54:25 +0300 Subject: [PATCH] Add support for HTTP-only HTTP proxies. GitOrigin-RevId: 7b8e3e7afb0f26c8e8fa97b0ffff5d1ad0267b5c --- td/generate/scheme/td_api.tl | 4 +- td/generate/scheme/td_api.tlo | Bin 130476 -> 130512 bytes td/mtproto/Handshake.h | 5 ++ td/mtproto/HttpTransport.cpp | 28 ++++++++-- td/mtproto/HttpTransport.h | 9 ++- td/mtproto/IStreamTransport.cpp | 2 +- td/mtproto/TcpTransport.h | 1 + td/telegram/cli.cpp | 4 +- td/telegram/net/ConnectionCreator.cpp | 77 +++++++++++++++++++------- td/telegram/net/ConnectionCreator.h | 16 +++++- td/telegram/net/DcOptionsSet.cpp | 31 ++++++----- td/telegram/net/DcOptionsSet.h | 6 +- 12 files changed, 134 insertions(+), 49 deletions(-) diff --git a/td/generate/scheme/td_api.tl b/td/generate/scheme/td_api.tl index 5d07c70c..02ab3fc3 100644 --- a/td/generate/scheme/td_api.tl +++ b/td/generate/scheme/td_api.tl @@ -2051,8 +2051,8 @@ textParseModeHTML = TextParseMode; //@description A SOCKS5 proxy server @username Username for logging in; may be empty @password Password for logging in; may be empty proxyTypeSocks5 username:string password:string = ProxyType; -//@description A HTTP transparent proxy server @username Username for logging in; may be empty @password Password for logging in; may be empty -proxyTypeHttp username:string password:string = ProxyType; +//@description A HTTP transparent proxy server @username Username for logging in; may be empty @password Password for logging in; may be empty @http_only Pass true, if the proxy supports only HTTP requests and doesn't support transparent TCP connections via HTTP CONNECT method +proxyTypeHttp username:string password:string http_only:Bool = ProxyType; //@description An MTProto proxy server @secret The proxy's secret in hexadecimal encoding proxyTypeMtproto secret:string = ProxyType; diff --git a/td/generate/scheme/td_api.tlo b/td/generate/scheme/td_api.tlo index bced32b08d1dcc89dc287867207387b9b41791ca..eaeefa0049c81c8a2eecd2c357500b5ab37f5bf5 100644 GIT binary patch delta 145 zcmZ4UoBhIX_6-VRES9sbKi;e&mh%G4nmpy7350RxUxY45jsXg;%5lY%A3wgBx1cD$ zqB5kiAl0L!q=10|q~_)^As^<+7yo%rR+!4ll>row&(F)LWSAUpCJQlb#eWqDqVV@@HMwY{W}(V_ +#include namespace td { namespace mtproto { @@ -47,9 +49,23 @@ void Transport::write(BufferWriter &&message, bool quick_ack) { * Host: url */ HttpHeaderCreator hc; - hc.init_post("/api"); - hc.add_header("Host", ""); - hc.set_keep_alive(); + Slice host; + Slice proxy_authorizarion; + std::tie(host, proxy_authorizarion) = split(Slice(secret_), '|'); + if (host.empty()) { + hc.init_post("/api"); + hc.add_header("Host", ""); + hc.set_keep_alive(); + } else { + hc.init_post(PSLICE() << "HTTP://" << host << ":80/api"); + hc.add_header("Host", host); + hc.add_header("User-Agent", "curl/7.35.0"); + hc.add_header("Accept", "*/*"); + hc.add_header("Proxy-Connection", "keep-alive"); + if (!proxy_authorizarion.empty()) { + hc.add_header("Proxy-Authorization", proxy_authorizarion); + } + } hc.set_content_size(message.size()); auto r_head = hc.finish(); CHECK(r_head.is_ok()); @@ -71,7 +87,11 @@ bool Transport::can_write() const { } size_t Transport::max_prepend_size() const { - return MAX_PREPEND_SIZE; + if (secret_.empty()) { + return 96; + } else { + return (secret_.size() + 1) / 2 * 4 + 156; + } } size_t Transport::max_append_size() const { diff --git a/td/mtproto/HttpTransport.h b/td/mtproto/HttpTransport.h index 6fc6084f..e625fe35 100644 --- a/td/mtproto/HttpTransport.h +++ b/td/mtproto/HttpTransport.h @@ -18,8 +18,12 @@ namespace td { namespace mtproto { namespace http { + class Transport : public IStreamTransport { public: + explicit Transport(string secret) : secret_(std::move(secret)) { + } + Result read_next(BufferSlice *message, uint32 *quick_ack) override TD_WARN_UNUSED_RESULT; bool support_quick_ack() const override { return false; @@ -35,16 +39,15 @@ class Transport : public IStreamTransport { size_t max_prepend_size() const override; size_t max_append_size() const override; TransportType get_type() const override { - return {TransportType::Http, 0, ""}; + return {TransportType::Http, 0, secret_}; } private: + string secret_; HttpReader reader_; HttpQuery http_query_; ChainBufferWriter *output_; enum { Write, Read } turn_ = Write; - - enum { MAX_PREPEND_SIZE = 96 }; }; } // namespace http diff --git a/td/mtproto/IStreamTransport.cpp b/td/mtproto/IStreamTransport.cpp index 1184d166..71bafc8b 100644 --- a/td/mtproto/IStreamTransport.cpp +++ b/td/mtproto/IStreamTransport.cpp @@ -20,7 +20,7 @@ std::unique_ptr create_transport(TransportType type) { case TransportType::Tcp: return std::make_unique(); case TransportType::Http: - return std::make_unique(); + return std::make_unique(type.secret); } UNREACHABLE(); } diff --git a/td/mtproto/TcpTransport.h b/td/mtproto/TcpTransport.h index 2ffdaf8c..7cfd8235 100644 --- a/td/mtproto/TcpTransport.h +++ b/td/mtproto/TcpTransport.h @@ -19,6 +19,7 @@ namespace td { namespace mtproto { namespace tcp { + class ITransport { // Writes packet into message. // Returns 0 if everything is ok, and [expected_size] otherwise. diff --git a/td/telegram/cli.cpp b/td/telegram/cli.cpp index f4d6ed2d..6564691e 100644 --- a/td/telegram/cli.cpp +++ b/td/telegram/cli.cpp @@ -3232,7 +3232,7 @@ class CliClient final : public Actor { send_request(make_tl_object(as_proxy_id(args))); } else if (op == "rproxy") { send_request(make_tl_object(as_proxy_id(args))); - } else if (op == "aproxy" || op == "aeproxy") { + } else if (op == "aproxy" || op == "aeproxy" || op == "aeproxytcp") { string server; string port; string user; @@ -3245,7 +3245,7 @@ class CliClient final : public Actor { type = make_tl_object(user); } else { if (port == "80") { - type = make_tl_object(user, password); + type = make_tl_object(user, password, op != "aeproxytcp"); } else { type = make_tl_object(user, password); } diff --git a/td/telegram/net/ConnectionCreator.cpp b/td/telegram/net/ConnectionCreator.cpp index 8f68087a..5e322b10 100644 --- a/td/telegram/net/ConnectionCreator.cpp +++ b/td/telegram/net/ConnectionCreator.cpp @@ -27,6 +27,7 @@ #include "td/net/Socks5.h" #include "td/net/TransparentProxy.h" +#include "td/utils/base64.h" #include "td/utils/format.h" #include "td/utils/logging.h" #include "td/utils/misc.h" @@ -175,8 +176,11 @@ class ConnectionCreator::ProxyInfo { bool use_socks5_proxy() const { return proxy_type() == Proxy::Type::Socks5; } - bool use_http_proxy() const { - return proxy_type() == Proxy::Type::Http; + bool use_http_tcp_proxy() const { + return proxy_type() == Proxy::Type::HttpTcp; + } + bool use_http_caching_proxy() const { + return proxy_type() == Proxy::Type::HttpCaching; } bool use_mtproto_proxy() const { return proxy_type() == Proxy::Type::Mtproto; @@ -198,7 +202,7 @@ template void Proxy::parse(T &parser) { using td::parse; parse(type_, parser); - if (type_ == Proxy::Type::Socks5 || type_ == Proxy::Type::Http) { + if (type_ == Proxy::Type::Socks5 || type_ == Proxy::Type::HttpTcp || type_ == Proxy::Type::HttpCaching) { parse(server_, parser); parse(port_, parser); parse(user_, parser); @@ -216,7 +220,7 @@ template void Proxy::store(T &storer) const { using td::store; store(type_, storer); - if (type_ == Proxy::Type::Socks5 || type_ == Proxy::Type::Http) { + if (type_ == Proxy::Type::Socks5 || type_ == Proxy::Type::HttpTcp || type_ == Proxy::Type::HttpCaching) { store(server_, storer); store(port_, storer); store(user_, storer); @@ -234,8 +238,10 @@ StringBuilder &operator<<(StringBuilder &string_builder, const Proxy &proxy) { switch (proxy.type()) { case Proxy::Type::Socks5: return string_builder << "ProxySocks5 " << proxy.server() << ":" << proxy.port(); - case Proxy::Type::Http: - return string_builder << "ProxyHttp " << proxy.server() << ":" << proxy.port(); + case Proxy::Type::HttpTcp: + return string_builder << "ProxyHttpTcp " << proxy.server() << ":" << proxy.port(); + case Proxy::Type::HttpCaching: + return string_builder << "ProxyHttpCaching " << proxy.server() << ":" << proxy.port(); case Proxy::Type::Mtproto: return string_builder << "ProxyMtproto " << proxy.server() << ":" << proxy.port() << "/" << proxy.secret(); case Proxy::Type::None: @@ -306,7 +312,11 @@ void ConnectionCreator::add_proxy(string server, int32 port, bool enable, } case td_api::proxyTypeHttp::ID: { auto type = td_api::move_object_as(proxy_type); - new_proxy = Proxy::http(server, port, type->username_, type->password_); + if (type->http_only_) { + new_proxy = Proxy::http_caching(server, port, type->username_, type->password_); + } else { + new_proxy = Proxy::http_tcp(server, port, type->username_, type->password_); + } break; } case td_api::proxyTypeMtproto::ID: { @@ -395,7 +405,8 @@ void ConnectionCreator::get_proxy_link(int32 proxy_id, Promise promise) url += "socks"; is_socks = true; break; - case Proxy::Type::Http: + case Proxy::Type::HttpTcp: + case Proxy::Type::HttpCaching: return promise.set_error(Status::Error(400, "HTTP proxy can't have public link")); case Proxy::Type::Mtproto: url += "proxy"; @@ -426,7 +437,7 @@ void ConnectionCreator::ping_proxy(int32 proxy_id, Promise promise) { ProxyInfo proxy{nullptr, IPAddress()}; auto main_dc_id = G()->net_query_dispatcher().main_dc_id(); bool prefer_ipv6 = G()->shared_config().get_option_boolean("prefer_ipv6"); - auto infos = dc_options_set_.find_all_connections(main_dc_id, false, false, prefer_ipv6); + auto infos = dc_options_set_.find_all_connections(main_dc_id, false, false, prefer_ipv6, false); if (infos.empty()) { return promise.set_error(Status::Error(400, "Can't find valid DC address")); } @@ -506,7 +517,7 @@ void ConnectionCreator::ping_proxy_resolved(int32 proxy_id, IPAddress ip_address std::move(transport_type), std::move(promise)); }); CHECK(proxy.use_proxy()); - if (proxy.use_socks5_proxy() || proxy.use_http_proxy()) { + if (proxy.use_socks5_proxy() || proxy.use_http_tcp_proxy()) { class Callback : public TransparentProxy::Callback { public: explicit Callback(Promise promise) : promise_(std::move(promise)) { @@ -604,6 +615,7 @@ void ConnectionCreator::on_proxy_changed(bool from_db) { } } + VLOG(connections) << "Drop proxy IP address " << proxy_ip_address_; resolve_proxy_query_token_ = 0; resolve_proxy_timestamp_ = Timestamp(); proxy_ip_address_ = IPAddress(); @@ -658,8 +670,11 @@ td_api::object_ptr ConnectionCreator::get_proxy_object(int32 prox case Proxy::Type::Socks5: type = make_tl_object(proxy.user().str(), proxy.password().str()); break; - case Proxy::Type::Http: - type = make_tl_object(proxy.user().str(), proxy.password().str()); + case Proxy::Type::HttpTcp: + type = make_tl_object(proxy.user().str(), proxy.password().str(), false); + break; + case Proxy::Type::HttpCaching: + type = make_tl_object(proxy.user().str(), proxy.password().str(), true); break; case Proxy::Type::Mtproto: type = make_tl_object(proxy.secret().str()); @@ -674,13 +689,17 @@ td_api::object_ptr ConnectionCreator::get_proxy_object(int32 prox } void ConnectionCreator::on_network(bool network_flag, uint32 network_generation) { + VLOG(connections) << "Receive network flag " << network_flag << " with generation " << network_generation; network_flag_ = network_flag; auto old_generation = network_generation_; network_generation_ = network_generation; if (network_flag_) { - resolve_proxy_query_token_ = 0; - resolve_proxy_timestamp_ = Timestamp(); - get_proxy_info_timestamp_ = Timestamp(); + if (old_generation != network_generation_) { + VLOG(connections) << "Set proxy query token to 0: " << old_generation << " " << network_generation_; + resolve_proxy_query_token_ = 0; + resolve_proxy_timestamp_ = Timestamp(); + get_proxy_info_timestamp_ = Timestamp(); + } for (auto &client : clients_) { client.second.backoff.clear(); client.second.flood_control.clear_events(); @@ -695,6 +714,7 @@ void ConnectionCreator::on_network(bool network_flag, uint32 network_generation) } void ConnectionCreator::on_online(bool online_flag) { + VLOG(connections) << "Receive online flag " << online_flag; online_flag_ = online_flag; if (online_flag_) { for (auto &client : clients_) { @@ -768,6 +788,16 @@ Result ConnectionCreator::get_transport_type(const Proxy TRY_RESULT(secret, hex_decode(proxy.proxy().secret())); return mtproto::TransportType{mtproto::TransportType::ObfuscatedTcp, raw_dc_id, std::move(secret)}; } + if (proxy.use_http_caching_proxy()) { + CHECK(info.option != nullptr); + string proxy_authorization; + if (!proxy.proxy().user().empty() || !proxy.proxy().password().empty()) { + proxy_authorization = + "|basic " + td::base64_encode(PSLICE() << proxy.proxy().user() << ':' << proxy.proxy().password()); + } + return mtproto::TransportType{mtproto::TransportType::Http, 0, + PSTRING() << info.option->get_ip_address().get_ip_str() << proxy_authorization}; + } if (info.use_http) { return mtproto::TransportType{mtproto::TransportType::Http, 0, ""}; @@ -781,8 +811,9 @@ Result ConnectionCreator::find_connection(const ProxyInfo &proxy, DcId extra.debug_str = PSTRING() << "Failed to find valid IP for " << dc_id; bool prefer_ipv6 = G()->shared_config().get_option_boolean("prefer_ipv6") || (proxy.use_proxy() && proxy.ip_address().is_ipv6()); - TRY_RESULT(info, dc_options_set_.find_connection(dc_id, allow_media_only, - proxy.use_proxy() && proxy.use_socks5_proxy(), prefer_ipv6)); + bool only_http = proxy.use_http_caching_proxy(); + TRY_RESULT(info, dc_options_set_.find_connection( + dc_id, allow_media_only, proxy.use_proxy() && proxy.use_socks5_proxy(), prefer_ipv6, only_http)); extra.stat = info.stat; TRY_RESULT(transport_type, get_transport_type(proxy, info)); extra.transport_type = std::move(transport_type); @@ -799,7 +830,7 @@ Result ConnectionCreator::find_connection(const ProxyInfo &proxy, DcId extra.check_mode |= info.should_check; - if (proxy.use_socks5_proxy() || proxy.use_http_proxy()) { + if (proxy.use_proxy()) { extra.mtproto_ip = info.option->get_ip_address(); extra.debug_str = PSTRING() << (proxy.use_socks5_proxy() ? "Socks5 " : "HTTP ") << proxy.ip_address() << " --> " << extra.mtproto_ip << extra.debug_str; @@ -815,14 +846,17 @@ Result ConnectionCreator::find_connection(const ProxyInfo &proxy, DcId void ConnectionCreator::client_loop(ClientInfo &client) { CHECK(client.hash != 0); if (!network_flag_) { + VLOG(connections) << "Exit client_loop, because there is no network"; return; } if (close_flag_) { + VLOG(connections) << "Exit client_loop, because of closing"; return; } ProxyInfo proxy{active_proxy_id_ == 0 ? nullptr : &proxies_[active_proxy_id_], proxy_ip_address_}; if (proxy.use_proxy() && !proxy.ip_address().is_valid()) { + VLOG(connections) << "Exit client_loop, because there is no valid IP address for proxy: " << proxy.ip_address(); return; } @@ -929,7 +963,7 @@ void ConnectionCreator::client_loop(ClientInfo &client) { client.is_media ? media_net_stats_callback_ : common_net_stats_callback_, actor_id(this), client.hash, extra.stat); - if (proxy.use_socks5_proxy() || proxy.use_http_proxy()) { + if (proxy.use_socks5_proxy() || proxy.use_http_tcp_proxy()) { class Callback : public TransparentProxy::Callback { public: explicit Callback(Promise promise, std::unique_ptr stats_callback) @@ -1267,6 +1301,7 @@ void ConnectionCreator::loop() { resolve_proxy_query_token_ = next_token(); const Proxy &proxy = proxies_[active_proxy_id_]; bool prefer_ipv6 = G()->shared_config().get_option_boolean("prefer_ipv6"); + VLOG(connections) << "Resolve IP address " << resolve_proxy_query_token_ << " of " << proxy.server(); send_closure( get_host_by_name_actor_, &GetHostByNameActor::run, proxy.server().str(), proxy.port(), prefer_ipv6, PromiseCreator::lambda([actor_id = create_reference(resolve_proxy_query_token_)](Result result) { @@ -1349,6 +1384,8 @@ void ConnectionCreator::schedule_get_proxy_info(int32 expires) { void ConnectionCreator::on_proxy_resolved(Result r_ip_address, bool dummy) { if (get_link_token() != resolve_proxy_query_token_) { + VLOG(connections) << "Ignore unneeded proxy IP address " << get_link_token() << ", expected " + << resolve_proxy_query_token_; return; } @@ -1358,10 +1395,12 @@ void ConnectionCreator::on_proxy_resolved(Result r_ip_address, bool d resolve_proxy_query_token_ = 0; if (r_ip_address.is_error()) { + VLOG(connections) << "Receive error for resolving proxy IP address: " << r_ip_address.error(); resolve_proxy_timestamp_ = Timestamp::in(1 * 60); return; } proxy_ip_address_ = r_ip_address.move_as_ok(); + VLOG(connections) << "Set proxy IP address to " << proxy_ip_address_; resolve_proxy_timestamp_ = Timestamp::in(5 * 60); for (auto &client : clients_) { client_loop(client.second); diff --git a/td/telegram/net/ConnectionCreator.h b/td/telegram/net/ConnectionCreator.h index d95ef940..d234a1a5 100644 --- a/td/telegram/net/ConnectionCreator.h +++ b/td/telegram/net/ConnectionCreator.h @@ -60,9 +60,19 @@ class Proxy { return proxy; } - static Proxy http(string server, int32 port, string user, string password) { + static Proxy http_tcp(string server, int32 port, string user, string password) { Proxy proxy; - proxy.type_ = Type::Http; + proxy.type_ = Type::HttpTcp; + proxy.server_ = std::move(server); + proxy.port_ = std::move(port); + proxy.user_ = std::move(user); + proxy.password_ = std::move(password); + return proxy; + } + + static Proxy http_caching(string server, int32 port, string user, string password) { + Proxy proxy; + proxy.type_ = Type::HttpCaching; proxy.server_ = std::move(server); proxy.port_ = std::move(port); proxy.user_ = std::move(user); @@ -99,7 +109,7 @@ class Proxy { return secret_; } - enum class Type : int32 { None, Socks5, Mtproto, Http }; + enum class Type : int32 { None, Socks5, Mtproto, HttpTcp, HttpCaching }; Type type() const { return type_; } diff --git a/td/telegram/net/DcOptionsSet.cpp b/td/telegram/net/DcOptionsSet.cpp index 8c32625c..e4210768 100644 --- a/td/telegram/net/DcOptionsSet.cpp +++ b/td/telegram/net/DcOptionsSet.cpp @@ -44,7 +44,8 @@ DcOptions DcOptionsSet::get_dc_options() const { } vector DcOptionsSet::find_all_connections(DcId dc_id, bool allow_media_only, - bool use_static, bool prefer_ipv6) { + bool use_static, bool prefer_ipv6, + bool only_http) { std::vector options; std::vector static_options; @@ -72,18 +73,22 @@ vector DcOptionsSet::find_all_connections(DcId dc_ OptionStat *option_stat = get_option_stat(option_info.get()); - info.use_http = false; - info.stat = &option_stat->tcp_stat; - if (option.is_static()) { - static_options.push_back(info); - } else { - options.push_back(info); + if (!only_http) { + info.use_http = false; + info.stat = &option_stat->tcp_stat; + if (option.is_static()) { + static_options.push_back(info); + } else { + options.push_back(info); + } } - if (!option.is_obfuscated_tcp_only() && !option.is_static() && false) { // TODO fix HTTP-mode and enable it - info.use_http = true; - info.stat = &option_stat->http_stat; - options.push_back(info); + if (only_http) { + if (!option.is_obfuscated_tcp_only() && !option.is_static()) { + info.use_http = true; + info.stat = &option_stat->http_stat; + options.push_back(info); + } } } @@ -121,8 +126,8 @@ vector DcOptionsSet::find_all_connections(DcId dc_ } Result DcOptionsSet::find_connection(DcId dc_id, bool allow_media_only, bool use_static, - bool prefer_ipv6) { - auto options = find_all_connections(dc_id, allow_media_only, use_static, prefer_ipv6); + bool prefer_ipv6, bool only_http) { + auto options = find_all_connections(dc_id, allow_media_only, use_static, prefer_ipv6, only_http); if (options.empty()) { return Status::Error(PSLICE() << "No such connection: " << tag("dc_id", dc_id) diff --git a/td/telegram/net/DcOptionsSet.h b/td/telegram/net/DcOptionsSet.h index 75690aee..be8469b5 100644 --- a/td/telegram/net/DcOptionsSet.h +++ b/td/telegram/net/DcOptionsSet.h @@ -61,9 +61,11 @@ class DcOptionsSet { Stat *stat{nullptr}; }; - vector find_all_connections(DcId dc_id, bool allow_media_only, bool use_static, bool prefer_ipv6); + vector find_all_connections(DcId dc_id, bool allow_media_only, bool use_static, bool prefer_ipv6, + bool only_http); - Result find_connection(DcId dc_id, bool allow_media_only, bool use_static, bool prefer_ipv6); + Result find_connection(DcId dc_id, bool allow_media_only, bool use_static, bool prefer_ipv6, + bool only_http); void reset(); private: