Add support for transparent HTTP proxies.

GitOrigin-RevId: 5ae836625a60d1f84b0ca8df2c41bf07e8ba59fd

td/generate/scheme/td_api.tl

@@ -2051,6 +2051,9 @@ textParseModeHTML = TextParseMode;
 //@description A SOCKS5 proxy server @username Username for logging in; may be empty @password Password for logging in; may be empty
 proxyTypeSocks5 username:string password:string = ProxyType;
 
+//@description A HTTP transparent proxy server @username Username for logging in; may be empty @password Password for logging in; may be empty
+proxyTypeHttp username:string password:string = ProxyType;
+
 //@description An MTProto proxy server @secret The proxy's secret in hexadecimal encoding
 proxyTypeMtproto secret:string = ProxyType;
 

td/telegram/cli.cpp

@@ -3244,7 +3244,11 @@ class CliClient final : public Actor {
       if (!user.empty() && password.empty()) {
         type = make_tl_object<td_api::proxyTypeMtproto>(user);
       } else {
-        type = make_tl_object<td_api::proxyTypeSocks5>(user, password);
+        if (port == "80") {
+          type = make_tl_object<td_api::proxyTypeHttp>(user, password);
+        } else {
+          type = make_tl_object<td_api::proxyTypeSocks5>(user, password);
+        }
       }
       send_request(make_tl_object<td_api::addProxy>(server, to_integer<int32>(port), op == "aeproxy", std::move(type)));
     } else if (op == "gproxy" || op == "gproxies") {

td/telegram/net/ConnectionCreator.cpp

@@ -23,6 +23,7 @@
 #include "td/mtproto/RawConnection.h"
 
 #include "td/net/GetHostByNameActor.h"
+#include "td/net/HttpProxy.h"
 #include "td/net/Socks5.h"
 #include "td/net/TransparentProxy.h"
 
@@ -174,6 +175,9 @@ class ConnectionCreator::ProxyInfo {
   bool use_socks5_proxy() const {
     return proxy_type() == Proxy::Type::Socks5;
   }
+  bool use_http_proxy() const {
+    return proxy_type() == Proxy::Type::Http;
+  }
   bool use_mtproto_proxy() const {
     return proxy_type() == Proxy::Type::Mtproto;
   }
@@ -194,7 +198,7 @@ template <class T>
 void Proxy::parse(T &parser) {
   using td::parse;
   parse(type_, parser);
-  if (type_ == Proxy::Type::Socks5) {
+  if (type_ == Proxy::Type::Socks5 || type_ == Proxy::Type::Http) {
     parse(server_, parser);
     parse(port_, parser);
     parse(user_, parser);
@@ -212,7 +216,7 @@ template <class T>
 void Proxy::store(T &storer) const {
   using td::store;
   store(type_, storer);
-  if (type_ == Proxy::Type::Socks5) {
+  if (type_ == Proxy::Type::Socks5 || type_ == Proxy::Type::Http) {
     store(server_, storer);
     store(port_, storer);
     store(user_, storer);
@@ -230,6 +234,8 @@ StringBuilder &operator<<(StringBuilder &string_builder, const Proxy &proxy) {
   switch (proxy.type()) {
     case Proxy::Type::Socks5:
       return string_builder << "ProxySocks5 " << proxy.server() << ":" << proxy.port();
+    case Proxy::Type::Http:
+      return string_builder << "ProxyHttp " << proxy.server() << ":" << proxy.port();
     case Proxy::Type::Mtproto:
       return string_builder << "ProxyMtproto " << proxy.server() << ":" << proxy.port() << "/" << proxy.secret();
     case Proxy::Type::None:
@@ -298,6 +304,11 @@ void ConnectionCreator::add_proxy(string server, int32 port, bool enable,
       new_proxy = Proxy::socks5(server, port, type->username_, type->password_);
       break;
     }
+    case td_api::proxyTypeHttp::ID: {
+      auto type = td_api::move_object_as<td_api::proxyTypeHttp>(proxy_type);
+      new_proxy = Proxy::http(server, port, type->username_, type->password_);
+      break;
+    }
     case td_api::proxyTypeMtproto::ID: {
       auto type = td_api::move_object_as<td_api::proxyTypeMtproto>(proxy_type);
       if (hex_decode(type->secret_).is_error()) {
@@ -384,6 +395,8 @@ void ConnectionCreator::get_proxy_link(int32 proxy_id, Promise<string> promise)
       url += "socks";
       is_socks = true;
       break;
+    case Proxy::Type::Http:
+      return promise.set_error(Status::Error(400, "HTTP proxy can't have public link"));
     case Proxy::Type::Mtproto:
       url += "proxy";
       break;
@@ -493,7 +506,7 @@ void ConnectionCreator::ping_proxy_resolved(int32 proxy_id, IPAddress ip_address
                      std::move(transport_type), std::move(promise));
       });
   CHECK(proxy.use_proxy());
-  if (proxy.use_socks5_proxy()) {
+  if (proxy.use_socks5_proxy() || proxy.use_http_proxy()) {
     class Callback : public TransparentProxy::Callback {
      public:
       explicit Callback(Promise<SocketFd> promise) : promise_(std::move(promise)) {
@@ -507,12 +520,19 @@ void ConnectionCreator::ping_proxy_resolved(int32 proxy_id, IPAddress ip_address
      private:
       Promise<SocketFd> promise_;
     };
-    LOG(INFO) << "Start socks5: " << extra.debug_str;
+    LOG(INFO) << "Start ping proxy: " << extra.debug_str;
     auto token = next_token();
-    children_[token] = {
+    if (proxy.use_socks5_proxy()) {
-        false, create_actor<Socks5>("PingSocks5", std::move(socket_fd), extra.mtproto_ip, proxy.proxy().user().str(),
+      children_[token] = {false, create_actor<Socks5>("PingSocks5", std::move(socket_fd), extra.mtproto_ip,
-                                    proxy.proxy().password().str(),
+                                                      proxy.proxy().user().str(), proxy.proxy().password().str(),
-                                    std::make_unique<Callback>(std::move(socket_fd_promise)), create_reference(token))};
+                                                      std::make_unique<Callback>(std::move(socket_fd_promise)),
+                                                      create_reference(token))};
+    } else {
+      children_[token] = {false, create_actor<HttpProxy>("PingHttpProxy", std::move(socket_fd), extra.mtproto_ip,
+                                                         proxy.proxy().user().str(), proxy.proxy().password().str(),
+                                                         std::make_unique<Callback>(std::move(socket_fd_promise)),
+                                                         create_reference(token))};
+    }
   } else {
     socket_fd_promise.set_value(std::move(socket_fd));
   }
@@ -574,7 +594,7 @@ void ConnectionCreator::disable_proxy_impl() {
 
 void ConnectionCreator::on_proxy_changed(bool from_db) {
   send_closure(G()->state_manager(), &StateManager::on_proxy,
-               active_proxy_id_ != 0 && proxies_[active_proxy_id_].type() == Proxy::Type::Socks5);
+               active_proxy_id_ != 0 && proxies_[active_proxy_id_].type() != Proxy::Type::Mtproto);
 
   if (!from_db) {
     for (auto &child : children_) {
@@ -638,6 +658,9 @@ td_api::object_ptr<td_api::proxy> ConnectionCreator::get_proxy_object(int32 prox
     case Proxy::Type::Socks5:
       type = make_tl_object<td_api::proxyTypeSocks5>(proxy.user().str(), proxy.password().str());
       break;
+    case Proxy::Type::Http:
+      type = make_tl_object<td_api::proxyTypeHttp>(proxy.user().str(), proxy.password().str());
+      break;
     case Proxy::Type::Mtproto:
       type = make_tl_object<td_api::proxyTypeMtproto>(proxy.secret().str());
       break;
@@ -776,9 +799,10 @@ Result<SocketFd> ConnectionCreator::find_connection(const ProxyInfo &proxy, DcId
 
   extra.check_mode |= info.should_check;
 
-  if (proxy.use_socks5_proxy()) {
+  if (proxy.use_socks5_proxy() || proxy.use_http_proxy()) {
     extra.mtproto_ip = info.option->get_ip_address();
-    extra.debug_str = PSTRING() << "Socks5 " << proxy.ip_address() << " --> " << extra.mtproto_ip << extra.debug_str;
+    extra.debug_str = PSTRING() << (proxy.use_socks5_proxy() ? "Socks5 " : "HTTP ") << proxy.ip_address() << " --> "
+                                << extra.mtproto_ip << extra.debug_str;
     LOG(INFO) << "Create: " << extra.debug_str;
     return SocketFd::open(proxy.ip_address());
   } else {
@@ -905,7 +929,7 @@ void ConnectionCreator::client_loop(ClientInfo &client) {
         client.is_media ? media_net_stats_callback_ : common_net_stats_callback_, actor_id(this), client.hash,
         extra.stat);
 
-    if (proxy.use_socks5_proxy()) {
+    if (proxy.use_socks5_proxy() || proxy.use_http_proxy()) {
       class Callback : public TransparentProxy::Callback {
        public:
         explicit Callback(Promise<ConnectionData> promise, std::unique_ptr<detail::StatsCallback> stats_callback)
@@ -937,13 +961,21 @@ void ConnectionCreator::client_loop(ClientInfo &client) {
         bool was_connected_{false};
         std::unique_ptr<detail::StatsCallback> stats_callback_;
       };
-      LOG(INFO) << "Start socks5: " << extra.debug_str;
+      LOG(INFO) << "Start " << (proxy.use_socks5_proxy() ? "Socks5" : "HTTP") << ": " << extra.debug_str;
       auto token = next_token();
-      children_[token] = {
+      if (proxy.use_socks5_proxy()) {
-          true, create_actor<Socks5>("Socks5", std::move(socket_fd), extra.mtproto_ip, proxy.proxy().user().str(),
+        children_[token] = {
-                                     proxy.proxy().password().str(),
+            true, create_actor<Socks5>("Socks5", std::move(socket_fd), extra.mtproto_ip, proxy.proxy().user().str(),
-                                     std::make_unique<Callback>(std::move(promise), std::move(stats_callback)),
+                                       proxy.proxy().password().str(),
-                                     create_reference(token))};
+                                       std::make_unique<Callback>(std::move(promise), std::move(stats_callback)),
+                                       create_reference(token))};
+      } else {
+        children_[token] = {
+            true, create_actor<HttpProxy>("HttpProxy", std::move(socket_fd), extra.mtproto_ip,
+                                          proxy.proxy().user().str(), proxy.proxy().password().str(),
+                                          std::make_unique<Callback>(std::move(promise), std::move(stats_callback)),
+                                          create_reference(token))};
+      }
     } else {
       ConnectionData data;
       data.socket_fd = std::move(socket_fd);

td/telegram/net/ConnectionCreator.h

@@ -60,6 +60,16 @@ class Proxy {
     return proxy;
   }
 
+  static Proxy http(string server, int32 port, string user, string password) {
+    Proxy proxy;
+    proxy.type_ = Type::Http;
+    proxy.server_ = std::move(server);
+    proxy.port_ = std::move(port);
+    proxy.user_ = std::move(user);
+    proxy.password_ = std::move(password);
+    return proxy;
+  }
+
   static Proxy mtproto(string server, int32 port, string secret) {
     Proxy proxy;
     proxy.type_ = Type::Mtproto;
@@ -89,7 +99,7 @@ class Proxy {
     return secret_;
   }
 
-  enum class Type : int32 { None, Socks5, Mtproto };
+  enum class Type : int32 { None, Socks5, Mtproto, Http };
   Type type() const {
     return type_;
   }

tdnet/CMakeLists.txt

@@ -14,6 +14,7 @@ set(TDNET_SOURCE
   td/net/HttpFile.cpp
   td/net/HttpInboundConnection.cpp
   td/net/HttpOutboundConnection.cpp
+  td/net/HttpProxy.cpp
   td/net/HttpQuery.cpp
   td/net/HttpReader.cpp
   td/net/Socks5.cpp
@@ -30,6 +31,7 @@ set(TDNET_SOURCE
   td/net/HttpHeaderCreator.h
   td/net/HttpInboundConnection.h
   td/net/HttpOutboundConnection.h
+  td/net/HttpProxy.h
   td/net/HttpQuery.h
   td/net/HttpReader.h
   td/net/NetStats.h

tdnet/td/net/HttpProxy.cpp

@@ -0,0 +1,103 @@
+//
+// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#include "td/net/HttpProxy.h"
+
+#include "td/utils/base64.h"
+#include "td/utils/logging.h"
+#include "td/utils/misc.h"
+#include "td/utils/port/Fd.h"
+#include "td/utils/Slice.h"
+
+namespace td {
+
+void HttpProxy::send_connect() {
+  VLOG(proxy) << "Send CONNECT to proxy";
+  CHECK(state_ == State::SendConnect);
+  state_ = State::WaitConnectResponse;
+
+  string host = PSTRING() << ip_address_.get_ip_str() << ':' << ip_address_.get_port();
+  string proxy_authorization;
+  if (!username_.empty() || !password_.empty()) {
+    auto userinfo = PSTRING() << username_ << ':' << password_;
+    proxy_authorization = PSTRING() << "Proxy-Authorization: basic " << td::base64_encode(userinfo) << "\r\n";
+  }
+  fd_.output_buffer().append(PSLICE() << "CONNECT " << host << " HTTP/1.1\r\n"
+                                      << "Host: " << host << "\r\n"
+                                      << proxy_authorization << "\r\n");
+}
+
+Status HttpProxy::wait_connect_response() {
+  CHECK(state_ == State::WaitConnectResponse);
+  auto it = fd_.input_buffer().clone();
+  VLOG(proxy) << "Receive CONNECT response of size " << it.size();
+  if (it.size() < 12 + 1 + 1) {
+    return Status::OK();
+  }
+  char begin_buf[12];
+  MutableSlice begin(begin_buf, 12);
+  it.advance(12, begin);
+  if ((begin.substr(0, 10) != "HTTP/1.1 2" && begin.substr(0, 10) != "HTTP/1.0 2") || !is_digit(begin[10]) ||
+      !is_digit(begin[11])) {
+    return Status::Error("Failed to connect");
+  }
+
+  size_t total_size = 12;
+  char c;
+  MutableSlice c_slice(&c, 1);
+  while (!it.empty()) {
+    it.advance(1, c_slice);
+    total_size++;
+    if (c == '\n') {
+      break;
+    }
+  }
+  if (it.empty()) {
+    return Status::OK();
+  }
+
+  char prev = '\n';
+  size_t pos = 0;
+  bool found = false;
+  while (!it.empty()) {
+    it.advance(1, c_slice);
+    total_size++;
+    if (c == '\n') {
+      if (pos == 0 || (pos == 1 && prev == '\r')) {
+        found = true;
+        break;
+      }
+      pos = 0;
+    } else {
+      pos++;
+    }
+    prev = c;
+  }
+  if (!found) {
+    CHECK(it.empty());
+    return Status::OK();
+  }
+
+  fd_.input_buffer().advance(total_size);
+  stop();
+  return Status::OK();
+}
+
+Status HttpProxy::loop_impl() {
+  switch (state_) {
+    case State::SendConnect:
+      send_connect();
+      break;
+    case State::WaitConnectResponse:
+      TRY_STATUS(wait_connect_response());
+      break;
+    default:
+      UNREACHABLE();
+  }
+  return Status::OK();
+}
+
+}  // namespace td

tdnet/td/net/HttpProxy.h

@@ -0,0 +1,28 @@
+//
+// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2018
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#pragma once
+
+#include "td/net/TransparentProxy.h"
+
+#include "td/utils/Status.h"
+
+namespace td {
+
+class HttpProxy : public TransparentProxy {
+ public:
+  using TransparentProxy::TransparentProxy;
+
+ private:
+  enum class State { SendConnect, WaitConnectResponse } state_ = State::SendConnect;
+
+  void send_connect();
+  Status wait_connect_response();
+
+  Status loop_impl() override;
+};
+
+}  // namespace td

tdnet/td/net/Socks5.cpp

@@ -44,7 +44,6 @@ Status Socks5::wait_greeting_response() {
   }
   auto authentication_method = slice[1];
   if (authentication_method == '\0') {
-    state_ = State::SendIpAddress;
     send_ip_address();
     return Status::OK();
   }
@@ -90,14 +89,12 @@ Status Socks5::wait_password_response() {
     return Status::Error("Wrong username or password");
   }
 
-  state_ = State::SendIpAddress;
   send_ip_address();
   return Status::OK();
 }
 
 void Socks5::send_ip_address() {
   VLOG(proxy) << "Send IP address";
-  CHECK(state_ == State::SendIpAddress);
   callback_->on_connected();
   string request;
   request += '\x05';
@@ -182,8 +179,7 @@ Status Socks5::loop_impl() {
     case State::WaitIpAddressResponse:
       TRY_STATUS(wait_ip_address_response());
       break;
-    case State::SendIpAddress:
+    default:
-    case State::Stop:
       UNREACHABLE();
   }
   return Status::OK();

tdnet/td/net/Socks5.h

@@ -21,9 +21,7 @@ class Socks5 : public TransparentProxy {
     SendGreeting,
     WaitGreetingResponse,
     WaitPasswordResponse,
-    SendIpAddress,
+    WaitIpAddressResponse
-    WaitIpAddressResponse,
-    Stop
   } state_ = State::SendGreeting;
 
   void send_greeting();

tdnet/td/net/TransparentProxy.cpp

@@ -15,7 +15,7 @@ namespace td {
 int VERBOSITY_NAME(proxy) = VERBOSITY_NAME(DEBUG);
 
 TransparentProxy::TransparentProxy(SocketFd socket_fd, IPAddress ip_address, string username, string password,
-               std::unique_ptr<Callback> callback, ActorShared<> parent)
+                                   std::unique_ptr<Callback> callback, ActorShared<> parent)
     : fd_(std::move(socket_fd))
     , ip_address_(std::move(ip_address))
     , username_(std::move(username))

tdnet/td/net/TransparentProxy.h

@@ -31,8 +31,8 @@ class TransparentProxy : public Actor {
     virtual void on_connected() = 0;
   };
 
-  TransparentProxy(SocketFd socket_fd, IPAddress ip_address, string username, string password, std::unique_ptr<Callback> callback,
+  TransparentProxy(SocketFd socket_fd, IPAddress ip_address, string username, string password,
-         ActorShared<> parent);
+                   std::unique_ptr<Callback> callback, ActorShared<> parent);
 
  protected:
   BufferedFd<SocketFd> fd_;

tdutils/td/utils/port/IPAddress.cpp

@@ -405,7 +405,7 @@ Status IPAddress::init_peer_address(const SocketFd &socket_fd) {
 }
 
 static CSlice get_ip_str(int family, const void *addr) {
-  const int buf_size = INET6_ADDRSTRLEN;  //, INET_ADDRSTRLEN;
+  const int buf_size = INET6_ADDRSTRLEN;
   static TD_THREAD_LOCAL char *buf;
   init_thread_local<char[]>(buf, buf_size);