2018-12-31 20:04:05 +01:00
|
|
|
//
|
2021-01-01 13:57:46 +01:00
|
|
|
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2021
|
2018-12-31 20:04:05 +01:00
|
|
|
//
|
|
|
|
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
|
|
|
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
|
|
|
//
|
|
|
|
#include "td/utils/HttpUrl.h"
|
|
|
|
|
|
|
|
#include "td/utils/format.h"
|
|
|
|
#include "td/utils/logging.h"
|
|
|
|
#include "td/utils/misc.h"
|
|
|
|
#include "td/utils/Parser.h"
|
2020-05-16 21:53:19 +02:00
|
|
|
#include "td/utils/port/IPAddress.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
namespace td {
|
|
|
|
|
|
|
|
string HttpUrl::get_url() const {
|
|
|
|
string result;
|
|
|
|
switch (protocol_) {
|
2020-06-15 03:23:47 +02:00
|
|
|
case Protocol::Http:
|
2018-12-31 20:04:05 +01:00
|
|
|
result += "http://";
|
|
|
|
break;
|
2020-06-15 03:23:47 +02:00
|
|
|
case Protocol::Https:
|
2018-12-31 20:04:05 +01:00
|
|
|
result += "https://";
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
UNREACHABLE();
|
|
|
|
}
|
|
|
|
if (!userinfo_.empty()) {
|
|
|
|
result += userinfo_;
|
|
|
|
result += '@';
|
|
|
|
}
|
|
|
|
result += host_;
|
|
|
|
if (specified_port_ > 0) {
|
|
|
|
result += ':';
|
|
|
|
result += to_string(specified_port_);
|
|
|
|
}
|
2019-06-27 19:08:58 +02:00
|
|
|
LOG_CHECK(!query_.empty() && query_[0] == '/') << query_;
|
2018-12-31 20:04:05 +01:00
|
|
|
result += query_;
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2019-08-05 11:42:42 +02:00
|
|
|
Result<HttpUrl> parse_url(Slice url, HttpUrl::Protocol default_protocol) {
|
2018-12-31 20:04:05 +01:00
|
|
|
// url == [https?://][userinfo@]host[:port]
|
2019-08-05 11:42:42 +02:00
|
|
|
ConstParser parser(url);
|
2020-05-07 02:12:47 +02:00
|
|
|
string protocol_str = to_lower(parser.read_till_nofail(":/?#@[]"));
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
HttpUrl::Protocol protocol;
|
2020-07-21 13:24:55 +02:00
|
|
|
if (parser.try_skip("://")) {
|
2018-12-31 20:04:05 +01:00
|
|
|
if (protocol_str == "http") {
|
2020-06-15 03:23:47 +02:00
|
|
|
protocol = HttpUrl::Protocol::Http;
|
2018-12-31 20:04:05 +01:00
|
|
|
} else if (protocol_str == "https") {
|
2020-06-15 03:23:47 +02:00
|
|
|
protocol = HttpUrl::Protocol::Https;
|
2018-12-31 20:04:05 +01:00
|
|
|
} else {
|
|
|
|
return Status::Error("Unsupported URL protocol");
|
|
|
|
}
|
|
|
|
} else {
|
2019-08-05 11:42:42 +02:00
|
|
|
parser = ConstParser(url);
|
2018-12-31 20:04:05 +01:00
|
|
|
protocol = default_protocol;
|
|
|
|
}
|
|
|
|
Slice userinfo_host_port = parser.read_till_nofail("/?#");
|
|
|
|
|
|
|
|
int port = 0;
|
|
|
|
const char *colon = userinfo_host_port.end() - 1;
|
|
|
|
while (colon > userinfo_host_port.begin() && *colon != ':' && *colon != ']' && *colon != '@') {
|
|
|
|
colon--;
|
|
|
|
}
|
|
|
|
Slice userinfo_host;
|
|
|
|
if (colon > userinfo_host_port.begin() && *colon == ':') {
|
2021-05-26 00:20:49 +02:00
|
|
|
auto r_port = to_integer_safe<int>(Slice(colon + 1, userinfo_host_port.end()));
|
|
|
|
if (r_port.is_error() || r_port.ok() == 0) {
|
|
|
|
port = -1;
|
|
|
|
} else {
|
|
|
|
port = r_port.ok();
|
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
userinfo_host = Slice(userinfo_host_port.begin(), colon);
|
|
|
|
} else {
|
|
|
|
userinfo_host = userinfo_host_port;
|
|
|
|
}
|
|
|
|
if (port < 0 || port > 65535) {
|
|
|
|
return Status::Error("Wrong port number specified in the URL");
|
|
|
|
}
|
|
|
|
|
|
|
|
auto at_pos = userinfo_host.rfind('@');
|
|
|
|
Slice userinfo = at_pos == static_cast<size_t>(-1) ? "" : userinfo_host.substr(0, at_pos);
|
|
|
|
Slice host = userinfo_host.substr(at_pos + 1);
|
|
|
|
|
|
|
|
bool is_ipv6 = false;
|
|
|
|
if (!host.empty() && host[0] == '[' && host.back() == ']') {
|
2020-05-16 21:53:19 +02:00
|
|
|
IPAddress ip_address;
|
|
|
|
if (ip_address.init_ipv6_port(host.str(), 1).is_error()) {
|
|
|
|
return Status::Error("Wrong IPv6 address specified in the URL");
|
|
|
|
}
|
|
|
|
CHECK(ip_address.is_ipv6());
|
2018-12-31 20:04:05 +01:00
|
|
|
is_ipv6 = true;
|
|
|
|
}
|
|
|
|
if (host.empty()) {
|
|
|
|
return Status::Error("URL host is empty");
|
|
|
|
}
|
2019-02-12 15:53:00 +01:00
|
|
|
if (host == ".") {
|
|
|
|
return Status::Error("Host is invalid");
|
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
int specified_port = port;
|
|
|
|
if (port == 0) {
|
2020-06-15 03:23:47 +02:00
|
|
|
if (protocol == HttpUrl::Protocol::Http) {
|
2018-12-31 20:04:05 +01:00
|
|
|
port = 80;
|
|
|
|
} else {
|
2020-06-15 03:23:47 +02:00
|
|
|
CHECK(protocol == HttpUrl::Protocol::Https);
|
2018-12-31 20:04:05 +01:00
|
|
|
port = 443;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Slice query = parser.read_all();
|
|
|
|
while (!query.empty() && is_space(query.back())) {
|
|
|
|
query.remove_suffix(1);
|
|
|
|
}
|
|
|
|
if (query.empty()) {
|
2019-02-13 00:29:52 +01:00
|
|
|
query = Slice("/");
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
string query_str;
|
|
|
|
if (query[0] != '/') {
|
|
|
|
query_str = '/';
|
|
|
|
}
|
|
|
|
for (auto c : query) {
|
|
|
|
if (static_cast<unsigned char>(c) <= 0x20) {
|
|
|
|
query_str += '%';
|
|
|
|
query_str += "0123456789ABCDEF"[c / 16];
|
|
|
|
query_str += "0123456789ABCDEF"[c % 16];
|
|
|
|
} else {
|
|
|
|
query_str += c;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
string host_str = to_lower(host);
|
|
|
|
for (size_t i = 0; i < host_str.size(); i++) {
|
|
|
|
char c = host_str[i];
|
2018-07-01 01:29:36 +02:00
|
|
|
if (is_ipv6) {
|
2021-05-26 00:20:49 +02:00
|
|
|
if (i == 0 || i + 1 == host_str.size() || c == ':' || ('0' <= c && c <= '9') || ('a' <= c && c <= 'f') ||
|
|
|
|
c == '.') {
|
2018-07-01 01:29:36 +02:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
return Status::Error("Wrong IPv6 URL host");
|
|
|
|
}
|
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
if (('a' <= c && c <= 'z') || c == '.' || ('0' <= c && c <= '9') || c == '-' || c == '_' || c == '!' || c == '$' ||
|
|
|
|
c == ',' || c == '~' || c == '*' || c == '\'' || c == '(' || c == ')' || c == ';' || c == '&' || c == '+' ||
|
|
|
|
c == '=') {
|
|
|
|
// symbols allowed by RFC 7230 and RFC 3986
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (c == '%') {
|
|
|
|
c = host_str[++i];
|
|
|
|
if (('a' <= c && c <= 'f') || ('0' <= c && c <= '9')) {
|
|
|
|
c = host_str[++i];
|
|
|
|
if (('a' <= c && c <= 'f') || ('0' <= c && c <= '9')) {
|
|
|
|
// percent encoded symbol as allowed by RFC 7230 and RFC 3986
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
2018-07-01 01:29:36 +02:00
|
|
|
return Status::Error("Wrong percent-encoded symbol in URL host");
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
2018-07-01 01:29:36 +02:00
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
// all other symbols aren't allowed
|
|
|
|
unsigned char uc = static_cast<unsigned char>(c);
|
|
|
|
if (uc >= 128) {
|
|
|
|
// but we allow plain UTF-8 symbols
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
return Status::Error("Wrong URL host");
|
|
|
|
}
|
|
|
|
|
|
|
|
return HttpUrl{protocol, userinfo.str(), std::move(host_str), is_ipv6, specified_port, port, std::move(query_str)};
|
|
|
|
}
|
|
|
|
|
|
|
|
StringBuilder &operator<<(StringBuilder &sb, const HttpUrl &url) {
|
2020-06-15 03:23:47 +02:00
|
|
|
sb << tag("protocol", url.protocol_ == HttpUrl::Protocol::Http ? "HTTP" : "HTTPS") << tag("userinfo", url.userinfo_)
|
2018-12-31 20:04:05 +01:00
|
|
|
<< tag("host", url.host_) << tag("port", url.port_) << tag("query", url.query_);
|
|
|
|
return sb;
|
|
|
|
}
|
|
|
|
|
2021-05-24 22:47:18 +02:00
|
|
|
HttpUrlQuery parse_url_query(Slice query) {
|
2021-05-25 03:39:41 +02:00
|
|
|
if (!query.empty() && query[0] == '/') {
|
|
|
|
query.remove_prefix(1);
|
|
|
|
}
|
|
|
|
|
2021-05-24 22:47:18 +02:00
|
|
|
size_t path_size = 0;
|
|
|
|
while (path_size < query.size() && query[path_size] != '?' && query[path_size] != '#') {
|
|
|
|
path_size++;
|
|
|
|
}
|
|
|
|
|
|
|
|
HttpUrlQuery result;
|
2021-05-25 03:39:41 +02:00
|
|
|
result.path_ = full_split(url_decode(query.substr(0, path_size), false), '/');
|
|
|
|
if (!result.path_.empty() && result.path_.back().empty()) {
|
|
|
|
result.path_.pop_back();
|
|
|
|
}
|
2021-05-24 22:47:18 +02:00
|
|
|
|
|
|
|
if (path_size < query.size() && query[path_size] == '?') {
|
|
|
|
query = query.substr(path_size + 1);
|
|
|
|
query.truncate(query.find('#'));
|
|
|
|
|
|
|
|
ConstParser parser(query);
|
|
|
|
while (!parser.data().empty()) {
|
|
|
|
auto key_value = split(parser.read_till_nofail('&'), '=');
|
|
|
|
parser.skip_nofail('&');
|
|
|
|
auto key = url_decode(key_value.first, true);
|
|
|
|
if (!key.empty()) {
|
|
|
|
result.args_.emplace_back(std::move(key), url_decode(key_value.second, true));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
CHECK(parser.status().is_ok());
|
|
|
|
}
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
Slice HttpUrlQuery::get_arg(Slice key) const {
|
|
|
|
auto it =
|
|
|
|
std::find_if(args_.begin(), args_.end(), [&key](const std::pair<string, string> &s) { return s.first == key; });
|
|
|
|
return it == args_.end() ? Slice() : it->second;
|
|
|
|
}
|
|
|
|
|
2018-02-19 23:54:50 +01:00
|
|
|
string get_url_query_file_name(const string &query) {
|
|
|
|
Slice query_slice = query;
|
|
|
|
query_slice.truncate(query.find_first_of("?#"));
|
|
|
|
|
|
|
|
auto slash_pos = query_slice.rfind('/');
|
|
|
|
if (slash_pos < query_slice.size()) {
|
|
|
|
return query_slice.substr(slash_pos + 1).str();
|
|
|
|
}
|
|
|
|
return query_slice.str();
|
|
|
|
}
|
|
|
|
|
2019-08-05 11:56:28 +02:00
|
|
|
string get_url_file_name(Slice url) {
|
|
|
|
auto r_http_url = parse_url(url);
|
2018-02-20 00:29:19 +01:00
|
|
|
if (r_http_url.is_error()) {
|
|
|
|
LOG(WARNING) << "Receive wrong URL \"" << url << '"';
|
|
|
|
return string();
|
|
|
|
}
|
|
|
|
return get_url_query_file_name(r_http_url.ok().query_);
|
|
|
|
}
|
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
} // namespace td
|