// // Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2020 // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // #include "td/utils/tests.h" #include "td/net/HttpChunkedByteFlow.h" #include "td/net/HttpHeaderCreator.h" #include "td/net/HttpQuery.h" #include "td/net/HttpReader.h" #include "td/utils/AesCtrByteFlow.h" #include "td/utils/base64.h" #include "td/utils/buffer.h" #include "td/utils/BufferedFd.h" #include "td/utils/ByteFlow.h" #include "td/utils/common.h" #include "td/utils/crypto.h" #include "td/utils/format.h" #include "td/utils/Gzip.h" #include "td/utils/GzipByteFlow.h" #include "td/utils/logging.h" #include "td/utils/misc.h" #include "td/utils/port/detail/PollableFd.h" #include "td/utils/port/FileFd.h" #include "td/utils/port/path.h" #include "td/utils/port/PollFlags.h" #include "td/utils/port/thread_local.h" #include "td/utils/Random.h" #include "td/utils/Slice.h" #include "td/utils/Status.h" #include "td/utils/UInt.h" #include "data.h" #include <algorithm> #include <cstdlib> #include <limits> REGISTER_TESTS(http) using namespace td; static string make_chunked(string str) { auto v = rand_split(str); string res; for (auto &s : v) { res += PSTRING() << format::as_hex_dump(static_cast<int32>(s.size())); res += "\r\n"; res += s; res += "\r\n"; } res += "0\r\n\r\n"; return res; } static string gen_http_content() { int t = Random::fast(0, 2); int len; if (t == 0) { len = Random::fast(1, 10); } else if (t == 1) { len = Random::fast(100, 200); } else { len = Random::fast(1000, 20000); } return rand_string(std::numeric_limits<char>::min(), std::numeric_limits<char>::max(), len); } static string make_http_query(string content, bool is_chunked, bool is_gzip, double gzip_k = 5, string zip_override = "") { HttpHeaderCreator hc; hc.init_post("/"); hc.add_header("jfkdlsahhjk", rand_string('a', 'z', Random::fast(1, 2000))); if (is_gzip) { BufferSlice zip; if (zip_override.empty()) { zip = gzencode(content, gzip_k); } else { zip = BufferSlice(zip_override); } if (!zip.empty()) { hc.add_header("content-encoding", "gzip"); content = zip.as_slice().str(); } } if (is_chunked) { hc.add_header("transfer-encoding", "chunked"); content = make_chunked(content); } else { hc.set_content_size(content.size()); } string res; auto r_header = hc.finish(); CHECK(r_header.is_ok()); res += r_header.ok().str(); res += content; return res; } static string rand_http_query(string content) { bool is_chunked = Random::fast(0, 1) == 0; bool is_gzip = Random::fast(0, 1) == 0; return make_http_query(std::move(content), is_chunked, is_gzip); } static string join(const std::vector<string> &v) { string res; for (auto &s : v) { res += s; } return res; } TEST(Http, stack_overflow) { ChainBufferWriter writer; BufferSlice slice(string(256, 'A')); for (int i = 0; i < 1000000; i++) { ChainBufferWriter tmp_writer; writer.append(slice.clone()); } { auto reader = writer.extract_reader(); reader.sync_with_writer(); } } TEST(Http, reader) { #if TD_ANDROID || TD_TIZEN return; #endif clear_thread_locals(); SET_VERBOSITY_LEVEL(VERBOSITY_NAME(ERROR)); auto start_mem = BufferAllocator::get_buffer_mem(); { td::ChainBufferWriter input_writer; auto input = input_writer.extract_reader(); HttpReader reader; int max_post_size = 10000; reader.init(&input, max_post_size, 0); std::srand(4); std::vector<string> contents(1000); std::generate(contents.begin(), contents.end(), gen_http_content); auto v = td::transform(contents, rand_http_query); auto vec_str = rand_split(join(v)); HttpQuery q; std::vector<string> res; for (auto &str : vec_str) { input_writer.append(str); input.sync_with_writer(); while (true) { auto r_state = reader.read_next(&q); LOG_IF(ERROR, r_state.is_error()) << r_state.error() << tag("ok", res.size()); ASSERT_TRUE(r_state.is_ok()); auto state = r_state.ok(); if (state == 0) { if (q.files_.empty()) { ASSERT_TRUE(td::narrow_cast<int>(q.content_.size()) <= max_post_size); auto expected = contents[res.size()]; ASSERT_EQ(expected, q.content_.str()); res.push_back(q.content_.str()); } else { auto r_fd = FileFd::open(q.files_[0].temp_file_name, FileFd::Read); ASSERT_TRUE(r_fd.is_ok()); auto fd = r_fd.move_as_ok(); string content(td::narrow_cast<size_t>(q.files_[0].size), '\0'); auto r_size = fd.read(MutableSlice(content)); ASSERT_TRUE(r_size.is_ok()); ASSERT_TRUE(r_size.ok() == content.size()); ASSERT_TRUE(td::narrow_cast<int>(content.size()) > max_post_size); ASSERT_EQ(contents[res.size()], content); res.push_back(content); fd.close(); } } else { break; } } } ASSERT_EQ(contents.size(), res.size()); ASSERT_EQ(contents, res); } clear_thread_locals(); ASSERT_EQ(start_mem, BufferAllocator::get_buffer_mem()); } TEST(Http, gzip_bomb) { #if TD_ANDROID || TD_TIZEN || TD_EMSCRIPTEN // the test should be disabled on low-memory systems return; #endif auto gzip_bomb_str = gzdecode(gzdecode(base64url_decode(Slice(gzip_bomb, gzip_bomb_size)).ok()).as_slice()).as_slice().str(); auto query = make_http_query("", false, true, 0.01, gzip_bomb_str); auto parts = rand_split(query); td::ChainBufferWriter input_writer; auto input = input_writer.extract_reader(); HttpReader reader; HttpQuery q; reader.init(&input, 100000000, 0); for (auto &part : parts) { input_writer.append(part); input.sync_with_writer(); auto r_state = reader.read_next(&q); if (r_state.is_error()) { LOG(INFO) << r_state.error(); return; } ASSERT_TRUE(r_state.ok() != 0); } } TEST(Http, aes_ctr_encode_decode_flow) { auto str = rand_string('a', 'z', 1000000); auto parts = rand_split(str); td::ChainBufferWriter input_writer; auto input = input_writer.extract_reader(); ByteFlowSource source(&input); UInt256 key; UInt128 iv; Random::secure_bytes(key.raw, sizeof(key)); Random::secure_bytes(iv.raw, sizeof(iv)); AesCtrByteFlow aes_encode; aes_encode.init(key, iv); AesCtrByteFlow aes_decode; aes_decode.init(key, iv); ByteFlowSink sink; source >> aes_encode >> aes_decode >> sink; ASSERT_TRUE(!sink.is_ready()); for (auto &part : parts) { input_writer.append(part); source.wakeup(); } ASSERT_TRUE(!sink.is_ready()); source.close_input(Status::OK()); ASSERT_TRUE(sink.is_ready()); LOG_IF(ERROR, sink.status().is_error()) << sink.status(); ASSERT_TRUE(sink.status().is_ok()); ASSERT_EQ(str, sink.result()->move_as_buffer_slice().as_slice().str()); } TEST(Http, aes_file_encryption) { auto str = rand_string('a', 'z', 1000000); CSlice name = "test_encryption"; unlink(name).ignore(); UInt256 key; UInt128 iv; Random::secure_bytes(key.raw, sizeof(key)); Random::secure_bytes(iv.raw, sizeof(iv)); { BufferedFdBase<FileFd> fd(FileFd::open(name, FileFd::Write | FileFd::Create).move_as_ok()); auto parts = rand_split(str); ChainBufferWriter output_writer; auto output_reader = output_writer.extract_reader(); ByteFlowSource source(&output_reader); AesCtrByteFlow aes_encode; aes_encode.init(key, iv); ByteFlowSink sink; source >> aes_encode >> sink; fd.set_output_reader(sink.get_output()); for (auto &part : parts) { output_writer.append(part); source.wakeup(); fd.flush_write().ensure(); } fd.close(); } { BufferedFdBase<FileFd> fd(FileFd::open(name, FileFd::Read).move_as_ok()); ChainBufferWriter input_writer; auto input_reader = input_writer.extract_reader(); ByteFlowSource source(&input_reader); AesCtrByteFlow aes_encode; aes_encode.init(key, iv); ByteFlowSink sink; source >> aes_encode >> sink; fd.set_input_writer(&input_writer); fd.get_poll_info().add_flags(PollFlags::Read()); while (can_read(fd)) { fd.flush_read(4096).ensure(); source.wakeup(); } fd.close(); source.close_input(Status::OK()); ASSERT_TRUE(sink.is_ready()); LOG_IF(ERROR, sink.status().is_error()) << sink.status(); ASSERT_TRUE(sink.status().is_ok()); auto result = sink.result()->move_as_buffer_slice().as_slice().str(); ASSERT_EQ(str, result); } } TEST(Http, chunked_flow) { auto str = rand_string('a', 'z', 100); auto parts = rand_split(make_chunked(str)); td::ChainBufferWriter input_writer; auto input = input_writer.extract_reader(); ByteFlowSource source(&input); HttpChunkedByteFlow chunked_flow; ByteFlowSink sink; source >> chunked_flow >> sink; for (auto &part : parts) { input_writer.append(part); source.wakeup(); } source.close_input(Status::OK()); ASSERT_TRUE(sink.is_ready()); LOG_IF(ERROR, sink.status().is_error()) << sink.status(); ASSERT_TRUE(sink.status().is_ok()); auto res = sink.result()->move_as_buffer_slice().as_slice().str(); ASSERT_EQ(str.size(), res.size()); ASSERT_EQ(str, res); } TEST(Http, chunked_flow_error) { auto str = rand_string('a', 'z', 100000); for (int d = 1; d < 100; d += 10) { auto new_str = make_chunked(str); new_str.resize(str.size() - d); auto parts = rand_split(new_str); td::ChainBufferWriter input_writer; auto input = input_writer.extract_reader(); ByteFlowSource source(&input); HttpChunkedByteFlow chunked_flow; ByteFlowSink sink; source >> chunked_flow >> sink; for (auto &part : parts) { input_writer.append(part); source.wakeup(); } ASSERT_TRUE(!sink.is_ready()); source.close_input(Status::OK()); ASSERT_TRUE(sink.is_ready()); ASSERT_TRUE(!sink.status().is_ok()); } } TEST(Http, gzip_chunked_flow) { auto str = rand_string('a', 'z', 1000000); auto parts = rand_split(make_chunked(gzencode(str).as_slice().str())); td::ChainBufferWriter input_writer; auto input = input_writer.extract_reader(); ByteFlowSource source(&input); HttpChunkedByteFlow chunked_flow; GzipByteFlow gzip_flow(Gzip::Decode); ByteFlowSink sink; source >> chunked_flow >> gzip_flow >> sink; for (auto &part : parts) { input_writer.append(part); source.wakeup(); } source.close_input(Status::OK()); ASSERT_TRUE(sink.is_ready()); LOG_IF(ERROR, sink.status().is_error()) << sink.status(); ASSERT_TRUE(sink.status().is_ok()); ASSERT_EQ(str, sink.result()->move_as_buffer_slice().as_slice().str()); }