2018-12-31 20:04:05 +01:00
|
|
|
//
|
2024-01-01 01:07:21 +01:00
|
|
|
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2024
|
2018-12-31 20:04:05 +01:00
|
|
|
//
|
|
|
|
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
|
|
|
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
|
|
|
//
|
|
|
|
#include "td/telegram/files/FileUploader.h"
|
|
|
|
|
2018-04-03 19:49:07 +02:00
|
|
|
#include "td/telegram/files/FileLoaderUtils.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
#include "td/telegram/Global.h"
|
2018-06-26 01:43:11 +02:00
|
|
|
#include "td/telegram/net/DcId.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
#include "td/telegram/net/NetQueryDispatcher.h"
|
2018-12-27 19:38:43 +01:00
|
|
|
#include "td/telegram/SecureStorage.h"
|
2021-10-27 16:32:09 +02:00
|
|
|
#include "td/telegram/telegram_api.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
#include "td/utils/buffer.h"
|
2019-02-12 22:26:36 +01:00
|
|
|
#include "td/utils/common.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
#include "td/utils/crypto.h"
|
|
|
|
#include "td/utils/format.h"
|
|
|
|
#include "td/utils/logging.h"
|
|
|
|
#include "td/utils/misc.h"
|
2018-04-03 19:49:07 +02:00
|
|
|
#include "td/utils/port/path.h"
|
2018-12-31 20:04:05 +01:00
|
|
|
#include "td/utils/Random.h"
|
|
|
|
#include "td/utils/ScopeGuard.h"
|
|
|
|
|
|
|
|
namespace td {
|
2018-12-26 17:11:15 +01:00
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
FileUploader::FileUploader(const LocalFileLocation &local, const RemoteFileLocation &remote, int64 expected_size,
|
|
|
|
const FileEncryptionKey &encryption_key, std::vector<int> bad_parts,
|
2018-09-27 03:19:03 +02:00
|
|
|
unique_ptr<Callback> callback)
|
2018-12-31 20:04:05 +01:00
|
|
|
: local_(local)
|
|
|
|
, remote_(remote)
|
|
|
|
, expected_size_(expected_size)
|
|
|
|
, encryption_key_(encryption_key)
|
|
|
|
, bad_parts_(std::move(bad_parts))
|
|
|
|
, callback_(std::move(callback)) {
|
2018-04-03 19:49:07 +02:00
|
|
|
if (encryption_key_.is_secret()) {
|
2018-12-31 20:04:05 +01:00
|
|
|
iv_ = encryption_key_.mutable_iv();
|
|
|
|
generate_iv_ = encryption_key_.iv_slice().str();
|
|
|
|
}
|
2021-08-26 20:20:55 +02:00
|
|
|
if (remote_.type() == RemoteFileLocation::Type::Partial && encryption_key_.is_secure() &&
|
|
|
|
remote_.partial().part_count_ != remote_.partial().ready_part_count_) {
|
2018-05-11 21:13:18 +02:00
|
|
|
remote_ = RemoteFileLocation{};
|
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
Result<FileLoader::FileInfo> FileUploader::init() {
|
2018-01-20 15:57:52 +01:00
|
|
|
if (remote_.type() == RemoteFileLocation::Type::Full) {
|
2018-12-31 20:04:05 +01:00
|
|
|
return Status::Error("File is already uploaded");
|
|
|
|
}
|
|
|
|
|
2019-01-01 18:13:44 +01:00
|
|
|
// file_size is needed only for partial local locations, but for uploaded partial files
|
|
|
|
// size is yet unknown or local location is full, so we can always pass 0 here
|
|
|
|
TRY_RESULT(prefix_info, on_update_local_location(local_, 0));
|
2018-12-31 20:04:05 +01:00
|
|
|
(void)prefix_info;
|
|
|
|
|
|
|
|
int offset = 0;
|
|
|
|
int part_size = 0;
|
2018-01-20 15:57:52 +01:00
|
|
|
if (remote_.type() == RemoteFileLocation::Type::Partial) {
|
2018-12-31 20:04:05 +01:00
|
|
|
const auto &partial = remote_.partial();
|
|
|
|
file_id_ = partial.file_id_;
|
|
|
|
part_size = partial.part_size_;
|
|
|
|
big_flag_ = partial.is_big_ != 0;
|
|
|
|
offset = partial.ready_part_count_;
|
|
|
|
} else {
|
|
|
|
file_id_ = Random::secure_int64();
|
2018-12-27 16:09:09 +01:00
|
|
|
big_flag_ = is_file_big(file_type_, expected_size_);
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<bool> ok(offset, true);
|
|
|
|
for (auto bad_id : bad_parts_) {
|
|
|
|
if (bad_id >= 0 && bad_id < offset) {
|
|
|
|
ok[bad_id] = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
std::vector<int> parts;
|
|
|
|
for (int i = 0; i < offset; i++) {
|
|
|
|
if (ok[i]) {
|
|
|
|
parts.push_back(i);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!ok.empty() && !ok[0]) {
|
|
|
|
parts.clear();
|
2023-07-05 00:13:46 +02:00
|
|
|
part_size = 0;
|
|
|
|
remote_ = RemoteFileLocation();
|
|
|
|
file_id_ = Random::secure_int64();
|
|
|
|
big_flag_ = is_file_big(file_type_, expected_size_);
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
2023-07-05 00:13:46 +02:00
|
|
|
|
|
|
|
LOG(DEBUG) << "Init file uploader for " << remote_ << " with offset = " << offset << " and part size = " << part_size;
|
2018-12-31 20:04:05 +01:00
|
|
|
FileInfo res;
|
|
|
|
res.size = local_size_;
|
|
|
|
res.is_size_final = local_is_ready_;
|
|
|
|
res.part_size = part_size;
|
|
|
|
res.ready_parts = std::move(parts);
|
2019-07-31 17:04:38 +02:00
|
|
|
res.is_upload = true;
|
2018-12-31 20:04:05 +01:00
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2018-12-26 17:11:15 +01:00
|
|
|
Result<FileLoader::PrefixInfo> FileUploader::on_update_local_location(const LocalFileLocation &location,
|
|
|
|
int64 file_size) {
|
2018-12-31 20:04:05 +01:00
|
|
|
SCOPE_EXIT {
|
|
|
|
try_release_fd();
|
|
|
|
};
|
2018-04-03 19:49:07 +02:00
|
|
|
|
|
|
|
if (encryption_key_.is_secure() && !fd_path_.empty()) {
|
|
|
|
return Status::Error("Can't change local location for Secure file");
|
|
|
|
}
|
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
string path;
|
2019-02-23 16:29:04 +01:00
|
|
|
int64 local_size = -1;
|
2018-12-31 20:04:05 +01:00
|
|
|
bool local_is_ready{false};
|
|
|
|
FileType file_type{FileType::Temp};
|
2018-04-03 19:49:07 +02:00
|
|
|
if (location.type() == LocalFileLocation::Type::Empty ||
|
|
|
|
(location.type() == LocalFileLocation::Type::Partial && encryption_key_.is_secure())) {
|
2018-12-31 20:04:05 +01:00
|
|
|
path = "";
|
|
|
|
local_size = 0;
|
|
|
|
local_is_ready = false;
|
|
|
|
file_type = FileType::Temp;
|
2018-01-20 15:57:52 +01:00
|
|
|
} else if (location.type() == LocalFileLocation::Type::Partial) {
|
2018-12-31 20:04:05 +01:00
|
|
|
path = location.partial().path_;
|
2018-11-11 12:38:04 +01:00
|
|
|
local_size = Bitmask(Bitmask::Decode{}, location.partial().ready_bitmask_)
|
2018-12-26 17:11:15 +01:00
|
|
|
.get_ready_prefix_size(0, location.partial().part_size_, file_size);
|
2018-12-31 20:04:05 +01:00
|
|
|
local_is_ready = false;
|
2018-01-20 15:57:52 +01:00
|
|
|
file_type = location.partial().file_type_;
|
2018-12-31 20:04:05 +01:00
|
|
|
} else {
|
|
|
|
path = location.full().path_;
|
2019-07-30 20:27:39 +02:00
|
|
|
if (path.empty()) {
|
2019-07-30 20:29:52 +02:00
|
|
|
return Status::Error("FullLocalFileLocation with empty path");
|
2019-07-30 20:27:39 +02:00
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
local_is_ready = true;
|
2018-01-20 15:57:52 +01:00
|
|
|
file_type = location.full().file_type_;
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
|
2019-01-01 18:35:08 +01:00
|
|
|
LOG(INFO) << "In FileUploader::on_update_local_location with " << location << ". Have path = \"" << path
|
|
|
|
<< "\", local_size = " << local_size << ", local_is_ready = " << local_is_ready
|
|
|
|
<< " and file type = " << file_type;
|
|
|
|
|
|
|
|
file_type_ = file_type;
|
|
|
|
|
2018-04-03 19:49:07 +02:00
|
|
|
bool is_temp = false;
|
2021-08-26 20:20:55 +02:00
|
|
|
if (encryption_key_.is_secure() && local_is_ready && remote_.type() == RemoteFileLocation::Type::Empty) {
|
2018-04-03 19:49:07 +02:00
|
|
|
TRY_RESULT(file_fd_path, open_temp_file(FileType::Temp));
|
|
|
|
file_fd_path.first.close();
|
|
|
|
auto new_path = std::move(file_fd_path.second);
|
|
|
|
TRY_RESULT(hash, secure_storage::encrypt_file(encryption_key_.secret(), path, new_path));
|
2018-04-23 17:05:02 +02:00
|
|
|
LOG(INFO) << "ENCRYPT " << path << " " << new_path;
|
2018-04-03 19:49:07 +02:00
|
|
|
callback_->on_hash(hash.as_slice().str());
|
|
|
|
path = new_path;
|
|
|
|
is_temp = true;
|
|
|
|
}
|
|
|
|
|
2019-08-20 09:42:21 +02:00
|
|
|
if (!path.empty() && (path != fd_path_ || fd_.empty())) {
|
2018-12-31 20:04:05 +01:00
|
|
|
auto res_fd = FileFd::open(path, FileFd::Read);
|
|
|
|
|
|
|
|
// Race: partial location could be already deleted. Just ignore such locations
|
|
|
|
if (res_fd.is_error()) {
|
2018-01-20 15:57:52 +01:00
|
|
|
if (location.type() == LocalFileLocation::Type::Partial) {
|
2019-01-01 18:35:08 +01:00
|
|
|
LOG(INFO) << "Ignore partial local location: " << res_fd.error();
|
2018-12-31 20:04:05 +01:00
|
|
|
PrefixInfo info;
|
|
|
|
info.size = local_size_;
|
|
|
|
info.is_ready = local_is_ready_;
|
|
|
|
return info;
|
|
|
|
}
|
|
|
|
return res_fd.move_as_error();
|
|
|
|
}
|
|
|
|
|
|
|
|
fd_.close();
|
|
|
|
fd_ = res_fd.move_as_ok();
|
|
|
|
fd_path_ = path;
|
2018-04-03 19:49:07 +02:00
|
|
|
is_temp_ = is_temp;
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
if (local_is_ready) {
|
|
|
|
CHECK(!fd_.empty());
|
2019-12-08 07:57:33 +01:00
|
|
|
TRY_RESULT_ASSIGN(local_size, fd_.get_size());
|
2019-02-23 20:34:50 +01:00
|
|
|
LOG(INFO) << "Set file local_size to " << local_size;
|
2018-12-31 20:04:05 +01:00
|
|
|
if (local_size == 0) {
|
|
|
|
return Status::Error("Can't upload empty file");
|
|
|
|
}
|
|
|
|
} else if (!fd_.empty()) {
|
2019-06-19 16:31:25 +02:00
|
|
|
TRY_RESULT(real_local_size, fd_.get_size());
|
2018-12-31 20:04:05 +01:00
|
|
|
if (real_local_size < local_size) {
|
|
|
|
LOG(ERROR) << tag("real_local_size", real_local_size) << " < " << tag("local_size", local_size);
|
|
|
|
PrefixInfo info;
|
|
|
|
info.size = local_size_;
|
|
|
|
info.is_ready = local_is_ready_;
|
|
|
|
return info;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
local_size_ = local_size;
|
2021-01-22 16:13:38 +01:00
|
|
|
if (expected_size_ < local_size_ && (expected_size_ != (10 << 20) || local_size_ >= (30 << 20))) {
|
2018-01-29 18:46:06 +01:00
|
|
|
expected_size_ = local_size_;
|
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
local_is_ready_ = local_is_ready;
|
|
|
|
|
|
|
|
PrefixInfo info;
|
|
|
|
info.size = local_size_;
|
|
|
|
info.is_ready = local_is_ready_;
|
|
|
|
return info;
|
|
|
|
}
|
|
|
|
|
|
|
|
Status FileUploader::on_ok(int64 size) {
|
|
|
|
fd_.close();
|
2018-04-03 19:49:07 +02:00
|
|
|
if (is_temp_) {
|
2018-04-23 17:05:02 +02:00
|
|
|
LOG(INFO) << "UNLINK " << fd_path_;
|
2018-04-03 19:49:07 +02:00
|
|
|
unlink(fd_path_).ignore();
|
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
return Status::OK();
|
|
|
|
}
|
2018-05-24 00:30:47 +02:00
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
void FileUploader::on_error(Status status) {
|
|
|
|
fd_.close();
|
2018-04-03 19:49:07 +02:00
|
|
|
if (is_temp_) {
|
2018-04-23 17:05:02 +02:00
|
|
|
LOG(INFO) << "UNLINK " << fd_path_;
|
2018-04-03 19:49:07 +02:00
|
|
|
unlink(fd_path_).ignore();
|
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
callback_->on_error(std::move(status));
|
|
|
|
}
|
|
|
|
|
|
|
|
Status FileUploader::generate_iv_map() {
|
2019-02-21 16:58:20 +01:00
|
|
|
LOG(INFO) << "Generate iv_map " << generate_offset_ << " " << local_size_;
|
2018-12-31 20:04:05 +01:00
|
|
|
auto part_size = get_part_size();
|
|
|
|
auto encryption_key = FileEncryptionKey(encryption_key_.key_slice(), generate_iv_);
|
|
|
|
BufferSlice bytes(part_size);
|
|
|
|
if (iv_map_.empty()) {
|
|
|
|
iv_map_.push_back(encryption_key.mutable_iv());
|
|
|
|
}
|
|
|
|
CHECK(!fd_.empty());
|
|
|
|
for (; generate_offset_ + static_cast<int64>(part_size) < local_size_;
|
|
|
|
generate_offset_ += static_cast<int64>(part_size)) {
|
2023-02-02 15:40:47 +01:00
|
|
|
TRY_RESULT(read_size, fd_.pread(bytes.as_mutable_slice(), generate_offset_));
|
2018-12-31 20:04:05 +01:00
|
|
|
if (read_size != part_size) {
|
|
|
|
return Status::Error("Failed to read file part (for iv_map)");
|
|
|
|
}
|
2023-02-02 15:40:47 +01:00
|
|
|
aes_ige_encrypt(as_slice(encryption_key.key()), as_mutable_slice(encryption_key.mutable_iv()), bytes.as_slice(),
|
|
|
|
bytes.as_mutable_slice());
|
2018-12-31 20:04:05 +01:00
|
|
|
iv_map_.push_back(encryption_key.mutable_iv());
|
|
|
|
}
|
|
|
|
generate_iv_ = encryption_key.iv_slice().str();
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
Status FileUploader::before_start_parts() {
|
|
|
|
auto status = acquire_fd();
|
|
|
|
if (status.is_error() && !local_is_ready_) {
|
2022-09-20 01:02:28 +02:00
|
|
|
return Status::Error(-1, "Can't open temporary file");
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
return status;
|
|
|
|
}
|
2021-08-26 20:44:29 +02:00
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
void FileUploader::after_start_parts() {
|
|
|
|
try_release_fd();
|
|
|
|
}
|
|
|
|
|
2020-01-08 18:15:49 +01:00
|
|
|
Result<std::pair<NetQueryPtr, bool>> FileUploader::start_part(Part part, int32 part_count, int64 streaming_offset) {
|
2018-12-31 20:04:05 +01:00
|
|
|
auto padded_size = part.size;
|
2018-04-03 19:49:07 +02:00
|
|
|
if (encryption_key_.is_secret()) {
|
2018-12-31 20:04:05 +01:00
|
|
|
padded_size = (padded_size + 15) & ~15;
|
|
|
|
}
|
|
|
|
BufferSlice bytes(padded_size);
|
2023-02-02 15:40:47 +01:00
|
|
|
TRY_RESULT(size, fd_.pread(bytes.as_mutable_slice().truncate(part.size), part.offset));
|
2018-04-03 19:49:07 +02:00
|
|
|
if (encryption_key_.is_secret()) {
|
2023-02-02 15:40:47 +01:00
|
|
|
Random::secure_bytes(bytes.as_mutable_slice().substr(part.size));
|
2018-12-31 20:04:05 +01:00
|
|
|
if (next_offset_ == part.offset) {
|
2023-02-02 15:40:47 +01:00
|
|
|
aes_ige_encrypt(as_slice(encryption_key_.key()), as_mutable_slice(iv_), bytes.as_slice(),
|
|
|
|
bytes.as_mutable_slice());
|
2018-12-31 20:04:05 +01:00
|
|
|
next_offset_ += static_cast<int64>(bytes.size());
|
|
|
|
} else {
|
|
|
|
if (part.id >= static_cast<int32>(iv_map_.size())) {
|
|
|
|
TRY_STATUS(generate_iv_map());
|
|
|
|
}
|
|
|
|
CHECK(part.id < static_cast<int32>(iv_map_.size()) && part.id >= 0);
|
|
|
|
auto iv = iv_map_[part.id];
|
2023-02-02 15:40:47 +01:00
|
|
|
aes_ige_encrypt(as_slice(encryption_key_.key()), as_mutable_slice(iv), bytes.as_slice(),
|
|
|
|
bytes.as_mutable_slice());
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (size != part.size) {
|
|
|
|
return Status::Error("Failed to read file part");
|
|
|
|
}
|
|
|
|
|
|
|
|
NetQueryPtr net_query;
|
|
|
|
if (big_flag_) {
|
2018-01-30 13:33:02 +01:00
|
|
|
auto query =
|
|
|
|
telegram_api::upload_saveBigFilePart(file_id_, part.id, local_is_ready_ ? part_count : -1, std::move(bytes));
|
2022-02-01 16:51:20 +01:00
|
|
|
net_query = G()->net_query_creator().create(query, {}, DcId::main(), NetQuery::Type::Upload);
|
2018-12-31 20:04:05 +01:00
|
|
|
} else {
|
|
|
|
auto query = telegram_api::upload_saveFilePart(file_id_, part.id, std::move(bytes));
|
2022-02-01 16:51:20 +01:00
|
|
|
net_query = G()->net_query_creator().create(query, {}, DcId::main(), NetQuery::Type::Upload);
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
net_query->file_type_ = narrow_cast<int32>(file_type_);
|
|
|
|
return std::make_pair(std::move(net_query), false);
|
|
|
|
}
|
|
|
|
|
|
|
|
Result<size_t> FileUploader::process_part(Part part, NetQueryPtr net_query) {
|
2018-06-27 18:24:42 +02:00
|
|
|
Result<bool> result = [&] {
|
|
|
|
if (big_flag_) {
|
2023-09-02 21:39:26 +02:00
|
|
|
return fetch_result<telegram_api::upload_saveBigFilePart>(std::move(net_query));
|
2018-06-27 18:24:42 +02:00
|
|
|
} else {
|
2023-09-02 21:39:26 +02:00
|
|
|
return fetch_result<telegram_api::upload_saveFilePart>(std::move(net_query));
|
2018-06-27 18:24:42 +02:00
|
|
|
}
|
|
|
|
}();
|
2018-12-31 20:04:05 +01:00
|
|
|
if (result.is_error()) {
|
|
|
|
return result.move_as_error();
|
|
|
|
}
|
|
|
|
if (!result.ok()) {
|
|
|
|
// TODO: it is possible
|
2021-06-25 20:21:17 +02:00
|
|
|
return Status::Error(500, "Internal Server Error during file upload");
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
return part.size;
|
|
|
|
}
|
|
|
|
|
2018-12-26 22:42:26 +01:00
|
|
|
void FileUploader::on_progress(Progress progress) {
|
|
|
|
callback_->on_partial_upload(PartialRemoteFileLocation{file_id_, progress.part_count, progress.part_size,
|
|
|
|
progress.ready_part_count, big_flag_},
|
|
|
|
progress.ready_size);
|
|
|
|
if (progress.is_ready) {
|
2018-12-31 20:04:05 +01:00
|
|
|
callback_->on_ok(file_type_,
|
2018-12-26 22:42:26 +01:00
|
|
|
PartialRemoteFileLocation{file_id_, progress.part_count, progress.part_size,
|
|
|
|
progress.ready_part_count, big_flag_},
|
2018-12-31 20:04:05 +01:00
|
|
|
local_size_);
|
|
|
|
}
|
|
|
|
}
|
2021-08-26 20:44:29 +02:00
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
void FileUploader::keep_fd_flag(bool keep_fd) {
|
|
|
|
keep_fd_ = keep_fd;
|
|
|
|
try_release_fd();
|
|
|
|
}
|
|
|
|
|
|
|
|
void FileUploader::try_release_fd() {
|
|
|
|
if (!keep_fd_ && !fd_.empty()) {
|
|
|
|
fd_.close();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Status FileUploader::acquire_fd() {
|
|
|
|
if (fd_.empty()) {
|
2019-12-08 07:57:33 +01:00
|
|
|
TRY_RESULT_ASSIGN(fd_, FileFd::open(fd_path_, FileFd::Read));
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace td
|