2018-12-31 20:04:05 +01:00
|
|
|
//
|
2018-12-31 23:02:34 +01:00
|
|
|
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2019
|
2018-12-31 20:04:05 +01:00
|
|
|
//
|
|
|
|
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
|
|
|
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
|
|
|
//
|
|
|
|
#include "td/telegram/files/PartsManager.h"
|
|
|
|
|
|
|
|
#include "td/utils/format.h"
|
|
|
|
#include "td/utils/logging.h"
|
|
|
|
#include "td/utils/misc.h"
|
|
|
|
|
|
|
|
#include <limits>
|
|
|
|
#include <numeric>
|
|
|
|
|
|
|
|
namespace td {
|
|
|
|
/*** PartsManager ***/
|
2018-01-30 13:33:02 +01:00
|
|
|
|
|
|
|
namespace {
|
2018-04-26 22:41:34 +02:00
|
|
|
int64 calc_part_count(int64 size, int64 part_size) {
|
2018-01-30 13:33:02 +01:00
|
|
|
CHECK(part_size != 0);
|
|
|
|
return (size + part_size - 1) / part_size;
|
|
|
|
}
|
|
|
|
} // namespace
|
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
Status PartsManager::init_known_prefix(int64 known_prefix, size_t part_size, const std::vector<int> &ready_parts) {
|
|
|
|
known_prefix_flag_ = true;
|
|
|
|
known_prefix_size_ = known_prefix;
|
|
|
|
return init_no_size(part_size, ready_parts);
|
|
|
|
}
|
|
|
|
|
2018-11-11 12:38:04 +01:00
|
|
|
void PartsManager::set_streaming_offset(int64 offset) {
|
2018-12-26 17:11:15 +01:00
|
|
|
if (offset < 0 || need_check_ || (!unknown_size_flag_ && get_size() < offset)) {
|
2018-11-11 12:38:04 +01:00
|
|
|
streaming_offset_ = 0;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
auto part_i = offset / part_size_;
|
2018-12-26 17:11:15 +01:00
|
|
|
if (part_i >= MAX_PART_COUNT) {
|
2018-11-11 12:38:04 +01:00
|
|
|
streaming_offset_ = 0;
|
|
|
|
// error?
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
streaming_offset_ = offset;
|
|
|
|
first_streaming_empty_part_ = narrow_cast<int>(part_i);
|
2018-12-26 22:41:52 +01:00
|
|
|
if (part_count_ < first_streaming_empty_part_) {
|
|
|
|
part_count_ = first_streaming_empty_part_;
|
2018-11-11 12:38:04 +01:00
|
|
|
part_status_.resize(part_count_, PartStatus::Empty);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-12-31 20:04:05 +01:00
|
|
|
Status PartsManager::init_no_size(size_t part_size, const std::vector<int> &ready_parts) {
|
|
|
|
unknown_size_flag_ = true;
|
|
|
|
size_ = 0;
|
|
|
|
min_size_ = 0;
|
|
|
|
max_size_ = std::numeric_limits<int64>::max();
|
|
|
|
|
|
|
|
if (part_size != 0) {
|
|
|
|
part_size_ = part_size;
|
|
|
|
} else {
|
|
|
|
part_size_ = 32 * (1 << 10);
|
2018-04-26 22:41:34 +02:00
|
|
|
while (use_part_count_limit_ && calc_part_count(expected_size_, part_size_) > MAX_PART_COUNT) {
|
2018-01-30 13:33:02 +01:00
|
|
|
part_size_ *= 2;
|
|
|
|
CHECK(part_size_ <= MAX_PART_SIZE);
|
|
|
|
}
|
|
|
|
// just in case if expected_size_ is wrong
|
|
|
|
if (part_size_ < MAX_PART_SIZE) {
|
|
|
|
part_size_ *= 2;
|
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
2018-12-26 22:41:52 +01:00
|
|
|
part_count_ =
|
|
|
|
std::accumulate(ready_parts.begin(), ready_parts.end(), 0, [](auto a, auto b) { return max(a, b + 1); });
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
init_common(ready_parts);
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2018-01-30 13:33:02 +01:00
|
|
|
Status PartsManager::init(int64 size, int64 expected_size, bool is_size_final, size_t part_size,
|
|
|
|
const std::vector<int> &ready_parts, bool use_part_count_limit) {
|
|
|
|
CHECK(expected_size >= size);
|
2018-12-31 20:04:05 +01:00
|
|
|
use_part_count_limit_ = use_part_count_limit;
|
2018-01-30 13:33:02 +01:00
|
|
|
expected_size_ = expected_size;
|
|
|
|
if (expected_size_ > MAX_FILE_SIZE) {
|
|
|
|
return Status::Error("Too big file");
|
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
if (!is_size_final) {
|
|
|
|
return init_known_prefix(size, part_size, ready_parts);
|
|
|
|
}
|
|
|
|
if (size == 0) {
|
|
|
|
return init_no_size(part_size, ready_parts);
|
|
|
|
}
|
2019-02-12 17:17:20 +01:00
|
|
|
LOG_CHECK(size > 0) << tag("size", size);
|
2018-12-31 20:04:05 +01:00
|
|
|
unknown_size_flag_ = false;
|
|
|
|
size_ = size;
|
|
|
|
|
|
|
|
if (part_size != 0) {
|
|
|
|
part_size_ = part_size;
|
2018-04-26 22:41:34 +02:00
|
|
|
if (use_part_count_limit_ && calc_part_count(expected_size_, part_size_) > MAX_PART_COUNT) {
|
2018-01-29 18:46:06 +01:00
|
|
|
return Status::Error("FILE_UPLOAD_RESTART");
|
2018-01-29 15:41:40 +01:00
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
} else {
|
|
|
|
// TODO choose part_size_ depending on size
|
|
|
|
part_size_ = 64 * (1 << 10);
|
2018-04-26 22:41:34 +02:00
|
|
|
while (use_part_count_limit && calc_part_count(expected_size_, part_size_) > MAX_PART_COUNT) {
|
2018-12-31 20:04:05 +01:00
|
|
|
part_size_ *= 2;
|
|
|
|
CHECK(part_size_ <= MAX_PART_SIZE);
|
|
|
|
}
|
|
|
|
}
|
2019-02-12 17:17:20 +01:00
|
|
|
LOG_CHECK(1 <= size_) << tag("size_", size_);
|
|
|
|
LOG_CHECK(!use_part_count_limit || calc_part_count(expected_size_, part_size_) <= MAX_PART_COUNT)
|
2018-01-30 13:33:02 +01:00
|
|
|
<< tag("size_", size_) << tag("expected_size", size_) << tag("is_size_final", is_size_final)
|
|
|
|
<< tag("part_size_", part_size_) << tag("ready_parts", ready_parts.size());
|
2018-04-26 22:41:34 +02:00
|
|
|
part_count_ = static_cast<int>(calc_part_count(size_, part_size_));
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
init_common(ready_parts);
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
bool PartsManager::unchecked_ready() {
|
|
|
|
VLOG(files) << "Check readiness. Ready size is " << ready_size_ << ", total size is " << size_
|
|
|
|
<< ", unknown_size_flag = " << unknown_size_flag_ << ", need_check = " << need_check_
|
|
|
|
<< ", checked_prefix_size = " << checked_prefix_size_;
|
|
|
|
return !unknown_size_flag_ && ready_size_ == size_;
|
|
|
|
}
|
|
|
|
bool PartsManager::ready() {
|
|
|
|
return unchecked_ready() && (!need_check_ || checked_prefix_size_ == size_);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status PartsManager::finish() {
|
|
|
|
if (!ready()) {
|
|
|
|
return Status::Error("File transferring not finished");
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
void PartsManager::update_first_empty_part() {
|
|
|
|
while (first_empty_part_ < part_count_ && part_status_[first_empty_part_] != PartStatus::Empty) {
|
|
|
|
first_empty_part_++;
|
|
|
|
}
|
2018-11-11 12:38:04 +01:00
|
|
|
|
|
|
|
if (streaming_offset_ == 0) {
|
|
|
|
first_streaming_empty_part_ = first_empty_part_;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
while (first_streaming_empty_part_ < part_count_ && part_status_[first_streaming_empty_part_] != PartStatus::Empty) {
|
|
|
|
first_streaming_empty_part_++;
|
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void PartsManager::update_first_not_ready_part() {
|
|
|
|
while (first_not_ready_part_ < part_count_ && part_status_[first_not_ready_part_] == PartStatus::Ready) {
|
|
|
|
first_not_ready_part_++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-05-19 23:56:40 +02:00
|
|
|
int32 PartsManager::get_unchecked_ready_prefix_count() {
|
2018-12-31 20:04:05 +01:00
|
|
|
update_first_not_ready_part();
|
2018-05-19 23:56:40 +02:00
|
|
|
return first_not_ready_part_;
|
|
|
|
}
|
|
|
|
|
|
|
|
int32 PartsManager::get_ready_prefix_count() {
|
|
|
|
auto res = get_unchecked_ready_prefix_count();
|
2018-12-31 20:04:05 +01:00
|
|
|
if (need_check_) {
|
|
|
|
auto checked_parts = narrow_cast<int32>(checked_prefix_size_ / part_size_);
|
|
|
|
if (checked_parts < res) {
|
2018-05-19 23:56:40 +02:00
|
|
|
return checked_parts;
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return res;
|
|
|
|
}
|
2018-12-27 19:06:30 +01:00
|
|
|
string PartsManager::get_bitmask() {
|
2018-12-27 10:12:24 +01:00
|
|
|
int32 prefix_count = -1;
|
|
|
|
if (need_check_) {
|
|
|
|
prefix_count = narrow_cast<int32>(checked_prefix_size_ / part_size_);
|
|
|
|
}
|
|
|
|
return bitmask_.encode(prefix_count);
|
2018-11-11 12:38:04 +01:00
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
Result<Part> PartsManager::start_part() {
|
|
|
|
update_first_empty_part();
|
2018-11-11 12:38:04 +01:00
|
|
|
auto part_i = first_streaming_empty_part_;
|
2018-12-26 22:41:52 +01:00
|
|
|
if (known_prefix_flag_ && part_i >= static_cast<int>(known_prefix_size_ / part_size_)) {
|
|
|
|
return Status::Error(1, "Wait for prefix to be known");
|
|
|
|
}
|
2018-11-11 12:38:04 +01:00
|
|
|
if (part_i == part_count_) {
|
2018-12-31 20:04:05 +01:00
|
|
|
if (unknown_size_flag_) {
|
2018-12-26 22:41:52 +01:00
|
|
|
part_count_++;
|
|
|
|
if (part_count_ > MAX_PART_COUNT) {
|
|
|
|
return Status::Error("Too big file with unknown size");
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
2018-12-26 22:41:52 +01:00
|
|
|
part_status_.push_back(PartStatus::Empty);
|
2018-12-31 20:04:05 +01:00
|
|
|
} else {
|
2018-11-11 12:38:04 +01:00
|
|
|
if (first_empty_part_ < part_count_) {
|
|
|
|
part_i = first_empty_part_;
|
|
|
|
} else {
|
|
|
|
return get_empty_part();
|
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
}
|
2018-11-11 12:38:04 +01:00
|
|
|
CHECK(part_status_[part_i] == PartStatus::Empty);
|
|
|
|
on_part_start(part_i);
|
|
|
|
return get_part(part_i);
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
|
2018-01-30 13:33:02 +01:00
|
|
|
Status PartsManager::set_known_prefix(size_t size, bool is_ready) {
|
2018-12-27 07:45:19 +01:00
|
|
|
if (!known_prefix_flag_ || size < static_cast<size_t>(known_prefix_size_)) {
|
2018-12-26 22:41:52 +01:00
|
|
|
return Status::Error("FILE_UPLOAD_RESTART");
|
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
known_prefix_size_ = narrow_cast<int64>(size);
|
2018-02-12 11:37:54 +01:00
|
|
|
expected_size_ = max(known_prefix_size_, expected_size_);
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
CHECK(static_cast<size_t>(part_count_) == part_status_.size());
|
|
|
|
if (is_ready) {
|
2018-04-26 22:41:34 +02:00
|
|
|
part_count_ = static_cast<int>(calc_part_count(size, part_size_));
|
2018-12-31 20:04:05 +01:00
|
|
|
|
|
|
|
size_ = narrow_cast<int64>(size);
|
|
|
|
unknown_size_flag_ = false;
|
2018-12-26 22:41:52 +01:00
|
|
|
known_prefix_flag_ = false;
|
2018-12-31 20:04:05 +01:00
|
|
|
} else {
|
|
|
|
part_count_ = static_cast<int>(size / part_size_);
|
|
|
|
}
|
2019-02-12 17:17:20 +01:00
|
|
|
LOG_CHECK(static_cast<size_t>(part_count_) >= part_status_.size())
|
2018-01-28 21:38:55 +01:00
|
|
|
<< size << " " << is_ready << " " << part_count_ << " " << part_size_ << " " << part_status_.size();
|
2018-12-31 20:04:05 +01:00
|
|
|
part_status_.resize(part_count_);
|
2018-04-26 22:41:34 +02:00
|
|
|
if (use_part_count_limit_ && calc_part_count(expected_size_, part_size_) > MAX_PART_COUNT) {
|
2018-01-30 13:33:02 +01:00
|
|
|
return Status::Error("FILE_UPLOAD_RESTART");
|
|
|
|
}
|
|
|
|
return Status::OK();
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
Status PartsManager::on_part_ok(int32 id, size_t part_size, size_t actual_size) {
|
|
|
|
CHECK(part_status_[id] == PartStatus::Pending);
|
|
|
|
pending_count_--;
|
|
|
|
|
|
|
|
part_status_[id] = PartStatus::Ready;
|
2018-12-28 14:16:14 +01:00
|
|
|
if (actual_size != 0) {
|
|
|
|
bitmask_.set(id);
|
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
ready_size_ += narrow_cast<int64>(actual_size);
|
|
|
|
|
|
|
|
VLOG(files) << "Transferred part " << id << " of size " << part_size << ", total ready size = " << ready_size_;
|
|
|
|
|
|
|
|
int64 offset = narrow_cast<int64>(part_size_) * id;
|
|
|
|
int64 end_offset = offset + narrow_cast<int64>(actual_size);
|
|
|
|
if (unknown_size_flag_) {
|
|
|
|
CHECK(part_size == part_size_);
|
|
|
|
if (actual_size < part_size_) {
|
2018-02-12 11:37:54 +01:00
|
|
|
max_size_ = min(max_size_, end_offset);
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
if (actual_size) {
|
2018-02-12 11:37:54 +01:00
|
|
|
min_size_ = max(min_size_, end_offset);
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
if (min_size_ > max_size_) {
|
|
|
|
auto status = Status::Error(PSLICE() << "Failed to transfer file: " << tag("min_size", min_size_)
|
|
|
|
<< tag("max_size", max_size_));
|
|
|
|
LOG(ERROR) << status;
|
|
|
|
return status;
|
|
|
|
} else if (min_size_ == max_size_) {
|
|
|
|
unknown_size_flag_ = false;
|
|
|
|
size_ = min_size_;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if ((actual_size < part_size && offset < size_) || (offset >= size_ && actual_size > 0)) {
|
|
|
|
auto status = Status::Error(PSLICE() << "Failed to transfer file: " << tag("size", size_) << tag("offset", offset)
|
|
|
|
<< tag("transferred size", actual_size) << tag("part size", part_size));
|
|
|
|
LOG(ERROR) << status;
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
void PartsManager::on_part_failed(int32 id) {
|
|
|
|
CHECK(part_status_[id] == PartStatus::Pending);
|
|
|
|
pending_count_--;
|
|
|
|
part_status_[id] = PartStatus::Empty;
|
|
|
|
if (id < first_empty_part_) {
|
|
|
|
first_empty_part_ = id;
|
|
|
|
}
|
2018-11-11 12:38:04 +01:00
|
|
|
if (streaming_offset_ == 0) {
|
|
|
|
first_streaming_empty_part_ = id;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
auto part_i = narrow_cast<int>(streaming_offset_ / part_size_);
|
|
|
|
if (id >= part_i && id < first_streaming_empty_part_) {
|
|
|
|
first_streaming_empty_part_ = id;
|
|
|
|
}
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
int64 PartsManager::get_size() const {
|
|
|
|
CHECK(!unknown_size_flag_);
|
|
|
|
return size_;
|
|
|
|
}
|
|
|
|
int64 PartsManager::get_size_or_zero() const {
|
|
|
|
return size_;
|
|
|
|
}
|
|
|
|
|
|
|
|
int64 PartsManager::get_ready_size() const {
|
|
|
|
return ready_size_;
|
|
|
|
}
|
|
|
|
|
|
|
|
int64 PartsManager::get_expected_size() const {
|
|
|
|
if (unknown_size_flag_) {
|
2018-02-12 11:37:54 +01:00
|
|
|
return max(static_cast<int64>(512 * (1 << 10)), get_ready_size() * 2);
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
return get_size();
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t PartsManager::get_part_size() const {
|
|
|
|
return part_size_;
|
|
|
|
}
|
|
|
|
|
|
|
|
int32 PartsManager::get_part_count() const {
|
|
|
|
return part_count_;
|
|
|
|
}
|
|
|
|
|
|
|
|
void PartsManager::init_common(const std::vector<int> &ready_parts) {
|
|
|
|
ready_size_ = 0;
|
|
|
|
pending_count_ = 0;
|
|
|
|
first_empty_part_ = 0;
|
|
|
|
first_not_ready_part_ = 0;
|
|
|
|
part_status_ = vector<PartStatus>(part_count_);
|
|
|
|
|
|
|
|
for (auto i : ready_parts) {
|
2019-02-12 17:17:20 +01:00
|
|
|
LOG_CHECK(0 <= i && i < part_count_) << tag("i", i) << tag("part_count", part_count_);
|
2018-12-31 20:04:05 +01:00
|
|
|
part_status_[i] = PartStatus::Ready;
|
2018-11-11 12:38:04 +01:00
|
|
|
bitmask_.set(i);
|
2018-12-31 20:04:05 +01:00
|
|
|
auto part = get_part(i);
|
|
|
|
ready_size_ += narrow_cast<int64>(part.size);
|
|
|
|
}
|
|
|
|
|
|
|
|
checked_prefix_size_ = get_ready_prefix_count() * narrow_cast<int64>(part_size_);
|
|
|
|
}
|
|
|
|
|
|
|
|
void PartsManager::set_need_check() {
|
|
|
|
need_check_ = true;
|
2018-11-11 12:38:04 +01:00
|
|
|
set_streaming_offset(0);
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void PartsManager::set_checked_prefix_size(int64 size) {
|
|
|
|
checked_prefix_size_ = size;
|
|
|
|
}
|
|
|
|
|
|
|
|
int64 PartsManager::get_checked_prefix_size() const {
|
|
|
|
return checked_prefix_size_;
|
|
|
|
}
|
|
|
|
int64 PartsManager::get_unchecked_ready_prefix_size() {
|
|
|
|
update_first_not_ready_part();
|
|
|
|
auto count = first_not_ready_part_;
|
|
|
|
if (count == 0) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
auto part = get_part(count - 1);
|
|
|
|
int64 res = part.offset;
|
|
|
|
if (!unknown_size_flag_) {
|
|
|
|
res += narrow_cast<int64>(part.size);
|
2018-02-12 11:37:54 +01:00
|
|
|
res = min(res, get_size());
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
Part PartsManager::get_part(int id) {
|
|
|
|
int64 offset = narrow_cast<int64>(part_size_) * id;
|
|
|
|
int64 size = narrow_cast<int64>(part_size_);
|
|
|
|
if (!unknown_size_flag_) {
|
|
|
|
auto total_size = get_size();
|
|
|
|
if (total_size < offset) {
|
|
|
|
size = 0;
|
|
|
|
} else {
|
2018-02-12 11:37:54 +01:00
|
|
|
size = min(size, total_size - offset);
|
2018-12-31 20:04:05 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return Part{id, offset, static_cast<size_t>(size)};
|
|
|
|
}
|
|
|
|
|
|
|
|
Part PartsManager::get_empty_part() {
|
|
|
|
return Part{-1, 0, 0};
|
|
|
|
}
|
|
|
|
|
|
|
|
void PartsManager::on_part_start(int32 id) {
|
|
|
|
CHECK(part_status_[id] == PartStatus::Empty);
|
|
|
|
part_status_[id] = PartStatus::Pending;
|
|
|
|
pending_count_++;
|
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace td
|