tdlight-team/tdlight
tdlight-team
/
tdlight
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

File streaming fixes. 

GitOrigin-RevId: 86c6d1394a31d6bb108a12e14f4d981133ece235
This commit is contained in:
levlam 2018-12-26 19:11:15 +03:00
parent 3b238f6fba
commit ac3fa705a2
28 changed files with 206 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
memprof/memprof.cpp
View File

@ -280,7 +280,7 @@ void *memalign(std::size_t aligment, std::size_t size) {
}
}
// c++14 guarantees than it is enough to override this two operators.
// c++14 guarantees that it is enough to override these two operators.
void *operator new(std::size_t count) {
return malloc_with_frame(count, get_backtrace());
}

16
td/generate/scheme/td_api.tl
View File

@ -126,8 +126,8 @@ temporaryPasswordState has_password:Bool valid_for:int32 = TemporaryPasswordStat
//@can_be_deleted True, if the file can be deleted
//@is_downloading_active True, if the file is currently being downloaded (or a local copy is being generated by some other means)
//@is_downloading_completed True, if the local copy is fully available
//@download_offset Download will be started from this offset. downloaded_prefix_size is calculated from this offset.
//@downloaded_prefix_size If is_downloading_completed is false, then only some prefix of the file is ready to be read. downloaded_prefix_size is the size of that prefix
//@download_offset Download will be started from this offset. downloaded_prefix_size is calculated from this offset
//@downloaded_prefix_size If is_downloading_completed is false, then only some prefix of the file starting from download_offset is ready to be read. downloaded_prefix_size is the size of that prefix
//@downloaded_size Total downloaded file bytes. Should be used only for calculating download progress. The actual file size may be bigger, and some parts of it may contain garbage
localFile path:string can_be_downloaded:Bool can_be_deleted:Bool is_downloading_active:Bool is_downloading_completed:Bool download_offset:int32 downloaded_prefix_size:int32 downloaded_size:int32 = LocalFile;
@ -2949,18 +2949,14 @@ setPinnedChats chat_ids:vector<int53> = Ok;
//@description Asynchronously downloads a file from the cloud. updateFile will be used to notify about the download progress and successful completion of the download. Returns file state just after the download has been started
//@file_id Identifier of the file to download
//@priority Priority of the download (1-32). The higher the priority, the earlier the file will be downloaded. If the priorities of two files are equal, then the last one for which downloadFile was called will be downloaded first
//@offset File will be downloaded starting from offset first. Supposed to be used for streaming.
//@offset File will be downloaded starting from that offset in bytes first. Supposed to be used for streaming
downloadFile file_id:int32 priority:int32 offset:int32 = File;
//@description Set offset for file downloading
//@file_id Identifier of file
//@offset File download offset
//@description Sets offset for file downloading @file_id Identifier of the file to change download offset @offset New file download offset
setFileDownloadOffset file_id:int32 offset:int32 = File;
//@description Get downloaded prefix from a given offset
//@file_id Identifier of file
//@offset Offset from which downloaded prefix is calculated
getFileDownloadedPrefix file_id:int32 offset:int32 = Count;
//@description Returns file downloaded prefix size from a given offset @file_id Identifier of the file @offset Offset from which downloaded prefix size should be calculated
getFileDownloadedPrefixSize file_id:int32 offset:int32 = Count;
//@description Stops the downloading of a file. If a file has already been downloaded, does nothing @file_id Identifier of a file to stop downloading @only_if_pending Pass true to stop downloading only if it hasn't been started, i.e. request hasn't been sent to server
cancelDownloadFile file_id:int32 only_if_pending:Bool = Ok;

BIN
td/generate/scheme/td_api.tlo
View File

Binary file not shown.

3
td/telegram/MessagesManager.cpp
View File

@ -9956,6 +9956,9 @@ void MessagesManager::try_restore_dialog_reply_markup(Dialog *d, const Message *
void MessagesManager::set_dialog_pinned_message_notification(Dialog *d, MessageId message_id) {
auto old_message_id = d->pinned_message_notification_message_id;
if (!old_message_id.is_valid() && !message_id.is_valid()) {
return;
}
CHECK(old_message_id != message_id);
VLOG(notifications) << "Change pinned message notification in " << d->dialog_id << " from " << old_message_id
<< " to " << message_id;

2
td/telegram/SecretChatActor.h
View File

@ -103,7 +103,7 @@ class SecretChatActor : public NetQueryCallback {
SecretChatActor(int32 id, unique_ptr<Context> context, bool can_be_empty);
// First query to new chat must be on of this two
// First query to new chat must be on of these two
void update_chat(telegram_api::object_ptr<telegram_api::EncryptedChat> chat);
void create_chat(int32 user_id, int64 user_access_hash, int32 random_id, Promise<SecretChatId> promise);
void cancel_chat(Promise<> promise);

2
td/telegram/StickersManager.cpp
View File

@ -3040,7 +3040,7 @@ Result<std::tuple<FileId, bool, bool>> StickersManager::prepare_input_file(
if (file_view.has_url()) {
is_url = true;
} else {
if (file_view.has_local_location() && file_view.local_size() > MAX_STICKER_FILE_SIZE) {
if (file_view.has_local_location() && file_view.expected_size() > MAX_STICKER_FILE_SIZE) {
return Status::Error(400, "File is too big");
}
is_local = true;

4
td/telegram/Td.cpp
View File

@ -4858,13 +4858,13 @@ void Td::on_request(uint64 id, const td_api::getFile &request) {
send_closure(actor_id(this), &Td::send_result, id, file_manager_->get_file_object(FileId(request.file_id_, 0)));
}
void Td::on_request(uint64 id, const td_api::getFileDownloadedPrefix &request) {
void Td::on_request(uint64 id, const td_api::getFileDownloadedPrefixSize &request) {
auto file_view = file_manager_->get_file_view(FileId(request.file_id_, 0));
if (file_view.empty()) {
return send_closure(actor_id(this), &Td::send_error, id, Status::Error(10, "Unknown file id"));
}
send_closure(actor_id(this), &Td::send_result, id,
td_api::make_object<td_api::count>(static_cast<int32>(file_view.downloaded_prefix(request.offset_))));
td_api::make_object<td_api::count>(narrow_cast<int32>(file_view.downloaded_prefix(request.offset_))));
}
void Td::on_request(uint64 id, td_api::getRemoteFile &request) {

2
td/telegram/Td.h
View File

@ -442,7 +442,7 @@ class Td final : public NetQueryCallback {
void on_request(uint64 id, const td_api::getFile &request);
void on_request(uint64 id, const td_api::getFileDownloadedPrefix &request);
void on_request(uint64 id, const td_api::getFileDownloadedPrefixSize &request);
void on_request(uint64 id, td_api::getRemoteFile &request);

40
td/telegram/cli.cpp
View File

@ -2251,11 +2251,11 @@ class CliClient final : public Actor {
send_request(make_tl_object<td_api::getChatMessageByDate>(as_chat_id(chat_id), to_integer<int32>(date)));
} else if (op == "gf" || op == "GetFile") {
send_request(make_tl_object<td_api::getFile>(as_file_id(args)));
} else if (op == "gfp" || op == "GetFileDownloadedPrefix") {
} else if (op == "gfdps") {
string file_id;
string offset;
std::tie(file_id, offset) = split(args);
send_request(make_tl_object<td_api::getFileDownloadedPrefix>(as_file_id(file_id), to_integer<int32>(offset)));
send_request(make_tl_object<td_api::getFileDownloadedPrefixSize>(as_file_id(file_id), to_integer<int32>(offset)));
} else if (op == "grf") {
send_request(make_tl_object<td_api::getRemoteFile>(args, nullptr));
} else if (op == "gmtf") {
@ -2277,36 +2277,34 @@ class CliClient final : public Actor {
as_location(latitude, longitude), to_integer<int32>(zoom), to_integer<int32>(width),
to_integer<int32>(height), to_integer<int32>(scale), as_chat_id(chat_id)));
} else if (op == "sfdo" || op == "SetDownloadFileOffset") {
string file_id_str;
string file_id;
string offset;
std::tie(file_id_str, offset) = split(args);
std::tie(file_id, offset) = split(args);
auto file_id = as_file_id(file_id_str);
send_request(make_tl_object<td_api::setFileDownloadOffset>(file_id, to_integer<int32>(offset)));
send_request(make_tl_object<td_api::setFileDownloadOffset>(as_file_id(file_id), to_integer<int32>(offset)));
} else if (op == "df" || op == "DownloadFile") {
string file_id_str;
string priority;
string offset;
std::tie(file_id_str, args) = split(args);
std::tie(priority, offset) = split(args);
if (priority.empty()) {
priority = "1";
}
auto file_id = as_file_id(file_id_str);
send_request(
make_tl_object<td_api::downloadFile>(file_id, to_integer<int32>(priority), to_integer<int32>(offset)));
} else if (op == "dff") {
string file_id;
string priority;
string offset;
std::tie(file_id, args) = split(args);
std::tie(priority, offset) = split(args);
std::tie(offset, priority) = split(args);
if (priority.empty()) {
priority = "1";
}
for (int i = 1; i <= as_file_id(file_id); i++) {
send_request(make_tl_object<td_api::downloadFile>(as_file_id(file_id), to_integer<int32>(priority),
to_integer<int32>(offset)));
} else if (op == "dff") {
string max_file_id;
string priority;
string offset;
std::tie(max_file_id, args) = split(args);
std::tie(offset, priority) = split(args);
if (priority.empty()) {
priority = "1";
}
for (int i = 1; i <= as_file_id(max_file_id); i++) {
send_request(make_tl_object<td_api::downloadFile>(i, to_integer<int32>(priority), to_integer<int32>(offset)));
}
} else if (op == "cdf") {

67
td/telegram/files/FileBitmask.cpp
View File

@ -5,80 +5,97 @@
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/telegram/files/FileBitmask.h"
#include "td/utils/common.h"
#include "td/utils/misc.h"
namespace td {
Bitmask::Bitmask(Decode, Slice data) : data_(zero_one_decode(data)) {
}
Bitmask::Bitmask(Ones, int64 count) : data_((count + 7) / 8, '\0') {
Bitmask::Bitmask(Ones, int64 count) : data_(narrow_cast<size_t>((count + 7) / 8), '\0') {
for (int64 i = 0; i < count; i++) {
set(i);
}
}
std::string Bitmask::encode() const {
// remove zeroes in the end to make encoding deteministic
// remove zeroes in the end to make encoding deterministic
td::Slice data(data_);
while (!data.empty() && data.back() == 0) {
while (!data.empty() && data.back() == '\0') {
data.remove_suffix(1);
}
return zero_one_encode(data_);
return zero_one_encode(data);
}
Bitmask::ReadySize Bitmask::get_ready_size(int64 offset, int64 part_size) const {
ReadySize res;
res.offset = offset;
int64 Bitmask::get_ready_prefix_size(int64 offset, int64 part_size, int64 file_size) const {
auto offset_part = offset / part_size;
auto ones = get_ready_parts(offset_part);
if (ones == 0) {
res.ready_size = 0;
} else {
res.ready_size = (offset_part + ones) * part_size - offset;
return 0;
}
CHECK(res.ready_size >= 0);
auto ready_parts_end = (offset_part + ones) * part_size;
if (file_size != 0 && ready_parts_end > file_size) {
ready_parts_end = file_size;
if (offset > file_size) {
offset = file_size;
}
}
auto res = ready_parts_end - offset;
CHECK(res >= 0);
return res;
}
int64 Bitmask::get_total_size(int64 part_size) const {
int64 res = 0;
for (int64 i = 0; i < size(); i++) {
res += get(i);
res += static_cast<int64>(get(i));
}
return res * part_size;
}
bool Bitmask::get(int64 offset) const {
if (offset < 0) {
bool Bitmask::get(int64 offset_part) const {
if (offset_part < 0) {
return 0;
}
if (offset / 8 >= narrow_cast<int64>(data_.size())) {
auto index = narrow_cast<size_t>(offset_part / 8);
if (index >= data_.size()) {
return 0;
}
return (data_[offset / 8] & (1 << (offset % 8))) != 0;
return (static_cast<uint8>(data_[index]) & (1 << static_cast<int>(offset_part % 8))) != 0;
}
int64 Bitmask::get_ready_parts(int64 offset) const {
int64 Bitmask::get_ready_parts(int64 offset_part) const {
int64 res = 0;
while (get(offset + res)) {
while (get(offset_part + res)) {
res++;
}
return res;
};
}
std::vector<int32> Bitmask::as_vector() const {
std::vector<int32> res;
for (int32 i = 0; i < narrow_cast<int32>(data_.size() * 8); i++) {
auto size = narrow_cast<int32>(data_.size() * 8);
for (int32 i = 0; i < size; i++) {
if (get(i)) {
res.push_back(i);
}
}
return res;
}
void Bitmask::set(int64 offset) {
auto need_size = narrow_cast<size_t>(offset / 8 + 1);
void Bitmask::set(int64 offset_part) {
CHECK(offset_part >= 0);
auto need_size = narrow_cast<size_t>(offset_part / 8 + 1);
if (need_size > data_.size()) {
data_.resize(need_size, 0);
data_.resize(need_size, '\0');
}
data_[need_size - 1] |= (1 << (offset % 8));
data_[need_size - 1] |= (1 << (offset_part % 8));
}
int64 Bitmask::size() const {
return data_.size() * 8;
return static_cast<int64>(data_.size() * 8);
}
} // namespace td

22
td/telegram/files/FileBitmask.h
View File

@ -5,34 +5,29 @@
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#pragma once
#include "td/utils/common.h"
#include "td/utils/Slice.h"
#include "td/utils/StringBuilder.h"
namespace td {
class Bitmask {
public:
struct ReadySize {
int64 offset{-1};
int64 ready_size{-1};
bool empty() const {
return offset == -1;
}
};
struct Decode {};
struct Ones {};
Bitmask() = default;
Bitmask(Decode, Slice data);
Bitmask(Ones, int64 count);
std::string encode() const;
ReadySize get_ready_size(int64 offset, int64 part_size) const;
int64 get_ready_prefix_size(int64 offset, int64 part_size, int64 file_size) const;
int64 get_total_size(int64 part_size) const;
bool get(int64 offset) const;
bool get(int64 offset_part) const;
int64 get_ready_parts(int64 offset) const;
int64 get_ready_parts(int64 offset_part) const;
std::vector<int32> as_vector() const;
void set(int64 offset);
void set(int64 offset_part);
int64 size() const;
private:
@ -40,11 +35,10 @@ class Bitmask {
};
inline StringBuilder &operator<<(StringBuilder &sb, const Bitmask &mask) {
std::string res;
for (int64 i = 0; i < mask.size(); i++) {
res += mask.get(i) ? "1" : "0";
sb << (mask.get(i) ? '1' : '0');
}
return sb << res;
return sb;
}
} // namespace td

17
td/telegram/files/FileDownloader.cpp
View File

@ -104,10 +104,11 @@ Result<FileLoader::FileInfo> FileDownloader::init() {
res.ready_parts = bitmask.as_vector();
res.use_part_count_limit = false;
res.only_check = only_check_;
res.need_delay = !is_small_ && (remote_.file_type_ == FileType::VideoNote ||
remote_.file_type_ == FileType::VoiceNote || remote_.file_type_ == FileType::Audio ||
remote_.file_type_ == FileType::Video || remote_.file_type_ == FileType::Animation ||
(remote_.file_type_ == FileType::Encrypted && size_ > (1 << 20)));
res.need_delay =
!is_small_ && (remote_.file_type_ == FileType::VideoNote || remote_.file_type_ == FileType::Document ||
remote_.file_type_ == FileType::VoiceNote || remote_.file_type_ == FileType::Audio ||
remote_.file_type_ == FileType::Video || remote_.file_type_ == FileType::Animation ||
(remote_.file_type_ == FileType::Encrypted && size_ > (1 << 20)));
res.offset = offset_;
return res;
}
@ -304,7 +305,6 @@ Result<size_t> FileDownloader::process_part(Part part, NetQueryPtr net_query) {
if (encryption_key_.is_secret()) {
padded_size = (part.size + 15) & ~15;
}
LOG(INFO) << "Got " << bytes.size() << " bytes, padded_size = " << padded_size << " for " << path_;
if (bytes.size() > padded_size) {
return Status::Error("Part size is more than requested");
}
@ -338,6 +338,7 @@ Result<size_t> FileDownloader::process_part(Part part, NetQueryPtr net_query) {
auto slice = bytes.as_slice().truncate(part.size);
TRY_STATUS(acquire_fd());
LOG(INFO) << "Got " << slice.size() << " bytes at " << part.offset << " for \"" << path_ << '"';
TRY_RESULT(written, fd_.pwrite(slice, part.offset));
// may write less than part.size, when size of downloadable file is unknown
if (written != slice.size()) {
@ -345,7 +346,7 @@ Result<size_t> FileDownloader::process_part(Part part, NetQueryPtr net_query) {
}
return written;
}
void FileDownloader::on_progress(int32 part_count, int32 part_size, int32 ready_part_count, string ready_bitmask,
void FileDownloader::on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask,
bool is_ready, int64 ready_size) {
if (is_ready) {
// do not send partial location. will lead to wrong local_size
@ -355,7 +356,7 @@ void FileDownloader::on_progress(int32 part_count, int32 part_size, int32 ready_
return;
}
if (encryption_key_.empty() || encryption_key_.is_secure()) {
callback_->on_partial_download(PartialLocalFileLocation{remote_.file_type_, path_, part_size, "", ready_bitmask},
callback_->on_partial_download(PartialLocalFileLocation{remote_.file_type_, part_size, path_, "", ready_bitmask},
ready_size);
} else if (encryption_key_.is_secret()) {
UInt256 iv;
@ -365,7 +366,7 @@ void FileDownloader::on_progress(int32 part_count, int32 part_size, int32 ready_
LOG(FATAL) << tag("ready_part_count", ready_part_count) << tag("next_part", next_part_);
}
callback_->on_partial_download(
PartialLocalFileLocation{remote_.file_type_, path_, part_size, as_slice(iv).str(), ready_bitmask}, ready_size);
PartialLocalFileLocation{remote_.file_type_, part_size, path_, as_slice(iv).str(), ready_bitmask}, ready_size);
} else {
UNREACHABLE();
}

4
td/telegram/files/FileDownloader.h
View File

@ -85,8 +85,8 @@ class FileDownloader : public FileLoader {
Result<bool> should_restart_part(Part part, NetQueryPtr &net_query) override TD_WARN_UNUSED_RESULT;
Result<std::pair<NetQueryPtr, bool>> start_part(Part part, int32 part_count) override TD_WARN_UNUSED_RESULT;
Result<size_t> process_part(Part part, NetQueryPtr net_query) override TD_WARN_UNUSED_RESULT;
void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, string ready_bitmask, bool is_ready,
int64 ready_size) override;
void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask,
bool is_ready, int64 ready_size) override;
FileLoader::Callback *get_callback() override;
Status process_check_query(NetQueryPtr net_query) override;
Result<CheckInfo> check_loop(int64 checked_prefix_size, int64 ready_prefix_size, bool is_ready) override;

4
td/telegram/files/FileGenerateManager.cpp
View File

@ -303,8 +303,8 @@ class FileExternalGenerateActor : public FileGenerateActor {
if (local_prefix_size < 0) {
return Status::Error(1, "Invalid local prefix size");
}
callback_->on_partial_generate(PartialLocalFileLocation{generate_location_.file_type_, path_, local_prefix_size,
Bitmask(Bitmask::Ones{}, 1).encode(), ""},
callback_->on_partial_generate(PartialLocalFileLocation{generate_location_.file_type_, local_prefix_size, path_, "",
Bitmask(Bitmask::Ones{}, 1).encode()},
expected_size);
return Status::OK();
}

4
td/telegram/files/FileLoader.cpp
View File

@ -20,6 +20,7 @@
#include <utility>
namespace td {
void FileLoader::set_resource_manager(ActorShared<ResourceManager> resource_manager) {
resource_manager_ = std::move(resource_manager);
send_closure(resource_manager_, &ResourceManager::update_resources, resource_state_);
@ -47,7 +48,7 @@ void FileLoader::hangup() {
}
void FileLoader::update_local_file_location(const LocalFileLocation &local) {
auto r_prefix_info = on_update_local_location(local);
auto r_prefix_info = on_update_local_location(local, parts_manager_.get_size_or_zero());
if (r_prefix_info.is_error()) {
on_error(r_prefix_info.move_as_error());
stop_flag_ = true;
@ -302,4 +303,5 @@ void FileLoader::on_progress_impl(size_t size) {
parts_manager_.get_ready_prefix_count(), parts_manager_.get_bitmask(), parts_manager_.ready(),
parts_manager_.get_ready_size());
}
} // namespace td

8
td/telegram/files/FileLoader.h
View File

@ -24,6 +24,7 @@
#include <utility>
namespace td {
class FileLoader : public FileLoaderActor {
public:
class Callback {
@ -69,10 +70,11 @@ class FileLoader : public FileLoaderActor {
virtual void after_start_parts() {
}
virtual Result<size_t> process_part(Part part, NetQueryPtr net_query) TD_WARN_UNUSED_RESULT = 0;
virtual void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, std::string ready_bitmask,
virtual void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask,
bool is_ready, int64 ready_size) = 0;
virtual Callback *get_callback() = 0;
virtual Result<PrefixInfo> on_update_local_location(const LocalFileLocation &location) TD_WARN_UNUSED_RESULT {
virtual Result<PrefixInfo> on_update_local_location(const LocalFileLocation &location,
int64 file_size) TD_WARN_UNUSED_RESULT {
return Status::Error("unsupported");
}
virtual Result<bool> should_restart_part(Part part, NetQueryPtr &net_query) TD_WARN_UNUSED_RESULT {
@ -103,7 +105,6 @@ class FileLoader : public FileLoaderActor {
PartsManager parts_manager_;
uint64 blocking_id_{0};
std::map<uint64, std::pair<Part, ActorShared<>>> part_map_;
// std::map<uint64, std::pair<Part, NetQueryRef>> part_map_;
bool ordered_flag_ = false;
OrderedEventsProcessor<std::pair<Part, NetQueryPtr>> ordered_parts_;
ActorOwn<DelayDispatcher> delay_dispatcher_;
@ -127,4 +128,5 @@ class FileLoader : public FileLoaderActor {
void on_common_query(NetQueryPtr query);
Status try_on_part_query(Part part, NetQueryPtr query);
};
} // namespace td

2
td/telegram/files/FileLoaderActor.h
View File

@ -22,7 +22,7 @@ class FileLoaderActor : public NetQueryCallback {
virtual void update_priority(int8 priority) = 0;
virtual void update_resources(const ResourceState &other) = 0;
// TODO: existence of this two functions is a dirty hack. Refactoring is highly appreciated
// TODO: existence of these two functions is a dirty hack. Refactoring is highly appreciated
virtual void update_local_file_location(const LocalFileLocation &local) {
}
virtual void update_download_offset(int64 offset) {

6
td/telegram/files/FileLocation.h
View File

@ -6,11 +6,11 @@
//
#pragma once
#include "td/telegram/files/FileBitmask.h"
#include "td/telegram/td_api.h"
#include "td/telegram/telegram_api.h"
#include "td/telegram/DialogId.h"
#include "td/telegram/files/FileBitmask.h"
#include "td/telegram/net/DcId.h"
#include "td/telegram/SecureStorage.h"
@ -908,8 +908,8 @@ inline bool operator!=(const EmptyLocalFileLocation &lhs, const EmptyLocalFileLo
struct PartialLocalFileLocation {
FileType file_type_;
string path_;
int32 part_size_;
string path_;
string iv_;
string ready_bitmask_;
@ -939,6 +939,8 @@ struct PartialLocalFileLocation {
if (deprecated_ready_part_count == -1) {
parse(ready_bitmask_, parser);
} else {
CHECK(0 <= deprecated_ready_part_count);
CHECK(deprecated_ready_part_count <= (1 << 22));
ready_bitmask_ = Bitmask(Bitmask::Ones{}, deprecated_ready_part_count).encode();
}
}

82
td/telegram/files/FileManager.cpp
View File

@ -66,17 +66,16 @@ FileNodePtr::operator bool() const {
return file_manager_ != nullptr && get_unsafe() != nullptr;
}
void FileNode::recalc_ready_prefix_size(Bitmask::ReadySize ready_prefix_size) {
void FileNode::recalc_ready_prefix_size(int64 prefix_offset, int64 ready_prefix_size) {
if (local_.type() != LocalFileLocation::Type::Partial) {
return;
}
int64 new_local_ready_prefix_size;
if (download_offset_ == ready_prefix_size.offset) {
new_local_ready_prefix_size = ready_prefix_size.ready_size;
if (download_offset_ == prefix_offset) {
new_local_ready_prefix_size = ready_prefix_size;
} else {
new_local_ready_prefix_size = Bitmask(Bitmask::Decode{}, local_.partial().ready_bitmask_)
.get_ready_size(download_offset_, local_.partial().part_size_)
.ready_size;
.get_ready_prefix_size(download_offset_, local_.partial().part_size_, size_);
}
if (new_local_ready_prefix_size != local_ready_prefix_size_) {
local_ready_prefix_size_ = new_local_ready_prefix_size;
@ -89,7 +88,7 @@ void FileNode::init_ready_size() {
return;
}
auto bitmask = Bitmask(Bitmask::Decode{}, local_.partial().ready_bitmask_);
local_ready_prefix_size_ = bitmask.get_ready_size(0, local_.partial().part_size_).ready_size;
local_ready_prefix_size_ = bitmask.get_ready_prefix_size(0, local_.partial().part_size_, size_);
local_ready_size_ = bitmask.get_total_size(local_.partial().part_size_);
}
@ -102,22 +101,27 @@ void FileNode::set_download_offset(int64 download_offset) {
}
download_offset_ = download_offset;
is_download_offset_dirty_ = true;
recalc_ready_prefix_size({});
recalc_ready_prefix_size(-1, -1);
on_info_changed();
}
void FileNode::set_local_location(const LocalFileLocation &local, int64 ready_size,
Bitmask::ReadySize ready_prefix_size) {
void FileNode::drop_local_location() {
set_local_location(LocalFileLocation(), 0, -1, -1);
}
void FileNode::set_local_location(const LocalFileLocation &local, int64 ready_size, int64 prefix_offset,
int64 ready_prefix_size) {
if (local_ready_size_ != ready_size) {
VLOG(update_file) << "File " << main_file_id_ << " has changed local ready size from " << local_ready_size_
<< " to " << ready_size;
local_ready_size_ = ready_size;
VLOG(update_file) << "File " << main_file_id_ << " has changed local ready size";
on_info_changed();
}
if (local_ != local) {
VLOG(update_file) << "File " << main_file_id_ << " has changed local location";
local_ = local;
recalc_ready_prefix_size(ready_prefix_size);
recalc_ready_prefix_size(prefix_offset, ready_prefix_size);
on_changed();
}
@ -357,22 +361,25 @@ int64 FileView::downloaded_prefix(int64 offset) const {
}
return 0;
case LocalFileLocation::Type::Partial:
if (is_encrypted_secure()) {
// File is not decrypted and verified yet
return 0;
}
return Bitmask(Bitmask::Decode{}, node_->local_.partial().ready_bitmask_)
.get_ready_size(offset, node_->local_.partial().part_size_)
.ready_size;
.get_ready_prefix_size(offset, node_->local_.partial().part_size_, node_->size_);
default:
UNREACHABLE();
return 0;
}
}
int64 FileView::local_size() const {
int64 FileView::local_prefix_size() const {
switch (node_->local_.type()) {
case LocalFileLocation::Type::Full:
return node_->size_;
return node_->download_offset_ <= node_->size_ ? node_->size_ - node_->download_offset_ : 0;
case LocalFileLocation::Type::Partial: {
if (is_encrypted_secure()) {
// File is not decrypted yet
// File is not decrypted and verified yet
return 0;
}
return node_->local_ready_prefix_size_;
@ -660,7 +667,7 @@ Status FileManager::check_local_location(FileNodePtr node) {
status = check_partial_local_location(node->local_.partial());
}
if (status.is_error()) {
node->set_local_location(LocalFileLocation(), 0);
node->drop_local_location();
try_flush_node(node);
}
return status;
@ -719,7 +726,7 @@ void FileManager::on_file_unlink(const FullLocalFileLocation &location) {
auto file_id = it->second;
auto file_node = get_sync_file_node(file_id);
CHECK(file_node);
file_node->set_local_location(LocalFileLocation(), 0);
file_node->drop_local_location();
try_flush_node_info(file_node);
}
@ -1114,15 +1121,17 @@ Result<FileId> FileManager::merge(FileId x_file_id, FileId y_file_id, bool no_sy
<< ", main_file_id_i = " << main_file_id_i;
if (local_i == other_node_i) {
cancel_download(node);
node->set_local_location(other_node->local_, other_node->local_ready_size_);
node->set_download_offset(other_node->download_offset_);
node->set_local_location(other_node->local_, other_node->local_ready_size_, other_node->download_offset_,
other_node->local_ready_prefix_size_);
node->download_id_ = other_node->download_id_;
node->is_download_started_ |= other_node->is_download_started_;
node->set_download_priority(other_node->download_priority_);
node->set_download_offset(other_node->download_offset_);
other_node->download_id_ = 0;
other_node->is_download_started_ = false;
other_node->download_priority_ = 0;
other_node->download_offset_ = 0;
other_node->local_ready_prefix_size_ = 0;
//cancel_generate(node);
//node->set_generate_location(std::move(other_node->generate_));
@ -1500,7 +1509,7 @@ void FileManager::delete_file(FileId file_id, Promise<Unit> promise, const char
unlink(file_view.local_location().path_).ignore();
context_->on_new_file(-file_view.size());
node->set_local_location(LocalFileLocation(), 0);
node->drop_local_location();
try_flush_node(node);
}
} else {
@ -1510,7 +1519,7 @@ void FileManager::delete_file(FileId file_id, Promise<Unit> promise, const char
if (node->local_.type() == LocalFileLocation::Type::Partial) {
LOG(INFO) << "Unlink partial file " << file_id << " at " << node->local_.partial().path_;
unlink(node->local_.partial().path_).ignore();
node->set_local_location(LocalFileLocation(), 0);
node->drop_local_location();
try_flush_node(node);
}
}
@ -1566,6 +1575,10 @@ void FileManager::download(FileId file_id, std::shared_ptr<DownloadCallback> cal
}
LOG(INFO) << "Change download priority of file " << file_id << " to " << new_priority;
if (file_view.is_encrypted_any()) {
// TODO: we need to set it always and just download from the beginning
offset = 0;
}
node->set_download_offset(offset);
auto *file_info = get_file_id_info(file_id);
CHECK(new_priority == 0 || callback);
@ -1585,7 +1598,7 @@ void FileManager::download_set_offset(FileId file_id, int64 offset) {
LOG(INFO) << "File " << file_id << " not found";
return;
}
if (FileView(file_node).is_encrypted()) {
if (FileView(file_node).is_encrypted_any()) {
offset = 0;
}
file_node->set_download_offset(offset);
@ -1622,10 +1635,6 @@ void FileManager::run_download(FileNodePtr node) {
}
return;
}
if (file_view.is_encrypted()) {
node->set_download_offset(0);
}
auto offset = node->download_offset_;
bool need_update_offset = node->is_download_offset_dirty_;
node->is_download_offset_dirty_ = false;
@ -1633,7 +1642,8 @@ void FileManager::run_download(FileNodePtr node) {
CHECK(node->download_id_ != 0);
send_closure(file_load_manager_, &FileLoadManager::update_priority, node->download_id_, priority);
if (need_update_offset) {
send_closure(file_load_manager_, &FileLoadManager::update_download_offset, node->download_id_, offset);
send_closure(file_load_manager_, &FileLoadManager::update_download_offset, node->download_id_,
node->download_offset_);
}
return;
}
@ -2030,7 +2040,7 @@ tl_object_ptr<td_api::file> FileManager::get_file_object(FileId file_id, bool wi
int32 size = narrow_cast<int32>(file_view.size());
int32 expected_size = narrow_cast<int32>(file_view.expected_size());
int32 download_offset = narrow_cast<int32>(file_view.download_offset());
int32 local_size = narrow_cast<int32>(file_view.local_size());
int32 local_prefix_size = narrow_cast<int32>(file_view.local_prefix_size());
int32 local_total_size = narrow_cast<int32>(file_view.local_total_size());
int32 remote_size = narrow_cast<int32>(file_view.remote_size());
string path = file_view.path();
@ -2053,7 +2063,7 @@ tl_object_ptr<td_api::file> FileManager::get_file_object(FileId file_id, bool wi
result_file_id.get(), size, expected_size,
td_api::make_object<td_api::localFile>(std::move(path), can_be_downloaded, can_be_deleted,
file_view.is_downloading(), file_view.has_local_location(),
download_offset, local_size, local_total_size),
download_offset, local_prefix_size, local_total_size),
td_api::make_object<td_api::remoteFile>(std::move(persistent_file_id), file_view.is_uploading(),
is_uploading_completed, remote_size));
}
@ -2272,9 +2282,7 @@ void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLo
auto file_id = query->file_id_;
auto file_node = get_file_node(file_id);
LOG(DEBUG) << "Receive on_parial_download for file " << file_id;
LOG(ERROR) << "Receive on_parital_generate for file " << file_id << ": " << partial_local.path_ << " "
<< Bitmask(Bitmask::Decode{}, partial_local.ready_bitmask_);
LOG(DEBUG) << "Receive on_partial_download for file " << file_id;
if (!file_node) {
return;
}
@ -2282,7 +2290,7 @@ void FileManager::on_partial_download(QueryId query_id, const PartialLocalFileLo
return;
}
file_node->set_local_location(LocalFileLocation(partial_local), ready_size);
file_node->set_local_location(LocalFileLocation(partial_local), ready_size, -1, -1 /* TODO */);
try_flush_node(file_node);
}
@ -2445,7 +2453,7 @@ void FileManager::on_partial_generate(QueryId query_id, const PartialLocalFileLo
auto file_id = query->file_id_;
auto file_node = get_file_node(file_id);
LOG(DEBUG) << "Receive on_parital_generate for file " << file_id << ": " << partial_local.path_ << " "
LOG(DEBUG) << "Receive on_partial_generate for file " << file_id << ": " << partial_local.path_ << " "
<< Bitmask(Bitmask::Decode{}, partial_local.ready_bitmask_);
if (!file_node) {
return;
@ -2453,7 +2461,7 @@ void FileManager::on_partial_generate(QueryId query_id, const PartialLocalFileLo
if (file_node->generate_id_ != query_id) {
return;
}
file_node->set_local_location(LocalFileLocation(partial_local), 0);
file_node->set_local_location(LocalFileLocation(partial_local), 0, -1, -1 /* TODO */);
// TODO check for size and local_size, abort generation if needed
if (expected_size > 0) {
file_node->set_expected_size(expected_size);
@ -2551,7 +2559,7 @@ void FileManager::on_error_impl(FileNodePtr node, FileManager::Query::Type type,
if (node->local_.type() == LocalFileLocation::Type::Partial && status.message() != "FILE_UPLOAD_RESTART") {
LOG(INFO) << "Unlink file " << node->local_.partial().path_;
unlink(node->local_.partial().path_).ignore();
node->set_local_location(LocalFileLocation(), 0);
node->drop_local_location();
}
if (node->remote_.type() == RemoteFileLocation::Type::Partial) {
node->set_remote_location(RemoteFileLocation(), FileLocationSource::None, 0);

16
td/telegram/files/FileManager.h
View File

@ -58,7 +58,9 @@ class FileNode {
, main_file_id_priority_(main_file_id_priority) {
init_ready_size();
}
void set_local_location(const LocalFileLocation &local, int64 ready_size, Bitmask::ReadySize ready_prefix_size = {});
void drop_local_location();
void set_local_location(const LocalFileLocation &local, int64 ready_size, int64 prefix_offset,
int64 ready_prefix_size);
void set_remote_location(const RemoteFileLocation &remote, FileLocationSource source, int64 ready_size);
void set_generate_location(unique_ptr<FullGenerateFileLocation> &&generate);
void set_size(int64 size);
@ -73,7 +75,6 @@ class FileNode {
void set_generate_priority(int8 download_priority, int8 upload_priority);
void set_download_offset(int64 download_offset);
void recalc_ready_prefix_size(Bitmask::ReadySize ready_prefix_size);
void on_changed();
void on_info_changed();
@ -95,7 +96,7 @@ class FileNode {
FileLoadManager::QueryId upload_id_ = 0;
int64 local_ready_size_ = 0;
int64 download_offset_ = 0;
int64 local_ready_prefix_size_ = 0;
int64 local_ready_prefix_size_ = 0; // PartialLocal only
RemoteFileLocation remote_;
FileLoadManager::QueryId download_id_ = 0;
@ -124,10 +125,11 @@ class FileNode {
int8 generate_upload_priority_ = 0;
int8 main_file_id_priority_ = 0;
bool is_download_offset_dirty_ = false;
FileLocationSource remote_source_ = FileLocationSource::FromUser;
bool is_download_offset_dirty_ = false;
bool get_by_hash_ = false;
bool can_search_locally_{true};
@ -138,9 +140,10 @@ class FileNode {
bool pmc_changed_flag_{false};
bool info_changed_flag_{false};
bool offset_changed_flags_{false};
void init_ready_size();
void recalc_ready_prefix_size(int64 prefix_offset, int64 ready_prefix_size);
};
class FileManager;
@ -221,7 +224,7 @@ class FileView {
bool is_downloading() const;
int64 download_offset() const;
int64 downloaded_prefix(int64 offset) const;
int64 local_size() const;
int64 local_prefix_size() const;
int64 local_total_size() const;
bool is_uploading() const;
int64 remote_size() const;
@ -515,4 +518,5 @@ class FileManager : public FileLoadManager::Callback {
friend class FileNodePtr;
};
} // namespace td

11
td/telegram/files/FileUploader.cpp
View File

@ -24,6 +24,7 @@
#include "td/utils/ScopeGuard.h"
namespace td {
FileUploader::FileUploader(const LocalFileLocation &local, const RemoteFileLocation &remote, int64 expected_size,
const FileEncryptionKey &encryption_key, std::vector<int> bad_parts,
unique_ptr<Callback> callback)
@ -47,7 +48,7 @@ Result<FileLoader::FileInfo> FileUploader::init() {
return Status::Error("File is already uploaded");
}
TRY_RESULT(prefix_info, on_update_local_location(local_));
TRY_RESULT(prefix_info, on_update_local_location(local_, 0 /* TODO(now) */));
(void)prefix_info;
int offset = 0;
@ -86,7 +87,8 @@ Result<FileLoader::FileInfo> FileUploader::init() {
return res;
}
Result<FileLoader::PrefixInfo> FileUploader::on_update_local_location(const LocalFileLocation &location) {
Result<FileLoader::PrefixInfo> FileUploader::on_update_local_location(const LocalFileLocation &location,
int64 file_size) {
SCOPE_EXIT {
try_release_fd();
};
@ -108,8 +110,7 @@ Result<FileLoader::PrefixInfo> FileUploader::on_update_local_location(const Loca
} else if (location.type() == LocalFileLocation::Type::Partial) {
path = location.partial().path_;
local_size = Bitmask(Bitmask::Decode{}, location.partial().ready_bitmask_)
.get_ready_size(0, location.partial().part_size_)
.ready_size;
.get_ready_prefix_size(0, location.partial().part_size_, file_size);
local_is_ready = false;
file_type = location.partial().file_type_;
} else {
@ -294,7 +295,7 @@ Result<size_t> FileUploader::process_part(Part part, NetQueryPtr net_query) {
return part.size;
}
void FileUploader::on_progress(int32 part_count, int32 part_size, int32 ready_part_count, std::string bitmask,
void FileUploader::on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask,
bool is_ready, int64 ready_size) {
callback_->on_partial_upload(PartialRemoteFileLocation{file_id_, part_count, part_size, ready_part_count, big_flag_},
ready_size);

9
td/telegram/files/FileUploader.h
View File

@ -16,6 +16,7 @@
#include <utility>
namespace td {
class FileUploader : public FileLoader {
public:
class Callback : public FileLoader::Callback {
@ -62,10 +63,11 @@ class FileUploader : public FileLoader {
void after_start_parts() override;
Result<std::pair<NetQueryPtr, bool>> start_part(Part part, int32 part_count) override TD_WARN_UNUSED_RESULT;
Result<size_t> process_part(Part part, NetQueryPtr net_query) override TD_WARN_UNUSED_RESULT;
void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, string ready_bitmask, bool is_ready,
int64 ready_size) override;
void on_progress(int32 part_count, int32 part_size, int32 ready_part_count, const string &ready_bitmask,
bool is_ready, int64 ready_size) override;
FileLoader::Callback *get_callback() override;
Result<PrefixInfo> on_update_local_location(const LocalFileLocation &location) override TD_WARN_UNUSED_RESULT;
Result<PrefixInfo> on_update_local_location(const LocalFileLocation &location,
int64 file_size) override TD_WARN_UNUSED_RESULT;
Status generate_iv_map();
@ -74,4 +76,5 @@ class FileUploader : public FileLoader {
void try_release_fd();
Status acquire_fd() TD_WARN_UNUSED_RESULT;
};
} // namespace td

4
td/telegram/files/PartsManager.cpp
View File

@ -30,13 +30,13 @@ Status PartsManager::init_known_prefix(int64 known_prefix, size_t part_size, con
}
void PartsManager::set_streaming_offset(int64 offset) {
if (need_check_ || (!unknown_size_flag_ && get_size() < offset)) {
if (offset < 0 || need_check_ || (!unknown_size_flag_ && get_size() < offset)) {
streaming_offset_ = 0;
return;
}
auto part_i = offset / part_size_;
if (part_i > MAX_PART_COUNT || part_i < 0) {
if (part_i >= MAX_PART_COUNT) {
streaming_offset_ = 0;
// error?
return;

4
td/telegram/files/PartsManager.h
View File

@ -6,11 +6,11 @@
//
#pragma once
#include "td/telegram/files/FileBitmask.h"
#include "td/utils/common.h"
#include "td/utils/Status.h"
#include "td/telegram/files/FileBitmask.h"
namespace td {
/*** PartsManager***/

2
td/telegram/logevent/SecretChatEvent.h
View File

@ -323,7 +323,7 @@ class OutboundSecretMessage : public LogEventHelper<OutboundSecretMessage, Secre
// 1. is_service // use messages_sendEncryptedsService
// 3. can_rewrite_with_empty // false for almost all service messages
// TODO: combine this two functions into one macros hell. Or lambda hell.
// TODO: combine these two functions into one macros hell. Or a lambda hell.
template <class T>
void store(T &storer) const {
using td::store;

2
td/telegram/net/Session.h
View File

@ -115,7 +115,7 @@ class Session final
std::unordered_set<uint64> unknown_queries_;
std::vector<int64> to_cancel_;
// Do not invalidate iterators of this two containers!
// Do not invalidate iterators of these two containers!
// TODO: better data structures
std::deque<NetQueryPtr> pending_queries_;
std::map<uint64, Query> sent_queries_;

24
tdutils/td/utils/misc.cpp
View File

@ -125,7 +125,9 @@ string url_encode(Slice str) {
CHECK(result.size() == length);
return result;
}
namespace detail {
namespace {
template <class F>
string x_decode(Slice s, F &&f) {
string res;
@ -139,6 +141,7 @@ string x_decode(Slice s, F &&f) {
}
return res;
}
template <class F>
string x_encode(Slice s, F &&f) {
string res;
@ -149,31 +152,34 @@ string x_encode(Slice s, F &&f) {
while (cnt < 250 && i + cnt < n && s[i + cnt] == s[i]) {
cnt++;
}
res.push_back(cnt);
res.push_back(static_cast<char>(cnt));
i += cnt - 1;
}
}
return res;
}
bool is_zero(uint8 c) {
bool is_zero(unsigned char c) {
return c == 0;
}
bool is_zero_or_one(uint8 c) {
bool is_zero_or_one(unsigned char c) {
return c == 0 || c == 0xff;
}
} // namespace detail
} // namespace
std::string zero_encode(Slice data) {
return detail::x_encode(data, detail::is_zero);
return x_encode(data, is_zero);
}
std::string zero_decode(Slice data) {
return detail::x_decode(data, detail::is_zero);
return x_decode(data, is_zero);
}
std::string zero_one_encode(Slice data) {
return detail::x_encode(data, detail::is_zero_or_one);
return x_encode(data, is_zero_or_one);
}
std::string zero_one_decode(Slice data) {
return detail::x_decode(data, detail::is_zero_or_one);
return x_decode(data, is_zero_or_one);
}
} // namespace td

12
tdutils/td/utils/misc.h
View File

@ -380,8 +380,12 @@ detail::reversion_wrapper<T> reversed(T &iterable) {
return {iterable};
}
std::string zero_encode(Slice data);
std::string zero_decode(Slice data);
std::string zero_one_encode(Slice data);
std::string zero_one_decode(Slice data);
string zero_encode(Slice data);
string zero_decode(Slice data);
string zero_one_encode(Slice data);
string zero_one_decode(Slice data);
} // namespace td