tdlight/td/telegram/VoiceNotesManager.cpp

//
// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2022
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
#include "td/telegram/VoiceNotesManager.h"

#include "td/telegram/AccessRights.h"
#include "td/telegram/DialogId.h"
#include "td/telegram/Dimensions.h"
#include "td/telegram/files/FileManager.h"
#include "td/telegram/Global.h"
#include "td/telegram/MessagesManager.h"
#include "td/telegram/secret_api.h"
#include "td/telegram/Td.h"
#include "td/telegram/td_api.h"
#include "td/telegram/telegram_api.h"

#include "td/utils/buffer.h"
#include "td/utils/logging.h"

namespace td {

class TranscribeAudioQuery final : public Td::ResultHandler {
  DialogId dialog_id_;
  FileId file_id_;

 public:
  void send(FileId file_id, FullMessageId full_message_id) {
    dialog_id_ = full_message_id.get_dialog_id();
    file_id_ = file_id;
    auto input_peer = td_->messages_manager_->get_input_peer(dialog_id_, AccessRights::Read);
    if (input_peer == nullptr) {
      return on_error(Status::Error(400, "Can't access the chat"));
    }
    send_query(G()->net_query_creator().create(telegram_api::messages_transcribeAudio(
        std::move(input_peer), full_message_id.get_message_id().get_server_message_id().get())));
  }

  void on_result(BufferSlice packet) final {
    auto result_ptr = fetch_result<telegram_api::messages_transcribeAudio>(packet);
    if (result_ptr.is_error()) {
      return on_error(result_ptr.move_as_error());
    }

    auto result = result_ptr.move_as_ok();
    LOG(INFO) << "Receive result for TranscribeAudioQuery: " << to_string(result);
    if (result->transcription_id_ == 0) {
      return on_error(Status::Error(500, "Receive no recognition identifier"));
    }
    td_->voice_notes_manager_->on_voice_note_transcribed(file_id_, std::move(result->text_), result->transcription_id_,
                                                         !result->pending_);
  }

  void on_error(Status status) final {
    td_->messages_manager_->on_get_dialog_error(dialog_id_, status, "TranscribeAudioQuery");
    td_->voice_notes_manager_->on_voice_note_transcription_failed(file_id_, std::move(status));
  }
};

class RateTranscribedAudioQuery final : public Td::ResultHandler {
  Promise<Unit> promise_;
  DialogId dialog_id_;

 public:
  explicit RateTranscribedAudioQuery(Promise<Unit> &&promise) : promise_(std::move(promise)) {
  }

  void send(FullMessageId full_message_id, int64 transcription_id, bool is_good) {
    dialog_id_ = full_message_id.get_dialog_id();
    auto input_peer = td_->messages_manager_->get_input_peer(dialog_id_, AccessRights::Read);
    if (input_peer == nullptr) {
      return on_error(Status::Error(400, "Can't access the chat"));
    }
    send_query(G()->net_query_creator().create(telegram_api::messages_rateTranscribedAudio(
        std::move(input_peer), full_message_id.get_message_id().get_server_message_id().get(), transcription_id,
        is_good)));
  }

  void on_result(BufferSlice packet) final {
    auto result_ptr = fetch_result<telegram_api::messages_rateTranscribedAudio>(packet);
    if (result_ptr.is_error()) {
      return on_error(result_ptr.move_as_error());
    }

    bool result = result_ptr.ok();
    LOG(INFO) << "Receive result for RateTranscribedAudioQuery: " << result;
    promise_.set_value(Unit());
  }

  void on_error(Status status) final {
    td_->messages_manager_->on_get_dialog_error(dialog_id_, status, "RateTranscribedAudioQuery");
    promise_.set_error(std::move(status));
  }
};

VoiceNotesManager::VoiceNotesManager(Td *td, ActorShared<> parent) : td_(td), parent_(std::move(parent)) {
  voice_note_transcription_timeout_.set_callback(on_voice_note_transcription_timeout_callback);
  voice_note_transcription_timeout_.set_callback_data(static_cast<void *>(this));
}

VoiceNotesManager::~VoiceNotesManager() {
  Scheduler::instance()->destroy_on_scheduler(G()->get_gc_scheduler_id(), voice_notes_, voice_note_messages_,
                                              message_voice_notes_);
}

void VoiceNotesManager::tear_down() {
  parent_.reset();
}

void VoiceNotesManager::on_voice_note_transcription_timeout_callback(void *voice_notes_manager_ptr,
                                                                     int64 transcription_id) {
  if (G()->close_flag()) {
    return;
  }

  auto voice_notes_manager = static_cast<VoiceNotesManager *>(voice_notes_manager_ptr);
  send_closure_later(voice_notes_manager->actor_id(voice_notes_manager),
                     &VoiceNotesManager::on_pending_voice_note_transcription_failed, transcription_id,
                     Status::Error(500, "Timeout expired"));
}

int32 VoiceNotesManager::get_voice_note_duration(FileId file_id) const {
  auto it = voice_notes_.find(file_id);
  if (it == voice_notes_.end()) {
    return 0;
  }
  return it->second->duration;
}

tl_object_ptr<td_api::voiceNote> VoiceNotesManager::get_voice_note_object(FileId file_id) const {
  if (!file_id.is_valid()) {
    return nullptr;
  }

  auto it = voice_notes_.find(file_id);
  CHECK(it != voice_notes_.end());
  auto voice_note = it->second.get();
  CHECK(voice_note != nullptr);

  auto speech_recognition_result = [this, voice_note]() -> td_api::object_ptr<td_api::SpeechRecognitionResult> {
    if (voice_note->is_transcribed) {
      return td_api::make_object<td_api::speechRecognitionResultText>(voice_note->text);
    }
    if (speech_recognition_queries_.count(voice_note->file_id) != 0) {
      return td_api::make_object<td_api::speechRecognitionResultPending>(voice_note->text);
    }
    if (voice_note->last_transcription_error.is_error()) {
      return td_api::make_object<td_api::speechRecognitionResultError>(
          td_api::make_object<td_api::error>(voice_note->last_transcription_error.error().code(),
                                             voice_note->last_transcription_error.error().message().str()));
    }
    return nullptr;
  }();

  return make_tl_object<td_api::voiceNote>(voice_note->duration, voice_note->waveform, voice_note->mime_type,
                                           std::move(speech_recognition_result),
                                           td_->file_manager_->get_file_object(file_id));
}

FileId VoiceNotesManager::on_get_voice_note(unique_ptr<VoiceNote> new_voice_note, bool replace) {
  auto file_id = new_voice_note->file_id;
  CHECK(file_id.is_valid());
  LOG(INFO) << "Receive voice note " << file_id;
  auto &v = voice_notes_[file_id];
  if (v == nullptr) {
    v = std::move(new_voice_note);
  } else if (replace) {
    CHECK(v->file_id == new_voice_note->file_id);
    if (v->mime_type != new_voice_note->mime_type) {
      LOG(DEBUG) << "Voice note " << file_id << " info has changed";
      v->mime_type = new_voice_note->mime_type;
    }
    if (v->duration != new_voice_note->duration || v->waveform != new_voice_note->waveform) {
      LOG(DEBUG) << "Voice note " << file_id << " info has changed";
      v->duration = new_voice_note->duration;
      v->waveform = new_voice_note->waveform;
    }
    if (new_voice_note->is_transcribed && v->transcription_id == 0) {
      CHECK(!v->is_transcribed);
      CHECK(new_voice_note->transcription_id != 0);
      v->is_transcribed = true;
      v->transcription_id = new_voice_note->transcription_id;
      v->text = std::move(new_voice_note->text);
      v->last_transcription_error = Status::OK();
      on_voice_note_transcription_updated(file_id);
    }
  }

  return file_id;
}

VoiceNotesManager::VoiceNote *VoiceNotesManager::get_voice_note(FileId file_id) {
  auto voice_note = voice_notes_.find(file_id);
  if (voice_note == voice_notes_.end()) {
    return nullptr;
  }

  CHECK(voice_note->second->file_id == file_id);
  return voice_note->second.get();
}

const VoiceNotesManager::VoiceNote *VoiceNotesManager::get_voice_note(FileId file_id) const {
  auto voice_note = voice_notes_.find(file_id);
  if (voice_note == voice_notes_.end()) {
    return nullptr;
  }

  CHECK(voice_note->second->file_id == file_id);
  return voice_note->second.get();
}

FileId VoiceNotesManager::dup_voice_note(FileId new_id, FileId old_id) {
  const VoiceNote *old_voice_note = get_voice_note(old_id);
  CHECK(old_voice_note != nullptr);
  auto &new_voice_note = voice_notes_[new_id];
  CHECK(new_voice_note == nullptr);
  new_voice_note = make_unique<VoiceNote>();
  new_voice_note->file_id = new_id;
  new_voice_note->mime_type = old_voice_note->mime_type;
  new_voice_note->duration = old_voice_note->duration;
  new_voice_note->waveform = old_voice_note->waveform;
  if (old_voice_note->is_transcribed) {
    new_voice_note->is_transcribed = old_voice_note->is_transcribed;
    new_voice_note->text = old_voice_note->text;
  }
  return new_id;
}

void VoiceNotesManager::merge_voice_notes(FileId new_id, FileId old_id, bool can_delete_old) {
  CHECK(old_id.is_valid() && new_id.is_valid());
  CHECK(new_id != old_id);

  LOG(INFO) << "Merge voice notes " << new_id << " and " << old_id;
  const VoiceNote *old_ = get_voice_note(old_id);
  CHECK(old_ != nullptr);

  auto new_it = voice_notes_.find(new_id);
  if (new_it == voice_notes_.end()) {
    auto &old = voice_notes_[old_id];
    if (!can_delete_old) {
      dup_voice_note(new_id, old_id);
    } else {
      old->file_id = new_id;
      voice_notes_.emplace(new_id, std::move(old));
    }
  } else {
    VoiceNote *new_ = new_it->second.get();
    CHECK(new_ != nullptr);

    if (!old_->mime_type.empty() && old_->mime_type != new_->mime_type) {
      LOG(INFO) << "Voice note has changed: mime_type = (" << old_->mime_type << ", " << new_->mime_type << ")";
    }
  }
  LOG_STATUS(td_->file_manager_->merge(new_id, old_id));
  if (can_delete_old) {
    voice_notes_.erase(old_id);
  }
}

void VoiceNotesManager::create_voice_note(FileId file_id, string mime_type, int32 duration, string waveform,
                                          bool replace) {
  auto v = make_unique<VoiceNote>();
  v->file_id = file_id;
  v->mime_type = std::move(mime_type);
  v->duration = max(duration, 0);
  v->waveform = std::move(waveform);
  on_get_voice_note(std::move(v), replace);
}

void VoiceNotesManager::register_voice_note(FileId voice_note_file_id, FullMessageId full_message_id,
                                            const char *source) {
  if (full_message_id.get_message_id().is_scheduled() || !full_message_id.get_message_id().is_server()) {
    return;
  }
  LOG(INFO) << "Register voice note " << voice_note_file_id << " from " << full_message_id << " from " << source;
  bool is_inserted = voice_note_messages_[voice_note_file_id].insert(full_message_id).second;
  LOG_CHECK(is_inserted) << source << ' ' << voice_note_file_id << ' ' << full_message_id;
  is_inserted = message_voice_notes_.emplace(full_message_id, voice_note_file_id).second;
  CHECK(is_inserted);
}

void VoiceNotesManager::unregister_voice_note(FileId voice_note_file_id, FullMessageId full_message_id,
                                              const char *source) {
  if (full_message_id.get_message_id().is_scheduled() || !full_message_id.get_message_id().is_server()) {
    return;
  }
  LOG(INFO) << "Unregister voice note " << voice_note_file_id << " from " << full_message_id << " from " << source;
  auto &message_ids = voice_note_messages_[voice_note_file_id];
  auto is_deleted = message_ids.erase(full_message_id) > 0;
  LOG_CHECK(is_deleted) << source << ' ' << voice_note_file_id << ' ' << full_message_id;
  if (message_ids.empty()) {
    voice_note_messages_.erase(voice_note_file_id);
  }
  is_deleted = message_voice_notes_.erase(full_message_id) > 0;
  CHECK(is_deleted);
}

void VoiceNotesManager::recognize_speech(FullMessageId full_message_id, Promise<Unit> &&promise) {
  if (!td_->messages_manager_->have_message_force(full_message_id, "recognize_speech")) {
    return promise.set_error(Status::Error(400, "Message not found"));
  }

  auto it = message_voice_notes_.find(full_message_id);
  if (it == message_voice_notes_.end()) {
    return promise.set_error(Status::Error(400, "Invalid message specified"));
  }

  auto file_id = it->second;
  auto voice_note = get_voice_note(file_id);
  CHECK(voice_note != nullptr);
  if (voice_note->is_transcribed) {
    return promise.set_value(Unit());
  }
  auto &queries = speech_recognition_queries_[file_id];
  queries.push_back(std::move(promise));
  if (queries.size() == 1) {
    td_->create_handler<TranscribeAudioQuery>()->send(file_id, full_message_id);
    voice_note->last_transcription_error = Status::OK();
    on_voice_note_transcription_updated(file_id);
  }
}

void VoiceNotesManager::on_voice_note_transcribed(FileId file_id, string &&text, int64 transcription_id,
                                                  bool is_final) {
  auto voice_note = get_voice_note(file_id);
  CHECK(voice_note != nullptr);
  CHECK(!voice_note->is_transcribed);
  CHECK(voice_note->transcription_id == 0 || voice_note->transcription_id == transcription_id);
  bool is_changed = voice_note->is_transcribed != is_final || voice_note->text != text;
  voice_note->transcription_id = transcription_id;
  voice_note->is_transcribed = is_final;
  voice_note->text = std::move(text);
  voice_note->last_transcription_error = Status::OK();

  if (is_final) {
    auto it = speech_recognition_queries_.find(file_id);
    CHECK(it != speech_recognition_queries_.end());
    CHECK(!it->second.empty());
    auto promises = std::move(it->second);
    speech_recognition_queries_.erase(it);

    on_voice_note_transcription_updated(file_id);
    set_promises(promises);
  } else {
    if (is_changed) {
      on_voice_note_transcription_updated(file_id);
    }

    if (pending_voice_note_transcription_queries_.count(transcription_id) != 0) {
      on_pending_voice_note_transcription_failed(transcription_id,
                                                 Status::Error(500, "Receive duplicate recognition identifier"));
    }
    bool is_inserted = pending_voice_note_transcription_queries_.emplace(transcription_id, file_id).second;
    CHECK(is_inserted);
    voice_note_transcription_timeout_.set_timeout_in(transcription_id, TRANSCRIPTION_TIMEOUT);
  }
}

void VoiceNotesManager::on_voice_note_transcription_failed(FileId file_id, Status &&error) {
  auto voice_note = get_voice_note(file_id);
  CHECK(voice_note != nullptr);
  CHECK(!voice_note->is_transcribed);
  CHECK(pending_voice_note_transcription_queries_.count(voice_note->transcription_id) == 0);

  voice_note->transcription_id = 0;
  voice_note->text.clear();
  voice_note->last_transcription_error = error.clone();

  auto it = speech_recognition_queries_.find(file_id);
  CHECK(it != speech_recognition_queries_.end());
  CHECK(!it->second.empty());
  auto promises = std::move(it->second);
  speech_recognition_queries_.erase(it);

  on_voice_note_transcription_updated(file_id);
  fail_promises(promises, std::move(error));
}

void VoiceNotesManager::on_update_transcribed_audio(string &&text, int64 transcription_id, bool is_final) {
  auto it = pending_voice_note_transcription_queries_.find(transcription_id);
  if (it == pending_voice_note_transcription_queries_.end()) {
    return;
  }
  auto file_id = it->second;
  pending_voice_note_transcription_queries_.erase(it);
  voice_note_transcription_timeout_.cancel_timeout(transcription_id);

  on_voice_note_transcribed(file_id, std::move(text), transcription_id, is_final);
}

void VoiceNotesManager::on_pending_voice_note_transcription_failed(int64 transcription_id, Status &&error) {
  auto it = pending_voice_note_transcription_queries_.find(transcription_id);
  if (it == pending_voice_note_transcription_queries_.end()) {
    return;
  }
  auto file_id = it->second;
  pending_voice_note_transcription_queries_.erase(it);
  voice_note_transcription_timeout_.cancel_timeout(transcription_id);

  on_voice_note_transcription_failed(file_id, std::move(error));
}

void VoiceNotesManager::on_voice_note_transcription_updated(FileId file_id) {
  auto it = voice_note_messages_.find(file_id);
  if (it != voice_note_messages_.end()) {
    for (const auto &full_message_id : it->second) {
      td_->messages_manager_->on_external_update_message_content(full_message_id);
    }
  }
}

void VoiceNotesManager::rate_speech_recognition(FullMessageId full_message_id, bool is_good, Promise<Unit> &&promise) {
  if (!td_->messages_manager_->have_message_force(full_message_id, "rate_speech_recognition")) {
    return promise.set_error(Status::Error(400, "Message not found"));
  }

  auto it = message_voice_notes_.find(full_message_id);
  if (it == message_voice_notes_.end()) {
    return promise.set_error(Status::Error(400, "Invalid message specified"));
  }

  auto file_id = it->second;
  auto voice_note = get_voice_note(file_id);
  CHECK(voice_note != nullptr);
  if (!voice_note->is_transcribed) {
    return promise.set_value(Unit());
  }
  CHECK(voice_note->transcription_id != 0);
  td_->create_handler<RateTranscribedAudioQuery>(std::move(promise))
      ->send(full_message_id, voice_note->transcription_id, is_good);
}

SecretInputMedia VoiceNotesManager::get_secret_input_media(FileId voice_note_file_id,
                                                           tl_object_ptr<telegram_api::InputEncryptedFile> input_file,
                                                           const string &caption, int32 layer) const {
  auto file_view = td_->file_manager_->get_file_view(voice_note_file_id);
  if (!file_view.is_encrypted_secret() || file_view.encryption_key().empty()) {
    return SecretInputMedia{};
  }
  if (file_view.has_remote_location()) {
    input_file = file_view.main_remote_location().as_input_encrypted_file();
  }
  if (!input_file) {
    return SecretInputMedia{};
  }

  auto *voice_note = get_voice_note(voice_note_file_id);
  CHECK(voice_note != nullptr);
  vector<tl_object_ptr<secret_api::DocumentAttribute>> attributes;
  attributes.push_back(make_tl_object<secret_api::documentAttributeAudio>(
      secret_api::documentAttributeAudio::VOICE_MASK | secret_api::documentAttributeAudio::WAVEFORM_MASK,
      false /*ignored*/, voice_note->duration, "", "", BufferSlice(voice_note->waveform)));

  return {std::move(input_file), BufferSlice(), Dimensions(), voice_note->mime_type, file_view,
          std::move(attributes), caption,       layer};
}

tl_object_ptr<telegram_api::InputMedia> VoiceNotesManager::get_input_media(
    FileId file_id, tl_object_ptr<telegram_api::InputFile> input_file) const {
  auto file_view = td_->file_manager_->get_file_view(file_id);
  if (file_view.is_encrypted()) {
    return nullptr;
  }
  if (file_view.has_remote_location() && !file_view.main_remote_location().is_web() && input_file == nullptr) {
    return make_tl_object<telegram_api::inputMediaDocument>(0, file_view.main_remote_location().as_input_document(), 0,
                                                            string());
  }
  if (file_view.has_url()) {
    return make_tl_object<telegram_api::inputMediaDocumentExternal>(0, file_view.url(), 0);
  }

  if (input_file != nullptr) {
    const VoiceNote *voice_note = get_voice_note(file_id);
    CHECK(voice_note != nullptr);

    vector<tl_object_ptr<telegram_api::DocumentAttribute>> attributes;
    int32 flags = telegram_api::documentAttributeAudio::VOICE_MASK;
    if (!voice_note->waveform.empty()) {
      flags |= telegram_api::documentAttributeAudio::WAVEFORM_MASK;
    }
    attributes.push_back(make_tl_object<telegram_api::documentAttributeAudio>(
        flags, false /*ignored*/, voice_note->duration, "", "", BufferSlice(voice_note->waveform)));
    string mime_type = voice_note->mime_type;
    if (mime_type != "audio/ogg" && mime_type != "audio/mpeg" && mime_type != "audio/mp4") {
      mime_type = "audio/ogg";
    }
    return make_tl_object<telegram_api::inputMediaUploadedDocument>(
        0, false /*ignored*/, false /*ignored*/, std::move(input_file), nullptr, mime_type, std::move(attributes),
        vector<tl_object_ptr<telegram_api::InputDocument>>(), 0);
  } else {
    CHECK(!file_view.has_remote_location());
  }

  return nullptr;
}

}  // namespace td