diff --git a/td/telegram/ContactsManager.cpp b/td/telegram/ContactsManager.cpp
index 65b0fdb41..ed63f67b2 100644
--- a/td/telegram/ContactsManager.cpp
+++ b/td/telegram/ContactsManager.cpp
@@ -5723,7 +5723,7 @@ UserId ContactsManager::search_user_by_phone_number(string phone_number, Promise
 
 void ContactsManager::on_resolved_phone_number(const string &phone_number, UserId user_id) {
   if (!user_id.is_valid()) {
-    resolved_phone_numbers_[phone_number] = user_id;  // negative cache
+    resolved_phone_numbers_.emplace(phone_number, user_id);  // negative cache
     return;
   }
 
@@ -5737,7 +5737,7 @@ void ContactsManager::on_resolved_phone_number(const string &phone_number, UserI
   }
 
   LOG(ERROR) << "Resolve phone number \"" << phone_number << "\" to " << user_id << ", but doesn't have it";
-  resolved_phone_numbers_[phone_number] = user_id;
+  resolved_phone_numbers_[phone_number] = user_id;  // always update cached value
 }
 
 void ContactsManager::share_phone_number(UserId user_id, Promise<Unit> &&promise) {
@@ -8608,10 +8608,6 @@ void ContactsManager::on_binlog_user_event(BinlogEvent &&event) {
   User *u = add_user(user_id, "on_binlog_user_event");
   *u = std::move(log_event.u);  // users come from binlog before all other events, so just add them
 
-  if (!u->phone_number.empty()) {
-    resolved_phone_numbers_[u->phone_number] = user_id;
-  }
-
   u->log_event_id = event.id_;
 
   update_user(u, user_id, true, false);
@@ -8740,10 +8736,6 @@ void ContactsManager::on_load_user_from_database(UserId user_id, string value, b
 
       log_event_parse(*u, value).ensure();
 
-      if (!u->phone_number.empty()) {
-        resolved_phone_numbers_[u->phone_number] = user_id;
-      }
-
       u->is_saved = true;
       u->is_status_saved = true;
       update_user(u, user_id, true, true);
@@ -9990,6 +9982,12 @@ void ContactsManager::update_user(User *u, UserId user_id, bool from_binlog, boo
     });
     u->is_photo_changed = false;
   }
+  if (u->is_phone_number_changed) {
+    if (!u->phone_number.empty() && !td_->auth_manager_->is_bot()) {
+      resolved_phone_numbers_[u->phone_number] = user_id;
+    }
+    u->is_phone_number_changed = false;
+  }
   if (u->is_status_changed && user_id != get_my_id()) {
     auto left_time = get_user_was_online(u, user_id) - G()->server_time_cached();
     if (left_time >= 0 && left_time < 30 * 86400) {
@@ -11165,17 +11163,17 @@ void ContactsManager::on_update_user_phone_number(User *u, UserId user_id, strin
   clean_phone_number(phone_number);
   if (u->phone_number != phone_number) {
     if (!u->phone_number.empty()) {
-      auto erased_count = resolved_phone_numbers_.erase(u->phone_number) > 0;
-      CHECK(erased_count == 1);
+      auto it = resolved_phone_numbers_.find(u->phone_number);
+      CHECK(it != resolved_phone_numbers_.end());
+      if (it->second == user_id) {
+        resolved_phone_numbers_.erase(it);
+      }
     }
 
     u->phone_number = std::move(phone_number);
+    u->is_phone_number_changed = true;
     LOG(DEBUG) << "Phone number has changed for " << user_id;
     u->is_changed = true;
-
-    if (!u->phone_number.empty()) {
-      resolved_phone_numbers_[u->phone_number] = user_id;
-    }
   }
 }
 
diff --git a/td/telegram/ContactsManager.h b/td/telegram/ContactsManager.h
index 8f1c5e41d..1d2f59541 100644
--- a/td/telegram/ContactsManager.h
+++ b/td/telegram/ContactsManager.h
@@ -659,6 +659,7 @@ class ContactsManager final : public Actor {
     bool is_name_changed = true;
     bool is_username_changed = true;
     bool is_photo_changed = true;
+    bool is_phone_number_changed = true;
     bool is_is_contact_changed = true;
     bool is_is_deleted_changed = true;
     bool is_changed = true;             // have new changes that need to be sent to the client and database