Move min and reference databases from memory

src/danog/MadelineProto/Db/DbPropertiesTrait.php

@@ -22,18 +22,6 @@ trait DbPropertiesTrait
                 $this->{$property} = yield DbPropertiesFabric::get($dbSettings, $prefix, $type, $property, $this->{$property});
             }
         }
-
-        if (!$reset && yield $this->usernames->count() === 0) {
-            $this->logger('Filling database cache. This can take few minutes.', Logger::WARNING);
-            $iterator = $this->chats->getIterator();
-            while (yield $iterator->advance()) {
-                [$id, $chat] = $iterator->getCurrent();
-                if (isset($chat['username'])) {
-                    $this->usernames[\strtolower($chat['username'])] = $this->getId($chat);
-                }
-            }
-            $this->logger('Cache filled.', Logger::WARNING);
-        }
     }
 
     private static function getSessionId(MTProto $madelineProto): string

src/danog/MadelineProto/MTProto.php

@@ -272,7 +272,7 @@ class MTProto extends AsyncConstruct implements TLCallback
     /**
      * Internal peer database.
      *
-     * @var DbArray
+     * @var DbArray|Promise[]
      */
     public $chats;
 
@@ -492,7 +492,10 @@ class MTProto extends AsyncConstruct implements TLCallback
      */
     public function __sleep(): array
     {
-        if ($this->settings['serialization']['cleanup_before_serialization']) {
+        if (
+            $this->settings['serialization']['cleanup_before_serialization']
+            && $this->settings['db']['type'] === 'memory'
+        ) {
             $this->cleanup();
         }
         return [
@@ -562,9 +565,9 @@ class MTProto extends AsyncConstruct implements TLCallback
     /**
      * Cleanup memory and session file.
      *
-     * @return self
+     * @return void
      */
-    public function cleanup(): self
+    public function cleanup(): void
     {
         $this->referenceDatabase = new ReferenceDatabase($this);
         $callbacks = [$this, $this->referenceDatabase];
@@ -572,8 +575,23 @@ class MTProto extends AsyncConstruct implements TLCallback
             $callbacks[] = $this->minDatabase;
         }
         $this->TL->updateCallbacks($callbacks);
-        return $this;
     }
+
+    private function fillUsernamesCache(): \Generator
+    {
+        if (yield $this->usernames->count() === 0) {
+            $this->logger('Filling database cache. This can take few minutes.', Logger::WARNING);
+            $iterator = $this->chats->getIterator();
+            while (yield $iterator->advance()) {
+                [$id, $chat] = $iterator->getCurrent();
+                if (isset($chat['username'])) {
+                    $this->usernames[\strtolower($chat['username'])] = $this->getId($chat);
+                }
+            }
+            $this->logger('Cache filled.', Logger::WARNING);
+        }
+    }
+
     /**
      * Logger.
      *
@@ -821,6 +839,7 @@ class MTProto extends AsyncConstruct implements TLCallback
         }
 
         yield from $this->initDb($this);
+        yield from $this->fillUsernamesCache();
     }
 
     /**

src/danog/MadelineProto/MTProtoTools/MinDatabase.php

@@ -19,23 +19,30 @@
 
 namespace danog\MadelineProto\MTProtoTools;
 
+use Amp\Loop;
+use Amp\Promise;
+use danog\MadelineProto\Db\DbArray;
+use danog\MadelineProto\Db\DbPropertiesTrait;
 use danog\MadelineProto\MTProto;
 use danog\MadelineProto\TL\TLCallback;
+use danog\MadelineProto\Tools;
 
 /**
  * Manages min peers.
  */
 class MinDatabase implements TLCallback
 {
+    use DbPropertiesTrait;
+
     const SWITCH_CONSTRUCTORS = ['inputChannel', 'inputUser', 'inputPeerUser', 'inputPeerChannel'];
     const CATCH_PEERS = ['message', 'messageService', 'peerUser', 'peerChannel', 'messageEntityMentionName', 'messageFwdHeader', 'messageActionChatCreate', 'messageActionChatAddUser', 'messageActionChatDeleteUser', 'messageActionChatJoinedByLink'];
     const ORIGINS = ['message', 'messageService'];
     /**
      * References indexed by location.
      *
-     * @var array
+     * @var DbArray|Promise[]
      */
-    private $db = [];
+    private $db;
     /**
      * Temporary cache during deserialization.
      *
@@ -48,6 +55,16 @@ class MinDatabase implements TLCallback
      * @var \danog\MadelineProto\MTProto
      */
     private $API;
+
+    /**
+     * List of properties stored in database (memory or external).
+     * @see DbPropertiesFabric
+     * @var array
+     */
+    protected array $dbProperies = [
+        'db' => 'array',
+    ];
+
     public function __construct(MTProto $API)
     {
         $this->API = $API;
@@ -63,11 +80,16 @@ class MinDatabase implements TLCallback
     }
     public function init()
     {
-        foreach ($this->db as $id => $origin) {
+        Tools::wait($this->initDb($this->API));
-            if (!isset($origin['peer']) || $origin['peer'] === $id) {
+        Loop::defer(function() {
-                unset($this->db[$id]);
+            $iterator = $this->db->getIterator();
+            while (yield $iterator->advance()) {
+                [$id, $origin] = $iterator->getCurrent();
+                if (!isset($origin['peer']) || $origin['peer'] === $id) {
+                    $this->db->offsetUnset($id);
+                }
             }
-        }
+        });
     }
     public function getMethodCallbacks(): array
     {
@@ -202,7 +224,7 @@ class MinDatabase implements TLCallback
             return $object;
         }
         $id = $this->API->getId($object);
-        if (isset($this->db[$id])) {
+        if (yield $this->db[$id]) {
             $new = \array_merge($object, $this->db[$id]);
             $new['_'] .= 'FromMessage';
             $new['peer'] = (yield from $this->API->getInfo($new['peer']))['InputPeer'];
@@ -215,16 +237,17 @@ class MinDatabase implements TLCallback
         $this->API->logger->logger("Don't have origin info with min peer {$id}, this may fail");
         return $object;
     }
+
     /**
      * Check if location info is available for peer.
      *
      * @param float|int $id Peer ID
      *
-     * @return boolean
+     * @return boolean<Promise>
      */
-    public function hasPeer($id): bool
+    public function hasPeer($id): Promise
     {
-        return isset($this->db[$id]);
+        return $this->db->isset($id);
     }
     public function __debugInfo()
     {

src/danog/MadelineProto/MTProtoTools/PeerHandler.php

@@ -518,7 +518,7 @@ trait PeerHandler
                 }
             }
             if (yield $this->chats[$id]) {
-                if (((yield $this->chats[$id])['min'] ?? false) && $this->minDatabase->hasPeer($id) && !isset($this->caching_full_info[$id])) {
+                if (((yield $this->chats[$id])['min'] ?? false) && yield $this->minDatabase->hasPeer($id) && !isset($this->caching_full_info[$id])) {
                     $this->caching_full_info[$id] = true;
                     $this->logger->logger("Only have min peer for {$id} in database, trying to fetch full info");
                     try {

src/danog/MadelineProto/MTProtoTools/ReferenceDatabase.php

@@ -19,15 +19,21 @@
 
 namespace danog\MadelineProto\MTProtoTools;
 
+use Amp\Loop;
+use Amp\Promise;
+use danog\MadelineProto\Db\DbArray;
+use danog\MadelineProto\Db\DbPropertiesTrait;
 use danog\MadelineProto\Exception;
 use danog\MadelineProto\MTProto;
 use danog\MadelineProto\TL\TLCallback;
+use danog\MadelineProto\Tools;
 
 /**
  * Manages upload and download of files.
  */
 class ReferenceDatabase implements TLCallback
 {
+    use DbPropertiesTrait;
 
     // Reference from a document
     const DOCUMENT_LOCATION = 0;
@@ -65,15 +71,25 @@ class ReferenceDatabase implements TLCallback
     /**
      * References indexed by location.
      *
-     * @var array
+     * @var DbArray|Promise[]
      */
-    private $db = [];
+    private $db;
     private $cache = [];
     private $cacheContexts = [];
     private $refreshed = [];
     private $API;
     private $refresh = false;
     private $refreshCount = 0;
+
+    /**
+     * List of properties stored in database (memory or external).
+     * @see DbPropertiesFabric
+     * @var array
+     */
+    protected array $dbProperies = [
+        'db' => 'array',
+    ];
+
     public function __construct(MTProto $API)
     {
         $this->API = $API;
@@ -89,6 +105,7 @@ class ReferenceDatabase implements TLCallback
     }
     public function init()
     {
+        Tools::wait($this->initDb($this->API));
     }
     public function getMethodCallbacks(): array
     {
@@ -355,11 +372,14 @@ class ReferenceDatabase implements TLCallback
     }
     public function storeReference(string $location, string $reference, int $originType, array $origin)
     {
-        if (!isset($this->db[$location])) {
+        $locationValue = Tools::wait($this->db[$location]);
-            $this->db[$location] = ['origins' => []];
+        if (!$locationValue) {
+            $locationValue = ['origins' => []];
         }
-        $this->db[$location]['reference'] = $reference;
+        $locationValue['reference'] = $reference;
-        $this->db[$location]['origins'][$originType] = $origin;
+        $locationValue['origins'][$originType] = $origin;
+        $this->db[$location] = $locationValue;
+
         if ($this->refresh) {
             $this->refreshed[$location] = true;
         }
@@ -393,11 +413,13 @@ class ReferenceDatabase implements TLCallback
     {
         if (isset($this->refreshed[$location])) {
             $this->API->logger->logger('Reference already refreshed!', \danog\MadelineProto\Logger::VERBOSE);
-            return $this->db[$location]['reference'];
+            return (yield $this->db[$location])['reference'];
         }
-        \ksort($this->db[$location]['origins']);
+        $locationValue = yield $this->db[$location];
+        \ksort($locationValue['origins']);
+        $this->db[$location] = $locationValue;
         $count = 0;
-        foreach ($this->db[$location]['origins'] as $originType => &$origin) {
+        foreach ((yield $this->db[$location]['origins']) as $originType => &$origin) {
             $count++;
             $this->API->logger->logger("Try {$count} refreshing file reference with origin type {$originType}", \danog\MadelineProto\Logger::VERBOSE);
             switch ($originType) {
@@ -447,7 +469,7 @@ class ReferenceDatabase implements TLCallback
                     throw new \danog\MadelineProto\Exception("Unknown origin type {$originType}");
             }
             if (isset($this->refreshed[$location])) {
-                return $this->db[$location]['reference'];
+                return (yield $this->db[$location])['reference'];
             }
         }
         throw new Exception('Did not refresh reference');
@@ -460,7 +482,7 @@ class ReferenceDatabase implements TLCallback
     public function getReference(int $locationType, array $location)
     {
         $locationString = $this->serializeLocation($locationType, $location);
-        if (!isset($this->db[$locationString]['reference'])) {
+        if (!isset((yield $this->db[$locationString])['reference'])) {
             if (isset($location['file_reference'])) {
                 $this->API->logger->logger("Using outdated file reference for location of type {$locationType} object {$location['_']}", \danog\MadelineProto\Logger::ULTRA_VERBOSE);
                 return $location['file_reference'];
@@ -475,7 +497,7 @@ class ReferenceDatabase implements TLCallback
         if ($this->refresh) {
             return $this->refreshReferenceInternal($locationString);
         }
-        return $this->db[$locationString]['reference'];
+        return (yield $this->db[$locationString])['reference'];
     }
     private function serializeLocation(int $locationType, array $location)
     {