From b90e92ee68b46937983924240df810bf9f9c672a Mon Sep 17 00:00:00 2001 From: Alexander Pankratov Date: Mon, 31 Aug 2020 01:39:25 +0300 Subject: [PATCH] Move min and reference databases from memory --- .../MadelineProto/Db/DbPropertiesTrait.php | 12 ----- src/danog/MadelineProto/MTProto.php | 29 ++++++++++-- .../MTProtoTools/MinDatabase.php | 43 +++++++++++++---- .../MTProtoTools/PeerHandler.php | 2 +- .../MTProtoTools/ReferenceDatabase.php | 46 ++++++++++++++----- 5 files changed, 92 insertions(+), 40 deletions(-) diff --git a/src/danog/MadelineProto/Db/DbPropertiesTrait.php b/src/danog/MadelineProto/Db/DbPropertiesTrait.php index 8463eba0..bbf2dc45 100644 --- a/src/danog/MadelineProto/Db/DbPropertiesTrait.php +++ b/src/danog/MadelineProto/Db/DbPropertiesTrait.php @@ -22,18 +22,6 @@ trait DbPropertiesTrait $this->{$property} = yield DbPropertiesFabric::get($dbSettings, $prefix, $type, $property, $this->{$property}); } } - - if (!$reset && yield $this->usernames->count() === 0) { - $this->logger('Filling database cache. This can take few minutes.', Logger::WARNING); - $iterator = $this->chats->getIterator(); - while (yield $iterator->advance()) { - [$id, $chat] = $iterator->getCurrent(); - if (isset($chat['username'])) { - $this->usernames[\strtolower($chat['username'])] = $this->getId($chat); - } - } - $this->logger('Cache filled.', Logger::WARNING); - } } private static function getSessionId(MTProto $madelineProto): string diff --git a/src/danog/MadelineProto/MTProto.php b/src/danog/MadelineProto/MTProto.php index e5b6ae24..08eb73e5 100644 --- a/src/danog/MadelineProto/MTProto.php +++ b/src/danog/MadelineProto/MTProto.php @@ -272,7 +272,7 @@ class MTProto extends AsyncConstruct implements TLCallback /** * Internal peer database. * - * @var DbArray + * @var DbArray|Promise[] */ public $chats; @@ -492,7 +492,10 @@ class MTProto extends AsyncConstruct implements TLCallback */ public function __sleep(): array { - if ($this->settings['serialization']['cleanup_before_serialization']) { + if ( + $this->settings['serialization']['cleanup_before_serialization'] + && $this->settings['db']['type'] === 'memory' + ) { $this->cleanup(); } return [ @@ -562,9 +565,9 @@ class MTProto extends AsyncConstruct implements TLCallback /** * Cleanup memory and session file. * - * @return self + * @return void */ - public function cleanup(): self + public function cleanup(): void { $this->referenceDatabase = new ReferenceDatabase($this); $callbacks = [$this, $this->referenceDatabase]; @@ -572,8 +575,23 @@ class MTProto extends AsyncConstruct implements TLCallback $callbacks[] = $this->minDatabase; } $this->TL->updateCallbacks($callbacks); - return $this; } + + private function fillUsernamesCache(): \Generator + { + if (yield $this->usernames->count() === 0) { + $this->logger('Filling database cache. This can take few minutes.', Logger::WARNING); + $iterator = $this->chats->getIterator(); + while (yield $iterator->advance()) { + [$id, $chat] = $iterator->getCurrent(); + if (isset($chat['username'])) { + $this->usernames[\strtolower($chat['username'])] = $this->getId($chat); + } + } + $this->logger('Cache filled.', Logger::WARNING); + } + } + /** * Logger. * @@ -821,6 +839,7 @@ class MTProto extends AsyncConstruct implements TLCallback } yield from $this->initDb($this); + yield from $this->fillUsernamesCache(); } /** diff --git a/src/danog/MadelineProto/MTProtoTools/MinDatabase.php b/src/danog/MadelineProto/MTProtoTools/MinDatabase.php index 3927876b..71299f03 100644 --- a/src/danog/MadelineProto/MTProtoTools/MinDatabase.php +++ b/src/danog/MadelineProto/MTProtoTools/MinDatabase.php @@ -19,23 +19,30 @@ namespace danog\MadelineProto\MTProtoTools; +use Amp\Loop; +use Amp\Promise; +use danog\MadelineProto\Db\DbArray; +use danog\MadelineProto\Db\DbPropertiesTrait; use danog\MadelineProto\MTProto; use danog\MadelineProto\TL\TLCallback; +use danog\MadelineProto\Tools; /** * Manages min peers. */ class MinDatabase implements TLCallback { + use DbPropertiesTrait; + const SWITCH_CONSTRUCTORS = ['inputChannel', 'inputUser', 'inputPeerUser', 'inputPeerChannel']; const CATCH_PEERS = ['message', 'messageService', 'peerUser', 'peerChannel', 'messageEntityMentionName', 'messageFwdHeader', 'messageActionChatCreate', 'messageActionChatAddUser', 'messageActionChatDeleteUser', 'messageActionChatJoinedByLink']; const ORIGINS = ['message', 'messageService']; /** * References indexed by location. * - * @var array + * @var DbArray|Promise[] */ - private $db = []; + private $db; /** * Temporary cache during deserialization. * @@ -48,6 +55,16 @@ class MinDatabase implements TLCallback * @var \danog\MadelineProto\MTProto */ private $API; + + /** + * List of properties stored in database (memory or external). + * @see DbPropertiesFabric + * @var array + */ + protected array $dbProperies = [ + 'db' => 'array', + ]; + public function __construct(MTProto $API) { $this->API = $API; @@ -63,11 +80,16 @@ class MinDatabase implements TLCallback } public function init() { - foreach ($this->db as $id => $origin) { - if (!isset($origin['peer']) || $origin['peer'] === $id) { - unset($this->db[$id]); + Tools::wait($this->initDb($this->API)); + Loop::defer(function() { + $iterator = $this->db->getIterator(); + while (yield $iterator->advance()) { + [$id, $origin] = $iterator->getCurrent(); + if (!isset($origin['peer']) || $origin['peer'] === $id) { + $this->db->offsetUnset($id); + } } - } + }); } public function getMethodCallbacks(): array { @@ -202,7 +224,7 @@ class MinDatabase implements TLCallback return $object; } $id = $this->API->getId($object); - if (isset($this->db[$id])) { + if (yield $this->db[$id]) { $new = \array_merge($object, $this->db[$id]); $new['_'] .= 'FromMessage'; $new['peer'] = (yield from $this->API->getInfo($new['peer']))['InputPeer']; @@ -215,16 +237,17 @@ class MinDatabase implements TLCallback $this->API->logger->logger("Don't have origin info with min peer {$id}, this may fail"); return $object; } + /** * Check if location info is available for peer. * * @param float|int $id Peer ID * - * @return boolean + * @return boolean */ - public function hasPeer($id): bool + public function hasPeer($id): Promise { - return isset($this->db[$id]); + return $this->db->isset($id); } public function __debugInfo() { diff --git a/src/danog/MadelineProto/MTProtoTools/PeerHandler.php b/src/danog/MadelineProto/MTProtoTools/PeerHandler.php index 9ecbec28..68c1b8c2 100644 --- a/src/danog/MadelineProto/MTProtoTools/PeerHandler.php +++ b/src/danog/MadelineProto/MTProtoTools/PeerHandler.php @@ -518,7 +518,7 @@ trait PeerHandler } } if (yield $this->chats[$id]) { - if (((yield $this->chats[$id])['min'] ?? false) && $this->minDatabase->hasPeer($id) && !isset($this->caching_full_info[$id])) { + if (((yield $this->chats[$id])['min'] ?? false) && yield $this->minDatabase->hasPeer($id) && !isset($this->caching_full_info[$id])) { $this->caching_full_info[$id] = true; $this->logger->logger("Only have min peer for {$id} in database, trying to fetch full info"); try { diff --git a/src/danog/MadelineProto/MTProtoTools/ReferenceDatabase.php b/src/danog/MadelineProto/MTProtoTools/ReferenceDatabase.php index caeaeacc..3c815f42 100644 --- a/src/danog/MadelineProto/MTProtoTools/ReferenceDatabase.php +++ b/src/danog/MadelineProto/MTProtoTools/ReferenceDatabase.php @@ -19,15 +19,21 @@ namespace danog\MadelineProto\MTProtoTools; +use Amp\Loop; +use Amp\Promise; +use danog\MadelineProto\Db\DbArray; +use danog\MadelineProto\Db\DbPropertiesTrait; use danog\MadelineProto\Exception; use danog\MadelineProto\MTProto; use danog\MadelineProto\TL\TLCallback; +use danog\MadelineProto\Tools; /** * Manages upload and download of files. */ class ReferenceDatabase implements TLCallback { + use DbPropertiesTrait; // Reference from a document const DOCUMENT_LOCATION = 0; @@ -65,15 +71,25 @@ class ReferenceDatabase implements TLCallback /** * References indexed by location. * - * @var array + * @var DbArray|Promise[] */ - private $db = []; + private $db; private $cache = []; private $cacheContexts = []; private $refreshed = []; private $API; private $refresh = false; private $refreshCount = 0; + + /** + * List of properties stored in database (memory or external). + * @see DbPropertiesFabric + * @var array + */ + protected array $dbProperies = [ + 'db' => 'array', + ]; + public function __construct(MTProto $API) { $this->API = $API; @@ -89,6 +105,7 @@ class ReferenceDatabase implements TLCallback } public function init() { + Tools::wait($this->initDb($this->API)); } public function getMethodCallbacks(): array { @@ -355,11 +372,14 @@ class ReferenceDatabase implements TLCallback } public function storeReference(string $location, string $reference, int $originType, array $origin) { - if (!isset($this->db[$location])) { - $this->db[$location] = ['origins' => []]; + $locationValue = Tools::wait($this->db[$location]); + if (!$locationValue) { + $locationValue = ['origins' => []]; } - $this->db[$location]['reference'] = $reference; - $this->db[$location]['origins'][$originType] = $origin; + $locationValue['reference'] = $reference; + $locationValue['origins'][$originType] = $origin; + $this->db[$location] = $locationValue; + if ($this->refresh) { $this->refreshed[$location] = true; } @@ -393,11 +413,13 @@ class ReferenceDatabase implements TLCallback { if (isset($this->refreshed[$location])) { $this->API->logger->logger('Reference already refreshed!', \danog\MadelineProto\Logger::VERBOSE); - return $this->db[$location]['reference']; + return (yield $this->db[$location])['reference']; } - \ksort($this->db[$location]['origins']); + $locationValue = yield $this->db[$location]; + \ksort($locationValue['origins']); + $this->db[$location] = $locationValue; $count = 0; - foreach ($this->db[$location]['origins'] as $originType => &$origin) { + foreach ((yield $this->db[$location]['origins']) as $originType => &$origin) { $count++; $this->API->logger->logger("Try {$count} refreshing file reference with origin type {$originType}", \danog\MadelineProto\Logger::VERBOSE); switch ($originType) { @@ -447,7 +469,7 @@ class ReferenceDatabase implements TLCallback throw new \danog\MadelineProto\Exception("Unknown origin type {$originType}"); } if (isset($this->refreshed[$location])) { - return $this->db[$location]['reference']; + return (yield $this->db[$location])['reference']; } } throw new Exception('Did not refresh reference'); @@ -460,7 +482,7 @@ class ReferenceDatabase implements TLCallback public function getReference(int $locationType, array $location) { $locationString = $this->serializeLocation($locationType, $location); - if (!isset($this->db[$locationString]['reference'])) { + if (!isset((yield $this->db[$locationString])['reference'])) { if (isset($location['file_reference'])) { $this->API->logger->logger("Using outdated file reference for location of type {$locationType} object {$location['_']}", \danog\MadelineProto\Logger::ULTRA_VERBOSE); return $location['file_reference']; @@ -475,7 +497,7 @@ class ReferenceDatabase implements TLCallback if ($this->refresh) { return $this->refreshReferenceInternal($locationString); } - return $this->db[$locationString]['reference']; + return (yield $this->db[$locationString])['reference']; } private function serializeLocation(int $locationType, array $location) {