From 1e23970ab3e4413bb79a10885482c38ee2f3a28c Mon Sep 17 00:00:00 2001 From: Alexander Pankratov Date: Sun, 7 Jun 2020 01:00:44 +0300 Subject: [PATCH] DbArray refactoring and convertation improvement --- src/danog/MadelineProto/Db/DbArray.php | 2 +- .../MadelineProto/Db/DbPropertiesFabric.php | 21 ++----- .../MadelineProto/Db/DbPropertiesTrait.php | 52 +++++++++++++++++ src/danog/MadelineProto/Db/MemoryArray.php | 13 +++-- src/danog/MadelineProto/Db/MysqlArray.php | 57 ++++++++----------- src/danog/MadelineProto/MTProto.php | 33 ++--------- 6 files changed, 96 insertions(+), 82 deletions(-) create mode 100644 src/danog/MadelineProto/Db/DbPropertiesTrait.php diff --git a/src/danog/MadelineProto/Db/DbArray.php b/src/danog/MadelineProto/Db/DbArray.php index edfaf2b3..b7d68586 100644 --- a/src/danog/MadelineProto/Db/DbArray.php +++ b/src/danog/MadelineProto/Db/DbArray.php @@ -7,7 +7,7 @@ use Amp\Promise; interface DbArray extends DbType, \ArrayAccess, \Countable { - public function getArrayCopy(): array; + public function getArrayCopy(): Promise; public function offsetExists($offset): Promise; public function offsetGet($offset): Promise; public function offsetSet($offset, $value); diff --git a/src/danog/MadelineProto/Db/DbPropertiesFabric.php b/src/danog/MadelineProto/Db/DbPropertiesFabric.php index 40c75d92..7e85f5db 100644 --- a/src/danog/MadelineProto/Db/DbPropertiesFabric.php +++ b/src/danog/MadelineProto/Db/DbPropertiesFabric.php @@ -3,12 +3,12 @@ namespace danog\MadelineProto\Db; use Amp\Promise; -use danog\MadelineProto\MTProto; class DbPropertiesFabric { /** - * @param MTProto $madelineProto + * @param array $dbSettings + * @param string $namePrefix * @param string $propertyType * @param string $name * @param $value @@ -19,10 +19,10 @@ class DbPropertiesFabric * @uses \danog\MadelineProto\Db\SharedMemoryArray * @uses \danog\MadelineProto\Db\MysqlArray */ - public static function get(MTProto $madelineProto, string $propertyType, string $name, $value = null): Promise + public static function get(array $dbSettings, string $namePrefix, string $propertyType, string $name, $value = null): Promise { $class = __NAMESPACE__; - $dbSettings = $madelineProto->settings['db']; + switch (strtolower($dbSettings['type'])) { case 'memory': $class .= '\Memory'; @@ -44,18 +44,7 @@ class DbPropertiesFabric throw new \InvalidArgumentException("Unknown $propertyType: {$propertyType}"); } - $prefix = static::getSessionId($madelineProto); - return $class::getInstance($name, $value, $prefix, $dbSettings[$dbSettings['type']]??[]); - } - - private static function getSessionId(MTProto $madelineProto): string - { - $result = $madelineProto->getSelf()['id'] ?? null; - if (!$result) { - $result = 'tmp_'; - $result .= str_replace('0','', spl_object_hash($madelineProto)); - } - return (string) $result; + return $class::getInstance($name, $value, $namePrefix, $dbSettings[$dbSettings['type']]??[]); } } \ No newline at end of file diff --git a/src/danog/MadelineProto/Db/DbPropertiesTrait.php b/src/danog/MadelineProto/Db/DbPropertiesTrait.php new file mode 100644 index 00000000..4c5a6f33 --- /dev/null +++ b/src/danog/MadelineProto/Db/DbPropertiesTrait.php @@ -0,0 +1,52 @@ +dbProperies)) { + throw new \LogicException(__CLASS__ . ' must have a $dbProperies'); + } + $dbSettings = $MadelineProto->settings['db']; + $prefix = static::getSessionId($MadelineProto); + + foreach ($this->dbProperies as $property => $type) { + if ($reset) { + unset($this->{$property}); + } else { + $this->{$property} = yield DbPropertiesFabric::get($dbSettings, $prefix, $type, $property, $this->{$property}); + } + } + + if (!$reset && yield $this->usernames->count() === 0) { + $this->logger('Filling database cache. This can take few minutes.', Logger::WARNING); + $iterator = $this->chats->getIterator(); + while (yield $iterator->advance()) { + [$id, $chat] = $iterator->getCurrent(); + if (isset($chat['username'])) { + $this->usernames[\strtolower($chat['username'])] = $this->getId($chat); + } + } + $this->logger('Cache filled.', Logger::WARNING); + } + } + + private static function getSessionId(MTProto $madelineProto): string + { + $result = $madelineProto->getSelf()['id'] ?? null; + if (!$result) { + $result = 'tmp_'; + $result .= str_replace('0','', spl_object_hash($madelineProto)); + } + + $className = explode('\\',__CLASS__); + $result .= '_' . end($className); + return $result; + } +} \ No newline at end of file diff --git a/src/danog/MadelineProto/Db/MemoryArray.php b/src/danog/MadelineProto/Db/MemoryArray.php index a7f65584..24f319c3 100644 --- a/src/danog/MadelineProto/Db/MemoryArray.php +++ b/src/danog/MadelineProto/Db/MemoryArray.php @@ -4,6 +4,7 @@ namespace danog\MadelineProto\Db; use Amp\Producer; use Amp\Promise; +use danog\MadelineProto\Logger; use function Amp\call; class MemoryArray extends \ArrayIterator implements DbArray @@ -15,9 +16,13 @@ class MemoryArray extends \ArrayIterator implements DbArray public static function getInstance(string $name, $value = null, string $tablePrefix = '', array $settings = []): Promise { - return call(function() use ($value) { + return call(static function() use ($value) { + if ($value instanceof MemoryArray) { + return $value; + } if ($value instanceof DbArray) { - $value = $value->getArrayCopy(); + Logger::log("Loading database to memory. Please wait.", Logger::WARNING); + $value = yield $value->getArrayCopy(); } return new static($value); }); @@ -43,9 +48,9 @@ class MemoryArray extends \ArrayIterator implements DbArray return call(fn() => parent::count()); } - public function getArrayCopy(): array + public function getArrayCopy(): Promise { - return parent::getArrayCopy(); + return call(fn() => parent::getArrayCopy()); } public function getIterator(): Producer diff --git a/src/danog/MadelineProto/Db/MysqlArray.php b/src/danog/MadelineProto/Db/MysqlArray.php index 801a680b..8c842a7a 100644 --- a/src/danog/MadelineProto/Db/MysqlArray.php +++ b/src/danog/MadelineProto/Db/MysqlArray.php @@ -60,7 +60,7 @@ class MysqlArray implements DbArray return call(static function() use($instance, $value) { yield from static::renameTmpTable($instance, $value); yield from $instance->prepareTable(); - Loop::defer(fn() => static::migrateDataToDb($instance, $value)); + yield from static::migrateDataToDb($instance, $value); return $instance; }); @@ -76,11 +76,11 @@ class MysqlArray implements DbArray { if ($value instanceof static && $value->table) { if ( - mb_strpos($value->table, 'tmp') === 0 && + $value->table !== $instance->table && mb_strpos($instance->table, 'tmp') !== 0 ) { yield from $instance->renameTable($value->table, $instance->table); - } elseif (mb_strpos($instance->table, 'tmp') === 0) { + } else { $instance->table = $value->table; } } @@ -95,17 +95,21 @@ class MysqlArray implements DbArray */ private static function migrateDataToDb(MysqlArray $instance, $value): \Generator { - if (!empty($value) && !$value instanceof static) { + if (!empty($value) && !$value instanceof MysqlArray) { Logger::log('Converting database.', Logger::ERROR); - $value = (array) $value; + if ($value instanceof DbArray) { + $value = yield $value->getArrayCopy(); + } else { + $value = (array) $value; + } $counter = 0; $total = count($value); - foreach ((array) $value as $key => $item) { + foreach ($value as $key => $item) { $counter++; if ($counter % 100 === 0) { yield $instance->offsetSet($key, $item); - Logger::log("Converting database. $counter/$total", Logger::WARNING); + Logger::log("Loading data to table {$instance->table}: $counter/$total", Logger::WARNING); } else { $instance->offsetSet($key, $item); } @@ -212,20 +216,20 @@ class MysqlArray implements DbArray /** * Get array copy * - * @link https://php.net/manual/en/arrayiterator.getarraycopy.php - * @return array A copy of the array, or array of public properties - * if ArrayIterator refers to an object. + * @return Promise * @throws \Throwable */ - public function getArrayCopy(): array + public function getArrayCopy(): Promise { - $rows = $this->syncRequest("SELECT `key`, `value` FROM `{$this->table}`"); - $result = []; - foreach ($rows as $row) { - $result[$row['key']] = $this->getValue($row); - } - - return $result; + return call(function(){ + $iterator = $this->getIterator(); + $result = []; + while (yield $iterator->advance()) { + [$key, $value] = $iterator->getCurrent(); + $result[$key] = $value; + } + return $result; + }); } public function getIterator(): Producer @@ -235,7 +239,6 @@ class MysqlArray implements DbArray while (yield $request->advance()) { $row = $request->getCurrent(); - yield $emit([$row['key'], $this->getValue($row)]); } }); @@ -317,21 +320,7 @@ class MysqlArray implements DbArray } /** - * Perform blocking request to db - * - * @param string $query - * @param array $params - * - * @return array|null - * @throws \Throwable - */ - private function syncRequest(string $query, array $params = []): array - { - return wait($this->request($query, $params)); - } - - /** - * Perform blocking request to db + * Perform async request to db * * @param string $query * @param array $params diff --git a/src/danog/MadelineProto/MTProto.php b/src/danog/MadelineProto/MTProto.php index 8656aaa9..aeb69538 100644 --- a/src/danog/MadelineProto/MTProto.php +++ b/src/danog/MadelineProto/MTProto.php @@ -26,6 +26,7 @@ use Amp\Promise; use danog\MadelineProto\Async\AsyncConstruct; use danog\MadelineProto\Db\DbArray; use danog\MadelineProto\Db\DbPropertiesFabric; +use danog\MadelineProto\Db\DbPropertiesTrait; use danog\MadelineProto\Db\Mysql; use danog\MadelineProto\Loop\Generic\PeriodicLoop; use danog\MadelineProto\Loop\Update\FeedLoop; @@ -72,6 +73,7 @@ class MTProto extends AsyncConstruct implements TLCallback use \danog\MadelineProto\Wrappers\Start; use \danog\MadelineProto\Wrappers\Templates; use \danog\MadelineProto\Wrappers\TOS; + use DbPropertiesTrait; /** * Old internal version of MadelineProto. * @@ -424,7 +426,7 @@ class MTProto extends AsyncConstruct implements TLCallback * @see DbPropertiesFabric * @var array */ - private array $dbProperies = [ + protected array $dbProperies = [ 'chats' => 'array', 'full_chats' => 'array', 'channel_participants' => 'array', @@ -566,29 +568,6 @@ class MTProto extends AsyncConstruct implements TLCallback ]; } - public function initDb(bool $reset = false): \Generator - { - foreach ($this->dbProperies as $property => $type) { - if ($reset) { - unset($this->{$property}); - } else { - $this->{$property} = yield DbPropertiesFabric::get($this, $type, $property, $this->{$property}); - } - } - - if (!$reset && yield $this->usernames->count() === 0) { - $this->logger('Filling database cache. This can take few minutes.', Logger::WARNING); - $iterator = $this->chats->getIterator(); - while (yield $iterator->advance()) { - [$id, $chat] = $iterator->getCurrent(); - if (isset($chat['username'])) { - $this->usernames[\strtolower($chat['username'])] = $this->getId($chat); - } - } - $this->logger('Cache filled.', Logger::WARNING); - } - } - /** * Cleanup memory and session file. * @@ -801,7 +780,7 @@ class MTProto extends AsyncConstruct implements TLCallback $this->TL->init($this->settings['tl_schema']['src'], $callbacks); } - yield from $this->initDb(); + yield from $this->initDb($this); } @@ -834,7 +813,7 @@ class MTProto extends AsyncConstruct implements TLCallback unset($settings['authorization']['rsa_key']); } - yield from $this->initDb(); + yield from $this->initDb($this); if (!isset($this->secret_chats)) { $this->secret_chats = []; @@ -1552,7 +1531,7 @@ class MTProto extends AsyncConstruct implements TLCallback $this->updates = []; $this->secret_chats = []; - yield from $this->initDb(true); + yield from $this->initDb($this,true); $this->tos = ['expires' => 0, 'accepted' => true]; $this->referenceDatabase = new ReferenceDatabase($this);