DbArray refactoring and convertation improvement

This commit is contained in:
Alexander Pankratov 2020-06-07 01:00:44 +03:00
parent e0832390ce
commit 1e23970ab3
6 changed files with 96 additions and 82 deletions

View File

@ -7,7 +7,7 @@ use Amp\Promise;
interface DbArray extends DbType, \ArrayAccess, \Countable interface DbArray extends DbType, \ArrayAccess, \Countable
{ {
public function getArrayCopy(): array; public function getArrayCopy(): Promise;
public function offsetExists($offset): Promise; public function offsetExists($offset): Promise;
public function offsetGet($offset): Promise; public function offsetGet($offset): Promise;
public function offsetSet($offset, $value); public function offsetSet($offset, $value);

View File

@ -3,12 +3,12 @@
namespace danog\MadelineProto\Db; namespace danog\MadelineProto\Db;
use Amp\Promise; use Amp\Promise;
use danog\MadelineProto\MTProto;
class DbPropertiesFabric class DbPropertiesFabric
{ {
/** /**
* @param MTProto $madelineProto * @param array $dbSettings
* @param string $namePrefix
* @param string $propertyType * @param string $propertyType
* @param string $name * @param string $name
* @param $value * @param $value
@ -19,10 +19,10 @@ class DbPropertiesFabric
* @uses \danog\MadelineProto\Db\SharedMemoryArray * @uses \danog\MadelineProto\Db\SharedMemoryArray
* @uses \danog\MadelineProto\Db\MysqlArray * @uses \danog\MadelineProto\Db\MysqlArray
*/ */
public static function get(MTProto $madelineProto, string $propertyType, string $name, $value = null): Promise public static function get(array $dbSettings, string $namePrefix, string $propertyType, string $name, $value = null): Promise
{ {
$class = __NAMESPACE__; $class = __NAMESPACE__;
$dbSettings = $madelineProto->settings['db'];
switch (strtolower($dbSettings['type'])) { switch (strtolower($dbSettings['type'])) {
case 'memory': case 'memory':
$class .= '\Memory'; $class .= '\Memory';
@ -44,18 +44,7 @@ class DbPropertiesFabric
throw new \InvalidArgumentException("Unknown $propertyType: {$propertyType}"); throw new \InvalidArgumentException("Unknown $propertyType: {$propertyType}");
} }
$prefix = static::getSessionId($madelineProto); return $class::getInstance($name, $value, $namePrefix, $dbSettings[$dbSettings['type']]??[]);
return $class::getInstance($name, $value, $prefix, $dbSettings[$dbSettings['type']]??[]);
}
private static function getSessionId(MTProto $madelineProto): string
{
$result = $madelineProto->getSelf()['id'] ?? null;
if (!$result) {
$result = 'tmp_';
$result .= str_replace('0','', spl_object_hash($madelineProto));
}
return (string) $result;
} }
} }

View File

@ -0,0 +1,52 @@
<?php
namespace danog\MadelineProto\Db;
use danog\MadelineProto\Logger;
use danog\MadelineProto\MTProto;
trait DbPropertiesTrait
{
public function initDb(MTProto $MadelineProto, bool $reset = false): \Generator
{
if (empty($this->dbProperies)) {
throw new \LogicException(__CLASS__ . ' must have a $dbProperies');
}
$dbSettings = $MadelineProto->settings['db'];
$prefix = static::getSessionId($MadelineProto);
foreach ($this->dbProperies as $property => $type) {
if ($reset) {
unset($this->{$property});
} else {
$this->{$property} = yield DbPropertiesFabric::get($dbSettings, $prefix, $type, $property, $this->{$property});
}
}
if (!$reset && yield $this->usernames->count() === 0) {
$this->logger('Filling database cache. This can take few minutes.', Logger::WARNING);
$iterator = $this->chats->getIterator();
while (yield $iterator->advance()) {
[$id, $chat] = $iterator->getCurrent();
if (isset($chat['username'])) {
$this->usernames[\strtolower($chat['username'])] = $this->getId($chat);
}
}
$this->logger('Cache filled.', Logger::WARNING);
}
}
private static function getSessionId(MTProto $madelineProto): string
{
$result = $madelineProto->getSelf()['id'] ?? null;
if (!$result) {
$result = 'tmp_';
$result .= str_replace('0','', spl_object_hash($madelineProto));
}
$className = explode('\\',__CLASS__);
$result .= '_' . end($className);
return $result;
}
}

View File

@ -4,6 +4,7 @@ namespace danog\MadelineProto\Db;
use Amp\Producer; use Amp\Producer;
use Amp\Promise; use Amp\Promise;
use danog\MadelineProto\Logger;
use function Amp\call; use function Amp\call;
class MemoryArray extends \ArrayIterator implements DbArray class MemoryArray extends \ArrayIterator implements DbArray
@ -15,9 +16,13 @@ class MemoryArray extends \ArrayIterator implements DbArray
public static function getInstance(string $name, $value = null, string $tablePrefix = '', array $settings = []): Promise public static function getInstance(string $name, $value = null, string $tablePrefix = '', array $settings = []): Promise
{ {
return call(function() use ($value) { return call(static function() use ($value) {
if ($value instanceof MemoryArray) {
return $value;
}
if ($value instanceof DbArray) { if ($value instanceof DbArray) {
$value = $value->getArrayCopy(); Logger::log("Loading database to memory. Please wait.", Logger::WARNING);
$value = yield $value->getArrayCopy();
} }
return new static($value); return new static($value);
}); });
@ -43,9 +48,9 @@ class MemoryArray extends \ArrayIterator implements DbArray
return call(fn() => parent::count()); return call(fn() => parent::count());
} }
public function getArrayCopy(): array public function getArrayCopy(): Promise
{ {
return parent::getArrayCopy(); return call(fn() => parent::getArrayCopy());
} }
public function getIterator(): Producer public function getIterator(): Producer

View File

@ -60,7 +60,7 @@ class MysqlArray implements DbArray
return call(static function() use($instance, $value) { return call(static function() use($instance, $value) {
yield from static::renameTmpTable($instance, $value); yield from static::renameTmpTable($instance, $value);
yield from $instance->prepareTable(); yield from $instance->prepareTable();
Loop::defer(fn() => static::migrateDataToDb($instance, $value)); yield from static::migrateDataToDb($instance, $value);
return $instance; return $instance;
}); });
@ -76,11 +76,11 @@ class MysqlArray implements DbArray
{ {
if ($value instanceof static && $value->table) { if ($value instanceof static && $value->table) {
if ( if (
mb_strpos($value->table, 'tmp') === 0 && $value->table !== $instance->table &&
mb_strpos($instance->table, 'tmp') !== 0 mb_strpos($instance->table, 'tmp') !== 0
) { ) {
yield from $instance->renameTable($value->table, $instance->table); yield from $instance->renameTable($value->table, $instance->table);
} elseif (mb_strpos($instance->table, 'tmp') === 0) { } else {
$instance->table = $value->table; $instance->table = $value->table;
} }
} }
@ -95,17 +95,21 @@ class MysqlArray implements DbArray
*/ */
private static function migrateDataToDb(MysqlArray $instance, $value): \Generator private static function migrateDataToDb(MysqlArray $instance, $value): \Generator
{ {
if (!empty($value) && !$value instanceof static) { if (!empty($value) && !$value instanceof MysqlArray) {
Logger::log('Converting database.', Logger::ERROR); Logger::log('Converting database.', Logger::ERROR);
$value = (array) $value; if ($value instanceof DbArray) {
$value = yield $value->getArrayCopy();
} else {
$value = (array) $value;
}
$counter = 0; $counter = 0;
$total = count($value); $total = count($value);
foreach ((array) $value as $key => $item) { foreach ($value as $key => $item) {
$counter++; $counter++;
if ($counter % 100 === 0) { if ($counter % 100 === 0) {
yield $instance->offsetSet($key, $item); yield $instance->offsetSet($key, $item);
Logger::log("Converting database. $counter/$total", Logger::WARNING); Logger::log("Loading data to table {$instance->table}: $counter/$total", Logger::WARNING);
} else { } else {
$instance->offsetSet($key, $item); $instance->offsetSet($key, $item);
} }
@ -212,20 +216,20 @@ class MysqlArray implements DbArray
/** /**
* Get array copy * Get array copy
* *
* @link https://php.net/manual/en/arrayiterator.getarraycopy.php * @return Promise<array>
* @return array A copy of the array, or array of public properties
* if ArrayIterator refers to an object.
* @throws \Throwable * @throws \Throwable
*/ */
public function getArrayCopy(): array public function getArrayCopy(): Promise
{ {
$rows = $this->syncRequest("SELECT `key`, `value` FROM `{$this->table}`"); return call(function(){
$result = []; $iterator = $this->getIterator();
foreach ($rows as $row) { $result = [];
$result[$row['key']] = $this->getValue($row); while (yield $iterator->advance()) {
} [$key, $value] = $iterator->getCurrent();
$result[$key] = $value;
return $result; }
return $result;
});
} }
public function getIterator(): Producer public function getIterator(): Producer
@ -235,7 +239,6 @@ class MysqlArray implements DbArray
while (yield $request->advance()) { while (yield $request->advance()) {
$row = $request->getCurrent(); $row = $request->getCurrent();
yield $emit([$row['key'], $this->getValue($row)]); yield $emit([$row['key'], $this->getValue($row)]);
} }
}); });
@ -317,21 +320,7 @@ class MysqlArray implements DbArray
} }
/** /**
* Perform blocking request to db * Perform async request to db
*
* @param string $query
* @param array $params
*
* @return array|null
* @throws \Throwable
*/
private function syncRequest(string $query, array $params = []): array
{
return wait($this->request($query, $params));
}
/**
* Perform blocking request to db
* *
* @param string $query * @param string $query
* @param array $params * @param array $params

View File

@ -26,6 +26,7 @@ use Amp\Promise;
use danog\MadelineProto\Async\AsyncConstruct; use danog\MadelineProto\Async\AsyncConstruct;
use danog\MadelineProto\Db\DbArray; use danog\MadelineProto\Db\DbArray;
use danog\MadelineProto\Db\DbPropertiesFabric; use danog\MadelineProto\Db\DbPropertiesFabric;
use danog\MadelineProto\Db\DbPropertiesTrait;
use danog\MadelineProto\Db\Mysql; use danog\MadelineProto\Db\Mysql;
use danog\MadelineProto\Loop\Generic\PeriodicLoop; use danog\MadelineProto\Loop\Generic\PeriodicLoop;
use danog\MadelineProto\Loop\Update\FeedLoop; use danog\MadelineProto\Loop\Update\FeedLoop;
@ -72,6 +73,7 @@ class MTProto extends AsyncConstruct implements TLCallback
use \danog\MadelineProto\Wrappers\Start; use \danog\MadelineProto\Wrappers\Start;
use \danog\MadelineProto\Wrappers\Templates; use \danog\MadelineProto\Wrappers\Templates;
use \danog\MadelineProto\Wrappers\TOS; use \danog\MadelineProto\Wrappers\TOS;
use DbPropertiesTrait;
/** /**
* Old internal version of MadelineProto. * Old internal version of MadelineProto.
* *
@ -424,7 +426,7 @@ class MTProto extends AsyncConstruct implements TLCallback
* @see DbPropertiesFabric * @see DbPropertiesFabric
* @var array * @var array
*/ */
private array $dbProperies = [ protected array $dbProperies = [
'chats' => 'array', 'chats' => 'array',
'full_chats' => 'array', 'full_chats' => 'array',
'channel_participants' => 'array', 'channel_participants' => 'array',
@ -566,29 +568,6 @@ class MTProto extends AsyncConstruct implements TLCallback
]; ];
} }
public function initDb(bool $reset = false): \Generator
{
foreach ($this->dbProperies as $property => $type) {
if ($reset) {
unset($this->{$property});
} else {
$this->{$property} = yield DbPropertiesFabric::get($this, $type, $property, $this->{$property});
}
}
if (!$reset && yield $this->usernames->count() === 0) {
$this->logger('Filling database cache. This can take few minutes.', Logger::WARNING);
$iterator = $this->chats->getIterator();
while (yield $iterator->advance()) {
[$id, $chat] = $iterator->getCurrent();
if (isset($chat['username'])) {
$this->usernames[\strtolower($chat['username'])] = $this->getId($chat);
}
}
$this->logger('Cache filled.', Logger::WARNING);
}
}
/** /**
* Cleanup memory and session file. * Cleanup memory and session file.
* *
@ -801,7 +780,7 @@ class MTProto extends AsyncConstruct implements TLCallback
$this->TL->init($this->settings['tl_schema']['src'], $callbacks); $this->TL->init($this->settings['tl_schema']['src'], $callbacks);
} }
yield from $this->initDb(); yield from $this->initDb($this);
} }
@ -834,7 +813,7 @@ class MTProto extends AsyncConstruct implements TLCallback
unset($settings['authorization']['rsa_key']); unset($settings['authorization']['rsa_key']);
} }
yield from $this->initDb(); yield from $this->initDb($this);
if (!isset($this->secret_chats)) { if (!isset($this->secret_chats)) {
$this->secret_chats = []; $this->secret_chats = [];
@ -1552,7 +1531,7 @@ class MTProto extends AsyncConstruct implements TLCallback
$this->updates = []; $this->updates = [];
$this->secret_chats = []; $this->secret_chats = [];
yield from $this->initDb(true); yield from $this->initDb($this,true);
$this->tos = ['expires' => 0, 'accepted' => true]; $this->tos = ['expires' => 0, 'accepted' => true];
$this->referenceDatabase = new ReferenceDatabase($this); $this->referenceDatabase = new ReferenceDatabase($this);