danog
/
MadelineProto
2
2
Fork 0
Code Issues Pull Requests Releases Wiki Activity

DbArray refactoring and convertation improvement

This commit is contained in:
Alexander Pankratov 2020-06-07 01:00:44 +03:00
parent e0832390ce
commit 1e23970ab3
6 changed files with 96 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/danog/MadelineProto/Db/DbArray.php
View File

@ -7,7 +7,7 @@ use Amp\Promise;
interface DbArray extends DbType, \ArrayAccess, \Countable
{
public function getArrayCopy(): array;
public function getArrayCopy(): Promise;
public function offsetExists($offset): Promise;
public function offsetGet($offset): Promise;
public function offsetSet($offset, $value);

21
src/danog/MadelineProto/Db/DbPropertiesFabric.php
View File

@ -3,12 +3,12 @@
namespace danog\MadelineProto\Db;
use Amp\Promise;
use danog\MadelineProto\MTProto;
class DbPropertiesFabric
{
/**
* @param MTProto $madelineProto
* @param array $dbSettings
* @param string $namePrefix
* @param string $propertyType
* @param string $name
* @param $value
@ -19,10 +19,10 @@ class DbPropertiesFabric
* @uses \danog\MadelineProto\Db\SharedMemoryArray
* @uses \danog\MadelineProto\Db\MysqlArray
*/
public static function get(MTProto $madelineProto, string $propertyType, string $name, $value = null): Promise
public static function get(array $dbSettings, string $namePrefix, string $propertyType, string $name, $value = null): Promise
{
$class = __NAMESPACE__;
$dbSettings = $madelineProto->settings['db'];
switch (strtolower($dbSettings['type'])) {
case 'memory':
$class .= '\Memory';
@ -44,18 +44,7 @@ class DbPropertiesFabric
throw new \InvalidArgumentException("Unknown $propertyType: {$propertyType}");
}
$prefix = static::getSessionId($madelineProto);
return $class::getInstance($name, $value, $prefix, $dbSettings[$dbSettings['type']]??[]);
}
private static function getSessionId(MTProto $madelineProto): string
{
$result = $madelineProto->getSelf()['id'] ?? null;
if (!$result) {
$result = 'tmp_';
$result .= str_replace('0','', spl_object_hash($madelineProto));
}
return (string) $result;
return $class::getInstance($name, $value, $namePrefix, $dbSettings[$dbSettings['type']]??[]);
}
}

52
src/danog/MadelineProto/Db/DbPropertiesTrait.php Normal file
View File

@ -0,0 +1,52 @@
<?php
namespace danog\MadelineProto\Db;
use danog\MadelineProto\Logger;
use danog\MadelineProto\MTProto;
trait DbPropertiesTrait
{
public function initDb(MTProto $MadelineProto, bool $reset = false): \Generator
{
if (empty($this->dbProperies)) {
throw new \LogicException(__CLASS__ . ' must have a $dbProperies');
}
$dbSettings = $MadelineProto->settings['db'];
$prefix = static::getSessionId($MadelineProto);
foreach ($this->dbProperies as $property => $type) {
if ($reset) {
unset($this->{$property});
} else {
$this->{$property} = yield DbPropertiesFabric::get($dbSettings, $prefix, $type, $property, $this->{$property});
}
}
if (!$reset && yield $this->usernames->count() === 0) {
$this->logger('Filling database cache. This can take few minutes.', Logger::WARNING);
$iterator = $this->chats->getIterator();
while (yield $iterator->advance()) {
[$id, $chat] = $iterator->getCurrent();
if (isset($chat['username'])) {
$this->usernames[\strtolower($chat['username'])] = $this->getId($chat);
}
}
$this->logger('Cache filled.', Logger::WARNING);
}
}
private static function getSessionId(MTProto $madelineProto): string
{
$result = $madelineProto->getSelf()['id'] ?? null;
if (!$result) {
$result = 'tmp_';
$result .= str_replace('0','', spl_object_hash($madelineProto));
}
$className = explode('\\',__CLASS__);
$result .= '_' . end($className);
return $result;
}
}

13
src/danog/MadelineProto/Db/MemoryArray.php
View File

@ -4,6 +4,7 @@ namespace danog\MadelineProto\Db;
use Amp\Producer;
use Amp\Promise;
use danog\MadelineProto\Logger;
use function Amp\call;
class MemoryArray extends \ArrayIterator implements DbArray
@ -15,9 +16,13 @@ class MemoryArray extends \ArrayIterator implements DbArray
public static function getInstance(string $name, $value = null, string $tablePrefix = '', array $settings = []): Promise
{
return call(function() use ($value) {
return call(static function() use ($value) {
if ($value instanceof MemoryArray) {
return $value;
}
if ($value instanceof DbArray) {
$value = $value->getArrayCopy();
Logger::log("Loading database to memory. Please wait.", Logger::WARNING);
$value = yield $value->getArrayCopy();
}
return new static($value);
});
@ -43,9 +48,9 @@ class MemoryArray extends \ArrayIterator implements DbArray
return call(fn() => parent::count());
}
public function getArrayCopy(): array
public function getArrayCopy(): Promise
{
return parent::getArrayCopy();
return call(fn() => parent::getArrayCopy());
}
public function getIterator(): Producer

57
src/danog/MadelineProto/Db/MysqlArray.php
View File

@ -60,7 +60,7 @@ class MysqlArray implements DbArray
return call(static function() use($instance, $value) {
yield from static::renameTmpTable($instance, $value);
yield from $instance->prepareTable();
Loop::defer(fn() => static::migrateDataToDb($instance, $value));
yield from static::migrateDataToDb($instance, $value);
return $instance;
});
@ -76,11 +76,11 @@ class MysqlArray implements DbArray
{
if ($value instanceof static && $value->table) {
if (
mb_strpos($value->table, 'tmp') === 0 &&
$value->table !== $instance->table &&
mb_strpos($instance->table, 'tmp') !== 0
) {
yield from $instance->renameTable($value->table, $instance->table);
} elseif (mb_strpos($instance->table, 'tmp') === 0) {
} else {
$instance->table = $value->table;
}
}
@ -95,17 +95,21 @@ class MysqlArray implements DbArray
*/
private static function migrateDataToDb(MysqlArray $instance, $value): \Generator
{
if (!empty($value) && !$value instanceof static) {
if (!empty($value) && !$value instanceof MysqlArray) {
Logger::log('Converting database.', Logger::ERROR);
$value = (array) $value;
if ($value instanceof DbArray) {
$value = yield $value->getArrayCopy();
} else {
$value = (array) $value;
}
$counter = 0;
$total = count($value);
foreach ((array) $value as $key => $item) {
foreach ($value as $key => $item) {
$counter++;
if ($counter % 100 === 0) {
yield $instance->offsetSet($key, $item);
Logger::log("Converting database. $counter/$total", Logger::WARNING);
Logger::log("Loading data to table {$instance->table}: $counter/$total", Logger::WARNING);
} else {
$instance->offsetSet($key, $item);
}
@ -212,20 +216,20 @@ class MysqlArray implements DbArray
/**
* Get array copy
*
* @link https://php.net/manual/en/arrayiterator.getarraycopy.php
* @return array A copy of the array, or array of public properties
* if ArrayIterator refers to an object.
* @return Promise<array>
* @throws \Throwable
*/
public function getArrayCopy(): array
public function getArrayCopy(): Promise
{
$rows = $this->syncRequest("SELECT `key`, `value` FROM `{$this->table}`");
$result = [];
foreach ($rows as $row) {
$result[$row['key']] = $this->getValue($row);
}
return $result;
return call(function(){
$iterator = $this->getIterator();
$result = [];
while (yield $iterator->advance()) {
[$key, $value] = $iterator->getCurrent();
$result[$key] = $value;
}
return $result;
});
}
public function getIterator(): Producer
@ -235,7 +239,6 @@ class MysqlArray implements DbArray
while (yield $request->advance()) {
$row = $request->getCurrent();
yield $emit([$row['key'], $this->getValue($row)]);
}
});
@ -317,21 +320,7 @@ class MysqlArray implements DbArray
}
/**
* Perform blocking request to db
*
* @param string $query
* @param array $params
*
* @return array|null
* @throws \Throwable
*/
private function syncRequest(string $query, array $params = []): array
{
return wait($this->request($query, $params));
}
/**
* Perform blocking request to db
* Perform async request to db
*
* @param string $query
* @param array $params

33
src/danog/MadelineProto/MTProto.php
View File

@ -26,6 +26,7 @@ use Amp\Promise;
use danog\MadelineProto\Async\AsyncConstruct;
use danog\MadelineProto\Db\DbArray;
use danog\MadelineProto\Db\DbPropertiesFabric;
use danog\MadelineProto\Db\DbPropertiesTrait;
use danog\MadelineProto\Db\Mysql;
use danog\MadelineProto\Loop\Generic\PeriodicLoop;
use danog\MadelineProto\Loop\Update\FeedLoop;
@ -72,6 +73,7 @@ class MTProto extends AsyncConstruct implements TLCallback
use \danog\MadelineProto\Wrappers\Start;
use \danog\MadelineProto\Wrappers\Templates;
use \danog\MadelineProto\Wrappers\TOS;
use DbPropertiesTrait;
/**
* Old internal version of MadelineProto.
*
@ -424,7 +426,7 @@ class MTProto extends AsyncConstruct implements TLCallback
* @see DbPropertiesFabric
* @var array
*/
private array $dbProperies = [
protected array $dbProperies = [
'chats' => 'array',
'full_chats' => 'array',
'channel_participants' => 'array',
@ -566,29 +568,6 @@ class MTProto extends AsyncConstruct implements TLCallback
];
}
public function initDb(bool $reset = false): \Generator
{
foreach ($this->dbProperies as $property => $type) {
if ($reset) {
unset($this->{$property});
} else {
$this->{$property} = yield DbPropertiesFabric::get($this, $type, $property, $this->{$property});
}
}
if (!$reset && yield $this->usernames->count() === 0) {
$this->logger('Filling database cache. This can take few minutes.', Logger::WARNING);
$iterator = $this->chats->getIterator();
while (yield $iterator->advance()) {
[$id, $chat] = $iterator->getCurrent();
if (isset($chat['username'])) {
$this->usernames[\strtolower($chat['username'])] = $this->getId($chat);
}
}
$this->logger('Cache filled.', Logger::WARNING);
}
}
/**
* Cleanup memory and session file.
*
@ -801,7 +780,7 @@ class MTProto extends AsyncConstruct implements TLCallback
$this->TL->init($this->settings['tl_schema']['src'], $callbacks);
}
yield from $this->initDb();
yield from $this->initDb($this);
}
@ -834,7 +813,7 @@ class MTProto extends AsyncConstruct implements TLCallback
unset($settings['authorization']['rsa_key']);
}
yield from $this->initDb();
yield from $this->initDb($this);
if (!isset($this->secret_chats)) {
$this->secret_chats = [];
@ -1552,7 +1531,7 @@ class MTProto extends AsyncConstruct implements TLCallback
$this->updates = [];
$this->secret_chats = [];
yield from $this->initDb(true);
yield from $this->initDb($this,true);
$this->tos = ['expires' => 0, 'accepted' => true];
$this->referenceDatabase = new ReferenceDatabase($this);