Improve performance of memory backend and misc fixes

This commit is contained in:
Daniil Gentili 2020-09-12 15:14:26 +02:00
parent 1c45c6c65b
commit 5567490c7a
Signed by: danog
GPG Key ID: 8C1BE3B34B230CA7
9 changed files with 76 additions and 103 deletions

View File

@ -2,20 +2,26 @@
namespace danog\MadelineProto\Db; namespace danog\MadelineProto\Db;
use danog\MadelineProto\Logger;
use danog\MadelineProto\MTProto; use danog\MadelineProto\MTProto;
trait DbPropertiesTrait trait DbPropertiesTrait
{ {
/**
* Initialize database instance.
*
* @param MTProto $MadelineProto
* @param boolean $reset
* @return \Generator
*/
public function initDb(MTProto $MadelineProto, bool $reset = false): \Generator public function initDb(MTProto $MadelineProto, bool $reset = false): \Generator
{ {
if (empty($this->dbProperies)) { if (empty(static::$dbProperties)) {
throw new \LogicException(__CLASS__ . ' must have a $dbProperies'); throw new \LogicException(static::class.' must have $dbProperties');
} }
$dbSettings = $MadelineProto->settings['db']; $dbSettings = $MadelineProto->settings['db'];
$prefix = static::getSessionId($MadelineProto); $prefix = static::getSessionId($MadelineProto);
foreach ($this->dbProperies as $property => $type) { foreach (static::$dbProperties as $property => $type) {
if ($reset) { if ($reset) {
unset($this->{$property}); unset($this->{$property});
} else { } else {
@ -32,8 +38,8 @@ trait DbPropertiesTrait
$result .= \str_replace('0', '', \spl_object_hash($madelineProto)); $result .= \str_replace('0', '', \spl_object_hash($madelineProto));
} }
$className = \explode('\\', __CLASS__); $className = \explode('\\', static::class);
$result .= '_' . \end($className); $result .= '_'.\end($className);
return $result; return $result;
} }
} }

View File

@ -4,6 +4,7 @@ namespace danog\MadelineProto\Db;
use Amp\Producer; use Amp\Producer;
use Amp\Promise; use Amp\Promise;
use Amp\Success;
use danog\MadelineProto\Logger; use danog\MadelineProto\Logger;
use function Amp\call; use function Amp\call;
@ -35,27 +36,27 @@ class MemoryArray extends \ArrayIterator implements DbArray
public function isset($key): Promise public function isset($key): Promise
{ {
return call(fn () => parent::offsetExists($key)); return new Success(parent::offsetExists($key));
} }
public function offsetGet($offset): Promise public function offsetGet($offset): Promise
{ {
return call(fn () => parent::offsetExists($offset) ? parent::offsetGet($offset) : null); return new Success(parent::offsetExists($offset) ? parent::offsetGet($offset) : null);
} }
public function offsetUnset($offset): Promise public function offsetUnset($offset): Promise
{ {
return call(fn () => parent::offsetUnset($offset)); return new Success(parent::offsetUnset($offset));
} }
public function count(): Promise public function count(): Promise
{ {
return call(fn () => parent::count()); return new Success(parent::count());
} }
public function getArrayCopy(): Promise public function getArrayCopy(): Promise
{ {
return call(fn () => parent::getArrayCopy()); return new Success(parent::getArrayCopy());
} }
public function getIterator(): Producer public function getIterator(): Producer

View File

@ -6,14 +6,12 @@ use Amp\Mysql\ConnectionConfig;
use Amp\Mysql\Pool; use Amp\Mysql\Pool;
use Amp\Sql\Common\ConnectionPool; use Amp\Sql\Common\ConnectionPool;
use danog\MadelineProto\Logger; use danog\MadelineProto\Logger;
use function Amp\call;
use function Amp\Mysql\Pool; use function Amp\Mysql\Pool;
use function Amp\Promise\wait;
class Mysql class Mysql
{ {
/** @var Pool[] */ /** @var Pool[] */
private static array $connections; private static array $connections = [];
/** /**
* @param string $host * @param string $host
@ -25,10 +23,11 @@ class Mysql
* @param int $maxConnections * @param int $maxConnections
* @param int $idleTimeout * @param int $idleTimeout
* *
* @return Pool
* @throws \Amp\Sql\ConnectionException * @throws \Amp\Sql\ConnectionException
* @throws \Amp\Sql\FailureException * @throws \Amp\Sql\FailureException
* @throws \Throwable * @throws \Throwable
*
* @return \Generator<Pool>
*/ */
public static function getConnection( public static function getConnection(
string $host = '127.0.0.1', string $host = '127.0.0.1',
@ -38,14 +37,14 @@ class Mysql
string $db = 'MadelineProto', string $db = 'MadelineProto',
int $maxConnections = ConnectionPool::DEFAULT_MAX_CONNECTIONS, int $maxConnections = ConnectionPool::DEFAULT_MAX_CONNECTIONS,
int $idleTimeout = ConnectionPool::DEFAULT_IDLE_TIMEOUT int $idleTimeout = ConnectionPool::DEFAULT_IDLE_TIMEOUT
): Pool { ): \Generator {
$dbKey = "$host:$port:$db"; $dbKey = "$host:$port:$db";
if (empty(static::$connections[$dbKey])) { if (empty(static::$connections[$dbKey])) {
$config = ConnectionConfig::fromString( $config = ConnectionConfig::fromString(
"host={$host} port={$port} user={$user} password={$password} db={$db}" "host={$host} port={$port} user={$user} password={$password} db={$db}"
); );
static::createDb($config); yield from static::createDb($config);
static::$connections[$dbKey] = pool($config, $maxConnections, $idleTimeout); static::$connections[$dbKey] = pool($config, $maxConnections, $idleTimeout);
} }
@ -58,10 +57,11 @@ class Mysql
* @throws \Amp\Sql\ConnectionException * @throws \Amp\Sql\ConnectionException
* @throws \Amp\Sql\FailureException * @throws \Amp\Sql\FailureException
* @throws \Throwable * @throws \Throwable
*
* @return \Generator
*/ */
private static function createDb(ConnectionConfig $config) private static function createDb(ConnectionConfig $config): \Generator
{ {
wait(call(static function () use ($config) {
try { try {
$db = $config->getDatabase(); $db = $config->getDatabase();
$connection = pool($config->withDatabase(null)); $connection = pool($config->withDatabase(null));
@ -74,6 +74,5 @@ class Mysql
} catch (\Throwable $e) { } catch (\Throwable $e) {
Logger::log($e->getMessage(), Logger::ERROR); Logger::log($e->getMessage(), Logger::ERROR);
} }
}));
} }
} }

View File

@ -6,6 +6,7 @@ use Amp\Mysql\Pool;
use Amp\Producer; use Amp\Producer;
use Amp\Promise; use Amp\Promise;
use Amp\Sql\ResultSet; use Amp\Sql\ResultSet;
use Amp\Success;
use danog\MadelineProto\Logger; use danog\MadelineProto\Logger;
use function Amp\call; use function Amp\call;
@ -17,24 +18,9 @@ class MysqlArray implements DbArray
private array $settings; private array $settings;
private Pool $db; private Pool $db;
public function __serialize(): array public function __sleep(): array
{ {
return [ return ['table', 'settings'];
'table' => $this->table,
'settings' => $this->settings
];
}
public function __unserialize($data): void
{
foreach ($data as $property => $value) {
$this->{$property} = $value;
}
try {
$this->db = static::getDbConnection($this->settings);
} catch (\Throwable $e) {
Logger::log($e->getMessage(), Logger::ERROR);
}
} }
/** /**
@ -56,12 +42,12 @@ class MysqlArray implements DbArray
} }
$instance->settings = $settings; $instance->settings = $settings;
$instance->db = static::getDbConnection($settings);
$instance->ttl = $settings['cache_ttl'] ?? $instance->ttl; $instance->ttl = $settings['cache_ttl'] ?? $instance->ttl;
$instance->startCacheCleanupLoop(); $instance->startCacheCleanupLoop();
return call(static function () use ($instance, $value) { return call(static function () use ($instance, $value, $settings) {
$instance->db = yield from static::getDbConnection($settings);
yield from $instance->prepareTable(); yield from $instance->prepareTable();
//Skip migrations if its same object //Skip migrations if its same object
@ -180,7 +166,7 @@ class MysqlArray implements DbArray
public function offsetSet($index, $value): Promise public function offsetSet($index, $value): Promise
{ {
if ($this->getCache($index) === $value) { if ($this->getCache($index) === $value) {
return call(fn () =>null); return new Success();
} }
$this->setCache($index, $value); $this->setCache($index, $value);
@ -286,7 +272,7 @@ class MysqlArray implements DbArray
return null; return null;
} }
public static function getDbConnection(array $settings): Pool public static function getDbConnection(array $settings): \Generator
{ {
return Mysql::getConnection( return Mysql::getConnection(
$settings['host'], $settings['host'],
@ -355,9 +341,9 @@ class MysqlArray implements DbArray
if ( if (
!empty($params['index']) !empty($params['index'])
&& !mb_check_encoding($params['index'], 'UTF-8') && !\mb_check_encoding($params['index'], 'UTF-8')
) { ) {
$params['index'] = mb_convert_encoding($params['index'], 'UTF-8'); $params['index'] = \mb_convert_encoding($params['index'], 'UTF-8');
} }
try { try {

View File

@ -6,14 +6,12 @@ use Amp\Postgres\ConnectionConfig;
use Amp\Postgres\Pool; use Amp\Postgres\Pool;
use Amp\Sql\Common\ConnectionPool; use Amp\Sql\Common\ConnectionPool;
use danog\MadelineProto\Logger; use danog\MadelineProto\Logger;
use function Amp\call;
use function Amp\Postgres\Pool; use function Amp\Postgres\Pool;
use function Amp\Promise\wait;
class Postgres class Postgres
{ {
/** @var Pool[] */ /** @var Pool[] */
private static array $connections; private static array $connections = [];
/** /**
* @param string $host * @param string $host
@ -25,10 +23,11 @@ class Postgres
* @param int $maxConnections * @param int $maxConnections
* @param int $idleTimeout * @param int $idleTimeout
* *
* @return Pool
* @throws \Amp\Sql\ConnectionException * @throws \Amp\Sql\ConnectionException
* @throws \Amp\Sql\FailureException * @throws \Amp\Sql\FailureException
* @throws \Throwable * @throws \Throwable
*
* @return \Generator<Pool>
*/ */
public static function getConnection( public static function getConnection(
string $host = '127.0.0.1', string $host = '127.0.0.1',
@ -38,14 +37,14 @@ class Postgres
string $db = 'MadelineProto', string $db = 'MadelineProto',
int $maxConnections = ConnectionPool::DEFAULT_MAX_CONNECTIONS, int $maxConnections = ConnectionPool::DEFAULT_MAX_CONNECTIONS,
int $idleTimeout = ConnectionPool::DEFAULT_IDLE_TIMEOUT int $idleTimeout = ConnectionPool::DEFAULT_IDLE_TIMEOUT
): Pool { ): \Generator {
$dbKey = "$host:$port:$db"; $dbKey = "$host:$port:$db";
if (empty(static::$connections[$dbKey])) { if (empty(static::$connections[$dbKey])) {
$config = ConnectionConfig::fromString( $config = ConnectionConfig::fromString(
"host={$host} port={$port} user={$user} password={$password} db={$db}" "host={$host} port={$port} user={$user} password={$password} db={$db}"
); );
static::createDb($config); yield from static::createDb($config);
static::$connections[$dbKey] = pool($config, $maxConnections, $idleTimeout); static::$connections[$dbKey] = pool($config, $maxConnections, $idleTimeout);
} }
@ -59,9 +58,8 @@ class Postgres
* @throws \Amp\Sql\FailureException * @throws \Amp\Sql\FailureException
* @throws \Throwable * @throws \Throwable
*/ */
private static function createDb(ConnectionConfig $config) private static function createDb(ConnectionConfig $config): \Generator
{ {
wait(call(static function () use ($config) {
try { try {
$db = $config->getDatabase(); $db = $config->getDatabase();
$user = $config->getUser(); $user = $config->getUser();
@ -96,6 +94,5 @@ class Postgres
} catch (\Throwable $e) { } catch (\Throwable $e) {
Logger::log($e->getMessage(), Logger::ERROR); Logger::log($e->getMessage(), Logger::ERROR);
} }
}));
} }
} }

View File

@ -57,7 +57,7 @@ class PostgresArray implements DbArray
return $request; return $request;
} }
public static function getDbConnection(array $settings): Pool public static function getDbConnection(array $settings): \Generator
{ {
return Postgres::getConnection( return Postgres::getConnection(
$settings['host'], $settings['host'],
@ -103,24 +103,9 @@ class PostgresArray implements DbArray
"); ");
} }
public function __serialize(): array public function __sleep()
{ {
return [ return ['table', 'settings'];
'table' => $this->table,
'settings' => $this->settings
];
}
public function __unserialize($data): void
{
foreach ($data as $property => $value) {
$this->{$property} = $value;
}
try {
$this->db = static::getDbConnection($this->settings);
} catch (\Throwable $e) {
Logger::log($e->getMessage(), Logger::ERROR);
}
} }
/** /**
@ -142,12 +127,13 @@ class PostgresArray implements DbArray
} }
$instance->settings = $settings; $instance->settings = $settings;
$instance->db = static::getDbConnection($settings);
$instance->ttl = $settings['cache_ttl'] ?? $instance->ttl; $instance->ttl = $settings['cache_ttl'] ?? $instance->ttl;
$instance->startCacheCleanupLoop(); $instance->startCacheCleanupLoop();
return call(static function () use ($instance, $value) { return call(static function () use ($instance, $value, $settings) {
$instance->db = yield from static::getDbConnection($settings);
yield from $instance->prepareTable(); yield from $instance->prepareTable();
//Skip migrations if its same object //Skip migrations if its same object
@ -367,9 +353,9 @@ class PostgresArray implements DbArray
if ( if (
!empty($params['index']) !empty($params['index'])
&& !mb_check_encoding($params['index'], 'UTF-8') && !\mb_check_encoding($params['index'], 'UTF-8')
) { ) {
$params['index'] = mb_convert_encoding($params['index'], 'UTF-8'); $params['index'] = \mb_convert_encoding($params['index'], 'UTF-8');
} }
try { try {

View File

@ -417,7 +417,7 @@ class MTProto extends AsyncConstruct implements TLCallback
* @see DbPropertiesFactory * @see DbPropertiesFactory
* @var array * @var array
*/ */
protected array $dbProperies = [ protected static array $dbProperties = [
'chats' => 'array', 'chats' => 'array',
'full_chats' => 'array', 'full_chats' => 'array',
'channel_participants' => 'array', 'channel_participants' => 'array',
@ -569,14 +569,12 @@ class MTProto extends AsyncConstruct implements TLCallback
*/ */
public function cleanup(): void public function cleanup(): void
{ {
/*
// :)
$this->referenceDatabase = new ReferenceDatabase($this); $this->referenceDatabase = new ReferenceDatabase($this);
$callbacks = [$this, $this->referenceDatabase]; $callbacks = [$this, $this->referenceDatabase];
if (!($this->authorization['user']['bot'] ?? false)) { if (!($this->authorization['user']['bot'] ?? false)) {
$callbacks[] = $this->minDatabase; $callbacks[] = $this->minDatabase;
} }
$this->TL->updateCallbacks($callbacks);*/ $this->TL->updateCallbacks($callbacks);
} }
private function fillUsernamesCache(): \Generator private function fillUsernamesCache(): \Generator

View File

@ -59,7 +59,7 @@ class MinDatabase implements TLCallback
* @see DbPropertiesFactory * @see DbPropertiesFactory
* @var array * @var array
*/ */
protected array $dbProperies = [ protected static array $dbProperties = [
'db' => 'array', 'db' => 'array',
]; ];

View File

@ -84,7 +84,7 @@ class ReferenceDatabase implements TLCallback
* @see DbPropertiesFactory * @see DbPropertiesFactory
* @var array * @var array
*/ */
protected array $dbProperies = [ protected static array $dbProperties = [
'db' => 'array', 'db' => 'array',
]; ];