Parallelize downloads
This commit is contained in:
parent
8bfeb13176
commit
698d0cb59f
2
docs
2
docs
@ -1 +1 @@
|
|||||||
Subproject commit 7a17668c568e4a17e6b08f02347bd3f139d8554f
|
Subproject commit 02967b3019032399364a9fc134f7bb0e674c099a
|
4
phar.php
4
phar.php
@ -61,10 +61,10 @@ function ___install_madeline()
|
|||||||
$release_branch = '';
|
$release_branch = '';
|
||||||
}
|
}
|
||||||
$release_fallback_branch = '';
|
$release_fallback_branch = '';
|
||||||
if (isset($_SERVER['SERVER_ADMIN']) && strpos($_SERVER['SERVER_ADMIN'], '000webhost.io') && $custom_branch === null) {
|
/*if (isset($_SERVER['SERVER_ADMIN']) && strpos($_SERVER['SERVER_ADMIN'], '000webhost.io') && $custom_branch === null) {
|
||||||
$release_branch = '-deprecated';
|
$release_branch = '-deprecated';
|
||||||
$release_fallback_branch = '-deprecated';
|
$release_fallback_branch = '-deprecated';
|
||||||
}
|
}*/
|
||||||
|
|
||||||
if (PHP_MAJOR_VERSION <= 5) {
|
if (PHP_MAJOR_VERSION <= 5) {
|
||||||
$release_branch = '5'.$release_branch;
|
$release_branch = '5'.$release_branch;
|
||||||
|
@ -20,6 +20,10 @@
|
|||||||
namespace danog\MadelineProto;
|
namespace danog\MadelineProto;
|
||||||
|
|
||||||
use Amp\Deferred;
|
use Amp\Deferred;
|
||||||
|
use function Amp\File\put;
|
||||||
|
use function Amp\File\rename;
|
||||||
|
use function Amp\File\get;
|
||||||
|
use function Amp\File\exists;
|
||||||
|
|
||||||
class API extends APIFactory
|
class API extends APIFactory
|
||||||
{
|
{
|
||||||
@ -62,21 +66,15 @@ class API extends APIFactory
|
|||||||
$realpaths = Serialization::realpaths($params);
|
$realpaths = Serialization::realpaths($params);
|
||||||
$this->session = $realpaths['file'];
|
$this->session = $realpaths['file'];
|
||||||
|
|
||||||
if (file_exists($realpaths['file'])) {
|
if (yield exists($realpaths['file'])) {
|
||||||
if (!file_exists($realpaths['lockfile'])) {
|
Logger::log('Waiting for shared lock of serialization lockfile...');
|
||||||
touch($realpaths['lockfile']);
|
$unlock = yield Tools::flock($realpaths['lockfile'], LOCK_SH);
|
||||||
clearstatcache();
|
Logger::log('Shared lock acquired, deserializing...');
|
||||||
}
|
|
||||||
$realpaths['lockfile'] = fopen($realpaths['lockfile'], 'r');
|
|
||||||
\danog\MadelineProto\Logger::log('Waiting for shared lock of serialization lockfile...');
|
|
||||||
flock($realpaths['lockfile'], LOCK_SH);
|
|
||||||
\danog\MadelineProto\Logger::log('Shared lock acquired, deserializing...');
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$tounserialize = file_get_contents($realpaths['file']);
|
$tounserialize = yield get($realpaths['file']);
|
||||||
} finally {
|
} finally {
|
||||||
flock($realpaths['lockfile'], LOCK_UN);
|
$unlock();
|
||||||
fclose($realpaths['lockfile']);
|
|
||||||
}
|
}
|
||||||
\danog\MadelineProto\Magic::class_exists();
|
\danog\MadelineProto\Magic::class_exists();
|
||||||
|
|
||||||
@ -137,11 +135,11 @@ class API extends APIFactory
|
|||||||
$deferred->resolve();
|
$deferred->resolve();
|
||||||
yield $this->API->initAsync();
|
yield $this->API->initAsync();
|
||||||
$this->APIFactory();
|
$this->APIFactory();
|
||||||
\danog\MadelineProto\Logger::log('Ping...', Logger::ULTRA_VERBOSE);
|
Logger::log('Ping...', Logger::ULTRA_VERBOSE);
|
||||||
$this->asyncInitPromise = null;
|
$this->asyncInitPromise = null;
|
||||||
$pong = yield $this->ping(['ping_id' => 3], ['async' => true]);
|
$pong = yield $this->ping(['ping_id' => 3], ['async' => true]);
|
||||||
\danog\MadelineProto\Logger::log('Pong: '.$pong['ping_id'], Logger::ULTRA_VERBOSE);
|
Logger::log('Pong: '.$pong['ping_id'], Logger::ULTRA_VERBOSE);
|
||||||
\danog\MadelineProto\Logger::log(\danog\MadelineProto\Lang::$current_lang['madelineproto_ready'], Logger::NOTICE);
|
Logger::log(\danog\MadelineProto\Lang::$current_lang['madelineproto_ready'], Logger::NOTICE);
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -158,14 +156,14 @@ class API extends APIFactory
|
|||||||
$this->API = new MTProto($params);
|
$this->API = new MTProto($params);
|
||||||
$this->APIFactory();
|
$this->APIFactory();
|
||||||
$deferred->resolve();
|
$deferred->resolve();
|
||||||
\danog\MadelineProto\Logger::log(\danog\MadelineProto\Lang::$current_lang['apifactory_start'], Logger::VERBOSE);
|
Logger::log(\danog\MadelineProto\Lang::$current_lang['apifactory_start'], Logger::VERBOSE);
|
||||||
yield $this->API->initAsync();
|
yield $this->API->initAsync();
|
||||||
$this->APIFactory();
|
$this->APIFactory();
|
||||||
$this->asyncInitPromise = null;
|
$this->asyncInitPromise = null;
|
||||||
\danog\MadelineProto\Logger::log('Ping...', Logger::ULTRA_VERBOSE);
|
Logger::log('Ping...', Logger::ULTRA_VERBOSE);
|
||||||
$pong = yield $this->ping(['ping_id' => 3], ['async' => true]);
|
$pong = yield $this->ping(['ping_id' => 3], ['async' => true]);
|
||||||
\danog\MadelineProto\Logger::log('Pong: '.$pong['ping_id'], Logger::ULTRA_VERBOSE);
|
Logger::log('Pong: '.$pong['ping_id'], Logger::ULTRA_VERBOSE);
|
||||||
\danog\MadelineProto\Logger::log(\danog\MadelineProto\Lang::$current_lang['madelineproto_ready'], Logger::NOTICE);
|
Logger::log(\danog\MadelineProto\Lang::$current_lang['madelineproto_ready'], Logger::NOTICE);
|
||||||
}
|
}
|
||||||
|
|
||||||
public function async($async)
|
public function async($async)
|
||||||
@ -312,14 +310,11 @@ class API extends APIFactory
|
|||||||
}
|
}
|
||||||
$this->serialized = time();
|
$this->serialized = time();
|
||||||
$realpaths = Serialization::realpaths($filename);
|
$realpaths = Serialization::realpaths($filename);
|
||||||
if (!file_exists($realpaths['lockfile'])) {
|
Logger::log('Waiting for exclusive lock of serialization lockfile...');
|
||||||
touch($realpaths['lockfile']);
|
|
||||||
clearstatcache();
|
$unlock = yield Tools::flock($realpaths['lockfile'], LOCK_EX);
|
||||||
}
|
|
||||||
$realpaths['lockfile'] = fopen($realpaths['lockfile'], 'w');
|
Logger::log('Lock acquired, serializing');
|
||||||
\danog\MadelineProto\Logger::log('Waiting for exclusive lock of serialization lockfile...');
|
|
||||||
flock($realpaths['lockfile'], LOCK_EX);
|
|
||||||
\danog\MadelineProto\Logger::log('Lock acquired, serializing');
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (!$this->getting_api_id) {
|
if (!$this->getting_api_id) {
|
||||||
@ -332,17 +327,16 @@ class API extends APIFactory
|
|||||||
$this->API->settings['logger']['logger_param'] = [$this->API, 'noop'];
|
$this->API->settings['logger']['logger_param'] = [$this->API, 'noop'];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$wrote = file_put_contents($realpaths['tempfile'], serialize($this));
|
$wrote = yield put($realpaths['tempfile'], serialize($this));
|
||||||
rename($realpaths['tempfile'], $realpaths['file']);
|
yield rename($realpaths['tempfile'], $realpaths['file']);
|
||||||
} finally {
|
} finally {
|
||||||
if (!$this->getting_api_id) {
|
if (!$this->getting_api_id) {
|
||||||
$this->API->settings['updates']['callback'] = $update_closure;
|
$this->API->settings['updates']['callback'] = $update_closure;
|
||||||
$this->API->settings['logger']['logger_param'] = $logger_closure;
|
$this->API->settings['logger']['logger_param'] = $logger_closure;
|
||||||
}
|
}
|
||||||
flock($realpaths['lockfile'], LOCK_UN);
|
$unlock();
|
||||||
fclose($realpaths['lockfile']);
|
|
||||||
}
|
}
|
||||||
\danog\MadelineProto\Logger::log('Done serializing');
|
Logger::log('Done serializing');
|
||||||
|
|
||||||
return $wrote;
|
return $wrote;
|
||||||
})());
|
})());
|
||||||
|
@ -26,6 +26,7 @@ use danog\MadelineProto\MTProtoTools\Crypt;
|
|||||||
use danog\MadelineProto\NothingInTheSocketException;
|
use danog\MadelineProto\NothingInTheSocketException;
|
||||||
use danog\MadelineProto\Tools;
|
use danog\MadelineProto\Tools;
|
||||||
use Amp\ByteStream\StreamException;
|
use Amp\ByteStream\StreamException;
|
||||||
|
use Amp\ByteStream\PendingReadError;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Socket read loop.
|
* Socket read loop.
|
||||||
@ -57,7 +58,7 @@ class ReadLoop extends SignalLoop
|
|||||||
while (true) {
|
while (true) {
|
||||||
try {
|
try {
|
||||||
$error = yield $this->waitSignal($this->readMessage());
|
$error = yield $this->waitSignal($this->readMessage());
|
||||||
} catch (NothingInTheSocketException|StreamException $e) {
|
} catch (NothingInTheSocketException|StreamException|PendingReadError $e) {
|
||||||
if (isset($connection->old)) {
|
if (isset($connection->old)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -65,10 +66,6 @@ class ReadLoop extends SignalLoop
|
|||||||
$API->logger->logger("Got nothing in the socket in DC {$datacenter}, reconnecting...", Logger::ERROR);
|
$API->logger->logger("Got nothing in the socket in DC {$datacenter}, reconnecting...", Logger::ERROR);
|
||||||
yield $connection->reconnect();
|
yield $connection->reconnect();
|
||||||
continue;
|
continue;
|
||||||
} catch (ClosedException $e) {
|
|
||||||
$API->logger->logger($e->getMessage(), Logger::FATAL_ERROR);
|
|
||||||
|
|
||||||
throw $e;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (is_int($error)) {
|
if (is_int($error)) {
|
||||||
|
@ -73,7 +73,7 @@ class MTProto extends AsyncConstruct implements TLCallback
|
|||||||
/*
|
/*
|
||||||
const V = 71;
|
const V = 71;
|
||||||
*/
|
*/
|
||||||
const V = 124;
|
const V = 125;
|
||||||
const RELEASE = '4.0';
|
const RELEASE = '4.0';
|
||||||
const NOT_LOGGED_IN = 0;
|
const NOT_LOGGED_IN = 0;
|
||||||
const WAITING_CODE = 1;
|
const WAITING_CODE = 1;
|
||||||
@ -801,9 +801,11 @@ class MTProto extends AsyncConstruct implements TLCallback
|
|||||||
], 'upload' => [
|
], 'upload' => [
|
||||||
'allow_automatic_upload' => true,
|
'allow_automatic_upload' => true,
|
||||||
'part_size' => 512 * 1024,
|
'part_size' => 512 * 1024,
|
||||||
|
'parallel_chunks' => 20
|
||||||
], 'download' => [
|
], 'download' => [
|
||||||
'report_broken_media' => true,
|
'report_broken_media' => true,
|
||||||
'part_size' => 1024 * 1024,
|
'part_size' => 1024 * 1024,
|
||||||
|
'parallel_chunks' => 20
|
||||||
], 'pwr' => [
|
], 'pwr' => [
|
||||||
'pwr' => false,
|
'pwr' => false,
|
||||||
// Need info ?
|
// Need info ?
|
||||||
|
@ -19,11 +19,20 @@
|
|||||||
|
|
||||||
namespace danog\MadelineProto\MTProtoTools;
|
namespace danog\MadelineProto\MTProtoTools;
|
||||||
|
|
||||||
|
use Amp\ByteStream\OutputStream;
|
||||||
|
use Amp\ByteStream\ResourceOutputStream;
|
||||||
|
use Amp\ByteStream\StreamException;
|
||||||
use danog\MadelineProto\Async\AsyncParameters;
|
use danog\MadelineProto\Async\AsyncParameters;
|
||||||
use danog\MadelineProto\Exception;
|
use danog\MadelineProto\Exception;
|
||||||
|
use danog\MadelineProto\FileCallbackInterface;
|
||||||
use danog\MadelineProto\Logger;
|
use danog\MadelineProto\Logger;
|
||||||
use danog\MadelineProto\RPCErrorException;
|
use danog\MadelineProto\RPCErrorException;
|
||||||
|
use function Amp\File\open;
|
||||||
|
use function Amp\File\stat;
|
||||||
use function Amp\Promise\all;
|
use function Amp\Promise\all;
|
||||||
|
use Amp\Deferred;
|
||||||
|
use Amp\Success;
|
||||||
|
use Amp\File\StatCache;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Manages upload and download of files.
|
* Manages upload and download of files.
|
||||||
@ -33,7 +42,7 @@ trait Files
|
|||||||
public function upload_async($file, $file_name = '', $cb = null, $encrypted = false): \Generator
|
public function upload_async($file, $file_name = '', $cb = null, $encrypted = false): \Generator
|
||||||
{
|
{
|
||||||
if (is_object($file)) {
|
if (is_object($file)) {
|
||||||
if (!isset(class_implements($file)['danog\MadelineProto\FileCallbackInterface'])) {
|
if (!$file instanceof FileCallbackInterface) {
|
||||||
throw new \danog\MadelineProto\Exception('Provided object does not implement FileCallbackInterface');
|
throw new \danog\MadelineProto\Exception('Provided object does not implement FileCallbackInterface');
|
||||||
}
|
}
|
||||||
$cb = $file;
|
$cb = $file;
|
||||||
@ -511,7 +520,7 @@ trait Files
|
|||||||
|
|
||||||
public function download_to_dir_async($message_media, $dir, $cb = null)
|
public function download_to_dir_async($message_media, $dir, $cb = null)
|
||||||
{
|
{
|
||||||
if (is_object($dir) && class_implements($dir)['danog\MadelineProto\FileCallbackInterface']) {
|
if (is_object($dir) && $dir instanceof FileCallbackInterface) {
|
||||||
$cb = $dir;
|
$cb = $dir;
|
||||||
$dir = $dir->getFile();
|
$dir = $dir->getFile();
|
||||||
}
|
}
|
||||||
@ -523,7 +532,7 @@ trait Files
|
|||||||
|
|
||||||
public function download_to_file_async($message_media, $file, $cb = null)
|
public function download_to_file_async($message_media, $file, $cb = null)
|
||||||
{
|
{
|
||||||
if (is_object($file) && class_implements($file)['danog\MadelineProto\FileCallbackInterface']) {
|
if (is_object($file) && $file instanceof FileCallbackInterface) {
|
||||||
$cb = $file;
|
$cb = $file;
|
||||||
$file = $file->getFile();
|
$file = $file->getFile();
|
||||||
}
|
}
|
||||||
@ -533,22 +542,21 @@ trait Files
|
|||||||
}
|
}
|
||||||
$file = realpath($file);
|
$file = realpath($file);
|
||||||
$message_media = yield $this->get_download_info_async($message_media);
|
$message_media = yield $this->get_download_info_async($message_media);
|
||||||
$stream = fopen($file, 'r+b');
|
|
||||||
$size = fstat($stream)['size'];
|
StatCache::clear($file);
|
||||||
|
|
||||||
|
$size = (yield stat($file))['size'];
|
||||||
|
$stream = yield open($file, 'cb');
|
||||||
|
|
||||||
$this->logger->logger('Waiting for lock of file to download...');
|
$this->logger->logger('Waiting for lock of file to download...');
|
||||||
do {
|
$unlock = yield $this->flock($file, LOCK_EX);
|
||||||
$res = flock($stream, LOCK_EX|LOCK_NB);
|
|
||||||
if (!$res) {
|
|
||||||
yield $this->sleep(0.1);
|
|
||||||
}
|
|
||||||
} while (!$res);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
yield $this->download_to_stream_async($message_media, $stream, $cb, $size, -1);
|
yield $this->download_to_stream_async($message_media, $stream, $cb, $size, -1);
|
||||||
} finally {
|
} finally {
|
||||||
flock($stream, LOCK_UN);
|
$unlock();
|
||||||
fclose($stream);
|
yield $stream->close();
|
||||||
clearstatcache();
|
StatCache::clear($file);
|
||||||
}
|
}
|
||||||
|
|
||||||
return $file;
|
return $file;
|
||||||
@ -556,7 +564,9 @@ trait Files
|
|||||||
|
|
||||||
public function download_to_stream_async($message_media, $stream, $cb = null, $offset = 0, $end = -1)
|
public function download_to_stream_async($message_media, $stream, $cb = null, $offset = 0, $end = -1)
|
||||||
{
|
{
|
||||||
if (is_object($stream) && class_implements($stream)['danog\MadelineProto\FileCallbackInterface']) {
|
$message_media = yield $this->get_download_info_async($message_media);
|
||||||
|
|
||||||
|
if (is_object($stream) && $stream instanceof FileCallbackInterface) {
|
||||||
$cb = $stream;
|
$cb = $stream;
|
||||||
$stream = $stream->getFile();
|
$stream = $stream->getFile();
|
||||||
}
|
}
|
||||||
@ -567,25 +577,34 @@ trait Files
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
$message_media = yield $this->get_download_info_async($message_media);
|
/** @var $stream \Amp\ByteStream\OutputStream */
|
||||||
|
if (!is_object($stream)) {
|
||||||
try {
|
$stream = new ResourceOutputStream($stream);
|
||||||
if (stream_get_meta_data($stream)['seekable']) {
|
|
||||||
fseek($stream, $offset);
|
|
||||||
}
|
|
||||||
} catch (\danog\MadelineProto\Exception $e) {
|
|
||||||
}
|
}
|
||||||
$downloaded_size = 0;
|
if (!$stream instanceof OutputStream) {
|
||||||
|
throw new Exception("Invalid stream provided");
|
||||||
|
}
|
||||||
|
$seekable = false;
|
||||||
|
if (method_exists($stream, 'seek')) {
|
||||||
|
try {
|
||||||
|
yield $stream->seek($offset);
|
||||||
|
$seekable = true;
|
||||||
|
} catch (StreamException $e) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if ($end === -1 && isset($message_media['size'])) {
|
if ($end === -1 && isset($message_media['size'])) {
|
||||||
$end = $message_media['size'];
|
$end = $message_media['size'];
|
||||||
}
|
}
|
||||||
$size = $end - $offset;
|
|
||||||
$part_size = $this->settings['download']['part_size'];
|
$part_size = $this->settings['download']['part_size'];
|
||||||
$percent = 0;
|
$parallel_chunks = $this->settings['download']['parallel_chunks'];
|
||||||
|
|
||||||
$datacenter = isset($message_media['InputFileLocation']['dc_id']) ? $message_media['InputFileLocation']['dc_id'] : $this->settings['connection_settings']['default_dc'];
|
$datacenter = isset($message_media['InputFileLocation']['dc_id']) ? $message_media['InputFileLocation']['dc_id'] : $this->settings['connection_settings']['default_dc'];
|
||||||
if (isset($this->datacenter->sockets[$datacenter.'_media'])) {
|
if (isset($this->datacenter->sockets[$datacenter.'_media'])) {
|
||||||
$datacenter .= '_media';
|
$datacenter .= '_media';
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isset($message_media['key'])) {
|
if (isset($message_media['key'])) {
|
||||||
$digest = hash('md5', $message_media['key'].$message_media['iv'], true);
|
$digest = hash('md5', $message_media['key'].$message_media['iv'], true);
|
||||||
$fingerprint = $this->unpack_signed_int(substr($digest, 0, 4) ^ substr($digest, 4, 4));
|
$fingerprint = $this->unpack_signed_int(substr($digest, 0, 4) ^ substr($digest, 4, 4));
|
||||||
@ -597,44 +616,124 @@ trait Files
|
|||||||
$ige->setKey($message_media['key']);
|
$ige->setKey($message_media['key']);
|
||||||
$ige->enableContinuousBuffer();
|
$ige->enableContinuousBuffer();
|
||||||
}
|
}
|
||||||
$theend = false;
|
|
||||||
$cdn = false;
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
if ($start_at = $offset % $part_size) {
|
if ($offset === $end) {
|
||||||
$offset -= $start_at;
|
$cb(100);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
$params = [];
|
||||||
|
$start_at = $offset % $part_size;
|
||||||
|
$probable_end = $end !== -1 ? $end : 512 * 1024 * 3000;
|
||||||
|
|
||||||
|
$breakOut = false;
|
||||||
|
for ($x = $offset - $start_at; $x < $probable_end; $x += $part_size) {
|
||||||
|
$end_at = $part_size;
|
||||||
|
if ($end !== -1 && $x + $part_size >= $end) {
|
||||||
|
$end_at = $end % $part_size;
|
||||||
|
$breakOut = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$params[] = [
|
||||||
|
'offset' => $x,
|
||||||
|
'limit' => $part_size,
|
||||||
|
'part_start_at' => $start_at,
|
||||||
|
'part_end_at' => $end_at,
|
||||||
|
];
|
||||||
|
|
||||||
|
$start_at = 0;
|
||||||
|
if ($breakOut) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!$params) {
|
||||||
|
$cb(100);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
$count = count($params);
|
||||||
|
$cb = function () use ($cb, $count) {
|
||||||
|
static $cur = 0;
|
||||||
|
$cur++;
|
||||||
|
$this->callFork($cb($cur*100/$count));
|
||||||
|
};
|
||||||
|
|
||||||
|
$cdn = false;
|
||||||
|
|
||||||
|
$params[0]['previous_promise'] = new Success(true);
|
||||||
|
|
||||||
|
$start = microtime(true);
|
||||||
|
$size = yield $this->download_part($message_media, $cdn, $datacenter, $old_dc, $ige, $cb, array_shift($params), $stream, $seekable);
|
||||||
|
|
||||||
|
if ($params) {
|
||||||
|
$previous_promise = new Success(true);
|
||||||
|
|
||||||
|
$promises = [];
|
||||||
|
foreach ($params as $key => $param) {
|
||||||
|
$param['previous_promise'] = $previous_promise;
|
||||||
|
$previous_promise = $this->call($this->download_part($message_media, $cdn, $datacenter, $old_dc, $ige, $cb, $param, $stream, $seekable));
|
||||||
|
$previous_promise->onResolve(static function ($e, $res) use (&$size) {
|
||||||
|
if ($res) {
|
||||||
|
$size += $res;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
$promises []= $previous_promise;
|
||||||
|
|
||||||
|
if (!($key % $parallel_chunks)) { // 20 mb at a time, for a typical bandwidth of 1gbps
|
||||||
|
yield $this->all($promises);
|
||||||
|
$promises = [];
|
||||||
|
|
||||||
|
$time = microtime(true) - $start;
|
||||||
|
$speed = (int) (($size*8)/$time)/1000000;
|
||||||
|
$this->logger->logger("Partial download time: $time");
|
||||||
|
$this->logger->logger("Partial download speed: $speed mbps");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($promises) {
|
||||||
|
yield $this->all($promises);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$time = microtime(true) - $start;
|
||||||
|
$speed = (int) (($size*8)/$time)/1000000;
|
||||||
|
$this->logger->logger("Total download time: $time");
|
||||||
|
$this->logger->logger("Total download speed: $speed mbps");
|
||||||
|
|
||||||
|
if ($cdn) {
|
||||||
|
$this->clear_cdn_hashes($message_media['file_token']);
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function download_part(&$message_media, &$cdn, &$datacenter, &$old_dc, &$ige, $cb, $offset, $stream, $seekable, $postpone = false)
|
||||||
|
{
|
||||||
|
static $method = [
|
||||||
|
false => 'upload.getFile', // non-cdn
|
||||||
|
true => 'upload.getCdnFile', // cdn
|
||||||
|
];
|
||||||
|
do {
|
||||||
|
if (!$cdn) {
|
||||||
|
$basic_param = [
|
||||||
|
'location' => $message_media['InputFileLocation']
|
||||||
|
];
|
||||||
|
} else {
|
||||||
|
$basic_param = [
|
||||||
|
'file_token' => $message_media['file_token']
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$res = $cdn ?
|
$res = yield $this->method_call_async_read(
|
||||||
yield $this->method_call_async_read(
|
$method[$cdn],
|
||||||
'upload.getCdnFile',
|
$basic_param + $offset,
|
||||||
[
|
[
|
||||||
'file_token' => $message_media['file_token'],
|
'heavy' => true,
|
||||||
'offset' => $offset,
|
'file' => true,
|
||||||
'limit' => $part_size
|
'FloodWaitLimit' => 0,
|
||||||
],
|
'datacenter' => &$datacenter,
|
||||||
[
|
'postpone' => $postpone
|
||||||
'heavy' => true,
|
]
|
||||||
'file' => true,
|
);
|
||||||
'FloodWaitLimit' => 0,
|
|
||||||
'datacenter' => $datacenter
|
|
||||||
]
|
|
||||||
) :
|
|
||||||
yield $this->method_call_async_read(
|
|
||||||
'upload.getFile',
|
|
||||||
[
|
|
||||||
'location' => $message_media['InputFileLocation'],
|
|
||||||
'offset' => $offset,
|
|
||||||
'limit' => $part_size
|
|
||||||
],
|
|
||||||
[
|
|
||||||
'heavy' => true,
|
|
||||||
'file' => true,
|
|
||||||
'FloodWaitLimit' => 0,
|
|
||||||
'datacenter' => &$datacenter
|
|
||||||
]
|
|
||||||
);
|
|
||||||
} catch (\danog\MadelineProto\RPCErrorException $e) {
|
} catch (\danog\MadelineProto\RPCErrorException $e) {
|
||||||
if (strpos($e->rpc, 'FLOOD_WAIT_') === 0) {
|
if (strpos($e->rpc, 'FLOOD_WAIT_') === 0) {
|
||||||
if (isset($message_media['MessageMedia']) && !$this->authorization['user']['bot'] && $this->settings['download']['report_broken_media']) {
|
if (isset($message_media['MessageMedia']) && !$this->authorization['user']['bot'] && $this->settings['download']['report_broken_media']) {
|
||||||
@ -657,6 +756,7 @@ trait Files
|
|||||||
throw $e;
|
throw $e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($res['_'] === 'upload.fileCdnRedirect') {
|
if ($res['_'] === 'upload.fileCdnRedirect') {
|
||||||
$cdn = true;
|
$cdn = true;
|
||||||
$message_media['file_token'] = $res['file_token'];
|
$message_media['file_token'] = $res['file_token'];
|
||||||
@ -669,9 +769,7 @@ trait Files
|
|||||||
yield $this->get_config_async([], ['datacenter' => $this->datacenter->curdc]);
|
yield $this->get_config_async([], ['datacenter' => $this->datacenter->curdc]);
|
||||||
}
|
}
|
||||||
$this->logger->logger(\danog\MadelineProto\Lang::$current_lang['stored_on_cdn'], \danog\MadelineProto\Logger::NOTICE);
|
$this->logger->logger(\danog\MadelineProto\Lang::$current_lang['stored_on_cdn'], \danog\MadelineProto\Logger::NOTICE);
|
||||||
continue;
|
} else if ($res['_'] === 'upload.cdnFileReuploadNeeded') {
|
||||||
}
|
|
||||||
if ($res['_'] === 'upload.cdnFileReuploadNeeded') {
|
|
||||||
$this->logger->logger(\danog\MadelineProto\Lang::$current_lang['cdn_reupload'], \danog\MadelineProto\Logger::NOTICE);
|
$this->logger->logger(\danog\MadelineProto\Lang::$current_lang['cdn_reupload'], \danog\MadelineProto\Logger::NOTICE);
|
||||||
yield $this->get_config_async([], ['datacenter' => $this->datacenter->curdc]);
|
yield $this->get_config_async([], ['datacenter' => $this->datacenter->curdc]);
|
||||||
|
|
||||||
@ -690,51 +788,37 @@ trait Files
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if ($cdn === false && $res['type']['_'] === 'storage.fileUnknown' && $res['bytes'] === '') {
|
if ($cdn === false && $res['type']['_'] === 'storage.fileUnknown' && $res['bytes'] === '') {
|
||||||
$datacenter = 1;
|
$datacenter = 0;
|
||||||
}
|
}
|
||||||
while ($cdn === false && $res['type']['_'] === 'storage.fileUnknown' && $res['bytes'] === '') {
|
while ($cdn === false &&
|
||||||
$res = yield $this->method_call_async_read('upload.getFile', ['location' => $message_media['InputFileLocation'], 'offset' => $offset, 'limit' => $part_size], ['heavy' => true, 'datacenter' => $datacenter]);
|
$res['type']['_'] === 'storage.fileUnknown' &&
|
||||||
$datacenter++;
|
$res['bytes'] === '' &&
|
||||||
if (!isset($this->datacenter->sockets[$datacenter])) {
|
isset($this->datacenter->sockets[++$datacenter])
|
||||||
break;
|
) {
|
||||||
}
|
$res = yield $this->method_call_async_read('upload.getFile', $basic_param + $offset, ['heavy' => true, 'file' => true, 'FloodWaitLimit' => 0, 'datacenter' => $datacenter]);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isset($message_media['cdn_key'])) {
|
if (isset($message_media['cdn_key'])) {
|
||||||
$ivec = substr($message_media['cdn_iv'], 0, 12).pack('N', $offset >> 4);
|
$ivec = substr($message_media['cdn_iv'], 0, 12).pack('N', $offset['offset'] >> 4);
|
||||||
$res['bytes'] = $this->ctr_encrypt($res['bytes'], $message_media['cdn_key'], $ivec);
|
$res['bytes'] = $this->ctr_encrypt($res['bytes'], $message_media['cdn_key'], $ivec);
|
||||||
$this->check_cdn_hash($message_media['file_token'], $offset, $res['bytes'], $old_dc);
|
$this->check_cdn_hash($message_media['file_token'], $offset['offset'], $res['bytes'], $old_dc);
|
||||||
}
|
}
|
||||||
if (isset($message_media['key'])) {
|
if (isset($message_media['key'])) {
|
||||||
$res['bytes'] = $ige->decrypt($res['bytes']);
|
$res['bytes'] = $ige->decrypt($res['bytes']);
|
||||||
}
|
}
|
||||||
if ($start_at) {
|
if ($offset['part_start_at'] || $offset['part_end_at'] !== $offset['limit']) {
|
||||||
$res['bytes'] = substr($res['bytes'], $start_at);
|
$res['bytes'] = substr($res['bytes'], $offset['part_start_at'], $offset['part_end_at']- $offset['part_start_at']);
|
||||||
}
|
}
|
||||||
if ($end !== -1 && strlen($res['bytes']) + $downloaded_size >= $size) {
|
|
||||||
$res['bytes'] = substr($res['bytes'], 0, $size - $downloaded_size);
|
if (!$seekable) {
|
||||||
$theend = true;
|
yield $offset['previous_promise']->promise();
|
||||||
|
} else {
|
||||||
|
yield $stream->seek($offset['offset'] + $offset['part_start_at']);
|
||||||
}
|
}
|
||||||
if ($res['bytes'] === '') {
|
yield $stream->write($res['bytes']);
|
||||||
break;
|
$cb();
|
||||||
}
|
return strlen($res['bytes']);
|
||||||
$offset += strlen($res['bytes']);
|
} while (true);
|
||||||
$downloaded_size += strlen($res['bytes']);
|
|
||||||
$this->logger->logger(fwrite($stream, $res['bytes']), \danog\MadelineProto\Logger::ULTRA_VERBOSE);
|
|
||||||
if ($theend) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if ($end !== -1) {
|
|
||||||
$this->callFork($cb($percent = $downloaded_size * 100 / $size));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ($end === -1) {
|
|
||||||
$this->callFork($cb(100));
|
|
||||||
}
|
|
||||||
if ($cdn) {
|
|
||||||
$this->clear_cdn_hashes($message_media['file_token']);
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private $cdn_hashes = [];
|
private $cdn_hashes = [];
|
||||||
|
@ -32,6 +32,9 @@ use function Amp\Promise\wait;
|
|||||||
use function Amp\ByteStream\getStdin;
|
use function Amp\ByteStream\getStdin;
|
||||||
use function Amp\ByteStream\getStdout;
|
use function Amp\ByteStream\getStdout;
|
||||||
use function Amp\ByteStream\getOutputBufferStream;
|
use function Amp\ByteStream\getOutputBufferStream;
|
||||||
|
use function Amp\File\exists;
|
||||||
|
use function Amp\File\touch;
|
||||||
|
use Amp\File\StatCache;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Some tools.
|
* Some tools.
|
||||||
@ -344,7 +347,42 @@ trait Tools
|
|||||||
|
|
||||||
return $deferred->promise();
|
return $deferred->promise();
|
||||||
}
|
}
|
||||||
|
/**
|
||||||
|
* Asynchronously lock a file
|
||||||
|
* Resolves with a callbable that MUST eventually be called in order to release the lock
|
||||||
|
*
|
||||||
|
* @param string $file File to lock
|
||||||
|
* @param integer $operation Locking mode (see flock)
|
||||||
|
* @param numeric $polling Polling interval for lock
|
||||||
|
* @return Promise
|
||||||
|
*/
|
||||||
|
public static function flock(string $file, int $operation, $polling = 0.1): Promise
|
||||||
|
{
|
||||||
|
return self::call(self::flockAsync($file, $operation, $polling));
|
||||||
|
}
|
||||||
|
public static function flockAsync(string $file, int $operation, $polling)
|
||||||
|
{
|
||||||
|
if (!yield exists($file)) {
|
||||||
|
yield touch($file);
|
||||||
|
StatCache::clear($file);
|
||||||
|
}
|
||||||
|
$operation |= LOCK_NB;
|
||||||
|
$res = fopen($file, 'c');
|
||||||
|
do {
|
||||||
|
$result = flock($res, $operation);
|
||||||
|
if (!$result) {
|
||||||
|
yield self::sleep($polling);
|
||||||
|
}
|
||||||
|
} while (!$result);
|
||||||
|
|
||||||
|
return static function () use (&$res) {
|
||||||
|
if ($res) {
|
||||||
|
flock($res, LOCK_UN);
|
||||||
|
fclose($res);
|
||||||
|
$res = null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
public static function sleep($time)
|
public static function sleep($time)
|
||||||
{
|
{
|
||||||
return new \Amp\Delayed($time * 1000);
|
return new \Amp\Delayed($time * 1000);
|
||||||
|
Loading…
Reference in New Issue
Block a user