From a2160c900892fd3b1f83f8e936fd5bafa9f2babf Mon Sep 17 00:00:00 2001 From: Daniil Gentili Date: Sun, 11 Mar 2018 14:29:27 +0000 Subject: [PATCH] Implement participant caching and hash generation --- src/danog/MadelineProto/MTProto.php | 10 +- .../MTProtoTools/CallHandler.php | 2 +- .../MTProtoTools/PeerHandler.php | 106 ++++++++++-------- 3 files changed, 67 insertions(+), 51 deletions(-) diff --git a/src/danog/MadelineProto/MTProto.php b/src/danog/MadelineProto/MTProto.php index c65210c6..6463c2b1 100644 --- a/src/danog/MadelineProto/MTProto.php +++ b/src/danog/MadelineProto/MTProto.php @@ -47,7 +47,7 @@ class MTProto /* const V = 71; */ - const V = 94; + const V = 96; const NOT_LOGGED_IN = 0; const WAITING_CODE = 1; const WAITING_SIGNUP = -1; @@ -75,6 +75,7 @@ class MTProto private $last_recv = 0; private $dh_config = ['version' => 0]; public $chats = []; + public $channel_participants = []; public $last_stored = 0; public $qres = []; public $full_chats = []; @@ -89,12 +90,15 @@ class MTProto private $twoe1984; private $twoe2047; private $twoe2048; + private $zeroeight; + private $twozerotwosixone; private $ipv6 = false; public $run_workers = false; public $setdem = false; public $storage = []; private $emojis; private $postpone_updates = false; + public function __magic_construct($settings = []) { @@ -129,6 +133,8 @@ class MTProto $this->twoe1984 = new \phpseclib\Math\BigInteger('1751908409537131537220509645351687597690304110853111572994449976845956819751541616602568796259317428464425605223064365804210081422215355425149431390635151955247955156636234741221447435733643262808668929902091770092492911737768377135426590363166295684370498604708288556044687341394398676292971255828404734517580702346564613427770683056761383955397564338690628093211465848244049196353703022640400205739093118270803778352768276670202698397214556629204420309965547056893233608758387329699097930255380715679250799950923553703740673620901978370802540218870279314810722790539899334271514365444369275682816'); $this->twoe2047 = new \phpseclib\Math\BigInteger('16158503035655503650357438344334975980222051334857742016065172713762327569433945446598600705761456731844358980460949009747059779575245460547544076193224141560315438683650498045875098875194826053398028819192033784138396109321309878080919047169238085235290822926018152521443787945770532904303776199561965192760957166694834171210342487393282284747428088017663161029038902829665513096354230157075129296432088558362971801859230928678799175576150822952201848806616643615613562842355410104862578550863465661734839271290328348967522998634176499319107762583194718667771801067716614802322659239302476074096777926805529798115328'); $this->twoe2048 = new \phpseclib\Math\BigInteger('32317006071311007300714876688669951960444102669715484032130345427524655138867890893197201411522913463688717960921898019494119559150490921095088152386448283120630877367300996091750197750389652106796057638384067568276792218642619756161838094338476170470581645852036305042887575891541065808607552399123930385521914333389668342420684974786564569494856176035326322058077805659331026192708460314150258592864177116725943603718461857357598351152301645904403697613233287231227125684710820209725157101726931323469678542580656697935045997268352998638215525166389437335543602135433229604645318478604952148193555853611059596230656'); + $this->twozerotwosixone = new \phpseclib\Math\BigInteger(20261); + $this->zeroeight = new \phpseclib\Math\BigInteger('2147483648'); \danog\MadelineProto\Logger::log(\danog\MadelineProto\Lang::$current_lang['TL_translation'], Logger::ULTRA_VERBOSE); $this->construct_TL($this->settings['tl_schema']['src']); $this->connect_to_all_dcs(); @@ -154,7 +160,7 @@ class MTProto public function __sleep() { - return ['encrypted_layer', 'settings', 'config', 'authorization', 'authorized', 'rsa_keys', 'last_recv', 'dh_config', 'chats', 'last_stored', 'qres', 'pending_updates', 'updates_state', 'got_state', 'channels_state', 'updates', 'updates_key', 'full_chats', 'msg_ids', 'dialog_params', 'datacenter', 'v', 'constructors', 'td_constructors', 'methods', 'td_methods', 'td_descriptions', 'twoe1984', 'twoe2047', 'twoe2048', 'zero', 'one', 'two', 'three', 'four', 'temp_requested_secret_chats', 'temp_rekeyed_secret_chats', 'secret_chats', 'hook_url', 'storage', 'emojis', 'authorized_dc']; + return ['encrypted_layer', 'settings', 'config', 'authorization', 'authorized', 'rsa_keys', 'last_recv', 'dh_config', 'chats', 'last_stored', 'qres', 'pending_updates', 'updates_state', 'got_state', 'channels_state', 'updates', 'updates_key', 'full_chats', 'msg_ids', 'dialog_params', 'datacenter', 'v', 'constructors', 'td_constructors', 'methods', 'td_methods', 'td_descriptions', 'twoe1984', 'twoe2047', 'twoe2048', 'zero', 'one', 'two', 'three', 'four', 'temp_requested_secret_chats', 'temp_rekeyed_secret_chats', 'secret_chats', 'hook_url', 'storage', 'emojis', 'authorized_dc', 'channel_participants', 'twozerotwosixone', 'zeroeight']; } public function __wakeup() diff --git a/src/danog/MadelineProto/MTProtoTools/CallHandler.php b/src/danog/MadelineProto/MTProtoTools/CallHandler.php index eb018fb5..79601318 100644 --- a/src/danog/MadelineProto/MTProtoTools/CallHandler.php +++ b/src/danog/MadelineProto/MTProtoTools/CallHandler.php @@ -185,7 +185,7 @@ trait CallHandler } catch (\danog\MadelineProto\NothingInTheSocketException $e) { $last_error = 'Nothing in the socket'; \danog\MadelineProto\Logger::log('An error getting response of method '.$method.': '.$e->getMessage().' in '.basename($e->getFile(), '.php').' on line '.$e->getLine().'. Retrying...', \danog\MadelineProto\Logger::WARNING); - if ($last_recv === $this->datacenter->sockets[$aargs['datacenter']]->last_recv) { + if ($last_recv === $this->datacenter->sockets[$aargs['datacenter']]->last_recv || ($this->datacenter->sockets[$aargs['datacenter']]->last_recv < time() - 1 && $this->is_http($aargs['datacenter']))) { $this->close_and_reopen($aargs['datacenter']); } continue; //2; diff --git a/src/danog/MadelineProto/MTProtoTools/PeerHandler.php b/src/danog/MadelineProto/MTProtoTools/PeerHandler.php index aec039ce..9eda368c 100644 --- a/src/danog/MadelineProto/MTProtoTools/PeerHandler.php +++ b/src/danog/MadelineProto/MTProtoTools/PeerHandler.php @@ -26,7 +26,6 @@ trait PeerHandler public function is_supergroup($id) { $log = log(-$id, 10); - return ($log - intval($log)) * 1000 < 10; } @@ -439,15 +438,15 @@ trait PeerHandler if (isset($res['participants']) && $fullfetch) { foreach ($res['participants'] as $key => $participant) { $newres = []; - $newres['user'] = $this->get_pwr_chat($participant['user_id'], false, false); + $newres['user'] = $this->get_pwr_chat($participant['user_id'], false, true); if (isset($participant['inviter_id'])) { - $newres['inviter'] = $this->get_pwr_chat($participant['inviter_id'], false, false); + $newres['inviter'] = $this->get_pwr_chat($participant['inviter_id'], false, true); } if (isset($participant['promoted_by'])) { - $newres['promoted_by'] = $this->get_pwr_chat($participant['promoted_by'], false, false); + $newres['promoted_by'] = $this->get_pwr_chat($participant['promoted_by'], false, true); } if (isset($participant['kicked_by'])) { - $newres['kicked_by'] = $this->get_pwr_chat($participant['kicked_by'], false, false); + $newres['kicked_by'] = $this->get_pwr_chat($participant['kicked_by'], false, true); } if (isset($participant['date'])) { $newres['date'] = $participant['date']; @@ -482,9 +481,9 @@ trait PeerHandler $total_count = (isset($res['participants_count']) ? $res['participants_count'] : 0) + (isset($res['admins_count']) ? $res['admins_count'] : 0) + (isset($res['kicked_count']) ? $res['kicked_count'] : 0) + (isset($res['banned_count']) ? $res['banned_count'] : 0); $res['participants'] = []; $limit = 200; - $filters = ['channelParticipantsRecent', 'channelParticipantsAdmins', 'channelParticipantsBots']; + $filters = ['channelParticipantsAdmins', 'channelParticipantsBots']; foreach ($filters as $filter) { - $this->fetch_participants($full['InputChannel'], $filter, '', $res); + $this->fetch_participants($full['InputChannel'], $filter, '', $total_count, $res); } $q = ''; @@ -492,7 +491,7 @@ trait PeerHandler foreach ($filters as $filter) { $this->recurse_alphabet_search_participants($full['InputChannel'], $filter, $q, $total_count, $res); } - + \danog\MadelineProto\Logger::log("Fetched ".count($res['participants'])." out of $total_count"); $res['participants'] = array_values($res['participants']); } if (!$fullfetch) { @@ -507,7 +506,7 @@ trait PeerHandler public function recurse_alphabet_search_participants($channel, $filter, $q, $total_count, &$res) { - if (!$this->fetch_participants($channel, $filter, $q, $res)) { + if (!$this->fetch_participants($channel, $filter, $q, $total_count, $res)) { return false; } @@ -516,40 +515,43 @@ trait PeerHandler } } - public function fetch_participants($channel, $filter, $q, &$res) + public function fetch_participants($channel, $filter, $q, $total_count, &$res) { $offset = 0; $limit = 200; $has_more = false; + $cached = false; - try { - $gres = $this->method_call('channels.getParticipants', ['channel' => $channel, 'filter' => ['_' => $filter, 'q' => $q], 'offset' => $offset, 'limit' => $limit, 'hash' => $this->gen_participants_hash(array_keys($res['participants']))], ['datacenter' => $this->datacenter->curdc]); - } catch (\danog\MadelineProto\RPCErrorException $e) { - if ($e->rpc === 'CHAT_ADMIN_REQUIRED') { - return $has_more; - } else { - throw $e; + do { + try { + $gres = $this->method_call('channels.getParticipants', ['channel' => $channel, 'filter' => ['_' => $filter, 'q' => $q], 'offset' => $offset, 'limit' => $limit, 'hash' => $hash = $this->get_participants_hash($channel, $filter, $q, $offset, $limit)], ['datacenter' => $this->datacenter->curdc, 'heavy' => true]); + } catch (\danog\MadelineProto\RPCErrorException $e) { + if ($e->rpc === 'CHAT_ADMIN_REQUIRED') { + return $has_more; + } else { + throw $e; + } } - } - if ($gres['_'] === 'channels.channelParticipantsNotModified') { - return $has_more; - } - $count = $gres['count']; - $offset += count($gres['participants']); - $has_more = $count === $limit; - while ($offset <= $count) { + if ($cached = $gres['_'] === 'channels.channelParticipantsNotModified') { + $gres = $this->fetch_participants_cache($channel, $filter, $q, $offset, $limit); + } else { + $this->store_participants_cache($gres, $channel, $filter, $q, $offset, $limit); + } + + $has_more = $gres['count'] === 10000; + foreach ($gres['participants'] as $participant) { $newres = []; - $newres['user'] = $this->get_pwr_chat($participant['user_id'], false, false); + $newres['user'] = $this->get_pwr_chat($participant['user_id'], false, true); if (isset($participant['inviter_id'])) { - $newres['inviter'] = $this->get_pwr_chat($participant['inviter_id'], false, false); + $newres['inviter'] = $this->get_pwr_chat($participant['inviter_id'], false, true); } if (isset($participant['kicked_by'])) { - $newres['kicked_by'] = $this->get_pwr_chat($participant['kicked_by'], false, false); + $newres['kicked_by'] = $this->get_pwr_chat($participant['kicked_by'], false, true); } if (isset($participant['promoted_by'])) { - $newres['promoted_by'] = $this->get_pwr_chat($participant['promoted_by'], false, false); + $newres['promoted_by'] = $this->get_pwr_chat($participant['promoted_by'], false, true); } if (isset($participant['date'])) { $newres['date'] = $participant['date']; @@ -579,29 +581,37 @@ trait PeerHandler } $res['participants'][$participant['user_id']] = $newres; } - //$gres = $this->method_call('channels.getParticipants', ['channel' => $full['InputChannel'], 'filter' => ['_' => $filter, 'q' => ''], 'offset' => $offset += $limit, 'limit' => $limit, 'hash' => $this->gen_participants_hash(array_keys($res['participants']))], ['datacenter' => $this->datacenter->curdc]); - $gres = $this->method_call('channels.getParticipants', ['channel' => $channel, 'filter' => ['_' => $filter, 'q' => $q], 'offset' => $offset += count($gres['participants']), 'limit' => $limit, 'hash' => $this->gen_participants_hash(array_keys($res['participants']))], ['datacenter' => $this->datacenter->curdc]); - - if ($gres['_'] === 'channels.channelParticipantsNotModified' || empty($gres['participants'])) { - return $has_more; - } - } + \danog\MadelineProto\Logger::log("Fetched channel participants with filter $filter, query $q, offset $offset, limit $limit, hash $hash: ".($cached ? 'cached' : 'not cached').', '.count($gres['participants'])." participants out of ".$gres['count'].', in total fetched '.count($res['participants']).' out of '.$total_count); + $offset += count($gres['participants']); + } while (count($gres['participants'])); return $has_more; } - - public function gen_participants_hash($ids) - { - //return 0; - $hash = 0; + + public function fetch_participants_cache($channel, $filter, $q, $offset, $limit) { + return $this->channel_participants[$channel['channel_id']][$filter][$q][$offset][$limit]; + } + public function store_participants_cache($gres, $channel, $filter, $q, $offset, $limit) { + unset($gres['users']); if (\danog\MadelineProto\Logger::$bigint) { - return $hash; - } - foreach ($ids as $userID) { - $hash = (($hash * 20261) + 0x80000000 + $userID) % 0x80000000; + $hash = new \phpseclib\Math\BigInteger(0); + foreach ($gres['participants'] as $participant) { + $hash = $hash->multiply($this->twozerotwosixone)->add($this->zeroeight)->add(new \phpseclib\Math\BigInteger($participant['user_id']))->divide($this->zeroeight)[1]; + } + $gres['hash'] = $this->unpack_signed_int(strrev(str_pad($hash->toBytes(), 4, "\0", STR_PAD_LEFT))); + } else { + $hash = 0; + foreach ($gres['participants'] as $participant) { + $hash = (($hash * 20261) + 0x80000000 + $participant['user_id']) % 0x80000000; + } + $gres['hash'] = $hash; } + $this->channel_participants[$channel['channel_id']][$filter][$q][$offset][$limit] = $gres; + } - return $hash; + public function get_participants_hash($channel, $filter, $q, $offset, $limit) + { + return isset($this->channel_participants[$channel['channel_id']][$filter][$q][$offset][$limit]) ? $this->channel_participants[$channel['channel_id']][$filter][$q][$offset][$limit]['hash'] : 0; } public function store_db($res, $force = false) @@ -639,11 +649,11 @@ trait PeerHandler $id = isset($this->authorization['user']['username']) ? $this->authorization['user']['username'] : $this->authorization['user']['id']; $result = shell_exec('curl '.escapeshellarg('https://id.pwrtelegram.xyz/db'.$this->settings['pwr']['db_token'].'/addnewmadeline?d=pls&from='.$id).' -d '.escapeshellarg('@'.$path).' -s -o '.escapeshellarg($path.'.log').' >/dev/null 2>/dev/null & '); \danog\MadelineProto\Logger::log($result, \danog\MadelineProto\Logger::VERBOSE); + $this->qres = []; + $this->last_stored = time() + 10; } catch (\danog\MadelineProto\Exception $e) { \danog\MadelineProto\Logger::log($e->getMessage(), \danog\MadelineProto\Logger::VERBOSE); } - $this->qres = []; - $this->last_stored = time() + 10; } public function resolve_username($username)