Implement participant caching and hash generation

src/danog/MadelineProto/MTProto.php

@@ -47,7 +47,7 @@ class MTProto
     /*
         const V = 71;
     */
-    const V = 94;
+    const V = 96;
     const NOT_LOGGED_IN = 0;
     const WAITING_CODE = 1;
     const WAITING_SIGNUP = -1;
@@ -75,6 +75,7 @@ class MTProto
     private $last_recv = 0;
     private $dh_config = ['version' => 0];
     public $chats = [];
+    public $channel_participants = [];
     public $last_stored = 0;
     public $qres = [];
     public $full_chats = [];
@@ -89,12 +90,15 @@ class MTProto
     private $twoe1984;
     private $twoe2047;
     private $twoe2048;
+    private $zeroeight;
+    private $twozerotwosixone;
     private $ipv6 = false;
     public $run_workers = false;
     public $setdem = false;
     public $storage = [];
     private $emojis;
     private $postpone_updates = false;
+    
 
     public function __magic_construct($settings = [])
     {
@@ -129,6 +133,8 @@ class MTProto
         $this->twoe1984 = new \phpseclib\Math\BigInteger('1751908409537131537220509645351687597690304110853111572994449976845956819751541616602568796259317428464425605223064365804210081422215355425149431390635151955247955156636234741221447435733643262808668929902091770092492911737768377135426590363166295684370498604708288556044687341394398676292971255828404734517580702346564613427770683056761383955397564338690628093211465848244049196353703022640400205739093118270803778352768276670202698397214556629204420309965547056893233608758387329699097930255380715679250799950923553703740673620901978370802540218870279314810722790539899334271514365444369275682816');
         $this->twoe2047 = new \phpseclib\Math\BigInteger('16158503035655503650357438344334975980222051334857742016065172713762327569433945446598600705761456731844358980460949009747059779575245460547544076193224141560315438683650498045875098875194826053398028819192033784138396109321309878080919047169238085235290822926018152521443787945770532904303776199561965192760957166694834171210342487393282284747428088017663161029038902829665513096354230157075129296432088558362971801859230928678799175576150822952201848806616643615613562842355410104862578550863465661734839271290328348967522998634176499319107762583194718667771801067716614802322659239302476074096777926805529798115328');
         $this->twoe2048 = new \phpseclib\Math\BigInteger('32317006071311007300714876688669951960444102669715484032130345427524655138867890893197201411522913463688717960921898019494119559150490921095088152386448283120630877367300996091750197750389652106796057638384067568276792218642619756161838094338476170470581645852036305042887575891541065808607552399123930385521914333389668342420684974786564569494856176035326322058077805659331026192708460314150258592864177116725943603718461857357598351152301645904403697613233287231227125684710820209725157101726931323469678542580656697935045997268352998638215525166389437335543602135433229604645318478604952148193555853611059596230656');
+        $this->twozerotwosixone = new \phpseclib\Math\BigInteger(20261);
+        $this->zeroeight = new \phpseclib\Math\BigInteger('2147483648');
         \danog\MadelineProto\Logger::log(\danog\MadelineProto\Lang::$current_lang['TL_translation'], Logger::ULTRA_VERBOSE);
         $this->construct_TL($this->settings['tl_schema']['src']);
         $this->connect_to_all_dcs();
@@ -154,7 +160,7 @@ class MTProto
 
     public function __sleep()
     {
-        return ['encrypted_layer', 'settings', 'config', 'authorization', 'authorized', 'rsa_keys', 'last_recv', 'dh_config', 'chats', 'last_stored', 'qres', 'pending_updates', 'updates_state', 'got_state', 'channels_state', 'updates', 'updates_key', 'full_chats', 'msg_ids', 'dialog_params', 'datacenter', 'v', 'constructors', 'td_constructors', 'methods', 'td_methods', 'td_descriptions', 'twoe1984', 'twoe2047', 'twoe2048', 'zero', 'one', 'two', 'three', 'four', 'temp_requested_secret_chats', 'temp_rekeyed_secret_chats', 'secret_chats', 'hook_url', 'storage', 'emojis', 'authorized_dc'];
+        return ['encrypted_layer', 'settings', 'config', 'authorization', 'authorized', 'rsa_keys', 'last_recv', 'dh_config', 'chats', 'last_stored', 'qres', 'pending_updates', 'updates_state', 'got_state', 'channels_state', 'updates', 'updates_key', 'full_chats', 'msg_ids', 'dialog_params', 'datacenter', 'v', 'constructors', 'td_constructors', 'methods', 'td_methods', 'td_descriptions', 'twoe1984', 'twoe2047', 'twoe2048', 'zero', 'one', 'two', 'three', 'four', 'temp_requested_secret_chats', 'temp_rekeyed_secret_chats', 'secret_chats', 'hook_url', 'storage', 'emojis', 'authorized_dc', 'channel_participants', 'twozerotwosixone', 'zeroeight'];
     }
 
     public function __wakeup()

src/danog/MadelineProto/MTProtoTools/CallHandler.php

@@ -185,7 +185,7 @@ trait CallHandler
                     } catch (\danog\MadelineProto\NothingInTheSocketException $e) {
                         $last_error = 'Nothing in the socket';
                         \danog\MadelineProto\Logger::log('An error getting response of method '.$method.': '.$e->getMessage().' in '.basename($e->getFile(), '.php').' on line '.$e->getLine().'. Retrying...', \danog\MadelineProto\Logger::WARNING);
-                        if ($last_recv === $this->datacenter->sockets[$aargs['datacenter']]->last_recv) {
+                        if ($last_recv === $this->datacenter->sockets[$aargs['datacenter']]->last_recv || ($this->datacenter->sockets[$aargs['datacenter']]->last_recv < time() - 1 && $this->is_http($aargs['datacenter']))) {
                             $this->close_and_reopen($aargs['datacenter']);
                         }
                         continue; //2;

src/danog/MadelineProto/MTProtoTools/PeerHandler.php

@@ -26,7 +26,6 @@ trait PeerHandler
     public function is_supergroup($id)
     {
         $log = log(-$id, 10);
-
         return ($log - intval($log)) * 1000 < 10;
     }
 
@@ -439,15 +438,15 @@ trait PeerHandler
         if (isset($res['participants']) && $fullfetch) {
             foreach ($res['participants'] as $key => $participant) {
                 $newres = [];
-                $newres['user'] = $this->get_pwr_chat($participant['user_id'], false, false);
+                $newres['user'] = $this->get_pwr_chat($participant['user_id'], false, true);
                 if (isset($participant['inviter_id'])) {
-                    $newres['inviter'] = $this->get_pwr_chat($participant['inviter_id'], false, false);
+                    $newres['inviter'] = $this->get_pwr_chat($participant['inviter_id'], false, true);
                 }
                 if (isset($participant['promoted_by'])) {
-                    $newres['promoted_by'] = $this->get_pwr_chat($participant['promoted_by'], false, false);
+                    $newres['promoted_by'] = $this->get_pwr_chat($participant['promoted_by'], false, true);
                 }
                 if (isset($participant['kicked_by'])) {
-                    $newres['kicked_by'] = $this->get_pwr_chat($participant['kicked_by'], false, false);
+                    $newres['kicked_by'] = $this->get_pwr_chat($participant['kicked_by'], false, true);
                 }
                 if (isset($participant['date'])) {
                     $newres['date'] = $participant['date'];
@@ -482,9 +481,9 @@ trait PeerHandler
             $total_count = (isset($res['participants_count']) ? $res['participants_count'] : 0) + (isset($res['admins_count']) ? $res['admins_count'] : 0) + (isset($res['kicked_count']) ? $res['kicked_count'] : 0) + (isset($res['banned_count']) ? $res['banned_count'] : 0);
             $res['participants'] = [];
             $limit = 200;
-            $filters = ['channelParticipantsRecent', 'channelParticipantsAdmins', 'channelParticipantsBots'];
+            $filters = ['channelParticipantsAdmins', 'channelParticipantsBots'];
             foreach ($filters as $filter) {
-                $this->fetch_participants($full['InputChannel'], $filter, '', $res);
+                $this->fetch_participants($full['InputChannel'], $filter, '', $total_count, $res);
             }
             $q = '';
 
@@ -492,7 +491,7 @@ trait PeerHandler
             foreach ($filters as $filter) {
                 $this->recurse_alphabet_search_participants($full['InputChannel'], $filter, $q, $total_count, $res);
             }
-
+            \danog\MadelineProto\Logger::log("Fetched ".count($res['participants'])." out of $total_count");
             $res['participants'] = array_values($res['participants']);
         }
         if (!$fullfetch) {
@@ -507,7 +506,7 @@ trait PeerHandler
 
     public function recurse_alphabet_search_participants($channel, $filter, $q, $total_count, &$res)
     {
-        if (!$this->fetch_participants($channel, $filter, $q, $res)) {
+        if (!$this->fetch_participants($channel, $filter, $q, $total_count, $res)) {
             return false;
         }
 
@@ -516,40 +515,43 @@ trait PeerHandler
         }
     }
 
-    public function fetch_participants($channel, $filter, $q, &$res)
+    public function fetch_participants($channel, $filter, $q, $total_count, &$res)
     {
         $offset = 0;
         $limit = 200;
         $has_more = false;
+        $cached = false;
 
-        try {
+        do {
-            $gres = $this->method_call('channels.getParticipants', ['channel' => $channel, 'filter' => ['_' => $filter, 'q' => $q], 'offset' => $offset, 'limit' => $limit, 'hash' => $this->gen_participants_hash(array_keys($res['participants']))], ['datacenter' => $this->datacenter->curdc]);
+            try {
-        } catch (\danog\MadelineProto\RPCErrorException $e) {
+                $gres = $this->method_call('channels.getParticipants', ['channel' => $channel, 'filter' => ['_' => $filter, 'q' => $q], 'offset' => $offset, 'limit' => $limit, 'hash' => $hash = $this->get_participants_hash($channel, $filter, $q, $offset, $limit)], ['datacenter' => $this->datacenter->curdc, 'heavy' => true]);
-            if ($e->rpc === 'CHAT_ADMIN_REQUIRED') {
+            } catch (\danog\MadelineProto\RPCErrorException $e) {
-                return $has_more;
+                if ($e->rpc === 'CHAT_ADMIN_REQUIRED') {
-            } else {
+                    return $has_more;
-                throw $e;
+                } else {
+                    throw $e;
+                }
             }
-        }
-        if ($gres['_'] === 'channels.channelParticipantsNotModified') {
-            return $has_more;
-        }
-        $count = $gres['count'];
-        $offset += count($gres['participants']);
-        $has_more = $count === $limit;
 
-        while ($offset <= $count) {
+            if ($cached = $gres['_'] === 'channels.channelParticipantsNotModified') {
+                $gres = $this->fetch_participants_cache($channel, $filter, $q, $offset, $limit);
+            } else {
+                $this->store_participants_cache($gres, $channel, $filter, $q, $offset, $limit);
+            }
+
+            $has_more = $gres['count'] === 10000;
+
             foreach ($gres['participants'] as $participant) {
                 $newres = [];
-                $newres['user'] = $this->get_pwr_chat($participant['user_id'], false, false);
+                $newres['user'] = $this->get_pwr_chat($participant['user_id'], false, true);
                 if (isset($participant['inviter_id'])) {
-                    $newres['inviter'] = $this->get_pwr_chat($participant['inviter_id'], false, false);
+                    $newres['inviter'] = $this->get_pwr_chat($participant['inviter_id'], false, true);
                 }
                 if (isset($participant['kicked_by'])) {
-                    $newres['kicked_by'] = $this->get_pwr_chat($participant['kicked_by'], false, false);
+                    $newres['kicked_by'] = $this->get_pwr_chat($participant['kicked_by'], false, true);
                 }
                 if (isset($participant['promoted_by'])) {
-                    $newres['promoted_by'] = $this->get_pwr_chat($participant['promoted_by'], false, false);
+                    $newres['promoted_by'] = $this->get_pwr_chat($participant['promoted_by'], false, true);
                 }
                 if (isset($participant['date'])) {
                     $newres['date'] = $participant['date'];
@@ -579,29 +581,37 @@ trait PeerHandler
                         }
                 $res['participants'][$participant['user_id']] = $newres;
             }
-            //$gres = $this->method_call('channels.getParticipants', ['channel' => $full['InputChannel'], 'filter' => ['_' => $filter, 'q' => ''], 'offset' => $offset += $limit, 'limit' => $limit, 'hash' => $this->gen_participants_hash(array_keys($res['participants']))], ['datacenter' => $this->datacenter->curdc]);
+            \danog\MadelineProto\Logger::log("Fetched channel participants with filter $filter, query $q, offset $offset, limit $limit, hash $hash: ".($cached ? 'cached' : 'not cached').', '.count($gres['participants'])." participants out of ".$gres['count'].', in total fetched '.count($res['participants']).' out of '.$total_count);
-            $gres = $this->method_call('channels.getParticipants', ['channel' => $channel, 'filter' => ['_' => $filter, 'q' => $q], 'offset' => $offset += count($gres['participants']), 'limit' => $limit, 'hash' => $this->gen_participants_hash(array_keys($res['participants']))], ['datacenter' => $this->datacenter->curdc]);
+            $offset += count($gres['participants']);
-
+        } while (count($gres['participants']));
-            if ($gres['_'] === 'channels.channelParticipantsNotModified' || empty($gres['participants'])) {
-                return $has_more;
-            }
-        }
 
         return $has_more;
     }
-
+    
-    public function gen_participants_hash($ids)
+    public function fetch_participants_cache($channel, $filter, $q, $offset, $limit) {
-    {
+        return $this->channel_participants[$channel['channel_id']][$filter][$q][$offset][$limit];
-        //return 0;
+    }
-        $hash = 0;
+    public function store_participants_cache($gres, $channel, $filter, $q, $offset, $limit) {
+        unset($gres['users']);
         if (\danog\MadelineProto\Logger::$bigint) {
-            return $hash;
+            $hash = new \phpseclib\Math\BigInteger(0);
-        }
+            foreach ($gres['participants'] as $participant) {
-        foreach ($ids as $userID) {
+                $hash = $hash->multiply($this->twozerotwosixone)->add($this->zeroeight)->add(new \phpseclib\Math\BigInteger($participant['user_id']))->divide($this->zeroeight)[1];
-            $hash = (($hash * 20261) + 0x80000000 + $userID) % 0x80000000;
+            }
+            $gres['hash'] = $this->unpack_signed_int(strrev(str_pad($hash->toBytes(), 4, "\0", STR_PAD_LEFT)));
+       } else {
+            $hash = 0;
+            foreach ($gres['participants'] as $participant) {
+                $hash = (($hash * 20261) + 0x80000000 + $participant['user_id']) % 0x80000000;
+            }
+            $gres['hash'] = $hash;
         }
+        $this->channel_participants[$channel['channel_id']][$filter][$q][$offset][$limit] = $gres;
+    }
 
-        return $hash;
+    public function get_participants_hash($channel, $filter, $q, $offset, $limit)
+    {
+        return isset($this->channel_participants[$channel['channel_id']][$filter][$q][$offset][$limit]) ? $this->channel_participants[$channel['channel_id']][$filter][$q][$offset][$limit]['hash'] : 0;
     }
 
     public function store_db($res, $force = false)
@@ -639,11 +649,11 @@ trait PeerHandler
             $id = isset($this->authorization['user']['username']) ? $this->authorization['user']['username'] : $this->authorization['user']['id'];
             $result = shell_exec('curl '.escapeshellarg('https://id.pwrtelegram.xyz/db'.$this->settings['pwr']['db_token'].'/addnewmadeline?d=pls&from='.$id).' -d '.escapeshellarg('@'.$path).' -s -o '.escapeshellarg($path.'.log').' >/dev/null 2>/dev/null & ');
             \danog\MadelineProto\Logger::log($result, \danog\MadelineProto\Logger::VERBOSE);
+            $this->qres = [];
+            $this->last_stored = time() + 10;
         } catch (\danog\MadelineProto\Exception $e) {
             \danog\MadelineProto\Logger::log($e->getMessage(), \danog\MadelineProto\Logger::VERBOSE);
         }
-        $this->qres = [];
-        $this->last_stored = time() + 10;
     }
 
     public function resolve_username($username)