TL Schema moved to JSON mode.

Important comments added

mtproto.py

@@ -5,57 +5,33 @@ Created on Tue Sep  2 19:26:15 2014
 @author: agrigoryev
 """
 from binascii import crc32
-import struct
-import socket
-import re
 from datetime import datetime
-import sys
 import io
-import configparser
+import json
+import socket
+import struct
 
-current_module = sys.modules[__name__]
 
 def vis(bs):
-    l = len(bs)
+    """
-    n = len(bs) // 8
+    Function to visualize byte streams. Split into bytes, print to console.
+    :param bs: BYTE STRING
+    """
+    symbols_in_one_line = 8
+    n = len(bs) // symbols_in_one_line
     for i in range(n):
-        print(" ".join(["%02X" % b for b in bs[i*8:i*8+8]]))
+        print(" ".join(["%02X" % b for b in bs[i*symbols_in_one_line:(i+1)*symbols_in_one_line]])) # for every 8 symbols line
-    print(" ".join(["%02X" % b for b in bs[i*8+8:]])+"\n")
+    print(" ".join(["%02X" % b for b in bs[(i+1)*symbols_in_one_line:]])+"\n") # for last line
-
-
-class TlElement:
-    def __init__(self, type_string):
-        tl_re = re.compile("""([a-z]\w*)           #name
-                              \#?([0-9a-f]{8})?\s+ #id
-                              (\{.*\}\s+)?         #subtype
-                              (.*)\s+              #arguments list
-                              =\s+([A-Z]\w*)       #result
-                              (\s+(\w+))?;         #subresult""", re.X)
-        assert isinstance(type_string, str)
-        x = tl_re.match(type_string)
-        if x is not None:
-            self.name = x.groups()[0]
-            if x.groups()[1] is not None:
-                self.id = int(x.groups()[1], 16)
-            else:
-                self.id = crc32(re.sub("(#[0-9a-f]{8})|([;\n{}])", "", type_string).encode())
-            self.args = self.get_arg_list(x.groups()[3])
-            self.result = x.groups()[4]
-        else:
-            raise SyntaxError
-
-    @staticmethod
-    def get_arg_list(arg_string):
-        arg_re = re.compile("([\w0-9]+)?:?([\w0-9]+)(<([\w0-0]+)>)?")
-        res = []
-        for s in arg_re.findall(arg_string):
-            d = {'name': s[0], 'type': s[1], 'subtype': s[3] if s[2] is not None else None}
-            res.append(d)
-        return res
 
 
 class TL:
     def __init__(self):
+        with open("TL_schema.JSON", 'r') as f:
+            TL_dict = json.load(f)
+
+        self.methods = TL_dict['methods']
+        self.constructors = TL_dict['constructors']
+
         self.func_dict_id = {}
         self.func_dict_name = {}
         self.obj_dict_id = {}
@@ -63,83 +39,52 @@ class TL:
 
         # Read constructors
 
-        f = open("TL_schema", 'r')
+        for elem in self.constructors:
-        for line in f:
+            z = TlElement(elem)
-            if line.startswith("---functions---"):
+            self.func_dict_id = {}
-                break
-            try:
-                z = TlElement(line)
-                self.obj_dict_id[z.id] = z
-                self.obj_dict_name[z.result] = z
-            except SyntaxError:
-                pass
 
-        # Read methods
+    def serialize(self, type_, data):
+        # 1. Type constructor ID
+        # 2. type costructor params
 
-        for line in f:
+        # Bare types
-            if line.startswith("---functions---"):
+        if type_ == 'string':
-                break
+            assert isinstance(data, str)
-            try:
+            struct.pack("<L", len(data))
-                z = TlElement(line)
+        if type_ == 'int':
-                self.func_dict_id[z.id] = z
+            assert isinstance(data, str)
-                self.func_dict_name[z.name] = z
+            struct.pack("<L", len(data))
-            except SyntaxError:
-                pass
 
+    def deserialize(self, byte_string, type_=None, subtype=None):
 
-    def tl_serialize(self, elem, kwargs):
+        if isinstance(byte_string, io.BytesIO):
-        assert isinstance(elem, TlElement)
+            bytes_io = byte_string
-        for arg in elem.args:
+        elif isinstance(byte_string, bytes):
-            pass
+            bytes_io = io.BytesIO(byte_string)
-        return struct.pack("<L", )
-
-    def tl_function_generator(elem, session, tl):
-        def dummy(**kwargs):
-            message = tl_serialize(elem, kwargs)
-            session.send_message(message)
-            answer = session.recv_message()
-            return deserialize(answer, TL)
-
-    def class_generator(self, tl_element):
-        class Dummy:
-            def __init__(self, bstring):
-                assert isinstance(bstring, bytes)
-                f = io.BytesIO(bstring)
-                dict = self.deserialize(f, type=tl_element.type)
-
-    def serialize(self, type=None, subtype=None):
-        pass
-
-    def deserialize(self, string, type=None, subtype=None):
-
-        if isinstance(string, io.BytesIO):
-            bytes_io = string
-        elif isinstance(string, bytes):
-            bytes_io = io.BytesIO(string)
         else:
-            raise Exception("Bad input type, use bytes string or BytesIO object")
+            raise TypeError("Bad input type, use bytes string or BytesIO object")
 
         # Built-in bare types
 
-        if type == 'int':
+        if type_ == 'int':
             x = struct.unpack('<i', bytes_io.read(4))[0]
-        elif type == '#':
+        elif type_ == '#':
             x = struct.unpack('<I', bytes_io.read(4))[0]
-        elif type == 'long':
+        elif type_ == 'long':
             x = struct.unpack('<q', bytes_io.read(8))[0]
-        elif type == 'double':
+        elif type_ == 'double':
             x = struct.unpack('<d', bytes_io.read(8))[0]
-        elif type == 'int128':
+        elif type_ == 'int128':
             t = struct.unpack('<16s', bytes_io.read(16))[0]
             x = int.from_bytes(t, 'little')
-        elif type == 'int256':
+        elif type_ == 'int256':
             t = struct.unpack('<32s', bytes_io.read(32))[0]
             x = int.from_bytes(t, 'little')
-        elif type == 'bytes':
+        elif type_ == 'bytes':
             l = int.from_bytes(bytes_io.read(1), 'little')
             x = bytes_io.read(l)
             bytes_io.read(-(l+1) % 4)  # skip padding bytes
-        elif type == 'string':
+        elif type_ == 'string':
             l = int.from_bytes(bytes_io.read(1), 'little')
             assert l <=254
             if l == 254:
@@ -152,10 +97,10 @@ class TL:
                 x = bytes_io.read(l)
                 bytes_io.read(-(l+1) % 4)  # skip padding bytes
             assert isinstance(x, bytes)
-        elif type == 'vector':
+        elif type_ == 'vector':
             assert subtype is not None
             count = int.from_bytes(bytes_io.read(4), 'little')
-            x = [self.deserialize(bytes_io, type=subtype) for i in range(count)]
+            x = [self.deserialize(bytes_io, type_=subtype) for i in range(count)]
         else:
             # Boxed types
 
@@ -163,46 +108,72 @@ class TL:
             try:
                 tl_elem = self.obj_dict_id[i]
             except:
-                raise Exception("Could not extract type: %s" % type)
+                raise Exception("Could not extract type: %s" % type_)
             base_boxed_types = ["Vector", "Int", "Long", "Double", "String", "Int128", "Int256"]
             if tl_elem.result in base_boxed_types:
-                x = self.deserialize(bytes_io, type=tl_elem.name, subtype=subtype)
+                x = self.deserialize(bytes_io, type_=tl_elem.name, subtype=subtype)
 
             else:  # other types
                 x = {}
                 for arg in tl_elem.args:
-                    x[arg['name']] = self.deserialize(bytes_io, type=arg['type'], subtype=arg['subtype'])
+                    x[arg['name']] = self.deserialize(bytes_io, type_=arg['type'], subtype=arg['subtype'])
         return x
 
 
-
 class Session:
-    def __init__(self, credentials):
+    def __init__(self, ip, port):
-        config = configparser.ConfigParser()
+        # creating socket
-        config.read(credentials)
         self.sock = socket.socket()
-        self.sock.connect((config['App data']['ip_adress'], config['App data'].getint('port')))
+        self.sock.connect((ip, port))
-        self.api_id = config['App data'].getint('api_id')
+        self.auth_key_id = None
-        self.api_hash = config['App data']['api_hash']
-        self.auth_key_id = b'\x00\x00\x00\x00\x00\x00\x00\x00'
         self.number = 0
-        self.TL = TL()
 
-    def header(self, message):
+    def __del__(self):
-        return (self.auth_key_id +
+        # closing socket when session object is deleted
-                struct.pack('<Q', int(datetime.utcnow().timestamp()*2**32)) +
+        self.sock.close()
+
+    @staticmethod
+    def header_unencrypted(message):
+        """
+        Creating header for the unencrypted message:
+        :param message: byte string to send
+        """
+        # Basic instructions: https://core.telegram.org/mtproto/description#unencrypted-message
+
+        # Message id: https://core.telegram.org/mtproto/description#message-identifier-msg-id
+        msg_id = int(datetime.utcnow().timestamp()*2**30)*4
+
+        return (b'\x00\x00\x00\x00\x00\x00\x00\x00' +
+                struct.pack('<Q', msg_id) +
                 struct.pack('<L', len(message)))
 
+    # TCP Transport
+
+    # Instructions may be found here: https://core.telegram.org/mtproto#tcp-transport
+    # If a payload (packet) needs to be transmitted from server to client or from client to server,
+    # it is encapsulated as follows: 4 length bytes are added at the front (to include the length,
+    # the sequence number, and CRC32; always divisible by 4) and 4 bytes with the packet sequence number
+    # within this TCP connection (the first packet sent is numbered 0, the next one 1, etc.),
+    # and 4 CRC32 bytes at the end (length, sequence number, and payload together).
+
     def send_message(self, message):
+        """
+        Forming the message frame and sending message to server
+        :param message: byte string to send
+        """
+
         print('>>')
-        vis(message)
+        vis(message)  # Sending message visualisation to console
-        #self.number += 1
+        data = self.header_unencrypted(message) + message
-        data = self.header(message) + message
         step1 = struct.pack('<LL', len(data)+12, self.number) + data
         step2 = step1 + struct.pack('<L', crc32(step1))
         self.sock.send(step2)
+        self.number += 1
 
     def recv_message(self):
+        """
+        Reading socket and receiving message from server. Check the CRC32 and
+        """
         packet_length_data = self.sock.recv(4)
         if len(packet_length_data) > 0:  # if we have smth. in the socket
             packet_length = struct.unpack("<L", packet_length_data)[0]
@@ -213,7 +184,9 @@ class Session:
             message_length = struct.unpack("<I", packet[20:24])[0]
             data = packet[24:24+message_length]
             crc = packet[-4:]
-            if crc32(packet_length_data + packet[0:-4]).to_bytes(4,'little') == crc:
+
+            # Checking the CRC32 correctness of received data
+            if crc32(packet_length_data + packet[0:-4]).to_bytes(4, 'little') == crc:
                 print('<<')
-                vis(data)
+                vis(data)  # Received message visualisation to console
                 return data