Repository: mindsdb/mindsdb

Issue:
MYSQL API fails to return large packets
When we are returning too many prediction on a `predict` call the mysql mime proxy breaks, error is:


```
2020-06-25 21:34:22,721 - ERROR - ERROR while executing query: SELECT `rental_price`, `$select_data_query` FROM `demo_rental_price_predictor_mariadb` WHERE $select_data_query= 'SELECT * FROM test.home_rentals limit 1000'
Traceback (most recent call last):
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 805, in handle
    self.queryAnswer(sql)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 461, in queryAnswer
    return self.selectAnswer(query)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 726, in selectAnswer
    self.answerTableQuery(query)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 279, in answerTableQuery
    self.sendPackageGroup(packages)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 240, in sendPackageGroup
    string = b''.join([x.accum() for x in packages])
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 240, in <listcomp>
    string = b''.join([x.accum() for x in packages])
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 92, in accum
    string = self.getPacketString()
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 58, in getPacketString
    count_header = struct.pack('b', self.seq)
struct.error: byte format requires -128 <= number <= 127

byte format requires -128 <= number <= 127
ERROR:mindsdb_sql:ERROR while executing query: SELECT `rental_price`, `$select_data_query` FROM `demo_rental_price_predictor_mariadb` WHERE $select_data_query= 'SELECT * FROM test.home_rentals limit 1000'
Traceback (most recent call last):
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 805, in handle
    self.queryAnswer(sql)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 461, in queryAnswer
    return self.selectAnswer(query)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 726, in selectAnswer
    self.answerTableQuery(query)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 279, in answerTableQuery
    self.sendPackageGroup(packages)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 240, in sendPackageGroup
    string = b''.join([x.accum() for x in packages])
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 240, in <listcomp>
    string = b''.join([x.accum() for x in packages])
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 92, in accum
    string = self.getPacketString()
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 58, in getPacketString
    count_header = struct.pack('b', self.seq)
struct.error: byte format requires -128 <= number <= 127

byte format requires -128 <= number <= 127
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 37752)
Traceback (most recent call last):
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 805, in handle
    self.queryAnswer(sql)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 461, in queryAnswer
    return self.selectAnswer(query)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 726, in selectAnswer
    self.answerTableQuery(query)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 279, in answerTableQuery
    self.sendPackageGroup(packages)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 240, in sendPackageGroup
    string = b''.join([x.accum() for x in packages])
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 240, in <listcomp>
    string = b''.join([x.accum() for x in packages])
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 92, in accum
    string = self.getPacketString()
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 58, in getPacketString
    count_header = struct.pack('b', self.seq)
struct.error: byte format requires -128 <= number <= 127

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.7/socketserver.py", line 650, in process_request_thread
    self.finish_request(request, client_address)
  File "/usr/lib/python3.7/socketserver.py", line 360, in finish_request
    self.RequestHandlerClass(request, client_address, self)
  File "/usr/lib/python3.7/socketserver.py", line 720, in __init__
    self.handle()
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 815, in handle
    msg=str(e)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 86, in send
    string = self.getPacketString()
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 58, in getPacketString
    count_header = struct.pack('b', self.seq)
struct.error: byte format requires -128 <= number <= 127
----------------------------------------
```
MYSQL API fails to return large packets
When we are returning too many prediction on a `predict` call the mysql mime proxy breaks, error is:


```
2020-06-25 21:34:22,721 - ERROR - ERROR while executing query: SELECT `rental_price`, `$select_data_query` FROM `demo_rental_price_predictor_mariadb` WHERE $select_data_query= 'SELECT * FROM test.home_rentals limit 1000'
Traceback (most recent call last):
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 805, in handle
    self.queryAnswer(sql)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 461, in queryAnswer
    return self.selectAnswer(query)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 726, in selectAnswer
    self.answerTableQuery(query)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 279, in answerTableQuery
    self.sendPackageGroup(packages)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 240, in sendPackageGroup
    string = b''.join([x.accum() for x in packages])
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 240, in <listcomp>
    string = b''.join([x.accum() for x in packages])
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 92, in accum
    string = self.getPacketString()
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 58, in getPacketString
    count_header = struct.pack('b', self.seq)
struct.error: byte format requires -128 <= number <= 127

byte format requires -128 <= number <= 127
ERROR:mindsdb_sql:ERROR while executing query: SELECT `rental_price`, `$select_data_query` FROM `demo_rental_price_predictor_mariadb` WHERE $select_data_query= 'SELECT * FROM test.home_rentals limit 1000'
Traceback (most recent call last):
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 805, in handle
    self.queryAnswer(sql)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 461, in queryAnswer
    return self.selectAnswer(query)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 726, in selectAnswer
    self.answerTableQuery(query)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 279, in answerTableQuery
    self.sendPackageGroup(packages)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 240, in sendPackageGroup
    string = b''.join([x.accum() for x in packages])
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 240, in <listcomp>
    string = b''.join([x.accum() for x in packages])
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 92, in accum
    string = self.getPacketString()
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 58, in getPacketString
    count_header = struct.pack('b', self.seq)
struct.error: byte format requires -128 <= number <= 127

byte format requires -128 <= number <= 127
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 37752)
Traceback (most recent call last):
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 805, in handle
    self.queryAnswer(sql)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 461, in queryAnswer
    return self.selectAnswer(query)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 726, in selectAnswer
    self.answerTableQuery(query)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 279, in answerTableQuery
    self.sendPackageGroup(packages)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 240, in sendPackageGroup
    string = b''.join([x.accum() for x in packages])
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 240, in <listcomp>
    string = b''.join([x.accum() for x in packages])
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 92, in accum
    string = self.getPacketString()
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 58, in getPacketString
    count_header = struct.pack('b', self.seq)
struct.error: byte format requires -128 <= number <= 127

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.7/socketserver.py", line 650, in process_request_thread
    self.finish_request(request, client_address)
  File "/usr/lib/python3.7/socketserver.py", line 360, in finish_request
    self.RequestHandlerClass(request, client_address, self)
  File "/usr/lib/python3.7/socketserver.py", line 720, in __init__
    self.handle()
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py", line 815, in handle
    msg=str(e)
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 86, in send
    string = self.getPacketString()
  File "/home/george/mindsdb/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py", line 58, in getPacketString
    count_header = struct.pack('b', self.seq)
struct.error: byte format requires -128 <= number <= 127
----------------------------------------
```


Discussion:

Resolving patch:
diff --git a/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py b/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py
--- a/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py
+++ b/mindsdb/api/mysql/mysql_proxy/data_types/mysql_packet.py
@@ -48,14 +48,14 @@ def setBody(self, body_string):
 
     def loadFromPacketString(self, packet_string):
         len_header = struct.unpack('>i', struct.pack('1s', '') + packet_string[:3])[0]
-        count_header = struct.unpack('b', packet_string[3])[0]
+        count_header = struct.unpack('B', packet_string[3])[0]
         body = packet_string[4:]
         self.loadFromParams(length=len_header, seq=count_header, body=body)
 
     def getPacketString(self):
         body = self.body
         len_header = struct.pack('<i', self.length)[:3]  # keep it 3 bytes
-        count_header = struct.pack('b', self.seq)
+        count_header = struct.pack('B', self.seq)
         packet = len_header + count_header + body
         return packet
 
@@ -77,7 +77,7 @@ def get(self):
                 break
             body += self.mysql_socket.recv(len_header)
         self.session.logging.info(f'Got packet: {str(body)}')
-        self.proxy.count = int(count_header) + 1
+        self.proxy.count = (int(count_header) + 1) % 256
         self.setup(len(body), count_header, body)
         return True
 
diff --git a/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py b/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py
--- a/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py
+++ b/mindsdb/api/mysql/mysql_proxy/mysql_proxy.py
@@ -857,7 +857,7 @@ def packet(self, packetClass=Packet, **kwargs):
         :return:
         """
         p = packetClass(socket=self.socket, seq=self.count, session=self.session, proxy=self, **kwargs)
-        self.count += 1
+        self.count = (self.count + 1) % 256
         return p
 
     @staticmethod


Test patch: