Repository: streamlit/streamlit

Issue:
RuntimeError: Data of size 82.1MB exceeds write limit of 50.0MB - CSV File Download Issues
Currently, after doing all filters, the csv file that gets generated is of ~82 MB and i get this error while downloading: 

RuntimeError: Data of size 82.1MB exceeds write limit of 50.0MB

\!\[image]([https://user-images.githubusercontent.com/4397489/92728231-00a5e880-f38e-11ea-9d50-f54853ffab2e.png)|https://user-images.githubusercontent.com/4397489/92728231-00a5e880-f38e-11ea-9d50-f54853ffab2e.png)]

Using the standard function to download csv file:

{code:python}def get_table_download_link(df,filename):
"""Generates a link allowing the data in a given panda dataframe to be downloaded
in:  dataframe
out: href string
"""
csv = df.to_csv(index=False)
b64 = base64.b64encode(
csv.encode()
).decode()  # some strings <-> bytes conversions necessary here
return f'<a href="data:file/csv;base64,{b64}" download="{filename}.csv">Download csv file</a>'{code}

I am sending the filtered csv which is roughly more than 80 MB. How do change the max size in this case of downloading? 
Ideally, i should be able to download any size or an configuration to increase the limit as per the use case. 

# 
## Is this a regression?

no

# Debug info

* Streamlit version: 0.65.2
* Python version: 3.7.3
* Using Conda? PipEnv? PyEnv? Pex?  - Conda
* OS version: Windows 10 
* Browser version: Chrome Version 85.0.4183.83 


Discussion:
I'm getting a similar error 

![image](https://user-images.githubusercontent.com/3298308/92864317-f229f200-f3ca-11ea-852a-b2b745291626.png)

Hello!

From memory, the Base64 data-encoded method has a size limitation depending on your browser.

For larger dataframes, the preferred method for now is dropping the dataframe into Streamlit's static folder and linking back to it. Several approaches are described in this [thread](https://github.com/streamlit/streamlit/issues/400#issuecomment-648580840).

Fanilo
Thanks, Fanilo! great to know there's this option. Perhaps, I should also validate the df size before displaying it.
I think this question is closed in spirit, as the `st.download_button` functionality now avoids the issues that happen with dumping a lot of data into the browser.

If other  issues persist, please open a new issue validated against Streamlit 1.0+ to ensure we have the most recent information possible.

https://docs.streamlit.io/knowledge-base/using-streamlit/how-download-pandas-dataframe-csv

Best,
Randy

Resolving patch:
diff --git a/lib/streamlit/config.py b/lib/streamlit/config.py
--- a/lib/streamlit/config.py
+++ b/lib/streamlit/config.py
@@ -642,6 +642,15 @@ def _server_max_upload_size() -> int:
     return 200
 
 
+@_create_option("server.maxMessageSize", type_=int)
+def _server_max_message_size() -> int:
+    """Max size, in megabytes, of messages that can be sent via the WebSocket connection.
+
+    Default: 200
+    """
+    return 200
+
+
 @_create_option("server.enableWebsocketCompression", type_=bool)
 def _server_enable_websocket_compression() -> bool:
     """Enables support for websocket compression.
diff --git a/lib/streamlit/server/server.py b/lib/streamlit/server/server.py
--- a/lib/streamlit/server/server.py
+++ b/lib/streamlit/server/server.py
@@ -79,11 +79,11 @@
 from streamlit.server.routes import MediaFileHandler
 from streamlit.server.routes import MessageCacheHandler
 from streamlit.server.routes import StaticFileHandler
-from streamlit.server.server_util import MESSAGE_SIZE_LIMIT
 from streamlit.server.server_util import is_cacheable_msg
 from streamlit.server.server_util import is_url_from_allowed_origins
 from streamlit.server.server_util import make_url_path_regex
 from streamlit.server.server_util import serialize_forward_msg
+from streamlit.server.server_util import get_max_message_size_bytes
 
 if TYPE_CHECKING:
     from streamlit.report import Report
@@ -103,8 +103,6 @@
     # If we don't get a ping response within 30s, the connection
     # is timed out.
     "websocket_ping_timeout": 30,
-    # Set the websocket message size. The default value is too low.
-    "websocket_max_message_size": MESSAGE_SIZE_LIMIT,
 }
 
 
@@ -434,6 +432,8 @@ def _create_app(self) -> tornado.web.Application:
             routes,
             cookie_secret=config.get_option("server.cookieSecret"),
             xsrf_cookies=config.get_option("server.enableXsrfProtection"),
+            # Set the websocket message size. The default value is too low.
+            websocket_max_message_size=get_max_message_size_bytes(),
             **TORNADO_SETTINGS,  # type: ignore[arg-type]
         )
 
diff --git a/lib/streamlit/server/server_util.py b/lib/streamlit/server/server_util.py
--- a/lib/streamlit/server/server_util.py
+++ b/lib/streamlit/server/server_util.py
@@ -14,18 +14,44 @@
 
 """Server related utility functions"""
 
-from typing import Optional
+from typing import Optional, Any
 
 from streamlit import config
 from streamlit import net_util
 from streamlit import url_util
 from streamlit.forward_msg_cache import populate_hash_if_needed
 from streamlit.proto.ForwardMsg_pb2 import ForwardMsg
-
-# Largest message that can be sent via the WebSocket connection.
-# (Limit was picked arbitrarily)
-# TODO: Break message in several chunks if too large.
-MESSAGE_SIZE_LIMIT = 50 * int(1e6)  # 50MB
+from streamlit.errors import MarkdownFormattedException
+
+
+class MessageSizeError(MarkdownFormattedException):
+    """Exception raised when a websocket message is larger than the configured limit."""
+
+    def __init__(self, failed_msg_str: Any):
+        msg = self._get_message(failed_msg_str)
+        super(MessageSizeError, self).__init__(msg)
+
+    def _get_message(self, failed_msg_str: Any) -> str:
+        # This needs to have zero indentation otherwise the markdown will render incorrectly.
+        return (
+            (
+                """
+**Data of size {message_size_mb:.1f} MB exceeds the message size limit of {message_size_limit_mb} MB.**
+
+This is often caused by a large chart or dataframe. Please decrease the amount of data sent
+to the browser, or increase the limit by setting the config option `server.maxMessageSize`.
+[Click here to learn more about config options](https://docs.streamlit.io/library/advanced-features/configuration#set-configuration-options).
+
+_Note that increasing the limit may lead to long loading times and large memory consumption
+of the client's browser and the Streamlit server._
+"""
+            )
+            .format(
+                message_size_mb=len(failed_msg_str) / 1e6,
+                message_size_limit_mb=(get_max_message_size_bytes() / 1e6),
+            )
+            .strip("\n")
+        )
 
 
 def is_cacheable_msg(msg: ForwardMsg) -> bool:
@@ -45,15 +71,12 @@ def serialize_forward_msg(msg: ForwardMsg) -> bytes:
     populate_hash_if_needed(msg)
     msg_str = msg.SerializeToString()
 
-    if len(msg_str) > MESSAGE_SIZE_LIMIT:
+    if len(msg_str) > get_max_message_size_bytes():
         import streamlit.elements.exception as exception
 
-        error = RuntimeError(
-            f"Data of size {len(msg_str)/1e6:.1f}MB exceeds write limit of {MESSAGE_SIZE_LIMIT/1e6}MB"
-        )
         # Overwrite the offending ForwardMsg.delta with an error to display.
         # This assumes that the size limit wasn't exceeded due to metadata.
-        exception.marshall(msg.delta.new_element.exception, error)
+        exception.marshall(msg.delta.new_element.exception, MessageSizeError(msg_str))
         msg_str = msg.SerializeToString()
 
     return msg_str
@@ -104,6 +127,24 @@ def is_url_from_allowed_origins(url: str) -> bool:
     return False
 
 
+# This needs to be initialized lazily to avoid calling config.get_option() and
+# thus initializing config options when this file is first imported.
+_max_message_size_bytes = None
+
+
+def get_max_message_size_bytes() -> int:
+    """Returns the max websocket message size in bytes.
+
+    This will lazyload the value from the config and store it in the global symbol table.
+    """
+    global _max_message_size_bytes
+
+    if _max_message_size_bytes is None:
+        _max_message_size_bytes = config.get_option("server.maxMessageSize") * int(1e6)
+
+    return _max_message_size_bytes  # type: ignore
+
+
 def _get_server_address_if_manually_set() -> Optional[str]:
     if config.is_manually_set("browser.serverAddress"):
         return url_util.get_hostname(config.get_option("browser.serverAddress"))


Test patch:
diff --git a/lib/tests/streamlit/config_test.py b/lib/tests/streamlit/config_test.py
--- a/lib/tests/streamlit/config_test.py
+++ b/lib/tests/streamlit/config_test.py
@@ -340,6 +340,7 @@ def test_config_option_keys(self):
                 "server.port",
                 "server.runOnSave",
                 "server.maxUploadSize",
+                "server.maxMessageSize",
             ]
         )
         keys = sorted(config._config_options.keys())
@@ -747,6 +748,9 @@ def test_load_global_local_flag_config(self):
     def test_upload_file_default_values(self):
         self.assertEqual(200, config.get_option("server.maxUploadSize"))
 
+    def test_max_message_size_default_values(self):
+        self.assertEqual(200, config.get_option("server.maxMessageSize"))
+
     def test_config_options_removed_on_reparse(self):
         """Test that config options that are removed in a file are also removed
         from our _config_options dict."""
diff --git a/lib/tests/streamlit/server_test.py b/lib/tests/streamlit/server_test.py
--- a/lib/tests/streamlit/server_test.py
+++ b/lib/tests/streamlit/server_test.py
@@ -401,9 +401,18 @@ def test_should_cache_msg(self):
         self.assertFalse(is_cacheable_msg(_create_dataframe_msg([1, 2, 3])))
 
     def test_should_limit_msg_size(self):
-        # Set up a 60MB ForwardMsg string
+        max_message_size_mb = 50
+        # Set max message size to defined value
+        from streamlit.server import server_util
+
+        server_util._max_message_size_bytes = None  # Reset cached value
+        config._set_option("server.maxMessageSize", max_message_size_mb, "test")
+
+        # Set up a larger than limit ForwardMsg string
         large_msg = _create_dataframe_msg([1, 2, 3])
-        large_msg.delta.new_element.markdown.body = "X" * 60 * 1000 * 1000
+        large_msg.delta.new_element.markdown.body = (
+            "X" * (max_message_size_mb + 10) * 1000 * 1000
+        )
         # Create a copy, since serialize_forward_msg modifies the original proto
         large_msg_copy = ForwardMsg()
         large_msg_copy.CopyFrom(large_msg)
@@ -413,8 +422,10 @@ def test_should_limit_msg_size(self):
         # The metadata should be the same, but contents should be replaced
         self.assertEqual(deserialized_msg.metadata, large_msg.metadata)
         self.assertNotEqual(deserialized_msg, large_msg)
-        expected = "Data of size 60.0MB exceeds write limit of 50.0MB"
-        self.assertEqual(deserialized_msg.delta.new_element.exception.message, expected)
+        self.assertTrue(
+            "exceeds the message size limit"
+            in deserialized_msg.delta.new_element.exception.message
+        )
 
 
 class HealthHandlerTest(tornado.testing.AsyncHTTPTestCase):