Repository: streamlit/streamlit

Issue:
When using `add_rows` with `datetime` index, the chart screen is not refreshed.
### Summary

When using `add_rows` with `datetime` index, the chart screen is not refreshed.

### Steps to reproduce

Code snippet:

```
import os
import time
import random
import streamlit as st
import pandas as pd
import numpy as np 

current_time = pd.to_datetime("08:00:00 2021-01-01", utc=True)
simulation_step: pd.Timedelta = pd.Timedelta(seconds=10)
# current_time = 1
# simulation_step = 1

if 'run_mode' not in st.session_state:
    st.session_state['run_mode'] = False

bt1 = st.sidebar.button(f'Start/Stop')
if bt1:
    if st.session_state.run_mode:
        st.session_state.run_mode = False
    else:
        st.session_state.run_mode = True

data = pd.DataFrame(data=[[current_time, float(random.randint(1, 10))]], columns=['t', 'y']).set_index("t")
text_field = st.empty()
line_chart = st.line_chart(data=data, use_container_width=True)
my_table = st.table(data)

while st.session_state.run_mode is True:
    current_time += simulation_step
    text_field.text(f"time: {current_time}")

    data_dif = pd.DataFrame(data=[[current_time, float(random.randint(1, 10))]], columns=['t', 'y']).set_index("t")
    line_chart.add_rows(data_dif)
    my_table.add_rows(data_dif)
    time.sleep(1)

```

**Expected behavior:**

The screen should be refreshed on each iteration.

**Actual behavior:**

It behaves as expected if using numeric type for index column
[screen record](https://drive.google.com/file/d/1yJ7c_5-mBqKuyZiX9dNvfvtY5JPggVCq/view?usp=sharing)

### Debug info

- Streamlit version: 0.85.1
- Python version: 3.7.10
- Using: PyEnv
- OS version: MacOS Catalina (10.15.7_
- Browser version: Chrome 92.0.4515.107 (Official Build) (x86_64)



Discussion:
Hey @DmitryBe Thanks for the report! I was able to confirm this. Looks like an issue with our implementation of Arrow. The best workaround, for now, is to run `streamlit run script.py --global.dataFrameSerialization legacy` for now.

Resolving patch:
diff --git a/e2e/scripts/st_arrow_line_chart_add_rows_special.py b/e2e/scripts/st_arrow_line_chart_add_rows_special.py
new file mode 100644
--- /dev/null
+++ b/e2e/scripts/st_arrow_line_chart_add_rows_special.py
@@ -0,0 +1,40 @@
+# Copyright 2018-2021 Streamlit Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This example was failing due to an issue (#3653) in st._arrow_add_rows.
+In the previos implementation of Quiver, we were mutating the Quiver element
+in the addRows function, which prevented re-rendering of the line chart.
+This example reproduces the issue, so that we don't repeat the same mistake
+in the future.
+"""
+
+import time
+
+import pandas as pd
+import streamlit as st
+
+current_time = pd.to_datetime("08:00:00 2021-01-01", utc=True)
+simulation_step = pd.Timedelta(seconds=10)
+
+df1 = pd.DataFrame(data=[[current_time, 1]], columns=["t", "y"]).set_index("t")
+line_chart = st._arrow_line_chart(df1, use_container_width=True)
+
+for count in range(5):
+    current_time += simulation_step
+
+    df2 = pd.DataFrame(data=[[current_time, count]], columns=["t", "y"]).set_index("t")
+    # This issue is Arrow specific, that's why st._legacy_add_rows doesn't have a similar test.
+    line_chart._arrow_add_rows(df2)
+    time.sleep(1)


Test patch:
diff --git a/frontend/src/lib/Quiver.test.ts b/frontend/src/lib/Quiver.test.ts
--- a/frontend/src/lib/Quiver.test.ts
+++ b/frontend/src/lib/Quiver.test.ts
@@ -16,6 +16,8 @@
  */
 
 import { Utf8Vector, util } from "apache-arrow"
+import { cloneDeep } from "lodash"
+
 import { IndexTypeName, Quiver } from "src/lib/Quiver"
 import {
   // Types
@@ -1014,17 +1016,17 @@ describe("Quiver", () => {
         const mockElement = { data: CATEGORICAL }
         const q = new Quiver(mockElement)
 
-        q.addRows(q)
+        const qq = q.addRows(q)
 
-        expect(q.index).toEqual([["i1"], ["i2"], ["i1"], ["i2"]])
-        expect(q.columns).toEqual([["c1", "c2"]])
-        expect(q.data).toEqual([
+        expect(qq.index).toEqual([["i1"], ["i2"], ["i1"], ["i2"]])
+        expect(qq.columns).toEqual([["c1", "c2"]])
+        expect(qq.data).toEqual([
           ["foo", util.BN.new(new Int32Array([100, 0]))],
           ["bar", util.BN.new(new Int32Array([200, 0]))],
           ["foo", util.BN.new(new Int32Array([100, 0]))],
           ["bar", util.BN.new(new Int32Array([200, 0]))],
         ])
-        expect(q.types).toEqual({
+        expect(qq.types).toEqual({
           index: [
             {
               pandas_type: IndexTypeName.CategoricalIndex,
@@ -1054,18 +1056,18 @@ describe("Quiver", () => {
         const mockElement = { data: DATETIME }
         const q = new Quiver(mockElement)
 
-        q.addRows(q)
+        const qq = q.addRows(q)
 
-        expect(q.index).toEqual([
+        expect(qq.index).toEqual([
           [978220800000],
           [1009756800000],
           [978220800000],
           [1009756800000],
         ])
-        expect(q.columns).toEqual([
+        expect(qq.columns).toEqual([
           ["2000-12-31 00:00:00", "2001-12-31 00:00:00"],
         ])
-        expect(q.data).toEqual([
+        expect(qq.data).toEqual([
           [
             new Date("2020-01-02T00:00:00.000Z"),
             new Date("2020-10-20T00:00:00.000Z"),
@@ -1083,7 +1085,7 @@ describe("Quiver", () => {
             new Date("2020-10-20T00:00:00.000Z"),
           ],
         ])
-        expect(q.types).toEqual({
+        expect(qq.types).toEqual({
           index: [
             {
               pandas_type: IndexTypeName.DatetimeIndex,
@@ -1110,17 +1112,17 @@ describe("Quiver", () => {
         const mockElement = { data: FLOAT64 }
         const q = new Quiver(mockElement)
 
-        q.addRows(q)
+        const qq = q.addRows(q)
 
-        expect(q.index).toEqual([[1.24], [2.35], [1.24], [2.35]])
-        expect(q.columns).toEqual([["1.24", "2.35"]])
-        expect(q.data).toEqual([
+        expect(qq.index).toEqual([[1.24], [2.35], [1.24], [2.35]])
+        expect(qq.columns).toEqual([["1.24", "2.35"]])
+        expect(qq.data).toEqual([
           [1.2, 1.3],
           [1.4, 1.5],
           [1.2, 1.3],
           [1.4, 1.5],
         ])
-        expect(q.types).toEqual({
+        expect(qq.types).toEqual({
           index: [
             {
               pandas_type: IndexTypeName.Float64Index,
@@ -1147,16 +1149,16 @@ describe("Quiver", () => {
         const mockElement = { data: INT64 }
         const q = new Quiver(mockElement)
 
-        q.addRows(q)
+        const qq = q.addRows(q)
 
-        expect(q.index).toEqual([
+        expect(qq.index).toEqual([
           [util.BN.new(new Int32Array([1, 0]))],
           [util.BN.new(new Int32Array([2, 0]))],
           [util.BN.new(new Int32Array([1, 0]))],
           [util.BN.new(new Int32Array([2, 0]))],
         ])
-        expect(q.columns).toEqual([["1", "2"]])
-        expect(q.data).toEqual([
+        expect(qq.columns).toEqual([["1", "2"]])
+        expect(qq.data).toEqual([
           [
             util.BN.new(new Int32Array([0, 0])),
             util.BN.new(new Int32Array([1, 0])),
@@ -1174,7 +1176,7 @@ describe("Quiver", () => {
             util.BN.new(new Int32Array([3, 0])),
           ],
         ])
-        expect(q.types).toEqual({
+        expect(qq.types).toEqual({
           index: [
             {
               pandas_type: IndexTypeName.Int64Index,
@@ -1201,21 +1203,21 @@ describe("Quiver", () => {
         const mockElement = { data: INTERVAL_DATETIME64 }
         const q = new Quiver(mockElement)
 
-        q.addRows(q)
+        const qq = q.addRows(q)
 
-        expect(q.index.toString()).toEqual(
+        expect(qq.index.toString()).toEqual(
           '{ "left": 1483228800000, "right": 1483315200000 },{ "left": 1483315200000, "right": 1483401600000 },{ "left": 1483228800000, "right": 1483315200000 },{ "left": 1483315200000, "right": 1483401600000 }'
         )
-        expect(q.columns).toEqual([
+        expect(qq.columns).toEqual([
           ["(2017-01-01, 2017-01-02]", "(2017-01-02, 2017-01-03]"],
         ])
-        expect(q.data).toEqual([
+        expect(qq.data).toEqual([
           ["foo", util.BN.new(new Int32Array([100, 0]))],
           ["bar", util.BN.new(new Int32Array([200, 0]))],
           ["foo", util.BN.new(new Int32Array([100, 0]))],
           ["bar", util.BN.new(new Int32Array([200, 0]))],
         ])
-        expect(q.types).toEqual({
+        expect(qq.types).toEqual({
           index: [
             {
               pandas_type: "object",
@@ -1242,19 +1244,19 @@ describe("Quiver", () => {
         const mockElement = { data: INTERVAL_FLOAT64 }
         const q = new Quiver(mockElement)
 
-        q.addRows(q)
+        const qq = q.addRows(q)
 
-        expect(q.index.toString()).toEqual(
+        expect(qq.index.toString()).toEqual(
           '{ "left": 0, "right": 1.5 },{ "left": 1.5, "right": 3 },{ "left": 0, "right": 1.5 },{ "left": 1.5, "right": 3 }'
         )
-        expect(q.columns).toEqual([["(0.0, 1.5]", "(1.5, 3.0]"]])
-        expect(q.data).toEqual([
+        expect(qq.columns).toEqual([["(0.0, 1.5]", "(1.5, 3.0]"]])
+        expect(qq.data).toEqual([
           ["foo", util.BN.new(new Int32Array([100, 0]))],
           ["bar", util.BN.new(new Int32Array([200, 0]))],
           ["foo", util.BN.new(new Int32Array([100, 0]))],
           ["bar", util.BN.new(new Int32Array([200, 0]))],
         ])
-        expect(q.types).toEqual({
+        expect(qq.types).toEqual({
           index: [
             {
               pandas_type: "object",
@@ -1281,19 +1283,19 @@ describe("Quiver", () => {
         const mockElement = { data: INTERVAL_INT64 }
         const q = new Quiver(mockElement)
 
-        q.addRows(q)
+        const qq = q.addRows(q)
 
-        expect(q.index.toString()).toEqual(
+        expect(qq.index.toString()).toEqual(
           '{ "left": 0, "right": 1 },{ "left": 1, "right": 2 },{ "left": 0, "right": 1 },{ "left": 1, "right": 2 }'
         )
-        expect(q.columns).toEqual([["(0, 1]", "(1, 2]"]])
-        expect(q.data).toEqual([
+        expect(qq.columns).toEqual([["(0, 1]", "(1, 2]"]])
+        expect(qq.data).toEqual([
           ["foo", util.BN.new(new Int32Array([100, 0]))],
           ["bar", util.BN.new(new Int32Array([200, 0]))],
           ["foo", util.BN.new(new Int32Array([100, 0]))],
           ["bar", util.BN.new(new Int32Array([200, 0]))],
         ])
-        expect(q.types).toEqual({
+        expect(qq.types).toEqual({
           index: [
             {
               pandas_type: "object",
@@ -1320,19 +1322,19 @@ describe("Quiver", () => {
         const mockElement = { data: INTERVAL_UINT64 }
         const q = new Quiver(mockElement)
 
-        q.addRows(q)
+        const qq = q.addRows(q)
 
-        expect(q.index.toString()).toEqual(
+        expect(qq.index.toString()).toEqual(
           '{ "left": 0, "right": 1 },{ "left": 1, "right": 2 },{ "left": 0, "right": 1 },{ "left": 1, "right": 2 }'
         )
-        expect(q.columns).toEqual([["(0, 1]", "(1, 2]"]])
-        expect(q.data).toEqual([
+        expect(qq.columns).toEqual([["(0, 1]", "(1, 2]"]])
+        expect(qq.data).toEqual([
           ["foo", util.BN.new(new Int32Array([100, 0]))],
           ["bar", util.BN.new(new Int32Array([200, 0]))],
           ["foo", util.BN.new(new Int32Array([100, 0]))],
           ["bar", util.BN.new(new Int32Array([200, 0]))],
         ])
-        expect(q.types).toEqual({
+        expect(qq.types).toEqual({
           index: [
             {
               pandas_type: "object",
@@ -1359,17 +1361,17 @@ describe("Quiver", () => {
         const mockElement = { data: RANGE }
         const q = new Quiver(mockElement)
 
-        q.addRows(q)
+        const qq = q.addRows(q)
 
-        expect(q.index).toEqual([[0], [1], [2], [3]])
-        expect(q.columns).toEqual([["0", "1"]])
-        expect(q.data).toEqual([
+        expect(qq.index).toEqual([[0], [1], [2], [3]])
+        expect(qq.columns).toEqual([["0", "1"]])
+        expect(qq.data).toEqual([
           ["foo", "1"],
           ["bar", "2"],
           ["foo", "1"],
           ["bar", "2"],
         ])
-        expect(q.types).toEqual({
+        expect(qq.types).toEqual({
           index: [
             {
               pandas_type: IndexTypeName.RangeIndex,
@@ -1402,16 +1404,16 @@ describe("Quiver", () => {
         const mockElement = { data: UINT64 }
         const q = new Quiver(mockElement)
 
-        q.addRows(q)
+        const qq = q.addRows(q)
 
-        expect(q.index).toEqual([
+        expect(qq.index).toEqual([
           [util.BN.new(new Int32Array([1, 0]), false)],
           [util.BN.new(new Int32Array([2, 0]), false)],
           [util.BN.new(new Int32Array([1, 0]), false)],
           [util.BN.new(new Int32Array([2, 0]), false)],
         ])
-        expect(q.columns).toEqual([["1", "2"]])
-        expect(q.data).toEqual([
+        expect(qq.columns).toEqual([["1", "2"]])
+        expect(qq.data).toEqual([
           [
             util.BN.new(new Int32Array([1, 0])),
             util.BN.new(new Int32Array([2, 0])),
@@ -1429,7 +1431,7 @@ describe("Quiver", () => {
             util.BN.new(new Int32Array([4, 0])),
           ],
         ])
-        expect(q.types).toEqual({
+        expect(qq.types).toEqual({
           index: [
             {
               pandas_type: IndexTypeName.UInt64Index,
@@ -1456,17 +1458,17 @@ describe("Quiver", () => {
         const mockElement = { data: UNICODE }
         const q = new Quiver(mockElement)
 
-        q.addRows(q)
+        const qq = q.addRows(q)
 
-        expect(q.index).toEqual([["i1"], ["i2"], ["i1"], ["i2"]])
-        expect(q.columns).toEqual([["c1", "c2"]])
-        expect(q.data).toEqual([
+        expect(qq.index).toEqual([["i1"], ["i2"], ["i1"], ["i2"]])
+        expect(qq.columns).toEqual([["c1", "c2"]])
+        expect(qq.data).toEqual([
           ["foo", "1"],
           ["bar", "2"],
           ["foo", "1"],
           ["bar", "2"],
         ])
-        expect(q.types).toEqual({
+        expect(qq.types).toEqual({
           index: [
             {
               pandas_type: IndexTypeName.UnicodeIndex,
@@ -1491,29 +1493,38 @@ describe("Quiver", () => {
     })
 
     describe("Special cases", () => {
+      it("does not mutate the original element", () => {
+        const mockElement = { data: UNICODE }
+        const q = new Quiver(mockElement)
+        const qClone = cloneDeep(q)
+
+        q.addRows(q)
+        expect(q).toEqual(qClone)
+      })
+
       test("multi-index", () => {
         const mockElement = { data: MULTI }
         const q = new Quiver(mockElement)
 
-        q.addRows(q)
+        const qq = q.addRows(q)
 
-        expect(q.index).toEqual([
+        expect(qq.index).toEqual([
           [util.BN.new(new Int32Array([1, 0])), "red"],
           [util.BN.new(new Int32Array([2, 0])), "blue"],
           [util.BN.new(new Int32Array([1, 0])), "red"],
           [util.BN.new(new Int32Array([2, 0])), "blue"],
         ])
-        expect(q.columns).toEqual([
+        expect(qq.columns).toEqual([
           ["1", "2"],
           ["red", "blue"],
         ])
-        expect(q.data).toEqual([
+        expect(qq.data).toEqual([
           ["foo", "1"],
           ["bar", "2"],
           ["foo", "1"],
           ["bar", "2"],
         ])
-        expect(q.types).toEqual({
+        expect(qq.types).toEqual({
           index: [
             {
               pandas_type: IndexTypeName.Int64Index,
@@ -1547,17 +1558,17 @@ describe("Quiver", () => {
         const q1 = new Quiver(mockElement1)
         const q2 = new Quiver(mockElement2)
 
-        q1.addRows(q2)
+        const q1q2 = q1.addRows(q2)
 
-        expect(q1.index).toEqual([["i1"], ["i2"], ["i1"], ["i2"]])
-        expect(q1.columns).toEqual([["c1", "c2"]])
-        expect(q1.data).toEqual([
+        expect(q1q2.index).toEqual([["i1"], ["i2"], ["i1"], ["i2"]])
+        expect(q1q2.columns).toEqual([["c1", "c2"]])
+        expect(q1q2.data).toEqual([
           ["foo", "1"],
           ["bar", "2"],
           ["baz", "1"],
           ["qux", "2"],
         ])
-        expect(q1.types).toEqual({
+        expect(q1q2.types).toEqual({
           index: [
             {
               pandas_type: IndexTypeName.UnicodeIndex,
@@ -1586,8 +1597,8 @@ describe("Quiver", () => {
         const q1 = new Quiver(mockElement1)
         const q2 = new Quiver(mockElement2)
 
-        q1.addRows(q2)
-        expect(q1).toEqual(q2)
+        const q1q2 = q1.addRows(q2)
+        expect(q1q2).toEqual(q2)
       })
 
       it("shows df1 if df2 is empty", () => {
@@ -1596,8 +1607,8 @@ describe("Quiver", () => {
         const q1 = new Quiver(mockElement1)
         const q2 = new Quiver(mockElement2)
 
-        q2.addRows(q1)
-        expect(q2).toEqual(q2)
+        const q2q1 = q2.addRows(q1)
+        expect(q2q1).toEqual(q2)
       })
 
       it("shows an empty DataFrame if both df1 and df2 are empty", () => {
@@ -1605,8 +1616,8 @@ describe("Quiver", () => {
         const q1 = new Quiver(mockElement)
         const q2 = new Quiver(mockElement)
 
-        q1.addRows(q2)
-        expect(q1.isEmpty()).toBe(true)
+        const q1q2 = q1.addRows(q2)
+        expect(q1q2.isEmpty()).toBe(true)
       })
 
       it("uses df1 columns if df2 has more columns than df1", () => {
@@ -1615,12 +1626,12 @@ describe("Quiver", () => {
         const q1 = new Quiver(mockElement1)
         const q2 = new Quiver(mockElement2)
 
-        q1.addRows(q2)
+        const q1q2 = q1.addRows(q2)
 
-        expect(q1.index).toEqual([["i1"], ["i2"], ["i1"], ["i2"]])
-        expect(q1.columns).toEqual([["c1"]])
-        expect(q1.data).toEqual([["foo"], ["bar"], ["foo"], ["bar"]])
-        expect(q1.types).toEqual({
+        expect(q1q2.index).toEqual([["i1"], ["i2"], ["i1"], ["i2"]])
+        expect(q1q2.columns).toEqual([["c1"]])
+        expect(q1q2.data).toEqual([["foo"], ["bar"], ["foo"], ["bar"]])
+        expect(q1q2.types).toEqual({
           index: [
             {
               pandas_type: IndexTypeName.UnicodeIndex,