Repository: spyder-ide/spyder

Issue:
Spyder very slow when working on large python files
## Description

### What steps will reproduce the problem?

<!--- You can use Markdown here --->

A few things in spyder are slowing down spyder considerably, mostly whrn working with big files. The functions that take most of the time are:
 - the `paintEvent` from `spyder/plugins/editor/panels/scrollflag.py`
     - Mostly computing the flags and adding all the painters
 - `__mark_occurences` in `spyder/plugins/editor/widgets/codeeditor.py`




## Versions

* Spyder version: 5.0.0.dev0 acb3dbff4
* Python version: 3.7.5
* Qt version: 5.12.6
* PyQt5 version: 5.12.3
* Operating System: Darwin 19.0.0

### Dependencies

```
cloudpickle >=0.5.0          :  1.2.2 (OK)
pygments >=2.0               :  2.4.2 (OK)
qtconsole >=4.6.0            :  4.7.0.dev0 (OK)
nbconvert >=4.0              :  5.6.1 (OK)
sphinx >=0.6.6               :  2.2.1 (OK)
pylint >=0.25                :  2.4.4 (OK)
psutil >=0.3                 :  5.6.5 (OK)
qtawesome >=0.5.7            :  0.6.0 (OK)
qtpy >=1.5.0                 :  1.9.0 (OK)
pickleshare >=0.4            :  0.7.5 (OK)
zmq >=17                     :  18.1.1 (OK)
chardet >=2.0.0              :  3.0.4 (OK)
numpydoc >=0.6.0             :  0.9.1 (OK)
spyder_kernels >=1.8.1;<2.0.0:  1.9.0.dev0 (OK)
qdarkstyle >=2.7             :  2.7 (OK)
atomicwrites >=1.2.0         :  1.3.0 (OK)
diff_match_patch >=20181111  :  20181111 (OK)
intervaltree                 :  None (OK)
watchdog                     :  None (OK)
keyring                      :  None (OK)
pexpect >=4.4.0              :  4.7.0 (OK)
sympy >=0.7.3                :  1.4 (OK)
cython >=0.21                :  None (NOK)
IPython >=4.0                :  7.11.0.dev (OK)
matplotlib >=2.0.0           :  3.1.2 (OK)
pandas >=0.13.1              :  0.25.3 (OK)
numpy >=1.7                  :  1.17.4 (OK)
scipy >=0.17.0               :  1.3.3 (OK)
pyls >=0.31.2;<0.32.0        :  0.31.1 (NOK)
applaunchservices >=0.1.7    :  0.2.1 (OK)
```



Discussion:
An example: (rename to .py)

[pdc_lib copy.py.txt](https://github.com/spyder-ide/spyder/files/3945532/pdc_lib.copy.py.txt)

Could you help us with this one @impact27?
I think I can help with optimizing the scrollflag paintEvent if you need help with it.
That'd be really appreciated, thanks!
> 
> 
> That'd be really appreciated, thanks!

Ok I'm on it. I won't touch the `__mark_occurences` in `spyder/plugins/editor/widgets/codeeditor.py` though.
This would be related to #10992

Resolving patch:
diff --git a/spyder/plugins/editor/panels/scrollflag.py b/spyder/plugins/editor/panels/scrollflag.py
--- a/spyder/plugins/editor/panels/scrollflag.py
+++ b/spyder/plugins/editor/panels/scrollflag.py
@@ -7,12 +7,16 @@
 This module contains the Scroll Flag panel
 """
 
+# Standard library imports
+from __future__ import division
 from math import ceil
 
-from qtpy.QtCore import QSize, Qt, QRect
-from qtpy.QtGui import QPainter, QBrush, QColor, QCursor
+# Third party imports
+from qtpy.QtCore import QSize, Qt
+from qtpy.QtGui import QPainter, QColor, QCursor
 from qtpy.QtWidgets import (QStyle, QStyleOptionSlider, QApplication)
 
+# Local imports
 from spyder.api.panel import Panel
 from spyder.plugins.completion.languageserver import DiagnosticSeverity
 
@@ -34,6 +38,23 @@ def __init__(self, editor):
         self._range_indicator_is_visible = False
         self._alt_key_is_down = False
 
+        # Define permanent Qt colors that are needed for painting the flags
+        # and the slider range.
+        self._facecolors = {
+            'warning': QColor(editor.warning_color),
+            'error': QColor(editor.error_color),
+            'todo': QColor(editor.todo_color),
+            'breakpoint': QColor(editor.breakpoint_color),
+            'occurrence': QColor(editor.occurrence_color),
+            'found_results': QColor(editor.found_results_color)
+            }
+        self._edgecolors = {key: color.darker(120) for
+                            key, color in self._facecolors.items()}
+        self._slider_range_color = QColor(Qt.gray)
+        self._slider_range_color.setAlphaF(.85)
+        self._slider_range_brush = QColor(Qt.gray)
+        self._slider_range_brush.setAlphaF(.5)
+
         editor.sig_focus_changed.connect(self.update)
         editor.sig_key_pressed.connect(self.keyPressEvent)
         editor.sig_key_released.connect(self.keyReleaseEvent)
@@ -41,37 +62,47 @@ def __init__(self, editor):
         editor.sig_alt_mouse_moved.connect(self.mouseMoveEvent)
         editor.sig_leave_out.connect(self.update)
         editor.sig_flags_changed.connect(self.update)
+        editor.sig_theme_colors_changed.connect(self.update_flag_colors)
 
     @property
     def slider(self):
         """This property holds whether the vertical scrollbar is visible."""
         return self.editor.verticalScrollBar().isVisible()
 
-    @property
-    def offset(self):
-        """This property holds the vertical offset of the scroll flag area
-        relative to the top of the text editor."""
-        vsb = self.editor.verticalScrollBar()
-        style = QApplication.instance().style()
-        opt = QStyleOptionSlider()
-        vsb.initStyleOption(opt)
-
-        # Get the area in which the slider handle may move.
-        groove_rect = style.subControlRect(
-                QStyle.CC_ScrollBar, opt, QStyle.SC_ScrollBarGroove, self)
-
-        return groove_rect.y()
-
     def sizeHint(self):
         """Override Qt method"""
         return QSize(self.WIDTH, 0)
 
+    def update_flag_colors(self, color_dict):
+        """
+        Update the permanent Qt colors that are used for painting the flags
+        and the slider range with the new colors defined in the given dict.
+        """
+        for name, color in color_dict.items():
+            self._facecolors[name] = QColor(color)
+            self._edgecolors[name] = self._facecolors[name].darker(120)
+
     def paintEvent(self, event):
         """
         Override Qt method.
         Painting the scroll flag area
         """
-        make_flag = self.make_flag_qrect
+        # The area in which the slider handle of the scrollbar may move.
+        groove_rect = self.get_scrollbar_groove_rect()
+        # The scrollbar's scale factor ratio between pixel span height and
+        # value span height
+        scale_factor = groove_rect.height() / self.get_scrollbar_value_height()
+        # The vertical offset of the scroll flag area relative to the
+        # top of the text editor.
+        offset = groove_rect.y()
+
+        # Note that we calculate the pixel metrics required to draw the flags
+        # here instead of using the convenience methods of the ScrollFlagArea
+        # for performance reason.
+
+        rect_x = ceil(self.FLAGS_DX / 2)
+        rect_w = self.WIDTH - self.FLAGS_DX
+        rect_h = self.FLAGS_DY
 
         # Fill the whole painting area
         painter = QPainter(self)
@@ -84,60 +115,77 @@ def paintEvent(self, event):
             if data:
                 if data.code_analysis:
                     # Paint the warnings
-                    color = self.editor.warning_color
                     for source, code, severity, message in data.code_analysis:
                         error = severity == DiagnosticSeverity.ERROR
                         if error:
-                            color = self.editor.error_color
+                            painter.setBrush(self._facecolors['error'])
+                            painter.setPen(self._edgecolors['error'])
                             break
-                    self.set_painter(painter, color)
-                    painter.drawRect(make_flag(line_number))
+                    else:
+                        painter.setBrush(self._facecolors['warning'])
+                        painter.setPen(self._edgecolors['warning'])
+
+                    rect_y = self.calcul_flag_ypos(
+                        line_number, scale_factor, offset)
+                    painter.drawRect(rect_x, rect_y, rect_w, rect_h)
                 if data.todo:
                     # Paint the todos
-                    self.set_painter(painter, self.editor.todo_color)
-                    painter.drawRect(make_flag(line_number))
+                    rect_y = self.calcul_flag_ypos(
+                        line_number, scale_factor, offset)
+                    painter.setBrush(self._facecolors['todo'])
+                    painter.setPen(self._edgecolors['todo'])
+                    painter.drawRect(rect_x, rect_y, rect_w, rect_h)
                 if data.breakpoint:
                     # Paint the breakpoints
-                    self.set_painter(painter, self.editor.breakpoint_color)
-                    painter.drawRect(make_flag(line_number))
+                    rect_y = self.calcul_flag_ypos(
+                        line_number, scale_factor, offset)
+                    painter.setBrush(self._facecolors['breakpoint'])
+                    painter.setPen(self._edgecolors['breakpoint'])
+                    painter.drawRect(rect_x, rect_y, rect_w, rect_h)
             block = block.next()
 
         # Paint the occurrences
         if self.editor.occurrences:
-            self.set_painter(painter, self.editor.occurrence_color)
+            painter.setBrush(self._facecolors['occurrence'])
+            painter.setPen(self._edgecolors['occurrence'])
             for line_number in self.editor.occurrences:
-                painter.drawRect(make_flag(line_number))
+                rect_y = self.calcul_flag_ypos(
+                    line_number, scale_factor, offset)
+                painter.drawRect(rect_x, rect_y, rect_w, rect_h)
 
         # Paint the found results
         if self.editor.found_results:
-            self.set_painter(painter, self.editor.found_results_color)
+            painter.setBrush(self._facecolors['found_results'])
+            painter.setPen(self._edgecolors['found_results'])
             for line_number in self.editor.found_results:
-                painter.drawRect(make_flag(line_number))
+                rect_y = self.calcul_flag_ypos(
+                    line_number, scale_factor, offset)
+                painter.drawRect(rect_x, rect_y, rect_w, rect_h)
 
         # Paint the slider range
         if not self._unit_testing:
             alt = QApplication.queryKeyboardModifiers() & Qt.AltModifier
         else:
             alt = self._alt_key_is_down
-        cursor_pos = self.mapFromGlobal(QCursor().pos())
-        is_over_self = self.rect().contains(cursor_pos)
-        is_over_editor = self.editor.rect().contains(
+
+        if self.slider:
+            cursor_pos = self.mapFromGlobal(QCursor().pos())
+            is_over_self = self.rect().contains(cursor_pos)
+            is_over_editor = self.editor.rect().contains(
                 self.editor.mapFromGlobal(QCursor().pos()))
-        # We use QRect.contains instead of QWidget.underMouse method to
-        # determined if the cursor is over the editor or the flag scrollbar
-        # because the later gives a wrong result when a mouse button
-        # is pressed.
-        if ((is_over_self or (alt and is_over_editor)) and self.slider):
-            pen_color = QColor(Qt.gray)
-            pen_color.setAlphaF(.85)
-            painter.setPen(pen_color)
-            brush_color = QColor(Qt.gray)
-            brush_color.setAlphaF(.5)
-            painter.setBrush(QBrush(brush_color))
-            painter.drawRect(self.make_slider_range(cursor_pos))
-            self._range_indicator_is_visible = True
-        else:
-            self._range_indicator_is_visible = False
+            # We use QRect.contains instead of QWidget.underMouse method to
+            # determined if the cursor is over the editor or the flag scrollbar
+            # because the later gives a wrong result when a mouse button
+            # is pressed.
+            if is_over_self or (alt and is_over_editor):
+                painter.setPen(self._slider_range_color)
+                painter.setBrush(self._slider_range_brush)
+                x, y, width, height = self.make_slider_range(
+                    cursor_pos, scale_factor, offset, groove_rect)
+                painter.drawRect(x, y, width, height)
+                self._range_indicator_is_visible = True
+            else:
+                self._range_indicator_is_visible = False
 
     def enterEvent(self, event):
         """Override Qt method"""
@@ -170,9 +218,16 @@ def keyPressEvent(self, event):
             self._alt_key_is_down = True
             self.update()
 
-    def get_scrollbar_position_height(self):
-        """Return the pixel span height of the scrollbar area in which
-        the slider handle may move"""
+    def get_vertical_offset(self):
+        """
+        Return the vertical offset of the scroll flag area relative to the
+        top of the text editor.
+        """
+        groove_rect = self.get_scrollbar_groove_rect()
+        return groove_rect.y()
+
+    def get_scrollbar_groove_rect(self):
+        """Return the area in which the slider handle may move."""
         vsb = self.editor.verticalScrollBar()
         style = QApplication.instance().style()
         opt = QStyleOptionSlider()
@@ -180,14 +235,20 @@ def get_scrollbar_position_height(self):
 
         # Get the area in which the slider handle may move.
         groove_rect = style.subControlRect(
-                QStyle.CC_ScrollBar, opt, QStyle.SC_ScrollBarGroove, self)
+            QStyle.CC_ScrollBar, opt, QStyle.SC_ScrollBarGroove, self)
 
+        return groove_rect
+
+    def get_scrollbar_position_height(self):
+        """Return the pixel span height of the scrollbar area in which
+        the slider handle may move"""
+        groove_rect = self.get_scrollbar_groove_rect()
         return float(groove_rect.height())
 
     def get_scrollbar_value_height(self):
         """Return the value span height of the scrollbar"""
         vsb = self.editor.verticalScrollBar()
-        return vsb.maximum()-vsb.minimum()+vsb.pageStep()
+        return vsb.maximum() - vsb.minimum() + vsb.pageStep()
 
     def get_scale_factor(self):
         """Return scrollbar's scale factor:
@@ -195,69 +256,65 @@ def get_scale_factor(self):
         return (self.get_scrollbar_position_height() /
                 self.get_scrollbar_value_height())
 
-    def value_to_position(self, y):
+    def value_to_position(self, y, scale_factor, offset):
         """Convert value to position in pixels"""
         vsb = self.editor.verticalScrollBar()
-        return (y-vsb.minimum())*self.get_scale_factor()+self.offset
+        return (y - vsb.minimum()) * scale_factor + offset
 
     def position_to_value(self, y):
         """Convert position in pixels to value"""
         vsb = self.editor.verticalScrollBar()
-        return vsb.minimum()+max([0, (y-self.offset)/self.get_scale_factor()])
+        offset = self.get_vertical_offset()
+        return vsb.minimum() + max([0, (y - offset) / self.get_scale_factor()])
 
-    def make_flag_qrect(self, value):
-        """Make flag QRect"""
+    def calcul_flag_ypos(self, line_number, scale_factor, offset):
+        """Calcul the vertical position of a flag for the given line number."""
         if self.slider:
-            position = self.value_to_position(value+0.5)
+            position = self.value_to_position(
+                line_number + 0.5, scale_factor, offset)
             # The 0.5 offset is used to align the flags with the center of
             # their corresponding text edit block before scaling.
-
-            return QRect(ceil(self.FLAGS_DX/2), ceil(position-self.FLAGS_DY/2),
-                         self.WIDTH-self.FLAGS_DX, self.FLAGS_DY)
+            return ceil(position - self.FLAGS_DY / 2)
         else:
             # When the vertical scrollbar is not visible, the flags are
             # vertically aligned with the center of their corresponding
             # text block with no scaling.
-            block = self.editor.document().findBlockByNumber(value)
+            block = self.editor.document().findBlockByNumber(line_number)
             top = self.editor.blockBoundingGeometry(block).translated(
-                      self.editor.contentOffset()).top()
+                self.editor.contentOffset()).top()
             bottom = top + self.editor.blockBoundingRect(block).height()
             middle = (top + bottom)/2
 
-            return QRect(ceil(self.FLAGS_DX/2), ceil(middle-self.FLAGS_DY/2),
-                         self.WIDTH-self.FLAGS_DX, self.FLAGS_DY)
+            return ceil(middle-self.FLAGS_DY/2)
 
-    def make_slider_range(self, cursor_pos):
-        """Make slider range QRect"""
+    def make_slider_range(self, cursor_pos, scale_factor, offset, groove_rect):
+        """
+        Return the slider x and y positions and the slider width and height.
+        """
         # The slider range indicator position follows the mouse vertical
         # position while its height corresponds to the part of the file that
         # is currently visible on screen.
 
         vsb = self.editor.verticalScrollBar()
-        groove_height = self.get_scrollbar_position_height()
-        slider_height = self.value_to_position(vsb.pageStep())-self.offset
+        slider_height = self.value_to_position(
+            vsb.pageStep(), scale_factor, offset) - offset
 
         # Calcul the minimum and maximum y-value to constraint the slider
         # range indicator position to the height span of the scrollbar area
         # where the slider may move.
-        min_ypos = self.offset
-        max_ypos = groove_height + self.offset - slider_height
+        min_ypos = offset
+        max_ypos = groove_rect.height() + offset - slider_height
 
         # Determine the bounded y-position of the slider rect.
         slider_y = max(min_ypos, min(max_ypos,
                                      ceil(cursor_pos.y()-slider_height/2)))
 
-        return QRect(1, slider_y, self.WIDTH-2, slider_height)
+        return 1, slider_y, self.WIDTH - 2, slider_height
 
     def wheelEvent(self, event):
         """Override Qt method"""
         self.editor.wheelEvent(event)
 
-    def set_painter(self, painter, light_color):
-        """Set scroll flag area painter pen and brush colors"""
-        painter.setPen(QColor(light_color).darker(120))
-        painter.setBrush(QBrush(QColor(light_color)))
-
     def set_enabled(self, state):
         """Toggle scroll flag area visibility"""
         self.enabled = state
diff --git a/spyder/plugins/editor/widgets/codeeditor.py b/spyder/plugins/editor/widgets/codeeditor.py
--- a/spyder/plugins/editor/widgets/codeeditor.py
+++ b/spyder/plugins/editor/widgets/codeeditor.py
@@ -276,6 +276,9 @@ class CodeEditor(TextEditBaseWidget):
     #: Signal emitted when the flags need to be updated in the scrollflagarea
     sig_flags_changed = Signal()
 
+    #: Signal emitted when the syntax color theme of the editor.
+    sig_theme_colors_changed = Signal(dict)
+
     #: Signal emitted when a new text is set on the widget
     new_text_set = Signal()
 
@@ -450,15 +453,18 @@ def __init__(self, parent=None):
         # Linux Ubuntu. See spyder-ide/spyder#5215.
         self.setVerticalScrollBar(QScrollBar())
 
-        # Scrollbar flag area
-        self.scrollflagarea = self.panels.register(ScrollFlagArea(self),
-                                                   Panel.Position.RIGHT)
-        self.scrollflagarea.hide()
+        # Highlights and flag colors
         self.warning_color = "#FFAD07"
         self.error_color = "#EA2B0E"
         self.todo_color = "#B4D4F3"
         self.breakpoint_color = "#30E62E"
+        self.occurrence_color = QColor(Qt.yellow).lighter(160)
+        self.found_results_color = QColor(Qt.magenta).lighter(180)
 
+        # Scrollbar flag area
+        self.scrollflagarea = self.panels.register(ScrollFlagArea(self),
+                                                   Panel.Position.RIGHT)
+        self.scrollflagarea.hide()
         self.panels.refresh()
 
         self.document_id = id(self)
@@ -497,12 +503,10 @@ def __init__(self, parent=None):
         self.occurrence_timer.setInterval(1500)
         self.occurrence_timer.timeout.connect(self.__mark_occurrences)
         self.occurrences = []
-        self.occurrence_color = QColor(Qt.yellow).lighter(160)
 
         # Mark found results
         self.textChanged.connect(self.__text_has_changed)
         self.found_results = []
-        self.found_results_color = QColor(Qt.magenta).lighter(180)
 
         # Docstring
         self.writer_docstring = DocstringWriterExtension(self)
@@ -2052,6 +2056,9 @@ def _apply_highlighter_color_scheme(self):
             self.edge_line.update_color()
             self.indent_guides.update_color()
 
+            self.sig_theme_colors_changed.emit(
+                {'occurrence': self.occurrence_color})
+
     def apply_highlighter_settings(self, color_scheme=None):
         """Apply syntax highlighter settings"""
         if self.highlighter is not None:


Test patch: