Repository: spyder-ide/spyder

Issue:
KeyError when drawing folding regions
## Description

### What steps will reproduce the problem?

<!--- You can use Markdown here --->

```python
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
#pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 50)
import category_encoders as ce
#pd.set_option('display.width', 1000)
# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os, sys, time
import json, random
from tqdm import tqdm
from datetime import datetime as dt

KAGGLE=True
file_dir='/Users/hidemiasakura/betworks/kaggle/data-science-bowl-2019/'
if KAGGLE:
    file_dir='/kaggle/input/'



#start copy
# Any results you write to the current directory are saved as output.
class Dataset:
    totalObjects=0
    def __init__(self):
        Dataset.totalObjects=Dataset.totalObjects+1
        
    def elapsed_time(func):
        def wrapper(*args, **kw):
            start_time=time.time()
            result = func(*args, **kw)
            message=f'{func.__name__}: Elapsed time: {round(((time.time() - start_time) / 60), 2)} minutes. {dt.now()}'
            print(message)
            return result
        return wrapper
    
    @classmethod
    def showcount(cls):
        print("Total objects: ",cls.totalObjects)
    
    @elapsed_time
    def load(self):
        files=[]
        for dirname, _, filenames in os.walk(file_dir):
            for filename in filenames:
                file=os.path.join(dirname, filename)
                if '.csv' in file:
                    print(file)
                    files.append(file)
                
        self.files=files
        for f in files:
            fn=f.split('/')[-1].replace('.csv','')
            setattr(self, fn+'_df', pd.read_csv(f))
            print('Loaded', fn)
            #break
        
        


    @elapsed_time
    def preprocess(self):
        self.process_dataset()
        self.new_labels=self.get_labels(self.train_test_df)
        
        
    @elapsed_time
    def process_dataset(self):
        self.train_df['test_data']=False
        self.test_df['test_data']=True
        self.train_test_df=self.train_df.append(self.test_df)
        del self.train_df
        
        #timeseries start assessment - all assessment + 2000
        self.predict_test_df=self.test_df.groupby('installation_id').last()
        del self.test_df
        
        self.train_test_df.index=range(self.train_test_df.shape[0])
        self.train_test_df['timestamp'] = pd.to_datetime(self.train_test_df['timestamp'])
        self.specs=self.specs_df.set_index('event_id')
        
        train_test=self.train_test_df
        specs=self.specs_df
        train_labels=self.train_labels_df
        sample_submission=self.sample_submission_df
        predict_test=self.predict_test_df
        
        self.list_of_user_activities = list_of_user_activities= list(set(train_test['title'].unique()))
        self.list_of_event_codes = sorted(list(set(train_test['event_code'].unique())))
        self.list_of_event_ids = list(set(train_test['event_id'].unique()))
        self.list_of_worlds = list(set(train_test['world'].unique()))
        self.list_of_assessments = list(set(train_test[train_test['type'] == 'Assessment']['title'].unique()))
        self.activities_map = activities_map = dict(zip(list_of_user_activities, np.arange(len(list_of_user_activities))))
        
        #map activities
        self.train_test_df['map_title'] = train_test['title'].map(activities_map)
        
    @elapsed_time
    def get_labels(self, df):
        
        '''
        Assessment attempts are captured in event_code 4100 for all assessments except for Bird Measurer, which uses event_code 4110. If the attempt was correct, it contains "correct":true.
        '''
        assessments=df[((df.event_code==4100) & (df.type=='Assessment')) | ((df.event_code==4110) & (df.title=='Bird Measurer (Assessment)'))].copy()
        #test_all_ass=test[(test.event_code==4100) | ((test.event_code==4110) & (test.title=='Bird Measurer (Assessment)'))].copy()
        '''
        test_install_ids=test_all_ass.installation_id.unique()
        train_install_ids=sorted(train_all_ass.installation_id.unique())
        print(f'train_install_ids {len(train_install_ids)}')
        label_install_ids=sorted(train_labels.installation_id.unique())
        print(f'train_label_install_ids {len(label_install_ids)} variance {len(train_install_ids)-len(label_install_ids)}')
        print(f'test_install_ids {len(test_install_ids)} rows {test_all_ass.shape[0]}')
        print(f'sample_submission install ids {len(sample_submission.installation_id.unique())}')
        '''
        def get_scores(df):
            for i,idx in enumerate(df.index):
                row=df.loc[idx]
                #print(i, json.loads(row.event_data)['correct'])
                correct=0
                if json.loads(row.event_data)['correct']:
                    correct=1
                df.at[idx, 'correct']=correct
            return df
        
        assessments=get_scores(assessments)
        #test_all_ass=get_scores(test_all_ass)
        
        
        def get_new_labels(df):
            '''
            labels
            ['game_session', 'installation_id', 'title', 'num_correct',
                   'num_incorrect', 'accuracy', 'accuracy_group']
            '''
            
            new_train_labels=pd.DataFrame(df.groupby(['installation_id', 'game_session','title']).correct.count())
            new_train_labels.columns=['num_tries']
            new_train_labels['num_correct']=df.groupby(['installation_id', 'game_session','title']).correct.sum()
            new_train_labels['num_incorrect']=new_train_labels.num_tries-new_train_labels.num_correct
            new_train_labels['accuracy']=new_train_labels.num_correct/new_train_labels['num_tries']
            
            accuracy_dict={
                1:3,
                0.5:2,
                0:0,
                }
            
            for idx in new_train_labels.index:
                installation_id, game_session, title=idx
                #print(installation_id, game_session, title)
                if new_train_labels.loc[idx].accuracy in accuracy_dict:
                    new_train_labels.at[idx, 'accuracy_group']=accuracy_dict[new_train_labels.loc[idx].accuracy]
                else:
                    new_train_labels.at[idx, 'accuracy_group']=1
                #print(new_train_labels.loc[idx])
                #break
            return new_train_labels
        
        new_labels=get_new_labels(assessments)
        #train_labels=get_new_labels(train_all_ass)
        #assert len(new_train_labels.reset_index().installation_id.unique())==len(train_install_ids), 'installation_ids missing'
        #new_train_labels.to_csv('new_train_labels.csv')
        return new_labels
    
ds=Dataset()
ds.load()
ds.preprocess()



self=ds
train_test=self.train_test_df
specs=self.specs_df
train_labels=self.train_labels_df
sample_submission=self.sample_submission_df
predict_test=self.predict_test_df
new_labels=self.new_labels
list_of_assessments = self.list_of_assessments

#baserates
summary=new_labels.groupby('title', level=2).sum()
summary.accuracy=summary.num_correct/summary.num_tries
#hard-coded
summary.accuracy_group=summary.accuracy.rank()-1
summary.at['Cart Balancer (Assessment)', 'accuracy_group']=3
print(summary)
predict_test['accuracy_group']=predict_test.title.map(summary.accuracy_group.to_dict())
base_submission=predict_test[['accuracy_group']].astype(int)
base_submission.to_csv('submission.csv')
```

pasted above code into open script.
console showing following
QPainter::begin: A paint device can only be painted by one painter at a time.
QPainter::setCompositionMode: Painter not active
QBackingStore::endPaint() called with active painter on backingstore paint device
QBackingStore::endPaint() called with active painter on backingstore paint device


### Traceback
```python-traceback
  File "/Users/hidemiasakura/anaconda/envs/py37/lib/python3.7/site-packages/spyder/plugins/editor/panels/codefolding.py", line 256, in paintEvent



QPainter::setCompositionMode: Painter not active
Traceback (most recent call last):
  File "/Users/hidemiasakura/anaconda/envs/py37/lib/python3.7/site-packages/spyder/plugins/editor/panels/codefolding.py", line 256, in paintEvent
    self._draw_fold_region_background(block, painter)
  File "/Users/hidemiasakura/anaconda/envs/py37/lib/python3.7/site-packages/spyder/plugins/editor/panels/codefolding.py", line 300, in _draw_fold_region_background
    end = self.folding_regions[start]
KeyError: 166

QPainter::setCompositionMode: Painter not active
Traceback (most recent call last):
  File "/Users/hidemiasakura/anaconda/envs/py37/lib/python3.7/site-packages/spyder/plugins/editor/panels/codefolding.py", line 256, in paintEvent
    self._draw_fold_region_background(block, painter)
  File "/Users/hidemiasakura/anaconda/envs/py37/lib/python3.7/site-packages/spyder/plugins/editor/panels/codefolding.py", line 300, in _draw_fold_region_background

QPainter::setCompositionMode: Painter not active
QWidget::repaint: Recursive repaint detected

Traceback (most recent call last):

```

## Versions

* Spyder version: 4.0.0rc3 
* Python version: 3.7.0
* Qt version: 5.9.6
* PyQt5 version: 5.9.2
* Operating System: Darwin 17.7.0

### Dependencies

```
cloudpickle >=0.5.0          :  1.2.2 (OK)
pygments >=2.0               :  2.4.2 (OK)
qtconsole >=4.6.0            :  4.6.0 (OK)
nbconvert >=4.0              :  5.6.0 (OK)
sphinx >=0.6.6               :  2.2.0 (OK)
pylint >=0.25                :  2.4.2 (OK)
psutil >=0.3                 :  5.6.3 (OK)
qtawesome >=0.5.7            :  0.6.0 (OK)
qtpy >=1.5.0                 :  1.9.0 (OK)
pickleshare >=0.4            :  0.7.5 (OK)
zmq >=17                     :  18.1.0 (OK)
chardet >=2.0.0              :  3.0.4 (OK)
numpydoc >=0.6.0             :  0.9.1 (OK)
spyder_kernels >=1.8.0;<2.0.0:  1.8.0 (OK)
qdarkstyle >=2.7             :  2.7 (OK)
atomicwrites >=1.2.0         :  1.3.0 (OK)
diff_match_patch >=20181111  :  20181111 (OK)
intervaltree                 :  None (OK)
watchdog                     :  None (OK)
keyring                      :  None (OK)
pexpect >=4.4.0              :  4.7.0 (OK)
pympler                      :  None (OK)
sympy >=0.7.3                :  None (NOK)
cython >=0.21                :  0.29.13 (OK)
IPython >=4.0                :  7.8.0 (OK)
matplotlib >=2.0.0           :  3.1.1 (OK)
pandas >=0.13.1              :  0.25.2 (OK)
numpy >=1.7                  :  1.17.2 (OK)
scipy >=0.17.0               :  1.3.1 (OK)
pyls >=0.31.1;<0.32.0        :  0.31.1 (OK)
applaunchservices >=0.1.7    :  0.2.1 (OK)
rtree >=0.8.3                :  0.8.3 (OK)
```



Discussion:
@jnsebgosselin could this be related to the problem you were experiencing?
Maybe, it is hard to tell. I cannot reproduce this error on latest master using the provided script in the OP.
I can't reproduce this one either.

@hasakura511, please post a small video (an animated gif with a program called Licecap) to see how you're getting this error.
will do.

some more information:

1. this time the dialog box closed, so i was able to continue w/o closing
spyder.
2. When the error occurs, the window panes black out and when i hover my
mouse over them it starts flickering, but it eventually becomes
viewable again.
3. it prints the following in the terminal sometimes when i move the mouse
from the editor window to the ipython console, or scrolling up/down
QPainter::begin: A paint device can only be painted by one painter at a
time.
QPainter::setCompositionMode: Painter not active
QBackingStore::endPaint() called with active painter on backingstore paint
device
4. Both times when this error occurred I pasted large code block
(with undefined variables) into existing script that had bunch of undefined
variables.  But it dosen't seem to happen every time, so not sure what is
triggering it.

hope this helps.
Thanks.

On Thu, Dec 5, 2019 at 7:48 AM Carlos Cordoba <notifications@github.com>
wrote:

> I can't reproduce this one either.
>
> @hasakura511 <https://github.com/hasakura511>, please post a small video
> (an animated gif with a program called Licecap) to see how you're getting
> this error.
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/spyder-ide/spyder/issues/10918?email_source=notifications&email_token=ACM6IWOXFGD3FO4FCPX5VTDQXA6UJA5CNFSM4JUUVLX2YY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOEF65Z2I#issuecomment-561896681>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/ACM6IWMLWJAVWPUA4VHRMVLQXA6UJANCNFSM4JUUVLXQ>
> .
>


Resolving patch:
diff --git a/spyder/plugins/editor/panels/codefolding.py b/spyder/plugins/editor/panels/codefolding.py
--- a/spyder/plugins/editor/panels/codefolding.py
+++ b/spyder/plugins/editor/panels/codefolding.py
@@ -254,7 +254,12 @@ def paintEvent(self, event):
                 self._mouse_over_line)
             try:
                 self._draw_fold_region_background(block, painter)
-            except ValueError:
+            except (ValueError, KeyError):
+                # Catching the KeyError above is necessary to avoid
+                # issue spyder-ide/spyder#10918.
+                # It happens when users have the mouse on top of the
+                # folding panel and make some text modifications
+                # that trigger a folding recomputation.
                 pass
         # Draw fold triggers
         for top_position, line_number, block in self.editor.visible_blocks:


Test patch: