Repository: ydataai/ydata-profiling

Issue:
An Error is emitted when the ‘count’field is present in DataFrame
**Describe the bug**

1.When I load a local file with a count field, it looks like this:

![image](https://user-images.githubusercontent.com/39480659/110434491-d8086c80-80ec-11eb-9b4a-57963df424b6.png)

2. So I'm going to go ahead and execute, I met this error:
'cannot insert count, already exists'

its all:
`---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
~/anaconda3/lib/python3.7/site-packages/IPython/core/formatters.py in __call__(self, obj)
    343             method = get_real_method(obj, self.print_method)
    344             if method is not None:
--> 345                 return method()
    346             return None
    347         else:

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/profile_report.py in _repr_html_(self)
    434     def _repr_html_(self):
    435         """The ipython notebook widgets user interface gets called by the jupyter notebook."""
--> 436         self.to_notebook_iframe()
    437 
    438     def __repr__(self):

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/profile_report.py in to_notebook_iframe(self)
    414         with warnings.catch_warnings():
    415             warnings.simplefilter("ignore")
--> 416             display(get_notebook_iframe(self))
    417 
    418     def to_widgets(self):

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/report/presentation/flavours/widget/notebook.py in get_notebook_iframe(profile)
     63         output = get_notebook_iframe_src(profile)
     64     elif attribute == "srcdoc":
---> 65         output = get_notebook_iframe_srcdoc(profile)
     66     else:
     67         raise ValueError(

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/report/presentation/flavours/widget/notebook.py in get_notebook_iframe_srcdoc(profile)
     21     width = config["notebook"]["iframe"]["width"].get(str)
     22     height = config["notebook"]["iframe"]["height"].get(str)
---> 23     src = html.escape(profile.to_html())
     24 
     25     iframe = f'<iframe width="{width}" height="{height}" srcdoc="{src}" frameborder="0" allowfullscreen></iframe>'

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/profile_report.py in to_html(self)
    384 
    385         """
--> 386         return self.html
    387 
    388     def to_json(self) -> str:

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/profile_report.py in html(self)
    199     def html(self):
    200         if self._html is None:
--> 201             self._html = self._render_html()
    202         return self._html
    203 

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/profile_report.py in _render_html(self)
    306         from pandas_profiling.report.presentation.flavours import HTMLReport
    307 
--> 308         report = self.report
    309 
    310         disable_progress_bar = not config["progress_bar"].get(bool)

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/profile_report.py in report(self)
    193     def report(self):
    194         if self._report is None:
--> 195             self._report = get_report_structure(self.description_set)
    196         return self._report
    197 

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/profile_report.py in description_set(self)
    173         if self._description_set is None:
    174             self._description_set = describe_df(
--> 175                 self.title, self.df, self.summarizer, self.typeset, self._sample
    176             )
    177         return self._description_set

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/model/describe.py in describe(title, df, summarizer, typeset, sample)
    135         # Duplicates
    136         pbar.set_postfix_str("Locating duplicates")
--> 137         duplicates = get_duplicates(df, supported_columns)
    138         pbar.update()
    139 

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/model/duplicates.py in get_duplicates(df, supported_columns)
     23             .groupby(supported_columns)
     24             .size()
---> 25             .reset_index(name="count")
     26             .nlargest(n_head, "count")
     27         )

~/anaconda3/lib/python3.7/site-packages/pandas/core/series.py in reset_index(self, level, drop, name, inplace)
   1247         else:
   1248             df = self.to_frame(name)
-> 1249             return df.reset_index(level=level, drop=drop)
   1250 
   1251     # ----------------------------------------------------------------------

~/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py in reset_index(self, level, drop, inplace, col_level, col_fill)
   5011                 # to ndarray and maybe infer different dtype
   5012                 level_values = maybe_casted_values(lev, lab)
-> 5013                 new_obj.insert(0, name, level_values)
   5014 
   5015         new_obj.index = new_index

~/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py in insert(self, loc, column, value, allow_duplicates)
   3758         self._ensure_valid_index(value)
   3759         value = self._sanitize_column(column, value, broadcast=False)
-> 3760         self._mgr.insert(loc, column, value, allow_duplicates=allow_duplicates)
   3761 
   3762     def assign(self, **kwargs) -> DataFrame:

~/anaconda3/lib/python3.7/site-packages/pandas/core/internals/managers.py in insert(self, loc, item, value, allow_duplicates)
   1189         if not allow_duplicates and item in self.items:
   1190             # Should this be a different kind of error??
-> 1191             raise ValueError(f"cannot insert {item}, already exists")
   1192 
   1193         if not isinstance(loc, int):

ValueError: cannot insert count, already exists`


3. I removed the 'count' field in train_df as if there was a conflict between my original field and the subsequent naming
4. It seems to have worked

![image](https://user-images.githubusercontent.com/39480659/110435223-b491f180-80ed-11eb-8d48-735b05f85359.png)





An Error is emitted when the ‘count’field is present in DataFrame
**Describe the bug**

1.When I load a local file with a count field, it looks like this:

![image](https://user-images.githubusercontent.com/39480659/110434491-d8086c80-80ec-11eb-9b4a-57963df424b6.png)

2. So I'm going to go ahead and execute, I met this error:
'cannot insert count, already exists'

its all:
`---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
~/anaconda3/lib/python3.7/site-packages/IPython/core/formatters.py in __call__(self, obj)
    343             method = get_real_method(obj, self.print_method)
    344             if method is not None:
--> 345                 return method()
    346             return None
    347         else:

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/profile_report.py in _repr_html_(self)
    434     def _repr_html_(self):
    435         """The ipython notebook widgets user interface gets called by the jupyter notebook."""
--> 436         self.to_notebook_iframe()
    437 
    438     def __repr__(self):

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/profile_report.py in to_notebook_iframe(self)
    414         with warnings.catch_warnings():
    415             warnings.simplefilter("ignore")
--> 416             display(get_notebook_iframe(self))
    417 
    418     def to_widgets(self):

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/report/presentation/flavours/widget/notebook.py in get_notebook_iframe(profile)
     63         output = get_notebook_iframe_src(profile)
     64     elif attribute == "srcdoc":
---> 65         output = get_notebook_iframe_srcdoc(profile)
     66     else:
     67         raise ValueError(

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/report/presentation/flavours/widget/notebook.py in get_notebook_iframe_srcdoc(profile)
     21     width = config["notebook"]["iframe"]["width"].get(str)
     22     height = config["notebook"]["iframe"]["height"].get(str)
---> 23     src = html.escape(profile.to_html())
     24 
     25     iframe = f'<iframe width="{width}" height="{height}" srcdoc="{src}" frameborder="0" allowfullscreen></iframe>'

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/profile_report.py in to_html(self)
    384 
    385         """
--> 386         return self.html
    387 
    388     def to_json(self) -> str:

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/profile_report.py in html(self)
    199     def html(self):
    200         if self._html is None:
--> 201             self._html = self._render_html()
    202         return self._html
    203 

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/profile_report.py in _render_html(self)
    306         from pandas_profiling.report.presentation.flavours import HTMLReport
    307 
--> 308         report = self.report
    309 
    310         disable_progress_bar = not config["progress_bar"].get(bool)

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/profile_report.py in report(self)
    193     def report(self):
    194         if self._report is None:
--> 195             self._report = get_report_structure(self.description_set)
    196         return self._report
    197 

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/profile_report.py in description_set(self)
    173         if self._description_set is None:
    174             self._description_set = describe_df(
--> 175                 self.title, self.df, self.summarizer, self.typeset, self._sample
    176             )
    177         return self._description_set

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/model/describe.py in describe(title, df, summarizer, typeset, sample)
    135         # Duplicates
    136         pbar.set_postfix_str("Locating duplicates")
--> 137         duplicates = get_duplicates(df, supported_columns)
    138         pbar.update()
    139 

~/anaconda3/lib/python3.7/site-packages/pandas_profiling/model/duplicates.py in get_duplicates(df, supported_columns)
     23             .groupby(supported_columns)
     24             .size()
---> 25             .reset_index(name="count")
     26             .nlargest(n_head, "count")
     27         )

~/anaconda3/lib/python3.7/site-packages/pandas/core/series.py in reset_index(self, level, drop, name, inplace)
   1247         else:
   1248             df = self.to_frame(name)
-> 1249             return df.reset_index(level=level, drop=drop)
   1250 
   1251     # ----------------------------------------------------------------------

~/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py in reset_index(self, level, drop, inplace, col_level, col_fill)
   5011                 # to ndarray and maybe infer different dtype
   5012                 level_values = maybe_casted_values(lev, lab)
-> 5013                 new_obj.insert(0, name, level_values)
   5014 
   5015         new_obj.index = new_index

~/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py in insert(self, loc, column, value, allow_duplicates)
   3758         self._ensure_valid_index(value)
   3759         value = self._sanitize_column(column, value, broadcast=False)
-> 3760         self._mgr.insert(loc, column, value, allow_duplicates=allow_duplicates)
   3761 
   3762     def assign(self, **kwargs) -> DataFrame:

~/anaconda3/lib/python3.7/site-packages/pandas/core/internals/managers.py in insert(self, loc, item, value, allow_duplicates)
   1189         if not allow_duplicates and item in self.items:
   1190             # Should this be a different kind of error??
-> 1191             raise ValueError(f"cannot insert {item}, already exists")
   1192 
   1193         if not isinstance(loc, int):

ValueError: cannot insert count, already exists`


3. I removed the 'count' field in train_df as if there was a conflict between my original field and the subsequent naming
4. It seems to have worked

![image](https://user-images.githubusercontent.com/39480659/110435223-b491f180-80ed-11eb-8d48-735b05f85359.png)







Discussion:
This is a bug, "count" is currently reserved in `pandas_profiling/model/duplicates.py`(which it shouldn't be). You are welcome to contribute a patch if you are interested. Replacing count with "duplicate_count" would already significantly reduce the chance of collusion. Alternatively we could pick an unique name base on the existing columns.
This is a bug, "count" is currently reserved in `pandas_profiling/model/duplicates.py`(which it shouldn't be). You are welcome to contribute a patch if you are interested. Replacing count with "duplicate_count" would already significantly reduce the chance of collusion. Alternatively we could pick an unique name base on the existing columns.

Resolving patch:
diff --git a/src/pandas_profiling/model/duplicates.py b/src/pandas_profiling/model/duplicates.py
--- a/src/pandas_profiling/model/duplicates.py
+++ b/src/pandas_profiling/model/duplicates.py
@@ -18,11 +18,18 @@ def get_duplicates(df: pd.DataFrame, supported_columns) -> Optional[pd.DataFrame
     n_head = config["duplicates"]["head"].get(int)
 
     if n_head > 0 and supported_columns:
+        duplicates_key = config["duplicates"]["key"].get(str)
+        if duplicates_key in df.columns:
+            raise ValueError(
+                f"Duplicates key ({duplicates_key}) may not be part of the DataFrame. Either change the "
+                f" column name in the DataFrame or change the 'duplicates.key' parameter."
+            )
+
         return (
             df[df.duplicated(subset=supported_columns, keep=False)]
             .groupby(supported_columns)
             .size()
-            .reset_index(name="count")
-            .nlargest(n_head, "count")
+            .reset_index(name=duplicates_key)
+            .nlargest(n_head, duplicates_key)
         )
     return None


Test patch:
diff --git a/tests/issues/test_issue725.py b/tests/issues/test_issue725.py
new file mode 100644
--- /dev/null
+++ b/tests/issues/test_issue725.py
@@ -0,0 +1,31 @@
+"""
+Test for issue 725:
+https://github.com/pandas-profiling/pandas-profiling/issues/725
+"""
+import numpy as np
+import pandas as pd
+import pytest
+
+from pandas_profiling.model.duplicates import get_duplicates
+
+
+@pytest.fixture(scope="module")
+def test_data():
+    np.random.seed(5)
+    df = pd.DataFrame(
+        np.random.randint(1, 100, (100, 5)),
+        columns=["a", "b", "c", "duplicates", "count"],
+    )
+    df = pd.concat([df, df], axis=0)
+    return df
+
+
+def test_issue725(test_data):
+    duplicates = get_duplicates(test_data, list(test_data.columns))
+    assert set(duplicates.columns) == set(test_data.columns).union({"# duplicates"})
+
+
+def test_issue725_existing(test_data):
+    test_data = test_data.rename(columns={"count": "# duplicates"})
+    with pytest.raises(ValueError):
+        _ = get_duplicates(test_data, list(test_data.columns))