Repository: geopandas/geopandas

Issue:
REGR: mapclassify plot error if categories are equal after formatting
The change in #2019 breaks some mapclassify-based plots. The issue is triggered when the formatted labels of two bins are equal.

MRE:

```py
nybb = gpd.read_file(gpd.datasets.get_path("nybb"))
nybb['values'] = [0.001, 0.002, 0.003, 0.004, 0.005]
nybb.plot('values', scheme="quantiles")

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/var/folders/kp/fxnnw89x5qbb9gryn507p03c0000gn/T/ipykernel_18206/434612750.py in <module>
      1 nybb = gpd.read_file(gpd.datasets.get_path("nybb"))
      2 nybb['values'] = [0.001, 0.002, 0.003, 0.004, 0.005]
----> 3 nybb.plot('values', scheme="quantiles")

~/Git/geopandas/geopandas/plotting.py in __call__(self, *args, **kwargs)
    944         kind = kwargs.pop("kind", "geo")
    945         if kind == "geo":
--> 946             return plot_dataframe(data, *args, **kwargs)
    947         if kind in self._pandas_kinds:
    948             # Access pandas plots

~/Git/geopandas/geopandas/plotting.py in plot_dataframe(df, column, cmap, color, ax, cax, categorical, legend, scheme, k, vmin, vmax, markersize, figsize, legend_kwds, categories, classification_kwds, missing_kwds, aspect, **style_kwds)
    774                 labels = [c[1:-1] for c in labels]
    775 
--> 776         values = pd.Categorical([np.nan] * len(values), categories=labels, ordered=True)
    777         values[~nan_idx] = pd.Categorical.from_codes(
    778             binning.yb, categories=labels, ordered=True

/opt/miniconda3/envs/geo_dev/lib/python3.9/site-packages/pandas/core/arrays/categorical.py in __init__(self, values, categories, ordered, dtype, fastpath, copy)
    368     ):
    369 
--> 370         dtype = CategoricalDtype._from_values_or_dtype(
    371             values, categories, ordered, dtype
    372         )

/opt/miniconda3/envs/geo_dev/lib/python3.9/site-packages/pandas/core/dtypes/dtypes.py in _from_values_or_dtype(cls, values, categories, ordered, dtype)
    294             # Note: This could potentially have categories=None and
    295             # ordered=None.
--> 296             dtype = CategoricalDtype(categories, ordered)
    297 
    298         return cast(CategoricalDtype, dtype)

/opt/miniconda3/envs/geo_dev/lib/python3.9/site-packages/pandas/core/dtypes/dtypes.py in __init__(self, categories, ordered)
    181 
    182     def __init__(self, categories=None, ordered: Ordered = False):
--> 183         self._finalize(categories, ordered, fastpath=False)
    184 
    185     @classmethod

/opt/miniconda3/envs/geo_dev/lib/python3.9/site-packages/pandas/core/dtypes/dtypes.py in _finalize(self, categories, ordered, fastpath)
    335 
    336         if categories is not None:
--> 337             categories = self.validate_categories(categories, fastpath=fastpath)
    338 
    339         self._categories = categories

/opt/miniconda3/envs/geo_dev/lib/python3.9/site-packages/pandas/core/dtypes/dtypes.py in validate_categories(categories, fastpath)
    538 
    539             if not categories.is_unique:
--> 540                 raise ValueError("Categorical categories must be unique")
    541 
    542         if isinstance(categories, ABCCategoricalIndex):

ValueError: Categorical categories must be unique
```

It happens because we format values to 2 decimals by default and use the formatted values as categories in `pandas.Categorical` here: https://github.com/geopandas/geopandas/blob/1cabaaa860c6d407cae710b878e98f6ea83d1ae0/geopandas/plotting.py#L768-L776

This is what gets passed in there:

```py
binning = mapclassify.classify(
            [0.001, 0.002, 0.003, 0.004, 0.005], "quantiles"
        )
binning.get_legend_classes()

['[0.00, 0.00]',
 '(0.00, 0.00]',
 '(0.00, 0.00]',
 '(0.00, 0.00]',
 '(0.00, 0.01]']
```

The workaround is to pass custom `fmt` now but this should be resolved under the hood. It works in 0.9.0.

```py
nybb['values'] = [0.001, 0.002, 0.003, 0.004, 0.005]
nybb.plot('values', scheme="quantiles", legend_kwds=dict(fmt="{:.3}"))
```


Discussion:
So when we create a Categorical from the mapclassify output (in case of a `scheme`), this will always be a problem if some of the generated `labels` becomes duplicated while formatting them, since pandas doesn't allow duplicated categories.

Looking at the code, it seems that a bit further down, we get the codes from the Categorical anyway (`values = cat.codes[~nan_idx]`). So probably the better solution is to never convert it to a pandas categorical in the first place for the `scheme` case?
> So probably the better solution is to never convert it to a pandas categorical in the first place for the scheme case?

Then we have to find another way of fixing #2019. 

I think that another solution would be to use unformatted bins as categories and apply formatting only during the plotting of a legend.

Resolving patch:
diff --git a/geopandas/plotting.py b/geopandas/plotting.py
--- a/geopandas/plotting.py
+++ b/geopandas/plotting.py
@@ -773,9 +773,11 @@ def plot_dataframe(
             if not show_interval:
                 labels = [c[1:-1] for c in labels]
 
-        values = pd.Categorical([np.nan] * len(values), categories=labels, ordered=True)
+        values = pd.Categorical(
+            [np.nan] * len(values), categories=binning.bins, ordered=True
+        )
         values[~nan_idx] = pd.Categorical.from_codes(
-            binning.yb, categories=labels, ordered=True
+            binning.yb, categories=binning.bins, ordered=True
         )
         if cmap is None:
             cmap = "viridis"
@@ -884,6 +886,8 @@ def plot_dataframe(
             norm = Normalize(vmin=mn, vmax=mx)
         n_cmap = cm.ScalarMappable(norm=norm, cmap=cmap)
         if categorical:
+            if scheme is not None:
+                categories = labels
             patches = []
             for value, cat in enumerate(categories):
                 patches.append(


Test patch:
diff --git a/geopandas/tests/test_plotting.py b/geopandas/tests/test_plotting.py
--- a/geopandas/tests/test_plotting.py
+++ b/geopandas/tests/test_plotting.py
@@ -1073,6 +1073,8 @@ def setup_class(cls):
         cls.df["mid_vals"] = np.linspace(0.3, 0.7, cls.df.shape[0])
         cls.df["high_vals"] = np.linspace(0.7, 1.0, cls.df.shape[0])
         cls.df.loc[cls.df.index[:20:2], "high_vals"] = np.nan
+        cls.nybb = read_file(get_path("nybb"))
+        cls.nybb["vals"] = [0.001, 0.002, 0.003, 0.004, 0.005]
 
     def test_legend(self):
         with warnings.catch_warnings(record=True) as _:  # don't print warning
@@ -1333,6 +1335,35 @@ def test_empty_bins(self):
             line.get_markerfacecolor() for line in ax3.get_legend().get_lines()
         ] == legend_colors_exp
 
+    def test_equally_formatted_bins(self):
+        ax = self.nybb.plot(
+            "vals",
+            scheme="quantiles",
+            legend=True,
+        )
+        labels = [t.get_text() for t in ax.get_legend().get_texts()]
+        expected = [
+            "0.00, 0.00",
+            "0.00, 0.00",
+            "0.00, 0.00",
+            "0.00, 0.00",
+            "0.00, 0.01",
+        ]
+        assert labels == expected
+
+        ax2 = self.nybb.plot(
+            "vals", scheme="quantiles", legend=True, legend_kwds=dict(fmt="{:.3f}")
+        )
+        labels = [t.get_text() for t in ax2.get_legend().get_texts()]
+        expected = [
+            "0.001, 0.002",
+            "0.002, 0.003",
+            "0.003, 0.003",
+            "0.003, 0.004",
+            "0.004, 0.005",
+        ]
+        assert labels == expected
+
 
 class TestPlotCollections:
     def setup_method(self):