Repository: dask/dask

Issue:
`set_index` on a column containing NaN raises confusing error (`partition_quantiles` not robust to NaNs)
**Minimal Complete Verifiable Example**:

```python
import pandas as pd
import dask.dataframe as dd

pdf = pd.DataFrame({"A": [1, 2, 3, 4, 5], "B": ["a", "b", float('nan'), "a", float('nan')]})
df = dd.from_pandas(pdf)
df.set_index("B")
# TypeError: '>=' not supported between instances of 'str' and 'float'
```

<details><summary>Full traceback</summary>

```python
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-22-d04e3b506594> in <module>
----> 1 df.set_index("B")

~/dev/dask/dask/dataframe/core.py in set_index(***failed resolving arguments***)
   4342             from .shuffle import set_index
   4343 
-> 4344             return set_index(
   4345                 self,
   4346                 other,

~/dev/dask/dask/dataframe/shuffle.py in set_index(df, index, npartitions, shuffle, compute, drop, upsample, divisions, partition_size, **kwargs)
    189 
    190     if divisions is None:
--> 191         divisions, mins, maxes = _calculate_divisions(
    192             df, index2, repartition, npartitions, upsample, partition_size
    193         )

~/dev/dask/dask/dataframe/shuffle.py in _calculate_divisions(df, partition_col, repartition, npartitions, upsample, partition_size)
     38     mins = partition_col.map_partitions(M.min)
     39     maxes = partition_col.map_partitions(M.max)
---> 40     divisions, sizes, mins, maxes = base.compute(divisions, sizes, mins, maxes)
     41     divisions = methods.tolist(divisions)
     42     if type(sizes) is not list:

~/dev/dask/dask/base.py in compute(*args, **kwargs)
    568         postcomputes.append(x.__dask_postcompute__())
    569 
--> 570     results = schedule(dsk, keys, **kwargs)
    571     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    572 

~/dev/dask/dask/threaded.py in get(dsk, result, cache, num_workers, pool, **kwargs)
     77             pool = MultiprocessingPoolExecutor(pool)
     78 
---> 79     results = get_async(
     80         pool.submit,
     81         pool._max_workers,

~/dev/dask/dask/local.py in get_async(submit, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, chunksize, **kwargs)
    505                             _execute_task(task, data)  # Re-execute locally
    506                         else:
--> 507                             raise_exception(exc, tb)
    508                     res, worker_id = loads(res_info)
    509                     state["cache"][key] = res

~/dev/dask/dask/local.py in reraise(exc, tb)
    313     if exc.__traceback__ is not tb:
    314         raise exc.with_traceback(tb)
--> 315     raise exc
    316 
    317 

~/dev/dask/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    218     try:
    219         task, data = loads(task_info)
--> 220         result = _execute_task(task, data)
    221         id = get_id()
    222         result = dumps((result, id))

~/dev/dask/dask/core.py in _execute_task(arg, cache, dsk)
    117         # temporaries by their reference count and can execute certain
    118         # operations in-place.
--> 119         return func(*(_execute_task(a, cache) for a in args))
    120     elif not ishashable(arg):
    121         return arg

~/dev/dask/dask/optimization.py in __call__(self, *args)
    967         if not len(args) == len(self.inkeys):
    968             raise ValueError("Expected %d args, got %d" % (len(self.inkeys), len(args)))
--> 969         return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
    970 
    971     def __reduce__(self):

~/dev/dask/dask/core.py in get(dsk, out, cache)
    147     for key in toposort(dsk):
    148         task = dsk[key]
--> 149         result = _execute_task(task, cache)
    150         cache[key] = result
    151     result = _execute_task(out, cache)

~/dev/dask/dask/core.py in _execute_task(arg, cache, dsk)
    117         # temporaries by their reference count and can execute certain
    118         # operations in-place.
--> 119         return func(*(_execute_task(a, cache) for a in args))
    120     elif not ishashable(arg):
    121         return arg

~/dev/dask/dask/utils.py in apply(func, args, kwargs)
     35 def apply(func, args, kwargs=None):
     36     if kwargs:
---> 37         return func(*args, **kwargs)
     38     else:
     39         return func(*args)

~/dev/dask/dask/dataframe/core.py in apply_and_enforce(*args, **kwargs)
   5948     func = kwargs.pop("_func")
   5949     meta = kwargs.pop("_meta")
-> 5950     df = func(*args, **kwargs)
   5951     if is_dataframe_like(df) or is_series_like(df) or is_index_like(df):
   5952         if not len(df):

~/dev/dask/dask/utils.py in __call__(self, _methodcaller__obj, *args, **kwargs)
   1019 
   1020     def __call__(self, __obj, *args, **kwargs):
-> 1021         return getattr(__obj, self.method)(*args, **kwargs)
   1022 
   1023     def __reduce__(self):

~/miniconda3/envs/dask-dev/lib/python3.8/site-packages/pandas/core/generic.py in stat_func(self, axis, skipna, level, numeric_only, **kwargs)
  11472         if level is not None:
  11473             return self._agg_by_level(name, axis=axis, level=level, skipna=skipna)
> 11474         return self._reduce(
  11475             func, name=name, axis=axis, skipna=skipna, numeric_only=numeric_only
  11476         )

~/miniconda3/envs/dask-dev/lib/python3.8/site-packages/pandas/core/series.py in _reduce(self, op, name, axis, skipna, numeric_only, filter_type, **kwds)
   4247                 )
   4248             with np.errstate(all="ignore"):
-> 4249                 return op(delegate, skipna=skipna, **kwds)
   4250 
   4251     def _reindex_indexer(self, new_index, indexer, copy):

~/miniconda3/envs/dask-dev/lib/python3.8/site-packages/pandas/core/nanops.py in f(values, axis, skipna, **kwds)
    127                     result = alt(values, axis=axis, skipna=skipna, **kwds)
    128             else:
--> 129                 result = alt(values, axis=axis, skipna=skipna, **kwds)
    130 
    131             return result

~/miniconda3/envs/dask-dev/lib/python3.8/site-packages/pandas/core/nanops.py in reduction(values, axis, skipna, mask)
    871                 result = np.nan
    872         else:
--> 873             result = getattr(values, meth)(axis)
    874 
    875         result = _wrap_results(result, dtype, fill_value)

~/miniconda3/envs/dask-dev/lib/python3.8/site-packages/numpy/core/_methods.py in _amax(a, axis, out, keepdims, initial, where)
     28 def _amax(a, axis=None, out=None, keepdims=False,
     29           initial=_NoValue, where=True):
---> 30     return umr_maximum(a, axis, None, out, keepdims, initial, where)
     31 
     32 def _amin(a, axis=None, out=None, keepdims=False,

TypeError: '>=' not supported between instances of 'str' and 'float'
```

</details>

Note that the error occurs eagerly while trying to calculate good divisions.

Setting a column containing NaNs as the index is probably a bad idea, so if we still want this to error, that would be reasonable. But an error like ``"Cannot calculate division for column "B" since it contains NaN values. Consider `.dropna(subset=["B"])` first."`` would be much more helpful.

Though it _is_ a valid thing to do in Pandas:
```python
>>> pdf.set_index("B")
     A
B     
a    1
b    2
NaN  3
a    4
NaN  5
```

xref https://github.com/dask/dask/issues/5720?

**Environment**:

- Dask version: c2278fece0d4fb4af1e63b6ca26e6a90f63b0fc3
- Python version: 3.8.8
- Operating System: macOS
- Install method (conda, pip, source): source


Discussion:
Yeah I think raising a better error is probably good enough. 
I'm currently dealing with this, raising an error if any `NaN` is found. 

This is definitely what should be done in Dask, because `divisions` rely on being comparable and distinct from each other (except possibly `divisions[-1]`, see the discussion in #8845).

I'm not sure actually that Pandas should tolerate such a thing; whenever there are `NaNs`, comparisons and equality are terribly confusing. For example, what should `pdf.index.is_monotonic_increasing` yield on the example written by @gjoseph92 in view of the following? `False`, `NA`, and error raised or what?

```python

# Numpy's nan
>>> from numpy import nan
>>> nan == nan
False
>>> nan != nan
True
>>> nan is nan
True
>>> float('nan') is float('nan')     # note the difference with `nan`
False
>>> nan == float('nan')
False
>>> nan > nan
False
>>> nan >= nan
False
>>> nan < nan
False
>>> nan <= nan
False

# Pandas NA for extension types
from pandas import NA
>>> NA == NA
<NA>
>>> NA != NA
<NA>
>>> NA > NA
<NA>
>>> NA >= NA
<NA>
>>> NA < NA
<NA>
>>> NA <= NA
<NA>
>>> NA is NA
True
>>> NA == nan
<NA>
```

Hope to have a PR by next week!
Thanks for engaging with this @jorloplaz! I agree that nans are often confusing, but luckily for dask the correct answer is always "whatever pandas does". So for instance `is_monotonic_increasing` should be `False`
Yes, but in this case for Dask it's very important that the `index` is sorted (whenever `known_divisions` is `True`). That's why you have the `sorted` flag in `set_index`, all the `partitionquantiles` code, etc. And as a result the elements in `divisions` must be unique and, more importantly, comparable among each other. 

That's why I strongly advocate for raising an error if trying to set an index with some missing value in that column. Maybe in the future when we settle things regarding `divisions` and unicity, we can come back to this. So perhaps a `NotImplementedError` would be the best here?
Yeah that makes sense. A `NotImplementedError` until we can come up with a more robust plan seems like a good choice.
Just made a [PR with this](https://github.com/dask/dask/pull/8963). I think the result is rather neat, hope you find it so!

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -7,7 +7,7 @@
 from numbers import Integral, Number
 from operator import getitem
 from pprint import pformat
-from typing import ClassVar
+from typing import Any, Callable, ClassVar, Literal, Mapping
 
 import numpy as np
 import pandas as pd
@@ -417,7 +417,7 @@ def divisions(self, value):
         self._divisions = value
 
     @property
-    def npartitions(self):
+    def npartitions(self) -> int:
         """Return number of partitions"""
         return len(self.divisions) - 1
 
@@ -4519,14 +4519,14 @@ def select_dtypes(self, include=None, exclude=None):
 
     def sort_values(
         self,
-        by,
-        npartitions=None,
-        ascending=True,
-        na_position="last",
-        sort_function=None,
-        sort_function_kwargs=None,
+        by: str,
+        npartitions: int | Literal["auto"] | None = None,
+        ascending: bool = True,
+        na_position: Literal["first"] | Literal["last"] = "last",
+        sort_function: Callable[[pd.DataFrame], pd.DataFrame] | None = None,
+        sort_function_kwargs: Mapping[str, Any] | None = None,
         **kwargs,
-    ):
+    ) -> DataFrame:
         """Sort the dataset by a single column.
 
         Sorting a parallel dataset requires expensive shuffles and is generally
@@ -4571,12 +4571,12 @@ def sort_values(
 
     def set_index(
         self,
-        other,
-        drop=True,
-        sorted=False,
-        npartitions=None,
-        divisions=None,
-        inplace=False,
+        other: str | Series,
+        drop: bool = True,
+        sorted: bool = False,
+        npartitions: int | Literal["auto"] | None = None,
+        divisions: Sequence | None = None,
+        inplace: bool = False,
         **kwargs,
     ):
         """Set the DataFrame index (row labels) using an existing column.
@@ -4606,6 +4606,7 @@ def set_index(
         Parameters
         ----------
         other: string or Dask Series
+            Column to use as index.
         drop: boolean, default True
             Delete column to be used as the new index.
         sorted: bool, optional
@@ -4680,15 +4681,56 @@ def set_index(
         >>> # ^ Now copy-paste this and edit the line above to:
         >>> # ddf2 = ddf.set_index("name", divisions=["Alice", "Laura", "Ursula", "Zelda"])
         """
+
         if inplace:
             raise NotImplementedError("The inplace= keyword is not supported")
         pre_sorted = sorted
         del sorted
 
-        if isinstance(other, Series) and other._name == self.index._name:
-            # Index is equal to current index, no-op
-            return self
+        # Check other can be translated to column name or column object, possibly flattening it
+        if not isinstance(other, str):
+
+            # It may refer to several columns
+            if isinstance(other, Sequence):
+                # Accept ["a"], but not [["a"]]
+                if len(other) == 1 and (
+                    isinstance(other[0], str) or not isinstance(other[0], Sequence)
+                ):
+                    other = other[0]
+                else:
+                    raise NotImplementedError(
+                        "Dask dataframe does not yet support multi-indexes.\n"
+                        f"You tried to index with this index: {other}\n"
+                        "Indexes must be single columns only."
+                    )
+
+            # Or be a frame directly
+            elif isinstance(other, DataFrame):
+                raise NotImplementedError(
+                    "Dask dataframe does not yet support multi-indexes.\n"
+                    f"You tried to index with a frame with these columns: {list(other.columns)}\n"
+                    "Indexes must be single columns only."
+                )
+
+        # If already a series
+        if isinstance(other, Series):
+            # If it's already the index, there's nothing to do
+            if other._name == self.index._name:
+                return self
+
+        # If the name of a column/index
+        else:
+            # With the same name as the index, there's nothing to do either
+            if other == self.index.name:
+                return self
+
+            # If a missing column, KeyError
+            if other not in self.columns:
+                raise KeyError(
+                    f"Data has no column '{other}': use any column of {list(self.columns)}"
+                )
 
+        # Check divisions
         if divisions is not None:
             check_divisions(divisions)
         elif (
@@ -4702,6 +4744,7 @@ def set_index(
             pre_sorted = True
             divisions = other.divisions
 
+        # If index is already sorted, take advantage of that with set_sorted_index
         if pre_sorted:
             from dask.dataframe.shuffle import set_sorted_index
 
@@ -5969,7 +6012,10 @@ def handle_out(out, result):
         msg = (
             "The out parameter is not fully supported."
             " Received type %s, expected %s "
-            % (typename(type(out)), typename(type(result)))
+            % (
+                typename(type(out)),
+                typename(type(result)),
+            )
         )
         raise NotImplementedError(msg)
     else:
@@ -6398,9 +6444,9 @@ def map_partitions(
 
         args2.insert(0, BlockwiseDepDict(partition_info))
         orig_func = func
-        func = lambda partition_info, *args, **kwargs: orig_func(
-            *args, **kwargs, partition_info=partition_info
-        )
+
+        def func(partition_info, *args, **kwargs):
+            return orig_func(*args, **kwargs, partition_info=partition_info)
 
     if enforce_metadata:
         dsk = partitionwise_graph(
@@ -6893,12 +6939,7 @@ def repartition_divisions(a, b, name, out1, out2, force=False):
     ----------
     a : tuple
         old divisions
-    b : tuple, list
-        new divisions
-    name : str
-        name of old dataframe
-    out1 : str
-        name of temporary splits
+    b : tuple, listmypy
     out2 : str
         name of new dataframe
     force : bool, default False
diff --git a/dask/dataframe/io/io.py b/dask/dataframe/io/io.py
--- a/dask/dataframe/io/io.py
+++ b/dask/dataframe/io/io.py
@@ -3,6 +3,7 @@
 from math import ceil
 from operator import getitem
 from threading import Lock
+from typing import Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -141,7 +142,13 @@ def from_array(x, chunksize=50000, columns=None, meta=None):
     return new_dd_object(dsk, name, meta, divisions)
 
 
-def from_pandas(data, npartitions=None, chunksize=None, sort=True, name=None):
+def from_pandas(
+    data: Union[pd.DataFrame, pd.Series],
+    npartitions: Optional[int] = None,
+    chunksize: Optional[int] = None,
+    sort: bool = True,
+    name: Optional[str] = None,
+) -> DataFrame:
     """
     Construct a Dask DataFrame from a Pandas DataFrame
 
@@ -212,24 +219,38 @@ def from_pandas(data, npartitions=None, chunksize=None, sort=True, name=None):
         raise NotImplementedError("Dask does not support MultiIndex Dataframes.")
 
     if not has_parallel_type(data):
-        raise TypeError("Input must be a pandas DataFrame or Series")
+        raise TypeError("Input must be a pandas DataFrame or Series.")
 
-    if (npartitions is None) == (chunksize is None):
+    if (npartitions is None) == (none_chunksize := (chunksize is None)):
         raise ValueError("Exactly one of npartitions and chunksize must be specified.")
 
     nrows = len(data)
 
-    if chunksize is None:
+    if none_chunksize:
+        if not isinstance(npartitions, int):
+            raise TypeError(
+                "Please provide npartitions as an int, or possibly as None if you specify chunksize."
+            )
         chunksize = int(ceil(nrows / npartitions))
+    elif not isinstance(chunksize, int):
+        raise TypeError(
+            "Please provide chunksize as an int, or possibly as None if you specify npartitions."
+        )
 
     name = name or ("from_pandas-" + tokenize(data, chunksize))
 
     if not nrows:
         return new_dd_object({(name, 0): data}, name, data, [None, None])
 
-    if sort and not data.index.is_monotonic_increasing:
-        data = data.sort_index(ascending=True)
+    if data.index.isna().any() and not data.index.is_numeric():
+        raise NotImplementedError(
+            "Index in passed data is non-numeric and contains nulls, which Dask does not entirely support.\n"
+            "Consider passing `data.loc[~data.isna()]` instead."
+        )
+
     if sort:
+        if not data.index.is_monotonic_increasing:
+            data = data.sort_index(ascending=True)
         divisions, locations = sorted_division_locations(
             data.index, chunksize=chunksize
         )
diff --git a/dask/dataframe/shuffle.py b/dask/dataframe/shuffle.py
--- a/dask/dataframe/shuffle.py
+++ b/dask/dataframe/shuffle.py
@@ -5,10 +5,13 @@
 import tempfile
 import uuid
 import warnings
+from collections.abc import Sequence
+from typing import Any, Callable, List, Literal, Mapping, Optional, Tuple, Union
 
 import numpy as np
 import pandas as pd
 import tlz as toolz
+from pandas.api.types import is_numeric_dtype
 
 from dask import config
 from dask.base import compute, compute_as_if_collection, is_dask_collection, tokenize
@@ -25,13 +28,13 @@
 
 
 def _calculate_divisions(
-    df,
-    partition_col,
-    repartition,
-    npartitions,
-    upsample=1.0,
-    partition_size=128e6,
-):
+    df: DataFrame,
+    partition_col: Series,
+    repartition: bool,
+    npartitions: int,
+    upsample: float = 1.0,
+    partition_size: float = 128e6,
+) -> Tuple[List, List, List]:
     """
     Utility function to calculate divisions for calls to `map_partitions`
     """
@@ -39,14 +42,36 @@ def _calculate_divisions(
     divisions = partition_col._repartition_quantiles(npartitions, upsample=upsample)
     mins = partition_col.map_partitions(M.min)
     maxes = partition_col.map_partitions(M.max)
-    divisions, sizes, mins, maxes = compute(divisions, sizes, mins, maxes)
+
+    try:
+        divisions, sizes, mins, maxes = compute(divisions, sizes, mins, maxes)
+    except TypeError as e:
+        # When there are nulls and a column is non-numeric, a TypeError is sometimes raised as a result of
+        # 1) computing mins/maxes above, 2) every null being switched to NaN, and 3) NaN being a float.
+        # Also, Pandas ExtensionDtypes may cause TypeErrors when dealing with special nulls such as pd.NaT or pd.NA.
+        # If this happens, we hint the user about eliminating nulls beforehand.
+        if not is_numeric_dtype(partition_col.dtype):
+            obj, suggested_method = (
+                ("column", f"`.dropna(subset=['{partition_col.name}'])`")
+                if any(partition_col._name == df[c]._name for c in df)
+                else ("series", "`.loc[series[~series.isna()]]`")
+            )
+            raise NotImplementedError(
+                f"Divisions calculation failed for non-numeric {obj} '{partition_col.name}'.\n"
+                f"This is probably due to the presence of nulls, which Dask does not entirely support in the index.\n"
+                f"We suggest you try with {suggested_method}."
+            ) from e
+        # For numeric types there shouldn't be problems with nulls, so we raise as-it-is this particular TypeError
+        else:
+            raise e
+
     divisions = methods.tolist(divisions)
     if type(sizes) is not list:
         sizes = methods.tolist(sizes)
     mins = methods.tolist(mins)
     maxes = methods.tolist(maxes)
 
-    empty_dataframe_detected = pd.isnull(divisions).all()
+    empty_dataframe_detected = pd.isna(divisions).all()
     if repartition or empty_dataframe_detected:
         total = sum(sizes)
         npartitions = max(math.ceil(total / partition_size), 1)
@@ -76,17 +101,17 @@ def _calculate_divisions(
 
 
 def sort_values(
-    df,
-    by,
-    npartitions=None,
-    ascending=True,
-    na_position="last",
-    upsample=1.0,
-    partition_size=128e6,
-    sort_function=None,
-    sort_function_kwargs=None,
+    df: DataFrame,
+    by: Union[str, List[str]],
+    npartitions: Optional[Union[int, Literal["auto"]]] = None,
+    ascending: Union[bool, List[bool]] = True,
+    na_position: Union[Literal["first"], Literal["last"]] = "last",
+    upsample: float = 1.0,
+    partition_size: float = 128e6,
+    sort_function: Optional[Callable[[pd.DataFrame], pd.DataFrame]] = None,
+    sort_function_kwargs: Optional[Mapping[str, Any]] = None,
     **kwargs,
-):
+) -> DataFrame:
     """See DataFrame.sort_values for docstring"""
     if na_position not in ("first", "last"):
         raise ValueError("na_position must be either 'first' or 'last'")
@@ -174,35 +199,18 @@ def sort_values(
 
 
 def set_index(
-    df,
-    index,
-    npartitions=None,
-    shuffle=None,
-    compute=False,
-    drop=True,
-    upsample=1.0,
-    divisions=None,
-    partition_size=128e6,
+    df: DataFrame,
+    index: Union[str, Series],
+    npartitions: Optional[Union[int, Literal["auto"]]] = None,
+    shuffle: Optional[str] = None,
+    compute: bool = False,
+    drop: bool = True,
+    upsample: float = 1.0,
+    divisions: Optional[Sequence] = None,
+    partition_size: float = 128e6,
     **kwargs,
-):
+) -> DataFrame:
     """See _Frame.set_index for docstring"""
-    if isinstance(index, Series) and index._name == df.index._name:
-        return df
-    if isinstance(index, (DataFrame, tuple, list)):
-        # Accept ["a"], but not [["a"]]
-        if (
-            isinstance(index, list)
-            and len(index) == 1
-            and not isinstance(index[0], list)  # if index = [["a"]], leave it that way
-        ):
-            index = index[0]
-        else:
-            raise NotImplementedError(
-                "Dask dataframe does not yet support multi-indexes.\n"
-                "You tried to index with this index: %s\n"
-                "Indexes must be single columns only." % str(index)
-            )
-
     if npartitions == "auto":
         repartition = True
         npartitions = max(100, df.npartitions)
@@ -236,7 +244,7 @@ def set_index(
     )
 
 
-def remove_nans(divisions):
+def remove_nans(divisions: Sequence) -> List:
     """Remove nans from divisions
 
     These sometime pop up when we call min/max on an empty partition
@@ -266,8 +274,14 @@ def remove_nans(divisions):
 
 
 def set_partition(
-    df, index, divisions, max_branch=32, drop=True, shuffle=None, compute=None
-):
+    df: DataFrame,
+    index: Union[str, Series],
+    divisions: Sequence,
+    max_branch: int = 32,
+    drop: bool = True,
+    shuffle: Optional[str] = None,
+    compute: Optional[bool] = None,
+) -> DataFrame:
     """Group DataFrame by index
 
     Sets a new index and partitions data along that index according to
@@ -304,7 +318,7 @@ def set_partition(
         # pd.isna considers tuples to be scalars. Convert to a list.
         divisions = list(divisions)
 
-    if np.isscalar(index):
+    if not isinstance(index, Series):
         dtype = df[index].dtype
     else:
         dtype = index.dtype
@@ -321,7 +335,7 @@ def set_partition(
     else:
         divisions = df._meta._constructor_sliced(divisions, dtype=dtype)
 
-    if np.isscalar(index):
+    if not isinstance(index, Series):
         partitions = df[index].map_partitions(
             set_partitions_pre, divisions=divisions, meta=meta
         )
@@ -342,7 +356,7 @@ def set_partition(
         ignore_index=True,
     )
 
-    if np.isscalar(index):
+    if not isinstance(index, Series):
         df4 = df3.map_partitions(
             set_index_post_scalar,
             index_name=index,
@@ -983,7 +997,7 @@ def fix_overlap(ddf, mins, maxes, lens):
     """
     name = "fix-overlap-" + tokenize(ddf, mins, maxes, lens)
 
-    non_empties = [i for i, l in enumerate(lens) if l != 0]
+    non_empties = [i for i, length in enumerate(lens) if length != 0]
     # drop empty partitions by mapping each partition in a new graph to a particular
     # partition on the old graph.
     dsk = {(name, i): (ddf._name, div) for i, div in enumerate(non_empties)}
@@ -1016,7 +1030,9 @@ def fix_overlap(ddf, mins, maxes, lens):
     return new_dd_object(graph, name, ddf._meta, divisions)
 
 
-def _compute_partition_stats(column, allow_overlap=False, **kwargs) -> tuple:
+def _compute_partition_stats(
+    column: Series, allow_overlap: bool = False, **kwargs
+) -> Tuple[List, List, List[int]]:
     """For a given column, compute the min, max, and len of each partition.
 
     And make sure that the partitions are sorted relative to each other.
@@ -1028,8 +1044,8 @@ def _compute_partition_stats(column, allow_overlap=False, **kwargs) -> tuple:
     mins, maxes, lens = compute(mins, maxes, lens, **kwargs)
     mins = remove_nans(mins)
     maxes = remove_nans(maxes)
-    non_empty_mins = [m for m, l in zip(mins, lens) if l != 0]
-    non_empty_maxes = [m for m, l in zip(maxes, lens) if l != 0]
+    non_empty_mins = [m for m, length in zip(mins, lens) if length != 0]
+    non_empty_maxes = [m for m, length in zip(maxes, lens) if length != 0]
     if (
         sorted(non_empty_mins) != non_empty_mins
         or sorted(non_empty_maxes) != non_empty_maxes
@@ -1049,20 +1065,21 @@ def _compute_partition_stats(column, allow_overlap=False, **kwargs) -> tuple:
             f"for each partition are : {list(zip(mins, maxes, lens))}",
             UserWarning,
         )
+    lens = methods.tolist(lens)
     if not allow_overlap:
         return (mins, maxes, lens)
     else:
         return (non_empty_mins, non_empty_maxes, lens)
 
 
-def compute_divisions(df, col=None, **kwargs) -> tuple:
+def compute_divisions(df: DataFrame, col: Optional[Any] = None, **kwargs) -> Tuple:
     column = df.index if col is None else df[col]
     mins, maxes, _ = _compute_partition_stats(column, allow_overlap=False, **kwargs)
 
     return tuple(mins) + (maxes[-1],)
 
 
-def compute_and_set_divisions(df, **kwargs):
+def compute_and_set_divisions(df: DataFrame, **kwargs) -> DataFrame:
     mins, maxes, lens = _compute_partition_stats(df.index, allow_overlap=True, **kwargs)
     if len(mins) == len(df.divisions) - 1:
         df._divisions = tuple(mins) + (maxes[-1],)
@@ -1072,8 +1089,15 @@ def compute_and_set_divisions(df, **kwargs):
     return fix_overlap(df, mins, maxes, lens)
 
 
-def set_sorted_index(df, index, drop=True, divisions=None, **kwargs):
-    if isinstance(index, _Frame):
+def set_sorted_index(
+    df: DataFrame,
+    index: Union[str, Series],
+    drop: bool = True,
+    divisions: Optional[Sequence] = None,
+    **kwargs,
+) -> DataFrame:
+
+    if isinstance(index, Series):
         meta = df._meta.set_index(index._meta, drop=drop)
     else:
         meta = df._meta.set_index(index, drop=drop)


Test patch:
diff --git a/dask/dataframe/io/tests/test_io.py b/dask/dataframe/io/tests/test_io.py
--- a/dask/dataframe/io/tests/test_io.py
+++ b/dask/dataframe/io/tests/test_io.py
@@ -355,6 +355,27 @@ def test_from_pandas_with_datetime_index():
     assert_eq(df, ddf)
 
 
+@pytest.mark.parametrize("null_value", [None, pd.NaT, pd.NA])
+def test_from_pandas_with_index_nulls(null_value):
+    df = pd.DataFrame({"x": [1, 2, 3]}, index=["C", null_value, "A"])
+    with pytest.raises(NotImplementedError, match="is non-numeric and contains nulls"):
+        dd.from_pandas(df, npartitions=2, sort=False)
+
+
+def test_from_pandas_with_wrong_args():
+    df = pd.DataFrame({"x": [1, 2, 3]}, index=[3, 2, 1])
+    with pytest.raises(TypeError, match="must be a pandas DataFrame or Series"):
+        dd.from_pandas("foo")
+    with pytest.raises(
+        ValueError, match="one of npartitions and chunksize must be specified"
+    ):
+        dd.from_pandas(df)
+    with pytest.raises(TypeError, match="provide npartitions as an int"):
+        dd.from_pandas(df, npartitions=5.2, sort=False)
+    with pytest.raises(TypeError, match="provide chunksize as an int"):
+        dd.from_pandas(df, chunksize=18.27)
+
+
 def test_DataFrame_from_dask_array():
     x = da.ones((10, 3), chunks=(4, 2))
     pdf = pd.DataFrame(np.ones((10, 3)), columns=["a", "b", "c"])
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -3676,6 +3676,46 @@ def test_timeseries_sorted():
     assert_eq(ddf.set_index("index", sorted=True, drop=True), df)
 
 
+def test_index_errors():
+    df = _compat.makeTimeDataFrame()
+    ddf = dd.from_pandas(df.reset_index(), npartitions=2)
+    with pytest.raises(NotImplementedError, match="You tried to index with this index"):
+        ddf.set_index([["A"]])  # should index with ["A"] instead of [["A"]]
+    with pytest.raises(NotImplementedError, match="You tried to index with a frame"):
+        ddf.set_index(ddf[["A"]])  # should index with ddf["A"] instead of ddf[["A"]]
+    with pytest.raises(KeyError, match="has no column"):
+        ddf.set_index("foo")  # a column that doesn't exist
+    with pytest.raises(KeyError, match="has no column"):
+        # column name doesn't need to be a string, but anyhow a KeyError should be raised if not found
+        ddf.set_index(0)
+
+
+@pytest.mark.parametrize("null_value", [None, pd.NaT, pd.NA])
+def test_index_nulls(null_value):
+    "Setting the index with some non-numeric null raises error"
+    df = pd.DataFrame(
+        {"numeric": [1, 2, 3, 4], "non_numeric": ["foo", "bar", "foo", "bar"]}
+    )
+    # an object column all set to nulls fails
+    ddf = dd.from_pandas(df, npartitions=2).assign(**{"non_numeric": null_value})
+    with pytest.raises(NotImplementedError, match="presence of nulls"):
+        ddf.set_index("non_numeric")
+    # an object column with only some nulls also fails
+    ddf = dd.from_pandas(df, npartitions=2)
+    with pytest.raises(NotImplementedError, match="presence of nulls"):
+        ddf.set_index(ddf["non_numeric"].map({"foo": "foo", "bar": null_value}))
+
+
+def test_set_index_with_index():
+    "Setting the index with the existing index is a no-op"
+    df = pd.DataFrame({"x": [1, 2, 3, 4], "y": [1, 0, 1, 0]}).set_index("x")
+    ddf = dd.from_pandas(df, npartitions=2)
+    ddf2 = ddf.set_index(ddf.index)
+    assert ddf2 is ddf
+    ddf = ddf.set_index("x", drop=False)
+    assert ddf2 is ddf
+
+
 def test_column_assignment():
     df = pd.DataFrame({"x": [1, 2, 3, 4], "y": [1, 0, 1, 0]})
     ddf = dd.from_pandas(df, npartitions=2)
diff --git a/dask/dataframe/tests/test_shuffle.py b/dask/dataframe/tests/test_shuffle.py
--- a/dask/dataframe/tests/test_shuffle.py
+++ b/dask/dataframe/tests/test_shuffle.py
@@ -370,7 +370,7 @@ def test_set_index_with_explicit_divisions():
     def throw(*args, **kwargs):
         raise Exception()
 
-    with dask.config.set(get=throw):
+    with dask.config.set(scheduler=throw):
         ddf2 = ddf.set_index("x", divisions=[1, 3, 5])
     assert ddf2.divisions == (1, 3, 5)
 
@@ -434,11 +434,11 @@ def test_set_index_divisions_sorted():
     def throw(*args, **kwargs):
         raise Exception("Shouldn't have computed")
 
-    with dask.config.set(get=throw):
+    with dask.config.set(scheduler=throw):
         res = ddf.set_index("x", divisions=[10, 13, 16, 18], sorted=True)
     assert_eq(res, df.set_index("x"))
 
-    with dask.config.set(get=throw):
+    with dask.config.set(scheduler=throw):
         res = ddf.set_index("y", divisions=["a", "b", "d", "e"], sorted=True)
     assert_eq(res, df.set_index("y"))