Repository: dask/dask

Issue:
Exceptions encountered after upgrading to 2021.5.0 with optimization.fuse.active: False
**What happened**:

Some of our internal code started to fail with various exceptions when upgrading from dask, distributed `2021.4.2` to `2021.5.0`.

Example exceptions:
* `TypeError: string indices must be integers`
* `TypeError: Trying to convert dd.Scalar<series-..., dtype=bool> to a boolean value. Because Dask objects are lazily evaluated, they cannot be converted to a boolean value or used in boolean conditions like if statements. Try calling .compute() to force computation prior to converting to a boolean value or using in a conditional statement.`
* `AssertionError: error`

Configuring `optimization.fuse.active: True` or downgrading to `2021.4.2` resolves all exceptions.

**What you expected to happen**:

Code runs without throwing exception.

**Minimal Complete Verifiable Example**:

For the first exception encountered:

```python
from datetime import date, timedelta

import dask
from dask import dataframe as dd
from distributed import Client
import numpy as np
import pandas as pd


def random_date(start=date(2014, 1, 1), end=date(2015, 1, 1)):
    delta = end - start
    int_delta = (delta.days * 24 * 60 * 60) + delta.seconds
    random_second = np.random.randint(int_delta)
    return start + timedelta(seconds=random_second)


def generate_data(n_samples, start_date, end_date, npartitions):
    df = pd.DataFrame(
        {
            "dates": [
                pd.to_datetime(random_date(start_date, end_date))
                for _ in range(n_samples)
            ]
        }
    )
    ddf = dd.from_pandas(df, npartitions=npartitions)
    return ddf["dates"]


def generate_data_differently(n_samples, start_date, end_date, npartitions):
    series = pd.Series(
        [pd.to_datetime(random_date(start_date, end_date)) for _ in range(n_samples)]
    )
    dseries = dd.from_pandas(series, npartitions=npartitions)
    return dseries


def failing_manipulation(series, mid_value, len_series=True):
    print("fuse active:", dask.config.config["optimization"]["fuse"]["active"])
    istest = series < np.datetime64(mid_value)
    test_len = np.sum(istest)
    if len_series:
        print(len(series))
    print(test_len.compute())


def main():
    start_date = date(2014, 1, 1)
    end_date = date(2015, 1, 1)
    mid_date = date(2014, 6, 1)
    n_samples = 10_000
    npartitions = 10

    print("Fuse disabled, series selected from dataframe, prints length of istest:")
    with Client():
        with dask.config.set({"optimization.fuse.active": False}):
            dates = generate_data(n_samples, start_date, end_date, npartitions)
            failing_manipulation(dates, mid_date, len_series=False)

    print(
        "\nFuse disabled, series not selected from dataframe, prints length of series and istest:"
    )
    with Client():
        with dask.config.set({"optimization.fuse.active": False}):
            dates = generate_data_differently(
                n_samples, start_date, end_date, npartitions
            )
            failing_manipulation(dates, mid_date)

    print(
        "\nFuse enabled, series selected from dataframe, prints length of series and istest:"
    )
    with Client():
        with dask.config.set({"optimization.fuse.active": True}):
            dates = generate_data(n_samples, start_date, end_date, npartitions)
            failing_manipulation(dates, mid_date)

    print(
        "\nFuse disabled, series selected from dataframe, prints length of series and throws an exception:"
    )
    with Client():
        with dask.config.set({"optimization.fuse.active": False}):
            dates = generate_data(n_samples, start_date, end_date, npartitions)
            failing_manipulation(dates, mid_date)


if __name__ == "__main__":
    main()
```

Yields the following output:

```
Fuse disabled, series selected from dataframe, prints length of istest:
fuse active: False
4235

Fuse disabled, series not selected from dataframe, prints length of series and istest:
fuse active: False
10000
4127

Fuse enabled, series selected from dataframe, prints length of series and istest:
fuse active: True
10000
4108

Fuse disabled, series selected from dataframe, prints length of series and throws an exception:
distributed.worker - WARNING - Compute Failed
Function:  subgraph_callable
args:      (          dates
1000 2014-01-07
1001 2014-02-21
1002 2014-07-15
1003 2014-05-29
1004 2014-09-13
...         ...
1995 2014-10-23
1996 2014-11-23
1997 2014-08-04
1998 2014-01-21
1999 2014-12-14

[1000 rows x 1 columns])
kwargs:    {}
Exception: TypeError('string indices must be integers')

distributed.worker - WARNING - Compute Failed
Function:  subgraph_callable
args:      (          dates
2000 2014-09-19
2001 2014-09-26
2002 2014-08-31
2003 2014-12-04
2004 2014-02-08
...         ...
2995 2014-08-12
2996 2014-10-01
2997 2014-06-11
2998 2014-11-18
2999 2014-05-15

[1000 rows x 1 columns])
kwargs:    {}
Exception: TypeError('string indices must be integers')

distributed.worker - WARNING - Compute Failed
Function:  subgraph_callable
args:      (          dates
3000 2014-05-13
3001 2014-01-15
3002 2014-09-01
3003 2014-10-24
3004 2014-06-21
...         ...
3995 2014-03-19
3996 2014-09-01
3997 2014-12-11
3998 2014-10-15
3999 2014-06-08

[1000 rows x 1 columns])
kwargs:    {}
Exception: TypeError('string indices must be integers')

distributed.worker - WARNING - Compute Failed
Function:  subgraph_callable
args:      (         dates
0   2014-10-23
1   2014-07-04
2   2014-06-03
3   2014-01-14
4   2014-08-04
..         ...
995 2014-08-25
996 2014-03-07
997 2014-08-31
998 2014-09-04
999 2014-07-08

[1000 rows x 1 columns])
kwargs:    {}
Exception: TypeError('string indices must be integers')

distributed.worker - WARNING - Compute Failed
Function:  subgraph_callable
args:      (          dates
4000 2014-10-23
4001 2014-03-04
4002 2014-12-12
4003 2014-04-01
4004 2014-06-25
...         ...
4995 2014-01-30
4996 2014-07-11
4997 2014-11-14
4998 2014-12-20
4999 2014-06-20

[1000 rows x 1 columns])
kwargs:    {}
Exception: TypeError('string indices must be integers')

distributed.worker - WARNING - Compute Failed
Function:  subgraph_callable
args:      (          dates
5000 2014-11-19
5001 2014-06-21
5002 2014-08-09
5003 2014-11-14
5004 2014-06-16
...         ...
5995 2014-09-16
5996 2014-05-20
5997 2014-02-19
5998 2014-08-08
5999 2014-07-06

[1000 rows x 1 columns])
kwargs:    {}
Exception: TypeError('string indices must be integers')

distributed.worker - WARNING - Compute Failed
Function:  subgraph_callable
args:      (          dates
6000 2014-02-22
6001 2014-12-14
6002 2014-12-24
6003 2014-06-02
6004 2014-04-22
...         ...
6995 2014-03-25
6996 2014-02-26
6997 2014-09-09
6998 2014-12-01
6999 2014-12-10

[1000 rows x 1 columns])
kwargs:    {}
Exception: TypeError('string indices must be integers')

distributed.worker - WARNING - Compute Failed
Function:  subgraph_callable
args:      (          dates
7000 2014-10-05
7001 2014-10-29
7002 2014-10-05
7003 2014-07-17
7004 2014-11-13
...         ...
7995 2014-08-07
7996 2014-10-19
7997 2014-01-30
7998 2014-08-09
7999 2014-05-19

[1000 rows x 1 columns])
kwargs:    {}
Exception: TypeError('string indices must be integers')

distributed.worker - WARNING - Compute Failed
Function:  subgraph_callable
args:      (          dates
8000 2014-07-06
8001 2014-05-16
8002 2014-12-07
8003 2014-09-07
8004 2014-01-15
...         ...
8995 2014-01-16
8996 2014-05-21
8997 2014-06-06
8998 2014-09-22
8999 2014-05-28

[1000 rows x 1 columns])
kwargs:    {}
Exception: TypeError('string indices must be integers')

distributed.worker - WARNING - Compute Failed
Function:  subgraph_callable
args:      (          dates
9000 2014-09-27
9001 2014-07-02
9002 2014-09-01
9003 2014-06-13
9004 2014-06-01
...         ...
9995 2014-10-18
9996 2014-09-22
9997 2014-04-08
9998 2014-12-21
9999 2014-04-14

[1000 rows x 1 columns])
kwargs:    {}
Exception: TypeError('string indices must be integers')

fuse active: False
10000
Traceback (most recent call last):
  File "failing_example.py", line 88, in <module>
    main()
  File "failing_example.py", line 84, in main
    failing_manipulation(dates, mid_date)
  File "failing_example.py", line 44, in failing_manipulation
    print(test_len.compute())
  File "/Users/<user>/.pyenv/versions/miniconda3-latest/envs/particles-dev/lib/python3.8/site-packages/dask/base.py", line 285, in compute
    (result,) = compute(self, traverse=False, **kwargs)
  File "/Users/<user>/.pyenv/versions/miniconda3-latest/envs/particles-dev/lib/python3.8/site-packages/dask/base.py", line 567, in compute
    results = schedule(dsk, keys, **kwargs)
  File "/Users/<user>/.pyenv/versions/miniconda3-latest/envs/particles-dev/lib/python3.8/site-packages/distributed/client.py", line 2674, in get
    results = self.gather(packed, asynchronous=asynchronous, direct=direct)
  File "/Users/<user>/.pyenv/versions/miniconda3-latest/envs/particles-dev/lib/python3.8/site-packages/distributed/client.py", line 1983, in gather
    return self.sync(
  File "/Users/<user>/.pyenv/versions/miniconda3-latest/envs/particles-dev/lib/python3.8/site-packages/distributed/client.py", line 851, in sync
    return sync(
  File "/Users/<user>/.pyenv/versions/miniconda3-latest/envs/particles-dev/lib/python3.8/site-packages/distributed/utils.py", line 354, in sync
    raise exc.with_traceback(tb)
  File "/Users/<user>/.pyenv/versions/miniconda3-latest/envs/particles-dev/lib/python3.8/site-packages/distributed/utils.py", line 337, in f
    result[0] = yield future
  File "/Users/<user>/.pyenv/versions/miniconda3-latest/envs/particles-dev/lib/python3.8/site-packages/tornado/gen.py", line 762, in run
    value = future.result()
  File "/Users/<user>/.pyenv/versions/miniconda3-latest/envs/particles-dev/lib/python3.8/site-packages/distributed/client.py", line 1848, in _gather
    raise exception.with_traceback(traceback)
  File "/Users/<user>/.pyenv/versions/miniconda3-latest/envs/particles-dev/lib/python3.8/site-packages/dask/optimization.py", line 966, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/Users/<user>/.pyenv/versions/miniconda3-latest/envs/particles-dev/lib/python3.8/site-packages/dask/core.py", line 151, in get
    result = _execute_task(task, cache)
  File "/Users/<user>/.pyenv/versions/miniconda3-latest/envs/particles-dev/lib/python3.8/site-packages/dask/core.py", line 121, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
TypeError: string indices must be integers
```

**Anything else we need to know?**:

N/A

**Environment**:

- Dask version: 2021.5.0
- Python version: 3.8.10
- Operating System: OS X, Linux
- Install method (conda, pip, source): conda


Discussion:
Thanks for reporting and providing a nice minimal reproducer @dankerrigan! I'm able to reproduce locally with the latest `dask` `main` branch. cc @rjzamora for visibility 
Thank you for raising @dankerrigan - It looks like the problem goes away when `optimize_blockwise` is removed from the HLG optimization pass.  I'll try to figure out what is going wrong here :/

Resolving patch:
diff --git a/dask/blockwise.py b/dask/blockwise.py
--- a/dask/blockwise.py
+++ b/dask/blockwise.py
@@ -1405,7 +1405,7 @@ def rewrite_blockwise(inputs):
             new_indices.append(x)
 
     sub = {blockwise_token(k): blockwise_token(v) for k, v in sub.items()}
-    dsk = {k: subs(v, sub) for k, v in dsk.items()}
+    dsk = {k: subs(v, sub) for k, v in dsk.items() if k not in sub.keys()}
 
     indices_check = {k for k, v in indices if v is not None}
     numblocks = toolz.merge([inp.numblocks for inp in inputs.values()])


Test patch:
diff --git a/dask/tests/test_distributed.py b/dask/tests/test_distributed.py
--- a/dask/tests/test_distributed.py
+++ b/dask/tests/test_distributed.py
@@ -353,6 +353,25 @@ def test_blockwise_dataframe_io(c, tmpdir, io, fuse):
         dd.assert_eq(ddf, df, check_index=False)
 
 
+def test_blockwise_fusion_after_compute(c):
+    # See: https://github.com/dask/dask/issues/7720
+
+    pd = pytest.importorskip("pandas")
+    dd = pytest.importorskip("dask.dataframe")
+
+    # Simple sequence of Dask-Dataframe manipulations
+    df = pd.DataFrame({"x": [1, 2, 3] * 5})
+    series = dd.from_pandas(df, npartitions=2)["x"]
+    result = series < 3
+
+    # Trigger an optimization of the `series` graph
+    # (which `result` depends on), then compute `result`.
+    # This is essentially a test of `rewrite_blockwise`.
+    series_len = len(series)
+    assert series_len == 15
+    assert df.x[result.compute()].sum() == 15
+
+
 @gen_cluster(client=True)
 async def test_blockwise_numpy_args(c, s, a, b):
     """Test pack/unpack of blockwise that includes a NumPy literal argument"""