Repository: flutter
Issue: Scrollable content jumps when PageStorageKey set
Labels: framework,f: scrolling,has reproducible steps,P2,team-framework,triaged-framework,found in release: 3.13,found in release: 3.15

### Is there an existing issue for this?

- [X] I have searched the [existing issues](https://github.com/flutter/flutter/issues)
- [X] I have read the [guide to filing a bug](https://flutter.dev/docs/resources/bug-reports)

### Steps to reproduce

1. scroll to the bottom
2. press collapse
3. press wrap

### Expected results

Content doesn't jump.

### Actual results

With `CustomScrollView` content jumps after pressing on wrap. With `SingleChildScrollView` content jumps after pressing on collapse. 

### Code sample

<details><summary>Code with CustomScrollView</summary>

```dart
import 'package:flutter/cupertino.dart';
import 'package:flutter/material.dart';

void main() {
  runApp(const MyApp());
}

class MyApp extends StatelessWidget {
  const MyApp({
    super.key,
  });

  @override
  Widget build(BuildContext context) {
    return MaterialApp(
      title: 'Flutter Demo',
      theme: ThemeData(
        colorScheme: ColorScheme.fromSeed(seedColor: Colors.deepPurple),
        useMaterial3: true,
      ),
      home: const MyHomePage(),
    );
  }
}

class MyHomePage extends StatefulWidget {
  const MyHomePage({
    super.key,
  });

  @override
  State<MyHomePage> createState() => _MyHomePageState();
}

class _MyHomePageState extends State<MyHomePage> {
  final _scrollViewKey = const PageStorageKey('scroll_view');
  var _wrapped = false;
  var _collapsed = false;

  @override
  Widget build(BuildContext context) {
    return Scaffold(
      resizeToAvoidBottomInset: true,
      body: Container(
        width: double.infinity,
        height: double.infinity,
        color: Colors.grey[200],
        child: SafeArea(
          child: Builder(
            builder: (context) {
              Widget widget = Scrollbar(
                child: CustomScrollView(
                  key: _scrollViewKey,
                  primary: true,
                  slivers: [
                    SliverList.list(
                      children: [
                        if (!_collapsed)
                        const SizedBox(
                          height: 400,
                        ),
                        CupertinoButton(
                          child: const Text('wrap'),
                          onPressed: () {
                            setState(() {
                              _wrapped = !_wrapped;
                            });
                          },
                        ),
                        const SizedBox(
                          height: 50,
                        ),
                        CupertinoButton(
                          child: const Text('collapse'),
                          onPressed: () {
                            setState(() {
                              _collapsed = !_collapsed;
                            });
                          },
                        ),
                        if (!_collapsed)
                          const SizedBox(
                            height: 400,
                          ),
                      ],
                    ),
                  ],
                ),
              );

              if (_wrapped) {
                widget = Container(
                  width: double.infinity,
                  height: double.infinity,
                  color: Colors.grey[400],
                  child: widget,
                );
              }

              return widget;
            },
          ),
        ),
      ),
    );
  }
}
```

</details>

<details><summary>Code with SingleChildScrollView</summary>

```dart
import 'package:flutter/cupertino.dart';
import 'package:flutter/material.dart';

void main() {
  runApp(const MyApp());
}

class MyApp extends StatelessWidget {
  const MyApp({
    super.key,
  });

  @override
  Widget build(BuildContext context) {
    return MaterialApp(
      title: 'Flutter Demo',
      theme: ThemeData(
        colorScheme: ColorScheme.fromSeed(seedColor: Colors.deepPurple),
        useMaterial3: true,
      ),
      home: const MyHomePage(),
    );
  }
}

class MyHomePage extends StatefulWidget {
  const MyHomePage({
    super.key,
  });

  @override
  State<MyHomePage> createState() => _MyHomePageState();
}

class _MyHomePageState extends State<MyHomePage> {
  final _scrollViewKey = const PageStorageKey('scroll_view');
  var _wrapped = false;
  var _collapsed = false;

  @override
  Widget build(BuildContext context) {
    return Scaffold(
      resizeToAvoidBottomInset: true,
      body: Container(
        width: double.infinity,
        height: double.infinity,
        color: Colors.grey[200],
        child: SafeArea(
          child: Builder(
            builder: (context) {
              Widget widget = Scrollbar(
                child: SingleChildScrollView(
                  key: _scrollViewKey,
                  primary: true,
                  child: Column(
                    children: [
                      if (!_collapsed)
                        const SizedBox(
                          height: 400,
                        ),
                      CupertinoButton(
                        child: const Text('wrap'),
                        onPressed: () {
                          setState(() {
                            _wrapped = !_wrapped;
                          });
                        },
                      ),
                      const SizedBox(
                        height: 50,
                      ),
                      CupertinoButton(
                        child: const Text('collapse'),
                        onPressed: () {
                          setState(() {
                            _collapsed = !_collapsed;
                          });
                        },
                      ),
                      if (!_collapsed)
                        const SizedBox(
                          height: 400,
                        ),
                    ],
                  ),
                ),
              );

              if (_wrapped) {
                widget = Container(
                  width: double.infinity,
                  height: double.infinity,
                  color: Colors.grey[400],
                  child: widget,
                );
              }

              return widget;
            },
          ),
        ),
      ),
    );
  }
}
```

</details>

### Screenshots or Video

_No response_

### Logs

_No response_

### Flutter Doctor output

<details><summary>Doctor output</summary>

```console
[✓] Flutter (Channel master, 3.15.0-7.0.pre.10, on macOS 13.0 22A380 darwin-x64, locale en-GB)
    • Flutter version 3.15.0-7.0.pre.10 on channel master at /Users/admin/Programs/flutter
    • Upstream repository https://github.com/flutter/flutter.git
    • Framework revision 4ae2e91c64 (2 hours ago), 2023-09-25 12:00:01 -0400
    • Engine revision 2c4c1108aa
    • Dart version 3.2.0 (build 3.2.0-191.0.dev)
    • DevTools version 2.28.0-dev.12

[✓] Android toolchain - develop for Android devices (Android SDK version 32.1.0-rc1)
    • Android SDK at /Users/admin/Library/Android/sdk
    • Platform android-33, build-tools 32.1.0-rc1
    • ANDROID_HOME = /Users/admin/Library/Android/sdk
    • Java binary at: /Applications/Android Studio.app/Contents/jbr/Contents/Home/bin/java
    • Java version OpenJDK Runtime Environment (build 17.0.6+0-17.0.6b802.4-9586694)
    • All Android licenses accepted.

[✓] Xcode - develop for iOS and macOS (Xcode 14.3.1)
    • Xcode at /Applications/Xcode-14.3.1.app/Contents/Developer
    • Build 14E300c
    • CocoaPods version 1.12.1

[✓] Chrome - develop for the web
    • Chrome at /Applications/Google Chrome.app/Contents/MacOS/Google Chrome

[✓] Android Studio (version 2022.2)
    • Android Studio at /Applications/Android Studio.app/Contents
    • Flutter plugin can be installed from:
      🔨 https://plugins.jetbrains.com/plugin/9212-flutter
    • Dart plugin can be installed from:
      🔨 https://plugins.jetbrains.com/plugin/6351-dart
    • Java version OpenJDK Runtime Environment (build 17.0.6+0-17.0.6b802.4-9586694)

[✓] IntelliJ IDEA Ultimate Edition (version 2023.2.2)
    • IntelliJ at /Applications/IntelliJ IDEA.app
    • Flutter plugin version 75.1.4
    • Dart plugin version 232.9559.10

[✓] VS Code (version 1.82.2)
    • VS Code at /Applications/Visual Studio Code.app/Contents
    • Flutter extension version 3.72.0

[✓] Connected device (3 available)
    • iPhone 13 mini (mobile) • 4954563B-35D4-4F3C-A720-2AEC91615974 • ios            • com.apple.CoreSimulator.SimRuntime.iOS-16-4 (simulator)
    • macOS (desktop)         • macos                                • darwin-x64     • macOS 13.0 22A380 darwin-x64
    • Chrome (web)            • chrome                               • web-javascript • Google Chrome 116.0.5845.187

[✓] Network resources
    • All expected network resources are available.

• No issues found!
```

</details>