Repository: go-delve/delve
Pull request #2504 (2021-05-22 12:12:50)

terminal,service: Add filtering and grouping to goroutines command
Adds filtering and grouping to the goroutines command.

The current implementation of the goroutines command is modeled after
the threads command of gdb. It works well for programs that have up to
a couple dozen goroutines but becomes unusable quickly after that.

This commit adds the ability to filter and group goroutines by several
different properties, allowing a better debugging experience on
programs that have hundreds or thousands of goroutines.


aarzilli: This is what grouped goroutines look like:

![Example](https://i.imgur.com/Ynnbguv.png)
polinasok: #2522 was merged. It has a todo to share code with this PR.
https://github.com/go-delve/delve/pull/2522/files/1be89ec08d0cee550faefccd5d694b9d19ee7086#r650137044

aarzilli: > #2522 was merged. It has a todo to share code with this PR.
> https://github.com/go-delve/delve/pull/2522/files/1be89ec08d0cee550faefccd5d694b9d19ee7086#r650137044

Done.
derekparker: @polinasok mind giving this another look when you get a chance.

Patch:
diff --git a/Documentation/cli/README.md b/Documentation/cli/README.md
index cf1984e3..96ad677b 100644
--- a/Documentation/cli/README.md
+++ b/Documentation/cli/README.md
@@ -353,18 +353,62 @@ Aliases: gr
 ## goroutines
 List program goroutines.
 
-	goroutines [-u (default: user location)|-r (runtime location)|-g (go statement location)|-s (start location)] [-t (stack trace)] [-l (labels)]
+	goroutines [-u|-r|-g|-s] [-t [depth]] [-l] [-with loc expr] [-without loc expr] [-group argument]
 
 Print out info for every goroutine. The flag controls what information is shown along with each goroutine:
 
-	-u	displays location of topmost stackframe in user code
+	-u	displays location of topmost stackframe in user code (default)
 	-r	displays location of topmost stackframe (including frames inside private runtime functions)
 	-g	displays location of go instruction that created the goroutine
 	-s	displays location of the start function
-	-t	displays goroutine's stacktrace
+	-t	displays goroutine's stacktrace (an optional depth value can be specified, default: 10)
 	-l	displays goroutine's labels
 
-If no flag is specified the default is -u.
+If no flag is specified the default is -u, i.e. the first frame within the first 30 frames that is not executing a runtime private function.
+
+FILTERING
+
+If -with or -without are specified only goroutines that match the given condition are returned.
+
+To only display goroutines where the specified location contains (or does not contain, for -without and -wo) expr as a substring, use:
+
+	goroutines -with (userloc|curloc|goloc|startloc) expr
+	goroutines -w (userloc|curloc|goloc|startloc) expr
+	goroutines -without (userloc|curloc|goloc|startloc) expr
+	goroutines -wo (userloc|curloc|goloc|startloc) expr
+	
+To only display goroutines that have (or do not have) the specified label key and value, use:
+	
+
+	goroutines -with label key=value
+	goroutines -without label key=value
+	
+To only display goroutines that have (or do not have) the specified label key, use:
+
+	goroutines -with label key
+	goroutines -without label key
+	
+To only display goroutines that are running (or are not running) on a OS thread, use:
+
+
+	goroutines -with running
+	goroutines -without running
+	
+To only display user (or runtime) goroutines, use:
+
+	goroutines -with user
+	goroutines -without user
+
+GROUPING
+
+	goroutines -group (userloc|curloc|goloc|startloc|running|user)
+
+Groups goroutines by the given location, running status or user classification, up to 5 goroutines per group will be displayed as well as the total number of goroutines in the group.
+
+	goroutines -group label key
+
+Groups goroutines by the value of the label with the specified key.
+
 
 Aliases: grs
 
diff --git a/Documentation/cli/starlark.md b/Documentation/cli/starlark.md
index ee60d5dd..b797bed8 100644
--- a/Documentation/cli/starlark.md
+++ b/Documentation/cli/starlark.md
@@ -45,7 +45,7 @@ checkpoints() | Equivalent to API call [ListCheckpoints](https://godoc.org/githu
 dynamic_libraries() | Equivalent to API call [ListDynamicLibraries](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListDynamicLibraries)
 function_args(Scope, Cfg) | Equivalent to API call [ListFunctionArgs](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListFunctionArgs)
 functions(Filter) | Equivalent to API call [ListFunctions](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListFunctions)
-goroutines(Start, Count) | Equivalent to API call [ListGoroutines](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListGoroutines)
+goroutines(Start, Count, Filters, GoroutineGroupingOptions) | Equivalent to API call [ListGoroutines](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListGoroutines)
 local_vars(Scope, Cfg) | Equivalent to API call [ListLocalVars](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListLocalVars)
 package_vars(Filter, Cfg) | Equivalent to API call [ListPackageVars](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListPackageVars)
 packages_build_info(IncludeFiles) | Equivalent to API call [ListPackagesBuildInfo](https://godoc.org/github.com/go-delve/delve/service/rpc2#RPCServer.ListPackagesBuildInfo)
diff --git a/Documentation/usage/dlv_dap.md b/Documentation/usage/dlv_dap.md
index e194c6b9..3a87a251 100644
--- a/Documentation/usage/dlv_dap.md
+++ b/Documentation/usage/dlv_dap.md
@@ -13,10 +13,9 @@ to be launched or process to be attached to. The following modes are supported:
 - launch + debug (builds and launches, like 'dlv debug')
 - launch + test (builds and tests, like 'dlv test')
 - attach + local (attaches to a running process, like 'dlv attach')
-The server does not yet support asynchronous request-response communication, so features
-like pausing or setting breakpoints while the program is running are not yet available.
-The server does not accept multiple client connections (--accept-multiclient),
-a feature that is often relied on to enable --continue with remote debugging.
+The server does not yet accept multiple client connections (--accept-multiclient).
+While --continue is not supported, stopOnEntry launch/attach attribute can be used to control if
+execution is resumed at the start of the debug session.
 
 ```
 dlv dap
diff --git a/_fixtures/fatalerror.go b/_fixtures/fatalerror.go
new file mode 100644
index 00000000..ad1e9bfa
--- /dev/null
+++ b/_fixtures/fatalerror.go
@@ -0,0 +1,6 @@
+package main
+
+func main() {
+	var f func()
+	go f()
+}
diff --git a/_fixtures/goroutinegroup.go b/_fixtures/goroutinegroup.go
new file mode 100644
index 00000000..7b5bad86
--- /dev/null
+++ b/_fixtures/goroutinegroup.go
@@ -0,0 +1,75 @@
+package main
+
+import (
+	"context"
+	"runtime"
+	"runtime/pprof"
+	"sync"
+	"time"
+)
+
+func sleepyfunc(wg *sync.WaitGroup, lbl string) {
+	defer wg.Done()
+	pprof.SetGoroutineLabels(pprof.WithLabels(context.Background(), pprof.Labels("name", lbl)))
+	time.Sleep(10 * 60 * time.Second)
+}
+
+func gopoint1(wg *sync.WaitGroup, lbl string, f func(*sync.WaitGroup, string)) {
+	go f(wg, lbl)
+}
+
+func gopoint2(wg *sync.WaitGroup, lbl string, f func(*sync.WaitGroup, string)) {
+	go f(wg, lbl)
+}
+
+func gopoint3(wg *sync.WaitGroup, lbl string, f func(*sync.WaitGroup, string)) {
+	go f(wg, lbl)
+}
+
+func gopoint4(wg *sync.WaitGroup, lbl string, f func(*sync.WaitGroup, string)) {
+	go f(wg, lbl)
+}
+
+func gopoint5(wg *sync.WaitGroup, lbl string, f func(*sync.WaitGroup, string)) {
+	go f(wg, lbl)
+}
+
+func startpoint1(wg *sync.WaitGroup, lbl string) {
+	sleepyfunc(wg, lbl)
+}
+
+func startpoint2(wg *sync.WaitGroup, lbl string) {
+	sleepyfunc(wg, lbl)
+}
+
+func startpoint3(wg *sync.WaitGroup, lbl string) {
+	sleepyfunc(wg, lbl)
+}
+
+func startpoint4(wg *sync.WaitGroup, lbl string) {
+	sleepyfunc(wg, lbl)
+}
+
+func startpoint5(wg *sync.WaitGroup, lbl string) {
+	sleepyfunc(wg, lbl)
+}
+
+func main() {
+	var wg sync.WaitGroup
+
+	for _, lbl := range []string{"one", "two", "three", "four", "five"} {
+		for _, f := range []func(*sync.WaitGroup, string){startpoint1, startpoint2, startpoint3, startpoint4, startpoint5} {
+			for i := 0; i < 1000; i++ {
+				wg.Add(5)
+				gopoint1(&wg, lbl, f)
+				gopoint2(&wg, lbl, f)
+				gopoint3(&wg, lbl, f)
+				gopoint4(&wg, lbl, f)
+				gopoint5(&wg, lbl, f)
+			}
+		}
+	}
+	time.Sleep(2 * time.Second)
+	runtime.Breakpoint()
+	wg.Wait()
+}
diff --git a/cmd/dlv/cmds/commands.go b/cmd/dlv/cmds/commands.go
index 1f0c8a8e..e063c728 100644
--- a/cmd/dlv/cmds/commands.go
+++ b/cmd/dlv/cmds/commands.go
@@ -183,10 +183,9 @@ to be launched or process to be attached to. The following modes are supported:
 - launch + debug (builds and launches, like 'dlv debug')
 - launch + test (builds and tests, like 'dlv test')
 - attach + local (attaches to a running process, like 'dlv attach')
-The server does not yet support asynchronous request-response communication, so features
-like pausing or setting breakpoints while the program is running are not yet available.
-The server does not accept multiple client connections (--accept-multiclient),
-a feature that is often relied on to enable --continue with remote debugging.`,
+The server does not yet accept multiple client connections (--accept-multiclient).
+While --continue is not supported, stopOnEntry launch/attach attribute can be used to control if
+execution is resumed at the start of the debug session.`,
 		Run: dapCmd,
 	}
 	rootCommand.AddCommand(dapCommand)
diff --git a/pkg/proc/gdbserial/gdbserver.go b/pkg/proc/gdbserial/gdbserver.go
index 12123a63..d2312c15 100644
--- a/pkg/proc/gdbserial/gdbserver.go
+++ b/pkg/proc/gdbserial/gdbserver.go
@@ -1853,8 +1853,11 @@ func (t *gdbThread) SetReg(regNum uint64, reg *op.DwarfRegister) error {
 	gdbreg, ok := t.regs.regs[regName]
 	if !ok && strings.HasPrefix(regName, "xmm") {
 		// XMMn and YMMn are the same amd64 register (in different sizes), if we
-		// don't find XMMn try YMMn instead.
+		// don't find XMMn try YMMn or ZMMn instead.
 		gdbreg, ok = t.regs.regs["y"+regName[1:]]
+		if !ok {
+			gdbreg, ok = t.regs.regs["z"+regName[1:]]
+		}
 	}
 	if !ok {
 		return fmt.Errorf("could not set register %s: not found", regName)
@@ -1913,8 +1916,16 @@ func (regs *gdbRegisters) Slice(floatingPoint bool) ([]proc.Register, error) {
 
 			value := regs.regs[reginfo.Name].value
 			xmmName := "x" + reginfo.Name[1:]
-			r = proc.AppendBytesRegister(r, strings.ToUpper(xmmName), value[:16])
-			r = proc.AppendBytesRegister(r, strings.ToUpper(reginfo.Name), value[16:])
+			r = proc.AppendBytesRegister(r, strings.ToUpper(xmmName), value)
+
+		case reginfo.Bitsize == 512:
+			if !strings.HasPrefix(strings.ToLower(reginfo.Name), "zmm") || !floatingPoint {
+				continue
+			}
+
+			value := regs.regs[reginfo.Name].value
+			xmmName := "x" + reginfo.Name[1:]
+			r = proc.AppendBytesRegister(r, strings.ToUpper(xmmName), value)
 		}
 	}
 	return r, nil
diff --git a/pkg/proc/variables.go b/pkg/proc/variables.go
index 2190f7e2..442db634 100644
--- a/pkg/proc/variables.go
+++ b/pkg/proc/variables.go
@@ -521,6 +521,20 @@ func (g *G) StartLoc() Location {
 	return Location{PC: g.StartPC, File: f, Line: l, Fn: fn}
 }
 
+// System returns true if g is a system goroutine. See isSystemGoroutine in
+// $GOROOT/src/runtime/traceback.go.
+func (g *G) System() bool {
+	loc := g.StartLoc()
+	if loc.Fn == nil {
+		return false
+	}
+	switch loc.Fn.Name {
+	case "runtime.main", "runtime.handleAsyncEvent", "runtime.runfinq":
+		return false
+	}
+	return strings.HasPrefix(loc.Fn.Name, "runtime.")
+}
+
 func (g *G) Labels() map[string]string {
 	if g.labels != nil {
 		return *g.labels
diff --git a/pkg/terminal/command.go b/pkg/terminal/command.go
index 8c4e36d3..24b08f53 100644
--- a/pkg/terminal/command.go
+++ b/pkg/terminal/command.go
@@ -220,18 +220,62 @@ If called with the linespec argument it will delete all the breakpoints matching
 toggle <breakpoint name or id>`},
 		{aliases: []string{"goroutines", "grs"}, group: goroutineCmds, cmdFn: goroutines, helpMsg: `List program goroutines.
 
-	goroutines [-u (default: user location)|-r (runtime location)|-g (go statement location)|-s (start location)] [-t (stack trace)] [-l (labels)]
+	goroutines [-u|-r|-g|-s] [-t [depth]] [-l] [-with loc expr] [-without loc expr] [-group argument]
 
 Print out info for every goroutine. The flag controls what information is shown along with each goroutine:
 
-	-u	displays location of topmost stackframe in user code
+	-u	displays location of topmost stackframe in user code (default)
 	-r	displays location of topmost stackframe (including frames inside private runtime functions)
 	-g	displays location of go instruction that created the goroutine
 	-s	displays location of the start function
-	-t	displays goroutine's stacktrace
+	-t	displays goroutine's stacktrace (an optional depth value can be specified, default: 10)
 	-l	displays goroutine's labels
 
-If no flag is specified the default is -u.`},
+If no flag is specified the default is -u, i.e. the first frame within the first 30 frames that is not executing a runtime private function.
+
+FILTERING
+
+If -with or -without are specified only goroutines that match the given condition are returned.
+
+To only display goroutines where the specified location contains (or does not contain, for -without and -wo) expr as a substring, use:
+
+	goroutines -with (userloc|curloc|goloc|startloc) expr
+	goroutines -w (userloc|curloc|goloc|startloc) expr
+	goroutines -without (userloc|curloc|goloc|startloc) expr
+	goroutines -wo (userloc|curloc|goloc|startloc) expr
+	
+To only display goroutines that have (or do not have) the specified label key and value, use:
+	
+
+	goroutines -with label key=value
+	goroutines -without label key=value
+	
+To only display goroutines that have (or do not have) the specified label key, use:
+
+	goroutines -with label key
+	goroutines -without label key
+	
+To only display goroutines that are running (or are not running) on a OS thread, use:
+
+
+	goroutines -with running
+	goroutines -without running
+	
+To only display user (or runtime) goroutines, use:
+
+	goroutines -with user
+	goroutines -without user
+
+GROUPING
+
+	goroutines -group (userloc|curloc|goloc|startloc|running|user)
+
+Groups goroutines by the given location, running status or user classification, up to 5 goroutines per group will be displayed as well as the total number of goroutines in the group.
+
+	goroutines -group label key
+
+Groups goroutines by the value of the label with the specified key.
+`},
 		{aliases: []string{"goroutine", "gr"}, group: goroutineCmds, allowedPrefixes: onPrefix, cmdFn: c.goroutine, helpMsg: `Shows or changes current goroutine
 
 	goroutine
@@ -717,67 +761,116 @@ const (
 	printGoroutinesLabels
 )
 
-func printGoroutines(t *Term, gs []*api.Goroutine, fgl formatGoroutineLoc, flags printGoroutinesFlags, state *api.DebuggerState) error {
+func printGoroutines(t *Term, indent string, gs []*api.Goroutine, fgl formatGoroutineLoc, flags printGoroutinesFlags, depth int, state *api.DebuggerState) error {
 	for _, g := range gs {
-		prefix := "  "
+		prefix := indent + "  "
 		if state.SelectedGoroutine != nil && g.ID == state.SelectedGoroutine.ID {
-			prefix = "* "
+			prefix = indent + "* "
 		}
 		fmt.Printf("%sGoroutine %s\n", prefix, t.formatGoroutine(g, fgl))
 		if flags&printGoroutinesLabels != 0 {
-			writeGoroutineLabels(os.Stdout, g, "\t")
+			writeGoroutineLabels(os.Stdout, g, indent+"\t")
 		}
 		if flags&printGoroutinesStack != 0 {
-			stack, err := t.client.Stacktrace(g.ID, 10, 0, nil)
+			stack, err := t.client.Stacktrace(g.ID, depth, 0, nil)
 			if err != nil {
 				return err
 			}
-			printStack(t, os.Stdout, stack, "\t", false)
+			printStack(t, os.Stdout, stack, indent+"\t", false)
 		}
 	}
 	return nil
 }
 
+const (
+	maxGroupMembers    = 5
+	maxGoroutineGroups = 50
+)
+
 func goroutines(t *Term, ctx callContext, argstr string) error {
 	args := strings.Split(argstr, " ")
+	var filters []api.ListGoroutinesFilter
+	var group api.GoroutineGroupingOptions
 	var fgl = fglUserCurrent
 	var flags printGoroutinesFlags
+	var depth = 10
+	var batchSize = goroutineBatchSize
+
+	group.MaxGroupMembers = maxGroupMembers
+	group.MaxGroups = maxGoroutineGroups
+
+	for i := 0; i < len(args); i++ {
+		arg := args[i]
+		switch arg {
+		case "-u":
+			fgl = fglUserCurrent
+		case "-r":
+			fgl = fglRuntimeCurrent
+		case "-g":
+			fgl = fglGo
+		case "-s":
+			fgl = fglStart
+		case "-l":
+			flags |= printGoroutinesLabels
+		case "-t":
+			flags |= printGoroutinesStack
+			// optional depth argument
+			if i+1 < len(args) && len(args[i+1]) > 0 {
+				n, err := strconv.Atoi(args[i+1])
+				if err == nil {
+					depth = n
+					i++
+				}
+			}
 
-	switch len(args) {
-	case 0:
-		// nothing to do
-	case 1, 2:
-		for _, arg := range args {
-			switch arg {
-			case "-u":
-				fgl = fglUserCurrent
-			case "-r":
-				fgl = fglRuntimeCurrent
-			case "-g":
-				fgl = fglGo
-			case "-s":
-				fgl = fglStart
-			case "-t":
-				flags |= printGoroutinesStack
-			case "-l":
-				flags |= printGoroutinesLabels
-			case "":
-				// nothing to do
-			default:
-				return fmt.Errorf("wrong argument: '%s'", arg)
+		case "-w", "-with":
+			filter, err := readGoroutinesFilter(args, &i)
+			if err != nil {
+				return err
 			}
+			filters = append(filters, *filter)
+
+		case "-wo", "-without":
+			filter, err := readGoroutinesFilter(args, &i)
+			if err != nil {
+				return err
+			}
+			filter.Negated = true
+			filters = append(filters, *filter)
+
+		case "-group":
+			var err error
+			group.GroupBy, err = readGoroutinesFilterKind(args, i+1)
+			if err != nil {
+				return err
+			}
+			i++
+			if group.GroupBy == api.GoroutineLabel {
+				if i+1 >= len(args) {
+					return errors.New("-group label must be followed by an argument")
+				}
+				group.GroupByKey = args[i+1]
+				i++
+			}
+			batchSize = 0 // grouping only works well if run on all goroutines
+
+		case "":
+			// nothing to do
+		default:
+			return fmt.Errorf("wrong argument: '%s'", arg)
 		}
-	default:
-		return fmt.Errorf("too many arguments")
 	}
+
 	state, err := t.client.GetState()
 	if err != nil {
 		return err
 	}
 	var (
-		start = 0
-		gslen = 0
-		gs    []*api.Goroutine
+		start         = 0
+		gslen         = 0
+		gs            []*api.Goroutine
+		groups        []api.GoroutineGroup
+		tooManyGroups bool
 	)
 	t.longCommandStart()
 	for start >= 0 {
@@ -785,21 +878,86 @@ func goroutines(t *Term, ctx callContext, argstr string) error {
 			fmt.Printf("interrupted\n")
 			return nil
 		}
-		gs, start, err = t.client.ListGoroutines(start, goroutineBatchSize)
+		gs, groups, start, tooManyGroups, err = t.client.ListGoroutinesWithFilter(start, batchSize, filters, &group)
 		if err != nil {
 			return err
 		}
-		sort.Sort(byGoroutineID(gs))
-		err = printGoroutines(t, gs, fgl, flags, state)
-		if err != nil {
-			return err
+		if len(groups) > 0 {
+			for i := range groups {
+				fmt.Printf("%s\n", groups[i].Name)
+				err = printGoroutines(t, "\t", gs[groups[i].Offset:][:groups[i].Count], fgl, flags, depth, state)
+				if err != nil {
+					return err
+				}
+				fmt.Printf("\tTotal: %d\n", groups[i].Total)
+				if i != len(groups)-1 {
+					fmt.Printf("\n")
+				}
+			}
+			if tooManyGroups {
+				fmt.Printf("Too many groups\n")
+			}
+		} else {
+			sort.Sort(byGoroutineID(gs))
+			err = printGoroutines(t, "", gs, fgl, flags, depth, state)
+			if err != nil {
+				return err
+			}
+			gslen += len(gs)
 		}
-		gslen += len(gs)
 	}
-	fmt.Printf("[%d goroutines]\n", gslen)
+	if gslen > 0 {
+		fmt.Printf("[%d goroutines]\n", gslen)
+	}
 	return nil
 }
 
+func readGoroutinesFilterKind(args []string, i int) (api.GoroutineField, error) {
+	if i >= len(args) {
+		return api.GoroutineFieldNone, fmt.Errorf("%s must be followed by an argument", args[i-1])
+	}
+
+	switch args[i] {
+	case "curloc":
+		return api.GoroutineCurrentLoc, nil
+	case "userloc":
+		return api.GoroutineUserLoc, nil
+	case "goloc":
+		return api.GoroutineGoLoc, nil
+	case "startloc":
+		return api.GoroutineStartLoc, nil
+	case "label":
+		return api.GoroutineLabel, nil
+	case "running":
+		return api.GoroutineRunning, nil
+	case "user":
+		return api.GoroutineUser, nil
+	default:
+		return api.GoroutineFieldNone, fmt.Errorf("unrecognized argument to %s %s", args[i-1], args[i])
+	}
+}
+
+func readGoroutinesFilter(args []string, pi *int) (*api.ListGoroutinesFilter, error) {
+	r := new(api.ListGoroutinesFilter)
+	var err error
+	r.Kind, err = readGoroutinesFilterKind(args, *pi+1)
+	if err != nil {
+		return nil, err
+	}
+	*pi++
+	switch r.Kind {
+	case api.GoroutineRunning, api.GoroutineUser:
+		return r, nil
+	}
+	if *pi+1 >= len(args) {
+		return nil, fmt.Errorf("%s %s needs to be followed by an expression", args[*pi-1], args[*pi])
+	}
+	r.Arg = args[*pi+1]
+	*pi++
+
+	return r, nil
+}
+
 func selectedGID(state *api.DebuggerState) int {
 	if state.SelectedGoroutine == nil {
 		return 0
diff --git a/pkg/terminal/starbind/starlark_mapping.go b/pkg/terminal/starbind/starlark_mapping.go
index 8f0bb02b..24da56a0 100644
--- a/pkg/terminal/starbind/starlark_mapping.go
+++ b/pkg/terminal/starbind/starlark_mapping.go
@@ -912,6 +912,18 @@ func (env *Env) starlarkPredeclare() starlark.StringDict {
 				return starlark.None, decorateError(thread, err)
 			}
 		}
+		if len(args) > 2 && args[2] != starlark.None {
+			err := unmarshalStarlarkValue(args[2], &rpcArgs.Filters, "Filters")
+			if err != nil {
+				return starlark.None, decorateError(thread, err)
+			}
+		}
+		if len(args) > 3 && args[3] != starlark.None {
+			err := unmarshalStarlarkValue(args[3], &rpcArgs.GoroutineGroupingOptions, "GoroutineGroupingOptions")
+			if err != nil {
+				return starlark.None, decorateError(thread, err)
+			}
+		}
 		for _, kv := range kwargs {
 			var err error
 			switch kv[0].(starlark.String) {
@@ -919,6 +931,10 @@ func (env *Env) starlarkPredeclare() starlark.StringDict {
 				err = unmarshalStarlarkValue(kv[1], &rpcArgs.Start, "Start")
 			case "Count":
 				err = unmarshalStarlarkValue(kv[1], &rpcArgs.Count, "Count")
+			case "Filters":
+				err = unmarshalStarlarkValue(kv[1], &rpcArgs.Filters, "Filters")
+			case "GoroutineGroupingOptions":
+				err = unmarshalStarlarkValue(kv[1], &rpcArgs.GoroutineGroupingOptions, "GoroutineGroupingOptions")
 			default:
 				err = fmt.Errorf("unknown argument %q", kv[0])
 			}
diff --git a/service/api/types.go b/service/api/types.go
index 0767f457..b24cf83e 100644
--- a/service/api/types.go
+++ b/service/api/types.go
@@ -586,3 +586,41 @@ type DumpState struct {
 
 	Err string
 }
+
+// ListGoroutinesFilter describes a filtering condition for the
+// ListGoroutines API call.
+type ListGoroutinesFilter struct {
+	Kind    GoroutineField
+	Negated bool
+	Arg     string
+}
+
+// GoroutineField allows referring to a field of a goroutine object.
+type GoroutineField uint8
+
+const (
+	GoroutineFieldNone  GoroutineField = iota
+	GoroutineCurrentLoc                // the goroutine's CurrentLoc
+	GoroutineUserLoc                   // the goroutine's UserLoc
+	GoroutineGoLoc                     // the goroutine's GoStatementLoc
+	GoroutineStartLoc                  // the goroutine's StartLoc
+	GoroutineLabel                     // the goroutine's label
+	GoroutineRunning                   // the goroutine is running
+	GoroutineUser                      // the goroutine is a user goroutine
+)
+
+// GoroutineGroup represents a group of goroutines in the return value of
+// the ListGoroutines API call.
+type GoroutineGroup struct {
+	Name   string // name of this group
+	Offset int    // start offset in the list of goroutines of this group
+	Count  int    // number of goroutines that belong to this group in the list of goroutines
+	Total  int    // total number of goroutines that belong to this group
+}
+
+type GoroutineGroupingOptions struct {
+	GroupBy         GoroutineField
+	GroupByKey      string
+	MaxGroupMembers int
+	MaxGroups       int
+}
diff --git a/service/client.go b/service/client.go
index a83831ef..805b9fb4 100644
--- a/service/client.go
+++ b/service/client.go
@@ -114,6 +114,8 @@ type Client interface {
 
 	// ListGoroutines lists all goroutines.
 	ListGoroutines(start, count int) ([]*api.Goroutine, int, error)
+	// ListGoroutinesWithFilter lists goroutines matching the filters
+	ListGoroutinesWithFilter(start, count int, filters []api.ListGoroutinesFilter, group *api.GoroutineGroupingOptions) ([]*api.Goroutine, []api.GoroutineGroup, int, bool, error)
 
 	// Returns stacktrace
 	Stacktrace(goroutineID int, depth int, opts api.StacktraceOptions, cfg *api.LoadConfig) ([]api.Stackframe, error)
diff --git a/service/dap/server.go b/service/dap/server.go
index 2514b3aa..49be9feb 100644
--- a/service/dap/server.go
+++ b/service/dap/server.go
@@ -29,6 +29,7 @@ import (
 	"sync"
 
 	"github.com/go-delve/delve/pkg/gobuild"
+	"github.com/go-delve/delve/pkg/goversion"
 	"github.com/go-delve/delve/pkg/locspec"
 	"github.com/go-delve/delve/pkg/logflags"
 	"github.com/go-delve/delve/pkg/proc"
@@ -1549,12 +1550,9 @@ func (s *Server) onStackTraceRequest(request *dap.StackTraceRequest) {
 	}
 
 	// Determine if the goroutine is a system goroutine.
-	// TODO(suzmue): Use the System() method defined in: https://github.com/go-delve/delve/pull/2504
-	g, err := s.debugger.FindGoroutine(goroutineID)
-	var isSystemGoroutine bool
-	if err == nil {
-		userLoc := g.UserCurrent()
-		isSystemGoroutine = fnPackageName(&userLoc) == "runtime"
+	isSystemGoroutine := true
+	if g, _ := s.debugger.FindGoroutine(goroutineID); g != nil {
+		isSystemGoroutine = g.System()
 	}
 
 	stackFrames := make([]dap.StackFrame, len(frames))
@@ -2509,14 +2507,36 @@ func (s *Server) onExceptionInfoRequest(request *dap.ExceptionInfoRequest) {
 	if bpState != nil && bpState.Breakpoint != nil && (bpState.Breakpoint.Name == proc.FatalThrow || bpState.Breakpoint.Name == proc.UnrecoveredPanic) {
 		switch bpState.Breakpoint.Name {
 		case proc.FatalThrow:
-			// TODO(suzmue): add the fatal throw reason to body.Description.
 			body.ExceptionId = "fatal error"
+			// Attempt to get the value of the throw reason.
+			// This is not currently working for Go 1.16 or 1.17: https://github.com/golang/go/issues/46425.
+			handleError := func(err error) {
+				if err != nil {
+					body.Description = fmt.Sprintf("Error getting throw reason: %s", err.Error())
+				}
+				if goversion.ProducerAfterOrEqual(s.debugger.TargetGoVersion(), 1, 16) {
+					body.Description = "Throw reason unavailable, see https://github.com/golang/go/issues/46425"
+				}
+			}
+
+			exprVar, err := s.debugger.EvalVariableInScope(goroutineID, 1, 0, "s", DefaultLoadConfig)
+			if err == nil {
+				if exprVar.Value != nil {
+					body.Description = exprVar.Value.String()
+				} else {
+					handleError(exprVar.Unreadable)
+				}
+			} else {
+				handleError(err)
+			}
 		case proc.UnrecoveredPanic:
 			body.ExceptionId = "panic"
 			// Attempt to get the value of the panic message.
 			exprVar, err := s.debugger.EvalVariableInScope(goroutineID, 0, 0, "(*msgs).arg.(data)", DefaultLoadConfig)
 			if err == nil {
 				body.Description = exprVar.Value.String()
+			} else {
+				body.Description = fmt.Sprintf("Error getting panic message: %s", err.Error())
 			}
 		}
 	} else {
@@ -2545,7 +2565,7 @@ func (s *Server) onExceptionInfoRequest(request *dap.ExceptionInfoRequest) {
 	}
 
 	frames, err := s.debugger.Stacktrace(goroutineID, s.args.stackTraceDepth, 0)
-	if err == nil && len(frames) > 0 {
+	if err == nil {
 		apiFrames, err := s.debugger.ConvertStacktrace(frames, nil)
 		if err == nil {
 			var buf bytes.Buffer
@@ -2562,6 +2582,8 @@ func (s *Server) onExceptionInfoRequest(request *dap.ExceptionInfoRequest) {
 			})
 			body.Details.StackTrace = buf.String()
 		}
+	} else {
+		body.Details.StackTrace = fmt.Sprintf("Error getting stack trace: %s", err.Error())
 	}
 	response := &dap.ExceptionInfoResponse{
 		Response: *newResponse(request.Request),
diff --git a/service/debugger/debugger.go b/service/debugger/debugger.go
index 6b3d29c2..6d6a355f 100644
--- a/service/debugger/debugger.go
+++ b/service/debugger/debugger.go
@@ -237,6 +237,12 @@ func (d *Debugger) checkGoVersion() error {
 	return goversion.Compatible(producer)
 }
 
+func (d *Debugger) TargetGoVersion() string {
+	d.targetMutex.Lock()
+	defer d.targetMutex.Unlock()
+	return d.target.BinInfo().Producer()
+}
+
 // Launch will start a process with the given args and working directory.
 func (d *Debugger) Launch(processArgs []string, wd string) (*proc.Target, error) {
 	if err := verifyBinaryFormat(processArgs[0]); err != nil {
@@ -620,7 +626,33 @@ func (d *Debugger) state(retLoadCfg *proc.LoadConfig) (*api.DebuggerState, error
 	return state, nil
 }
 
-// CreateBreakpoint creates a breakpoint.
+// CreateBreakpoint creates a breakpoint using information from the provided `requestedBp`.
+// This function accepts several different ways of specifying where and how to create the
+// breakpoint that has been requested. Any error encountered during the attempt to set the
+// breakpoint will be returned to the caller.
+//
+// The ways of specifying a breakpoint are listed below in the order they are considered by
+// this function:
+//
+// - If requestedBp.TraceReturn is true then it is expected that
+// requestedBp.Addrs will contain the list of return addresses
+// supplied by the caller.
+//
+// - If requestedBp.File is not an empty string the breakpoint
+// will be created on the specified file:line location
+//
+// - If requestedBp.FunctionName is not an empty string
+// the breakpoint will be created on the specified function:line
+// location.
+//
+// - If requestedBp.Addrs is filled it will create a logical breakpoint
+// corresponding to all specified addresses.
+//
+// - Otherwise the value specified by arg.Breakpoint.Addr will be used.
+//
+// Note that this method will use the first successful method in order to
+// create a breakpoint, so mixing different fields will not result is multiple
+// breakpoints being set.
 func (d *Debugger) CreateBreakpoint(requestedBp *api.Breakpoint) (*api.Breakpoint, error) {
 	d.targetMutex.Lock()
 	defer d.targetMutex.Unlock()
@@ -1493,6 +1525,131 @@ func (d *Debugger) Goroutines(start, count int) ([]*proc.G, int, error) {
 	return proc.GoroutinesInfo(d.target, start, count)
 }
 
+// FilterGoroutines returns the goroutines in gs that satisfy the specified filters.
+func (d *Debugger) FilterGoroutines(gs []*proc.G, filters []api.ListGoroutinesFilter) []*proc.G {
+	if len(filters) == 0 {
+		return gs
+	}
+	d.targetMutex.Lock()
+	defer d.targetMutex.Unlock()
+	r := []*proc.G{}
+	for _, g := range gs {
+		ok := true
+		for i := range filters {
+			if !matchGoroutineFilter(g, &filters[i]) {
+				ok = false
+				break
+			}
+		}
+		if ok {
+			r = append(r, g)
+		}
+	}
+	return r
+}
+
+func matchGoroutineFilter(g *proc.G, filter *api.ListGoroutinesFilter) bool {
+	var val bool
+	switch filter.Kind {
+	default:
+		fallthrough
+	case api.GoroutineFieldNone:
+		val = true
+	case api.GoroutineCurrentLoc:
+		val = matchGoroutineLocFilter(g.CurrentLoc, filter.Arg)
+	case api.GoroutineUserLoc:
+		val = matchGoroutineLocFilter(g.UserCurrent(), filter.Arg)
+	case api.GoroutineGoLoc:
+		val = matchGoroutineLocFilter(g.Go(), filter.Arg)
+	case api.GoroutineStartLoc:
+		val = matchGoroutineLocFilter(g.StartLoc(), filter.Arg)
+	case api.GoroutineLabel:
+		idx := strings.Index(filter.Arg, "=")
+		if idx >= 0 {
+			val = g.Labels()[filter.Arg[:idx]] == filter.Arg[idx+1:]
+		} else {
+			_, val = g.Labels()[filter.Arg]
+		}
+	case api.GoroutineRunning:
+		val = g.Thread != nil
+	case api.GoroutineUser:
+		val = !g.System()
+	}
+	if filter.Negated {
+		val = !val
+	}
+	return val
+}
+
+func matchGoroutineLocFilter(loc proc.Location, arg string) bool {
+	return strings.Contains(formatLoc(loc), arg)
+}
+
+func formatLoc(loc proc.Location) string {
+	fnname := "?"
+	if loc.Fn != nil {
+		fnname = loc.Fn.Name
+	}
+	return fmt.Sprintf("%s:%d in %s", loc.File, loc.Line, fnname)
+}
+
+// GroupGoroutines divides goroutines in gs into groups as specified by groupBy and groupByArg.
+// A maximum of maxGoroutinesPerGroup are saved in each group, but the total
+// number of goroutines in each group is recorded.
+func (d *Debugger) GroupGoroutines(gs []*proc.G, group *api.GoroutineGroupingOptions) ([]*proc.G, []api.GoroutineGroup, bool) {
+	if group.GroupBy == api.GoroutineFieldNone {
+		return gs, nil, false
+	}
+	d.targetMutex.Lock()
+	defer d.targetMutex.Unlock()
+
+	groupMembers := map[string][]*proc.G{}
+	totals := map[string]int{}
+
+	for _, g := range gs {
+		var key string
+		switch group.GroupBy {
+		case api.GoroutineCurrentLoc:
+			key = formatLoc(g.CurrentLoc)
+		case api.GoroutineUserLoc:
+			key = formatLoc(g.UserCurrent())
+		case api.GoroutineGoLoc:
+			key = formatLoc(g.Go())
+		case api.GoroutineStartLoc:
+			key = formatLoc(g.StartLoc())
+		case api.GoroutineLabel:
+			key = fmt.Sprintf("%s=%s", group.GroupByKey, g.Labels()[group.GroupByKey])
+		case api.GoroutineRunning:
+			key = fmt.Sprintf("running=%v", g.Thread != nil)
+		case api.GoroutineUser:
+			key = fmt.Sprintf("user=%v", !g.System())
+		}
+		if len(groupMembers[key]) < group.MaxGroupMembers {
+			groupMembers[key] = append(groupMembers[key], g)
+		}
+		totals[key]++
+	}
+
+	keys := make([]string, 0, len(groupMembers))
+	for key := range groupMembers {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	tooManyGroups := false
+	gsout := []*proc.G{}
+	groups := []api.GoroutineGroup{}
+	for _, key := range keys {
+		if group.MaxGroups > 0 && len(groups) >= group.MaxGroups {
+			tooManyGroups = true
+			break
+		}
+		groups = append(groups, api.GoroutineGroup{Name: key, Offset: len(gsout), Count: len(groupMembers[key]), Total: totals[key]})
+		gsout = append(gsout, groupMembers[key]...)
+	}
+	return gsout, groups, tooManyGroups
+}
+
 // Stacktrace returns a list of Stackframes for the given goroutine. The
 // length of the returned list will be min(stack_len, depth).
 // If 'full' is true, then local vars, function args, etc will be returned as well.
diff --git a/service/rpc2/client.go b/service/rpc2/client.go
index 314274fd..21bf88f6 100644
--- a/service/rpc2/client.go
+++ b/service/rpc2/client.go
@@ -226,6 +226,10 @@ func (c *RPCClient) GetBreakpointByName(name string) (*api.Breakpoint, error) {
 	return &out.Breakpoint, err
 }
 
+// CreateBreakpoint will send a request to the RPC server to create a breakpoint.
+// Please refer to the documentation for `Debugger.CreateBreakpoint` for a description of how
+// the requested breakpoint parameters are interpreted and used:
+// https://pkg.go.dev/github.com/go-delve/delve/service/debugger#Debugger.CreateBreakpoint
 func (c *RPCClient) CreateBreakpoint(breakPoint *api.Breakpoint) (*api.Breakpoint, error) {
 	var out CreateBreakpointOut
 	err := c.call("CreateBreakpoint", CreateBreakpointIn{*breakPoint}, &out)
@@ -352,10 +356,19 @@ func (c *RPCClient) ListFunctionArgs(scope api.EvalScope, cfg api.LoadConfig) ([
 
 func (c *RPCClient) ListGoroutines(start, count int) ([]*api.Goroutine, int, error) {
 	var out ListGoroutinesOut
-	err := c.call("ListGoroutines", ListGoroutinesIn{start, count}, &out)
+	err := c.call("ListGoroutines", ListGoroutinesIn{start, count, nil, api.GoroutineGroupingOptions{}}, &out)
 	return out.Goroutines, out.Nextg, err
 }
 
+func (c *RPCClient) ListGoroutinesWithFilter(start, count int, filters []api.ListGoroutinesFilter, group *api.GoroutineGroupingOptions) ([]*api.Goroutine, []api.GoroutineGroup, int, bool, error) {
+	if group == nil {
+		group = &api.GoroutineGroupingOptions{}
+	}
+	var out ListGoroutinesOut
+	err := c.call("ListGoroutines", ListGoroutinesIn{start, count, filters, *group}, &out)
+	return out.Goroutines, out.Groups, out.Nextg, out.TooManyGroups, err
+}
+
 func (c *RPCClient) Stacktrace(goroutineId, depth int, opts api.StacktraceOptions, cfg *api.LoadConfig) ([]api.Stackframe, error) {
 	var out StacktraceOut
 	err := c.call("Stacktrace", StacktraceIn{goroutineId, depth, false, false, opts, cfg}, &out)
diff --git a/service/rpc2/server.go b/service/rpc2/server.go
index 05dbfbda..2a0607aa 100644
--- a/service/rpc2/server.go
+++ b/service/rpc2/server.go
@@ -235,19 +235,10 @@ type CreateBreakpointOut struct {
 	Breakpoint api.Breakpoint
 }
 
-// CreateBreakpoint creates a new breakpoint.
-//
-// - If arg.Breakpoint.File is not an empty string the breakpoint
-// will be created on the specified file:line location
-//
-// - If arg.Breakpoint.FunctionName is not an empty string
-// the breakpoint will be created on the specified function:line
-// location.
-//
-// - If arg.Breakpoint.Addrs is filled it will create a logical breakpoint
-// corresponding to all specified addresses.
-//
-// - Otherwise the value specified by arg.Breakpoint.Addr will be used.
+// CreateBreakpoint creates a new breakpoint. The client is expected to populate `CreateBreakpointIn`
+// with an `api.Breakpoint` struct describing where to set the breakpoing. For more information on
+// how to properly request a breakpoint via the `api.Breakpoint` struct see the documentation for
+// `debugger.CreateBreakpoint` here: https://pkg.go.dev/github.com/go-delve/delve/service/debugger#Debugger.CreateBreakpoint.
 func (s *RPCServer) CreateBreakpoint(arg CreateBreakpointIn, out *CreateBreakpointOut) error {
 	if err := api.ValidBreakpointName(arg.Breakpoint.Name); err != nil {
 		return err
@@ -594,11 +585,16 @@ func (s *RPCServer) ListTypes(arg ListTypesIn, out *ListTypesOut) error {
 type ListGoroutinesIn struct {
 	Start int
 	Count int
+
+	Filters []api.ListGoroutinesFilter
+	api.GoroutineGroupingOptions
 }
 
 type ListGoroutinesOut struct {
-	Goroutines []*api.Goroutine
-	Nextg      int
+	Goroutines    []*api.Goroutine
+	Nextg         int
+	Groups        []api.GoroutineGroup
+	TooManyGroups bool
 }
 
 // ListGoroutines lists all goroutines.
@@ -607,11 +603,37 @@ type ListGoroutinesOut struct {
 // parameter, to get more goroutines from ListGoroutines.
 // Passing a value of Start that wasn't returned by ListGoroutines will skip
 // an undefined number of goroutines.
+//
+// If arg.Filters are specified the list of returned goroutines is filtered
+// applying the specified filters.
+// For example:
+//    ListGoroutinesFilter{ Kind: ListGoroutinesFilterUserLoc, Negated: false, Arg: "afile.go" }
+// will only return goroutines whose UserLoc contains "afile.go" as a substring.
+// More specifically a goroutine matches a location filter if the specified
+// location, formatted like this:
+//    filename:lineno in function
+// contains Arg[0] as a substring.
+//
+// Filters can also be applied to goroutine labels:
+//    ListGoroutineFilter{ Kind: ListGoroutinesFilterLabel, Negated: false, Arg: "key=value" }
+// this filter will only return goroutines that have a key=value label.
+//
+// If arg.GroupBy is not GoroutineFieldNone then the goroutines will
+// be grouped with the specified criterion.
+// If the value of arg.GroupBy is GoroutineLabel goroutines will
+// be grouped by the value of the label with key GroupByKey.
+// For each group a maximum of MaxExamples example goroutines are
+// returned, as well as the total number of goroutines in the group.
 func (s *RPCServer) ListGoroutines(arg ListGoroutinesIn, out *ListGoroutinesOut) error {
+	//TODO(aarzilli): if arg contains a running goroutines filter (not negated)
+	// and start == 0 and count == 0 then we can optimize this by just looking
+	// at threads directly.
 	gs, nextg, err := s.debugger.Goroutines(arg.Start, arg.Count)
 	if err != nil {
 		return err
 	}
+	gs = s.debugger.FilterGoroutines(gs, arg.Filters)
+	gs, out.Groups, out.TooManyGroups = s.debugger.GroupGoroutines(gs, &arg.GoroutineGroupingOptions)
 	s.debugger.LockTarget()
 	defer s.debugger.UnlockTarget()
 	out.Goroutines = api.ConvertGoroutines(gs)


Test patch:
diff --git a/cmd/dlv/dlv_test.go b/cmd/dlv/dlv_test.go
index 00a082f5..2c5b23a7 100644
--- a/cmd/dlv/dlv_test.go
+++ b/cmd/dlv/dlv_test.go
@@ -326,6 +326,8 @@ func TestRedirect(t *testing.T) {
 	cmd.Wait()
 }
 
+const checkAutogenDocLongOutput = false
+
 func checkAutogenDoc(t *testing.T, filename, gencommand string, generated []byte) {
 	saved := slurpFile(t, filepath.Join(projectRoot(), filename))
 
@@ -333,13 +335,17 @@ func checkAutogenDoc(t *testing.T, filename, gencommand string, generated []byte
 	generated = bytes.ReplaceAll(generated, []byte("\r\n"), []byte{'\n'})
 
 	if len(saved) != len(generated) {
-		t.Logf("generated %q saved %q\n", generated, saved)
+		if checkAutogenDocLongOutput {
+			t.Logf("generated %q saved %q\n", generated, saved)
+		}
 		t.Fatalf("%s: needs to be regenerated; run %s", filename, gencommand)
 	}
 
 	for i := range saved {
 		if saved[i] != generated[i] {
-			t.Logf("generated %q saved %q\n", generated, saved)
+			if checkAutogenDocLongOutput {
+				t.Logf("generated %q saved %q\n", generated, saved)
+			}
 			t.Fatalf("%s: needs to be regenerated; run %s", filename, gencommand)
 		}
 	}
diff --git a/service/dap/server_test.go b/service/dap/server_test.go
index c615d86f..220685d0 100644
--- a/service/dap/server_test.go
+++ b/service/dap/server_test.go
@@ -3490,7 +3490,6 @@ func TestBadAccess(t *testing.T) {
 						if eInfo.Body.ExceptionId != "runtime error" || !strings.HasPrefix(eInfo.Body.Description, errorPrefix) {
 							t.Errorf("\ngot  %#v\nwant ExceptionId=\"runtime error\" Text=\"%s\"", eInfo, errorPrefix)
 						}
-
 					}
 
 					client.ContinueRequest(1)
@@ -3602,6 +3601,41 @@ func TestPanicBreakpointOnNext(t *testing.T) {
 }
 
 func TestFatalThrowBreakpoint(t *testing.T) {
+	runTest(t, "fatalerror", func(client *daptest.Client, fixture protest.Fixture) {
+		runDebugSessionWithBPs(t, client, "launch",
+			// Launch
+			func() {
+				client.LaunchRequest("exec", fixture.Path, !stopOnEntry)
+			},
+			// Set breakpoints
+			fixture.Source, []int{3},
+			[]onBreakpoint{{
+				execute: func() {
+					checkStop(t, client, 1, "main.main", 3)
+
+					client.ContinueRequest(1)
+					client.ExpectContinueResponse(t)
+
+					se := client.ExpectStoppedEvent(t)
+					if se.Body.ThreadId != 1 || se.Body.Reason != "exception" || se.Body.Description != "fatal error" {
+						t.Errorf("\ngot  %#v\nwant ThreadId=1 Reason=\"exception\" Description=\"fatal error\"", se)
+					}
+
+					// TODO(suzmue): Enable this test for 1.17 when https://github.com/golang/go/issues/46425 is fixed.
+					errorPrefix := "\"go of nil func value\""
+					if goversion.VersionAfterOrEqual(runtime.Version(), 1, 16) {
+						errorPrefix = "Throw reason unavailable, see https://github.com/golang/go/issues/46425"
+					}
+					client.ExceptionInfoRequest(1)
+					eInfo := client.ExpectExceptionInfoResponse(t)
+					if eInfo.Body.ExceptionId != "fatal error" || !strings.HasPrefix(eInfo.Body.Description, errorPrefix) {
+						t.Errorf("\ngot  %#v\nwant ExceptionId=\"runtime error\" Text=%s", eInfo, errorPrefix)
+					}
+
+				},
+				disconnect: true,
+			}})
+	})
 	runTest(t, "testdeadlock", func(client *daptest.Client, fixture protest.Fixture) {
 		runDebugSessionWithBPs(t, client, "launch",
 			// Launch
@@ -3621,6 +3655,10 @@ func TestFatalThrowBreakpoint(t *testing.T) {
 					if se.Body.Reason != "exception" || se.Body.Description != "fatal error" {
 						t.Errorf("\ngot  %#v\nwant Reason=\"exception\" Description=\"fatal error\"", se)
 					}
+
+					// TODO(suzmue): Get the exception info for the thread and check the description
+					// includes "all goroutines are asleep - deadlock!".
+					// Stopped events with no selected goroutines need to be supported to test deadlock.
 				},
 				disconnect: true,
 			}})
diff --git a/service/test/integration2_test.go b/service/test/integration2_test.go
index a59e13df..835671f7 100644
--- a/service/test/integration2_test.go
+++ b/service/test/integration2_test.go
@@ -1565,7 +1565,7 @@ func TestClientServer_FpRegisters(t *testing.T) {
 			if regtest.name == "XMM11" && !avx2 {
 				continue
 			}
-			if regtest.name == "XMM12" && !avx512 {
+			if regtest.name == "XMM12" && (!avx512 || testBackend == "rr") {
 				continue
 			}
 			found := false
@@ -2391,3 +2391,29 @@ func TestStopServerWithClosedListener(t *testing.T) {
 	listener.Close()
 	time.Sleep(1 * time.Second) // give time to server to panic
 }
+
+func TestGoroutinesGrouping(t *testing.T) {
+	// Tests the goroutine grouping and filtering feature
+	withTestClient2("goroutinegroup", t, func(c service.Client) {
+		state := <-c.Continue()
+		assertNoError(state.Err, t, "Continue")
+		_, ggrp, _, _, err := c.ListGoroutinesWithFilter(0, 0, nil, &api.GoroutineGroupingOptions{GroupBy: api.GoroutineLabel, GroupByKey: "name", MaxGroupMembers: 5, MaxGroups: 10})
+		assertNoError(err, t, "ListGoroutinesWithFilter (group by label)")
+		t.Logf("%#v\n", ggrp)
+		if len(ggrp) < 5 {
+			t.Errorf("not enough groups %d\n", len(ggrp))
+		}
+		var unnamedCount int
+		for i := range ggrp {
+			if ggrp[i].Name == "name=" {
+				unnamedCount = ggrp[i].Total
+				break
+			}
+		}
+		gs, _, _, _, err := c.ListGoroutinesWithFilter(0, 0, []api.ListGoroutinesFilter{{Kind: api.GoroutineLabel, Arg: "name="}}, nil)
+		assertNoError(err, t, "ListGoroutinesWithFilter (filter unnamed)")
+		if len(gs) != unnamedCount {
+			t.Errorf("wrong number of goroutines returned by filter: %d (expected %d)\n", len(gs), unnamedCount)
+		}
+	})
+}