Repository: jestjs/jest (TypeScript)

Issue:
[Bug]: Stackoverflow in toMatchObject when matching w/ recursive object
### Version

29.7.0

### Steps to reproduce

Using `expect` 29.7.0

Here is a contrived example (real usage [here](https://github.com/GoogleChrome/lighthouse/blob/b75a9d98875c95d156811b3b85e796aa4024a82c/core/test/network-records-to-devtools-log.js#L477) / [here](https://github.com/GoogleChrome/lighthouse/pull/15640#issuecomment-1836904534))

```js
const a = {
  v: 1,
};
const b = {
  v: 1,
};
const c1 = {
  ref: [],
};
c1.ref.push(c1);
const c2 = {
  ref: [],
};
c2.ref.push(c2);

a.ref = c1;
b.ref = c2;
expect(a).toMatchObject(b);
```

### Expected behavior

Either document that recursive objects are not supported, or support them.

### Actual behavior

`RangeError: Maximum call stack size exceeded`

### Additional context

_No response_

### Environment

```shell
n/a
```



Hints:
This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 30 days.
We should start by adding a warning to the docs I guess, but it would be nice to support this
As far as I've tested, the issue doesn't happen if the circular reference is not in an array nor if it's in a set. It also works if it's nested deeper in. I can take a crack at fixing this issue if no one else is working on it.

Patch:
diff --git a/packages/expect-utils/src/utils.ts b/packages/expect-utils/src/utils.ts
--- a/packages/expect-utils/src/utils.ts
+++ b/packages/expect-utils/src/utils.ts
@@ -355,12 +355,14 @@ export const subsetEquality = (
         return undefined;
       }
 
-      return getObjectKeys(subset).every(key => {
+      if (seenReferences.has(subset)) return undefined;
+      seenReferences.set(subset, true);
+
+      const matchResult = getObjectKeys(subset).every(key => {
         if (isObjectWithKeys(subset[key])) {
           if (seenReferences.has(subset[key])) {
             return equals(object[key], subset[key], filteredCustomTesters);
           }
-          seenReferences.set(subset[key], true);
         }
         const result =
           object != null &&
@@ -377,6 +379,8 @@ export const subsetEquality = (
         seenReferences.delete(subset[key]);
         return result;
       });
+      seenReferences.delete(subset);
+      return matchResult;
     };
 
   return subsetEqualityWithContext()(object, subset);


Test patch:
diff --git a/e2e/__tests__/circularRefInBuiltInObj.test.ts b/e2e/__tests__/circularRefInBuiltInObj.test.ts
new file mode 100644
--- /dev/null
+++ b/e2e/__tests__/circularRefInBuiltInObj.test.ts
@@ -0,0 +1,89 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+describe('matches circular references nested in:', () => {
+  interface CircularObj {
+    ref: unknown;
+    [prop: string]: unknown;
+  }
+
+  test('arrays', () => {
+    type CircularArray = CircularObj & {ref: Array<unknown>};
+
+    const a: CircularArray = {c: 1, ref: [1]};
+    const b: CircularArray = {c: 1, ref: [1]};
+
+    a.ref.push(a);
+    b.ref.push(b);
+    expect(a).toMatchObject(b);
+
+    b.ref = [];
+    expect(a).not.toMatchObject(b);
+
+    b.ref = [1];
+    expect(a).not.toMatchObject(b);
+  });
+
+  test('deeply nested array properties', () => {
+    type DeepCircularArray = CircularObj & {ref: {inner: Array<unknown>}};
+    const a: DeepCircularArray = {
+      c: 1,
+      ref: {
+        inner: [1],
+      },
+    };
+    const b: DeepCircularArray = {
+      c: 1,
+      ref: {
+        inner: [1],
+      },
+    };
+    a.ref.inner.push(a);
+    b.ref.inner.push(b);
+    expect(a).toMatchObject(b);
+
+    b.ref.inner = [];
+    expect(a).not.toMatchObject(b);
+
+    b.ref.inner = [1];
+    expect(a).not.toMatchObject(b);
+  });
+
+  test('sets', () => {
+    type CircularSet = CircularObj & {ref: Set<unknown>};
+
+    const a: CircularSet = {c: 1, ref: new Set()};
+    const b: CircularSet = {c: 1, ref: new Set()};
+
+    a.ref.add(a);
+    b.ref.add(b);
+    expect(a).toMatchObject(b);
+
+    b.ref.clear();
+    expect(a).not.toMatchObject(b);
+
+    b.ref.add(1);
+    expect(a).not.toMatchObject(b);
+  });
+
+  test('maps', () => {
+    type CircularMap = CircularObj & {ref: Map<string, unknown>};
+
+    const a: CircularMap = {c: 1, ref: new Map()};
+    const b: CircularMap = {c: 1, ref: new Map()};
+
+    a.ref.set('innerRef', a);
+    b.ref.set('innerRef', b);
+    expect(a).toMatchObject(b);
+
+    b.ref.clear();
+    expect(a).not.toMatchObject(b);
+
+    b.ref.set('innerRef', 1);
+    expect(a).not.toMatchObject(b);
+  });
+});