Repository: maxnowack/signaldb
Pull request #1426 (2025-02-19 08:04:41)

perf: improve index query performance

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/maxnowack/signaldb/pull/1426?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Max+Nowack) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 100.00%. Comparing base [(`c1bb7d7`)](https://app.codecov.io/gh/maxnowack/signaldb/commit/c1bb7d7424c2af073db7084f0e9afd27156ede1d?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Max+Nowack) to head [(`7b1e655`)](https://app.codecov.io/gh/maxnowack/signaldb/commit/7b1e65505cdaf0959fb56861cc2d953257cb9052?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Max+Nowack).

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff            @@
##              main     #1426   +/-   ##
=========================================
  Coverage   100.00%   100.00%           
=========================================
  Files           55        55           
  Lines         1723      1735   +12     
  Branches       390       389    -1     
=========================================
+ Hits          1723      1735   +12     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/maxnowack/signaldb/pull/1426?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Max+Nowack).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Max+Nowack).


Patch:
diff --git a/packages/base/core/CHANGELOG.md b/packages/base/core/CHANGELOG.md
index 5f77917..0e2ea67 100644
--- a/packages/base/core/CHANGELOG.md
+++ b/packages/base/core/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+* Improved index query performance
+
 ## [1.3.0] - 2025-02-18
 
 ### Added
diff --git a/packages/base/core/src/Collection/createIndex.ts b/packages/base/core/src/Collection/createIndex.ts
index b22e8ff..31e4d1d 100644
--- a/packages/base/core/src/Collection/createIndex.ts
+++ b/packages/base/core/src/Collection/createIndex.ts
@@ -20,19 +20,43 @@ export function createExternalIndex<T extends BaseItem<I> = BaseItem, I = any>(
       const keys = getMatchingKeys<T, I>(field, selector)
       if (keys.include == null && keys.exclude == null) return { matched: false }
 
-      const includedKeys = keys.include == null
-        // eslint-disable-next-line unicorn/prefer-array-flat
-        ? [...index.values()].reduce<number[]>((memo, set) => [...memo, ...set], [])
-        : keys.include.reduce<number[]>((memo, key) => [...memo, ...index.get(key) || []], [])
+      // Accumulate included positions
+      let includedPositions: number[] = []
+      if (keys.include == null) {
+        for (const set of index.values()) {
+          for (const pos of set) {
+            includedPositions.push(pos)
+          }
+        }
+      } else {
+        for (const key of keys.include) {
+          const posSet = index.get(key)
+          if (posSet) {
+            for (const pos of posSet) {
+              includedPositions.push(pos)
+            }
+          }
+        }
+      }
 
-      const itemPositions = keys.exclude == null
-        ? includedKeys
-        : keys.exclude.reduce<number[]>((memo, key) =>
-          memo.filter(i => !index.get(key)?.has(i)), includedKeys)
+      // If exclusion is specified, build a single set of all positions to exclude.
+      if (keys.exclude != null) {
+        const excludeSet = new Set<number>()
+        for (const key of keys.exclude) {
+          const posSet = index.get(key)
+          if (posSet) {
+            for (const pos of posSet) {
+              excludeSet.add(pos)
+            }
+          }
+        }
+        // Filter out any position that exists in the exclude set.
+        includedPositions = includedPositions.filter(pos => !excludeSet.has(pos))
+      }
 
       return {
         matched: true,
-        positions: itemPositions,
+        positions: includedPositions,
         fields: [field],
       }
     },
diff --git a/packages/base/sync/CHANGELOG.md b/packages/base/sync/CHANGELOG.md
index 7f346cf..53b3273 100644
--- a/packages/base/sync/CHANGELOG.md
+++ b/packages/base/sync/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+* Improved sync performance by reducing overhead in internal `applyChanges` method
+
 ## [1.1.2] - 2025-02-18
 
 ### Changed
diff --git a/packages/base/sync/src/applyChanges.ts b/packages/base/sync/src/applyChanges.ts
index f22871a..c0bbf11 100644
--- a/packages/base/sync/src/applyChanges.ts
+++ b/packages/base/sync/src/applyChanges.ts
@@ -1,5 +1,5 @@
-import type { BaseItem, Selector } from '@signaldb/core'
-import { Collection, modify } from '@signaldb/core'
+import type { BaseItem } from '@signaldb/core'
+import { modify } from '@signaldb/core'
 import type { Change } from './types'
 
 /**
@@ -8,40 +8,33 @@ import type { Change } from './types'
  * @param changes The changes to apply to the items
  * @returns The new items after applying the changes
  */
-export default async function applyChanges<ItemType extends BaseItem<IdType>, IdType>(
+export default function applyChanges<ItemType extends BaseItem<IdType>, IdType>(
   items: ItemType[],
   changes: Change<ItemType, IdType>[],
-) {
-  const collection = new Collection<ItemType, IdType>()
-  collection.batch(() => {
-    items.forEach(item => collection.insert(item))
-    changes.forEach((change) => {
-      if (change.type === 'remove') {
-        collection.removeOne({ id: change.data } as Selector<ItemType>)
-        return
-      }
+): ItemType[] {
+  // Create initial map of items by ID
+  const itemMap = new Map(items.map(item => [item.id, item]))
 
-      const selector = { id: change.data.id } as ItemType
-      const itemExists = collection.findOne(selector)
-
-      if (change.type === 'insert') {
-        if (itemExists) { // update item if it alread exists
-          collection.updateOne(selector, { $set: change.data })
-        } else { // insert item if it does not exist
-          collection.insert(change.data)
-        }
-        return
-      }
-
-      // change.type === 'update'
-      if (itemExists) { // update item if it exists
-        collection.updateOne(selector, change.data.modifier)
-      } else { // insert item if it does not exist
-        collection.insert(modify(selector, change.data.modifier))
-      }
-    })
+  changes.forEach((change) => {
+    if (change.type === 'remove') {
+      itemMap.delete(change.data)
+    } else if (change.type === 'insert') {
+      const existingItem = itemMap.get(change.data.id)
+      itemMap.set(
+        change.data.id,
+        existingItem ? { ...existingItem, ...change.data } : change.data,
+      )
+    } else { // change.type === 'update'
+      const existingItem = itemMap.get(change.data.id)
+      itemMap.set(
+        change.data.id,
+        existingItem
+          ? modify(existingItem, change.data.modifier)
+          : modify({ id: change.data.id } as ItemType, change.data.modifier),
+      )
+    }
   })
-  const result = collection.find().fetch()
-  await collection.dispose()
-  return result
+
+  // Convert map back to array
+  return [...itemMap.values()]
 }
diff --git a/packages/base/sync/src/sync.ts b/packages/base/sync/src/sync.ts
index a72c942..89cbc9b 100644
--- a/packages/base/sync/src/sync.ts
+++ b/packages/base/sync/src/sync.ts
@@ -72,10 +72,10 @@ export default async function sync<ItemType extends BaseItem<IdType>, IdType>({
   let newSnapshot = getSnapshot(lastSnapshot, newData)
   if (changes.length > 0) {
     // apply changes on last snapshot and check if there is a difference
-    const lastSnapshotWithChanges = await applyChanges(previousSnapshot, changes)
+    const lastSnapshotWithChanges = applyChanges(previousSnapshot, changes)
     if (hasDifference(previousSnapshot, lastSnapshotWithChanges)) {
       // if yes, apply the changes on the newSnapshot and check if there is a difference
-      const newSnapshotWithChanges = await applyChanges(newSnapshot, changes)
+      const newSnapshotWithChanges = applyChanges(newSnapshot, changes)
       const changesToPush = computeChanges(newSnapshot, newSnapshotWithChanges)
       if (hasChanges(changesToPush)) {
         // if yes, push the changes to the server


Test patch:
diff --git a/packages/base/sync/__tests__/applyChanges.spec.ts b/packages/base/sync/__tests__/applyChanges.spec.ts
index c2f22a1..cc9182e 100644
--- a/packages/base/sync/__tests__/applyChanges.spec.ts
+++ b/packages/base/sync/__tests__/applyChanges.spec.ts
@@ -24,7 +24,7 @@ it('should insert new items', async () => {
     { ...getDefaultChangeItem(), type: 'insert', data: { id: 2, name: 'Item 2' } },
   ]
 
-  const result = await applyChanges(items, changes)
+  const result = applyChanges(items, changes)
   expect(result).toEqual([
     { id: 1, name: 'Item 1' },
     { id: 2, name: 'Item 2' },
@@ -37,7 +37,7 @@ it('should update an item', async () => {
     { ...getDefaultChangeItem(), type: 'update', data: { id: 1, modifier: { $set: { name: 'Updated Item 1' } } } },
   ]
 
-  const result = await applyChanges(items, changes)
+  const result = applyChanges(items, changes)
   expect(result).toEqual([{ id: 1, name: 'Updated Item 1' }])
 })
 
@@ -47,7 +47,7 @@ it('should remove an item', async () => {
     { ...getDefaultChangeItem(), type: 'remove', data: 1 },
   ]
 
-  const result = await applyChanges(items, changes)
+  const result = applyChanges(items, changes)
   expect(result).toEqual([{ id: 2, name: 'Item 2' }])
 })
 
@@ -59,7 +59,7 @@ it('should handle a mix of inserts, updates, and removes', async () => {
     { ...getDefaultChangeItem('3'), type: 'remove', data: 2 },
   ]
 
-  const result = await applyChanges(items, changes)
+  const result = applyChanges(items, changes)
   expect(result).toEqual([{ id: 1, name: 'Updated Item 1' }])
 })
 
@@ -69,7 +69,7 @@ it('should update already existing items with insert', async () => {
     { ...getDefaultChangeItem(), type: 'insert', data: { id: 1, name: 'New Item Name' } },
   ]
 
-  const result = await applyChanges(items, changes)
+  const result = applyChanges(items, changes)
   expect(result).toEqual([
     { id: 1, name: 'New Item Name' },
   ])
@@ -81,6 +81,6 @@ it('should insert non-existing items on update', async () => {
     { ...getDefaultChangeItem(), type: 'update', data: { id: 1, modifier: { $set: { name: 'Updated Item 1' } } } },
   ]
 
-  const result = await applyChanges(items, changes)
+  const result = applyChanges(items, changes)
   expect(result).toEqual([{ id: 1, name: 'Updated Item 1' }])
 })
diff --git a/packages/base/sync/__tests__/sync.spec.ts b/packages/base/sync/__tests__/sync.spec.ts
index 177ca71..026578b 100644
--- a/packages/base/sync/__tests__/sync.spec.ts
+++ b/packages/base/sync/__tests__/sync.spec.ts
@@ -35,7 +35,7 @@ it('should apply changes to the last snapshot and push them to the server if the
   const batch = vi.fn().mockImplementation((fn: () => void) => fn())
 
   const snapshot = getSnapshot(lastSnapshot, data)
-  const newSnapshotWithChanges = await applyChanges(snapshot, changes)
+  const newSnapshotWithChanges = applyChanges(snapshot, changes)
   const changesToPush = computeChanges(snapshot, newSnapshotWithChanges)
 
   await sync({
@@ -72,7 +72,7 @@ it('should not push changes if there is no difference between snapshots', async
   const mockRemove = vi.fn<(id: number) => void>()
   const batch = vi.fn().mockImplementation((fn: () => void) => fn())
 
-  const lastSnapshotWithChanges = await applyChanges(lastSnapshot, changes)
+  const lastSnapshotWithChanges = applyChanges(lastSnapshot, changes)
 
   expect(lastSnapshotWithChanges).toEqual(lastSnapshot)