Repository: scimmyjs/scimmy (js)

Issue:
The SCIMMY attributes throw away null value, but for PatchOp can search null value attributes, which makes the filter never match
Still for the Entra integration, to add the member to a group, the request body looks like (refer to [this link](https://learn.microsoft.com/en-us/entra/identity/app-provisioning/use-scim-to-provision-users-and-groups#update-group-add-members)):
```json
{
    "schemas": ["urn:ietf:params:scim:api:messages:2.0:PatchOp"],
    "Operations": [{
        "op": "Add",
        "path": "members",
        "value": [{
            "$ref": null,
            "value": "f648f8d5ea4e4cd38e9c"
        }]
    }]
}
```

As a result to this [SCIMMY code](https://github.com/scimmyjs/scimmy/blob/18a96c4806dc1f685e48812ec42b3e9bfbdeffdd/src/lib/types/attribute.js#L440), the $ref part is thrown away and never reach to the ingress handler.

However, when remove group member with request body looks like (refer to [this link](https://learn.microsoft.com/en-us/entra/identity/app-provisioning/use-scim-to-provision-users-and-groups#update-group-remove-members)):
```json
{
    "schemas": ["urn:ietf:params:scim:api:messages:2.0:PatchOp"],
    "Operations": [{
        "op": "Remove",
        "path": "members",
        "value": [{
            "$ref": null,
            "value": "f648f8d5ea4e4cd38e9c"
        }]
    }]
}
```

Except the SCIMMY throws the issue mentioned in https://github.com/scimmyjs/scimmy/issues/33, it also fails the Filter match, as the $ref does not exist.

Pull request description:
Treat null as undefined in `match` method of `SCIMMY.Types.Filter` class
Currently, when a filter is instantiated with an expression that looks for equality of "empty" values (e.g. `value eq null`), the `match` method will strictly compare against the value `null`. This comparison will always fail, as `null` is treated as undefined in attribute coercion.

The `match` method has been updated to treat `null` as `undefined` for equality comparison purposes (fixes #34). Additionally, the internal tokenise method has been updated to correctly detect unescaped string values that begin with numbers. The test fixtures for the filter class have also been updated to verify the expected behaviour as above.

Patch:
diff --git a/src/lib/messages/patchop.js b/src/lib/messages/patchop.js
index 4bd7302..1506181 100644
--- a/src/lib/messages/patchop.js
+++ b/src/lib/messages/patchop.js
@@ -349,9 +349,9 @@ export class PatchOp {
                                 // Get rid of any empty values from the filter
                                 .filter(([, value]) => value !== undefined)
                                 // Turn it into an equity filter string
-                                .map(([key, value]) => (`${key} eq ${value}`)).join(" and "))
-                                .join(" or ")
-                            )
+                                .map(([key, value]) => (`${key} eq ${typeof value === "string" ? `"${value}"` : value}`))
+                                // Join all comparisons into one logical expression
+                                .join(" and ")).join(" or "))
                             // Get any matching values from the filter
                             .match(target[property])
                         ));
diff --git a/src/lib/types/filter.js b/src/lib/types/filter.js
index 57ff8f4..0ac2911 100644
--- a/src/lib/types/filter.js
+++ b/src/lib/types/filter.js
@@ -26,7 +26,7 @@ const comparators = ["eq", "ne", "co", "sw", "ew", "gt", "lt", "ge", "le", "pr",
 // Regular expressions that represent filter syntax
 const lexicon = [
     // White Space, Number Values
-    /(\s+)/, /([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)/,
+    /(\s+)/, /([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)(?![\w+-])/,
     // Boolean Values, Empty Values, String Values
     /(false|true)+/, /(null)+/, /("(?:[^"]|\\.|\n)*")/,
     // Logical Groups, Complex Attribute Value Filters
@@ -311,8 +311,9 @@ export class Filter extends Array {
                         const negate = (expression[0].toLowerCase() === "not");
                         let [comparator, expected] = expression.slice(((+negate) - expression.length));
                         
-                        // Cast true and false strings to boolean values
-                        expected = (expected === "false" ? false : (expected === "true" ? true : expected));
+                        // For equality tests, cast true and false strings to boolean values, maintaining EntraID support
+                        if (["eq", "ne"].includes(comparator.toLowerCase()) && typeof actual === "boolean" && typeof expected === "string")
+                            expected = (expected.toLowerCase() === "false" ? false : (expected.toLowerCase() === "true" ? true : expected));
                         
                         switch (comparator.toLowerCase()) {
                             default:
@@ -320,11 +321,11 @@ export class Filter extends Array {
                                 break;
                             
                             case "eq":
-                                result = (actual === expected);
+                                result = (actual === (expected ?? undefined));
                                 break;
                             
                             case "ne":
-                                result = (actual !== expected);
+                                result = (actual !== (expected ?? undefined));
                                 break;
                             
                             case "co":


Test patch:
diff --git a/test/lib/messages/patchop.js b/test/lib/messages/patchop.js
index 75b4b2d..41feb5d 100644
--- a/test/lib/messages/patchop.js
+++ b/test/lib/messages/patchop.js
@@ -19,7 +19,13 @@ const TestSchema = createSchemaClass({
         new Attribute("string", "nickName"), new Attribute("string", "password", {direction: "in", returned: false}),
         new Attribute("complex", "name", {}, [new Attribute("string", "formatted"), new Attribute("string", "honorificPrefix")]),
         new Attribute("complex", "emails", {multiValued: true}, [new Attribute("string", "value"), new Attribute("string", "type")]),
-        new Attribute("string", "throws"), new Attribute("dateTime", "date")
+        new Attribute("string", "throws"), new Attribute("dateTime", "date"),
+        new Attribute("complex", "members", {multiValued: true, uniqueness: false}, [
+            new Attribute("string", "value", {mutable: "immutable"}),
+            new Attribute("string", "display", {mutable: "immutable"}),
+            new Attribute("reference", "$ref", {mutable: "immutable", referenceTypes: ["User", "Group"]}),
+            new Attribute("string", "type", {mutable: "immutable", canonicalValues: ["User", "Group"]})
+        ])
     ]
 });
 
diff --git a/test/lib/messages/patchop.json b/test/lib/messages/patchop.json
index 1127d61..c563207 100644
--- a/test/lib/messages/patchop.json
+++ b/test/lib/messages/patchop.json
@@ -22,55 +22,115 @@
         "ops": [
           {"op": "add", "path": "name", "value": {"formatted": "Test"}}
         ]
+      },
+      {
+        "source": {"id": "1234", "userName": "asdf", "members": [{"value": "f648f8d5ea4e4cd38e9c"}, {"value": "123abc"}]},
+        "target": {"id": "1234", "userName": "asdf", "members": [{"value": "f648f8d5ea4e4cd38e9c"}, {"value": "123abc", "$ref": "User"}]},
+        "ops": [
+          {"op": "add", "path": "members[value eq 123abc].$ref", "value": "User"}
+        ]
       }
     ],
     "remove": [
       {
         "source": {"id": "1234", "userName": "asdf", "name": {"honorificPrefix": "Mr"}},
         "target": {"id": "1234", "userName": "asdf"},
-        "ops": [{"op": "remove", "path": "name"}]
+        "ops": [
+          {"op": "remove", "path": "name"}
+        ]
       },
       {
         "source": {"id": "1234", "userName": "asdf", "emails": [{"type": "work", "value": "test@example.com"}, {"type": "home", "value": "asdf@dsaf.com"}]},
         "target": {"id": "1234", "userName": "asdf", "emails": [{"type": "work", "value": "test@example.com"}]},
-        "ops": [{"op": "remove", "path": "emails", "value": {"type": "home"}}]
+        "ops": [
+          {"op": "remove", "path": "emails", "value": {"type": "home"}}
+        ]
       },
       {
         "source": {"id": "1234", "userName": "asdf", "emails": [{"type": "work", "value": "test@example.com"}]},
         "target": {"id": "1234", "userName": "asdf"},
-        "ops": [{"op": "remove", "path": "emails[type eq \"work\"]"}]
+        "ops": [
+          {"op": "remove", "path": "emails[type eq \"work\"]"}
+        ]
+      },
+      {
+        "source": {"id": "1234", "userName": "asdf", "members": [{"value": "f648f8d5ea4e4cd38e9c"}, {"$ref": "User", "value": "f648f8d5ea4e4cd38e9c"}]},
+        "target": {"id": "1234", "userName": "asdf", "members": [{"$ref": "User", "value": "f648f8d5ea4e4cd38e9c"}]},
+        "ops": [
+          {"op": "remove", "path": "members", "value": [{"$ref": null, "value": "f648f8d5ea4e4cd38e9c"}]}
+        ]
+      },
+      {
+        "source": {"id": "1234", "userName": "asdf", "members": [{"value": "f648f8d5ea4e4cd38e9c"}, {"$ref": "User", "value": "f648f8d5ea4e4cd38e9c"}]},
+        "target": {"id": "1234", "userName": "asdf", "members": [{"$ref": "User", "value": "f648f8d5ea4e4cd38e9c"}]},
+        "ops": [
+          {"op": "remove", "path": "members[$ref eq null and value eq \"f648f8d5ea4e4cd38e9c\"]"}
+        ]
+      },
+      {
+        "source": {"id": "1234", "userName": "asdf", "members": [{"value": "f648f8d5ea4e4cd38e9c"}, {"value": "123abc"}]},
+        "target": {"id": "1234", "userName": "asdf", "members": [{"value": "f648f8d5ea4e4cd38e9c"}]},
+        "ops": [
+          {"op": "remove", "path": "members", "value": [{"value": "123abc"}]}
+        ]
+      },
+      {
+        "source": {"id": "1234", "userName": "asdf", "members": [{"value": "f648f8d5ea4e4cd38e9c"}, {"value": "123abc"}]},
+        "target": {"id": "1234", "userName": "asdf", "members": [{"value": "f648f8d5ea4e4cd38e9c"}]},
+        "ops": [
+          {"op": "remove", "path": "members[value eq \"123abc\"]"}
+        ]
+      },
+      {
+        "source": {"id": "1234", "userName": "asdf", "members": [{"value": "f648f8d5ea4e4cd38e9c"}, {"value": "123abc"}]},
+        "target": {"id": "1234", "userName": "asdf", "members": [{"value": "f648f8d5ea4e4cd38e9c"}]},
+        "ops": [
+          {"op": "Remove", "path": "members[value eq 123abc]"}
+        ]
       }
     ],
     "replace": [
       {
         "source": {"id": "1234", "userName": "asdf", "name": {"honorificPrefix": "Mr"}},
         "target": {"id": "1234", "userName": "ghjk", "name": {"honorificPrefix": "Mr"}},
-        "ops": [{"op": "replace", "path": "userName", "value": "ghjk"}]
+        "ops": [
+          {"op": "replace", "path": "userName", "value": "ghjk"}
+        ]
       },
       {
         "source": {"id": "1234", "userName": "asdf", "name": {"honorificPrefix": "Mr"}},
         "target": {"id": "1234", "userName": "asdf", "name": {"formatted": "Test"}},
-        "ops": [{"op": "replace", "path": "name", "value": {"formatted": "Test"}}]
+        "ops": [
+          {"op": "replace", "path": "name", "value": {"formatted": "Test"}}
+        ]
       },
       {
         "source": {"id": "1234", "userName": "asdf", "emails": [{"type": "home", "value": "asdf@dsaf.com"}]},
         "target": {"id": "1234", "userName": "asdf", "emails": [{"type": "work", "value": "test@example.com"}]},
-        "ops": [{"op": "replace", "path": "emails", "value": {"type": "work", "value": "test@example.com"}}]
+        "ops": [
+          {"op": "replace", "path": "emails", "value": {"type": "work", "value": "test@example.com"}}
+        ]
       },
       {
         "source": {"id": "1234", "userName": "asdf", "emails": [{"type": "home", "value": "asdf@dsaf.com"}]},
         "target": {"id": "1234", "userName": "asdf", "emails": [{"type": "home", "value": "asdf@dsaf.com"}, {"type": "work", "value": "test@example.com"}]},
-        "ops": [{"op": "replace", "path": "emails[type eq \"work\"]", "value": {"type": "work", "value": "test@example.com"}}]
+        "ops": [
+          {"op": "replace", "path": "emails[type eq \"work\"]", "value": {"type": "work", "value": "test@example.com"}}
+        ]
       },
       {
         "source": {"id": "1234", "userName": "asdf", "emails": [{"type": "work", "value": "asdf@dsaf.com"}]},
         "target": {"id": "1234", "userName": "asdf", "emails": [{"type": "work", "value": "test@example.com"}]},
-        "ops": [{"op": "replace", "path": "emails[type eq \"work\"]", "value": {"type": "work", "value": "test@example.com"}}]
+        "ops": [
+          {"op": "replace", "path": "emails[type eq \"work\"]", "value": {"type": "work", "value": "test@example.com"}}
+        ]
       },
       {
         "source": {"id": "1234", "userName": "asdf", "emails": [{"type": "work", "value": "asdf@dsaf.com"}]},
         "target": {"id": "1234", "userName": "asdf", "emails": [{"type": "work", "value": "test@example.com"}]},
-        "ops": [{"op": "replace", "path": "emails[type eq \"work\"].value", "value": "test@example.com"}]
+        "ops": [
+          {"op": "replace", "path": "emails[type eq \"work\"].value", "value": "test@example.com"}
+        ]
       }
     ]
   }
diff --git a/test/lib/types/filter.json b/test/lib/types/filter.json
index 06837b1..09c3353 100644
--- a/test/lib/types/filter.json
+++ b/test/lib/types/filter.json
@@ -7,12 +7,16 @@
       {"source": "displayName co \"Bob\"", "target": [{"displayName": ["co", "Bob"]}]},
       {"source": "name.formatted sw \"Bob\"", "target": [{"name": {"formatted": ["sw", "Bob"]}}]},
       {"source": "quota gt 1.5", "target": [{"quota": ["gt", 1.5]}]},
+      {"source": "quota gt 2e2", "target": [{"quota": ["gt", 2e2]}]},
       {"source": "UserType eq null", "target": [{"userType": ["eq", null]}]},
       {"source": "$ref eq null", "target": [{"$ref": ["eq", null]}]},
       {"source": "valid$Name eq null", "target": [{"valid$Name": ["eq", null]}]},
       {"source": "-valid$Name eq -null", "target": [{"-valid$Name": ["eq", "-null"]}]},
       {"source": "active eq false", "target": [{"active": ["eq", false]}]},
-      {"source": "emails.primary eq true", "target": [{"emails": {"primary": ["eq", true]}}]}
+      {"source": "emails.primary eq true", "target": [{"emails": {"primary": ["eq", true]}}]},
+      {"source": "value eq 123abc", "target": [{"value": ["eq", "123abc"]}]},
+      {"source": "value eq 123", "target": [{"value": ["eq", 123]}]},
+      {"source": "value eq 123abc5e4", "target": [{"value": ["eq", "123abc5e4"]}]}
     ],
     "logical": [
       {"source": "not eq pr", "target": [{"eq": ["not", "pr"]}]},
@@ -287,7 +291,12 @@
         {"expression": {"userName": ["pr"]}, "expected": [1, 2, 3, 4]},
         {"expression": {"exists": ["pr"]}, "expected": [1, 3]},
         {"expression": {"userName": ["np"]}, "expected": []},
-        {"expression": {"exists": ["np"]}, "expected": [2, 4]}
+        {"expression": {"exists": ["np"]}, "expected": [2, 4]},
+        {"expression": {"exists": ["eq", null]}, "expected": [2, 4]},
+        {"expression": {"exists": ["ne", null]}, "expected": [1, 3]},
+        {"expression": {"exists": ["eq", true]}, "expected": [1]},
+        {"expression": {"exists": ["eq", "True"]}, "expected": [1]},
+        {"expression": {"exists": ["eq", "False"]}, "expected": [3]}
       ],
       "nesting": [
         {"expression": {"name": {"formatted": ["co", "a"]}}, "expected": [1, 2, 4]},