Repository: telefonicaid/iotagent-node-lib
Pull request #1702:  Error control in JEXL transformations
Author: login: GregorioBlazquez
Created: 2025-03-12T09:12:21Z | Merged: 2025-03-12T11:11:38Z

This PR was initially intended to add try-catch blocks to some functions, but it also includes additional test cases and suggests better error handling. Referenced Issue: #1701 


Diff:
diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE
index e69de29bb..294d352bd 100644
--- a/CHANGES_NEXT_RELEASE
+++ b/CHANGES_NEXT_RELEASE
@@ -0,0 +1 @@
+- Fix: modified JEXL transformations (toisostring, gettime, parseint, etc.) to return null instead of NaN when some unexpected situation occurs (#1701)
\ No newline at end of file
diff --git a/lib/jexlTranformsMap.js b/lib/jexlTranformsMap.js
index 89bc2ecd0..d2c7cba2f 100644
--- a/lib/jexlTranformsMap.js
+++ b/lib/jexlTranformsMap.js
@@ -26,28 +26,85 @@
 JEXL avaliable transformations*/
 
 const map = {
-    jsonparse: (str) => JSON.parse(str),
-    jsonstringify: (obj) => JSON.stringify(obj),
+    jsonparse: (str) => {
+        try {
+            return JSON.parse(str);
+        } catch (e) {
+            return null;
+        }
+    },
+    jsonstringify: (obj) => {
+        try {
+            return JSON.stringify(obj);
+        } catch (e) {
+            return null;
+        }
+    },
     indexOf: (val, char) => String(val).indexOf(char),
     length: (val) => String(val).length,
     trim: (val) => String(val).trim(),
     substr: (val, int1, int2) => String(val).substr(int1, int2),
-    addreduce: (arr) => arr.reduce((i, v) => i + v),
+    addreduce: (arr) => {
+        try {
+            return arr.reduce((i, v) => i + v);
+        } catch (e) {
+            return null;
+        }
+    },
     lengtharray: (arr) => arr.length,
     typeof: (val) => typeof val,
     isarray: (arr) => Array.isArray(arr),
     isnan: (val) => isNaN(val),
-    parseint: (val) => parseInt(val),
-    parsefloat: (val) => parseFloat(val),
-    toisodate: (val) => new Date(val).toISOString(),
-    timeoffset: (isostr) => new Date(isostr).getTimezoneOffset(),
-    tostring: (val) => val.toString(),
+    parseint: (val) => {
+        const parsed = parseInt(val);
+        return isNaN(parsed) ? null : parsed;
+    },
+    parsefloat: (val) => {
+        const parsed = parseFloat(val);
+        return isNaN(parsed) ? null : parsed;
+    },
+    toisodate: (val) => {
+        try {
+            return new Date(val).toISOString();
+        } catch (e) {
+            return null;
+        }
+    },
+    timeoffset: (isostr) => {
+        const date = new Date(isostr);
+        return isNaN(date.getTime()) ? null : date.getTimezoneOffset();
+    },
+    tostring: (val) => {
+        try {
+            return val.toString();
+        } catch (e) {
+            return null;
+        }
+    },
     urlencode: (val) => encodeURI(val),
-    urldecode: (val) => decodeURI(val),
+    urldecode: function (value) {
+        try {
+            return decodeURI(value);
+        } catch {
+            return null;
+        }
+    },
     replacestr: (str, from, to) => str.replace(from, to),
-    replaceregexp: (str, reg, to) => str.replace(new RegExp(reg), to),
+    replaceregexp: (str, reg, to) => {
+        try {
+            return str.replace(new RegExp(reg), to);
+        } catch (e) {
+            return null;
+        }
+    },
     replaceallstr: (str, from, to) => str.replaceAll(from, to),
-    replaceallregexp: (str, reg, to) => str.replaceAll(new RegExp(reg, 'g'), to),
+    replaceallregexp: (str, reg, to) => {
+        try {
+            return str.replaceAll(new RegExp(reg, 'g'), to);
+        } catch (e) {
+            return null;
+        }
+    },
     split: (str, ch) => str.split(ch),
     joinarrtostr: (arr, ch) => arr.join(ch),
     concatarr: (arr, arr2) => arr.concat(arr2),
@@ -73,16 +130,49 @@ const map = {
     floor: (val) => Math.floor(val),
     ceil: (val) => Math.ceil(val),
     round: (val) => Math.round(val),
-    tofixed: (val, decimals) => Number.parseFloat(val).toFixed(decimals),
-    gettime: (d) => new Date(d).getTime(),
-    toisostring: (d) => new Date(d).toISOString(),
+    tofixed: (val, decimals) => {
+        const num = Number.parseFloat(val);
+        const dec = Number.parseInt(decimals);
+
+        if (isNaN(num) || isNaN(dec)) return null;
+
+        return num.toFixed(dec);
+    },
+    gettime: function (value) {
+        const timestamp = new Date(value).getTime();
+        return isNaN(timestamp) ? null : timestamp;
+    },
+    toisostring: (d) => {
+        const date = new Date(d);
+        return isNaN(date.getTime()) ? null : date.toISOString();
+    },
     // https://developer.mozilla.org/es/docs/Web/JavaScript/Reference/Global_Objects/Date/toLocaleString
-    localestring: (d, timezone, options) => new Date(d).toLocaleString(timezone, options),
+    localestring: (d, timezone, options) => {
+        try {
+            return new Date(d).toLocaleString(timezone, options);
+        } catch (e) {
+            return null;
+        }
+    },
     now: () => Date.now(),
-    hextostring: (val) =>
-        new TextDecoder().decode(new Uint8Array(val.match(/.{1,2}/g).map((byte) => parseInt(byte, 16)))),
-    valuePicker: (val,pick) => Object.entries(val).filter(([, v]) => v === pick).map(([k,]) => k),
-    valuePickerMulti: (val,pick) => Object.entries(val).filter(([, v]) => pick.includes(v)).map(([k,]) => k)
+    hextostring: (val) => {
+        try {
+            if (typeof val !== 'string' || !/^[0-9a-fA-F]+$/.test(val) || val.length % 2 !== 0) {
+                return null;
+            }
+            return new TextDecoder().decode(new Uint8Array(val.match(/.{1,2}/g).map((byte) => parseInt(byte, 16))));
+        } catch (e) {
+            return null;
+        }
+    },
+    valuePicker: (val, pick) =>
+        Object.entries(val)
+            .filter(([, v]) => v === pick)
+            .map(([k]) => k),
+    valuePickerMulti: (val, pick) =>
+        Object.entries(val)
+            .filter(([, v]) => pick.includes(v))
+            .map(([k]) => k)
 };
 
 exports.map = map;
diff --git a/test/unit/expressions/jexlExpression-test.js b/test/unit/expressions/jexlExpression-test.js
index 4b187f76b..79ec7582b 100644
--- a/test/unit/expressions/jexlExpression-test.js
+++ b/test/unit/expressions/jexlExpression-test.js
@@ -40,7 +40,12 @@ describe('Jexl expression interpreter', function () {
         object: {
             name: 'John',
             surname: 'Doe'
-        }
+        },
+        invalidJson: '{"name": "John", surname": "Doe"}', // Invalid JSON for jsonparse
+        invalidDate: 'invalid-date', // Invalid date for toisodate and other date functions
+        emptyArray: [], // Empty array for addreduce
+        invalidHex: 'zz', // Invalid hex string for hextostring
+        nonNumeric: 'not-a-number' // Non-numeric input for parseint, parsefloat
     };
 
     describe('When a expression with a single value is parsed', function () {
@@ -392,4 +397,163 @@ describe('Jexl expression interpreter', function () {
             });
         });
     });
+
+    // Check errors
+    describe('When invalid inputs are used', function () {
+        describe('When an invalid JSON string is parsed', function () {
+            it('should return null for jsonparse', function (done) {
+                expressionParser.parse('invalidJson|jsonparse', scope, function (error, result) {
+                    should.not.exist(error);
+                    should(result).be.Null();
+                    done();
+                });
+            });
+        });
+
+        describe('When an invalid date string is parsed', function () {
+            it('should return null for toisodate', function (done) {
+                expressionParser.parse('invalidDate|toisodate', scope, function (error, result) {
+                    should.not.exist(error);
+                    should(result).be.Null();
+                    done();
+                });
+            });
+
+            it('should return null for timeoffset', function (done) {
+                expressionParser.parse('invalidDate|timeoffset', scope, function (error, result) {
+                    should.not.exist(error);
+                    should(result).be.Null();
+                    done();
+                });
+            });
+        });
+
+        describe('When an empty array is processed with addreduce', function () {
+            it('should return null for addreduce', function (done) {
+                expressionParser.parse('emptyArray|addreduce', scope, function (error, result) {
+                    should.not.exist(error);
+                    should(result).be.Null();
+                    done();
+                });
+            });
+        });
+
+        describe('When a non-numeric string is parsed by parseint and parsefloat', function () {
+            it('should return null for parseint', function (done) {
+                expressionParser.parse('nonNumeric|parseint', scope, function (error, result) {
+                    should.not.exist(error);
+                    should(result).be.Null();
+                    done();
+                });
+            });
+
+            it('should return null for parsefloat', function (done) {
+                expressionParser.parse('nonNumeric|parsefloat', scope, function (error, result) {
+                    should.not.exist(error);
+                    should(result).be.Null();
+                    done();
+                });
+            });
+        });
+
+        describe('When an invalid hex string is processed by hextostring', function () {
+            it('should return null for hextostring', function (done) {
+                expressionParser.parse('invalidHex|hextostring', scope, function (error, result) {
+                    should.not.exist(error);
+                    should(result).be.Null();
+                    done();
+                });
+            });
+        });
+
+        describe('When invalid regular expressions are used in replace functions', function () {
+            it('should return null for replaceregexp with invalid regex', function (done) {
+                expressionParser.parse(
+                    'theString|replaceregexp("[a-z", "replacement")',
+                    scope,
+                    function (error, result) {
+                        should.not.exist(error);
+                        should(result).be.Null();
+                        done();
+                    }
+                );
+            });
+
+            it('should return null for replaceallregexp with invalid regex', function (done) {
+                expressionParser.parse(
+                    'theString|replaceallregexp("[a-z", "replacement")',
+                    scope,
+                    function (error, result) {
+                        should.not.exist(error);
+                        should(result).be.Null();
+                        done();
+                    }
+                );
+            });
+        });
+
+        describe('When tostring is used with null or undefined', function () {
+            it('should return null for tostring(null)', function (done) {
+                expressionParser.parse('null|tostring', scope, function (error, result) {
+                    should.not.exist(error);
+                    should(result).be.Null();
+                    done();
+                });
+            });
+
+            it('should return null for tostring(undefined)', function (done) {
+                expressionParser.parse('undefined|tostring', scope, function (error, result) {
+                    should.not.exist(error);
+                    should(result).be.Null();
+                    done();
+                });
+            });
+        });
+
+        describe('When urldecode is used with an invalid URI', function () {
+            it('should return null for urldecode with malformed URI', function (done) {
+                expressionParser.parse('"%%%invalidURI"|urldecode', scope, function (error, result) {
+                    should.not.exist(error);
+                    should(result).be.Null();
+                    done();
+                });
+            });
+        });
+
+        describe('When tofixed is used with invalid inputs', function () {
+            it('should return null for tofixed with non-numeric value', function (done) {
+                expressionParser.parse('"notANumber"|tofixed(2)', scope, function (error, result) {
+                    should.not.exist(error);
+                    should(result).be.Null();
+                    done();
+                });
+            });
+
+            it('should return null for tofixed with invalid decimal value', function (done) {
+                expressionParser.parse('"123.456"|tofixed("invalid")', scope, function (error, result) {
+                    should.not.exist(error);
+                    should(result).be.Null();
+                    done();
+                });
+            });
+        });
+
+        describe('When gettime is used with invalid date', function () {
+            it('should return null for gettime with invalid date string', function (done) {
+                expressionParser.parse('"invalidDate"|gettime', scope, function (error, result) {
+                    should.not.exist(error);
+                    should(result).be.Null();
+                    done();
+                });
+            });
+
+            it('should return null for gettime with null', function (done) {
+                expressionParser.parse('null|gettime', scope, function (error, result) {
+                    should.not.exist(error);
+                    should(result).be.Null();
+                    done();
+                });
+            });
+        });
+    });
 });


Reviews:
user: login: AlvaroVega
body: LGTM
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-12T10:43:56Z
user: login: fgalan
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-12T11:00:57Z
user: login: fgalan
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-12T11:01:39Z
user: login: fgalan
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-12T11:11:11Z
user: login: fgalan
body: LGTM
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-12T11:11:32Z

Review comments:
diff_hunk: @@ -0,0 +1,3 @@
+- Fix: Added try-catch error handling to functions like jsonparse, jsonstringify, toisodate, etc. (#1702).
+- Fix: Modified functions (toisostring, gettime, parseint, etc.) to return null instead of NaN (#1702).
+- Add: Added tests for error scenarios, including a specific test for expressionParser.parse() (#1702).
path: CHANGES_NEXT_RELEASE
user: login: fgalan
body: The changelog should be redacted in a user-perspective. In this sense I'd suggest to simplify to:

```suggestion
- Fix: modified JEXL transformations (toisostring, gettime, parseint, etc.) to return null instead of NaN when some unexpected situation occurs (#1701)
```
created_at: 2025-03-12T11:00:57Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,3 @@
+- Fix: Added try-catch error handling to functions like jsonparse, jsonstringify, toisodate, etc. (#1702).
+- Fix: Modified functions (toisostring, gettime, parseint, etc.) to return null instead of NaN (#1702).
+- Add: Added tests for error scenarios, including a specific test for expressionParser.parse() (#1702).
path: CHANGES_NEXT_RELEASE
user: login: fgalan
body: Note that, if possible, the reference should be to the issue associated to the PR, not to the PR itself.
created_at: 2025-03-12T11:01:39Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1 @@
+- Fix: modified JEXL transformations (toisostring, gettime, parseint, etc.) to return null instead of NaN when some unexpected situation occurs (#1702).
path: CHANGES_NEXT_RELEASE
user: login: fgalan
body: ```suggestion
- Fix: modified JEXL transformations (toisostring, gettime, parseint, etc.) to return null instead of NaN when some unexpected situation occurs (#1701)
```
created_at: 2025-03-12T11:11:11Z
author_association: MEMBER

PR comments:
user: login: GregorioBlazquez
created_at: 2025-03-12T09:22:12Z
body: I’ve added exception handling (try-catch) to the following functions: _jsonparse_, _jsonstringify_, _addreduce_, _toisodate_, _tostring_, _urldecode_, _replaceregexp_, _replaceallregexp_, _localestring_ and _hextostring_.

Additionally, I’ve modified functions like _toisostring_, _gettime_, _tofixed_, _parseint_, and _parsefloat_ to return _null_ instead of _NaN_.

I’ve also added new tests to verify error scenarios, which were the driving force behind the decision to change some functions to return _null_ instead of _NaN_. There are still other functions that could potentially benefit from this change, especially most of the string transformation functions that haven’t been modified yet.
author_association: CONTRIBUTOR
user: login: GregorioBlazquez
created_at: 2025-03-12T09:30:00Z
body: The tests included the following case:
```
describe('When an expression with a parse error is parsed', function () {
      it('should raise an INVALID_EXPRESSION error', function (done) {
          /* eslint-disable-next-line  no-unused-vars */
          expressionParser.parse('"numb+sd ññ ((', scope, function (error, result) {
              should.exist(error);
              error.name.should.equal('INVALID_EXPRESSION');
              done();
          });
      });
  });
```

This test is verifying a function located in [jexlParser.js](iotagent-node-lib/lib/plugins/jexlParser.js) (line 40), specifically the expressionParser.parse() function. I wanted to ask if this function should be modified to return null instead of raising an INVALID_EXPRESSION error, or if that behavior is intentional.
author_association: CONTRIBUTOR
user: login: AlvaroVega
created_at: 2025-03-12T09:41:32Z
body: > The tests included the following case:
> 
> ```
> describe('When an expression with a parse error is parsed', function () {
>       it('should raise an INVALID_EXPRESSION error', function (done) {
>           /* eslint-disable-next-line  no-unused-vars */
>           expressionParser.parse('"numb+sd ññ ((', scope, function (error, result) {
>               should.exist(error);
>               error.name.should.equal('INVALID_EXPRESSION');
>               done();
>           });
>       });
>   });
> ```
> 
> This test is verifying a function located in [jexlParser.js](iotagent-node-lib/lib/plugins/jexlParser.js) (line 40), specifically the expressionParser.parse() function. I wanted to ask if this function should be modified to return null instead of raising an INVALID_EXPRESSION error, or if that behavior is intentional.

IMHO this behavior is intentional, so none changes are expected in that function.
author_association: MEMBER