Option to disable NULL_VALUE output processing

src/jsonata/jsonata.py

@@ -1921,6 +1921,7 @@ def __init__(self, expr: Optional[str]) -> None:
 
         self.input = None
         self.validate_input = True
+        self.output_convert_nulls = True
 
         # Note: now and millis are implemented in Functions
         #  environment.bind("now", defineFunction(function(picture, timezone) {
@@ -1957,6 +1958,24 @@ def is_validate_input(self) -> bool:
     def set_validate_input(self, validate_input: bool) -> None:
         self.validate_input = validate_input
 
+    #
+    # Checks whether output NULL_VALUE conversion is enabled
+    #
+    def is_output_convert_nulls(self) -> bool:
+        return self.output_convert_nulls
+
+    #
+    # Enable or disable output NULL_VALUE conversion. Enabled by default, which
+    # returns both "JSONata null" and "JSONata undefined" as Python None.
+    #
+    # When disabled, output values may contain Utils.NULL_VALUE indicating
+    # "JSONata null" while Python None indicates "JSONata undefined".
+    # Manually calling Utils.convert_nulls(result) on a raw result will yield
+    # the converted result.
+    #
+    def set_output_convert_nulls(self, output_convert_nulls: bool) -> None:
+        self.output_convert_nulls = output_convert_nulls
+
     def evaluate(self, input: Optional[Any], bindings: Optional[Frame] = None) -> Optional[Any]:
         # throw if the expression compiled with syntax errors
         if self.errors is not None:
@@ -1993,7 +2012,8 @@ def evaluate(self, input: Optional[Any], bindings: Optional[Frame] = None) -> Op
             #  if (typeof callback === "function") {
             #      callback(null, it)
             #  }
-            it = utils.Utils.convert_nulls(it)
+            if self.output_convert_nulls:
+                it = utils.Utils.convert_nulls(it)
             return it
         except Exception as err:
             # insert error message into structure

tests/jsonata_test.py

@@ -3,6 +3,23 @@
 import math
 import pathlib
 import traceback
+import uuid
+
+
+# Sentinel representing a JSONata undefined result (distinct from JSONata null,
+# which surfaces as Python None after Utils.convert_nulls).
+UNDEFINED = "__UNDEFINED__" + uuid.uuid4().hex
+
+
+def evaluate_jsonata(expr, data, bindings):
+    """Evaluate `expr`, returning UNDEFINED for an undefined result (to
+    distinguish it from JSONata null, which becomes Python None)."""
+    j = jsonata.Jsonata(expr)
+    j.set_output_convert_nulls(False)
+    result = j.evaluate(data, bindings)
+    if result is None:
+        result = UNDEFINED
+    return jsonata.Utils.convert_nulls(result)
 
 
 class TestJsonata:
@@ -27,20 +44,15 @@ def eval_expr(self, expr, data, bindings, expected, code):
                 for k, v in bindings.items():
                     binding_frame.bind(k, v)
 
-            jsonata_expr = jsonata.Jsonata(expr)
             if binding_frame is not None:
                 binding_frame.set_runtime_bounds(500000 if TestJsonata.debug else 10000, 303)
-            result = jsonata_expr.evaluate(data, binding_frame)
-            if code is not None:
-                success = False
 
-            if expected is not None and expected != result:
-                # if ((""+expected).equals(""+result))
-                #     System.out.println("Value equals failed, stringified equals = true. Result = "+result)
-                # else
+            result = evaluate_jsonata(expr, data, binding_frame)
+
+            if code is not None:
                 success = False
 
-            if expected is None and result is not None:
+            if expected != result:
                 success = False
 
             if TestJsonata.debug and success:
@@ -119,22 +131,6 @@ def run_test_suite(self, name):
             success &= self.run_test_case(name, test_case)
         return success
 
-    def replace_nulls(self, o):
-        if isinstance(o, list):
-            index = 0
-            for i in o:
-                if i is None:
-                    o[index] = jsonata.Utils.NULL_VALUE
-                else:
-                    self.replace_nulls(i)
-                index += 1
-        if isinstance(o, dict):
-            for k, v in o.items():
-                if v is None:
-                    o[k] = jsonata.Utils.NULL_VALUE
-                else:
-                    self.replace_nulls(v)
-
     testOverrides = None
 
     @staticmethod
@@ -174,20 +170,21 @@ def run_test_case(self, name, test_def):
         bindings = test_def.get("bindings")
         result = test_def.get("result")
 
-        # if (result == null)
-        #   if (testDef.containsKey("result"))
-        #     result = Jsonata.NULL_VALUE
-
-        # replaceNulls(result)
+        # Check if test is expected to return undefined result
+        if str(test_def.get("undefinedResult")).lower() == "true":
+            result = UNDEFINED
 
         code = test_def.get("code")
 
         if isinstance(test_def.get("error"), dict):
             code = test_def.get("error").get("code")
 
-        # System.out.println(""+bindings)
-
         data = test_def.get("data")
+        # Explicit `"data": null` means JSONata null input (not undefined), so
+        # feed it as NULL_VALUE since Python None would be read as undefined.
+        if "data" in test_def and data is None:
+            data = jsonata.Utils.NULL_VALUE
+
         if data is None and dataset is not None:
             data = self.read_json("jsonata/test/test-suite/datasets/" + dataset + ".json")
 

tests/null_safety_test.py

@@ -1,5 +1,14 @@
 import jsonata
 
+from tests.jsonata_test import TestJsonata as JsonataTestRunner, UNDEFINED, evaluate_jsonata
+
+
+def _execute_jsonata_raw(expr, input):
+    """Evaluate without output NULL_VALUE conversion."""
+    j = jsonata.Jsonata(expr)
+    j.set_output_convert_nulls(False)
+    return j.evaluate(input)
+
 
 class TestNullSafety:
 
@@ -38,3 +47,98 @@ def test_array_index_preserves_null(self):
         data = {"data": [[1, None, 3], [2, None, 4], [3, None, 5]]}
         res = jsonata.Jsonata("[$map(data, function($row) { $row[1] })]").evaluate(data)
         assert res == [None, None, None]
+
+    def test_output_convert_nulls(self):
+        j = jsonata.Jsonata("$")
+        j2 = jsonata.Jsonata("$")
+        j2.set_output_convert_nulls(False)
+
+        assert j.is_output_convert_nulls() is True
+        assert j2.is_output_convert_nulls() is False
+
+        res = j.evaluate(jsonata.Utils.NULL_VALUE)
+        res2 = j2.evaluate(jsonata.Utils.NULL_VALUE)
+        assert res is None
+        assert res2 is jsonata.Utils.NULL_VALUE
+
+        res = j.evaluate(None)
+        res2 = j2.evaluate(None)
+        assert res is None
+        assert res2 is None
+
+    def test_python_null_vs_undefined(self):
+        test = JsonataTestRunner()
+
+        assert test.run_test_case("test-undefined", {
+            "expr": "undefined",
+            "undefinedResult": True,
+        })
+
+        assert test.run_test_case("test-null", {
+            "expr": "null",
+            "result": None,
+        })
+
+        # raw vs cooked, returning null or undefined
+        res = _execute_jsonata_raw("null", None)
+        assert res is jsonata.Utils.NULL_VALUE
+
+        res = _execute_jsonata_raw("$", jsonata.Utils.NULL_VALUE)
+        assert res is jsonata.Utils.NULL_VALUE
+
+        res = _execute_jsonata_raw("$", None)
+        assert res is None
+
+        res = _execute_jsonata_raw("no_match", None)
+        assert res is None
+
+        res = evaluate_jsonata("null", None, None)
+        assert res is None
+
+        res = evaluate_jsonata("no_match", None, None)
+        assert res == UNDEFINED
+
+        res = evaluate_jsonata("{}.a", None, None)
+        assert res == UNDEFINED
+
+        res = _execute_jsonata_raw('{"a":null}.a', None)
+        assert res is jsonata.Utils.NULL_VALUE
+
+        res = evaluate_jsonata('{"a":null}.a', None, None)
+        assert res is None
+
+        res = _execute_jsonata_raw('{"a":null}.b', None)
+        assert res is None
+
+        res = evaluate_jsonata('{"a":null}.b', None, None)
+        assert res == UNDEFINED
+
+        res = evaluate_jsonata("[a,null,b][0]", None, None)
+        assert res is None
+
+        res = _execute_jsonata_raw("$[1]", [42, jsonata.Utils.NULL_VALUE])
+        assert res is jsonata.Utils.NULL_VALUE
+
+        res = _execute_jsonata_raw("$[2]", [42, jsonata.Utils.NULL_VALUE])
+        assert res is None
+
+        res = evaluate_jsonata("$[2]", [42, jsonata.Utils.NULL_VALUE], None)
+        assert res == UNDEFINED
+
+        res = jsonata.Jsonata("$").evaluate(jsonata.Utils.NULL_VALUE)
+        assert res is None
+
+        res = _execute_jsonata_raw("{'a':$}", jsonata.Utils.NULL_VALUE)
+        assert res["a"] is jsonata.Utils.NULL_VALUE
+
+        res = _execute_jsonata_raw("{'a':$}", None)
+        assert res == {}
+
+        res = _execute_jsonata_raw("{'a':{'b':$}}", None)
+        assert res == {"a": {}}
+
+        res = _execute_jsonata_raw("[$]", [jsonata.Utils.NULL_VALUE, jsonata.Utils.NULL_VALUE])
+        assert res == [jsonata.Utils.NULL_VALUE, jsonata.Utils.NULL_VALUE]
+
+        res = evaluate_jsonata("[$]", [jsonata.Utils.NULL_VALUE, jsonata.Utils.NULL_VALUE], None)
+        assert res == [None, None]