fix: serialize NaT in NumPy datetime64 arrays as JSON null

CHANGELOG.md

@@ -5,6 +5,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ## Unreleased
 
 ### Fixed
+- Serialize missing values in NumPy `datetime64` arrays as JSON `null` rather than the string `"NaT"` with both JSON engines ([#5829](https://github.com/plotly/plotly.py/issues/5829)).
 - Fix `mpl_to_plotly` tick marker mirroring, axis positioning, and trace visibility for twinned and overlaid subplots (such as `twinx` and `twiny`) [[#5310](https://github.com/plotly/plotly.py/pull/5310)], with thanks to @robertoffmoura for the contribution!
 - Fix `mpl_to_plotly` losing matplotlib axis line colors by exporting the color of each axis's displayed spine to the plotly x and y axis `linecolor` [[#5311](https://github.com/plotly/plotly.py/pull/5311)], with thanks to @robertoffmoura for the contribution!
 - Fix concurrent first access to lazily initialized graph object properties, which could raise `ValueError("Invalid value")` [[#5691](https://github.com/plotly/plotly.py/pull/5691)], with thanks to @hb1915 for the contribution!

_plotly_utils/utils.py

@@ -294,7 +294,9 @@ def encode_as_numpy(obj):
             return float("nan")
         elif isinstance(obj, numpy.ndarray) and obj.dtype.kind == "M":
             try:
-                return numpy.datetime_as_string(obj).tolist()
+                return numpy.where(
+                    numpy.isnat(obj), None, numpy.datetime_as_string(obj)
+                ).tolist()
             except TypeError:
                 pass
 

plotly/io/_json.py

@@ -74,6 +74,19 @@ def _safe(json_str, _swap):
     return out
 
 
+def _has_nat_datetime_array(obj, np):
+    """Return whether nested figure data contain missing NumPy datetimes."""
+    if np is None:
+        return False
+    if isinstance(obj, np.ndarray):
+        return obj.dtype.kind == "M" and bool(np.isnat(obj).any())
+    if isinstance(obj, dict):
+        return any(_has_nat_datetime_array(value, np) for value in obj.values())
+    if isinstance(obj, (list, tuple)):
+        return any(_has_nat_datetime_array(value, np) for value in obj)
+    return False
+
+
 def to_json_plotly(plotly_object, pretty=False, engine=None):
     """
     Convert a plotly/Dash object to a JSON string representation
@@ -153,13 +166,16 @@ def to_json_plotly(plotly_object, pretty=False, engine=None):
         except AttributeError:
             pass
 
-        # Try without cleaning
-        try:
-            return _safe(
-                orjson.dumps(plotly_object, option=opts).decode("utf8"), _swap_orjson
-            )
-        except TypeError:
-            pass
+        # The fast NumPy serializer can emit "NaT" or a spurious date for NaT.
+        # Normalize only when needed, preserving the fast path otherwise.
+        if not _has_nat_datetime_array(plotly_object, modules["np"]):
+            try:
+                return _safe(
+                    orjson.dumps(plotly_object, option=opts).decode("utf8"),
+                    _swap_orjson,
+                )
+            except TypeError:
+                pass
 
         cleaned = clean_to_json_compatible(
             plotly_object,
@@ -514,7 +530,9 @@ def clean_to_json_compatible(obj, **kwargs):
                 return np.ascontiguousarray(obj)
             elif obj.dtype.kind == "M":
                 # datetime64 array
-                return np.datetime_as_string(obj).tolist()
+                return np.where(
+                    np.isnat(obj), None, np.datetime_as_string(obj)
+                ).tolist()
             elif obj.dtype.kind == "U":
                 return obj.tolist()
             elif obj.dtype.kind == "O":

tests/test_io/test_to_from_plotly_json.py

@@ -266,3 +266,36 @@ def test_sanitize_json(engine):
 def test_na_values(engine, na_value):
     result = pio.to_json_plotly(na_value, engine=engine)
     assert result == "null"
+
+
+@pytest.mark.parametrize("unit", ["D", "us", "ns"])
+@pytest.mark.parametrize("two_dimensional", [False, True])
+def test_datetime64_array_nat_is_json_null(engine, unit, two_dimensional):
+    dates = np.array(["2020-01-01", "NaT", "2020-01-03"], dtype=f"datetime64[{unit}]")
+    if two_dimensional:
+        dates = dates.reshape(1, 3)
+
+    result = json.loads(pio.to_json_plotly(dates, engine=engine))
+    items = result[0] if two_dimensional else result
+    assert items[0].startswith("2020-01-01")
+    assert items[1] is None
+    assert items[2].startswith("2020-01-03")
+
+
+@pytest.mark.parametrize("unit", ["D", "us", "ns"])
+def test_zero_dimensional_nat_array_is_json_null(engine, unit):
+    value = np.array("NaT", dtype=f"datetime64[{unit}]")
+    assert json.loads(pio.to_json_plotly(value, engine=engine)) is None
+
+
+def test_pandas_datetime_series_nat_roundtrip(engine):
+    series = pd.Series(pd.to_datetime(["2020-01-01", None]))
+    fig = go.Figure(go.Scatter(x=series))
+    encoded = pio.to_json(fig, engine=engine)
+
+    values = json.loads(encoded)["data"][0]["x"]
+    assert values[0].startswith("2020-01-01")
+    assert values[1] is None
+
+    decoded = pio.from_json(encoded, engine=engine)
+    assert decoded.data[0].x[1] is None