diff --git a/CHANGELOG.md b/CHANGELOG.md index c92c3ea6d6..e6a7c24cc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased ### Fixed +- Serialize missing values in NumPy `datetime64` arrays as JSON `null` rather than the string `"NaT"` with both JSON engines ([#5829](https://github.com/plotly/plotly.py/issues/5829)). - Fix `mpl_to_plotly` tick marker mirroring, axis positioning, and trace visibility for twinned and overlaid subplots (such as `twinx` and `twiny`) [[#5310](https://github.com/plotly/plotly.py/pull/5310)], with thanks to @robertoffmoura for the contribution! - Fix `mpl_to_plotly` losing matplotlib axis line colors by exporting the color of each axis's displayed spine to the plotly x and y axis `linecolor` [[#5311](https://github.com/plotly/plotly.py/pull/5311)], with thanks to @robertoffmoura for the contribution! - Fix concurrent first access to lazily initialized graph object properties, which could raise `ValueError("Invalid value")` [[#5691](https://github.com/plotly/plotly.py/pull/5691)], with thanks to @hb1915 for the contribution! diff --git a/_plotly_utils/utils.py b/_plotly_utils/utils.py index 07a2b9e0b9..5cc43be109 100644 --- a/_plotly_utils/utils.py +++ b/_plotly_utils/utils.py @@ -294,7 +294,9 @@ def encode_as_numpy(obj): return float("nan") elif isinstance(obj, numpy.ndarray) and obj.dtype.kind == "M": try: - return numpy.datetime_as_string(obj).tolist() + return numpy.where( + numpy.isnat(obj), None, numpy.datetime_as_string(obj) + ).tolist() except TypeError: pass diff --git a/plotly/io/_json.py b/plotly/io/_json.py index 3e779f5cac..960ed43fd1 100644 --- a/plotly/io/_json.py +++ b/plotly/io/_json.py @@ -74,6 +74,19 @@ def _safe(json_str, _swap): return out +def _has_nat_datetime_array(obj, np): + """Return whether nested figure data contain missing NumPy datetimes.""" + if np is None: + return False + if isinstance(obj, np.ndarray): + return obj.dtype.kind == "M" and bool(np.isnat(obj).any()) + if isinstance(obj, dict): + return any(_has_nat_datetime_array(value, np) for value in obj.values()) + if isinstance(obj, (list, tuple)): + return any(_has_nat_datetime_array(value, np) for value in obj) + return False + + def to_json_plotly(plotly_object, pretty=False, engine=None): """ Convert a plotly/Dash object to a JSON string representation @@ -153,13 +166,16 @@ def to_json_plotly(plotly_object, pretty=False, engine=None): except AttributeError: pass - # Try without cleaning - try: - return _safe( - orjson.dumps(plotly_object, option=opts).decode("utf8"), _swap_orjson - ) - except TypeError: - pass + # The fast NumPy serializer can emit "NaT" or a spurious date for NaT. + # Normalize only when needed, preserving the fast path otherwise. + if not _has_nat_datetime_array(plotly_object, modules["np"]): + try: + return _safe( + orjson.dumps(plotly_object, option=opts).decode("utf8"), + _swap_orjson, + ) + except TypeError: + pass cleaned = clean_to_json_compatible( plotly_object, @@ -514,7 +530,9 @@ def clean_to_json_compatible(obj, **kwargs): return np.ascontiguousarray(obj) elif obj.dtype.kind == "M": # datetime64 array - return np.datetime_as_string(obj).tolist() + return np.where( + np.isnat(obj), None, np.datetime_as_string(obj) + ).tolist() elif obj.dtype.kind == "U": return obj.tolist() elif obj.dtype.kind == "O": diff --git a/tests/test_io/test_to_from_plotly_json.py b/tests/test_io/test_to_from_plotly_json.py index f8c5a71ee4..cdbb3fb0da 100644 --- a/tests/test_io/test_to_from_plotly_json.py +++ b/tests/test_io/test_to_from_plotly_json.py @@ -266,3 +266,36 @@ def test_sanitize_json(engine): def test_na_values(engine, na_value): result = pio.to_json_plotly(na_value, engine=engine) assert result == "null" + + +@pytest.mark.parametrize("unit", ["D", "us", "ns"]) +@pytest.mark.parametrize("two_dimensional", [False, True]) +def test_datetime64_array_nat_is_json_null(engine, unit, two_dimensional): + dates = np.array(["2020-01-01", "NaT", "2020-01-03"], dtype=f"datetime64[{unit}]") + if two_dimensional: + dates = dates.reshape(1, 3) + + result = json.loads(pio.to_json_plotly(dates, engine=engine)) + items = result[0] if two_dimensional else result + assert items[0].startswith("2020-01-01") + assert items[1] is None + assert items[2].startswith("2020-01-03") + + +@pytest.mark.parametrize("unit", ["D", "us", "ns"]) +def test_zero_dimensional_nat_array_is_json_null(engine, unit): + value = np.array("NaT", dtype=f"datetime64[{unit}]") + assert json.loads(pio.to_json_plotly(value, engine=engine)) is None + + +def test_pandas_datetime_series_nat_roundtrip(engine): + series = pd.Series(pd.to_datetime(["2020-01-01", None])) + fig = go.Figure(go.Scatter(x=series)) + encoded = pio.to_json(fig, engine=engine) + + values = json.loads(encoded)["data"][0]["x"] + assert values[0].startswith("2020-01-01") + assert values[1] is None + + decoded = pio.from_json(encoded, engine=engine) + assert decoded.data[0].x[1] is None