From 64f8f520c5ecf21d883155f0dd64c7403719bf46 Mon Sep 17 00:00:00 2001 From: dance858 Date: Sun, 15 Nov 2026 06:49:56 -0800 Subject: [PATCH 1/2] Limit sparse constants by nonzeros, not rows times columns A scipy.sparse constant is passed to the engine as CSR, so only its dimensions and nonzero count must fit in a C int. The check from #38 rejected it by rows times columns, which ruled out e.g. the 126015 x 78478 branch incidence matrix of a 78k-bus power network. Such a constant can be the matrix of a matmul or quad_form; any use that needs it dense is still rejected, now when the expression is compiled. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_012aJMgvtfPYcdJMWR6ox56t --- sparsediffpy/_compile.py | 4 +++- sparsediffpy/_expression.py | 21 +++++++++++++++++++-- tests/test_api.py | 20 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/sparsediffpy/_compile.py b/sparsediffpy/_compile.py index 710b0d1..9a66762 100644 --- a/sparsediffpy/_compile.py +++ b/sparsediffpy/_compile.py @@ -11,7 +11,8 @@ from sparsediffpy import _sparsediffengine as _C from sparsediffpy._expression import ( - Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, cast_to_const, + Constant, Parameter, Variable, _check_size, _flatten_shaped, _flatten_vector, + cast_to_const, ) @@ -74,6 +75,7 @@ def _build_one(self, e): cap = _C.make_parameter(d1, d2, e._offset, self.n_vars, values) self.params.append((e, cap)) elif isinstance(e, Constant): + _check_size(e.shape) # a sparse constant may be too large to densify values = e.dense().ravel(order="F") cap = _C.make_parameter(d1, d2, -1, self.n_vars, values) else: diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py index b178623..17937a6 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/_expression.py @@ -38,6 +38,14 @@ def _check_size(shape): f"the engine supports at most {_MAX_SIZE}") +def _check_sparse_size(shape, nnz): + """Raise if a sparse matrix has more rows, columns or nonzeros than that.""" + if max(*shape, nnz) > _MAX_SIZE: + raise ValueError(f"Sparse matrix of shape {shape} with {nnz} nonzeros; " + f"the engine supports at most {_MAX_SIZE} rows, " + f"columns and nonzeros") + + class Expression: """Base class for all expressions. Supports + - * @ ** [] and .T.""" @@ -117,13 +125,22 @@ def __init__(self, make, args, shape): class Constant(Expression): - """Fixed dense or scipy.sparse data.""" + """Fixed dense or scipy.sparse data. + + A sparse value is stored as CSR, so it may have more than INT_MAX entries + as long as its nonzeros fit. It can then only be the matrix of a matmul or + quad_form; any other use needs it dense and fails when compiled. + """ def __init__(self, value): if value is None: raise TypeError("Constant value cannot be None") if sps.issparse(value): self.value = sps.csr_matrix(value, dtype=np.float64) + self.shape = tuple(int(d) for d in self.value.shape) + if 0 in self.shape: + raise ValueError(f"Shape dimensions must be positive, got {self.shape}") + _check_sparse_size(self.shape, self.value.nnz) else: value = np.asarray(value, dtype=np.float64) if value.size == 1: @@ -133,7 +150,7 @@ def __init__(self, value): f"Constants must be scalars or 2-D arrays, (n, 1) or (1, n); " f"got shape {value.shape}. Use a[:, None] or a[None, :].") self.value = value - self.shape = _check_shape(*self.value.shape) + self.shape = _check_shape(*self.value.shape) def dense(self): """The value as a dense array.""" diff --git a/tests/test_api.py b/tests/test_api.py index 6c161e9..fad9dfe 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -4,6 +4,7 @@ import numpy as np import pytest +import scipy.sparse as sps import sparsediffpy as sp from sparsediffpy import _compile @@ -84,6 +85,25 @@ def test_too_many_entries(scope): assert sp.broadcast(x, (2**31 - 1, 1)).size == 2**31 - 1 +def test_large_sparse_constant(scope): + """A sparse constant is limited by its nonzeros, not by rows times columns.""" + m, n = 65536, 65537 # 2**32 + 65536 entries + rows, cols = np.array([0, 1, m - 1]), np.array([n - 1, 0, 2]) + A = sps.csr_matrix((np.array([1.0, 2.0, 3.0]), (rows, cols)), (m, n)) + x = scope.Variable(n, 1) + x.value = np.arange(n, dtype=float) + fn = sp.compile(A @ x) + np.testing.assert_array_equal(fn.forward().ravel(), A @ np.arange(n)) + assert abs(fn.jacobian() - A).max() == 0 + row = scope.Variable(1, m) + assert abs(sp.compile(row @ A).jacobian()[:, n:] - A.T).max() == 0 + # any other use would need the matrix dense + with pytest.raises(ValueError, match="at most"): + sp.compile(sp.sum(A) * x) + with pytest.raises(ValueError, match="at most"): + A * scope.Variable(1, 1) + + def test_reevaluates_at_new_values(scope): x = scope.Variable(2, 1) fn = sp.compile(sp.exp(x)) From 25b430010013c03d93589dce728171c0fa18e155 Mon Sep 17 00:00:00 2001 From: dance858 Date: Sun, 15 Nov 2026 07:25:44 -0800 Subject: [PATCH 2/2] Explain when a constant becomes a dense engine parameter Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_012aJMgvtfPYcdJMWR6ox56t --- sparsediffpy/_compile.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sparsediffpy/_compile.py b/sparsediffpy/_compile.py index 9a66762..3c42341 100644 --- a/sparsediffpy/_compile.py +++ b/sparsediffpy/_compile.py @@ -75,7 +75,11 @@ def _build_one(self, e): cap = _C.make_parameter(d1, d2, e._offset, self.n_vars, values) self.params.append((e, cap)) elif isinstance(e, Constant): - _check_size(e.shape) # a sparse constant may be too large to densify + # Only a constant used as an ordinary operand (e.g. sum(A) or A * x) + # gets here, and it becomes a dense engine parameter even if its + # value is sparse. The matrix of a matmul or quad_form does not: it + # is passed to its node as CSR. + _check_size(e.shape) values = e.dense().ravel(order="F") cap = _C.make_parameter(d1, d2, -1, self.n_vars, values) else: