diff --git a/sparsediffpy/_compile.py b/sparsediffpy/_compile.py index 710b0d1..3c42341 100644 --- a/sparsediffpy/_compile.py +++ b/sparsediffpy/_compile.py @@ -11,7 +11,8 @@ from sparsediffpy import _sparsediffengine as _C from sparsediffpy._expression import ( - Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, cast_to_const, + Constant, Parameter, Variable, _check_size, _flatten_shaped, _flatten_vector, + cast_to_const, ) @@ -74,6 +75,11 @@ def _build_one(self, e): cap = _C.make_parameter(d1, d2, e._offset, self.n_vars, values) self.params.append((e, cap)) elif isinstance(e, Constant): + # Only a constant used as an ordinary operand (e.g. sum(A) or A * x) + # gets here, and it becomes a dense engine parameter even if its + # value is sparse. The matrix of a matmul or quad_form does not: it + # is passed to its node as CSR. + _check_size(e.shape) values = e.dense().ravel(order="F") cap = _C.make_parameter(d1, d2, -1, self.n_vars, values) else: diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py index b178623..17937a6 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/_expression.py @@ -38,6 +38,14 @@ def _check_size(shape): f"the engine supports at most {_MAX_SIZE}") +def _check_sparse_size(shape, nnz): + """Raise if a sparse matrix has more rows, columns or nonzeros than that.""" + if max(*shape, nnz) > _MAX_SIZE: + raise ValueError(f"Sparse matrix of shape {shape} with {nnz} nonzeros; " + f"the engine supports at most {_MAX_SIZE} rows, " + f"columns and nonzeros") + + class Expression: """Base class for all expressions. Supports + - * @ ** [] and .T.""" @@ -117,13 +125,22 @@ def __init__(self, make, args, shape): class Constant(Expression): - """Fixed dense or scipy.sparse data.""" + """Fixed dense or scipy.sparse data. + + A sparse value is stored as CSR, so it may have more than INT_MAX entries + as long as its nonzeros fit. It can then only be the matrix of a matmul or + quad_form; any other use needs it dense and fails when compiled. + """ def __init__(self, value): if value is None: raise TypeError("Constant value cannot be None") if sps.issparse(value): self.value = sps.csr_matrix(value, dtype=np.float64) + self.shape = tuple(int(d) for d in self.value.shape) + if 0 in self.shape: + raise ValueError(f"Shape dimensions must be positive, got {self.shape}") + _check_sparse_size(self.shape, self.value.nnz) else: value = np.asarray(value, dtype=np.float64) if value.size == 1: @@ -133,7 +150,7 @@ def __init__(self, value): f"Constants must be scalars or 2-D arrays, (n, 1) or (1, n); " f"got shape {value.shape}. Use a[:, None] or a[None, :].") self.value = value - self.shape = _check_shape(*self.value.shape) + self.shape = _check_shape(*self.value.shape) def dense(self): """The value as a dense array.""" diff --git a/tests/test_api.py b/tests/test_api.py index 6c161e9..fad9dfe 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -4,6 +4,7 @@ import numpy as np import pytest +import scipy.sparse as sps import sparsediffpy as sp from sparsediffpy import _compile @@ -84,6 +85,25 @@ def test_too_many_entries(scope): assert sp.broadcast(x, (2**31 - 1, 1)).size == 2**31 - 1 +def test_large_sparse_constant(scope): + """A sparse constant is limited by its nonzeros, not by rows times columns.""" + m, n = 65536, 65537 # 2**32 + 65536 entries + rows, cols = np.array([0, 1, m - 1]), np.array([n - 1, 0, 2]) + A = sps.csr_matrix((np.array([1.0, 2.0, 3.0]), (rows, cols)), (m, n)) + x = scope.Variable(n, 1) + x.value = np.arange(n, dtype=float) + fn = sp.compile(A @ x) + np.testing.assert_array_equal(fn.forward().ravel(), A @ np.arange(n)) + assert abs(fn.jacobian() - A).max() == 0 + row = scope.Variable(1, m) + assert abs(sp.compile(row @ A).jacobian()[:, n:] - A.T).max() == 0 + # any other use would need the matrix dense + with pytest.raises(ValueError, match="at most"): + sp.compile(sp.sum(A) * x) + with pytest.raises(ValueError, match="at most"): + A * scope.Variable(1, 1) + + def test_reevaluates_at_new_values(scope): x = scope.Variable(2, 1) fn = sp.compile(sp.exp(x))