From 3afeb0eb58af59fa5605f52bd3785f4411640f09 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sat, 7 Nov 2026 14:36:21 -0800 Subject: [PATCH 01/17] Add a Python expression API (Scope, atoms, compile, Problem) Reworks the interface from #12 against current main: - Generic Node(make, args, shape) replaces ~30 node classes, the registry and the dispatch module; sp.* atoms are mostly one-liners. - compile(f) is built on the existing problem bindings (f as the only constraint, zero objective), so no new C code; _bindings/ is unchanged. - Parameters sync to the engine automatically when their values change. - Restricted-domain atoms validate that their arguments are plain Variables, matching the engine's requirements. - Tests check every atom's value, Jacobian and Hessian against NumPy and finite differences, and run the README examples. They run in the cibuildwheel step. - scipy becomes a runtime dependency (results are csr_matrix). Co-authored-by: dance858 Co-Authored-By: Claude Opus 5.5 --- README.md | 98 +++++++++++- conda.recipe/recipe.yaml | 1 + pyproject.toml | 6 +- sparsediffpy/__init__.py | 9 ++ sparsediffpy/_atoms.py | 170 ++++++++++++++++++++ sparsediffpy/_compile.py | 168 ++++++++++++++++++++ sparsediffpy/_expression.py | 307 ++++++++++++++++++++++++++++++++++++ tests/conftest.py | 39 +++++ tests/test_api.py | 182 +++++++++++++++++++++ tests/test_atoms.py | 228 ++++++++++++++++++++++++++ tests/test_readme.py | 14 ++ 11 files changed, 1218 insertions(+), 4 deletions(-) create mode 100644 sparsediffpy/_atoms.py create mode 100644 sparsediffpy/_compile.py create mode 100644 sparsediffpy/_expression.py create mode 100644 tests/conftest.py create mode 100644 tests/test_api.py create mode 100644 tests/test_atoms.py create mode 100644 tests/test_readme.py diff --git a/README.md b/README.md index 3a02ca6..c8de887 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # SparseDiffPy -Python bindings for [SparseDiffEngine](https://github.com/SparseDifferentiation/SparseDiffEngine), a C library for computing sparse Jacobians and Hessians. +Sparse Jacobians and Hessians of nonlinear expressions, computed by +[SparseDiffEngine](https://github.com/SparseDifferentiation/SparseDiffEngine), +a C library that works out the sparsity pattern once and then only computes +the structural nonzeros. Results are `scipy.sparse.csr_matrix`. ## Installation @@ -8,12 +11,101 @@ Python bindings for [SparseDiffEngine](https://github.com/SparseDifferentiation/ pip install sparsediffpy ``` -## Usage +From source: `git clone --recursive https://github.com/SparseDifferentiation/SparseDiffPy && pip install -e SparseDiffPy`. + +## Quick start + +```python +import numpy as np +import sparsediffpy as sp + +scope = sp.Scope() # owns the flat variable vector +x = scope.Variable(3, 1) # shapes are always 2-D: (rows, cols) + +f = sp.exp(x) + 2.0 * x # build an expression with operators and sp.* atoms +fn = sp.compile(f) # analyze sparsity once + +x.value = [1.0, 2.0, 3.0] +print(fn.forward()) # [ 4.718 11.389 26.086] +print(fn.jacobian().toarray()) # diag(exp(x) + 2), a 3x3 csr_matrix +print(fn.hessian(np.ones(3))) # Hessian of w' f(x) for weights w +``` + +`jacobian()` has shape `(f.size, n_vars)` and `hessian(w)` has shape +`(n_vars, n_vars)`, where `n_vars` is the total size of all variables in the +scope, in declaration order. Values are flattened column-major. + +**Parameters** are data you can change without recompiling: ```python -from sparsediffpy import _sparsediffengine +import numpy as np +import sparsediffpy as sp + +scope = sp.Scope() +x = scope.Variable(3, 1) +A = scope.Parameter(2, 3, value=np.ones((2, 3))) +fn = sp.compile(sp.sin(A @ x)) + +x.value = [0.1, 0.2, 0.3] +J1 = fn.jacobian() +A.value = np.arange(6.0).reshape(2, 3) # no recompile needed +J2 = fn.jacobian() ``` +## Using it in a solver + +`sp.Problem(objective, constraints)` exposes the callbacks NLP solvers ask for: +objective value and gradient, constraint values and Jacobian, and the Lagrangian +Hessian `obj_factor * H_f + sum_i lagrange_i * H_gi`. Here it drives SciPy: + +```python +import numpy as np +from scipy.optimize import NonlinearConstraint, minimize + +import sparsediffpy as sp + +# minimize sum(exp(x) - c * x) subject to ||x||^2 <= 1 +scope = sp.Scope() +x = scope.Variable(4, 1) +c = np.array([1.0, 2.0, 3.0, 4.0]) +prob = sp.Problem(sp.sum(sp.exp(x) - c * x), [sp.sum(x ** 2)]) + +def grad(u): + prob.objective_forward(u) + return prob.gradient() + +def obj_hess(u): + prob.objective_forward(u) + return prob.hessian(1.0, [0.0]) + +def con_jac(u): + prob.constraint_forward(u) + return prob.jacobian() + +def con_hess(u, v): + prob.constraint_forward(u) + return prob.hessian(0.0, v) + +con = NonlinearConstraint(prob.constraint_forward, -np.inf, 1.0, + jac=con_jac, hess=con_hess) +res = minimize(prob.objective_forward, np.zeros(4), jac=grad, hess=obj_hess, + method="trust-constr", constraints=con) +print(res.x) # approx. [0. 0.289 0.549 0.784], on the boundary ||x|| = 1 +``` + +## Supported operations + +- **Operators:** `+ - * @ **`, `.T`, and NumPy-style indexing (`x[1:3]`, `X[0, :]`, `X[[0, 2], 1]`). + Scalars, rows and columns broadcast. NumPy arrays and `scipy.sparse` matrices are accepted as constants. +- **Elementwise:** `sin cos tan exp log sinh tanh asinh atan atanh logistic normal_cdf entr xexp power atan2`. +- **Affine:** `sum prod reshape broadcast hstack vstack trace diag_vec index_flat`. +- **Other:** `quad_form quad_over_lin rel_entr`. + +`log`, `tan`, `atanh`, `entr`, `atan2`, `rel_entr`, `quad_over_lin` and `prod` +currently need plain Variables as arguments (an engine limitation). + +The low-level C bindings remain available as `sparsediffpy._sparsediffengine`. + ## Free-threaded Python Wheels are published for free-threaded CPython 3.14 (`cp314t`) as well as the diff --git a/conda.recipe/recipe.yaml b/conda.recipe/recipe.yaml index 70394b3..47b66da 100644 --- a/conda.recipe/recipe.yaml +++ b/conda.recipe/recipe.yaml @@ -56,6 +56,7 @@ requirements: run: - python - numpy >=2 + - scipy tests: - python: diff --git a/pyproject.toml b/pyproject.toml index 20133b0..d2ce46e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "sparsediffpy" version = "0.9.0" description = "Python bindings for SparseDiffEngine algorithmic differentiation" requires-python = ">=3.11" -dependencies = ["numpy >= 2.0.0"] +dependencies = ["numpy >= 2.0.0", "scipy"] license = "Apache-2.0" [project.readme] @@ -22,3 +22,7 @@ wheel.packages = ["sparsediffpy"] # Persistent build dir: avoids scikit-build-core's TemporaryDirectory, whose cleanup # intermittently fails on Windows CI with WinError 32 (lingering mspdbsrv/AV handle). build-dir = "build/{wheel_tag}" + +[tool.cibuildwheel] +test-requires = ["pytest"] +test-command = "pytest {project}/tests -q" diff --git a/sparsediffpy/__init__.py b/sparsediffpy/__init__.py index 9b73c36..08995f9 100644 --- a/sparsediffpy/__init__.py +++ b/sparsediffpy/__init__.py @@ -15,3 +15,12 @@ """ from sparsediffpy import _sparsediffengine # noqa: F401 +from sparsediffpy._expression import ( # noqa: F401 + DimensionError, Expression, Parameter, Scope, Variable, broadcast, index_flat, +) +from sparsediffpy._atoms import ( # noqa: F401 + asinh, atan, atan2, atanh, cos, diag_vec, entr, exp, hstack, log, logistic, + normal_cdf, power, prod, quad_form, quad_over_lin, rel_entr, reshape, sin, + sinh, sum, tan, tanh, trace, vstack, xexp, +) +from sparsediffpy._compile import CompiledExpression, Problem, compile # noqa: F401 diff --git a/sparsediffpy/_atoms.py b/sparsediffpy/_atoms.py new file mode 100644 index 0000000..16ba8d6 --- /dev/null +++ b/sparsediffpy/_atoms.py @@ -0,0 +1,170 @@ +"""Named functions: elementwise atoms, reductions, stacking, and special forms.""" + +import numpy as np +import scipy.sparse as sps + +from sparsediffpy import _sparsediffengine as _C +from sparsediffpy._expression import ( + Constant, Node, Parameter, Variable, _check_shape, _wrap, +) + + +def _require_variables(name, *xs): + """Restricted-domain atoms in the engine only accept plain Variables.""" + if not all(isinstance(x, Variable) for x in xs): + raise ValueError(f"{name} cannot be applied directly to a composite " + "expression; its arguments must be plain Variables") + if len(xs) == 2 and xs[0] is xs[1]: + raise ValueError(f"{name}: arguments must be distinct Variables") + + +def _elementwise(make, name, restricted=False): + def atom(x): + x = _wrap(x) + if restricted: + _require_variables(name, x) + return Node(make, [x], x.shape) + atom.__name__ = atom.__qualname__ = name + atom.__doc__ = f"Elementwise {name}." + return atom + + +sin = _elementwise(_C.make_sin, "sin") +cos = _elementwise(_C.make_cos, "cos") +tan = _elementwise(_C.make_tan, "tan", restricted=True) +exp = _elementwise(_C.make_exp, "exp") +log = _elementwise(_C.make_log, "log", restricted=True) +sinh = _elementwise(_C.make_sinh, "sinh") +tanh = _elementwise(_C.make_tanh, "tanh") +asinh = _elementwise(_C.make_asinh, "asinh") +atanh = _elementwise(_C.make_atanh, "atanh", restricted=True) +atan = _elementwise(_C.make_atan, "atan") +logistic = _elementwise(_C.make_logistic, "logistic") +normal_cdf = _elementwise(_C.make_normal_cdf, "normal_cdf") +entr = _elementwise(_C.make_entr, "entr", restricted=True) +xexp = _elementwise(_C.make_xexp, "xexp") + + +def power(x, p): + """Elementwise x ** p for a constant exponent p.""" + return _wrap(x) ** p + + +def atan2(y, x): + """Elementwise atan2(y, x) for distinct Variables of the same shape.""" + _require_variables("atan2", y, x) + if y.shape != x.shape: + raise ValueError(f"atan2: shape mismatch {y.shape} vs {x.shape}") + return Node(_C.make_atan2, [y, x], y.shape) + + +def _reduction_shape(x, axis): + """Output shape of sum/prod. The engine always returns a row vector.""" + d1, d2 = x.shape + shapes = {None: (1, 1), 0: (1, d2), 1: (1, d1)} + if axis not in shapes: + raise ValueError(f"Invalid axis {axis}, must be None, 0, or 1") + return shapes[axis] + + +def sum(x, axis=None): + """Sum of all entries (axis=None) -> (1, 1); axis=0 -> (1, d2); axis=1 -> (1, d1).""" + x = _wrap(x) + shape = _reduction_shape(x, axis) + c_axis = -1 if axis is None else axis + return Node(lambda c: _C.make_sum(c, c_axis), [x], shape) + + +def prod(x, axis=None): + """Product of entries; shapes as for :func:`sum`. ``x`` must be a Variable.""" + if not isinstance(x, Variable): + raise ValueError("prod requires its argument to be a plain Variable " + "(the engine has no chain rule for prod yet)") + shape = _reduction_shape(x, axis) + make = {None: _C.make_prod, 0: _C.make_prod_axis_zero, + 1: _C.make_prod_axis_one}[axis] + return Node(make, [x], shape) + + +def reshape(x, d1, d2): + """Column-major reshape.""" + x, shape = _wrap(x), _check_shape(d1, d2) + if x.size != shape[0] * shape[1]: + raise ValueError(f"Cannot reshape {x.shape} to {shape}") + return Node(lambda c: _C.make_reshape(c, *shape), [x], shape) + + +def trace(x): + x = _wrap(x) + if x.shape[0] != x.shape[1]: + raise ValueError(f"trace requires a square matrix, got {x.shape}") + return Node(_C.make_trace, [x], (1, 1)) + + +def diag_vec(x): + """(n, 1) column vector -> (n, n) diagonal matrix.""" + x = _wrap(x) + if x.shape[1] != 1: + raise ValueError(f"diag_vec requires a column vector, got {x.shape}") + return Node(_C.make_diag_vec, [x], (x.shape[0], x.shape[0])) + + +def hstack(exprs): + """Concatenate horizontally; all arguments need the same number of rows.""" + exprs = [_wrap(e) for e in exprs] + if not exprs or len({e.shape[0] for e in exprs}) != 1: + raise ValueError(f"hstack: need >=1 args with equal rows, got " + f"{[e.shape for e in exprs]}") + shape = (exprs[0].shape[0], np.sum([e.shape[1] for e in exprs]).item()) + return Node(lambda *caps: _C.make_hstack(list(caps)), exprs, shape) + + +def vstack(exprs): + """Concatenate vertically; all arguments need the same number of columns.""" + exprs = [_wrap(e) for e in exprs] + if not exprs or len({e.shape[1] for e in exprs}) != 1: + raise ValueError(f"vstack: need >=1 args with equal columns, got " + f"{[e.shape for e in exprs]}") + shape = (np.sum([e.shape[0] for e in exprs]).item(), exprs[0].shape[1]) + return Node(lambda *caps: _C.make_vstack(list(caps)), exprs, shape) + + +def quad_form(x, Q): + """x' Q x for a column vector x (n, 1) and a constant or Parameter Q (n, n).""" + x, Q = _wrap(x), _wrap(Q) + n = x.shape[0] + if x.shape[1] != 1 or Q.shape != (n, n): + raise ValueError(f"quad_form: need x (n, 1) and Q (n, n), " + f"got {x.shape} and {Q.shape}") + if isinstance(Q, Parameter): + return Node(lambda p, c: _C.make_quad_form(p, c, "dense", None, n), + [Q, x], (1, 1)) + if not isinstance(Q, Constant): + raise TypeError("quad_form: Q must be a constant or a Parameter") + V = sps.csr_matrix(Q.value) + return Node(lambda c: _C.make_quad_form( + None, c, "sparse", V.data, V.indices.astype(np.int32), + V.indptr.astype(np.int32), n, n), [x], (1, 1)) + + +def quad_over_lin(x, z): + """sum(x**2) / z for distinct Variables x and scalar z.""" + _require_variables("quad_over_lin", x, z) + if z.shape != (1, 1): + raise ValueError(f"quad_over_lin: z must be scalar, got {z.shape}") + return Node(_C.make_quad_over_lin, [x, z], (1, 1)) + + +def rel_entr(x, y): + """Elementwise x * log(x / y) for distinct Variables x, y. + + Shapes must match, or one of them must be scalar. + """ + _require_variables("rel_entr", x, y) + if x.shape == y.shape or y.shape == (1, 1): + shape = x.shape + elif x.shape == (1, 1): + shape = y.shape + else: + raise ValueError(f"rel_entr: incompatible shapes {x.shape} and {y.shape}") + return Node(_C.make_rel_entr, [x, y], shape) diff --git a/sparsediffpy/_compile.py b/sparsediffpy/_compile.py new file mode 100644 index 0000000..bdf11f9 --- /dev/null +++ b/sparsediffpy/_compile.py @@ -0,0 +1,168 @@ +"""Turn expressions into an engine problem and evaluate it. + +``compile(f)`` is a thin wrapper around ``Problem``: f becomes the only +constraint of a problem with a zero objective, so the engine's constraint +Jacobian is J_f and its Lagrangian Hessian (objective weight 0) is the Hessian +of w' f. +""" + +import numpy as np +import scipy.sparse as sps + +from sparsediffpy import _sparsediffengine as _C +from sparsediffpy._expression import Constant, Parameter, Variable, _wrap + + +def _find_scope(exprs): + """Return the single Scope shared by all Variables/Parameters in exprs.""" + scopes, has_var, seen, stack = set(), False, set(), list(exprs) + while stack: + e = stack.pop() + if id(e) in seen: + continue + seen.add(id(e)) + if isinstance(e, (Variable, Parameter)): + scopes.add(e._scope) + has_var |= isinstance(e, Variable) + stack.extend(e.args) + if not has_var: + raise ValueError("Expression must contain at least one Variable") + if len(scopes) > 1: + raise ValueError("All variables must belong to the same Scope") + return scopes.pop() + + +class _Builder: + """Builds C capsules bottom-up, sharing repeated subexpressions.""" + + def __init__(self, scope): + self.n_vars = scope.total_var_size + self.cache = {} + self.params = [] # (Parameter, capsule), each once + + def __call__(self, e): + if id(e) in self.cache: + return self.cache[id(e)] + d1, d2 = e.shape + if isinstance(e, Variable): + cap = _C.make_variable(d1, d2, e._offset, self.n_vars) + elif isinstance(e, Parameter): + values = e._flat if e._flat is not None else np.zeros(e.size) + cap = _C.make_parameter(d1, d2, e._offset, self.n_vars, values) + self.params.append((e, cap)) + elif isinstance(e, Constant): + values = e.dense().ravel(order="F") + cap = _C.make_parameter(d1, d2, -1, self.n_vars, values) + else: + cap = e.make(*[self(a) for a in e.args]) + if _C.get_expr_dimensions(cap) != e.shape: + raise ValueError(f"Engine shape {_C.get_expr_dimensions(cap)} != " + f"expected {e.shape}") + self.cache[id(e)] = cap + return cap + + +def _csr(triple): + data, indices, indptr, shape = triple + return sps.csr_matrix((data, indices, indptr), shape=shape) + + +class Problem: + """A scalar objective plus a list of constraint expressions. + + Typical solver loop (u is the flat variable vector):: + + prob.objective_forward(u); prob.constraint_forward(u) + prob.gradient(); prob.jacobian(); prob.hessian(obj_factor, lagrange) + """ + + def __init__(self, objective, constraints=(), verbose=False): + objective = _wrap(objective) + constraints = [_wrap(c) for c in constraints] + if objective.shape != (1, 1): + raise ValueError(f"Objective must be scalar, got {objective.shape}") + self._scope = _find_scope([objective, *constraints]) + build = _Builder(self._scope) + obj_cap = build(objective) + con_caps = [build(c) for c in constraints] + self._capsule = _C.make_problem(obj_cap, con_caps, verbose) + self._params = [p for p, _ in build.params] + if build.params: + _C.problem_register_params(self._capsule, [c for _, c in build.params]) + _C.problem_init_derivatives(self._capsule) + self._param_version = None + self.n_vars = self._scope.total_var_size + self.total_constraint_size = np.sum([c.size for c in constraints]).item() + + def _sync_params(self): + """Push Parameter values to the engine if any changed since last time.""" + if not self._params or self._param_version == self._scope._param_version: + return + theta = np.zeros(self._scope._n_params) + for p in self._params: + if p._flat is None: + raise ValueError(f"Parameter with shape {p.shape} has no value; " + "set parameter.value before evaluating") + theta[p._offset:p._offset + p.size] = p._flat + _C.problem_update_params(self._capsule, theta) + self._param_version = self._scope._param_version + + def _u(self, u): + return self._scope._values if u is None else np.asarray(u, np.float64) + + def objective_forward(self, u=None): + """Objective value at u (default: the scope's current values).""" + self._sync_params() + return _C.problem_objective_forward(self._capsule, self._u(u)) + + def constraint_forward(self, u=None): + """Stacked constraint values at u (default: the scope's current values).""" + self._sync_params() + return _C.problem_constraint_forward(self._capsule, self._u(u)) + + def gradient(self): + """Objective gradient at the point of the last objective_forward.""" + return _C.problem_gradient(self._capsule) + + def jacobian(self): + """Constraint Jacobian (csr_matrix) at the last constraint_forward point.""" + return _csr(_C.problem_jacobian(self._capsule)) + + def hessian(self, obj_factor, lagrange): + """Lagrangian Hessian obj_factor * H_f + sum_i lagrange_i * H_gi (csr_matrix). + + Call objective_forward and constraint_forward at the point first. + """ + lagrange = np.asarray(lagrange, dtype=np.float64).ravel(order="F") + if lagrange.size != self.total_constraint_size: + raise ValueError(f"lagrange has {lagrange.size} entries, expected " + f"{self.total_constraint_size}") + return _csr(_C.problem_hessian(self._capsule, float(obj_factor), lagrange)) + + +class CompiledExpression: + """A compiled expression f, evaluated at the scope's current values.""" + + def __init__(self, expr): + self.shape = expr.shape + self._problem = Problem(Constant(0.0), [expr]) + + def forward(self): + """f(x), flattened column-major.""" + return self._problem.constraint_forward() + + def jacobian(self): + """Sparse Jacobian df/dx, shape (f.size, n_vars).""" + self.forward() + return self._problem.jacobian() + + def hessian(self, weights): + """Sparse Hessian of w' f(x), shape (n_vars, n_vars).""" + self.forward() + self._problem.jacobian() + return self._problem.hessian(0.0, weights) + + +def compile(expr): + """Analyze the sparsity of ``expr`` once and return a CompiledExpression.""" + return CompiledExpression(_wrap(expr)) diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py new file mode 100644 index 0000000..c3b7d80 --- /dev/null +++ b/sparsediffpy/_expression.py @@ -0,0 +1,307 @@ +"""Expression graph: Scope, leaves (Variable/Parameter/Constant), and operators. + +Every non-leaf is a generic ``Node(make, args, shape)``: ``make`` receives the +C capsules of ``args`` and returns the node's own capsule. Shapes are always +2-D ``(d1, d2)`` and values are flattened column-major, like the C engine. +""" + +import numpy as np +import scipy.sparse as sps + +from sparsediffpy import _sparsediffengine as _C + + +class DimensionError(ValueError): + """Raised when a value has the wrong number of elements.""" + + +def _check_shape(d1, d2): + if not all(isinstance(d, (int, np.integer)) for d in (d1, d2)): + raise TypeError(f"Shape dimensions must be integers, got ({d1!r}, {d2!r})") + if d1 <= 0 or d2 <= 0: + raise ValueError(f"Shape dimensions must be positive, got ({d1}, {d2})") + return (int(d1), int(d2)) + + +class Expression: + """Base class for all expressions. Supports + - * @ ** [] and .T.""" + + __array_ufunc__ = None # make NumPy defer to our reflected operators + shape = None + args = () + + @property + def size(self): + return self.shape[0] * self.shape[1] + + @property + def T(self): + return Node(_C.make_transpose, [self], self.shape[::-1]) + + def __add__(self, other): + return _add(self, _wrap(other)) + + def __radd__(self, other): + return _add(_wrap(other), self) + + def __sub__(self, other): + return _add(self, -_wrap(other)) + + def __rsub__(self, other): + return _add(_wrap(other), -self) + + def __neg__(self): + return Node(_C.make_neg, [self], self.shape) + + def __mul__(self, other): + return _mul(self, _wrap(other)) + + def __rmul__(self, other): + return _mul(_wrap(other), self) + + def __matmul__(self, other): + return _matmul(self, _wrap(other)) + + def __rmatmul__(self, other): + return _matmul(_wrap(other), self) + + def __pow__(self, p): + if not isinstance(p, (int, float)): + raise TypeError("Exponent must be a constant number") + return Node(lambda c: _C.make_power(c, float(p)), [self], self.shape) + + def __getitem__(self, key): + return _index(self, key) + + +class Node(Expression): + """An operation: ``make(*child_capsules)`` builds its C capsule.""" + + def __init__(self, make, args, shape): + self.make = make + self.args = tuple(args) + self.shape = tuple(shape) + + +class Constant(Expression): + """Fixed dense or scipy.sparse data.""" + + def __init__(self, value): + if sps.issparse(value): + self.value = sps.csr_matrix(value, dtype=np.float64) + else: + value = np.asarray(value, dtype=np.float64) + if value.ndim > 2: + raise ValueError(f"Cannot wrap {value.ndim}D array as constant") + # scalars -> (1, 1), 1-D -> column vector + self.value = value.reshape(value.shape[0], -1) if value.ndim else \ + value.reshape(1, 1) + self.shape = _check_shape(*self.value.shape) + + def dense(self): + return self.value.toarray() if sps.issparse(self.value) else self.value + + +class Variable(Expression): + """A block of the scope's flat variable vector. Create via Scope.Variable.""" + + def __init__(self, scope, offset, shape): + self._scope, self._offset, self.shape = scope, offset, shape + + @property + def value(self): + flat = self._scope._values[self._offset:self._offset + self.size] + return flat.reshape(self.shape, order="F").copy() + + @value.setter + def value(self, val): + self._scope._values[self._offset:self._offset + self.size] = \ + _flatten(val, self.size) + + +class Parameter(Expression): + """Data that can change without recompiling. Create via Scope.Parameter.""" + + def __init__(self, scope, offset, shape, value=None): + self._scope, self._offset, self.shape = scope, offset, shape + self._flat = None + if value is not None: + self.value = value + + @property + def value(self): + if self._flat is None: + return None + return self._flat.reshape(self.shape, order="F").copy() + + @value.setter + def value(self, val): + self._flat = _flatten(val, self.size) + self._scope._param_version += 1 + + +def _flatten(val, size): + flat = np.asarray(val, dtype=np.float64).ravel(order="F") + if flat.size != size: + raise DimensionError(f"expected {size} elements, got {flat.size}") + return flat + + +class Scope: + """Owns the flat variable vector and the parameter vector.""" + + def __init__(self): + self._values = np.zeros(0) + self._parameters = [] + self._n_params = 0 + self._param_version = 0 + + def Variable(self, d1, d2): + """Create a (d1, d2) variable; its entries are appended to the flat vector.""" + shape = _check_shape(d1, d2) + var = Variable(self, self._values.size, shape) + self._values = np.concatenate([self._values, np.zeros(var.size)]) + return var + + def Parameter(self, d1, d2, value=None): + """Create a (d1, d2) parameter, optionally with an initial value.""" + shape = _check_shape(d1, d2) + param = Parameter(self, self._n_params, shape, value) + self._n_params += param.size + self._parameters.append(param) + return param + + @property + def total_var_size(self): + return self._values.size + + def set_values(self, array): + """Set all variables at once from a flat array (declaration order).""" + self._values[:] = _flatten(array, self._values.size) + + def get_values(self): + return self._values.copy() + + +def _wrap(value): + return value if isinstance(value, Expression) else Constant(value) + + +def _is_data(e): + return isinstance(e, (Constant, Parameter)) + + +# --------------------------------------------------------------------------- +# Operator dispatch +# --------------------------------------------------------------------------- + +def _broadcast_shape(a, b): + out = [] + for da, db in zip(a, b): + if da != db and 1 not in (da, db): + raise ValueError(f"Cannot broadcast shapes {a} and {b}") + out.append(max(da, db)) + return tuple(out) + + +def broadcast(x, shape): + """Broadcast a scalar, row or column expression to ``shape``.""" + x, shape = _wrap(x), _check_shape(*shape) + if x.shape == shape: + return x + if _broadcast_shape(x.shape, shape) != shape: + raise ValueError(f"Cannot broadcast {x.shape} to {shape}") + if isinstance(x, Constant): + return Constant(np.broadcast_to(x.dense(), shape)) + return Node(lambda c: _C.make_broadcast(c, *shape), [x], shape) + + +def _add(a, b): + shape = _broadcast_shape(a.shape, b.shape) + return Node(_C.make_add, [broadcast(a, shape), broadcast(b, shape)], shape) + + +def _mul(a, b): + if _is_data(b) and not _is_data(a): + a, b = b, a + if not _is_data(a): # both variable-dependent + shape = _broadcast_shape(a.shape, b.shape) + return Node(_C.make_multiply, + [broadcast(a, shape), broadcast(b, shape)], shape) + if a.shape == (1, 1): + return Node(_C.make_param_scalar_mult, [a, b], b.shape) + shape = _broadcast_shape(a.shape, b.shape) + a, b = broadcast(a, shape), broadcast(b, shape) + make = _C.make_param_vector_mult if _is_data(a) else _C.make_multiply + return Node(make, [a, b], shape) + + +def _matmul(a, b): + if a.shape[1] != b.shape[0]: + raise ValueError(f"Matmul shape mismatch: {a.shape} @ {b.shape}") + shape = (a.shape[0], b.shape[1]) + if _is_data(a): + return Node(_matmul_with_data(_C.make_left_matmul, a), _data_args(a, b), + shape) + if _is_data(b): + return Node(_matmul_with_data(_C.make_right_matmul, b), _data_args(b, a), + shape) + return Node(_C.make_matmul, [a, b], shape) + + +def _data_args(data, x): + return [data, x] if isinstance(data, Parameter) else [x] + + +def _matmul_with_data(make, A): + """Return make(child) / make(param, child) for a constant/parameter A.""" + m, n = A.shape + if isinstance(A, Parameter): + return lambda p, c: make(p, c, "dense", None, m, n) + if sps.issparse(A.value): + V = A.value + return lambda c: make(None, c, "sparse", V.data, V.indices.astype(np.int32), + V.indptr.astype(np.int32), m, n) + data = np.ascontiguousarray(A.value).ravel() # row-major + return lambda c: make(None, c, "dense", data, m, n) + + +def index_flat(x, flat_indices, shape): + """Gather entries of ``x`` by column-major flat indices into ``shape``.""" + idx = np.asarray(flat_indices, dtype=np.int32).ravel() + shape = _check_shape(*shape) + if idx.size != shape[0] * shape[1]: + raise ValueError(f"{idx.size} indices do not fill shape {shape}") + if idx.size and (idx.min() < 0 or idx.max() >= x.size): + raise IndexError(f"flat index out of range for size {x.size}") + return Node(lambda c: _C.make_index(c, shape[0], shape[1], idx), [x], shape) + + +def _axis_indices(key, n): + if isinstance(key, (int, np.integer)): + if not -n <= key < n: + raise IndexError(f"Index {key} out of range for axis of length {n}") + return np.array([key % n]) + if isinstance(key, slice): + return np.arange(n)[key] + if isinstance(key, (list, np.ndarray)): + key = np.asarray(key, dtype=np.int64) + if key.size and (key.min() < -n or key.max() >= n): + raise IndexError(f"Index out of range for axis of length {n}") + return key % n + raise IndexError(f"Unsupported index type: {type(key).__name__}") + + +def _index(x, key): + d1, d2 = x.shape + if isinstance(key, tuple): + if len(key) != 2: + raise IndexError("Only 1D or 2D indexing supported") + rows, cols = _axis_indices(key[0], d1), _axis_indices(key[1], d2) + flat = (rows[:, None] + cols[None, :] * d1).ravel(order="F") + return index_flat(x, flat, (rows.size, cols.size)) + if d1 == 1 and d2 > 1: # row vector: index along its columns + flat = _axis_indices(key, d2) + return index_flat(x, flat, (1, flat.size)) + flat = _axis_indices(key, x.size) # column vector / column-major flat + return index_flat(x, flat, (flat.size, 1)) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a290423 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,39 @@ +import numpy as np +import pytest + +import sparsediffpy as sp + + +@pytest.fixture +def scope(): + return sp.Scope() + + +@pytest.fixture +def rng(): + return np.random.default_rng(0) + + +def check_derivatives(fn, scope, x0, w, h=1e-6, rtol=1e-5, atol=1e-6): + """Check fn.jacobian() and fn.hessian(w) against central differences.""" + x0 = np.asarray(x0, dtype=np.float64) + + def at(x, f): + scope.set_values(x) + return f() + + def jac(x): + return at(x, fn.jacobian).toarray() + + J = jac(x0) + H = at(x0, lambda: fn.hessian(w)).toarray() + J_fd = np.zeros_like(J) + H_fd = np.zeros_like(H) + for j in range(x0.size): + e = np.zeros_like(x0) + e[j] = h + J_fd[:, j] = (at(x0 + e, fn.forward) - at(x0 - e, fn.forward)) / (2 * h) + H_fd[:, j] = (jac(x0 + e) - jac(x0 - e)).T @ w / (2 * h) + scope.set_values(x0) + np.testing.assert_allclose(J, J_fd, rtol=rtol, atol=atol, err_msg="Jacobian") + np.testing.assert_allclose(H, H_fd, rtol=rtol, atol=atol, err_msg="Hessian") diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..1172910 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,182 @@ +"""Scope/values, shape rules, validation errors, parameters, and Problem.""" + +import numpy as np +import pytest + +import sparsediffpy as sp + + +# --- scope and values --------------------------------------------------------- + +def test_values_are_column_major(scope): + X, y = scope.Variable(2, 2), scope.Variable(1, 1) + X.value = np.array([[1.0, 2.0], [3.0, 4.0]]) + y.value = 5.0 + np.testing.assert_array_equal(scope.get_values(), [1, 3, 2, 4, 5]) + scope.set_values(np.arange(5.0)) + np.testing.assert_array_equal(X.value, [[0, 2], [1, 3]]) + + +def test_wrong_sizes(scope): + x = scope.Variable(3, 1) + p = scope.Parameter(2, 1) + with pytest.raises(sp.DimensionError): + x.value = [1.0, 2.0] + with pytest.raises(sp.DimensionError): + p.value = [1.0, 2.0, 3.0] + with pytest.raises(sp.DimensionError): + scope.set_values(np.zeros(4)) + assert p.value is None + + +@pytest.mark.parametrize("shape", [(0, 1), (-1, 2), (2.0, 1)]) +def test_invalid_shapes(scope, shape): + with pytest.raises((ValueError, TypeError)): + scope.Variable(*shape) + + +def test_reevaluates_at_new_values(scope): + x = scope.Variable(2, 1) + fn = sp.compile(sp.exp(x)) + for v in ([0.0, 1.0], [2.0, -1.0]): + x.value = v + np.testing.assert_allclose(fn.forward(), np.exp(v)) + np.testing.assert_allclose(fn.jacobian().diagonal(), np.exp(v)) + np.testing.assert_allclose(fn.hessian([1, 1]).diagonal(), np.exp(v)) + + +def test_two_compiles_share_scope(scope): + x, y = scope.Variable(2, 1), scope.Variable(2, 1) + f, g = sp.compile(sp.sin(x)), sp.compile(x * y) + scope.set_values([1.0, 2.0, 3.0, 4.0]) + np.testing.assert_allclose(f.forward(), np.sin([1, 2])) + np.testing.assert_allclose(g.forward(), [3, 8]) + assert f.jacobian().shape == g.jacobian().shape == (2, 4) + + +def test_affine_hessian_is_zero(scope): + x = scope.Variable(3, 1) + fn = sp.compile(sp.sum(-x[0:2]) + 2.0) + x.value = [1.0, 2.0, 3.0] + assert not fn.hessian([1.0]).toarray().any() + + +# --- shapes and validation ---------------------------------------------------- + +def test_shapes(scope): + x, r, X = scope.Variable(3, 1), scope.Variable(1, 2), scope.Variable(3, 2) + assert (x + r).shape == (3, 2) + assert (X @ r.T).shape == (3, 1) + assert x.T.shape == (1, 3) + assert x[1].shape == (1, 1) and x[-2:].shape == (2, 1) + assert r[1].shape == (1, 1) and X[:, 1].shape == (3, 1) + assert sp.sum(X, axis=1).shape == (1, 3) + + +@pytest.mark.parametrize("make", [ + lambda s: s.Variable(3, 1) + s.Variable(2, 1), + lambda s: s.Variable(3, 2) @ s.Variable(3, 2), + lambda s: sp.trace(s.Variable(2, 3)), + lambda s: sp.diag_vec(s.Variable(1, 3)), + lambda s: sp.reshape(s.Variable(3, 2), 4, 2), + lambda s: sp.hstack([]), + lambda s: sp.vstack([s.Variable(2, 1), s.Variable(2, 2)]), + lambda s: sp.quad_form(s.Variable(3, 1), np.eye(2)), + lambda s: sp.sum(s.Variable(2, 2), axis=2), + lambda s: sp.rel_entr(s.Variable(3, 1), s.Variable(2, 1)), +]) +def test_shape_errors(scope, make): + with pytest.raises(ValueError): + make(scope) + + +@pytest.mark.parametrize("make", [ + lambda x: sp.log(x[1:3]), + lambda x: sp.entr(2.0 * x), + lambda x: sp.prod(sp.exp(x)), + lambda x: sp.quad_over_lin(x, x[0]), + lambda x: sp.rel_entr(x, x), +]) +def test_variable_only_atoms(scope, make): + with pytest.raises(ValueError): + make(scope.Variable(4, 1)) + + +@pytest.mark.parametrize("key", [4, -5, [0, 4], (0, 3), (3, 0)]) +def test_index_out_of_range(scope, key): + X = scope.Variable(2, 2) if isinstance(key, tuple) else scope.Variable(4, 1) + with pytest.raises(IndexError): + X[key] + + +def test_compile_errors(scope): + with pytest.raises(ValueError, match="at least one Variable"): + sp.compile(sp.Scope().Parameter(1, 1, value=1.0) * 2.0) + with pytest.raises(ValueError, match="same Scope"): + sp.compile(scope.Variable(1, 1) + sp.Scope().Variable(1, 1)) + with pytest.raises(TypeError): + scope.Variable(1, 1) ** scope.Variable(1, 1) + + +# --- parameters ---------------------------------------------------------------- + +def test_parameter_update_without_recompile(scope): + x = scope.Variable(3, 1) + A = scope.Parameter(2, 3, value=np.ones((2, 3))) + c = scope.Parameter(1, 1, value=1.0) + fn = sp.compile(c * sp.exp(A @ x)) + x.value = [0.1, 0.2, 0.3] + for Av, cv in [(np.ones((2, 3)), 1.0), (np.arange(6.0).reshape(2, 3), -2.0)]: + A.value, c.value = Av, cv + np.testing.assert_allclose(fn.forward(), cv * np.exp(Av @ x.value).ravel()) + J = cv * np.exp(Av @ x.value) * Av + np.testing.assert_allclose(fn.jacobian().toarray(), J) + + +def test_unset_parameter_raises(scope): + x, p = scope.Variable(2, 1), scope.Parameter(2, 1) + fn = sp.compile(p * x) + with pytest.raises(ValueError, match="no value"): + fn.forward() + p.value = [1.0, 2.0] + np.testing.assert_allclose(fn.forward(), [0, 0]) + + +# --- Problem ---------------------------------------------------------------------- + +def test_problem(scope): + x = scope.Variable(3, 1) + shared = sp.sin(x) + prob = sp.Problem(sp.sum(sp.exp(shared)), [x[0:2] * x[1:3], shared]) + u = np.array([0.3, -0.5, 0.8]) + s, c = np.sin(u), np.cos(u) + + assert prob.objective_forward(u) == pytest.approx(np.exp(s).sum()) + np.testing.assert_allclose(prob.gradient(), np.exp(s) * c) + np.testing.assert_allclose(prob.constraint_forward(u), + np.r_[u[:2] * u[1:], s]) + + J = prob.jacobian().toarray() + np.testing.assert_allclose(J[:2], [[u[1], u[0], 0], [0, u[2], u[1]]]) + np.testing.assert_allclose(J[2:], np.diag(c)) + + lam = np.array([1.0, 2.0, 0.5, -1.0, 3.0]) + H = prob.hessian(2.0, lam).toarray() + H_obj = np.diag(np.exp(s) * (c ** 2 - s)) + H_con = np.array([[0, 1, 0], [1, 0, 2], [0, 2, 0]]) + np.diag(-s * lam[2:]) + np.testing.assert_allclose(H, 2.0 * H_obj + H_con) + + with pytest.raises(ValueError, match="lagrange"): + prob.hessian(1.0, np.ones(4)) + with pytest.raises(ValueError, match="scalar"): + sp.Problem(x) + + +def test_problem_parameters(scope): + x, a = scope.Variable(2, 1), scope.Parameter(1, 1, value=2.0) + prob = sp.Problem(sp.sum(a * x ** 2)) + u = np.array([1.0, 3.0]) + assert prob.objective_forward(u) == pytest.approx(20.0) + a.value = 0.5 + assert prob.objective_forward(u) == pytest.approx(5.0) + np.testing.assert_allclose(prob.gradient(), u) diff --git a/tests/test_atoms.py b/tests/test_atoms.py new file mode 100644 index 0000000..466391d --- /dev/null +++ b/tests/test_atoms.py @@ -0,0 +1,228 @@ +"""Every atom: forward value vs NumPy, Jacobian and Hessian vs finite differences. + +Each case builds (expr, reference) on a fresh scope; ``reference()`` computes +the expected value from the variables' current values with NumPy. +""" + +import numpy as np +import pytest +import scipy.sparse as sps +from conftest import check_derivatives +from scipy.special import ndtr + +import sparsediffpy as sp + +A = np.array([[1.0, -2.0, 0.5], [0.0, 3.0, 1.0]]) +Q = np.array([[2.0, 1.0, 0.0], [1.0, 3.0, 0.5], [0.0, 0.5, 4.0]]) +CASES = {} # name -> (build, positive_domain) + + +def case(positive=False): + def register(build): + CASES[build.__name__] = (build, positive) + return build + return register + + +def _elementwise(name, f, ref, positive=False): + def build(s): + X = s.Variable(3, 2) + return f(X), lambda: ref(X.value) + build.__name__ = name + case(positive)(build) + + +for _name, _f, _ref in [ + ("sin", sp.sin, np.sin), ("cos", sp.cos, np.cos), ("exp", sp.exp, np.exp), + ("sinh", sp.sinh, np.sinh), ("tanh", sp.tanh, np.tanh), + ("asinh", sp.asinh, np.arcsinh), ("atan", sp.atan, np.arctan), + ("logistic", sp.logistic, lambda x: np.log1p(np.exp(x))), + ("normal_cdf", sp.normal_cdf, ndtr), ("xexp", sp.xexp, lambda x: x * np.exp(x)), + ("power3", lambda x: x ** 3, lambda x: x ** 3), +]: + _elementwise(_name, _f, _ref) + +for _name, _f, _ref in [ + ("log", sp.log, np.log), ("tan", sp.tan, np.tan), + ("atanh", sp.atanh, np.arctanh), ("entr", sp.entr, lambda x: -x * np.log(x)), + ("power_half", lambda x: sp.power(x, 0.5), np.sqrt), +]: + _elementwise(_name, _f, _ref, positive=True) + + +@case(positive=True) +def atan2(s): + y, x = s.Variable(3, 1), s.Variable(3, 1) + return sp.atan2(y, x), lambda: np.arctan2(y.value, x.value) + + +# --- affine ----------------------------------------------------------------- + +@case() +def add_broadcast(s): + a, r, c, X = s.Variable(1, 1), s.Variable(1, 2), s.Variable(3, 1), s.Variable(3, 2) + return a * X + r + c - 1.0, lambda: a.value * X.value + r.value + c.value - 1 + + +@case() +def neg_sub(s): + x, y = s.Variable(3, 1), s.Variable(3, 1) + return -x - y + np.arange(3.0), lambda: -x.value - y.value + np.c_[0:3.0] + + +@case() +def constant_mult(s): + X = s.Variable(3, 2) + B = np.arange(6.0).reshape(3, 2) + return 2.0 * X * B, lambda: 2 * X.value * B + + +@case() +def matmul_dense_left_right(s): + x, Y = s.Variable(3, 1), s.Variable(2, 3) + return A @ x + (Y @ A.T)[:, 0:1], lambda: A @ x.value + (Y.value @ A.T)[:, 0:1] + + +@case() +def matmul_sparse_left_right(s): + x, Y = s.Variable(3, 1), s.Variable(2, 3) + S = sps.csr_matrix(A) + return S @ x + (Y @ S.T)[:, 0:1], lambda: A @ x.value + (Y.value @ A.T)[:, 0:1] + + +@case() +def transpose_reshape(s): + X = s.Variable(3, 2) + return sp.reshape(X.T, 6, 1), lambda: X.value.T.reshape(6, 1, order="F") + + +@case() +def sums(s): + X = s.Variable(3, 2) + return sp.hstack([sp.sum(X), sp.sum(X, axis=0), sp.sum(X, axis=1)]), \ + lambda: np.r_[X.value.sum(), X.value.sum(0), X.value.sum(1)][None, :] + + +@case() +def trace_diag_vec(s): + X, x = s.Variable(3, 3), s.Variable(3, 1) + return sp.trace(X) + sp.diag_vec(x), \ + lambda: np.trace(X.value) + np.diag(x.value.ravel()) + + +@case() +def stacking(s): + x, y = s.Variable(2, 1), s.Variable(2, 2) + return sp.vstack([sp.hstack([x, y]), sp.hstack([y, x])]), \ + lambda: np.block([[x.value, y.value], [y.value, x.value]]) + + +@case() +def indexing(s): + X, r = s.Variable(3, 3), s.Variable(1, 4) + return sp.vstack([X[0:2, [2, 0]], X[-1, 1:], r[1:3]]), \ + lambda: np.r_[X.value[0:2][:, [2, 0]], X.value[-1:, 1:], r.value[:, 1:3]] + + +# --- bivariate / other -------------------------------------------------------- + +@case() +def multiply(s): + x, Y = s.Variable(3, 1), s.Variable(3, 2) + return sp.sin(x) * Y, lambda: np.sin(x.value) * Y.value + + +@case() +def matmul_variables(s): + X, Y = s.Variable(2, 3), s.Variable(3, 2) + return X @ Y, lambda: X.value @ Y.value + + +@case() +def x_transpose_x(s): + x = s.Variable(3, 1) + return x.T @ x, lambda: x.value.T @ x.value + + +@case() +def quad_forms(s): + x = s.Variable(3, 1) + P = s.Parameter(3, 3, value=2 * Q) + xv = lambda: x.value.ravel() # noqa: E731 + return sp.vstack([sp.quad_form(x, Q), sp.quad_form(x, sps.csr_matrix(Q)), + sp.quad_form(x, P)]), \ + lambda: np.c_[[xv() @ Q @ xv(), xv() @ Q @ xv(), 2 * xv() @ Q @ xv()]] + + +@case(positive=True) +def quad_over_lin(s): + x, z = s.Variable(3, 1), s.Variable(1, 1) + return sp.quad_over_lin(x, z), lambda: (x.value ** 2).sum() / z.value + + +@case(positive=True) +def rel_entr_vectors(s): + x, y = s.Variable(3, 1), s.Variable(3, 1) + return sp.rel_entr(x, y), lambda: x.value * np.log(x.value / y.value) + + +@case(positive=True) +def rel_entr_scalar_vector(s): + a, y = s.Variable(1, 1), s.Variable(3, 1) + return sp.rel_entr(a, y), lambda: a.value * np.log(a.value / y.value) + + +@case(positive=True) +def rel_entr_vector_scalar(s): + x, b = s.Variable(3, 1), s.Variable(1, 1) + return sp.rel_entr(x, b), lambda: x.value * np.log(x.value / b.value) + + +@case(positive=True) +def prods(s): + X = s.Variable(3, 2) + return sp.hstack([sp.prod(X), sp.prod(X, axis=0), sp.prod(X, axis=1)]), \ + lambda: np.r_[X.value.prod(), X.value.prod(0), X.value.prod(1)][None, :] + + +# --- compositions ------------------------------------------------------------- + +@case() +def parameter_matmul_and_scale(s): + x = s.Variable(3, 1) + B, c = s.Parameter(2, 3, value=A), s.Parameter(1, 1, value=3.0) + return c * sp.exp(B @ x) + B @ sp.sin(x), \ + lambda: 3 * np.exp(A @ x.value) + A @ np.sin(x.value) + + +@case() +def deep_chain(s): + x = s.Variable(4, 1) + f = x + for _ in range(5): + f = sp.tanh(0.9 * f + 0.1) + return sp.sum(f * f) + sp.cos(x[0]), None + + +@case() +def shared_subexpression(s): + x = s.Variable(3, 1) + u = sp.sin(A @ x) + return u * u + sp.exp(u), lambda: np.sin(A @ x.value) ** 2 + np.exp(np.sin(A @ x.value)) + + +@pytest.mark.parametrize("name", list(CASES)) +def test_atom(name, rng): + build, positive = CASES[name] + scope = sp.Scope() + f, ref = build(scope) + fn = sp.compile(f) + low, high = (0.2, 0.9) if positive else (-1.0, 1.0) + x0 = rng.uniform(low, high, scope.total_var_size) + scope.set_values(x0) + out = fn.forward() + assert out.shape == (f.size,) + if ref is not None: + expected = np.broadcast_to(ref(), f.shape).ravel(order="F") + np.testing.assert_allclose(out, expected, rtol=1e-10, atol=1e-12) + check_derivatives(fn, scope, x0, rng.standard_normal(f.size)) diff --git a/tests/test_readme.py b/tests/test_readme.py new file mode 100644 index 0000000..63426b3 --- /dev/null +++ b/tests/test_readme.py @@ -0,0 +1,14 @@ +"""Run every ```python block in README.md so the examples stay correct.""" + +import re +from pathlib import Path + +import pytest + +README = Path(__file__).resolve().parents[1] / "README.md" +BLOCKS = re.findall(r"```python\n(.*?)```", README.read_text(), re.S) + + +@pytest.mark.parametrize("code", BLOCKS, ids=[f"block{i}" for i in range(len(BLOCKS))]) +def test_readme_example(code): + exec(compile(code, str(README), "exec"), {}) From 330769a1ddf0e14823af43e3a7f4349a9fc09fc3 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sat, 7 Nov 2026 14:43:35 -0800 Subject: [PATCH 02/17] Fix Problem.hessian after forward-only calls; simplify dispatch The engine's Hessian reuses Jacobian values cached by eval_jacobian, so Problem.hessian() returned wrong values unless gradient() and jacobian() had been called at the same point (7 of 39 atom cases differed). It now refreshes them itself; a regression test covers both Problem and compile. Also drop the unused Scope._parameters list and fold redundant branches in _mul and _index. Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/_compile.py | 5 ++++- sparsediffpy/_expression.py | 17 ++++------------- tests/test_api.py | 22 ++++++++++++++++++++++ 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/sparsediffpy/_compile.py b/sparsediffpy/_compile.py index bdf11f9..25526d2 100644 --- a/sparsediffpy/_compile.py +++ b/sparsediffpy/_compile.py @@ -137,6 +137,10 @@ def hessian(self, obj_factor, lagrange): if lagrange.size != self.total_constraint_size: raise ValueError(f"lagrange has {lagrange.size} entries, expected " f"{self.total_constraint_size}") + # The engine's Hessian reuses Jacobian values cached by eval_jacobian, + # so those must be current at this point. + _C.problem_gradient(self._capsule) + _C.problem_jacobian(self._capsule) return _csr(_C.problem_hessian(self._capsule, float(obj_factor), lagrange)) @@ -159,7 +163,6 @@ def jacobian(self): def hessian(self, weights): """Sparse Hessian of w' f(x), shape (n_vars, n_vars).""" self.forward() - self._problem.jacobian() return self._problem.hessian(0.0, weights) diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py index c3b7d80..ad9686b 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/_expression.py @@ -152,7 +152,6 @@ class Scope: def __init__(self): self._values = np.zeros(0) - self._parameters = [] self._n_params = 0 self._param_version = 0 @@ -168,7 +167,6 @@ def Parameter(self, d1, d2, value=None): shape = _check_shape(d1, d2) param = Parameter(self, self._n_params, shape, value) self._n_params += param.size - self._parameters.append(param) return param @property @@ -223,12 +221,8 @@ def _add(a, b): def _mul(a, b): if _is_data(b) and not _is_data(a): - a, b = b, a - if not _is_data(a): # both variable-dependent - shape = _broadcast_shape(a.shape, b.shape) - return Node(_C.make_multiply, - [broadcast(a, shape), broadcast(b, shape)], shape) - if a.shape == (1, 1): + a, b = b, a # put the constant/parameter operand (if any) first + if _is_data(a) and a.shape == (1, 1): return Node(_C.make_param_scalar_mult, [a, b], b.shape) shape = _broadcast_shape(a.shape, b.shape) a, b = broadcast(a, shape), broadcast(b, shape) @@ -300,8 +294,5 @@ def _index(x, key): rows, cols = _axis_indices(key[0], d1), _axis_indices(key[1], d2) flat = (rows[:, None] + cols[None, :] * d1).ravel(order="F") return index_flat(x, flat, (rows.size, cols.size)) - if d1 == 1 and d2 > 1: # row vector: index along its columns - flat = _axis_indices(key, d2) - return index_flat(x, flat, (1, flat.size)) - flat = _axis_indices(key, x.size) # column vector / column-major flat - return index_flat(x, flat, (flat.size, 1)) + flat = _axis_indices(key, x.size) # column-major flat index + return index_flat(x, flat, (1, flat.size) if d1 == 1 else (flat.size, 1)) diff --git a/tests/test_api.py b/tests/test_api.py index 1172910..3e53dce 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -180,3 +180,25 @@ def test_problem_parameters(scope): a.value = 0.5 assert prob.objective_forward(u) == pytest.approx(5.0) np.testing.assert_allclose(prob.gradient(), u) + + +def test_hessian_without_prior_jacobian(scope): + """The engine's Hessian reuses cached Jacobian values; hessian() must refresh them.""" + x, y = scope.Variable(2, 1), scope.Variable(2, 1) + fn = sp.compile(x * y + sp.sin(x * y)) + prob = sp.Problem(sp.sum(x * y), [x * y]) + for u in ([0.1, 0.2, 0.3, 0.4], [1.0, -2.0, 0.5, 3.0]): + scope.set_values(u) + xy, xv, yv = np.multiply(u[:2], u[2:]), np.array(u[:2]), np.array(u[2:]) + w = np.array([1.0, 2.0]) + # d2/dx_i dy_i of w_i * (x_i y_i + sin(x_i y_i)) + d2 = lambda a, b: w * (-np.sin(xy) * a * b) # noqa: E731 + H = np.block([[np.diag(d2(yv, yv)), np.diag(w * (1 + np.cos(xy)) + d2(xv, yv))], + [np.diag(w * (1 + np.cos(xy)) + d2(xv, yv)), np.diag(d2(xv, xv))]]) + np.testing.assert_allclose(fn.hessian(w).toarray(), H, atol=1e-12) + + prob.objective_forward(u) + prob.constraint_forward(u) + H_prob = np.block([[np.zeros((2, 2)), np.diag(2.0 + w)], + [np.diag(2.0 + w), np.zeros((2, 2))]]) + np.testing.assert_allclose(prob.hessian(2.0, w).toarray(), H_prob, atol=1e-12) From 009485f22c18d933e23935e4be9ff0accde956e6 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sat, 7 Nov 2026 14:49:54 -0800 Subject: [PATCH 03/17] quad_form: use the dense path for NumPy Q and symmetrize constant Q A NumPy Q now goes to the engine's dense (BLAS) quad_form instead of being converted to CSR; scipy.sparse Q keeps the sparse path, matching matmul. The engine assumes Q is symmetric (gradient 2Qx), so a non-symmetric constant Q gave a correct value but wrong derivatives. Constant Q is now replaced by (Q + Q')/2, which leaves x'Qx unchanged. Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/_atoms.py | 13 +++++++++++-- tests/test_atoms.py | 9 +++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/sparsediffpy/_atoms.py b/sparsediffpy/_atoms.py index 16ba8d6..d3eacbb 100644 --- a/sparsediffpy/_atoms.py +++ b/sparsediffpy/_atoms.py @@ -130,7 +130,11 @@ def vstack(exprs): def quad_form(x, Q): - """x' Q x for a column vector x (n, 1) and a constant or Parameter Q (n, n).""" + """x' Q x for a column vector x (n, 1) and a constant or Parameter Q (n, n). + + A NumPy Q uses the engine's dense (BLAS) path, a scipy.sparse Q the sparse + path. Constant Q is symmetrized; a Parameter Q must be symmetric. + """ x, Q = _wrap(x), _wrap(Q) n = x.shape[0] if x.shape[1] != 1 or Q.shape != (n, n): @@ -141,7 +145,12 @@ def quad_form(x, Q): [Q, x], (1, 1)) if not isinstance(Q, Constant): raise TypeError("quad_form: Q must be a constant or a Parameter") - V = sps.csr_matrix(Q.value) + V = (Q.value + Q.value.T) / 2 # the engine assumes symmetric Q + if not sps.issparse(V): + data = np.ascontiguousarray(V).ravel() + return Node(lambda c: _C.make_quad_form(None, c, "dense", data, n), + [x], (1, 1)) + V = sps.csr_matrix(V) return Node(lambda c: _C.make_quad_form( None, c, "sparse", V.data, V.indices.astype(np.int32), V.indptr.astype(np.int32), n, n), [x], (1, 1)) diff --git a/tests/test_atoms.py b/tests/test_atoms.py index 466391d..578003f 100644 --- a/tests/test_atoms.py +++ b/tests/test_atoms.py @@ -154,6 +154,15 @@ def quad_forms(s): lambda: np.c_[[xv() @ Q @ xv(), xv() @ Q @ xv(), 2 * xv() @ Q @ xv()]] +@case() +def quad_form_nonsymmetric(s): + x = s.Variable(3, 1) + N = Q + np.triu(np.ones((3, 3)), 1) # not symmetric + xv = lambda: x.value.ravel() # noqa: E731 + return sp.quad_form(x, N) + sp.quad_form(x, sps.csr_matrix(N)), \ + lambda: 2 * xv() @ N @ xv() + + @case(positive=True) def quad_over_lin(s): x, z = s.Variable(3, 1), s.Variable(1, 1) From 79c0b5277f861f99d13b440f6c7d4ef5160cba75 Mon Sep 17 00:00:00 2001 From: dance858 Date: Sat, 7 Nov 2026 16:17:30 -0800 Subject: [PATCH 04/17] Check the variable vector length and require 2-D constants Problem._u now validates that an explicit u has exactly n_vars elements (the C bindings read the buffer unchecked, so a short array read past its end) and slices the scope's vector to n_vars so variables declared after compile do not change what the problem sees. Constants in expressions must be scalars or 2-D arrays. A 1-D array used to become a column silently, which turned row + np.ones(n) into an (n, n) outer sum. Every expression is now (n, 1) or (1, n), never (n,). CompiledExpression.forward() returns an array shaped like the expression (column-major reshape of the stacked values). Problem.constraint_forward() still returns the flat stacked vector solvers expect. Co-Authored-By: Claude Fable 5.1 --- README.md | 14 +++++++----- sparsediffpy/_compile.py | 11 +++++---- sparsediffpy/_expression.py | 12 +++++----- tests/conftest.py | 3 ++- tests/test_api.py | 45 ++++++++++++++++++++++++++++++++----- tests/test_atoms.py | 6 ++--- 6 files changed, 67 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index c8de887..421f4be 100644 --- a/README.md +++ b/README.md @@ -26,14 +26,15 @@ f = sp.exp(x) + 2.0 * x # build an expression with operators and sp.* ato fn = sp.compile(f) # analyze sparsity once x.value = [1.0, 2.0, 3.0] -print(fn.forward()) # [ 4.718 11.389 26.086] +print(fn.forward()) # [[4.718] [11.389] [26.086]], shaped like f print(fn.jacobian().toarray()) # diag(exp(x) + 2), a 3x3 csr_matrix print(fn.hessian(np.ones(3))) # Hessian of w' f(x) for weights w ``` -`jacobian()` has shape `(f.size, n_vars)` and `hessian(w)` has shape -`(n_vars, n_vars)`, where `n_vars` is the total size of all variables in the -scope, in declaration order. Values are flattened column-major. +`forward()` has the shape of `f`, `jacobian()` has shape `(f.size, n_vars)` +and `hessian(w)` has shape `(n_vars, n_vars)`, where `n_vars` is the total +size of all variables in the scope, in declaration order. Flat vectors (the +rows of the Jacobian, `x.value = [...]`, the weights `w`) are column-major. **Parameters** are data you can change without recompiling: @@ -67,7 +68,7 @@ import sparsediffpy as sp # minimize sum(exp(x) - c * x) subject to ||x||^2 <= 1 scope = sp.Scope() x = scope.Variable(4, 1) -c = np.array([1.0, 2.0, 3.0, 4.0]) +c = np.array([[1.0], [2.0], [3.0], [4.0]]) # constants are 2-D, like variables prob = sp.Problem(sp.sum(sp.exp(x) - c * x), [sp.sum(x ** 2)]) def grad(u): @@ -96,7 +97,8 @@ print(res.x) # approx. [0. 0.289 0.549 0.784], on the boundary ||x|| = 1 ## Supported operations - **Operators:** `+ - * @ **`, `.T`, and NumPy-style indexing (`x[1:3]`, `X[0, :]`, `X[[0, 2], 1]`). - Scalars, rows and columns broadcast. NumPy arrays and `scipy.sparse` matrices are accepted as constants. + Scalars, rows and columns broadcast. NumPy arrays and `scipy.sparse` matrices are accepted as + constants; like every expression they must be 2-D, so pass `a[:, None]` or `a[None, :]` for a 1-D `a`. - **Elementwise:** `sin cos tan exp log sinh tanh asinh atan atanh logistic normal_cdf entr xexp power atan2`. - **Affine:** `sum prod reshape broadcast hstack vstack trace diag_vec index_flat`. - **Other:** `quad_form quad_over_lin rel_entr`. diff --git a/sparsediffpy/_compile.py b/sparsediffpy/_compile.py index 25526d2..4c13367 100644 --- a/sparsediffpy/_compile.py +++ b/sparsediffpy/_compile.py @@ -10,7 +10,7 @@ import scipy.sparse as sps from sparsediffpy import _sparsediffengine as _C -from sparsediffpy._expression import Constant, Parameter, Variable, _wrap +from sparsediffpy._expression import Constant, Parameter, Variable, _flatten, _wrap def _find_scope(exprs): @@ -108,7 +108,10 @@ def _sync_params(self): self._param_version = self._scope._param_version def _u(self, u): - return self._scope._values if u is None else np.asarray(u, np.float64) + """Flat variable vector of length n_vars (the C side does not check it).""" + if u is None: + return self._scope._values[:self.n_vars] + return _flatten(u, self.n_vars) def objective_forward(self, u=None): """Objective value at u (default: the scope's current values).""" @@ -152,8 +155,8 @@ def __init__(self, expr): self._problem = Problem(Constant(0.0), [expr]) def forward(self): - """f(x), flattened column-major.""" - return self._problem.constraint_forward() + """f(x) as an array of shape f.shape.""" + return self._problem.constraint_forward().reshape(self.shape, order="F") def jacobian(self): """Sparse Jacobian df/dx, shape (f.size, n_vars).""" diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py index ad9686b..830f75c 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/_expression.py @@ -91,11 +91,13 @@ def __init__(self, value): self.value = sps.csr_matrix(value, dtype=np.float64) else: value = np.asarray(value, dtype=np.float64) - if value.ndim > 2: - raise ValueError(f"Cannot wrap {value.ndim}D array as constant") - # scalars -> (1, 1), 1-D -> column vector - self.value = value.reshape(value.shape[0], -1) if value.ndim else \ - value.reshape(1, 1) + if value.size == 1: + value = value.reshape(1, 1) + elif value.ndim != 2: + raise ValueError( + f"Constants must be scalars or 2-D arrays, (n, 1) or (1, n); " + f"got shape {value.shape}. Use a[:, None] or a[None, :].") + self.value = value self.shape = _check_shape(*self.value.shape) def dense(self): diff --git a/tests/conftest.py b/tests/conftest.py index a290423..c266207 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,7 +32,8 @@ def jac(x): for j in range(x0.size): e = np.zeros_like(x0) e[j] = h - J_fd[:, j] = (at(x0 + e, fn.forward) - at(x0 - e, fn.forward)) / (2 * h) + df = at(x0 + e, fn.forward) - at(x0 - e, fn.forward) + J_fd[:, j] = df.ravel(order="F") / (2 * h) H_fd[:, j] = (jac(x0 + e) - jac(x0 - e)).T @ w / (2 * h) scope.set_values(x0) np.testing.assert_allclose(J, J_fd, rtol=rtol, atol=atol, err_msg="Jacobian") diff --git a/tests/test_api.py b/tests/test_api.py index 3e53dce..9f82b1d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -40,7 +40,7 @@ def test_reevaluates_at_new_values(scope): fn = sp.compile(sp.exp(x)) for v in ([0.0, 1.0], [2.0, -1.0]): x.value = v - np.testing.assert_allclose(fn.forward(), np.exp(v)) + np.testing.assert_allclose(fn.forward(), np.exp(v)[:, None]) np.testing.assert_allclose(fn.jacobian().diagonal(), np.exp(v)) np.testing.assert_allclose(fn.hessian([1, 1]).diagonal(), np.exp(v)) @@ -49,11 +49,40 @@ def test_two_compiles_share_scope(scope): x, y = scope.Variable(2, 1), scope.Variable(2, 1) f, g = sp.compile(sp.sin(x)), sp.compile(x * y) scope.set_values([1.0, 2.0, 3.0, 4.0]) - np.testing.assert_allclose(f.forward(), np.sin([1, 2])) - np.testing.assert_allclose(g.forward(), [3, 8]) + np.testing.assert_allclose(f.forward(), np.sin([[1], [2]])) + np.testing.assert_allclose(g.forward(), [[3], [8]]) assert f.jacobian().shape == g.jacobian().shape == (2, 4) +def test_forward_has_expression_shape(scope): + x, M = scope.Variable(1, 3), scope.Variable(2, 2) + scope.set_values([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0]) + assert sp.compile(2.0 * x).forward().shape == (1, 3) + out = sp.compile(sp.exp(M)).forward() + assert out.shape == (2, 2) + np.testing.assert_allclose(out, np.exp([[4.0, 6.0], [5.0, 7.0]])) # column-major + + +def test_constants_must_be_2d(scope): + x = scope.Variable(3, 1) + for bad in (np.ones(3), [1.0, 2.0, 3.0], np.zeros((2, 2, 2))): + with pytest.raises(ValueError, match="2-D"): + x + bad + with pytest.raises(ValueError, match="2-D"): + bad * x + assert (x + np.array([2.0])).shape == (x + 2.0).shape == (3, 1) + assert (x + np.ones((3, 1))).shape == (3, 1) + + +def test_variable_declared_after_compile(scope): + x = scope.Variable(2, 1) + fn = sp.compile(sp.exp(x)) + scope.Variable(3, 1) + x.value = [0.0, 1.0] + np.testing.assert_allclose(fn.forward(), np.exp([[0.0], [1.0]])) + assert fn.jacobian().shape == (2, 2) + + def test_affine_hessian_is_zero(scope): x = scope.Variable(3, 1) fn = sp.compile(sp.sum(-x[0:2]) + 2.0) @@ -128,7 +157,7 @@ def test_parameter_update_without_recompile(scope): x.value = [0.1, 0.2, 0.3] for Av, cv in [(np.ones((2, 3)), 1.0), (np.arange(6.0).reshape(2, 3), -2.0)]: A.value, c.value = Av, cv - np.testing.assert_allclose(fn.forward(), cv * np.exp(Av @ x.value).ravel()) + np.testing.assert_allclose(fn.forward(), cv * np.exp(Av @ x.value)) J = cv * np.exp(Av @ x.value) * Av np.testing.assert_allclose(fn.jacobian().toarray(), J) @@ -139,7 +168,7 @@ def test_unset_parameter_raises(scope): with pytest.raises(ValueError, match="no value"): fn.forward() p.value = [1.0, 2.0] - np.testing.assert_allclose(fn.forward(), [0, 0]) + np.testing.assert_allclose(fn.forward(), [[0], [0]]) # --- Problem ---------------------------------------------------------------------- @@ -168,6 +197,12 @@ def test_problem(scope): with pytest.raises(ValueError, match="lagrange"): prob.hessian(1.0, np.ones(4)) + for bad in (np.zeros(2), np.zeros(4)): + with pytest.raises(ValueError, match="expected 3 elements"): + prob.objective_forward(bad) + with pytest.raises(ValueError, match="expected 3 elements"): + prob.constraint_forward(bad) + assert prob.objective_forward(u[:, None]) == pytest.approx(np.exp(s).sum()) with pytest.raises(ValueError, match="scalar"): sp.Problem(x) diff --git a/tests/test_atoms.py b/tests/test_atoms.py index 578003f..3778fc7 100644 --- a/tests/test_atoms.py +++ b/tests/test_atoms.py @@ -67,7 +67,7 @@ def add_broadcast(s): @case() def neg_sub(s): x, y = s.Variable(3, 1), s.Variable(3, 1) - return -x - y + np.arange(3.0), lambda: -x.value - y.value + np.c_[0:3.0] + return -x - y + np.c_[0:3.0], lambda: -x.value - y.value + np.c_[0:3.0] @case() @@ -230,8 +230,8 @@ def test_atom(name, rng): x0 = rng.uniform(low, high, scope.total_var_size) scope.set_values(x0) out = fn.forward() - assert out.shape == (f.size,) + assert out.shape == f.shape if ref is not None: - expected = np.broadcast_to(ref(), f.shape).ravel(order="F") + expected = np.broadcast_to(ref(), f.shape) np.testing.assert_allclose(out, expected, rtol=1e-10, atol=1e-12) check_derivatives(fn, scope, x0, rng.standard_normal(f.size)) From 4a1d692f40f89a994f6de0b8a2028adf318b6995 Mon Sep 17 00:00:00 2001 From: dance858 Date: Sat, 7 Nov 2026 16:32:31 -0800 Subject: [PATCH 05/17] Evaluate each stage once per point Problem remembers the point at which each stage (objective forward, constraint forward, gradient, Jacobian) was last computed and skips the engine call when asked again for the same point. A point is the scope's values version plus the parameter version for u=None, or the last explicit u (one copy kept, compared with array_equal) so scipy's separate fun/jac callbacks at the same x cost one forward. gradient()/jacobian() raise if no forward has run, and hessian() requires both forwards at the same point instead of silently mixing Jacobian values from two points. forward(); jacobian(); hessian(w) now runs one forward and one Jacobian in the engine instead of three and two. Co-Authored-By: Claude Fable 5.1 --- README.md | 12 ++++-- sparsediffpy/_compile.py | 74 +++++++++++++++++++++++++-------- sparsediffpy/_expression.py | 3 ++ tests/test_api.py | 81 +++++++++++++++++++++++++++++++++++++ 4 files changed, 150 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 421f4be..ed2c064 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,11 @@ J2 = fn.jacobian() `sp.Problem(objective, constraints)` exposes the callbacks NLP solvers ask for: objective value and gradient, constraint values and Jacobian, and the Lagrangian -Hessian `obj_factor * H_f + sum_i lagrange_i * H_gi`. Here it drives SciPy: +Hessian `obj_factor * H_f + sum_i lagrange_i * H_gi`. A stage runs in the engine +only when its point changed: a forward called again with the same `u` (or with +unchanged scope values and parameters) returns the cached result, and +`gradient()`/`jacobian()` are reused until the matching forward moves. Returned +arrays are owned by the problem, so do not modify them in place. Here it drives SciPy: ```python import numpy as np @@ -75,8 +79,8 @@ def grad(u): prob.objective_forward(u) return prob.gradient() -def obj_hess(u): - prob.objective_forward(u) +def obj_hess(u): # the Lagrangian Hessian needs both forwards at u (cached if done) + prob.objective_forward(u); prob.constraint_forward(u) return prob.hessian(1.0, [0.0]) def con_jac(u): @@ -84,7 +88,7 @@ def con_jac(u): return prob.jacobian() def con_hess(u, v): - prob.constraint_forward(u) + prob.objective_forward(u); prob.constraint_forward(u) return prob.hessian(0.0, v) con = NonlinearConstraint(prob.constraint_forward, -np.inf, 1.0, diff --git a/sparsediffpy/_compile.py b/sparsediffpy/_compile.py index 4c13367..e7d8ccc 100644 --- a/sparsediffpy/_compile.py +++ b/sparsediffpy/_compile.py @@ -74,6 +74,12 @@ class Problem: prob.objective_forward(u); prob.constraint_forward(u) prob.gradient(); prob.jacobian(); prob.hessian(obj_factor, lagrange) + + Each stage runs in the engine only when its point changed: a forward is + skipped when u (or the scope's values and parameters) is unchanged, and + gradient/jacobian are skipped when the matching forward has not moved. + Skipped calls return the cached object, which the problem owns: do not + modify it in place. """ def __init__(self, objective, constraints=(), verbose=False): @@ -92,7 +98,11 @@ def __init__(self, objective, constraints=(), verbose=False): _C.problem_init_derivatives(self._capsule) self._param_version = None self.n_vars = self._scope.total_var_size - self.total_constraint_size = np.sum([c.size for c in constraints]).item() + self.total_constraint_size = sum(c.size for c in constraints) + # Point keys: ("scope", values_version, param_version) or ("u", u_version, + # param_version). Each stage remembers the key it was last computed at. + self._last_u, self._u_version = None, 0 + self._obj_at = self._con_at = self._grad_at = self._jac_at = None def _sync_params(self): """Push Parameter values to the engine if any changed since last time.""" @@ -107,43 +117,74 @@ def _sync_params(self): _C.problem_update_params(self._capsule, theta) self._param_version = self._scope._param_version - def _u(self, u): - """Flat variable vector of length n_vars (the C side does not check it).""" + def _point(self, u): + """Return (key, flat u) for the evaluation point u (None: scope values). + + The C side does not check the length of u, so it is checked here. + """ if u is None: - return self._scope._values[:self.n_vars] - return _flatten(u, self.n_vars) + key = ("scope", self._scope._values_version) + u = self._scope._values[:self.n_vars] + else: + u = _flatten(u, self.n_vars) + if self._last_u is None or not np.array_equal(u, self._last_u): + self._last_u, self._u_version = u.copy(), self._u_version + 1 + key = ("u", self._u_version) + return key + (self._scope._param_version,), u def objective_forward(self, u=None): """Objective value at u (default: the scope's current values).""" - self._sync_params() - return _C.problem_objective_forward(self._capsule, self._u(u)) + key, u = self._point(u) + if self._obj_at != key: + self._sync_params() + self._obj_value = _C.problem_objective_forward(self._capsule, u) + self._obj_at = key + return self._obj_value def constraint_forward(self, u=None): """Stacked constraint values at u (default: the scope's current values).""" - self._sync_params() - return _C.problem_constraint_forward(self._capsule, self._u(u)) + key, u = self._point(u) + if self._con_at != key: + self._sync_params() + self._con_value = _C.problem_constraint_forward(self._capsule, u) + self._con_at = key + return self._con_value def gradient(self): """Objective gradient at the point of the last objective_forward.""" - return _C.problem_gradient(self._capsule) + if self._obj_at is None: + raise RuntimeError("call objective_forward before gradient") + if self._grad_at != self._obj_at: + self._grad_value = _C.problem_gradient(self._capsule) + self._grad_at = self._obj_at + return self._grad_value def jacobian(self): """Constraint Jacobian (csr_matrix) at the last constraint_forward point.""" - return _csr(_C.problem_jacobian(self._capsule)) + if self._con_at is None: + raise RuntimeError("call constraint_forward before jacobian") + if self._jac_at != self._con_at: + self._jac_value = _csr(_C.problem_jacobian(self._capsule)) + self._jac_at = self._con_at + return self._jac_value def hessian(self, obj_factor, lagrange): """Lagrangian Hessian obj_factor * H_f + sum_i lagrange_i * H_gi (csr_matrix). - Call objective_forward and constraint_forward at the point first. + Call objective_forward and constraint_forward at the same point first. """ lagrange = np.asarray(lagrange, dtype=np.float64).ravel(order="F") if lagrange.size != self.total_constraint_size: raise ValueError(f"lagrange has {lagrange.size} entries, expected " f"{self.total_constraint_size}") - # The engine's Hessian reuses Jacobian values cached by eval_jacobian, - # so those must be current at this point. - _C.problem_gradient(self._capsule) - _C.problem_jacobian(self._capsule) + # The engine's Hessian reads Jacobian values cached by eval_jacobian, so + # the gradient and Jacobian must be current (no-ops if they already are). + self.gradient() + if self.total_constraint_size: + self.jacobian() + if self._con_at != self._obj_at: + raise RuntimeError("objective_forward and constraint_forward must " + "be evaluated at the same point before hessian") return _csr(_C.problem_hessian(self._capsule, float(obj_factor), lagrange)) @@ -165,6 +206,7 @@ def jacobian(self): def hessian(self, weights): """Sparse Hessian of w' f(x), shape (n_vars, n_vars).""" + self._problem.objective_forward() # the zero objective; needed by gradient() self.forward() return self._problem.hessian(0.0, weights) diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py index 830f75c..2b5d090 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/_expression.py @@ -119,6 +119,7 @@ def value(self): def value(self, val): self._scope._values[self._offset:self._offset + self.size] = \ _flatten(val, self.size) + self._scope._values_version += 1 class Parameter(Expression): @@ -154,6 +155,7 @@ class Scope: def __init__(self): self._values = np.zeros(0) + self._values_version = 0 # bumped on every write, so problems can skip stale work self._n_params = 0 self._param_version = 0 @@ -178,6 +180,7 @@ def total_var_size(self): def set_values(self, array): """Set all variables at once from a flat array (declaration order).""" self._values[:] = _flatten(array, self._values.size) + self._values_version += 1 def get_values(self): return self._values.copy() diff --git a/tests/test_api.py b/tests/test_api.py index 9f82b1d..2f09862 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -4,6 +4,7 @@ import pytest import sparsediffpy as sp +from sparsediffpy import _compile # --- scope and values --------------------------------------------------------- @@ -171,6 +172,86 @@ def test_unset_parameter_raises(scope): np.testing.assert_allclose(fn.forward(), [[0], [0]]) +# --- Caching ------------------------------------------------------------------------ + +def count_calls(monkeypatch, name): + """Count calls to the engine function `name` as seen from _compile.""" + calls, real = [], getattr(_compile._C, name) + + def counted(*args): + calls.append(1) + return real(*args) + monkeypatch.setattr(_compile._C, name, counted) + return calls + + +def test_forward_runs_once_per_point(scope, monkeypatch): + n_fwd = count_calls(monkeypatch, "problem_constraint_forward") + x, p = scope.Variable(2, 1), scope.Parameter(1, 1, value=1.0) + fn = sp.compile(p * sp.exp(x)) + x.value = [0.0, 1.0] + for _ in range(3): + np.testing.assert_allclose(fn.forward(), np.exp([[0.0], [1.0]])) + assert len(n_fwd) == 1 + x.value = [1.0, 1.0] + fn.forward() + scope.set_values([2.0, 2.0]) + fn.forward() + assert len(n_fwd) == 3 + p.value = 3.0 + np.testing.assert_allclose(fn.forward(), 3.0 * np.exp([[2.0], [2.0]])) + assert len(n_fwd) == 4 + + +def test_derivatives_reuse_forward_and_jacobian(scope, monkeypatch): + n_fwd = count_calls(monkeypatch, "problem_constraint_forward") + n_jac = count_calls(monkeypatch, "problem_jacobian") + n_hess = count_calls(monkeypatch, "problem_hessian") + x = scope.Variable(2, 1) + fn = sp.compile(sp.exp(x)) + x.value = [0.5, -0.5] + fn.forward() + J = fn.jacobian() + assert fn.jacobian() is J + fn.hessian([1.0, 1.0]) + fn.hessian([2.0, 0.0]) + assert (len(n_fwd), len(n_jac), len(n_hess)) == (1, 1, 2) + x.value = [0.0, 0.0] + fn.hessian([1.0, 1.0]) + assert (len(n_fwd), len(n_jac), len(n_hess)) == (2, 2, 3) + + +def test_problem_skips_repeated_u(scope, monkeypatch): + n_fwd = count_calls(monkeypatch, "problem_constraint_forward") + x = scope.Variable(2, 1) + prob = sp.Problem(sp.sum(x), [sp.exp(x)]) + prob.constraint_forward(np.array([1.0, 2.0])) + prob.constraint_forward(np.array([1.0, 2.0])) # equal, distinct array + assert len(n_fwd) == 1 + prob.constraint_forward(np.array([1.0, 3.0])) + assert len(n_fwd) == 2 + prob.constraint_forward() # scope values are a different point + assert len(n_fwd) == 3 + np.testing.assert_allclose(prob.constraint_forward(), [1.0, 1.0]) + assert len(n_fwd) == 3 + + +def test_derivatives_require_forward(scope): + x = scope.Variable(2, 1) + prob = sp.Problem(sp.sum(x), [sp.exp(x)]) + with pytest.raises(RuntimeError, match="objective_forward"): + prob.gradient() + with pytest.raises(RuntimeError, match="constraint_forward"): + prob.jacobian() + prob.objective_forward([1.0, 2.0]) + prob.constraint_forward([1.0, 3.0]) + with pytest.raises(RuntimeError, match="same point"): + prob.hessian(1.0, [1.0, 1.0]) + prob.objective_forward([1.0, 2.0]) # a point is "the last u": redo both forwards + prob.constraint_forward([1.0, 2.0]) + assert prob.hessian(1.0, [1.0, 1.0]).shape == (2, 2) + + # --- Problem ---------------------------------------------------------------------- def test_problem(scope): From 70a839c18520c3b5ae52408576eb702778bf5296 Mon Sep 17 00:00:00 2001 From: dance858 Date: Sat, 7 Nov 2026 16:50:40 -0800 Subject: [PATCH 06/17] Require two indices on matrices and integer index arrays X[i] on a matrix used to gather by column-major flat position, a silent departure from NumPy's row selection. It now raises; vectors still take a single index and matrices take X[i, j], X[i, :] or X[:, j]. Boolean and multi-dimensional index arrays raise instead of being cast to integers. Co-Authored-By: Claude Fable 5.1 --- README.md | 3 ++- sparsediffpy/_expression.py | 13 ++++++++++--- tests/test_api.py | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ed2c064..144825b 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,8 @@ print(res.x) # approx. [0. 0.289 0.549 0.784], on the boundary ||x|| = 1 ## Supported operations -- **Operators:** `+ - * @ **`, `.T`, and NumPy-style indexing (`x[1:3]`, `X[0, :]`, `X[[0, 2], 1]`). +- **Operators:** `+ - * @ **`, `.T`, and indexing with ints, slices and integer lists: `x[1:3]` on a + vector, `X[0, :]` or `X[[0, 2], 1]` on a matrix (a matrix always takes two indices). Scalars, rows and columns broadcast. NumPy arrays and `scipy.sparse` matrices are accepted as constants; like every expression they must be 2-D, so pass `a[:, None]` or `a[None, :]` for a 1-D `a`. - **Elementwise:** `sin cos tan exp log sinh tanh asinh atan atanh logistic normal_cdf entr xexp power atan2`. diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py index 2b5d090..7185173 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/_expression.py @@ -284,7 +284,10 @@ def _axis_indices(key, n): if isinstance(key, slice): return np.arange(n)[key] if isinstance(key, (list, np.ndarray)): - key = np.asarray(key, dtype=np.int64) + key = np.asarray(key) + if key.ndim != 1 or not np.issubdtype(key.dtype, np.integer): + raise IndexError("Index arrays must be 1-D with integer entries " + f"(got shape {key.shape}, dtype {key.dtype})") if key.size and (key.min() < -n or key.max() >= n): raise IndexError(f"Index out of range for axis of length {n}") return key % n @@ -292,12 +295,16 @@ def _axis_indices(key, n): def _index(x, key): + """x[i, j] on anything; x[i] only on vectors, where it indexes along the axis.""" d1, d2 = x.shape if isinstance(key, tuple): if len(key) != 2: - raise IndexError("Only 1D or 2D indexing supported") + raise IndexError(f"Expected 2 indices for shape {x.shape}, got {len(key)}") rows, cols = _axis_indices(key[0], d1), _axis_indices(key[1], d2) flat = (rows[:, None] + cols[None, :] * d1).ravel(order="F") return index_flat(x, flat, (rows.size, cols.size)) - flat = _axis_indices(key, x.size) # column-major flat index + if d1 > 1 and d2 > 1: + raise IndexError(f"A single index on shape {x.shape} is ambiguous; " + "use X[i, j], X[i, :] or X[:, j]") + flat = _axis_indices(key, x.size) return index_flat(x, flat, (1, flat.size) if d1 == 1 else (flat.size, 1)) diff --git a/tests/test_api.py b/tests/test_api.py index 2f09862..c930561 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -139,6 +139,20 @@ def test_index_out_of_range(scope, key): X[key] +def test_index_must_be_explicit(scope): + x, X = scope.Variable(4, 1), scope.Variable(2, 3) + for key in (1, slice(0, 2), [0, 1]): + with pytest.raises(IndexError, match="ambiguous"): + X[key] + assert X[1, :].shape == (1, 3) and X[:, 1].shape == (2, 1) + with pytest.raises(IndexError, match="integer"): + x[[True, False, True, False]] + with pytest.raises(IndexError, match="1-D"): + x[np.array([[0, 1]])] + with pytest.raises(IndexError, match="2 indices"): + X[0, 0, 0] + + def test_compile_errors(scope): with pytest.raises(ValueError, match="at least one Variable"): sp.compile(sp.Scope().Parameter(1, 1, value=1.0) * 2.0) From e6619cec87f53370c65f64ff4d61bc1255a065b8 Mon Sep 17 00:00:00 2001 From: dance858 Date: Sun, 8 Nov 2026 05:42:36 -0800 Subject: [PATCH 07/17] Require exact shapes in value setters Value inputs used to be raveled column-major and only checked for element count, so a (3, 2) or flat (6,) array was silently accepted for a (2, 3) variable. Now: - Variable.value, Parameter.value and CompiledExpression.hessian(weights) take the exact shape of the variable, parameter or f. A 1-D array is also accepted for vector shapes and a scalar for (1, 1). - scope.set_values, the Problem point u and Problem.hessian lagrange must be 1-D with exactly n entries. Co-Authored-By: Claude Opus 5.5 --- README.md | 7 ++++-- sparsediffpy/_compile.py | 15 +++++++------ sparsediffpy/_expression.py | 32 +++++++++++++++++++------- tests/conftest.py | 4 +++- tests/test_api.py | 45 ++++++++++++++++++++++++++++++++++--- 5 files changed, 82 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 144825b..b11e3b7 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,11 @@ print(fn.hessian(np.ones(3))) # Hessian of w' f(x) for weights w `forward()` has the shape of `f`, `jacobian()` has shape `(f.size, n_vars)` and `hessian(w)` has shape `(n_vars, n_vars)`, where `n_vars` is the total -size of all variables in the scope, in declaration order. Flat vectors (the -rows of the Jacobian, `x.value = [...]`, the weights `w`) are column-major. +size of all variables in the scope, in declaration order. Values must have +the exact shape of what they set: `x.value` takes an array shaped like `x`, +`hessian(w)` takes `w` shaped like `f` (for vectors a 1-D array also works), +and flat vectors such as `scope.set_values(u)` must be 1-D. Flat vectors (the +rows of the Jacobian, the scope values) are column-major. **Parameters** are data you can change without recompiling: diff --git a/sparsediffpy/_compile.py b/sparsediffpy/_compile.py index e7d8ccc..604e244 100644 --- a/sparsediffpy/_compile.py +++ b/sparsediffpy/_compile.py @@ -10,7 +10,7 @@ import scipy.sparse as sps from sparsediffpy import _sparsediffengine as _C -from sparsediffpy._expression import Constant, Parameter, Variable, _flatten, _wrap +from sparsediffpy._expression import Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, _wrap def _find_scope(exprs): @@ -126,7 +126,7 @@ def _point(self, u): key = ("scope", self._scope._values_version) u = self._scope._values[:self.n_vars] else: - u = _flatten(u, self.n_vars) + u = _flatten_vector(u, self.n_vars, "u") if self._last_u is None or not np.array_equal(u, self._last_u): self._last_u, self._u_version = u.copy(), self._u_version + 1 key = ("u", self._u_version) @@ -173,10 +173,7 @@ def hessian(self, obj_factor, lagrange): Call objective_forward and constraint_forward at the same point first. """ - lagrange = np.asarray(lagrange, dtype=np.float64).ravel(order="F") - if lagrange.size != self.total_constraint_size: - raise ValueError(f"lagrange has {lagrange.size} entries, expected " - f"{self.total_constraint_size}") + lagrange = _flatten_vector(lagrange, self.total_constraint_size, "lagrange") # The engine's Hessian reads Jacobian values cached by eval_jacobian, so # the gradient and Jacobian must be current (no-ops if they already are). self.gradient() @@ -205,7 +202,11 @@ def jacobian(self): return self._problem.jacobian() def hessian(self, weights): - """Sparse Hessian of w' f(x), shape (n_vars, n_vars).""" + """Sparse Hessian of sum(weights * f(x)), shape (n_vars, n_vars). + + weights has f's shape (a 1-D array is also accepted when f is a vector). + """ + weights = _flatten_shaped(weights, self.shape) self._problem.objective_forward() # the zero objective; needed by gradient() self.forward() return self._problem.hessian(0.0, weights) diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py index 7185173..ac68b3b 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/_expression.py @@ -118,7 +118,7 @@ def value(self): @value.setter def value(self, val): self._scope._values[self._offset:self._offset + self.size] = \ - _flatten(val, self.size) + _flatten_shaped(val, self.shape) self._scope._values_version += 1 @@ -139,15 +139,31 @@ def value(self): @value.setter def value(self, val): - self._flat = _flatten(val, self.size) + self._flat = _flatten_shaped(val, self.shape) self._scope._param_version += 1 -def _flatten(val, size): - flat = np.asarray(val, dtype=np.float64).ravel(order="F") - if flat.size != size: - raise DimensionError(f"expected {size} elements, got {flat.size}") - return flat +def _flatten_shaped(val, shape): + """Flatten a value of shape ``shape`` column-major. + + Also accepts a 1-D array for a vector shape (n, 1) or (1, n), and a scalar + for (1, 1). Any other shape raises, even with the right element count. + """ + arr = np.asarray(val, dtype=np.float64) + if not (arr.shape == shape + or (arr.ndim == 0 and shape == (1, 1)) + or (arr.ndim == 1 and min(shape) == 1 and arr.size == shape[0] * shape[1])): + raise DimensionError(f"expected shape {shape}, got {arr.shape}") + return arr.ravel(order="F") + + +def _flatten_vector(val, n, name): + """Return ``val`` as a float64 1-D array of exactly n entries.""" + arr = np.asarray(val, dtype=np.float64) + if arr.ndim != 1 or arr.size != n: + raise DimensionError(f"{name} must be a 1-D array of {n} elements, " + f"got shape {arr.shape}") + return arr class Scope: @@ -179,7 +195,7 @@ def total_var_size(self): def set_values(self, array): """Set all variables at once from a flat array (declaration order).""" - self._values[:] = _flatten(array, self._values.size) + self._values[:] = _flatten_vector(array, self._values.size, "values") self._values_version += 1 def get_values(self): diff --git a/tests/conftest.py b/tests/conftest.py index c266207..bd8e38b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,7 +26,9 @@ def jac(x): return at(x, fn.jacobian).toarray() J = jac(x0) - H = at(x0, lambda: fn.hessian(w)).toarray() + w = np.asarray(w, dtype=np.float64).ravel(order="F") + W = w.reshape(fn.shape, order="F") # hessian() takes weights shaped like f + H = at(x0, lambda: fn.hessian(W)).toarray() J_fd = np.zeros_like(J) H_fd = np.zeros_like(H) for j in range(x0.size): diff --git a/tests/test_api.py b/tests/test_api.py index c930561..690d53c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -30,6 +30,42 @@ def test_wrong_sizes(scope): assert p.value is None +def test_value_shapes(scope): + """Values need the exact shape, not just the right number of elements.""" + X, r, c, y = (scope.Variable(2, 3), scope.Variable(1, 3), + scope.Variable(3, 1), scope.Variable(1, 1)) + P = scope.Parameter(2, 3) + for leaf in (X, P): + for bad in (np.zeros(6), np.zeros((3, 2)), np.zeros((6, 1)), 1.0): + with pytest.raises(sp.DimensionError, match="expected shape"): + leaf.value = bad + leaf.value = np.arange(6.0).reshape(2, 3) + np.testing.assert_array_equal(leaf.value, np.arange(6.0).reshape(2, 3)) + with pytest.raises(sp.DimensionError): + r.value = np.zeros((3, 1)) + with pytest.raises(sp.DimensionError): + c.value = np.zeros((1, 3)) + r.value, c.value, y.value = [1.0, 2.0, 3.0], [4.0, 5.0, 6.0], 7.0 + np.testing.assert_array_equal(r.value, [[1, 2, 3]]) + np.testing.assert_array_equal(c.value, [[4], [5], [6]]) + y.value = [8.0] + assert y.value.shape == (1, 1) + + n = scope.total_var_size + with pytest.raises(sp.DimensionError, match="values"): + scope.set_values(np.zeros((n, 1))) + scope.set_values(np.zeros(n)) + + fn = sp.compile(sp.exp(X)) + with pytest.raises(sp.DimensionError): + fn.hessian(np.ones(6)) + assert fn.hessian(np.ones((2, 3))).shape == (n, n) + fn = sp.compile(sp.exp(c)) + assert fn.hessian(np.ones(3)).shape == fn.hessian(np.ones((3, 1))).shape + with pytest.raises(sp.DimensionError): + fn.hessian(np.ones((1, 3))) + + @pytest.mark.parametrize("shape", [(0, 1), (-1, 2), (2.0, 1)]) def test_invalid_shapes(scope, shape): with pytest.raises((ValueError, TypeError)): @@ -293,11 +329,14 @@ def test_problem(scope): with pytest.raises(ValueError, match="lagrange"): prob.hessian(1.0, np.ones(4)) for bad in (np.zeros(2), np.zeros(4)): - with pytest.raises(ValueError, match="expected 3 elements"): + with pytest.raises(sp.DimensionError, match="1-D array of 3 elements"): prob.objective_forward(bad) - with pytest.raises(ValueError, match="expected 3 elements"): + with pytest.raises(sp.DimensionError, match="1-D array of 3 elements"): prob.constraint_forward(bad) - assert prob.objective_forward(u[:, None]) == pytest.approx(np.exp(s).sum()) + with pytest.raises(sp.DimensionError): + prob.objective_forward(u[:, None]) + with pytest.raises(sp.DimensionError, match="lagrange"): + prob.hessian(1.0, lam[:, None]) with pytest.raises(ValueError, match="scalar"): sp.Problem(x) From 1c9ebf95340b4962044304a04e0899f516fc963d Mon Sep 17 00:00:00 2001 From: dance858 Date: Sun, 8 Nov 2026 05:47:39 -0800 Subject: [PATCH 08/17] Reject bare boolean indices bool subclasses int, so x[True] and x[False] were treated as x[1] and x[0] instead of raising. Boolean index arrays were already rejected; now scalar bools (Python and NumPy) are rejected too. Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/_expression.py | 2 ++ tests/test_api.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py index ac68b3b..a9153ba 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/_expression.py @@ -293,6 +293,8 @@ def index_flat(x, flat_indices, shape): def _axis_indices(key, n): + if isinstance(key, (bool, np.bool_)): # bool subclasses int; x[True] is not x[1] + raise IndexError("Boolean indices are not supported; use integer indices") if isinstance(key, (int, np.integer)): if not -n <= key < n: raise IndexError(f"Index {key} out of range for axis of length {n}") diff --git a/tests/test_api.py b/tests/test_api.py index 690d53c..548c2f2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -183,6 +183,9 @@ def test_index_must_be_explicit(scope): assert X[1, :].shape == (1, 3) and X[:, 1].shape == (2, 1) with pytest.raises(IndexError, match="integer"): x[[True, False, True, False]] + for key in (True, False, np.True_, (True, 0), (0, False)): + with pytest.raises(IndexError, match="Boolean"): + x[key] with pytest.raises(IndexError, match="1-D"): x[np.array([[0, 1]])] with pytest.raises(IndexError, match="2 indices"): From 12e8d65b7946b0f9e8bc20cb09583e4e669e5ab1 Mon Sep 17 00:00:00 2001 From: dance858 Date: Sun, 8 Nov 2026 06:11:26 -0800 Subject: [PATCH 09/17] Build capsules iteratively so deep expressions compile _Builder.__call__ recursed once per tree level, so a sum built term by term in a Python loop raised RecursionError at compile time from about 1,000 terms. Raising the recursion limit does not help: Python 3.14 then reports a stack overflow around 7,000 levels. The builder now walks the tree with an explicit stack, like _find_scope. Nodes are built in the same left-to-right post-order as before, so the capsules and the parameter registration order are unchanged. Tests cover a 5,000-term sum with a parameter (values, Jacobian, Hessian, and a parameter update) and a 3,000-level nonlinear chain through both compile() and Problem, checked against closed forms. Remaining limits are in the engine, not the Python layer: its C recursion overflows the stack somewhere between 1e5 and 1e6 levels, problem_init_derivatives is roughly quadratic in depth for long add chains, memory grows with nodes x n_vars, and a shared subexpression is re-evaluated once per use. Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/_compile.py | 31 +++++++++++++++++---- tests/test_api.py | 60 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/sparsediffpy/_compile.py b/sparsediffpy/_compile.py index 604e244..1227358 100644 --- a/sparsediffpy/_compile.py +++ b/sparsediffpy/_compile.py @@ -33,16 +33,36 @@ def _find_scope(exprs): class _Builder: - """Builds C capsules bottom-up, sharing repeated subexpressions.""" + """Builds C capsules bottom-up, sharing repeated subexpressions. + + The walk uses an explicit stack rather than recursion, so deep expressions + (e.g. a long sum built in a Python loop) do not hit Python's recursion + limit. Nodes are built in the same order as a recursive left-to-right + post-order traversal. + """ def __init__(self, scope): self.n_vars = scope.total_var_size self.cache = {} self.params = [] # (Parameter, capsule), each once - def __call__(self, e): - if id(e) in self.cache: - return self.cache[id(e)] + def __call__(self, root): + stack = [root] + while stack: + e = stack[-1] + if id(e) in self.cache: + stack.pop() + continue + pending = [a for a in e.args if id(a) not in self.cache] + if pending: + stack.extend(reversed(pending)) # build the leftmost child first + continue + stack.pop() + self.cache[id(e)] = self._build_one(e) + return self.cache[id(root)] + + def _build_one(self, e): + """Build e's capsule; all of e's children must already be cached.""" d1, d2 = e.shape if isinstance(e, Variable): cap = _C.make_variable(d1, d2, e._offset, self.n_vars) @@ -54,11 +74,10 @@ def __call__(self, e): values = e.dense().ravel(order="F") cap = _C.make_parameter(d1, d2, -1, self.n_vars, values) else: - cap = e.make(*[self(a) for a in e.args]) + cap = e.make(*[self.cache[id(a)] for a in e.args]) if _C.get_expr_dimensions(cap) != e.shape: raise ValueError(f"Engine shape {_C.get_expr_dimensions(cap)} != " f"expected {e.shape}") - self.cache[id(e)] = cap return cap diff --git a/tests/test_api.py b/tests/test_api.py index 548c2f2..827a7b5 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,5 +1,7 @@ """Scope/values, shape rules, validation errors, parameters, and Problem.""" +import sys + import numpy as np import pytest @@ -374,3 +376,61 @@ def test_hessian_without_prior_jacobian(scope): H_prob = np.block([[np.zeros((2, 2)), np.diag(2.0 + w)], [np.diag(2.0 + w), np.zeros((2, 2))]]) np.testing.assert_allclose(prob.hessian(2.0, w).toarray(), H_prob, atol=1e-12) + + +# --- deep expressions ------------------------------------------------------------- + +def test_compile_deep_sum(scope): + """A sum built term by term in a loop is a chain deeper than the recursion limit.""" + x, p = scope.Variable(3, 1), scope.Parameter(1, 1, value=2.0) + n_terms = 5000 + assert n_terms > sys.getrecursionlimit() # each term adds one level + f = x[0] + for i in range(1, n_terms + 1): + f = f + p * sp.sin(x[i % 3]) + fn = sp.compile(f) + u = np.array([0.3, -0.7, 1.1]) + x.value = u + counts = np.bincount(np.arange(1, n_terms + 1) % 3, minlength=3) + for pv in (2.0, 3.0): + p.value = pv + np.testing.assert_allclose(fn.forward(), [[u[0] + pv * counts @ np.sin(u)]]) + np.testing.assert_allclose(fn.jacobian().toarray(), + [np.eye(3)[0] + pv * counts * np.cos(u)]) + np.testing.assert_allclose(fn.hessian(1.0).toarray(), + np.diag(-pv * counts * np.sin(u)), atol=1e-9) + + +def test_compile_deep_nonlinear_chain(scope): + """A deep chain of nonlinear atoms that all reuse one shared subexpression. + + The shared node is shallow on purpose: the engine re-evaluates a shared + subexpression once per use, so reusing a deep one would take exponential time. + """ + x = scope.Variable(1, 1) + n_levels, c = 3000, 1e-3 + shared = sp.sin(x) + g = x + for _ in range(n_levels): + g = sp.sin(g) + c * shared + + x0 = 0.7 + v, d1, d2 = x0, 1.0, 0.0 # g, dg/dx, d2g/dx2 by the chain rule + for _ in range(n_levels): + d2 = np.cos(v) * d2 - np.sin(v) * d1 ** 2 - c * np.sin(x0) + d1 = np.cos(v) * d1 + c * np.cos(x0) + v = np.sin(v) + c * np.sin(x0) + + x.value = x0 + fn = sp.compile(g) + np.testing.assert_allclose(fn.forward(), [[v]], rtol=1e-12) + np.testing.assert_allclose(fn.jacobian().toarray(), [[d1]], rtol=1e-10) + np.testing.assert_allclose(fn.hessian(1.0).toarray(), [[d2]], rtol=1e-8) + + prob = sp.Problem(sp.sum(g), [g]) + u = np.array([x0]) + assert prob.objective_forward(u) == pytest.approx(v, rel=1e-12) + np.testing.assert_allclose(prob.gradient(), [d1], rtol=1e-10) + np.testing.assert_allclose(prob.constraint_forward(u), [v], rtol=1e-12) + np.testing.assert_allclose(prob.jacobian().toarray(), [[d1]], rtol=1e-10) + np.testing.assert_allclose(prob.hessian(1.0, [2.0]).toarray(), [[3 * d2]], rtol=1e-8) From 621b4edcb79e09fd4ff6951a0015bf102c9cdaac Mon Sep 17 00:00:00 2001 From: dance858 Date: Sun, 8 Nov 2026 11:05:40 -0800 Subject: [PATCH 10/17] Fix review findings: CI test step, quad_form symmetry, input checks - Run pytest in the cibuildwheel test step. The [tool.cibuildwheel] test-command in pyproject.toml was overridden by the workflow's CIBW_TEST_COMMAND env var, so the suite never ran on the wheels. - quad_form with a Parameter Q: the engine assumes Q symmetric, so check symmetry when the atom is built and on every later Q.value update. - Expression.__iter__ raises TypeError instead of silently iterating via __getitem__ (a bare expression passed as `constraints` gave 0 constraints). - Reject None in Constant and in value setters instead of storing NaN. - index_flat wraps its argument like the other atoms. - __pow__ accepts any numbers.Real, including NumPy scalars. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/build-and-publish.yml | 6 ++-- pyproject.toml | 4 --- sparsediffpy/_atoms.py | 8 +++-- sparsediffpy/_expression.py | 26 ++++++++++++-- tests/test_api.py | 48 +++++++++++++++++++++++++ 5 files changed, 82 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 6b4a85f..644721c 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -74,8 +74,10 @@ jobs: CIBW_ENVIRONMENT_WINDOWS: "CMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake SKBUILD_CMAKE_ARGS=-DSDE_WARNINGS_AS_ERRORS=ON" # Bundle openblas.dll into the wheel (auditwheel/delocate handle this on Linux/macOS) CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel} --add-path C:/vcpkg/installed/x64-windows/bin" - # On free-threaded builds, assert the import did not re-enable the GIL. - CIBW_TEST_COMMAND: python -c "import sys, sysconfig, sparsediffpy; ft = bool(sysconfig.get_config_var('Py_GIL_DISABLED')); assert not ft or not sys._is_gil_enabled(), 'GIL re-enabled on import'; print('freethreaded_build=', ft)" + # On free-threaded builds, assert the import did not re-enable the GIL, + # then run the test suite against the installed wheel. + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: python -c "import sys, sysconfig, sparsediffpy; ft = bool(sysconfig.get_config_var('Py_GIL_DISABLED')); assert not ft or not sys._is_gil_enabled(), 'GIL re-enabled on import'; print('freethreaded_build=', ft)" && pytest {project}/tests -q - name: Check wheels shell: bash run: | diff --git a/pyproject.toml b/pyproject.toml index d2ce46e..4c87286 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,3 @@ wheel.packages = ["sparsediffpy"] # Persistent build dir: avoids scikit-build-core's TemporaryDirectory, whose cleanup # intermittently fails on Windows CI with WinError 32 (lingering mspdbsrv/AV handle). build-dir = "build/{wheel_tag}" - -[tool.cibuildwheel] -test-requires = ["pytest"] -test-command = "pytest {project}/tests -q" diff --git a/sparsediffpy/_atoms.py b/sparsediffpy/_atoms.py index d3eacbb..c352f0c 100644 --- a/sparsediffpy/_atoms.py +++ b/sparsediffpy/_atoms.py @@ -5,7 +5,7 @@ from sparsediffpy import _sparsediffengine as _C from sparsediffpy._expression import ( - Constant, Node, Parameter, Variable, _check_shape, _wrap, + Constant, Node, Parameter, Variable, _check_shape, _check_symmetric, _wrap, ) @@ -133,7 +133,8 @@ def quad_form(x, Q): """x' Q x for a column vector x (n, 1) and a constant or Parameter Q (n, n). A NumPy Q uses the engine's dense (BLAS) path, a scipy.sparse Q the sparse - path. Constant Q is symmetrized; a Parameter Q must be symmetric. + path. Constant Q is symmetrized; a Parameter Q must be symmetric, which is + checked here and on every later ``Q.value`` update. """ x, Q = _wrap(x), _wrap(Q) n = x.shape[0] @@ -141,6 +142,9 @@ def quad_form(x, Q): raise ValueError(f"quad_form: need x (n, 1) and Q (n, n), " f"got {x.shape} and {Q.shape}") if isinstance(Q, Parameter): + Q._symmetric = True # checked now and on every later Q.value update + if Q._flat is not None: + _check_symmetric(Q._flat, Q.shape) return Node(lambda p, c: _C.make_quad_form(p, c, "dense", None, n), [Q, x], (1, 1)) if not isinstance(Q, Constant): diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py index a9153ba..293e477 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/_expression.py @@ -5,6 +5,8 @@ 2-D ``(d1, d2)`` and values are flattened column-major, like the C engine. """ +import numbers + import numpy as np import scipy.sparse as sps @@ -66,13 +68,18 @@ def __rmatmul__(self, other): return _matmul(_wrap(other), self) def __pow__(self, p): - if not isinstance(p, (int, float)): + if not isinstance(p, numbers.Real): raise TypeError("Exponent must be a constant number") return Node(lambda c: _C.make_power(c, float(p)), [self], self.shape) def __getitem__(self, key): return _index(self, key) + def __iter__(self): + # Without this, Python would iterate via __getitem__ until IndexError. + raise TypeError("Expressions are not iterable; index with x[i, j] or " + "pass a list of expressions") + class Node(Expression): """An operation: ``make(*child_capsules)`` builds its C capsule.""" @@ -87,6 +94,8 @@ class Constant(Expression): """Fixed dense or scipy.sparse data.""" def __init__(self, value): + if value is None: + raise TypeError("Constant value cannot be None") if sps.issparse(value): self.value = sps.csr_matrix(value, dtype=np.float64) else: @@ -128,6 +137,7 @@ class Parameter(Expression): def __init__(self, scope, offset, shape, value=None): self._scope, self._offset, self.shape = scope, offset, shape self._flat = None + self._symmetric = False # set by quad_form; the engine assumes Q = Q' if value is not None: self.value = value @@ -139,16 +149,27 @@ def value(self): @value.setter def value(self, val): - self._flat = _flatten_shaped(val, self.shape) + flat = _flatten_shaped(val, self.shape) + if self._symmetric: + _check_symmetric(flat, self.shape) + self._flat = flat self._scope._param_version += 1 +def _check_symmetric(flat, shape): + M = flat.reshape(shape, order="F") + if not np.allclose(M, M.T): + raise ValueError("Parameter used in quad_form must be symmetric") + + def _flatten_shaped(val, shape): """Flatten a value of shape ``shape`` column-major. Also accepts a 1-D array for a vector shape (n, 1) or (1, n), and a scalar for (1, 1). Any other shape raises, even with the right element count. """ + if val is None: + raise TypeError("value cannot be None") arr = np.asarray(val, dtype=np.float64) if not (arr.shape == shape or (arr.ndim == 0 and shape == (1, 1)) @@ -283,6 +304,7 @@ def _matmul_with_data(make, A): def index_flat(x, flat_indices, shape): """Gather entries of ``x`` by column-major flat indices into ``shape``.""" + x = _wrap(x) idx = np.asarray(flat_indices, dtype=np.int32).ravel() shape = _check_shape(*shape) if idx.size != shape[0] * shape[1]: diff --git a/tests/test_api.py b/tests/test_api.py index 827a7b5..e2255c2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -203,6 +203,54 @@ def test_compile_errors(scope): scope.Variable(1, 1) ** scope.Variable(1, 1) +def test_expressions_are_not_iterable(scope): + x, X = scope.Variable(3, 1), scope.Variable(2, 2) + with pytest.raises(TypeError, match="not iterable"): + list(x) + with pytest.raises(TypeError, match="not iterable"): + sp.Problem(sp.sum(X), X) # constraints must be a list + assert sp.Problem(sp.sum(X), [X]).total_constraint_size == 4 + + +def test_none_is_rejected(scope): + x, p = scope.Variable(2, 1), scope.Parameter(2, 1) + with pytest.raises(TypeError, match="None"): + x + None + with pytest.raises(TypeError, match="None"): + x.value = None + with pytest.raises(TypeError, match="None"): + p.value = None + + +def test_index_flat_wraps_constants(scope): + x = scope.Variable(2, 1) + fn = sp.compile(sp.index_flat(np.array([[5.0], [6.0]]), [1], (1, 1)) + x[0]) + x.value = [1.0, 2.0] + np.testing.assert_array_equal(fn.forward(), [[7.0]]) + + +def test_numpy_scalar_exponent(scope): + x = scope.Variable(2, 1) + f1, f2 = sp.compile(x ** np.int64(2)), sp.compile(x ** np.float32(2)) + x.value = [1.5, -2.0] + np.testing.assert_array_equal(f1.forward(), x.value ** 2) + np.testing.assert_array_equal(f2.forward(), x.value ** 2) + + +def test_quad_form_parameter_must_be_symmetric(scope): + x = scope.Variable(2, 1) + N, S = np.array([[1.0, 5.0], [0.0, 1.0]]), np.array([[1.0, 2.0], [2.0, 3.0]]) + with pytest.raises(ValueError, match="symmetric"): + sp.quad_form(x, scope.Parameter(2, 2, value=N)) + P = scope.Parameter(2, 2, value=S) + fn = sp.compile(sp.quad_form(x, P)) + with pytest.raises(ValueError, match="symmetric"): + P.value = N + x.value = [1.0, 1.0] + P.value = 2 * S + np.testing.assert_allclose(fn.jacobian().toarray(), [2 * (2 * S) @ [1.0, 1.0]]) + + # --- parameters ---------------------------------------------------------------- def test_parameter_update_without_recompile(scope): From fd79744c1e8a12e63be9125e67c11ef7b318c056 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 16:59:57 -0800 Subject: [PATCH 11/17] Rename _wrap to cast_to_const, as in CVXPY Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/_atoms.py | 21 +++++++++++---------- sparsediffpy/_compile.py | 8 ++++---- sparsediffpy/_expression.py | 23 ++++++++++++----------- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/sparsediffpy/_atoms.py b/sparsediffpy/_atoms.py index c352f0c..5bf22a4 100644 --- a/sparsediffpy/_atoms.py +++ b/sparsediffpy/_atoms.py @@ -5,7 +5,8 @@ from sparsediffpy import _sparsediffengine as _C from sparsediffpy._expression import ( - Constant, Node, Parameter, Variable, _check_shape, _check_symmetric, _wrap, + Constant, Node, Parameter, Variable, _check_shape, _check_symmetric, + cast_to_const, ) @@ -20,7 +21,7 @@ def _require_variables(name, *xs): def _elementwise(make, name, restricted=False): def atom(x): - x = _wrap(x) + x = cast_to_const(x) if restricted: _require_variables(name, x) return Node(make, [x], x.shape) @@ -47,7 +48,7 @@ def atom(x): def power(x, p): """Elementwise x ** p for a constant exponent p.""" - return _wrap(x) ** p + return cast_to_const(x) ** p def atan2(y, x): @@ -69,7 +70,7 @@ def _reduction_shape(x, axis): def sum(x, axis=None): """Sum of all entries (axis=None) -> (1, 1); axis=0 -> (1, d2); axis=1 -> (1, d1).""" - x = _wrap(x) + x = cast_to_const(x) shape = _reduction_shape(x, axis) c_axis = -1 if axis is None else axis return Node(lambda c: _C.make_sum(c, c_axis), [x], shape) @@ -88,14 +89,14 @@ def prod(x, axis=None): def reshape(x, d1, d2): """Column-major reshape.""" - x, shape = _wrap(x), _check_shape(d1, d2) + x, shape = cast_to_const(x), _check_shape(d1, d2) if x.size != shape[0] * shape[1]: raise ValueError(f"Cannot reshape {x.shape} to {shape}") return Node(lambda c: _C.make_reshape(c, *shape), [x], shape) def trace(x): - x = _wrap(x) + x = cast_to_const(x) if x.shape[0] != x.shape[1]: raise ValueError(f"trace requires a square matrix, got {x.shape}") return Node(_C.make_trace, [x], (1, 1)) @@ -103,7 +104,7 @@ def trace(x): def diag_vec(x): """(n, 1) column vector -> (n, n) diagonal matrix.""" - x = _wrap(x) + x = cast_to_const(x) if x.shape[1] != 1: raise ValueError(f"diag_vec requires a column vector, got {x.shape}") return Node(_C.make_diag_vec, [x], (x.shape[0], x.shape[0])) @@ -111,7 +112,7 @@ def diag_vec(x): def hstack(exprs): """Concatenate horizontally; all arguments need the same number of rows.""" - exprs = [_wrap(e) for e in exprs] + exprs = [cast_to_const(e) for e in exprs] if not exprs or len({e.shape[0] for e in exprs}) != 1: raise ValueError(f"hstack: need >=1 args with equal rows, got " f"{[e.shape for e in exprs]}") @@ -121,7 +122,7 @@ def hstack(exprs): def vstack(exprs): """Concatenate vertically; all arguments need the same number of columns.""" - exprs = [_wrap(e) for e in exprs] + exprs = [cast_to_const(e) for e in exprs] if not exprs or len({e.shape[1] for e in exprs}) != 1: raise ValueError(f"vstack: need >=1 args with equal columns, got " f"{[e.shape for e in exprs]}") @@ -136,7 +137,7 @@ def quad_form(x, Q): path. Constant Q is symmetrized; a Parameter Q must be symmetric, which is checked here and on every later ``Q.value`` update. """ - x, Q = _wrap(x), _wrap(Q) + x, Q = cast_to_const(x), cast_to_const(Q) n = x.shape[0] if x.shape[1] != 1 or Q.shape != (n, n): raise ValueError(f"quad_form: need x (n, 1) and Q (n, n), " diff --git a/sparsediffpy/_compile.py b/sparsediffpy/_compile.py index 1227358..8554b10 100644 --- a/sparsediffpy/_compile.py +++ b/sparsediffpy/_compile.py @@ -10,7 +10,7 @@ import scipy.sparse as sps from sparsediffpy import _sparsediffengine as _C -from sparsediffpy._expression import Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, _wrap +from sparsediffpy._expression import Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, cast_to_const def _find_scope(exprs): @@ -102,8 +102,8 @@ class Problem: """ def __init__(self, objective, constraints=(), verbose=False): - objective = _wrap(objective) - constraints = [_wrap(c) for c in constraints] + objective = cast_to_const(objective) + constraints = [cast_to_const(c) for c in constraints] if objective.shape != (1, 1): raise ValueError(f"Objective must be scalar, got {objective.shape}") self._scope = _find_scope([objective, *constraints]) @@ -233,4 +233,4 @@ def hessian(self, weights): def compile(expr): """Analyze the sparsity of ``expr`` once and return a CompiledExpression.""" - return CompiledExpression(_wrap(expr)) + return CompiledExpression(cast_to_const(expr)) diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py index 293e477..878bfdf 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/_expression.py @@ -41,31 +41,31 @@ def T(self): return Node(_C.make_transpose, [self], self.shape[::-1]) def __add__(self, other): - return _add(self, _wrap(other)) + return _add(self, cast_to_const(other)) def __radd__(self, other): - return _add(_wrap(other), self) + return _add(cast_to_const(other), self) def __sub__(self, other): - return _add(self, -_wrap(other)) + return _add(self, -cast_to_const(other)) def __rsub__(self, other): - return _add(_wrap(other), -self) + return _add(cast_to_const(other), -self) def __neg__(self): return Node(_C.make_neg, [self], self.shape) def __mul__(self, other): - return _mul(self, _wrap(other)) + return _mul(self, cast_to_const(other)) def __rmul__(self, other): - return _mul(_wrap(other), self) + return _mul(cast_to_const(other), self) def __matmul__(self, other): - return _matmul(self, _wrap(other)) + return _matmul(self, cast_to_const(other)) def __rmatmul__(self, other): - return _matmul(_wrap(other), self) + return _matmul(cast_to_const(other), self) def __pow__(self, p): if not isinstance(p, numbers.Real): @@ -223,7 +223,8 @@ def get_values(self): return self._values.copy() -def _wrap(value): +def cast_to_const(value): + """Return ``value`` as an Expression, wrapping non-Expressions in a Constant.""" return value if isinstance(value, Expression) else Constant(value) @@ -246,7 +247,7 @@ def _broadcast_shape(a, b): def broadcast(x, shape): """Broadcast a scalar, row or column expression to ``shape``.""" - x, shape = _wrap(x), _check_shape(*shape) + x, shape = cast_to_const(x), _check_shape(*shape) if x.shape == shape: return x if _broadcast_shape(x.shape, shape) != shape: @@ -304,7 +305,7 @@ def _matmul_with_data(make, A): def index_flat(x, flat_indices, shape): """Gather entries of ``x`` by column-major flat indices into ``shape``.""" - x = _wrap(x) + x = cast_to_const(x) idx = np.asarray(flat_indices, dtype=np.int32).ravel() shape = _check_shape(*shape) if idx.size != shape[0] * shape[1]: From 48c59718af7f3ec161176e26f9060a8e424c5a87 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 8 Nov 2026 16:46:42 -0800 Subject: [PATCH 12/17] Tests: keep test_api within 85 columns Also name the deep-sum test's loop count n_adds rather than n_terms. Co-Authored-By: Claude Opus 5.5 --- tests/test_api.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index e2255c2..ec645e6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -275,7 +275,7 @@ def test_unset_parameter_raises(scope): np.testing.assert_allclose(fn.forward(), [[0], [0]]) -# --- Caching ------------------------------------------------------------------------ +# --- Caching ------------------------------------------------------------------ def count_calls(monkeypatch, name): """Count calls to the engine function `name` as seen from _compile.""" @@ -405,7 +405,7 @@ def test_problem_parameters(scope): def test_hessian_without_prior_jacobian(scope): - """The engine's Hessian reuses cached Jacobian values; hessian() must refresh them.""" + """hessian() must refresh the Jacobian values the engine's Hessian reuses.""" x, y = scope.Variable(2, 1), scope.Variable(2, 1) fn = sp.compile(x * y + sp.sin(x * y)) prob = sp.Problem(sp.sum(x * y), [x * y]) @@ -415,31 +415,32 @@ def test_hessian_without_prior_jacobian(scope): w = np.array([1.0, 2.0]) # d2/dx_i dy_i of w_i * (x_i y_i + sin(x_i y_i)) d2 = lambda a, b: w * (-np.sin(xy) * a * b) # noqa: E731 - H = np.block([[np.diag(d2(yv, yv)), np.diag(w * (1 + np.cos(xy)) + d2(xv, yv))], - [np.diag(w * (1 + np.cos(xy)) + d2(xv, yv)), np.diag(d2(xv, xv))]]) + off = np.diag(w * (1 + np.cos(xy)) + d2(xv, yv)) + H = np.block([[np.diag(d2(yv, yv)), off], [off, np.diag(d2(xv, xv))]]) np.testing.assert_allclose(fn.hessian(w).toarray(), H, atol=1e-12) prob.objective_forward(u) prob.constraint_forward(u) H_prob = np.block([[np.zeros((2, 2)), np.diag(2.0 + w)], [np.diag(2.0 + w), np.zeros((2, 2))]]) - np.testing.assert_allclose(prob.hessian(2.0, w).toarray(), H_prob, atol=1e-12) + np.testing.assert_allclose(prob.hessian(2.0, w).toarray(), H_prob, + atol=1e-12) # --- deep expressions ------------------------------------------------------------- def test_compile_deep_sum(scope): - """A sum built term by term in a loop is a chain deeper than the recursion limit.""" + """A loop-built sum is a chain of adds deeper than the recursion limit.""" x, p = scope.Variable(3, 1), scope.Parameter(1, 1, value=2.0) - n_terms = 5000 - assert n_terms > sys.getrecursionlimit() # each term adds one level + n_adds = 5000 + assert n_adds > sys.getrecursionlimit() # each add is one level deeper f = x[0] - for i in range(1, n_terms + 1): + for i in range(1, n_adds + 1): f = f + p * sp.sin(x[i % 3]) fn = sp.compile(f) u = np.array([0.3, -0.7, 1.1]) x.value = u - counts = np.bincount(np.arange(1, n_terms + 1) % 3, minlength=3) + counts = np.bincount(np.arange(1, n_adds + 1) % 3, minlength=3) for pv in (2.0, 3.0): p.value = pv np.testing.assert_allclose(fn.forward(), [[u[0] + pv * counts @ np.sin(u)]]) @@ -481,4 +482,5 @@ def test_compile_deep_nonlinear_chain(scope): np.testing.assert_allclose(prob.gradient(), [d1], rtol=1e-10) np.testing.assert_allclose(prob.constraint_forward(u), [v], rtol=1e-12) np.testing.assert_allclose(prob.jacobian().toarray(), [[d1]], rtol=1e-10) - np.testing.assert_allclose(prob.hessian(1.0, [2.0]).toarray(), [[3 * d2]], rtol=1e-8) + np.testing.assert_allclose(prob.hessian(1.0, [2.0]).toarray(), [[3 * d2]], + rtol=1e-8) From c09f141b070f0c9c8f88bfe4147c11b918c4a18e Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Mon, 9 Nov 2026 13:14:45 -0800 Subject: [PATCH 13/17] Problem: keep the objective and constraints on separate engine nodes A node shared by the objective and a constraint became one engine node, so a constraint_forward at another point overwrote values that gradient() then read. Build the objective and the constraints with separate builders. Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/_compile.py | 16 ++++++++++------ tests/test_api.py | 10 ++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/sparsediffpy/_compile.py b/sparsediffpy/_compile.py index 8554b10..4bc5f18 100644 --- a/sparsediffpy/_compile.py +++ b/sparsediffpy/_compile.py @@ -107,13 +107,17 @@ def __init__(self, objective, constraints=(), verbose=False): if objective.shape != (1, 1): raise ValueError(f"Objective must be scalar, got {objective.shape}") self._scope = _find_scope([objective, *constraints]) - build = _Builder(self._scope) - obj_cap = build(objective) - con_caps = [build(c) for c in constraints] + # The objective and the constraints are evaluated at separate points, so + # they must not share engine nodes: a constraint_forward would overwrite + # values that the objective's gradient still reads. + obj_build, con_build = _Builder(self._scope), _Builder(self._scope) + obj_cap = obj_build(objective) + con_caps = [con_build(c) for c in constraints] self._capsule = _C.make_problem(obj_cap, con_caps, verbose) - self._params = [p for p, _ in build.params] - if build.params: - _C.problem_register_params(self._capsule, [c for _, c in build.params]) + params = obj_build.params + con_build.params + self._params = [p for p, _ in params] + if params: + _C.problem_register_params(self._capsule, [c for _, c in params]) _C.problem_init_derivatives(self._capsule) self._param_version = None self.n_vars = self._scope.total_var_size diff --git a/tests/test_api.py b/tests/test_api.py index ec645e6..b414288 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -404,6 +404,16 @@ def test_problem_parameters(scope): np.testing.assert_allclose(prob.gradient(), u) +def test_objective_and_constraints_at_different_points(scope): + x, a = scope.Variable(3, 1), scope.Parameter(1, 1, value=2.0) + e = sp.exp(a * x) # shared by the objective and the constraint + prob = sp.Problem(sp.sum(e), [e]) + prob.objective_forward(np.zeros(3)) + prob.constraint_forward(np.ones(3)) + np.testing.assert_allclose(prob.gradient(), [2.0, 2.0, 2.0]) + np.testing.assert_allclose(prob.jacobian().diagonal(), 2 * np.exp([2.0] * 3)) + + def test_hessian_without_prior_jacobian(scope): """hessian() must refresh the Jacobian values the engine's Hessian reuses.""" x, y = scope.Variable(2, 1), scope.Variable(2, 1) From 50f17799a0054d01fbe5276e7b89243f94abcaa0 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Mon, 9 Nov 2026 16:16:51 -0800 Subject: [PATCH 14/17] Docstrings instead of section banners; keep lines within 85 columns Co-Authored-By: Claude Opus 5.5 --- sparsediffpy/_atoms.py | 7 ++++++- sparsediffpy/_compile.py | 8 ++++++-- sparsediffpy/_expression.py | 34 ++++++++++++++++++++++++++-------- tests/test_api.py | 12 ------------ 4 files changed, 38 insertions(+), 23 deletions(-) diff --git a/sparsediffpy/_atoms.py b/sparsediffpy/_atoms.py index 5bf22a4..58ce53e 100644 --- a/sparsediffpy/_atoms.py +++ b/sparsediffpy/_atoms.py @@ -20,6 +20,10 @@ def _require_variables(name, *xs): def _elementwise(make, name, restricted=False): + """Make the elementwise atom ``name`` from its engine constructor. + + A restricted atom (domain not all of R) only accepts a plain Variable. + """ def atom(x): x = cast_to_const(x) if restricted: @@ -69,7 +73,7 @@ def _reduction_shape(x, axis): def sum(x, axis=None): - """Sum of all entries (axis=None) -> (1, 1); axis=0 -> (1, d2); axis=1 -> (1, d1).""" + """Sum of entries: axis=None -> (1, 1), axis=0 -> (1, d2), axis=1 -> (1, d1).""" x = cast_to_const(x) shape = _reduction_shape(x, axis) c_axis = -1 if axis is None else axis @@ -96,6 +100,7 @@ def reshape(x, d1, d2): def trace(x): + """Sum of the diagonal of a square matrix, as a (1, 1) expression.""" x = cast_to_const(x) if x.shape[0] != x.shape[1]: raise ValueError(f"trace requires a square matrix, got {x.shape}") diff --git a/sparsediffpy/_compile.py b/sparsediffpy/_compile.py index 4bc5f18..37f25eb 100644 --- a/sparsediffpy/_compile.py +++ b/sparsediffpy/_compile.py @@ -10,7 +10,9 @@ import scipy.sparse as sps from sparsediffpy import _sparsediffengine as _C -from sparsediffpy._expression import Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, cast_to_const +from sparsediffpy._expression import ( + Constant, Parameter, Variable, _flatten_shaped, _flatten_vector, cast_to_const, +) def _find_scope(exprs): @@ -47,6 +49,7 @@ def __init__(self, scope): self.params = [] # (Parameter, capsule), each once def __call__(self, root): + """Return root's capsule, building any children not built yet.""" stack = [root] while stack: e = stack[-1] @@ -82,6 +85,7 @@ def _build_one(self, e): def _csr(triple): + """scipy CSR matrix from an engine (data, indices, indptr, shape) tuple.""" data, indices, indptr, shape = triple return sps.csr_matrix((data, indices, indptr), shape=shape) @@ -192,7 +196,7 @@ def jacobian(self): return self._jac_value def hessian(self, obj_factor, lagrange): - """Lagrangian Hessian obj_factor * H_f + sum_i lagrange_i * H_gi (csr_matrix). + """Lagrangian Hessian obj_factor H_f + sum_i lagrange_i H_gi (csr_matrix). Call objective_forward and constraint_forward at the same point first. """ diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py index 878bfdf..0f8fd10 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/_expression.py @@ -18,6 +18,7 @@ class DimensionError(ValueError): def _check_shape(d1, d2): + """Return (d1, d2) as a tuple of positive Python ints, or raise.""" if not all(isinstance(d, (int, np.integer)) for d in (d1, d2)): raise TypeError(f"Shape dimensions must be integers, got ({d1!r}, {d2!r})") if d1 <= 0 or d2 <= 0: @@ -34,10 +35,12 @@ class Expression: @property def size(self): + """Number of entries, d1 * d2.""" return self.shape[0] * self.shape[1] @property def T(self): + """Transpose.""" return Node(_C.make_transpose, [self], self.shape[::-1]) def __add__(self, other): @@ -110,6 +113,7 @@ def __init__(self, value): self.shape = _check_shape(*self.value.shape) def dense(self): + """The value as a dense array.""" return self.value.toarray() if sps.issparse(self.value) else self.value @@ -121,6 +125,7 @@ def __init__(self, scope, offset, shape): @property def value(self): + """The variable's current entries, shaped like the variable.""" flat = self._scope._values[self._offset:self._offset + self.size] return flat.reshape(self.shape, order="F").copy() @@ -143,6 +148,7 @@ def __init__(self, scope, offset, shape, value=None): @property def value(self): + """The parameter's current value, or None if it was never set.""" if self._flat is None: return None return self._flat.reshape(self.shape, order="F").copy() @@ -157,6 +163,7 @@ def value(self, val): def _check_symmetric(flat, shape): + """Raise unless the column-major ``flat`` is a symmetric matrix of ``shape``.""" M = flat.reshape(shape, order="F") if not np.allclose(M, M.T): raise ValueError("Parameter used in quad_form must be symmetric") @@ -173,7 +180,8 @@ def _flatten_shaped(val, shape): arr = np.asarray(val, dtype=np.float64) if not (arr.shape == shape or (arr.ndim == 0 and shape == (1, 1)) - or (arr.ndim == 1 and min(shape) == 1 and arr.size == shape[0] * shape[1])): + or (arr.ndim == 1 and min(shape) == 1 + and arr.size == shape[0] * shape[1])): raise DimensionError(f"expected shape {shape}, got {arr.shape}") return arr.ravel(order="F") @@ -192,12 +200,12 @@ class Scope: def __init__(self): self._values = np.zeros(0) - self._values_version = 0 # bumped on every write, so problems can skip stale work + self._values_version = 0 # bumped on every write; problems skip stale work self._n_params = 0 self._param_version = 0 def Variable(self, d1, d2): - """Create a (d1, d2) variable; its entries are appended to the flat vector.""" + """Create a (d1, d2) variable, appending its entries to the flat vector.""" shape = _check_shape(d1, d2) var = Variable(self, self._values.size, shape) self._values = np.concatenate([self._values, np.zeros(var.size)]) @@ -212,6 +220,7 @@ def Parameter(self, d1, d2, value=None): @property def total_var_size(self): + """Length of the flat variable vector.""" return self._values.size def set_values(self, array): @@ -220,6 +229,7 @@ def set_values(self, array): self._values_version += 1 def get_values(self): + """A copy of the flat variable vector (declaration order).""" return self._values.copy() @@ -229,14 +239,12 @@ def cast_to_const(value): def _is_data(e): + """Whether e is data (Constant or Parameter) rather than a function of x.""" return isinstance(e, (Constant, Parameter)) -# --------------------------------------------------------------------------- -# Operator dispatch -# --------------------------------------------------------------------------- - def _broadcast_shape(a, b): + """Shape of a op b when scalars, rows and columns broadcast, or raise.""" out = [] for da, db in zip(a, b): if da != db and 1 not in (da, db): @@ -258,11 +266,17 @@ def broadcast(x, shape): def _add(a, b): + """a + b, broadcasting both operands to the common shape first.""" shape = _broadcast_shape(a.shape, b.shape) return Node(_C.make_add, [broadcast(a, shape), broadcast(b, shape)], shape) def _mul(a, b): + """Elementwise a * b. + + A data operand (constant or parameter) uses the engine's scalar or vector + scaling atoms; two functions of x use the general product. + """ if _is_data(b) and not _is_data(a): a, b = b, a # put the constant/parameter operand (if any) first if _is_data(a) and a.shape == (1, 1): @@ -274,6 +288,7 @@ def _mul(a, b): def _matmul(a, b): + """a @ b: data times a function of x (either side), or two functions of x.""" if a.shape[1] != b.shape[0]: raise ValueError(f"Matmul shape mismatch: {a.shape} @ {b.shape}") shape = (a.shape[0], b.shape[1]) @@ -287,6 +302,7 @@ def _matmul(a, b): def _data_args(data, x): + """Children of a data matmul: a Parameter is a child, a Constant is baked in.""" return [data, x] if isinstance(data, Parameter) else [x] @@ -316,6 +332,7 @@ def index_flat(x, flat_indices, shape): def _axis_indices(key, n): + """Positions along an axis of length n chosen by an int, slice or int list.""" if isinstance(key, (bool, np.bool_)): # bool subclasses int; x[True] is not x[1] raise IndexError("Boolean indices are not supported; use integer indices") if isinstance(key, (int, np.integer)): @@ -340,7 +357,8 @@ def _index(x, key): d1, d2 = x.shape if isinstance(key, tuple): if len(key) != 2: - raise IndexError(f"Expected 2 indices for shape {x.shape}, got {len(key)}") + raise IndexError( + f"Expected 2 indices for shape {x.shape}, got {len(key)}") rows, cols = _axis_indices(key[0], d1), _axis_indices(key[1], d2) flat = (rows[:, None] + cols[None, :] * d1).ravel(order="F") return index_flat(x, flat, (rows.size, cols.size)) diff --git a/tests/test_api.py b/tests/test_api.py index b414288..7097c85 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -9,8 +9,6 @@ from sparsediffpy import _compile -# --- scope and values --------------------------------------------------------- - def test_values_are_column_major(scope): X, y = scope.Variable(2, 2), scope.Variable(1, 1) X.value = np.array([[1.0, 2.0], [3.0, 4.0]]) @@ -129,8 +127,6 @@ def test_affine_hessian_is_zero(scope): assert not fn.hessian([1.0]).toarray().any() -# --- shapes and validation ---------------------------------------------------- - def test_shapes(scope): x, r, X = scope.Variable(3, 1), scope.Variable(1, 2), scope.Variable(3, 2) assert (x + r).shape == (3, 2) @@ -251,8 +247,6 @@ def test_quad_form_parameter_must_be_symmetric(scope): np.testing.assert_allclose(fn.jacobian().toarray(), [2 * (2 * S) @ [1.0, 1.0]]) -# --- parameters ---------------------------------------------------------------- - def test_parameter_update_without_recompile(scope): x = scope.Variable(3, 1) A = scope.Parameter(2, 3, value=np.ones((2, 3))) @@ -275,8 +269,6 @@ def test_unset_parameter_raises(scope): np.testing.assert_allclose(fn.forward(), [[0], [0]]) -# --- Caching ------------------------------------------------------------------ - def count_calls(monkeypatch, name): """Count calls to the engine function `name` as seen from _compile.""" calls, real = [], getattr(_compile._C, name) @@ -355,8 +347,6 @@ def test_derivatives_require_forward(scope): assert prob.hessian(1.0, [1.0, 1.0]).shape == (2, 2) -# --- Problem ---------------------------------------------------------------------- - def test_problem(scope): x = scope.Variable(3, 1) shared = sp.sin(x) @@ -437,8 +427,6 @@ def test_hessian_without_prior_jacobian(scope): atol=1e-12) -# --- deep expressions ------------------------------------------------------------- - def test_compile_deep_sum(scope): """A loop-built sum is a chain of adds deeper than the recursion limit.""" x, p = scope.Variable(3, 1), scope.Parameter(1, 1, value=2.0) From cd001d26c4d538a561abf8fd1adf64218d06f84c Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Mon, 9 Nov 2026 16:16:51 -0800 Subject: [PATCH 15/17] Tests: one table of atom cases, each written once for sp and NumPy Each case is evaluated on Variables (the expression under test) and on their values with NumPy stand-ins (the expected result), instead of a registered function plus a separate reference per case. Co-Authored-By: Claude Opus 5.5 --- tests/test_atoms.py | 322 ++++++++++++++------------------------------ 1 file changed, 102 insertions(+), 220 deletions(-) diff --git a/tests/test_atoms.py b/tests/test_atoms.py index 3778fc7..1b7eff7 100644 --- a/tests/test_atoms.py +++ b/tests/test_atoms.py @@ -1,9 +1,14 @@ -"""Every atom: forward value vs NumPy, Jacobian and Hessian vs finite differences. +"""Each atom is wired to the right engine call. -Each case builds (expr, reference) on a fresh scope; ``reference()`` computes -the expected value from the variables' current values with NumPy. +A case is ``(name, shapes, f, positive)``. ``f(m, *xs)`` is evaluated once +with ``m = sp`` on Variables (the expression under test) and once with +``m = NP`` on their values (the expected result). A shape that is an array +declares a Parameter with that value. The value must match NumPy, and the +Jacobian and Hessian must match finite differences. """ +from types import SimpleNamespace + import numpy as np import pytest import scipy.sparse as sps @@ -14,224 +19,101 @@ A = np.array([[1.0, -2.0, 0.5], [0.0, 3.0, 1.0]]) Q = np.array([[2.0, 1.0, 0.0], [1.0, 3.0, 0.5], [0.0, 0.5, 4.0]]) -CASES = {} # name -> (build, positive_domain) - - -def case(positive=False): - def register(build): - CASES[build.__name__] = (build, positive) - return build - return register - - -def _elementwise(name, f, ref, positive=False): - def build(s): - X = s.Variable(3, 2) - return f(X), lambda: ref(X.value) - build.__name__ = name - case(positive)(build) - - -for _name, _f, _ref in [ - ("sin", sp.sin, np.sin), ("cos", sp.cos, np.cos), ("exp", sp.exp, np.exp), - ("sinh", sp.sinh, np.sinh), ("tanh", sp.tanh, np.tanh), - ("asinh", sp.asinh, np.arcsinh), ("atan", sp.atan, np.arctan), - ("logistic", sp.logistic, lambda x: np.log1p(np.exp(x))), - ("normal_cdf", sp.normal_cdf, ndtr), ("xexp", sp.xexp, lambda x: x * np.exp(x)), - ("power3", lambda x: x ** 3, lambda x: x ** 3), -]: - _elementwise(_name, _f, _ref) - -for _name, _f, _ref in [ - ("log", sp.log, np.log), ("tan", sp.tan, np.tan), - ("atanh", sp.atanh, np.arctanh), ("entr", sp.entr, lambda x: -x * np.log(x)), - ("power_half", lambda x: sp.power(x, 0.5), np.sqrt), -]: - _elementwise(_name, _f, _ref, positive=True) - - -@case(positive=True) -def atan2(s): - y, x = s.Variable(3, 1), s.Variable(3, 1) - return sp.atan2(y, x), lambda: np.arctan2(y.value, x.value) - - -# --- affine ----------------------------------------------------------------- - -@case() -def add_broadcast(s): - a, r, c, X = s.Variable(1, 1), s.Variable(1, 2), s.Variable(3, 1), s.Variable(3, 2) - return a * X + r + c - 1.0, lambda: a.value * X.value + r.value + c.value - 1 - - -@case() -def neg_sub(s): - x, y = s.Variable(3, 1), s.Variable(3, 1) - return -x - y + np.c_[0:3.0], lambda: -x.value - y.value + np.c_[0:3.0] - - -@case() -def constant_mult(s): - X = s.Variable(3, 2) - B = np.arange(6.0).reshape(3, 2) - return 2.0 * X * B, lambda: 2 * X.value * B - - -@case() -def matmul_dense_left_right(s): - x, Y = s.Variable(3, 1), s.Variable(2, 3) - return A @ x + (Y @ A.T)[:, 0:1], lambda: A @ x.value + (Y.value @ A.T)[:, 0:1] - - -@case() -def matmul_sparse_left_right(s): - x, Y = s.Variable(3, 1), s.Variable(2, 3) - S = sps.csr_matrix(A) - return S @ x + (Y @ S.T)[:, 0:1], lambda: A @ x.value + (Y.value @ A.T)[:, 0:1] - - -@case() -def transpose_reshape(s): - X = s.Variable(3, 2) - return sp.reshape(X.T, 6, 1), lambda: X.value.T.reshape(6, 1, order="F") - - -@case() -def sums(s): - X = s.Variable(3, 2) - return sp.hstack([sp.sum(X), sp.sum(X, axis=0), sp.sum(X, axis=1)]), \ - lambda: np.r_[X.value.sum(), X.value.sum(0), X.value.sum(1)][None, :] - - -@case() -def trace_diag_vec(s): - X, x = s.Variable(3, 3), s.Variable(3, 1) - return sp.trace(X) + sp.diag_vec(x), \ - lambda: np.trace(X.value) + np.diag(x.value.ravel()) - - -@case() -def stacking(s): - x, y = s.Variable(2, 1), s.Variable(2, 2) - return sp.vstack([sp.hstack([x, y]), sp.hstack([y, x])]), \ - lambda: np.block([[x.value, y.value], [y.value, x.value]]) - - -@case() -def indexing(s): - X, r = s.Variable(3, 3), s.Variable(1, 4) - return sp.vstack([X[0:2, [2, 0]], X[-1, 1:], r[1:3]]), \ - lambda: np.r_[X.value[0:2][:, [2, 0]], X.value[-1:, 1:], r.value[:, 1:3]] - - -# --- bivariate / other -------------------------------------------------------- - -@case() -def multiply(s): - x, Y = s.Variable(3, 1), s.Variable(3, 2) - return sp.sin(x) * Y, lambda: np.sin(x.value) * Y.value - - -@case() -def matmul_variables(s): - X, Y = s.Variable(2, 3), s.Variable(3, 2) - return X @ Y, lambda: X.value @ Y.value - - -@case() -def x_transpose_x(s): - x = s.Variable(3, 1) - return x.T @ x, lambda: x.value.T @ x.value - - -@case() -def quad_forms(s): - x = s.Variable(3, 1) - P = s.Parameter(3, 3, value=2 * Q) - xv = lambda: x.value.ravel() # noqa: E731 - return sp.vstack([sp.quad_form(x, Q), sp.quad_form(x, sps.csr_matrix(Q)), - sp.quad_form(x, P)]), \ - lambda: np.c_[[xv() @ Q @ xv(), xv() @ Q @ xv(), 2 * xv() @ Q @ xv()]] - - -@case() -def quad_form_nonsymmetric(s): - x = s.Variable(3, 1) - N = Q + np.triu(np.ones((3, 3)), 1) # not symmetric - xv = lambda: x.value.ravel() # noqa: E731 - return sp.quad_form(x, N) + sp.quad_form(x, sps.csr_matrix(N)), \ - lambda: 2 * xv() @ N @ xv() - - -@case(positive=True) -def quad_over_lin(s): - x, z = s.Variable(3, 1), s.Variable(1, 1) - return sp.quad_over_lin(x, z), lambda: (x.value ** 2).sum() / z.value - - -@case(positive=True) -def rel_entr_vectors(s): - x, y = s.Variable(3, 1), s.Variable(3, 1) - return sp.rel_entr(x, y), lambda: x.value * np.log(x.value / y.value) - - -@case(positive=True) -def rel_entr_scalar_vector(s): - a, y = s.Variable(1, 1), s.Variable(3, 1) - return sp.rel_entr(a, y), lambda: a.value * np.log(a.value / y.value) - - -@case(positive=True) -def rel_entr_vector_scalar(s): - x, b = s.Variable(3, 1), s.Variable(1, 1) - return sp.rel_entr(x, b), lambda: x.value * np.log(x.value / b.value) - - -@case(positive=True) -def prods(s): - X = s.Variable(3, 2) - return sp.hstack([sp.prod(X), sp.prod(X, axis=0), sp.prod(X, axis=1)]), \ - lambda: np.r_[X.value.prod(), X.value.prod(0), X.value.prod(1)][None, :] - - -# --- compositions ------------------------------------------------------------- - -@case() -def parameter_matmul_and_scale(s): - x = s.Variable(3, 1) - B, c = s.Parameter(2, 3, value=A), s.Parameter(1, 1, value=3.0) - return c * sp.exp(B @ x) + B @ sp.sin(x), \ - lambda: 3 * np.exp(A @ x.value) + A @ np.sin(x.value) - - -@case() -def deep_chain(s): - x = s.Variable(4, 1) - f = x - for _ in range(5): - f = sp.tanh(0.9 * f + 0.1) - return sp.sum(f * f) + sp.cos(x[0]), None - - -@case() -def shared_subexpression(s): - x = s.Variable(3, 1) - u = sp.sin(A @ x) - return u * u + sp.exp(u), lambda: np.sin(A @ x.value) ** 2 + np.exp(np.sin(A @ x.value)) - - -@pytest.mark.parametrize("name", list(CASES)) -def test_atom(name, rng): - build, positive = CASES[name] +N = Q + np.triu(np.ones((3, 3)), 1) # not symmetric + + +def _reduce(f): + """NumPy sum/prod with sp's shapes: (1, 1), or a row for axis 0 or 1.""" + return lambda X, axis=None: np.reshape(f(X, axis=axis), (1, -1)) + + +def _dense(M): + return M.toarray() if sps.issparse(M) else M + + +NP = SimpleNamespace( + sin=np.sin, cos=np.cos, exp=np.exp, sinh=np.sinh, tanh=np.tanh, log=np.log, + tan=np.tan, asinh=np.arcsinh, atan=np.arctan, atanh=np.arctanh, + logistic=lambda x: np.logaddexp(0, x), normal_cdf=ndtr, + xexp=lambda x: x * np.exp(x), entr=lambda x: -x * np.log(x), + power=lambda x, p: x ** p, atan2=np.arctan2, + sum=_reduce(np.sum), prod=_reduce(np.prod), hstack=np.hstack, vstack=np.vstack, + reshape=lambda x, d1, d2: x.reshape(d1, d2, order="F"), + trace=lambda X: np.trace(X).reshape(1, 1), diag_vec=lambda x: np.diag(x.ravel()), + quad_form=lambda x, P: x.T @ _dense(P) @ x, + quad_over_lin=lambda x, z: (x ** 2).sum() / z, + rel_entr=lambda x, y: x * np.log(x / y), +) + +ELEMENTWISE = ["sin", "cos", "exp", "sinh", "tanh", "asinh", "atan", "logistic", + "normal_cdf", "xexp"] +POSITIVE_ELEMENTWISE = ["log", "tan", "atanh", "entr"] + +CASES = [ + *[(a, [(3, 2)], lambda m, x, a=a: getattr(m, a)(x), False) for a in ELEMENTWISE], + *[(a, [(3, 2)], lambda m, x, a=a: getattr(m, a)(x), True) + for a in POSITIVE_ELEMENTWISE], + ("power", [(3, 2)], lambda m, x: x ** 3 + m.power(x, 0.5), True), + ("atan2", [(3, 1), (3, 1)], lambda m, y, x: m.atan2(y, x), True), + # affine + ("add_broadcast", [(1, 1), (1, 2), (3, 1), (3, 2)], + lambda m, a, r, c, X: a * X + r + c - 1.0, False), + ("neg_sub", [(3, 1), (3, 1)], lambda m, x, y: -x - y + np.c_[0:3.0], False), + ("constant_mult", [(3, 2)], lambda m, X: 2.0 * X * np.arange(6.0).reshape(3, 2), + False), + ("matmul_dense", [(3, 1), (2, 3)], lambda m, x, Y: A @ x + (Y @ A.T)[:, 0:1], + False), + ("matmul_sparse", [(3, 1), (2, 3)], + lambda m, x, Y: sps.csr_matrix(A) @ x + (Y @ sps.csr_matrix(A.T))[:, 0:1], + False), + ("transpose_reshape", [(3, 2)], lambda m, X: m.reshape(X.T, 6, 1), False), + ("sums", [(3, 2)], + lambda m, X: m.hstack([m.sum(X), m.sum(X, axis=0), m.sum(X, axis=1)]), False), + ("trace_diag_vec", [(3, 3), (3, 1)], lambda m, X, x: m.trace(X) + m.diag_vec(x), + False), + ("stacking", [(2, 1), (2, 2)], + lambda m, x, y: m.vstack([m.hstack([x, y]), m.hstack([y, x])]), False), + ("indexing", [(3, 3), (1, 4)], + lambda m, X, r: m.vstack([X[0:2, [2, 0]], X[2:3, 1:], r[:, 1:3]] + if m is NP else [X[0:2, [2, 0]], X[-1, 1:], r[1:3]]), + False), + # bivariate and other + ("multiply", [(3, 1), (3, 2)], lambda m, x, Y: m.sin(x) * Y, False), + ("matmul_variables", [(2, 3), (3, 2)], lambda m, X, Y: X @ Y, False), + ("x_transpose_x", [(3, 1)], lambda m, x: x.T @ x, False), + ("quad_form", [(3, 1), 2 * Q], + lambda m, x, P: m.vstack([m.quad_form(x, Q), m.quad_form(x, sps.csr_matrix(Q)), + m.quad_form(x, P)]), False), + ("quad_form_nonsymmetric", [(3, 1)], + lambda m, x: m.quad_form(x, N) + m.quad_form(x, sps.csr_matrix(N)), False), + ("quad_over_lin", [(3, 1), (1, 1)], lambda m, x, z: m.quad_over_lin(x, z), True), + ("rel_entr", [(3, 1), (3, 1)], lambda m, x, y: m.rel_entr(x, y), True), + ("rel_entr_scalar_vector", [(1, 1), (3, 1)], lambda m, x, y: m.rel_entr(x, y), + True), + ("rel_entr_vector_scalar", [(3, 1), (1, 1)], lambda m, x, y: m.rel_entr(x, y), + True), + ("prods", [(3, 2)], + lambda m, X: m.hstack([m.prod(X), m.prod(X, axis=0), m.prod(X, axis=1)]), True), + # compositions + ("parameter_matmul_and_scale", [(3, 1), A, np.array([[3.0]])], + lambda m, x, B, c: c * m.exp(B @ x) + B @ m.sin(x), False), + ("shared_subexpression", [(3, 1)], + lambda m, x: (lambda u: u * u + m.exp(u))(m.sin(A @ x)), False), +] + + +@pytest.mark.parametrize("name, shapes, f, positive", CASES, + ids=[c[0] for c in CASES]) +def test_atom(name, shapes, f, positive, rng): scope = sp.Scope() - f, ref = build(scope) - fn = sp.compile(f) + leaves = [scope.Parameter(*s.shape, value=s) if isinstance(s, np.ndarray) + else scope.Variable(*s) for s in shapes] + expr = f(sp, *leaves) + fn = sp.compile(expr) low, high = (0.2, 0.9) if positive else (-1.0, 1.0) x0 = rng.uniform(low, high, scope.total_var_size) scope.set_values(x0) - out = fn.forward() - assert out.shape == f.shape - if ref is not None: - expected = np.broadcast_to(ref(), f.shape) - np.testing.assert_allclose(out, expected, rtol=1e-10, atol=1e-12) - check_derivatives(fn, scope, x0, rng.standard_normal(f.size)) + expected = f(NP, *[leaf.value for leaf in leaves]) + np.testing.assert_allclose(fn.forward(), np.broadcast_to(expected, expr.shape), + rtol=1e-10, atol=1e-12) + check_derivatives(fn, scope, x0, rng.standard_normal(expr.size)) From 0c31a70c049906a71a9d57724a713c5cab55918c Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Mon, 9 Nov 2026 16:16:58 -0800 Subject: [PATCH 16/17] Tests: drop test_readme Co-Authored-By: Claude Opus 5.5 --- tests/test_readme.py | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 tests/test_readme.py diff --git a/tests/test_readme.py b/tests/test_readme.py deleted file mode 100644 index 63426b3..0000000 --- a/tests/test_readme.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Run every ```python block in README.md so the examples stay correct.""" - -import re -from pathlib import Path - -import pytest - -README = Path(__file__).resolve().parents[1] / "README.md" -BLOCKS = re.findall(r"```python\n(.*?)```", README.read_text(), re.S) - - -@pytest.mark.parametrize("code", BLOCKS, ids=[f"block{i}" for i in range(len(BLOCKS))]) -def test_readme_example(code): - exec(compile(code, str(README), "exec"), {}) From 5ad9e3351958170aa4303c4dc0371585dc161097 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Mon, 9 Nov 2026 16:28:23 -0800 Subject: [PATCH 17/17] Support division by a constant x / c for a number or array c is x * (1 / c). Dividing by an expression stays unsupported (the error points to x * d ** -1), and dividing by zero raises. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- sparsediffpy/_expression.py | 10 ++++++++++ tests/test_api.py | 10 ++++++++++ tests/test_atoms.py | 2 ++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b11e3b7..87a77bb 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ print(res.x) # approx. [0. 0.289 0.549 0.784], on the boundary ||x|| = 1 ## Supported operations -- **Operators:** `+ - * @ **`, `.T`, and indexing with ints, slices and integer lists: `x[1:3]` on a +- **Operators:** `+ - * @ **`, `/` by a constant, `.T`, and indexing with ints, slices and integer lists: `x[1:3]` on a vector, `X[0, :]` or `X[[0, 2], 1]` on a matrix (a matrix always takes two indices). Scalars, rows and columns broadcast. NumPy arrays and `scipy.sparse` matrices are accepted as constants; like every expression they must be 2-D, so pass `a[:, None]` or `a[None, :]` for a 1-D `a`. diff --git a/sparsediffpy/_expression.py b/sparsediffpy/_expression.py index 0f8fd10..6cecaf4 100644 --- a/sparsediffpy/_expression.py +++ b/sparsediffpy/_expression.py @@ -64,6 +64,16 @@ def __mul__(self, other): def __rmul__(self, other): return _mul(cast_to_const(other), self) + def __truediv__(self, other): + """self / c for a constant c (a number or array), as self * (1 / c).""" + if isinstance(other, Expression) or sps.issparse(other): + raise TypeError("Can only divide by a constant number or dense array; " + "for an expression d use x * d ** -1") + c = np.asarray(other, dtype=np.float64) + if np.any(c == 0): + raise ZeroDivisionError("division of an expression by zero") + return self * (1.0 / c) + def __matmul__(self, other): return _matmul(self, cast_to_const(other)) diff --git a/tests/test_api.py b/tests/test_api.py index 7097c85..9329d82 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -233,6 +233,16 @@ def test_numpy_scalar_exponent(scope): np.testing.assert_array_equal(f2.forward(), x.value ** 2) +def test_division_only_by_nonzero_constants(scope): + x = scope.Variable(2, 1) + with pytest.raises(TypeError, match="d \\*\\* -1"): + x / x + with pytest.raises(TypeError): + 1.0 / x + with pytest.raises(ZeroDivisionError): + x / np.array([[1.0], [0.0]]) + + def test_quad_form_parameter_must_be_symmetric(scope): x = scope.Variable(2, 1) N, S = np.array([[1.0, 5.0], [0.0, 1.0]]), np.array([[1.0, 2.0], [2.0, 3.0]]) diff --git a/tests/test_atoms.py b/tests/test_atoms.py index 1b7eff7..48d4e8b 100644 --- a/tests/test_atoms.py +++ b/tests/test_atoms.py @@ -61,6 +61,8 @@ def _dense(M): ("neg_sub", [(3, 1), (3, 1)], lambda m, x, y: -x - y + np.c_[0:3.0], False), ("constant_mult", [(3, 2)], lambda m, X: 2.0 * X * np.arange(6.0).reshape(3, 2), False), + ("constant_div", [(3, 2), (1, 1)], + lambda m, X, a: X / 2.0 + a / np.arange(1.0, 7.0).reshape(3, 2), False), ("matmul_dense", [(3, 1), (2, 3)], lambda m, x, Y: A @ x + (Y @ A.T)[:, 0:1], False), ("matmul_sparse", [(3, 1), (2, 3)],